From d74f4cd12405aa9d3320b28a54e3bfd89de3dd7a Mon Sep 17 00:00:00 2001 From: Fabian Fulga Date: Fri, 15 May 2026 16:57:07 +0300 Subject: [PATCH 1/2] Use flake8+ruff for format --- ruff.toml | 26 ++++++++++++++++++++++++++ test-requirements.txt | 1 + tox.ini | 27 +++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..2bc6554a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,26 @@ +line-length = 88 +target-version = "py310" + +extend-exclude = [ + "doc", + "tools", + "*openstack/common*", + "*lib/python*", + "*egg", +] + +[lint] +select = ["E", "W", "F"] + +ignore = [ + "E251", + "E305", + "E731", + "E117", + "W605", + "F632", + "E501", +] + +[format] +quote-style = "preserve" diff --git a/test-requirements.txt b/test-requirements.txt index 5e7d1300..1dae6478 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,3 +10,4 @@ oslotest>=3.8.0 # Apache-2.0 stestr>=4.2.1 # Apache-2.0 flake8>=7.0.0 pyflakes>=3.2.0 +ruff>=0.5.0 # MIT diff --git a/tox.ini b/tox.ini index 43804ad1..6e4aa15d 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,9 @@ minversion = 4.0.2 envlist = py3,pep8,cover skipsdist = True +[vars] +all_path = {toxinidir} + [testenv] usedevelop = True setenv = VIRTUAL_ENV={envdir} @@ -12,8 +15,25 @@ deps = git+https://github.com/cloudbase/python-coriolisclient.git commands = stestr run --slowest --exclude-regex coriolis.tests.integration {posargs} +[testenv:fmt] +description = Format the code based on the coding style standards. +deps = + ruff +commands = + ruff check --select I --fix {[vars]all_path} + ruff format {[vars]all_path} + [testenv:pep8] -commands = flake8 {posargs} +description = Verify the coding style (pep8). +deps = + ruff + flake8>=7.0.0 + pyflakes>=3.2.0 + hacking>=7.0.0,<7.1.0 +commands = + ruff format --diff {[vars]all_path} + ruff check {[vars]all_path} + flake8 {[vars]all_path} [testenv:cover] setenv = @@ -48,6 +68,9 @@ omit = coriolis/tests/* [flake8] # E125 is deliberately excluded. See https://github.com/jcrocholl/pep8/issues/126 # E251 Skipped due to https://github.com/jcrocholl/pep8/issues/301 +# +# E501 (line too long) is left to ruff +# E203 (whitespace before ':') conflicts with how ruff formats slices. -ignore = E125,E251,W503,W504,E305,E731,E117,W605,F632,H401,H403,H404,H405,H202 +ignore = E125,E251,W503,W504,E305,E731,E117,W605,F632,H401,H403,H404,H405,H202,E501,E203,H301,H306 exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,tools From 4bc248c382689bb63ae82b608061d237e5051f8c Mon Sep 17 00:00:00 2001 From: Fabian Fulga Date: Mon, 18 May 2026 15:56:08 +0300 Subject: [PATCH 2/2] Format code with Ruff + Flake8 --- coriolis/api-refs/source/conf.py | 10 +- coriolis/api/__init__.py | 45 +- coriolis/api/common.py | 6 +- coriolis/api/middleware/auth.py | 37 +- coriolis/api/middleware/fault.py | 28 +- coriolis/api/v1/deployment_actions.py | 2 +- coriolis/api/v1/deployments.py | 66 +- coriolis/api/v1/diagnostics.py | 13 +- coriolis/api/v1/endpoint_actions.py | 19 +- ...ndpoint_destination_minion_pool_options.py | 25 +- .../api/v1/endpoint_destination_options.py | 18 +- coriolis/api/v1/endpoint_instances.py | 32 +- coriolis/api/v1/endpoint_inventory.py | 12 +- coriolis/api/v1/endpoint_networks.py | 15 +- .../v1/endpoint_source_minion_pool_options.py | 22 +- coriolis/api/v1/endpoint_source_options.py | 18 +- coriolis/api/v1/endpoint_storage.py | 15 +- coriolis/api/v1/endpoints.py | 49 +- coriolis/api/v1/minion_pool_actions.py | 28 +- coriolis/api/v1/minion_pools.py | 232 +- coriolis/api/v1/provider_schemas.py | 7 +- coriolis/api/v1/providers.py | 7 +- coriolis/api/v1/regions.py | 31 +- coriolis/api/v1/router.py | 325 +- coriolis/api/v1/services.py | 41 +- coriolis/api/v1/transfer_actions.py | 14 +- coriolis/api/v1/transfer_schedules.py | 80 +- .../v1/transfer_tasks_execution_actions.py | 8 +- coriolis/api/v1/transfer_tasks_executions.py | 46 +- coriolis/api/v1/transfers.py | 272 +- coriolis/api/v1/utils.py | 73 +- coriolis/api/v1/views/deployment_view.py | 6 +- .../api/v1/views/endpoint_options_view.py | 15 +- .../api/v1/views/endpoint_resources_view.py | 6 +- coriolis/api/v1/views/endpoint_view.py | 6 +- coriolis/api/v1/views/minion_pool_view.py | 12 +- coriolis/api/v1/views/region_view.py | 9 +- coriolis/api/v1/views/service_view.py | 6 +- .../api/v1/views/transfer_schedule_view.py | 3 +- .../v1/views/transfer_tasks_execution_view.py | 16 +- coriolis/api/v1/views/transfer_view.py | 4 +- coriolis/api/v1/views/utils.py | 5 +- coriolis/api/wsgi.py | 200 +- coriolis/cache.py | 10 +- coriolis/cmd/api.py | 17 +- coriolis/cmd/conductor.py | 19 +- coriolis/cmd/db_sync.py | 5 +- coriolis/cmd/deployer_manager.py | 19 +- coriolis/cmd/minion_manager.py | 22 +- coriolis/cmd/scheduler.py | 22 +- coriolis/cmd/transfer_cron.py | 14 +- coriolis/cmd/worker.py | 20 +- coriolis/conductor/rpc/client.py | 640 ++-- coriolis/conductor/rpc/server.py | 2995 +++++++++------ coriolis/conductor/rpc/utils.py | 57 +- coriolis/conf.py | 8 +- coriolis/constants.py | 112 +- coriolis/context.py | 63 +- coriolis/cron/cron.py | 81 +- coriolis/data_transfer.py | 64 +- coriolis/db/api.py | 941 +++-- coriolis/db/sqlalchemy/api.py | 5 +- .../migrate_repo/versions/001_initial.py | 168 +- .../versions/002_adds_endpoints.py | 40 +- .../migrate_repo/versions/003_adds_notes.py | 6 +- .../versions/004_adds_replica_schedules.py | 37 +- .../versions/005_adds_transfer_result.py | 6 +- .../versions/006_adds_network_map.py | 6 +- .../versions/007_adds_storage_mappings.py | 6 +- .../versions/008_adds_source_environment.py | 6 +- .../versions/009_Migrate_info_to_blob.py | 3 +- .../versions/010_adds_reservation_id.py | 6 +- .../versions/011_adds_execution_type.py | 6 +- .../012_adds_migration_sync_fields.py | 11 +- .../versions/013_adds_task_index.py | 3 +- .../014_adds_worker_service_regions.py | 85 +- .../015_adds_action_last_execution_status.py | 14 +- .../versions/016_adds_minion_vm_pools.py | 241 +- .../versions/017_adds_user_scripts_column.py | 10 +- .../versions/018_adds_task_progress_idices.py | 13 +- .../019_add_replica_scenario_field.py | 9 +- .../versions/020_rename_tables.py | 47 +- .../versions/021_add_deployment_defaults.py | 13 +- .../versions/022_adds_auto_deploy_column.py | 10 +- .../versions/023_add_deployer_id.py | 11 +- coriolis/db/sqlalchemy/migration.py | 12 +- coriolis/db/sqlalchemy/models.py | 544 +-- coriolis/db/sqlalchemy/types.py | 7 +- coriolis/deployer_manager/rpc/client.py | 29 +- coriolis/deployer_manager/rpc/server.py | 84 +- coriolis/deployments/api.py | 55 +- coriolis/diagnostics/api.py | 2 +- coriolis/endpoint_options/api.py | 33 +- coriolis/endpoint_resources/api.py | 41 +- coriolis/endpoints/api.py | 47 +- coriolis/events.py | 84 +- coriolis/exception.py | 86 +- coriolis/keystone.py | 113 +- coriolis/licensing/client.py | 170 +- coriolis/minion_manager/rpc/client.py | 255 +- coriolis/minion_manager/rpc/server.py | 1952 ++++++---- coriolis/minion_manager/rpc/tasks.py | 1345 ++++--- coriolis/minion_manager/rpc/utils.py | 23 +- coriolis/minion_pools/api.py | 47 +- coriolis/osmorphing/amazon.py | 10 +- coriolis/osmorphing/base.py | 318 +- coriolis/osmorphing/centos.py | 10 +- coriolis/osmorphing/coreos.py | 4 +- coriolis/osmorphing/debian.py | 64 +- coriolis/osmorphing/manager.py | 207 +- coriolis/osmorphing/netpreserver/base.py | 1 - coriolis/osmorphing/netpreserver/factory.py | 15 +- coriolis/osmorphing/netpreserver/ifcfg.py | 16 +- .../osmorphing/netpreserver/interfaces.py | 35 +- coriolis/osmorphing/netpreserver/netplan.py | 55 +- .../osmorphing/netpreserver/nmconnection.py | 34 +- coriolis/osmorphing/openwrt.py | 5 +- coriolis/osmorphing/oracle.py | 44 +- coriolis/osmorphing/osdetect/amazon.py | 5 +- coriolis/osmorphing/osdetect/base.py | 50 +- coriolis/osmorphing/osdetect/centos.py | 29 +- coriolis/osmorphing/osdetect/coreos.py | 5 +- coriolis/osmorphing/osdetect/debian.py | 13 +- coriolis/osmorphing/osdetect/manager.py | 80 +- coriolis/osmorphing/osdetect/openwrt.py | 9 +- coriolis/osmorphing/osdetect/oracle.py | 12 +- coriolis/osmorphing/osdetect/redhat.py | 18 +- coriolis/osmorphing/osdetect/rocky.py | 23 +- coriolis/osmorphing/osdetect/suse.py | 14 +- coriolis/osmorphing/osdetect/ubuntu.py | 5 +- coriolis/osmorphing/osdetect/windows.py | 84 +- coriolis/osmorphing/osmount/base.py | 249 +- coriolis/osmorphing/osmount/factory.py | 31 +- coriolis/osmorphing/osmount/redhat.py | 12 +- coriolis/osmorphing/osmount/suse.py | 20 +- coriolis/osmorphing/osmount/ubuntu.py | 11 +- coriolis/osmorphing/osmount/windows.py | 109 +- coriolis/osmorphing/redhat.py | 115 +- coriolis/osmorphing/rocky.py | 8 +- coriolis/osmorphing/suse.py | 139 +- coriolis/osmorphing/ubuntu.py | 69 +- coriolis/osmorphing/windows.py | 434 ++- coriolis/policies/base.py | 9 +- coriolis/policies/deployments.py | 41 +- coriolis/policies/diagnostics.py | 14 +- coriolis/policies/endpoints.py | 103 +- coriolis/policies/general.py | 11 +- coriolis/policies/minion_pools.py | 62 +- coriolis/policies/regions.py | 41 +- coriolis/policies/services.py | 41 +- coriolis/policies/transfer_schedules.py | 41 +- .../policies/transfer_tasks_executions.py | 39 +- coriolis/policies/transfers.py | 49 +- coriolis/policy.py | 69 +- coriolis/providers/api.py | 3 +- coriolis/providers/backup_writers.py | 463 +-- coriolis/providers/base.py | 79 +- coriolis/providers/factory.py | 55 +- coriolis/providers/provider_utils.py | 110 +- coriolis/providers/replicator.py | 477 +-- coriolis/qemu.py | 58 +- coriolis/qemu_reader.py | 28 +- coriolis/regions/api.py | 6 +- coriolis/rpc.py | 45 +- coriolis/scheduler/filters/base.py | 7 +- coriolis/scheduler/filters/trivial_filters.py | 59 +- coriolis/scheduler/rpc/client.py | 166 +- coriolis/scheduler/rpc/server.py | 103 +- coriolis/scheduler/scheduler_utils.py | 32 +- coriolis/schemas.py | 92 +- coriolis/schemas_exceptions.py | 13 +- coriolis/secrets.py | 10 +- coriolis/service.py | 89 +- coriolis/services/api.py | 10 +- coriolis/taskflow/base.py | 202 +- coriolis/taskflow/runner.py | 71 +- coriolis/taskflow/utils.py | 7 +- coriolis/tasks/base.py | 148 +- coriolis/tasks/factory.py | 178 +- coriolis/tasks/migration_tasks.py | 70 +- coriolis/tasks/minion_pool_tasks.py | 580 +-- coriolis/tasks/osmorphing_tasks.py | 102 +- coriolis/tasks/replica_tasks.py | 692 ++-- coriolis/tests/api/middleware/test_auth.py | 95 +- coriolis/tests/api/test_common.py | 4 +- coriolis/tests/api/test_wsgi.py | 2 +- coriolis/tests/api/v1/test_diagnostics.py | 6 +- .../tests/api/v1/test_endpoint_actions.py | 48 +- ...ndpoint_destination_minion_pool_options.py | 74 +- .../v1/test_endpoint_destination_options.py | 45 +- .../tests/api/v1/test_endpoint_instances.py | 62 +- .../tests/api/v1/test_endpoint_inventory.py | 14 +- .../tests/api/v1/test_endpoint_networks.py | 28 +- ...est_endpoint_source_minion_pool_options.py | 54 +- .../api/v1/test_endpoint_source_options.py | 39 +- .../tests/api/v1/test_endpoint_storage.py | 28 +- coriolis/tests/api/v1/test_endpoints.py | 186 +- .../tests/api/v1/test_minion_pool_actions.py | 98 +- coriolis/tests/api/v1/test_minion_pools.py | 280 +- .../tests/api/v1/test_provider_schemas.py | 20 +- coriolis/tests/api/v1/test_providers.py | 14 +- coriolis/tests/api/v1/test_regions.py | 129 +- coriolis/tests/api/v1/test_router.py | 158 +- coriolis/tests/api/v1/test_services.py | 146 +- .../tests/api/v1/test_transfer_actions.py | 54 +- .../tests/api/v1/test_transfer_schedules.py | 297 +- .../test_transfer_tasks_execution_actions.py | 30 +- .../api/v1/test_transfer_tasks_executions.py | 137 +- coriolis/tests/api/v1/test_transfers.py | 338 +- coriolis/tests/api/v1/test_utils.py | 130 +- .../api/v1/views/test_diagnostic_view.py | 22 +- .../v1/views/test_endpoint_options_view.py | 15 +- .../v1/views/test_endpoint_resources_view.py | 16 +- .../tests/api/v1/views/test_endpoint_view.py | 88 +- .../api/v1/views/test_minion_pool_view.py | 83 +- .../tests/api/v1/views/test_region_view.py | 111 +- .../tests/api/v1/views/test_service_view.py | 88 +- .../test_transfer_task_execution_view.py | 46 +- .../tests/api/v1/views/test_transfer_view.py | 140 +- coriolis/tests/api/v1/views/test_utils.py | 15 +- coriolis/tests/cmd/test_api.py | 43 +- coriolis/tests/cmd/test_conductor.py | 48 +- coriolis/tests/cmd/test_db_sync.py | 12 +- coriolis/tests/cmd/test_minion_manager.py | 25 +- coriolis/tests/cmd/test_replica_cron.py | 25 +- coriolis/tests/cmd/test_scheduler.py | 25 +- coriolis/tests/cmd/test_worker.py | 51 +- coriolis/tests/conductor/rpc/test_client.py | 197 +- coriolis/tests/conductor/rpc/test_server.py | 3232 +++++++---------- coriolis/tests/conductor/rpc/test_utils.py | 57 +- coriolis/tests/cron/test_cron.py | 253 +- coriolis/tests/db/sqlalchemy/test_api.py | 76 +- .../tests/db/sqlalchemy/test_migration.py | 18 +- coriolis/tests/db/sqlalchemy/test_models.py | 113 +- coriolis/tests/db/sqlalchemy/test_types.py | 101 +- coriolis/tests/db/test_api.py | 641 ++-- coriolis/tests/diagnostics/test_api.py | 27 +- coriolis/tests/endpoint_options/test_api.py | 70 +- coriolis/tests/endpoint_resources/test_api.py | 71 +- coriolis/tests/endpoints/test_api.py | 204 +- coriolis/tests/integration/base.py | 63 +- .../deployments/test_deployment.py | 22 +- .../deployments/test_osmorphing.py | 1 - coriolis/tests/integration/harness.py | 143 +- .../management/test_diagnostics.py | 9 +- .../integration/management/test_providers.py | 10 +- .../integration/management/test_region.py | 10 +- .../integration/management/test_service.py | 10 +- coriolis/tests/integration/test_endpoints.py | 40 +- .../integration/test_failure_recovery.py | 3 +- .../tests/integration/test_minion_pools.py | 10 +- coriolis/tests/integration/test_pagination.py | 139 +- .../tests/integration/test_provider/exp.py | 93 +- .../tests/integration/test_provider/imp.py | 224 +- .../test_provider/osmorphing/__init__.py | 1 - coriolis/tests/integration/test_smoke.py | 4 +- .../integration/transfers/test_executions.py | 47 +- .../integration/transfers/test_schedules.py | 20 +- .../integration/transfers/test_transfer.py | 6 +- coriolis/tests/integration/utils.py | 99 +- coriolis/tests/licensing/test_client.py | 257 +- .../tests/minion_manager/rpc/test_client.py | 151 +- .../tests/minion_manager/rpc/test_server.py | 218 +- .../tests/minion_manager/rpc/test_tasks.py | 1984 +++++----- coriolis/tests/minion_pools/test_api.py | 70 +- .../osmorphing/netpreserver/test_factory.py | 64 +- .../osmorphing/netpreserver/test_ifcfg.py | 78 +- .../netpreserver/test_interfaces.py | 48 +- .../osmorphing/netpreserver/test_netplan.py | 49 +- .../netpreserver/test_nmconnection.py | 123 +- .../tests/osmorphing/osdetect/test_amazon.py | 13 +- .../tests/osmorphing/osdetect/test_base.py | 93 +- .../tests/osmorphing/osdetect/test_centos.py | 22 +- .../tests/osmorphing/osdetect/test_coreos.py | 13 +- .../tests/osmorphing/osdetect/test_debian.py | 40 +- .../tests/osmorphing/osdetect/test_manager.py | 156 +- .../tests/osmorphing/osdetect/test_openwrt.py | 18 +- .../tests/osmorphing/osdetect/test_oracle.py | 11 +- .../tests/osmorphing/osdetect/test_redhat.py | 21 +- .../tests/osmorphing/osdetect/test_rocky.py | 20 +- .../tests/osmorphing/osdetect/test_suse.py | 37 +- .../tests/osmorphing/osdetect/test_ubuntu.py | 17 +- .../tests/osmorphing/osdetect/test_windows.py | 77 +- .../tests/osmorphing/osmount/test_base.py | 800 ++-- .../tests/osmorphing/osmount/test_factory.py | 90 +- .../tests/osmorphing/osmount/test_redhat.py | 19 +- .../tests/osmorphing/osmount/test_suse.py | 54 +- .../tests/osmorphing/osmount/test_ubuntu.py | 25 +- .../tests/osmorphing/osmount/test_windows.py | 245 +- coriolis/tests/osmorphing/test_amazon.py | 8 +- coriolis/tests/osmorphing/test_base.py | 865 +++-- coriolis/tests/osmorphing/test_centos.py | 8 +- coriolis/tests/osmorphing/test_coreos.py | 14 +- coriolis/tests/osmorphing/test_debian.py | 212 +- coriolis/tests/osmorphing/test_manager.py | 212 +- coriolis/tests/osmorphing/test_openwrt.py | 17 +- coriolis/tests/osmorphing/test_oracle.py | 84 +- coriolis/tests/osmorphing/test_redhat.py | 420 ++- coriolis/tests/osmorphing/test_rocky.py | 10 +- coriolis/tests/osmorphing/test_suse.py | 260 +- coriolis/tests/osmorphing/test_ubuntu.py | 97 +- coriolis/tests/osmorphing/test_windows.py | 903 ++--- coriolis/tests/providers/base.py | 31 +- coriolis/tests/providers/test_api.py | 15 +- .../tests/providers/test_backup_writers.py | 847 +++-- coriolis/tests/providers/test_factory.py | 56 +- .../tests/providers/test_provider_utils.py | 232 +- coriolis/tests/providers/test_replicator.py | 964 +++-- coriolis/tests/regions/test_api.py | 16 +- coriolis/tests/scheduler/filters/test_base.py | 26 +- .../scheduler/filters/test_trivial_filters.py | 62 +- coriolis/tests/scheduler/rpc/test_client.py | 182 +- coriolis/tests/scheduler/rpc/test_server.py | 133 +- .../tests/scheduler/test_scheduler_utils.py | 61 +- coriolis/tests/services/test_api.py | 15 +- coriolis/tests/taskflow/test_base.py | 279 +- coriolis/tests/taskflow/test_runner.py | 76 +- coriolis/tests/tasks/test_base.py | 141 +- coriolis/tests/tasks/test_factory.py | 15 +- coriolis/tests/tasks/test_migration_tasks.py | 65 +- .../tests/tasks/test_minion_pool_tasks.py | 582 +-- coriolis/tests/tasks/test_osmorphing_tasks.py | 143 +- coriolis/tests/tasks/test_replica_tasks.py | 1359 ++++--- coriolis/tests/test_base.py | 17 +- coriolis/tests/test_cache.py | 13 +- coriolis/tests/test_context.py | 46 +- coriolis/tests/test_data_transfer.py | 124 +- coriolis/tests/test_events.py | 178 +- coriolis/tests/test_exception.py | 16 +- coriolis/tests/test_keystone.py | 210 +- coriolis/tests/test_policy.py | 54 +- coriolis/tests/test_rpc.py | 107 +- coriolis/tests/test_schemas.py | 48 +- coriolis/tests/test_secrets.py | 31 +- coriolis/tests/test_service.py | 140 +- coriolis/tests/test_utils.py | 1148 +++--- coriolis/tests/test_wsman.py | 113 +- coriolis/tests/testutils.py | 12 +- .../tests/transfer_cron/rpc/test_client.py | 9 +- .../tests/transfer_cron/rpc/test_server.py | 139 +- coriolis/tests/transfer_cron/test_api.py | 52 +- .../transfer_tasks_executions/test_api.py | 45 +- coriolis/tests/transfers/test_api.py | 80 +- coriolis/tests/worker/rpc/test_client.py | 63 +- coriolis/tests/worker/rpc/test_server.py | 388 +- coriolis/transfer_cron/api.py | 34 +- coriolis/transfer_cron/rpc/client.py | 6 +- coriolis/transfer_cron/rpc/server.py | 57 +- coriolis/transfer_tasks_executions/api.py | 31 +- coriolis/transfers/api.py | 78 +- coriolis/utils.py | 357 +- coriolis/worker/rpc/client.py | 245 +- coriolis/worker/rpc/server.py | 486 ++- coriolis/wsman.py | 112 +- setup.py | 10 +- 355 files changed, 25073 insertions(+), 21503 deletions(-) diff --git a/coriolis/api-refs/source/conf.py b/coriolis/api-refs/source/conf.py index 1e814007..91b71b4f 100644 --- a/coriolis/api-refs/source/conf.py +++ b/coriolis/api-refs/source/conf.py @@ -1,15 +1,11 @@ - -extensions = [ - 'openstackdocstheme', - 'os_api_ref' -] +extensions = ['openstackdocstheme', 'os_api_ref'] source_suffix = '.rst' master_doc = 'index' -project = u'Coriolis API Reference' -copyright = u'2018-present, Cloudbase Solutions S.R.L' +project = 'Coriolis API Reference' +copyright = '2018-present, Cloudbase Solutions S.R.L' repository_name = 'cloudbase/coriolis' bug_project = 'coriolis' diff --git a/coriolis/api/__init__.py b/coriolis/api/__init__.py index 36bdb95a..6351c1bd 100644 --- a/coriolis/api/__init__.py +++ b/coriolis/api/__init__.py @@ -18,16 +18,14 @@ WSGI middleware for OpenStack API controllers. """ -from paste import urlmap import routes - from oslo_log import log as logging from oslo_service import wsgi as base_wsgi +from paste import urlmap -from coriolis.api import wsgi from coriolis import exception -from coriolis.i18n import _, _LW # noqa - +from coriolis.api import wsgi +from coriolis.i18n import _LW, _ # noqa LOG = logging.getLogger(__name__) @@ -61,16 +59,13 @@ def resource(self, member_name, collection_name, **kwargs): parent_resource = kwargs['parent_resource'] p_collection = parent_resource['collection_name'] p_member = parent_resource['member_name'] - kwargs['path_prefix'] = '{project_id}/%s/:%s_id' % (p_collection, - p_member) - routes.Mapper.resource(self, - member_name, - collection_name, - **kwargs) + kwargs['path_prefix'] = '{project_id}/%s/:%s_id' % (p_collection, p_member) + routes.Mapper.resource(self, member_name, collection_name, **kwargs) class APIRouter(base_wsgi.Router): """Routes requests on the API to the appropriate controller and method.""" + ExtensionManager = None # override in subclasses @classmethod @@ -83,7 +78,8 @@ def __init__(self, ext_mgr=None): ext_mgr = self.ExtensionManager() else: raise exception.CoriolisException( - _("Must specify an ExtensionManager class")) + _("Must specify an ExtensionManager class") + ) mapper = ProjectMapper() self.resources = {} @@ -94,15 +90,15 @@ def __init__(self, ext_mgr=None): def _setup_ext_routes(self, mapper, ext_mgr): for resource in ext_mgr.get_resources(): - LOG.debug('Extended resource: %s', - resource.collection) + LOG.debug('Extended resource: %s', resource.collection) wsgi_resource = wsgi.Resource(resource.controller) self.resources[resource.collection] = wsgi_resource kargs = dict( controller=wsgi_resource, collection=resource.collection_actions, - member=resource.member_actions) + member=resource.member_actions, + ) if resource.parent: kargs['parent_resource'] = resource.parent @@ -118,16 +114,19 @@ def _setup_extensions(self, ext_mgr): controller = extension.controller if collection not in self.resources: - LOG.warning(_LW('Extension %(ext_name)s: Cannot extend ' - 'resource %(collection)s: No such resource'), - {'ext_name': extension.extension.name, - 'collection': collection}) + LOG.warning( + _LW( + 'Extension %(ext_name)s: Cannot extend ' + 'resource %(collection)s: No such resource' + ), + {'ext_name': extension.extension.name, 'collection': collection}, + ) continue - LOG.debug('Extension %(ext_name)s extending resource: ' - '%(collection)s', - {'ext_name': extension.extension.name, - 'collection': collection}) + LOG.debug( + 'Extension %(ext_name)s extending resource: %(collection)s', + {'ext_name': extension.extension.name, 'collection': collection}, + ) resource = self.resources[collection] resource.register_actions(controller) diff --git a/coriolis/api/common.py b/coriolis/api/common.py index 5582fea2..86422465 100644 --- a/coriolis/api/common.py +++ b/coriolis/api/common.py @@ -12,9 +12,9 @@ def get_paging_params(req): return marker, limit -def get_sort_params(req, - default_keys=('created_at', 'id'), - default_dirs=('desc', 'desc')): +def get_sort_params( + req, default_keys=('created_at', 'id'), default_dirs=('desc', 'desc') +): """Retrieves sort keys/directions parameters. Processes the parameters to create a list of sort keys and sort directions diff --git a/coriolis/api/middleware/auth.py b/coriolis/api/middleware/auth.py index d60ea92d..abea1fa4 100644 --- a/coriolis/api/middleware/auth.py +++ b/coriolis/api/middleware/auth.py @@ -1,13 +1,13 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +import webob from oslo_log import log as logging from oslo_middleware import request_id from oslo_serialization import jsonutils -import webob -from coriolis.api import wsgi from coriolis import context +from coriolis.api import wsgi from coriolis.i18n import _ LOG = logging.getLogger(__name__) @@ -23,14 +23,16 @@ def _get_project_id(self, req): return req.headers['X_TENANT'] else: raise webob.exc.HTTPBadRequest( - explanation=_("No 'X_TENANT_ID' or 'X_TENANT' passed.")) + explanation=_("No 'X_TENANT_ID' or 'X_TENANT' passed.") + ) def _get_user(self, req): user = req.headers.get('X_USER') user = req.headers.get('X_USER_ID', user) if user is None: raise webob.exc.HTTPUnauthorized( - explanation=_("Neither X_USER_ID nor X_USER found in request")) + explanation=_("Neither X_USER_ID nor X_USER found in request") + ) return user @webob.dec.wsgify(RequestClass=wsgi.Request) @@ -64,18 +66,21 @@ def __call__(self, req): service_catalog = jsonutils.loads(catalog_header) except ValueError: raise webob.exc.HTTPInternalServerError( - explanation=_('Invalid service catalog json.')) - - ctx = context.RequestContext(user, - project_id, - project_name=project_name, - project_domain_name=project_domain_name, - user_domain_name=user_domain_name, - roles=roles, - auth_token=auth_token, - remote_address=remote_address, - service_catalog=service_catalog, - request_id=req_id) + explanation=_('Invalid service catalog json.') + ) + + ctx = context.RequestContext( + user, + project_id, + project_name=project_name, + project_domain_name=project_domain_name, + user_domain_name=user_domain_name, + roles=roles, + auth_token=auth_token, + remote_address=remote_address, + service_catalog=service_catalog, + request_id=req_id, + ) req.environ['coriolis.context'] = ctx return self.application diff --git a/coriolis/api/middleware/fault.py b/coriolis/api/middleware/fault.py index e56740b6..1cadcc6a 100644 --- a/coriolis/api/middleware/fault.py +++ b/coriolis/api/middleware/fault.py @@ -14,16 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo_log import log as logging import six import webob.dec import webob.exc +from oslo_log import log as logging +from coriolis import exception, utils from coriolis.api import wsgi -from coriolis import exception -from coriolis.i18n import _, _LE, _LI # noqa -from coriolis import utils - +from coriolis.i18n import _LE, _LI, _ # noqa LOG = logging.getLogger(__name__) @@ -39,12 +37,14 @@ def status_to_type(status): for clazz in utils.walk_class_hierarchy(webob.exc.HTTPError): FaultWrapper._status_to_type[clazz.code] = clazz return FaultWrapper._status_to_type.get( - status, webob.exc.HTTPInternalServerError)() + status, webob.exc.HTTPInternalServerError + )() def _error(self, inner, req): - LOG.exception(_LE("Caught error: %(type)s %(error)s"), - {'type': type(inner), - 'error': inner}) + LOG.exception( + _LE("Caught error: %(type)s %(error)s"), + {'type': type(inner), 'error': inner}, + ) safe = getattr(inner, 'safe', False) headers = getattr(inner, 'headers', None) status = getattr(inner, 'code', 500) @@ -57,10 +57,12 @@ def _error(self, inner, req): if headers: outer.headers = headers if safe: - msg = (inner.msg if isinstance(inner, exception.CoriolisException) - else six.text_type(inner)) - params = {'exception': inner.__class__.__name__, - 'explanation': msg} + msg = ( + inner.msg + if isinstance(inner, exception.CoriolisException) + else six.text_type(inner) + ) + params = {'exception': inner.__class__.__name__, 'explanation': msg} outer.explanation = _('%(exception)s: %(explanation)s') % params return wsgi.Fault(outer) diff --git a/coriolis/api/v1/deployment_actions.py b/coriolis/api/v1/deployment_actions.py index c7f5034f..7e88c2d3 100644 --- a/coriolis/api/v1/deployment_actions.py +++ b/coriolis/api/v1/deployment_actions.py @@ -3,9 +3,9 @@ from webob import exc +from coriolis import exception from coriolis.api import wsgi as api_wsgi from coriolis.deployments import api -from coriolis import exception from coriolis.policies import deployments as deployment_policies diff --git a/coriolis/api/v1/deployments.py b/coriolis/api/v1/deployments.py index 2f62ddd7..9a116b9d 100644 --- a/coriolis/api/v1/deployments.py +++ b/coriolis/api/v1/deployments.py @@ -4,16 +4,15 @@ from oslo_log import log as logging from webob import exc +from coriolis import exception from coriolis.api import common +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import deployment_view -from coriolis.api import wsgi as api_wsgi from coriolis.deployments import api from coriolis.endpoints import api as endpoints_api -from coriolis import exception from coriolis.policies import deployments as deployment_policies - LOG = logging.getLogger(__name__) @@ -27,23 +26,24 @@ def show(self, req, id): context = req.environ["coriolis.context"] context.can(deployment_policies.get_deployments_policy_label("show")) include_task_info = api_utils.get_bool_url_arg( - req, "include_task_info", default=False) + req, "include_task_info", default=False + ) deployment = self._deployment_api.get_deployment( - context, id, - include_task_info=include_task_info) + context, id, include_task_info=include_task_info + ) if not deployment: raise exc.HTTPNotFound() return deployment_view.single(deployment) def _list(self, req): - show_deleted = api_utils.get_bool_url_arg( - req, "show_deleted", default=False) + show_deleted = api_utils.get_bool_url_arg(req, "show_deleted", default=False) context = req.environ["coriolis.context"] context.show_deleted = show_deleted context.can(deployment_policies.get_deployments_policy_label("list")) include_task_info = api_utils.get_bool_url_arg( - req, "include_task_info", default=False) + req, "include_task_info", default=False + ) marker, limit = common.get_paging_params(req) sort_keys, sort_dirs = common.get_sort_params(req) @@ -53,9 +53,12 @@ def _list(self, req): context, include_tasks=include_task_info, include_task_info=include_task_info, - marker=marker, limit=limit, - sort_keys=sort_keys, sort_dirs=sort_dirs, - )) + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + ) + ) def index(self, req): return self._list(req) @@ -72,36 +75,51 @@ def _validate_deployment_input(self, context, body): if not transfer_id: raise exc.HTTPBadRequest( explanation="Missing 'transfer_id' field from deployment " - "body. A deployment can be created strictly " - "based on an existing Transfer.") + "body. A deployment can be created strictly " + "based on an existing Transfer." + ) clone_disks = deployment.get("clone_disks", True) force = deployment.get("force", False) skip_os_morphing = deployment.get("skip_os_morphing", False) instance_osmorphing_minion_pool_mappings = deployment.get( - 'instance_osmorphing_minion_pool_mappings', {}) + 'instance_osmorphing_minion_pool_mappings', {} + ) user_scripts = deployment.get('user_scripts', {}) api_utils.validate_user_scripts(user_scripts) return ( - transfer_id, force, clone_disks, skip_os_morphing, + transfer_id, + force, + clone_disks, + skip_os_morphing, instance_osmorphing_minion_pool_mappings, - user_scripts) + user_scripts, + ) def create(self, req, body): context = req.environ['coriolis.context'] context.can(deployment_policies.get_deployments_policy_label("create")) - (transfer_id, force, clone_disks, skip_os_morphing, - instance_osmorphing_minion_pool_mappings, - user_scripts) = self._validate_deployment_input( - context, body) + ( + transfer_id, + force, + clone_disks, + skip_os_morphing, + instance_osmorphing_minion_pool_mappings, + user_scripts, + ) = self._validate_deployment_input(context, body) # NOTE: destination environment for transfer should have been # validated upon its creation. deployment = self._deployment_api.deploy_transfer_instances( - context, transfer_id, instance_osmorphing_minion_pool_mappings, - clone_disks, force, skip_os_morphing, - user_scripts=user_scripts) + context, + transfer_id, + instance_osmorphing_minion_pool_mappings, + clone_disks, + force, + skip_os_morphing, + user_scripts=user_scripts, + ) return deployment_view.single(deployment) diff --git a/coriolis/api/v1/diagnostics.py b/coriolis/api/v1/diagnostics.py index 6ba735d7..a0a78a95 100644 --- a/coriolis/api/v1/diagnostics.py +++ b/coriolis/api/v1/diagnostics.py @@ -1,14 +1,13 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import diagnostic_view +import logging + from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import diagnostic_view from coriolis.diagnostics import api from coriolis.policies import diagnostics -import logging - - LOG = logging.getLogger(__name__) @@ -19,11 +18,9 @@ def __init__(self): def index(self, req): context = req.environ['coriolis.context'] - context.can( - diagnostics.get_diagnostics_policy_label("get")) + context.can(diagnostics.get_diagnostics_policy_label("get")) - return diagnostic_view.collection( - self._diag_api.get(context)) + return diagnostic_view.collection(self._diag_api.get(context)) def create_resource(): diff --git a/coriolis/api/v1/endpoint_actions.py b/coriolis/api/v1/endpoint_actions.py index 7ab7b188..f04c6b8b 100644 --- a/coriolis/api/v1/endpoint_actions.py +++ b/coriolis/api/v1/endpoint_actions.py @@ -1,13 +1,13 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from webob import exc + +from coriolis import exception from coriolis.api import wsgi as api_wsgi from coriolis.endpoints import api -from coriolis import exception from coriolis.policies import endpoints as endpoint_policies -from webob import exc - class EndpointActionsController(api_wsgi.Controller): def __init__(self): @@ -17,15 +17,12 @@ def __init__(self): @api_wsgi.action('validate-connection') def _validate_connection(self, req, id, body): context = req.environ['coriolis.context'] - context.can("%s:validate_connection" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can( + "%s:validate_connection" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) + ) try: - is_valid, message = self._endpoint_api.validate_connection( - context, id) - return { - "validate-connection": - {"valid": is_valid, "message": message} - } + is_valid, message = self._endpoint_api.validate_connection(context, id) + return {"validate-connection": {"valid": is_valid, "message": message}} except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: diff --git a/coriolis/api/v1/endpoint_destination_minion_pool_options.py b/coriolis/api/v1/endpoint_destination_minion_pool_options.py index c8f07227..9f381206 100644 --- a/coriolis/api/v1/endpoint_destination_minion_pool_options.py +++ b/coriolis/api/v1/endpoint_destination_minion_pool_options.py @@ -1,14 +1,13 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_options_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging - LOG = logging.getLogger(__name__) @@ -20,8 +19,10 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_destination_minion_pool_options" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can( + "%s:list_destination_minion_pool_options" + % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) + ) env = req.GET.get("env") if env is not None: @@ -35,11 +36,11 @@ def index(self, req, endpoint_id): else: options = {} - return (endpoint_options_view. - destination_minion_pool_options_collection)( - (self._minion_pool_options_api. - get_endpoint_destination_minion_pool_options)( - context, endpoint_id, env=env, option_names=options)) + return (endpoint_options_view.destination_minion_pool_options_collection)( + ( + self._minion_pool_options_api.get_endpoint_destination_minion_pool_options + )(context, endpoint_id, env=env, option_names=options) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_destination_options.py b/coriolis/api/v1/endpoint_destination_options.py index fef035b8..664787f1 100644 --- a/coriolis/api/v1/endpoint_destination_options.py +++ b/coriolis/api/v1/endpoint_destination_options.py @@ -1,14 +1,13 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_options_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging - LOG = logging.getLogger(__name__) @@ -20,8 +19,9 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_destination_options" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can( + "%s:list_destination_options" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) + ) env = req.GET.get("env") if env is not None: @@ -37,7 +37,9 @@ def index(self, req, endpoint_id): return endpoint_options_view.destination_options_collection( self._destination_options_api.get_endpoint_destination_options( - context, endpoint_id, env=env, option_names=options)) + context, endpoint_id, env=env, option_names=options + ) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_instances.py b/coriolis/api/v1/endpoint_instances.py index a131c9f9..79470f31 100644 --- a/coriolis/api/v1/endpoint_instances.py +++ b/coriolis/api/v1/endpoint_instances.py @@ -1,15 +1,15 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import common +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import endpoint_resources_view -from coriolis.api import wsgi as api_wsgi from coriolis.endpoint_resources import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging LOG = logging.getLogger(__name__) @@ -21,12 +21,10 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_instances" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can("%s:list_instances" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) marker, limit = common.get_paging_params(req) instance_name_pattern = req.GET.get("name") - refresh = api_utils.get_bool_url_arg( - req, "refresh", default=False) + refresh = api_utils.get_bool_url_arg(req, "refresh", default=False) env = req.GET.get("env") if env is not None: @@ -36,13 +34,19 @@ def index(self, req, endpoint_id): return endpoint_resources_view.instances_collection( self._instance_api.get_endpoint_instances( - context, endpoint_id, env, marker, limit, - instance_name_pattern, refresh=refresh)) + context, + endpoint_id, + env, + marker, + limit, + instance_name_pattern, + refresh=refresh, + ) + ) def show(self, req, endpoint_id, id): context = req.environ['coriolis.context'] - context.can("%s:get_instance" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can("%s:get_instance" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) # WSGI does not allow encoded / chars (%2F) in the url # See e.g.: https://github.com/pallets/flask/issues/900 @@ -56,7 +60,9 @@ def show(self, req, endpoint_id, id): return endpoint_resources_view.instance_single( self._instance_api.get_endpoint_instance( - req.environ['coriolis.context'], endpoint_id, env, id)) + req.environ['coriolis.context'], endpoint_id, env, id + ) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_inventory.py b/coriolis/api/v1/endpoint_inventory.py index 616d60d3..9bdaf746 100644 --- a/coriolis/api/v1/endpoint_inventory.py +++ b/coriolis/api/v1/endpoint_inventory.py @@ -1,12 +1,12 @@ # Copyright 2026 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi from coriolis.endpoint_resources import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging LOG = logging.getLogger(__name__) @@ -20,8 +20,7 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:export_inventory" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can("%s:export_inventory" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) env = req.GET.get("env") if env is not None: @@ -30,7 +29,8 @@ def index(self, req, endpoint_id): env = {} csv_content = self._endpoint_resources_api.get_endpoint_inventory_csv( - context, endpoint_id, env) + context, endpoint_id, env + ) return api_wsgi.ResponseObject(csv_content) diff --git a/coriolis/api/v1/endpoint_networks.py b/coriolis/api/v1/endpoint_networks.py index f34f14be..908a0fe4 100644 --- a/coriolis/api/v1/endpoint_networks.py +++ b/coriolis/api/v1/endpoint_networks.py @@ -1,13 +1,13 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_resources_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_resources_view from coriolis.endpoint_resources import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging LOG = logging.getLogger(__name__) @@ -19,8 +19,7 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_networks" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can("%s:list_networks" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) env = req.GET.get("env") if env is not None: env = utils.decode_base64_param(env, is_json=True) @@ -28,8 +27,8 @@ def index(self, req, endpoint_id): env = {} return endpoint_resources_view.networks_collection( - self._network_api.get_endpoint_networks( - context, endpoint_id, env)) + self._network_api.get_endpoint_networks(context, endpoint_id, env) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_source_minion_pool_options.py b/coriolis/api/v1/endpoint_source_minion_pool_options.py index 2c526d9f..e85c61e3 100644 --- a/coriolis/api/v1/endpoint_source_minion_pool_options.py +++ b/coriolis/api/v1/endpoint_source_minion_pool_options.py @@ -1,14 +1,13 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_options_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging - LOG = logging.getLogger(__name__) @@ -20,8 +19,10 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_source_minion_pool_options" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can( + "%s:list_source_minion_pool_options" + % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) + ) env = req.GET.get("env") if env is not None: @@ -36,9 +37,10 @@ def index(self, req, endpoint_id): options = {} return endpoint_options_view.source_minion_pool_options_collection( - (self._minion_pool_options_api. - get_endpoint_source_minion_pool_options)( - context, endpoint_id, env=env, option_names=options)) + (self._minion_pool_options_api.get_endpoint_source_minion_pool_options)( + context, endpoint_id, env=env, option_names=options + ) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_source_options.py b/coriolis/api/v1/endpoint_source_options.py index b3c7b409..1b085098 100644 --- a/coriolis/api/v1/endpoint_source_options.py +++ b/coriolis/api/v1/endpoint_source_options.py @@ -1,14 +1,13 @@ # Copyright 2019 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_options_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging - LOG = logging.getLogger(__name__) @@ -20,8 +19,9 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_source_options" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can( + "%s:list_source_options" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) + ) env = req.GET.get("env") if env is not None: @@ -37,7 +37,9 @@ def index(self, req, endpoint_id): return endpoint_options_view.source_options_collection( self._source_options_api.get_endpoint_source_options( - context, endpoint_id, env=env, option_names=options)) + context, endpoint_id, env=env, option_names=options + ) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoint_storage.py b/coriolis/api/v1/endpoint_storage.py index 3465fcc2..d6a7f85d 100644 --- a/coriolis/api/v1/endpoint_storage.py +++ b/coriolis/api/v1/endpoint_storage.py @@ -1,13 +1,13 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import endpoint_resources_view +from oslo_log import log as logging + +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import endpoint_resources_view from coriolis.endpoint_resources import api from coriolis.policies import endpoints as endpoint_policies -from coriolis import utils - -from oslo_log import log as logging LOG = logging.getLogger(__name__) @@ -19,8 +19,7 @@ def __init__(self): def index(self, req, endpoint_id): context = req.environ['coriolis.context'] - context.can("%s:list_storage" % ( - endpoint_policies.ENDPOINTS_POLICY_PREFIX)) + context.can("%s:list_storage" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) env = req.GET.get("env") if env is not None: env = utils.decode_base64_param(env, is_json=True) @@ -28,8 +27,8 @@ def index(self, req, endpoint_id): env = {} return endpoint_resources_view.storage_collection( - self._storage_api.get_endpoint_storage( - context, endpoint_id, env)) + self._storage_api.get_endpoint_storage(context, endpoint_id, env) + ) def create_resource(): diff --git a/coriolis/api/v1/endpoints.py b/coriolis/api/v1/endpoints.py index a5fe63f0..a2a1a97b 100644 --- a/coriolis/api/v1/endpoints.py +++ b/coriolis/api/v1/endpoints.py @@ -1,17 +1,16 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging +from webob import exc + +from coriolis import exception +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import endpoint_view -from coriolis.api import wsgi as api_wsgi from coriolis.endpoints import api -from coriolis import exception from coriolis.policies import endpoints as endpoint_policies -from oslo_log import log as logging -from webob import exc - - LOG = logging.getLogger(__name__) @@ -32,8 +31,7 @@ def show(self, req, id): def index(self, req): context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("list")) - return endpoint_view.collection( - self._endpoint_api.get_endpoints(context)) + return endpoint_view.collection(self._endpoint_api.get_endpoints(context)) @api_utils.format_keyerror_message(resource='endpoint', method='create') def _validate_create_body(self, body): @@ -43,34 +41,43 @@ def _validate_create_body(self, body): endpoint_type = endpoint["type"] connection_info = endpoint["connection_info"] mapped_regions = endpoint.get("mapped_regions", []) - return ( - name, endpoint_type, description, connection_info, - mapped_regions) + return (name, endpoint_type, description, connection_info, mapped_regions) def create(self, req, body): context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("create")) - (name, endpoint_type, description, - connection_info, mapped_regions) = self._validate_create_body(body) - return endpoint_view.single(self._endpoint_api.create( - context, name, endpoint_type, description, connection_info, - mapped_regions)) + (name, endpoint_type, description, connection_info, mapped_regions) = ( + self._validate_create_body(body) + ) + return endpoint_view.single( + self._endpoint_api.create( + context, + name, + endpoint_type, + description, + connection_info, + mapped_regions, + ) + ) @api_utils.format_keyerror_message(resource='endpoint', method='update') def _validate_update_body(self, body): endpoint = body["endpoint"] return { k: endpoint[k] - for k in endpoint.keys() & { - "name", "description", "connection_info", - "mapped_regions"}} + for k in endpoint.keys() + & {"name", "description", "connection_info", "mapped_regions"} + } def update(self, req, id, body): context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("update")) updated_values = self._validate_update_body(body) - return endpoint_view.single(self._endpoint_api.update( - req.environ['coriolis.context'], id, updated_values)) + return endpoint_view.single( + self._endpoint_api.update( + req.environ['coriolis.context'], id, updated_values + ) + ) def delete(self, req, id): context = req.environ["coriolis.context"] diff --git a/coriolis/api/v1/minion_pool_actions.py b/coriolis/api/v1/minion_pool_actions.py index cfb87ce3..b377b1eb 100644 --- a/coriolis/api/v1/minion_pool_actions.py +++ b/coriolis/api/v1/minion_pool_actions.py @@ -3,9 +3,9 @@ from webob import exc -from coriolis.api.v1.views import minion_pool_view -from coriolis.api import wsgi as api_wsgi from coriolis import exception +from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import minion_pool_view from coriolis.minion_pools import api from coriolis.policies import minion_pools as minion_pool_policies @@ -18,13 +18,11 @@ def __init__(self): @api_wsgi.action('allocate') def _allocate_pool(self, req, id, body): context = req.environ['coriolis.context'] - context.can( - minion_pool_policies.get_minion_pools_policy_label( - "allocate")) + context.can(minion_pool_policies.get_minion_pools_policy_label("allocate")) try: return minion_pool_view.single( - self.minion_pool_api.allocate_minion_pool( - context, id)) + self.minion_pool_api.allocate_minion_pool(context, id) + ) except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: @@ -33,13 +31,11 @@ def _allocate_pool(self, req, id, body): @api_wsgi.action('refresh') def _refresh_pool(self, req, id, body): context = req.environ['coriolis.context'] - context.can( - minion_pool_policies.get_minion_pools_policy_label( - "refresh")) + context.can(minion_pool_policies.get_minion_pools_policy_label("refresh")) try: return minion_pool_view.single( - self.minion_pool_api.refresh_minion_pool( - context, id)) + self.minion_pool_api.refresh_minion_pool(context, id) + ) except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: @@ -48,14 +44,12 @@ def _refresh_pool(self, req, id, body): @api_wsgi.action('deallocate') def _deallocate_pool(self, req, id, body): context = req.environ['coriolis.context'] - context.can( - minion_pool_policies.get_minion_pools_policy_label( - "deallocate")) + context.can(minion_pool_policies.get_minion_pools_policy_label("deallocate")) force = (body["deallocate"] or {}).get("force", False) try: return minion_pool_view.single( - self.minion_pool_api.deallocate_minion_pool( - context, id, force=force)) + self.minion_pool_api.deallocate_minion_pool(context, id, force=force) + ) except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: diff --git a/coriolis/api/v1/minion_pools.py b/coriolis/api/v1/minion_pools.py index 48e9a459..ac7df5ce 100644 --- a/coriolis/api/v1/minion_pools.py +++ b/coriolis/api/v1/minion_pools.py @@ -1,18 +1,17 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging +from webob import exc + +from coriolis import constants, exception +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import minion_pool_view -from coriolis.api import wsgi as api_wsgi -from coriolis import constants from coriolis.endpoints import api as endpoints_api -from coriolis import exception from coriolis.minion_pools import api from coriolis.policies import minion_pools as pools_policies -from oslo_log import log as logging -from webob import exc - LOG = logging.getLogger(__name__) @@ -35,46 +34,54 @@ def index(self, req): context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("list")) return minion_pool_view.collection( - self._minion_pool_api.get_minion_pools(context)) + self._minion_pool_api.get_minion_pools(context) + ) def _check_pool_retention_strategy(self, pool_retention_strategy): if not pool_retention_strategy: LOG.debug( "Ignoring void minion pool retention strategy '%s'", - pool_retention_strategy) + pool_retention_strategy, + ) valid_strats = [ constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE, - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF] + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF, + ] if pool_retention_strategy not in valid_strats: raise Exception( "Invalid minion pool retention strategy '%s'. Must be one of " - "the following: %s" % (pool_retention_strategy, valid_strats)) + "the following: %s" % (pool_retention_strategy, valid_strats) + ) def _check_pool_numeric_values( - self, minimum_minions, maximum_minions, minion_max_idle_time): + self, minimum_minions, maximum_minions, minion_max_idle_time + ): if minimum_minions is not None: if minimum_minions <= 0: raise Exception( "'minimum_minions' must be a strictly positive integer. " - "Got: %s" % minimum_minions) + "Got: %s" % minimum_minions + ) if maximum_minions is not None: if maximum_minions <= 0: raise Exception( "'maximum_minions' must be a strictly positive integer. " - "Got: %s" % maximum_minions) + "Got: %s" % maximum_minions + ) if maximum_minions < minimum_minions: raise Exception( "'maximum_minions' value (%s) must be at least as large as" - " the 'minimum_minions' value (%s)." % - (maximum_minions, minimum_minions)) + " the 'minimum_minions' value (%s)." + % (maximum_minions, minimum_minions) + ) if minion_max_idle_time is not None: if minion_max_idle_time <= 0: raise Exception( "'minion_max_idle_time' must be a strictly positive " - "integer. Got: %s" % maximum_minions) + "integer. Got: %s" % maximum_minions + ) - @api_utils.format_keyerror_message(resource='minion_pool', - method='create') + @api_utils.format_keyerror_message(resource='minion_pool', method='create') def _validate_create_body(self, ctxt, body): minion_pool = body["minion_pool"] name = minion_pool["name"] @@ -83,116 +90,155 @@ def _validate_create_body(self, ctxt, body): if pool_os_type not in constants.VALID_OS_TYPES: raise Exception( "The provided pool OS type '%s' is invalid. Must be one " - "of the following: %s" % ( - pool_os_type, constants.VALID_OS_TYPES)) + "of the following: %s" % (pool_os_type, constants.VALID_OS_TYPES) + ) pool_platform = minion_pool["platform"] supported_pool_platforms = [ constants.PROVIDER_PLATFORM_SOURCE, - constants.PROVIDER_PLATFORM_DESTINATION] + constants.PROVIDER_PLATFORM_DESTINATION, + ] if pool_platform not in supported_pool_platforms: raise Exception( "The provided pool platform ('%s') is invalid. Must be one" - " of the following: %s" % ( - pool_platform, supported_pool_platforms)) + " of the following: %s" % (pool_platform, supported_pool_platforms) + ) if pool_platform == constants.PROVIDER_PLATFORM_SOURCE and ( - pool_os_type != constants.OS_TYPE_LINUX): + pool_os_type != constants.OS_TYPE_LINUX + ): raise Exception( "Source Minion Pools are required to be of OS type " - "'%s', not '%s'." % ( - constants.OS_TYPE_LINUX, pool_os_type)) + "'%s', not '%s'." % (constants.OS_TYPE_LINUX, pool_os_type) + ) environment_options = minion_pool["environment_options"] if pool_platform == constants.PROVIDER_PLATFORM_SOURCE: self._endpoints_api.validate_endpoint_source_minion_pool_options( - ctxt, endpoint_id, environment_options) + ctxt, endpoint_id, environment_options + ) elif pool_platform == constants.PROVIDER_PLATFORM_DESTINATION: - (self._endpoints_api. - validate_endpoint_destination_minion_pool_options)( - ctxt, endpoint_id, environment_options) + (self._endpoints_api.validate_endpoint_destination_minion_pool_options)( + ctxt, endpoint_id, environment_options + ) minimum_minions = minion_pool.get("minimum_minions", 1) - maximum_minions = minion_pool.get( - "maximum_minions", minimum_minions) - minion_max_idle_time = minion_pool.get( - "minion_max_idle_time", 1) + maximum_minions = minion_pool.get("maximum_minions", minimum_minions) + minion_max_idle_time = minion_pool.get("minion_max_idle_time", 1) self._check_pool_numeric_values( - minimum_minions, maximum_minions, minion_max_idle_time) + minimum_minions, maximum_minions, minion_max_idle_time + ) minion_retention_strategy = minion_pool.get( "minion_retention_strategy", - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE) - self._check_pool_retention_strategy( - minion_retention_strategy) + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE, + ) + self._check_pool_retention_strategy(minion_retention_strategy) notes = minion_pool.get("notes") skip_allocation = minion_pool.get('skip_allocation', False) return ( - name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes, - skip_allocation) + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes, + skip_allocation, + ) def create(self, req, body): context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("create")) - (name, endpoint_id, pool_platform, pool_os_type, environment_options, - minimum_minions, maximum_minions, minion_max_idle_time, - minion_retention_strategy, notes, skip_allocation) = ( - self._validate_create_body(context, body)) - return minion_pool_view.single(self._minion_pool_api.create( - context, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=notes, - skip_allocation=skip_allocation)) - - def _validate_updated_environment_options(self, context, minion_pool, - environment_options): - if minion_pool['platform'] == ( - constants.PROVIDER_PLATFORM_SOURCE): - (self._endpoints_api. - validate_endpoint_source_minion_pool_options)( - context, minion_pool['endpoint_id'], environment_options) - elif minion_pool['platform'] == ( - constants.PROVIDER_PLATFORM_DESTINATION): - (self._endpoints_api. - validate_endpoint_destination_minion_pool_options)( - context, minion_pool['endpoint_id'], environment_options) + ( + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes, + skip_allocation, + ) = self._validate_create_body(context, body) + return minion_pool_view.single( + self._minion_pool_api.create( + context, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=notes, + skip_allocation=skip_allocation, + ) + ) + + def _validate_updated_environment_options( + self, context, minion_pool, environment_options + ): + if minion_pool['platform'] == (constants.PROVIDER_PLATFORM_SOURCE): + (self._endpoints_api.validate_endpoint_source_minion_pool_options)( + context, minion_pool['endpoint_id'], environment_options + ) + elif minion_pool['platform'] == (constants.PROVIDER_PLATFORM_DESTINATION): + (self._endpoints_api.validate_endpoint_destination_minion_pool_options)( + context, minion_pool['endpoint_id'], environment_options + ) else: - raise Exception( - "Unknown pool platform: %s" % minion_pool[ - 'platform']) + raise Exception("Unknown pool platform: %s" % minion_pool['platform']) - @api_utils.format_keyerror_message(resource='minion_pool', - method='update') + @api_utils.format_keyerror_message(resource='minion_pool', method='update') def _validate_update_body(self, id, context, body): minion_pool = body["minion_pool"] if 'endpoint_id' in minion_pool: - raise Exception( - "The 'endpoint_id' of a minion pool cannot be updated.") + raise Exception("The 'endpoint_id' of a minion pool cannot be updated.") if 'platform' in minion_pool: - raise Exception( - "The 'platform' of a minion pool cannot be updated.") - vals = {k: minion_pool[k] for k in minion_pool.keys() & - {"name", "environment_options", "minimum_minions", - "maximum_minions", "minion_max_idle_time", - "minion_retention_strategy", "notes", "os_type"}} + raise Exception("The 'platform' of a minion pool cannot be updated.") + vals = { + k: minion_pool[k] + for k in minion_pool.keys() + & { + "name", + "environment_options", + "minimum_minions", + "maximum_minions", + "minion_max_idle_time", + "minion_retention_strategy", + "notes", + "os_type", + } + } if 'minion_retention_strategy' in vals: - self._check_pool_retention_strategy( - vals['minion_retention_strategy']) - if any([ - f in vals for f in [ - 'environment_options', 'minimum_minions', - 'maximum_minions', 'minion_max_idle_time']]): - minion_pool = self._minion_pool_api.get_minion_pool( - context, id) + self._check_pool_retention_strategy(vals['minion_retention_strategy']) + if any( + [ + f in vals + for f in [ + 'environment_options', + 'minimum_minions', + 'maximum_minions', + 'minion_max_idle_time', + ] + ] + ): + minion_pool = self._minion_pool_api.get_minion_pool(context, id) self._check_pool_numeric_values( - vals.get( - 'minimum_minions', minion_pool['minimum_minions']), - vals.get( - 'maximum_minions', minion_pool['maximum_minions']), - vals.get('minion_max_idle_time')) + vals.get('minimum_minions', minion_pool['minimum_minions']), + vals.get('maximum_minions', minion_pool['maximum_minions']), + vals.get('minion_max_idle_time'), + ) if 'environment_options' in vals: self._validate_updated_environment_options( - context, minion_pool, vals['environment_options']) + context, minion_pool, vals['environment_options'] + ) return vals def update(self, req, id, body): @@ -201,7 +247,9 @@ def update(self, req, id, body): updated_values = self._validate_update_body(id, context, body) return minion_pool_view.single( self._minion_pool_api.update( - req.environ['coriolis.context'], id, updated_values)) + req.environ['coriolis.context'], id, updated_values + ) + ) def delete(self, req, id): context = req.environ["coriolis.context"] diff --git a/coriolis/api/v1/provider_schemas.py b/coriolis/api/v1/provider_schemas.py index f36bf1c4..92c02973 100644 --- a/coriolis/api/v1/provider_schemas.py +++ b/coriolis/api/v1/provider_schemas.py @@ -15,8 +15,11 @@ def __init__(self): super(ProviderSchemasController, self).__init__() def index(self, req, platform_name, provider_type): - return {"schemas": self._provider_api.get_provider_schemas( - req.environ["coriolis.context"], platform_name, provider_type)} + return { + "schemas": self._provider_api.get_provider_schemas( + req.environ["coriolis.context"], platform_name, provider_type + ) + } def create_resource(): diff --git a/coriolis/api/v1/providers.py b/coriolis/api/v1/providers.py index 7fe52da2..b5462358 100644 --- a/coriolis/api/v1/providers.py +++ b/coriolis/api/v1/providers.py @@ -1,12 +1,12 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging + from coriolis.api import wsgi as api_wsgi from coriolis.policies import general as general_policies from coriolis.providers import api -from oslo_log import log as logging - LOG = logging.getLogger(__name__) @@ -18,8 +18,7 @@ def __init__(self): def index(self, req): context = req.environ['coriolis.context'] context.can(general_policies.get_providers_policy_label('list')) - return { - "providers": self._provider_api.get_available_providers(context)} + return {"providers": self._provider_api.get_available_providers(context)} def create_resource(): diff --git a/coriolis/api/v1/regions.py b/coriolis/api/v1/regions.py index 4cadb4c6..770b3398 100644 --- a/coriolis/api/v1/regions.py +++ b/coriolis/api/v1/regions.py @@ -1,16 +1,16 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging +from webob import exc + +from coriolis import exception +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import region_view -from coriolis.api import wsgi as api_wsgi -from coriolis import exception from coriolis.policies import regions as region_policies from coriolis.regions import api -from oslo_log import log as logging -from webob import exc - LOG = logging.getLogger(__name__) @@ -31,8 +31,7 @@ def show(self, req, id): def index(self, req): context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("list")) - return region_view.collection( - self._region_api.get_regions(context)) + return region_view.collection(self._region_api.get_regions(context)) @api_utils.format_keyerror_message(resource='region', method='create') def _validate_create_body(self, body): @@ -46,22 +45,26 @@ def create(self, req, body): context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("create")) (name, description, enabled) = self._validate_create_body(body) - return region_view.single(self._region_api.create( - context, region_name=name, description=description, - enabled=enabled)) + return region_view.single( + self._region_api.create( + context, region_name=name, description=description, enabled=enabled + ) + ) @api_utils.format_keyerror_message(resource='region', method='update') def _validate_update_body(self, body): region = body["region"] - return {k: region[k] for k in region.keys() & - {"name", "description", "enabled"}} + return { + k: region[k] for k in region.keys() & {"name", "description", "enabled"} + } def update(self, req, id, body): context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("update")) updated_values = self._validate_update_body(body) - return region_view.single(self._region_api.update( - req.environ['coriolis.context'], id, updated_values)) + return region_view.single( + self._region_api.update(req.environ['coriolis.context'], id, updated_values) + ) def delete(self, req, id): context = req.environ["coriolis.context"] diff --git a/coriolis/api/v1/router.py b/coriolis/api/v1/router.py index 168270e6..53497a53 100644 --- a/coriolis/api/v1/router.py +++ b/coriolis/api/v1/router.py @@ -4,30 +4,32 @@ from oslo_log import log as logging from coriolis import api -from coriolis.api.v1 import deployment_actions -from coriolis.api.v1 import deployments -from coriolis.api.v1 import diagnostics -from coriolis.api.v1 import endpoint_actions -from coriolis.api.v1 import endpoint_destination_minion_pool_options -from coriolis.api.v1 import endpoint_destination_options -from coriolis.api.v1 import endpoint_instances -from coriolis.api.v1 import endpoint_inventory -from coriolis.api.v1 import endpoint_networks -from coriolis.api.v1 import endpoint_source_minion_pool_options -from coriolis.api.v1 import endpoint_source_options -from coriolis.api.v1 import endpoint_storage -from coriolis.api.v1 import endpoints -from coriolis.api.v1 import minion_pool_actions -from coriolis.api.v1 import minion_pools -from coriolis.api.v1 import provider_schemas -from coriolis.api.v1 import providers -from coriolis.api.v1 import regions -from coriolis.api.v1 import services -from coriolis.api.v1 import transfer_actions -from coriolis.api.v1 import transfer_schedules -from coriolis.api.v1 import transfer_tasks_execution_actions -from coriolis.api.v1 import transfer_tasks_executions -from coriolis.api.v1 import transfers +from coriolis.api.v1 import ( + deployment_actions, + deployments, + diagnostics, + endpoint_actions, + endpoint_destination_minion_pool_options, + endpoint_destination_options, + endpoint_instances, + endpoint_inventory, + endpoint_networks, + endpoint_source_minion_pool_options, + endpoint_source_options, + endpoint_storage, + endpoints, + minion_pool_actions, + minion_pools, + provider_schemas, + providers, + regions, + services, + transfer_actions, + transfer_schedules, + transfer_tasks_execution_actions, + transfer_tasks_executions, + transfers, +) LOG = logging.getLogger(__name__) @@ -47,163 +49,212 @@ def _setup_routes(self, mapper, ext_mgr): mapper.redirect("", "/") self.resources['providers'] = providers.create_resource() - mapper.resource('provider', 'providers', - controller=self.resources['providers']) + mapper.resource('provider', 'providers', controller=self.resources['providers']) self.resources['regions'] = regions.create_resource() - mapper.resource('region', 'regions', - controller=self.resources['regions'], - collection={'detail': 'GET'}) + mapper.resource( + 'region', + 'regions', + controller=self.resources['regions'], + collection={'detail': 'GET'}, + ) self.resources['endpoints'] = endpoints.create_resource() - mapper.resource('endpoint', 'endpoints', - controller=self.resources['endpoints'], - collection={'detail': 'GET'}, - member={'action': 'POST'}) + mapper.resource( + 'endpoint', + 'endpoints', + controller=self.resources['endpoints'], + collection={'detail': 'GET'}, + member={'action': 'POST'}, + ) self.resources['services'] = services.create_resource() - mapper.resource('service', 'services', - controller=self.resources['services'], - collection={'detail': 'GET'}) + mapper.resource( + 'service', + 'services', + controller=self.resources['services'], + collection={'detail': 'GET'}, + ) self.resources['minion_pools'] = minion_pools.create_resource() - mapper.resource('minion_pool', 'minion_pools', - controller=self.resources['minion_pools'], - collection={'detail': 'GET'}) + mapper.resource( + 'minion_pool', + 'minion_pools', + controller=self.resources['minion_pools'], + collection={'detail': 'GET'}, + ) minion_pool_actions_resource = minion_pool_actions.create_resource() self.resources['minion_pool_actions'] = minion_pool_actions_resource minion_pool_path = '/{project_id}/minion_pools/{id}' - mapper.connect('minion_pool_actions', - minion_pool_path + '/actions', - controller=self.resources['minion_pool_actions'], - action='action', - conditions={'method': 'POST'}) - - self.resources['endpoint_source_minion_pool_options'] = \ + mapper.connect( + 'minion_pool_actions', + minion_pool_path + '/actions', + controller=self.resources['minion_pool_actions'], + action='action', + conditions={'method': 'POST'}, + ) + + self.resources['endpoint_source_minion_pool_options'] = ( endpoint_source_minion_pool_options.create_resource() - mapper.resource('minion_pool_options', - 'endpoints/{endpoint_id}/source-minion-pool-options', - controller=( - self.resources[ - 'endpoint_source_minion_pool_options'])) + ) + mapper.resource( + 'minion_pool_options', + 'endpoints/{endpoint_id}/source-minion-pool-options', + controller=(self.resources['endpoint_source_minion_pool_options']), + ) - self.resources['endpoint_destination_minion_pool_options'] = \ + self.resources['endpoint_destination_minion_pool_options'] = ( endpoint_destination_minion_pool_options.create_resource() + ) mapper.resource( 'minion_pool_options', 'endpoints/{endpoint_id}/destination-minion-pool-options', - controller=(self.resources - ['endpoint_destination_minion_pool_options'])) + controller=(self.resources['endpoint_destination_minion_pool_options']), + ) endpoint_actions_resource = endpoint_actions.create_resource() self.resources['endpoint_actions'] = endpoint_actions_resource endpoint_path = '/{project_id}/endpoints/{id}' - mapper.connect('endpoint_actions', - endpoint_path + '/actions', - controller=self.resources['endpoint_actions'], - action='action', - conditions={'method': 'POST'}) - - self.resources['endpoint_instances'] = \ - endpoint_instances.create_resource() - mapper.resource('instance', 'endpoints/{endpoint_id}/instances', - controller=self.resources['endpoint_instances']) - - self.resources['endpoint_inventory'] = \ - endpoint_inventory.create_resource() - mapper.resource('inventory', 'endpoints/{endpoint_id}/inventory', - controller=self.resources['endpoint_inventory']) - - self.resources['endpoint_networks'] = \ - endpoint_networks.create_resource() - mapper.resource('network', 'endpoints/{endpoint_id}/networks', - controller=self.resources['endpoint_networks']) - - self.resources['endpoint_storage'] = \ - endpoint_storage.create_resource() - mapper.resource('storage', 'endpoints/{endpoint_id}/storage', - controller=self.resources['endpoint_storage']) - - self.resources['endpoint_destination_options'] = \ + mapper.connect( + 'endpoint_actions', + endpoint_path + '/actions', + controller=self.resources['endpoint_actions'], + action='action', + conditions={'method': 'POST'}, + ) + + self.resources['endpoint_instances'] = endpoint_instances.create_resource() + mapper.resource( + 'instance', + 'endpoints/{endpoint_id}/instances', + controller=self.resources['endpoint_instances'], + ) + + self.resources['endpoint_inventory'] = endpoint_inventory.create_resource() + mapper.resource( + 'inventory', + 'endpoints/{endpoint_id}/inventory', + controller=self.resources['endpoint_inventory'], + ) + + self.resources['endpoint_networks'] = endpoint_networks.create_resource() + mapper.resource( + 'network', + 'endpoints/{endpoint_id}/networks', + controller=self.resources['endpoint_networks'], + ) + + self.resources['endpoint_storage'] = endpoint_storage.create_resource() + mapper.resource( + 'storage', + 'endpoints/{endpoint_id}/storage', + controller=self.resources['endpoint_storage'], + ) + + self.resources['endpoint_destination_options'] = ( endpoint_destination_options.create_resource() - mapper.resource('destination_options', - 'endpoints/{endpoint_id}/destination-options', - controller=( - self.resources['endpoint_destination_options'])) + ) + mapper.resource( + 'destination_options', + 'endpoints/{endpoint_id}/destination-options', + controller=(self.resources['endpoint_destination_options']), + ) - self.resources['endpoint_source_options'] = \ + self.resources['endpoint_source_options'] = ( endpoint_source_options.create_resource() - mapper.resource('source_options', - 'endpoints/{endpoint_id}/source-options', - controller=( - self.resources['endpoint_source_options'])) + ) + mapper.resource( + 'source_options', + 'endpoints/{endpoint_id}/source-options', + controller=(self.resources['endpoint_source_options']), + ) - self.resources['provider_schemas'] = \ - provider_schemas.create_resource() - mapper.resource('provider_schemas', - 'providers/{platform_name}/schemas/{provider_type}', - controller=self.resources['provider_schemas']) + self.resources['provider_schemas'] = provider_schemas.create_resource() + mapper.resource( + 'provider_schemas', + 'providers/{platform_name}/schemas/{provider_type}', + controller=self.resources['provider_schemas'], + ) self.resources['deployments'] = deployments.create_resource() - mapper.resource('deployment', 'deployments', - controller=self.resources['deployments'], - collection={'detail': 'GET'}, - member={'action': 'POST'}) + mapper.resource( + 'deployment', + 'deployments', + controller=self.resources['deployments'], + collection={'detail': 'GET'}, + member={'action': 'POST'}, + ) deployments_actions_resource = deployment_actions.create_resource() self.resources['deployment_actions'] = deployments_actions_resource deployment_path = '/{project_id}/deployments/{id}' - mapper.connect('deployment_actions', - deployment_path + '/actions', - controller=self.resources['deployment_actions'], - action='action', - conditions={'method': 'POST'}) + mapper.connect( + 'deployment_actions', + deployment_path + '/actions', + controller=self.resources['deployment_actions'], + action='action', + conditions={'method': 'POST'}, + ) self.resources['transfers'] = transfers.create_resource() - mapper.resource('transfer', 'transfers', - controller=self.resources['transfers'], - collection={'detail': 'GET'}, - member={'action': 'POST'}) + mapper.resource( + 'transfer', + 'transfers', + controller=self.resources['transfers'], + collection={'detail': 'GET'}, + member={'action': 'POST'}, + ) transfer_actions_resource = transfer_actions.create_resource() self.resources['transfer_actions'] = transfer_actions_resource migration_path = '/{project_id}/transfers/{id}' - mapper.connect('transfer_actions', - migration_path + '/actions', - controller=self.resources['transfer_actions'], - action='action', - conditions={'method': 'POST'}) - - self.resources['transfer_tasks_executions'] = \ + mapper.connect( + 'transfer_actions', + migration_path + '/actions', + controller=self.resources['transfer_actions'], + action='action', + conditions={'method': 'POST'}, + ) + + self.resources['transfer_tasks_executions'] = ( transfer_tasks_executions.create_resource() - mapper.resource('execution', 'transfers/{transfer_id}/executions', - controller=self.resources['transfer_tasks_executions'], - collection={'detail': 'GET'}, - member={'action': 'POST'}) - - transfer_tasks_execution_actions_resource = \ + ) + mapper.resource( + 'execution', + 'transfers/{transfer_id}/executions', + controller=self.resources['transfer_tasks_executions'], + collection={'detail': 'GET'}, + member={'action': 'POST'}, + ) + + transfer_tasks_execution_actions_resource = ( transfer_tasks_execution_actions.create_resource() - self.resources['transfer_tasks_execution_actions'] = \ + ) + self.resources['transfer_tasks_execution_actions'] = ( transfer_tasks_execution_actions_resource - migration_path = ('/{project_id}/transfers/{transfer_id}/' - 'executions/{id}') - mapper.connect('transfer_tasks_execution_actions', - migration_path + '/actions', - controller=self.resources[ - 'transfer_tasks_execution_actions'], - action='action', - conditions={'method': 'POST'}) + ) + migration_path = '/{project_id}/transfers/{transfer_id}/executions/{id}' + mapper.connect( + 'transfer_tasks_execution_actions', + migration_path + '/actions', + controller=self.resources['transfer_tasks_execution_actions'], + action='action', + conditions={'method': 'POST'}, + ) sched = transfer_schedules.create_resource() self.resources['transfer_schedules'] = sched - mapper.resource('transfer_schedule', - 'transfers/{transfer_id}/schedules', - controller=self.resources['transfer_schedules'], - collection={'index': 'GET'}, - member={'action': 'POST'}) + mapper.resource( + 'transfer_schedule', + 'transfers/{transfer_id}/schedules', + controller=self.resources['transfer_schedules'], + collection={'index': 'GET'}, + member={'action': 'POST'}, + ) diag = diagnostics.create_resource() self.resources['diagnostics'] = diag - mapper.resource('diagnostics', 'diagnostics', - controller=self.resources['diagnostics']) + mapper.resource( + 'diagnostics', 'diagnostics', controller=self.resources['diagnostics'] + ) diff --git a/coriolis/api/v1/services.py b/coriolis/api/v1/services.py index b2bd267b..92b99341 100644 --- a/coriolis/api/v1/services.py +++ b/coriolis/api/v1/services.py @@ -1,16 +1,16 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging +from webob import exc + +from coriolis import exception +from coriolis.api import wsgi as api_wsgi from coriolis.api.v1 import utils as api_utils from coriolis.api.v1.views import service_view -from coriolis.api import wsgi as api_wsgi -from coriolis import exception from coriolis.policies import services as service_policies from coriolis.services import api -from oslo_log import log as logging -from webob import exc - LOG = logging.getLogger(__name__) @@ -31,8 +31,7 @@ def show(self, req, id): def index(self, req): context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("list")) - return service_view.collection( - self._service_api.get_services(context)) + return service_view.collection(self._service_api.get_services(context)) @api_utils.format_keyerror_message(resource='service', method='create') def _validate_create_body(self, body): @@ -47,24 +46,34 @@ def _validate_create_body(self, body): def create(self, req, body): context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("create")) - (host, binary, topic, mapped_regions, enabled) = ( - self._validate_create_body(body)) - return service_view.single(self._service_api.create( - context, host=host, binary=binary, topic=topic, - mapped_regions=mapped_regions, enabled=enabled)) + (host, binary, topic, mapped_regions, enabled) = self._validate_create_body( + body + ) + return service_view.single( + self._service_api.create( + context, + host=host, + binary=binary, + topic=topic, + mapped_regions=mapped_regions, + enabled=enabled, + ) + ) @api_utils.format_keyerror_message(resource='service', method='update') def _validate_update_body(self, body): service = body["service"] - return {k: service[k] for k in service.keys() & { - "enabled", "mapped_regions"}} + return {k: service[k] for k in service.keys() & {"enabled", "mapped_regions"}} def update(self, req, id, body): context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("update")) updated_values = self._validate_update_body(body) - return service_view.single(self._service_api.update( - req.environ['coriolis.context'], id, updated_values)) + return service_view.single( + self._service_api.update( + req.environ['coriolis.context'], id, updated_values + ) + ) def delete(self, req, id): context = req.environ["coriolis.context"] diff --git a/coriolis/api/v1/transfer_actions.py b/coriolis/api/v1/transfer_actions.py index 487e0099..c5f2b230 100644 --- a/coriolis/api/v1/transfer_actions.py +++ b/coriolis/api/v1/transfer_actions.py @@ -1,14 +1,14 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import transfer_tasks_execution_view -from coriolis.api import wsgi as api_wsgi +from webob import exc + from coriolis import exception +from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import transfer_tasks_execution_view from coriolis.policies import transfers as transfer_policies from coriolis.transfers import api -from webob import exc - class TransferActionsController(api_wsgi.Controller): def __init__(self): @@ -18,11 +18,11 @@ def __init__(self): @api_wsgi.action('delete-disks') def _delete_disks(self, req, id, body): context = req.environ['coriolis.context'] - context.can( - transfer_policies.get_transfers_policy_label("delete_disks")) + context.can(transfer_policies.get_transfers_policy_label("delete_disks")) try: return transfer_tasks_execution_view.single( - self._transfer_api.delete_disks(context, id)) + self._transfer_api.delete_disks(context, id) + ) except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: diff --git a/coriolis/api/v1/transfer_schedules.py b/coriolis/api/v1/transfer_schedules.py index e602390a..2e096444 100644 --- a/coriolis/api/v1/transfer_schedules.py +++ b/coriolis/api/v1/transfer_schedules.py @@ -1,19 +1,16 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.api.v1.views import transfer_schedule_view -from coriolis.api import wsgi as api_wsgi -from coriolis import exception -from coriolis.policies import transfer_schedules as schedules_policies -from coriolis import schemas -from coriolis.transfer_cron import api - import jsonschema from oslo_log import log as logging -from oslo_utils import strutils -from oslo_utils import timeutils +from oslo_utils import strutils, timeutils from webob import exc +from coriolis import exception, schemas +from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1.views import transfer_schedule_view +from coriolis.policies import transfer_schedules as schedules_policies +from coriolis.transfer_cron import api LOG = logging.getLogger(__name__) @@ -25,8 +22,7 @@ def __init__(self): def show(self, req, transfer_id, id): context = req.environ["coriolis.context"] - context.can( - schedules_policies.get_transfer_schedules_policy_label("show")) + context.can(schedules_policies.get_transfer_schedules_policy_label("show")) schedule = self._schedule_api.get_schedule(context, transfer_id, id) if not schedule: raise exc.HTTPNotFound() @@ -35,14 +31,14 @@ def show(self, req, transfer_id, id): def index(self, req, transfer_id): context = req.environ["coriolis.context"] - context.can( - schedules_policies.get_transfer_schedules_policy_label("list")) + context.can(schedules_policies.get_transfer_schedules_policy_label("list")) show_expired = strutils.bool_from_string( - req.GET.get("show_expired", True), strict=True) + req.GET.get("show_expired", True), strict=True + ) return transfer_schedule_view.collection( - self._schedule_api.get_schedules( - context, transfer_id, expired=show_expired)) + self._schedule_api.get_schedules(context, transfer_id, expired=show_expired) + ) def _validate_schedule(self, schedule): schema = schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"] @@ -52,23 +48,22 @@ def _validate_schedule(self, schedule): def _validate_expiration_date(self, expiration_date): if expiration_date is None: return expiration_date - exp = timeutils.normalize_time( - timeutils.parse_isotime(expiration_date)) + exp = timeutils.normalize_time(timeutils.parse_isotime(expiration_date)) now = timeutils.utcnow() if now > exp: - raise exception.InvalidInput( - "expiration_date is in the past") + raise exception.InvalidInput("expiration_date is in the past") return exp def _validate_create_body(self, body): schedule = body.get("schedule") if schedule is None: - raise exception.InvalidInput( - "schedule is required") + raise exception.InvalidInput("schedule is required") schedule = self._validate_schedule(schedule) schemas.validate_value( - body, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=jsonschema.FormatChecker()) + body, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=jsonschema.FormatChecker(), + ) enabled = body.get("enabled", True) exp = body.get("expiration_date", None) @@ -94,52 +89,53 @@ def _validate_update_body(self, update_body): if auto_deploy is not None: body['auto_deploy'] = auto_deploy schemas.validate_value( - body, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=jsonschema.FormatChecker()) + body, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=jsonschema.FormatChecker(), + ) exp = None if "expiration_date" in update_body: - exp = self._validate_expiration_date( - update_body.get("expiration_date")) + exp = self._validate_expiration_date(update_body.get("expiration_date")) body["expiration_date"] = exp return body def create(self, req, transfer_id, body): context = req.environ["coriolis.context"] - context.can( - schedules_policies.get_transfer_schedules_policy_label("create")) + context.can(schedules_policies.get_transfer_schedules_policy_label("create")) LOG.debug("Got request: %r %r %r" % (req, transfer_id, body)) try: schedule, enabled, exp_date, shutdown, auto_deploy = ( - self._validate_create_body(body)) + self._validate_create_body(body) + ) except Exception as err: raise exception.InvalidInput(err) - return transfer_schedule_view.single(self._schedule_api.create( - context, transfer_id, schedule, enabled, exp_date, shutdown, - auto_deploy)) + return transfer_schedule_view.single( + self._schedule_api.create( + context, transfer_id, schedule, enabled, exp_date, shutdown, auto_deploy + ) + ) def update(self, req, transfer_id, id, body): context = req.environ["coriolis.context"] - context.can( - schedules_policies.get_transfer_schedules_policy_label("update")) + context.can(schedules_policies.get_transfer_schedules_policy_label("update")) - LOG.debug("Got request: %r %r %r %r" % ( - req, transfer_id, id, body)) + LOG.debug("Got request: %r %r %r %r" % (req, transfer_id, id, body)) try: update_values = self._validate_update_body(body) except Exception as err: raise exception.InvalidInput(err) - return transfer_schedule_view.single(self._schedule_api.update( - context, transfer_id, id, update_values)) + return transfer_schedule_view.single( + self._schedule_api.update(context, transfer_id, id, update_values) + ) def delete(self, req, transfer_id, id): context = req.environ["coriolis.context"] - context.can( - schedules_policies.get_transfer_schedules_policy_label("delete")) + context.can(schedules_policies.get_transfer_schedules_policy_label("delete")) self._schedule_api.delete(context, transfer_id, id) raise exc.HTTPNoContent() diff --git a/coriolis/api/v1/transfer_tasks_execution_actions.py b/coriolis/api/v1/transfer_tasks_execution_actions.py index 27998046..1f81049a 100644 --- a/coriolis/api/v1/transfer_tasks_execution_actions.py +++ b/coriolis/api/v1/transfer_tasks_execution_actions.py @@ -3,8 +3,8 @@ from webob import exc -from coriolis.api import wsgi as api_wsgi from coriolis import exception +from coriolis.api import wsgi as api_wsgi from coriolis.policies import transfer_tasks_executions as execution_policies from coriolis.transfer_tasks_executions import api @@ -17,13 +17,11 @@ def __init__(self): @api_wsgi.action('cancel') def _cancel(self, req, transfer_id, id, body): context = req.environ['coriolis.context'] - context.can( - execution_policies.get_transfer_executions_policy_label('cancel')) + context.can(execution_policies.get_transfer_executions_policy_label('cancel')) try: force = (body["cancel"] or {}).get("force", False) - self._transfer_tasks_execution_api.cancel( - context, transfer_id, id, force) + self._transfer_tasks_execution_api.cancel(context, transfer_id, id, force) raise exc.HTTPNoContent() except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) diff --git a/coriolis/api/v1/transfer_tasks_executions.py b/coriolis/api/v1/transfer_tasks_executions.py index 6fcb5eef..8c0bfaf7 100644 --- a/coriolis/api/v1/transfer_tasks_executions.py +++ b/coriolis/api/v1/transfer_tasks_executions.py @@ -1,15 +1,15 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from webob import exc + +from coriolis import exception from coriolis.api import common -from coriolis.api.v1.views import transfer_tasks_execution_view from coriolis.api import wsgi as api_wsgi -from coriolis import exception +from coriolis.api.v1.views import transfer_tasks_execution_view from coriolis.policies import transfer_tasks_executions as executions_policies from coriolis.transfer_tasks_executions import api -from webob import exc - class TransferTasksExecutionController(api_wsgi.Controller): def __init__(self): @@ -18,10 +18,10 @@ def __init__(self): def show(self, req, transfer_id, id): context = req.environ["coriolis.context"] - context.can( - executions_policies.get_transfer_executions_policy_label("show")) + context.can(executions_policies.get_transfer_executions_policy_label("show")) execution = self._transfer_tasks_execution_api.get_execution( - context, transfer_id, id) + context, transfer_id, id + ) if not execution: raise exc.HTTPNotFound() @@ -29,31 +29,36 @@ def show(self, req, transfer_id, id): def index(self, req, transfer_id): context = req.environ["coriolis.context"] - context.can( - executions_policies.get_transfer_executions_policy_label("list")) + context.can(executions_policies.get_transfer_executions_policy_label("list")) marker, limit = common.get_paging_params(req) sort_keys, sort_dirs = common.get_sort_params(req) return transfer_tasks_execution_view.collection( self._transfer_tasks_execution_api.get_executions( - context, transfer_id, include_tasks=False, - marker=marker, limit=limit, - sort_keys=sort_keys, sort_dirs=sort_dirs)) + context, + transfer_id, + include_tasks=False, + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + ) + ) def detail(self, req, transfer_id): context = req.environ["coriolis.context"] - context.can( - executions_policies.get_transfer_executions_policy_label("show")) + context.can(executions_policies.get_transfer_executions_policy_label("show")) return transfer_tasks_execution_view.collection( self._transfer_tasks_execution_api.get_executions( - context, transfer_id, include_tasks=True)) + context, transfer_id, include_tasks=True + ) + ) def create(self, req, transfer_id, body): context = req.environ["coriolis.context"] - context.can( - executions_policies.get_transfer_executions_policy_label("create")) + context.can(executions_policies.get_transfer_executions_policy_label("create")) # TODO(alexpilotti): validate body @@ -63,12 +68,13 @@ def create(self, req, transfer_id, body): return transfer_tasks_execution_view.single( self._transfer_tasks_execution_api.create( - context, transfer_id, shutdown_instances, auto_deploy)) + context, transfer_id, shutdown_instances, auto_deploy + ) + ) def delete(self, req, transfer_id, id): context = req.environ["coriolis.context"] - context.can( - executions_policies.get_transfer_executions_policy_label("delete")) + context.can(executions_policies.get_transfer_executions_policy_label("delete")) try: self._transfer_tasks_execution_api.delete(context, transfer_id, id) diff --git a/coriolis/api/v1/transfers.py b/coriolis/api/v1/transfers.py index 92df1fde..4fbf75c9 100644 --- a/coriolis/api/v1/transfers.py +++ b/coriolis/api/v1/transfers.py @@ -1,26 +1,24 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from oslo_log import log as logging +from webob import exc + +from coriolis import constants, exception from coriolis.api import common -from coriolis.api.v1 import utils as api_utils -from coriolis.api.v1.views import transfer_tasks_execution_view -from coriolis.api.v1.views import transfer_view from coriolis.api import wsgi as api_wsgi -from coriolis import constants +from coriolis.api.v1 import utils as api_utils +from coriolis.api.v1.views import transfer_tasks_execution_view, transfer_view from coriolis.endpoints import api as endpoints_api -from coriolis import exception from coriolis.policies import transfers as transfer_policies from coriolis.transfers import api -from oslo_log import log as logging -from webob import exc - - LOG = logging.getLogger(__name__) SUPPORTED_TRANSFER_SCENARIOS = [ constants.TRANSFER_SCENARIO_REPLICA, - constants.TRANSFER_SCENARIO_LIVE_MIGRATION] + constants.TRANSFER_SCENARIO_LIVE_MIGRATION, +] class TransferController(api_wsgi.Controller): @@ -33,23 +31,24 @@ def show(self, req, id): context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("show")) include_task_info = api_utils.get_bool_url_arg( - req, "include_task_info", default=False) + req, "include_task_info", default=False + ) transfer = self._transfer_api.get_transfer( - context, id, - include_task_info=include_task_info) + context, id, include_task_info=include_task_info + ) if not transfer: raise exc.HTTPNotFound() return transfer_view.single(transfer) def _list(self, req): - show_deleted = api_utils.get_bool_url_arg( - req, "show_deleted", default=False) + show_deleted = api_utils.get_bool_url_arg(req, "show_deleted", default=False) context = req.environ["coriolis.context"] context.show_deleted = show_deleted context.can(transfer_policies.get_transfers_policy_label("list")) include_task_info = api_utils.get_bool_url_arg( - req, "include_task_info", default=False) + req, "include_task_info", default=False + ) marker, limit = common.get_paging_params(req) sort_keys, sort_dirs = common.get_sort_params(req) return transfer_view.collection( @@ -57,9 +56,12 @@ def _list(self, req): context, include_tasks_executions=include_task_info, include_task_info=include_task_info, - marker=marker, limit=limit, - sort_keys=sort_keys, sort_dirs=sort_dirs, - )) + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + ) + ) def index(self, req): return self._list(req) @@ -76,42 +78,46 @@ def _validate_create_body(self, context, body): if scenario not in SUPPORTED_TRANSFER_SCENARIOS: raise exc.HTTPBadRequest( explanation=f"Unsupported Transfer creation scenario " - f"'{scenario}', must be one of: " # noqa - f"{SUPPORTED_TRANSFER_SCENARIOS}") # noqa + f"'{scenario}', must be one of: " # noqa + f"{SUPPORTED_TRANSFER_SCENARIOS}" + ) # noqa else: scenario = constants.TRANSFER_SCENARIO_REPLICA LOG.warn( "No Transfer 'scenario' field set in Transfer body, " - f"defaulting to: '{scenario}'") + f"defaulting to: '{scenario}'" + ) origin_endpoint_id = transfer["origin_endpoint_id"] destination_endpoint_id = transfer["destination_endpoint_id"] - destination_environment = transfer.get( - "destination_environment", {}) + destination_environment = transfer.get("destination_environment", {}) instances = api_utils.validate_instances_list_for_transfer( - transfer.get('instances')) + transfer.get('instances') + ) notes = transfer.get("notes") source_environment = transfer.get("source_environment", {}) self._endpoints_api.validate_source_environment( - context, origin_endpoint_id, source_environment) + context, origin_endpoint_id, source_environment + ) - origin_minion_pool_id = transfer.get( - 'origin_minion_pool_id') - destination_minion_pool_id = transfer.get( - 'destination_minion_pool_id') + origin_minion_pool_id = transfer.get('origin_minion_pool_id') + destination_minion_pool_id = transfer.get('destination_minion_pool_id') instance_osmorphing_minion_pool_mappings = transfer.get( - 'instance_osmorphing_minion_pool_mappings', {}) + 'instance_osmorphing_minion_pool_mappings', {} + ) extras = [ instance for instance in instance_osmorphing_minion_pool_mappings - if instance not in instances] + if instance not in instances + ] if extras: raise ValueError( "One or more instance OSMorphing pool mappings were " "provided for instances (%s) which are not part of the " - "Transfer's declared instances (%s)" % (extras, instances)) + "Transfer's declared instances (%s)" % (extras, instances) + ) # TODO(aznashwan): until the provider plugin interface is updated # to have separate 'network_map' and 'storage_mappings' fields, @@ -120,7 +126,8 @@ def _validate_create_body(self, context, body): api_utils.validate_network_map(network_map) destination_environment['network_map'] = network_map self._endpoints_api.validate_target_environment( - context, destination_endpoint_id, destination_environment) + context, destination_endpoint_id, destination_environment + ) user_scripts = transfer.get('user_scripts', {}) api_utils.validate_user_scripts(user_scripts) @@ -137,32 +144,66 @@ def _validate_create_body(self, context, body): clone_disks = transfer.get('clone_disks', True) skip_os_morphing = transfer.get('skip_os_morphing', False) - return (scenario, origin_endpoint_id, destination_endpoint_id, - source_environment, destination_environment, instances, - network_map, storage_mappings, notes, - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, user_scripts, - clone_disks, skip_os_morphing) + return ( + scenario, + origin_endpoint_id, + destination_endpoint_id, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + user_scripts, + clone_disks, + skip_os_morphing, + ) def create(self, req, body): context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("create")) - (scenario, origin_endpoint_id, destination_endpoint_id, - source_environment, destination_environment, instances, network_map, - storage_mappings, notes, origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, user_scripts, clone_disks, - skip_os_morphing) = ( - self._validate_create_body(context, body)) - - return transfer_view.single(self._transfer_api.create( - context, scenario, origin_endpoint_id, destination_endpoint_id, - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, source_environment, - destination_environment, instances, network_map, - storage_mappings, notes, user_scripts, clone_disks, - skip_os_morphing)) + ( + scenario, + origin_endpoint_id, + destination_endpoint_id, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + user_scripts, + clone_disks, + skip_os_morphing, + ) = self._validate_create_body(context, body) + + return transfer_view.single( + self._transfer_api.create( + context, + scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes, + user_scripts, + clone_disks, + skip_os_morphing, + ) + ) def delete(self, req, id): context = req.environ["coriolis.context"] @@ -174,46 +215,44 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) @staticmethod - def _update_storage_mappings(original_storage_mappings, - new_storage_mappings): + def _update_storage_mappings(original_storage_mappings, new_storage_mappings): - backend_mappings = original_storage_mappings.get( - 'backend_mappings', []) - new_backend_mappings = new_storage_mappings.get( - 'backend_mappings', []) - new_backend_mapping_sources = [mapping['source'] for mapping in - new_backend_mappings] + backend_mappings = original_storage_mappings.get('backend_mappings', []) + new_backend_mappings = new_storage_mappings.get('backend_mappings', []) + new_backend_mapping_sources = [ + mapping['source'] for mapping in new_backend_mappings + ] disk_mappings = original_storage_mappings.get('disk_mappings', []) new_disk_mappings = new_storage_mappings.get('disk_mappings', []) - new_disk_mappings_disk_ids = [mapping['disk_id'] for mapping in - new_disk_mappings] + new_disk_mappings_disk_ids = [ + mapping['disk_id'] for mapping in new_disk_mappings + ] non_duplicates_backend_mapping = [] for mapping in backend_mappings: if mapping['source'] not in new_backend_mapping_sources: non_duplicates_backend_mapping.append(mapping) else: - LOG.info("Storage Backend Mapping %s will be overwritten." % - mapping) + LOG.info("Storage Backend Mapping %s will be overwritten." % mapping) non_duplicates_disk_mappings = [] for mapping in disk_mappings: if mapping['disk_id'] not in new_disk_mappings_disk_ids: non_duplicates_disk_mappings.append(mapping) else: - LOG.info("Storage Disk Mapping %s will be overwritten" % - mapping) + LOG.info("Storage Disk Mapping %s will be overwritten" % mapping) non_duplicates_backend_mapping.extend(new_backend_mappings) non_duplicates_disk_mappings.extend(new_disk_mappings) storage_mappings = { 'backend_mappings': non_duplicates_backend_mapping, - 'disk_mappings': non_duplicates_disk_mappings} + 'disk_mappings': non_duplicates_disk_mappings, + } - default_storage_backend = ( - new_storage_mappings.get('default', None) or - original_storage_mappings.get('default', None)) + default_storage_backend = new_storage_mappings.get( + 'default', None + ) or original_storage_mappings.get('default', None) if default_storage_backend: storage_mappings['default'] = default_storage_backend @@ -239,7 +278,7 @@ def _get_updated_user_scripts(original_user_scripts, new_user_scripts): return user_scripts def _get_merged_transfer_values(self, transfer, updated_values): - """ Looks for the following keys in the original transfer body and + """Looks for the following keys in the original transfer body and updated values (preferring the updated values where needed, but using `.update()` on dicts): "source_environment", "destination_environment", "network_map", "notes" @@ -250,9 +289,7 @@ def _get_merged_transfer_values(self, transfer, updated_values): final_values = {} # NOTE: this just replaces options at the top-level and does not do # merging of container types (ex: lists, dicts) - for option in [ - "source_environment", "destination_environment", - "network_map"]: + for option in ["source_environment", "destination_environment", "network_map"]: before = transfer.get(option) after = updated_values.get(option) # NOTE: for Transfers created before the separation of these fields @@ -272,14 +309,18 @@ def _get_merged_transfer_values(self, transfer, updated_values): if new_storage_mappings is None: new_storage_mappings = {} final_values['storage_mappings'] = self._update_storage_mappings( - original_storage_mappings, new_storage_mappings) + original_storage_mappings, new_storage_mappings + ) original_user_scripts = api_utils.validate_user_scripts( - transfer.get('user_scripts', {})) + transfer.get('user_scripts', {}) + ) new_user_scripts = api_utils.validate_user_scripts( - updated_values.get('user_scripts', {})) + updated_values.get('user_scripts', {}) + ) final_values['user_scripts'] = self._get_updated_user_scripts( - original_user_scripts, new_user_scripts) + original_user_scripts, new_user_scripts + ) if 'notes' in updated_values: final_values['notes'] = updated_values.get('notes', '') @@ -299,19 +340,24 @@ def _get_merged_transfer_values(self, transfer, updated_values): final_storage_mappings = final_values['storage_mappings'] final_network_map = final_values['network_map'] if final_storage_mappings: - final_values['destination_environment'][ - 'storage_mappings'] = final_storage_mappings + final_values['destination_environment']['storage_mappings'] = ( + final_storage_mappings + ) if final_network_map: - final_values['destination_environment'][ - 'network_map'] = final_network_map + final_values['destination_environment']['network_map'] = final_network_map minion_pool_fields = [ - "origin_minion_pool_id", "destination_minion_pool_id", - "instance_osmorphing_minion_pool_mappings"] - final_values.update({ - mpf: updated_values[mpf] - for mpf in minion_pool_fields - if mpf in updated_values}) + "origin_minion_pool_id", + "destination_minion_pool_id", + "instance_osmorphing_minion_pool_mappings", + ] + final_values.update( + { + mpf: updated_values[mpf] + for mpf in minion_pool_fields + if mpf in updated_values + } + ) return final_values @@ -323,46 +369,44 @@ def _validate_update_body(self, id, context, body): if scenario and scenario != transfer["scenario"]: raise exc.HTTPBadRequest( explanation=f"Changing Transfer creation scenario is not " - f"supported (original scenario is " # noqa - f"{transfer['scenario']}, received '{scenario}')") # noqa + f"supported (original scenario is " # noqa + f"{transfer['scenario']}, received '{scenario}')" + ) # noqa transfer_body = body['transfer'] origin_endpoint_id = transfer_body.get('origin_endpoint_id', None) - destination_endpoint_id = transfer_body.get( - 'destination_endpoint_id', None) + destination_endpoint_id = transfer_body.get('destination_endpoint_id', None) instances = body['transfer'].get('instances', None) if origin_endpoint_id or destination_endpoint_id: raise exc.HTTPBadRequest( explanation="The source or destination endpoints for a " - "Coriolis Transfer cannot be updated after its " - "creation. If the credentials of any of the " - "Transfer's endpoints need updating, please " - "update the endpoints themselves.") + "Coriolis Transfer cannot be updated after its " + "creation. If the credentials of any of the " + "Transfer's endpoints need updating, please " + "update the endpoints themselves." + ) if instances: raise exc.HTTPBadRequest( - explanation="The list of instances of a Transfer cannot be " - "updated") + explanation="The list of instances of a Transfer cannot be updated" + ) - merged_body = self._get_merged_transfer_values( - transfer, transfer_body) + merged_body = self._get_merged_transfer_values(transfer, transfer_body) transfer_origin_endpoint_id = transfer["origin_endpoint_id"] - transfer_destination_endpoint_id = transfer[ - "destination_endpoint_id"] + transfer_destination_endpoint_id = transfer["destination_endpoint_id"] self._endpoints_api.validate_source_environment( - context, transfer_origin_endpoint_id, - merged_body["source_environment"]) + context, transfer_origin_endpoint_id, merged_body["source_environment"] + ) destination_environment = merged_body["destination_environment"] self._endpoints_api.validate_target_environment( - context, transfer_destination_endpoint_id, - destination_environment) + context, transfer_destination_endpoint_id, destination_environment + ) api_utils.validate_network_map(merged_body["network_map"]) - api_utils.validate_storage_mappings( - merged_body["storage_mappings"]) + api_utils.validate_storage_mappings(merged_body["storage_mappings"]) user_scripts = merged_body['user_scripts'] api_utils.validate_user_scripts(user_scripts) @@ -376,8 +420,10 @@ def update(self, req, id, body): updated_values = self._validate_update_body(id, context, body) try: return transfer_tasks_execution_view.single( - self._transfer_api.update(req.environ['coriolis.context'], - id, updated_values)) + self._transfer_api.update( + req.environ['coriolis.context'], id, updated_values + ) + ) except exception.NotFound as ex: raise exc.HTTPNotFound(explanation=ex.msg) except exception.InvalidParameterValue as ex: diff --git a/coriolis/api/v1/utils.py b/coriolis/api/v1/utils.py index 4daff9c5..ba086156 100644 --- a/coriolis/api/v1/utils.py +++ b/coriolis/api/v1/utils.py @@ -7,10 +7,7 @@ from oslo_log import log as logging from webob import exc -from coriolis import constants -from coriolis import exception -from coriolis import schemas - +from coriolis import constants, exception, schemas LOG = logging.getLogger(__name__) @@ -22,29 +19,26 @@ def get_bool_url_arg(req, arg_name, default=False): if type(parsed_val) is bool: return parsed_val except Exception as err: - LOG.warn( - "failed to parse %s: %s" % (arg_name, err)) + LOG.warn("failed to parse %s: %s" % (arg_name, err)) return default def validate_network_map(network_map): - """ Validates the JSON schema for the network_map. """ + """Validates the JSON schema for the network_map.""" try: - schemas.validate_value( - network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA) + schemas.validate_value(network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA) except exception.SchemaValidationException as ex: - raise exc.HTTPBadRequest( - explanation="Invalid network_map: %s" % str(ex)) + raise exc.HTTPBadRequest(explanation="Invalid network_map: %s" % str(ex)) def validate_storage_mappings(storage_mappings): - """ Validates the JSON schema for the storage_mappings. """ + """Validates the JSON schema for the storage_mappings.""" try: schemas.validate_value( - storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA) + storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA + ) except exception.SchemaValidationException as ex: - raise exc.HTTPBadRequest( - explanation="Invalid storage_mappings: %s" % str(ex)) + raise exc.HTTPBadRequest(explanation="Invalid storage_mappings: %s" % str(ex)) def format_keyerror_message(resource='', method=''): @@ -57,8 +51,7 @@ def _wrapper(*args, **kwargs): LOG.exception(err) if err.args: key = err.args[0] - exc_message = _build_keyerror_message( - resource, method, key) + exc_message = _build_keyerror_message(resource, method, key) else: exc_message = str(err) raise exception.InvalidInput(exc_message) @@ -66,7 +59,9 @@ def _wrapper(*args, **kwargs): LOG.exception(err) msg = getattr(err, "msg", str(err)) raise exception.InvalidInput(reason=msg) + return _wrapper + return _format_keyerror_message @@ -74,14 +69,21 @@ def _build_keyerror_message(resource, method, key): msg = '' method_mapping = { "create": "creation", - "update": "update", } + "update": "update", + } if resource == key: msg = 'The %s %s body needs to be encased inside the "%s" key' % ( - resource, method_mapping[method], key) + resource, + method_mapping[method], + key, + ) else: msg = 'The %s %s body lacks a required attribute: "%s"' % ( - resource, method_mapping[method], key) + resource, + method_mapping[method], + key, + ) return msg @@ -91,26 +93,30 @@ def validate_user_scripts(user_scripts): user_scripts = {} if not isinstance(user_scripts, dict): raise exception.InvalidInput( - reason='"user_scripts" must be of JSON object format') + reason='"user_scripts" must be of JSON object format' + ) global_scripts = user_scripts.get('global', {}) if not isinstance(global_scripts, dict): raise exception.InvalidInput( reason='"global" must be a mapping between the identifiers of the ' - 'supported OS types and their respective scripts.') + 'supported OS types and their respective scripts.' + ) for os_type in global_scripts.keys(): if os_type not in constants.VALID_OS_TYPES: raise exception.InvalidInput( reason='The provided global user script os_type "%s" is ' - 'invalid. Must be one of the ' - 'following: %s' % (os_type, constants.VALID_OS_TYPES)) + 'invalid. Must be one of the ' + 'following: %s' % (os_type, constants.VALID_OS_TYPES) + ) instance_scripts = user_scripts.get('instances', {}) if not isinstance(instance_scripts, dict): raise exception.InvalidInput( reason='"instances" must be a mapping between the identifiers of ' - 'the instances in the Replica/Migration and their ' - 'respective scripts.') + 'the instances in the Replica/Migration and their ' + 'respective scripts.' + ) return user_scripts @@ -118,12 +124,13 @@ def validate_user_scripts(user_scripts): def validate_instances_list_for_transfer(instances): if not instances: raise exception.InvalidInput( - "No instance identifiers provided for transfer action.") + "No instance identifiers provided for transfer action." + ) if not isinstance(instances, list): raise exception.InvalidInput( - "Instances must be a list. Got type %s: %s" % ( - type(instances), instances)) + "Instances must be a list. Got type %s: %s" % (type(instances), instances) + ) appearances = {} for instance in instances: @@ -131,11 +138,11 @@ def validate_instances_list_for_transfer(instances): appearances[instance] = appearances[instance] + 1 else: appearances[instance] = 1 - duplicates = { - inst: count for (inst, count) in appearances.items() if count > 1} + duplicates = {inst: count for (inst, count) in appearances.items() if count > 1} if duplicates: raise exception.InvalidInput( - "Transfer action instances (%s) list contained duplicates: %s " % - (instances, duplicates)) + "Transfer action instances (%s) list contained duplicates: %s " + % (instances, duplicates) + ) return instances diff --git a/coriolis/api/v1/views/deployment_view.py b/coriolis/api/v1/views/deployment_view.py index 1a32f562..382ef99b 100644 --- a/coriolis/api/v1/views/deployment_view.py +++ b/coriolis/api/v1/views/deployment_view.py @@ -10,7 +10,8 @@ def _format_deployment(deployment, keys=None): if len(deployment_dict.get("executions", [])): execution = view.format_transfer_tasks_execution( - deployment_dict["executions"][0], keys) + deployment_dict["executions"][0], keys + ) del deployment_dict["executions"] else: execution = {} @@ -27,6 +28,5 @@ def single(deployment, keys=None): def collection(deployments, keys=None): - formatted_deployments = [_format_deployment(m, keys) - for m in deployments] + formatted_deployments = [_format_deployment(m, keys) for m in deployments] return {'deployments': formatted_deployments} diff --git a/coriolis/api/v1/views/endpoint_options_view.py b/coriolis/api/v1/views/endpoint_options_view.py index 6c580e83..c3391a99 100644 --- a/coriolis/api/v1/views/endpoint_options_view.py +++ b/coriolis/api/v1/views/endpoint_options_view.py @@ -4,26 +4,23 @@ from coriolis.api.v1.views import utils as view_utils -def destination_minion_pool_options_collection(destination_pool_options, - keys=None): +def destination_minion_pool_options_collection(destination_pool_options, keys=None): formatted_opts = [ - view_utils.format_opt(opt, keys) for opt in destination_pool_options] + view_utils.format_opt(opt, keys) for opt in destination_pool_options + ] return {'destination_minion_pool_options': formatted_opts} def destination_options_collection(destination_options, keys=None): - formatted_opts = [ - view_utils.format_opt(opt, keys) for opt in destination_options] + formatted_opts = [view_utils.format_opt(opt, keys) for opt in destination_options] return {'destination_options': formatted_opts} def source_minion_pool_options_collection(source_pool_options, keys=None): - formatted_opts = [ - view_utils.format_opt(opt, keys) for opt in source_pool_options] + formatted_opts = [view_utils.format_opt(opt, keys) for opt in source_pool_options] return {'source_minion_pool_options': formatted_opts} def source_options_collection(source_options, keys=None): - formatted_opts = [ - view_utils.format_opt(opt, keys) for opt in source_options] + formatted_opts = [view_utils.format_opt(opt, keys) for opt in source_options] return {'source_options': formatted_opts} diff --git a/coriolis/api/v1/views/endpoint_resources_view.py b/coriolis/api/v1/views/endpoint_resources_view.py index 65e47436..8fb7a331 100644 --- a/coriolis/api/v1/views/endpoint_resources_view.py +++ b/coriolis/api/v1/views/endpoint_resources_view.py @@ -9,8 +9,7 @@ def instance_single(instance, keys=None): def instances_collection(instances, keys=None): - formatted_instances = [view_utils.format_opt(m, keys) - for m in instances] + formatted_instances = [view_utils.format_opt(m, keys) for m in instances] return {'instances': formatted_instances} @@ -19,8 +18,7 @@ def network_single(network, keys=None): def networks_collection(networks, keys=None): - formatted_networks = [view_utils.format_opt(m, keys) - for m in networks] + formatted_networks = [view_utils.format_opt(m, keys) for m in networks] return {'networks': formatted_networks} diff --git a/coriolis/api/v1/views/endpoint_view.py b/coriolis/api/v1/views/endpoint_view.py index a069b029..1b5d312e 100644 --- a/coriolis/api/v1/views/endpoint_view.py +++ b/coriolis/api/v1/views/endpoint_view.py @@ -7,8 +7,7 @@ def _format_endpoint(endpoint, keys=None): endpoint_dict = view_utils.format_opt(endpoint, keys) mapped_regions = endpoint_dict.get('mapped_regions', []) - endpoint_dict['mapped_regions'] = [ - reg['id'] for reg in mapped_regions] + endpoint_dict['mapped_regions'] = [reg['id'] for reg in mapped_regions] return endpoint_dict @@ -18,6 +17,5 @@ def single(endpoint, keys=None): def collection(endpoints, keys=None): - formatted_endpoints = [_format_endpoint(m, keys) - for m in endpoints] + formatted_endpoints = [_format_endpoint(m, keys) for m in endpoints] return {'endpoints': formatted_endpoints} diff --git a/coriolis/api/v1/views/minion_pool_view.py b/coriolis/api/v1/views/minion_pool_view.py index 84036e92..0db80a54 100644 --- a/coriolis/api/v1/views/minion_pool_view.py +++ b/coriolis/api/v1/views/minion_pool_view.py @@ -17,17 +17,18 @@ def _hide_minion_creds(minion_conn): if minion_conn.get('certificates'): for key in minion_conn['certificates']: minion_conn['certificates'][key] = '***' + if 'minion_machines' in minion_pool_dict: for machine in minion_pool_dict['minion_machines']: if 'connection_info' in machine: _hide_minion_creds(machine['connection_info']) if 'backup_writer_connection_info' in machine: if machine.get('backup_writer_connection_info') and ( - 'connection_details' in machine[ - 'backup_writer_connection_info']): + 'connection_details' in machine['backup_writer_connection_info'] + ): _hide_minion_creds( - machine['backup_writer_connection_info'][ - 'connection_details']) + machine['backup_writer_connection_info']['connection_details'] + ) return minion_pool_dict @@ -37,6 +38,5 @@ def single(minion_pool, keys=None): def collection(minion_pools, keys=None): - formatted_minion_pools = [ - _format_minion_pool(r, keys) for r in minion_pools] + formatted_minion_pools = [_format_minion_pool(r, keys) for r in minion_pools] return {'minion_pools': formatted_minion_pools} diff --git a/coriolis/api/v1/views/region_view.py b/coriolis/api/v1/views/region_view.py index 6752c1c7..d66339f1 100644 --- a/coriolis/api/v1/views/region_view.py +++ b/coriolis/api/v1/views/region_view.py @@ -8,12 +8,10 @@ def _format_region(region, keys=None): region_dict = view_utils.format_opt(region, keys) mapped_endpoints = region_dict.get('mapped_endpoints', []) - region_dict['mapped_endpoints'] = [ - endp['id'] for endp in mapped_endpoints] + region_dict['mapped_endpoints'] = [endp['id'] for endp in mapped_endpoints] mapped_services = region_dict.get('mapped_services', []) - region_dict['mapped_services'] = [ - svc['id'] for svc in mapped_services] + region_dict['mapped_services'] = [svc['id'] for svc in mapped_services] return region_dict @@ -23,6 +21,5 @@ def single(region, keys=None): def collection(regions, keys=None): - formatted_regions = [ - _format_region(r, keys) for r in regions] + formatted_regions = [_format_region(r, keys) for r in regions] return {'regions': formatted_regions} diff --git a/coriolis/api/v1/views/service_view.py b/coriolis/api/v1/views/service_view.py index f223967c..7dc1d1c9 100644 --- a/coriolis/api/v1/views/service_view.py +++ b/coriolis/api/v1/views/service_view.py @@ -8,8 +8,7 @@ def _format_service(service, keys=None): service_dict = view_utils.format_opt(service, keys) mapped_regions = service_dict.get('mapped_regions', []) - service_dict['mapped_regions'] = [ - mapping['id'] for mapping in mapped_regions] + service_dict['mapped_regions'] = [mapping['id'] for mapping in mapped_regions] return service_dict @@ -19,6 +18,5 @@ def single(service, keys=None): def collection(services, keys=None): - formatted_services = [ - _format_service(r, keys) for r in services] + formatted_services = [_format_service(r, keys) for r in services] return {'services': formatted_services} diff --git a/coriolis/api/v1/views/transfer_schedule_view.py b/coriolis/api/v1/views/transfer_schedule_view.py index ad850cb4..6965bda6 100644 --- a/coriolis/api/v1/views/transfer_schedule_view.py +++ b/coriolis/api/v1/views/transfer_schedule_view.py @@ -9,6 +9,5 @@ def single(schedule, keys=None): def collection(schedules, keys=None): - formatted_schedules = [view_utils.format_opt(m, keys) - for m in schedules] + formatted_schedules = [view_utils.format_opt(m, keys) for m in schedules] return {'schedules': formatted_schedules} diff --git a/coriolis/api/v1/views/transfer_tasks_execution_view.py b/coriolis/api/v1/views/transfer_tasks_execution_view.py index 3873d187..9b30029d 100644 --- a/coriolis/api/v1/views/transfer_tasks_execution_view.py +++ b/coriolis/api/v1/views/transfer_tasks_execution_view.py @@ -4,9 +4,8 @@ from oslo_log import log as logging -from coriolis.api.v1.views import utils as view_utils from coriolis import constants - +from coriolis.api.v1.views import utils as view_utils LOG = logging.getLogger(__name__) @@ -18,11 +17,9 @@ def _sort_tasks(tasks, filter_error_only_tasks=True): """ if filter_error_only_tasks: tasks = [ - t for t in tasks - if t['status'] != ( - constants.TASK_STATUS_ON_ERROR_ONLY)] - return sorted( - tasks, key=lambda t: t.get('index', 0)) + t for t in tasks if t['status'] != (constants.TASK_STATUS_ON_ERROR_ONLY) + ] + return sorted(tasks, key=lambda t: t.get('index', 0)) def format_transfer_tasks_execution(execution, keys=None): @@ -39,6 +36,7 @@ def single(execution, keys=None): def collection(executions, keys=None): - formatted_executions = [format_transfer_tasks_execution(m, keys) - for m in executions] + formatted_executions = [ + format_transfer_tasks_execution(m, keys) for m in executions + ] return {'executions': formatted_executions} diff --git a/coriolis/api/v1/views/transfer_view.py b/coriolis/api/v1/views/transfer_view.py index 2fcefe4f..7cc93cb5 100644 --- a/coriolis/api/v1/views/transfer_view.py +++ b/coriolis/api/v1/views/transfer_view.py @@ -10,8 +10,8 @@ def _format_transfer(transfer, keys=None): executions = transfer_dict.get('executions', []) transfer_dict['executions'] = [ - view.format_transfer_tasks_execution(ex) - for ex in executions] + view.format_transfer_tasks_execution(ex) for ex in executions + ] return transfer_dict diff --git a/coriolis/api/v1/views/utils.py b/coriolis/api/v1/views/utils.py index 2f057ed2..7c0a889b 100644 --- a/coriolis/api/v1/views/utils.py +++ b/coriolis/api/v1/views/utils.py @@ -10,5 +10,6 @@ def transform(key, value): return yield (key, value) - return dict(itertools.chain.from_iterable( - transform(k, v) for k, v in option.items())) + return dict( + itertools.chain.from_iterable(transform(k, v) for k, v in option.items()) + ) diff --git a/coriolis/api/wsgi.py b/coriolis/api/wsgi.py index 65f96d2d..226cccff 100644 --- a/coriolis/api/wsgi.py +++ b/coriolis/api/wsgi.py @@ -18,17 +18,15 @@ import math import time -from oslo_log import log as logging -from oslo_serialization import jsonutils -from oslo_utils import excutils import six import webob import webob.dec +from oslo_log import log as logging +from oslo_serialization import jsonutils +from oslo_utils import excutils -from coriolis import exception -from coriolis import i18n -from coriolis.i18n import _, _LE, _LI # noqa - +from coriolis import exception, i18n +from coriolis.i18n import _LE, _LI, _ # noqa LOG = logging.getLogger(__name__) @@ -228,12 +226,12 @@ def cache_db_volume(self, volume): self.cache_db_items(self.path, [volume], 'id') def get_db_volumes(self): - return (self.get_db_items('volumes') or - self.get_db_items(self.path)) + return self.get_db_items('volumes') or self.get_db_items(self.path) def get_db_volume(self, volume_id): - return (self.get_db_item('volumes', volume_id) or - self.get_db_item(self.path, volume_id)) + return self.get_db_item('volumes', volume_id) or self.get_db_item( + self.path, volume_id + ) def cache_db_volume_types(self, volume_types): self.cache_db_items('volume_types', volume_types, 'id') @@ -287,8 +285,9 @@ def best_match_content_type(self): if not content_type: content_type = self.accept.best_match(SUPPORTED_CONTENT_TYPES) - self.environ['coriolis.best_content_type'] = (content_type or - 'application/json') + self.environ['coriolis.best_content_type'] = ( + content_type or 'application/json' + ) return self.environ['coriolis.best_content_type'] @@ -346,8 +345,10 @@ def factory(cls, global_config, **local_config): You could of course re-implement the `factory` method in subclasses, but using the kwarg passing it shouldn't be necessary. """ + def _factory(app): return cls(app, **local_config) + return _factory def __init__(self, application): @@ -399,7 +400,6 @@ def default(self, datastring): class JSONDeserializer(TextDeserializer): - def _from_json(self, datastring): try: return jsonutils.loads(datastring) @@ -448,6 +448,7 @@ def decorator(func): func.wsgi_serializers = {} func.wsgi_serializers.update(serializers) return func + return decorator @@ -464,6 +465,7 @@ def decorator(func): func.wsgi_deserializers = {} func.wsgi_deserializers.update(deserializers) return func + return decorator @@ -478,6 +480,7 @@ def response(code): def decorator(func): func.wsgi_code = code return func + return decorator @@ -567,8 +570,7 @@ def preserialize(self, content_type, default_serializers=None): be accessed by extensions for, e.g., template extension. """ - mtype, serializer = self.get_serializer(content_type, - default_serializers) + mtype, serializer = self.get_serializer(content_type, default_serializers) self.media_type = mtype self.serializer = serializer() @@ -588,8 +590,7 @@ def serialize(self, request, content_type, default_serializers=None): if self.serializer: serializer = self.serializer else: - _mtype, _serializer = self.get_serializer(content_type, - default_serializers) + _mtype, _serializer = self.get_serializer(content_type, default_serializers) serializer = _serializer() response = webob.Response() @@ -654,13 +655,16 @@ def __exit__(self, ex_type, ex_value, ex_traceback): if isinstance(ex_value, exception.NotAuthorized): raise Fault(webob.exc.HTTPForbidden(explanation=ex_value.msg)) elif isinstance(ex_value, exception.Invalid): - raise Fault(exception.ConvertedException( - code=ex_value.code, explanation=ex_value.msg)) + raise Fault( + exception.ConvertedException( + code=ex_value.code, explanation=ex_value.msg + ) + ) elif isinstance(ex_value, TypeError): exc_info = (ex_type, ex_value, ex_traceback) - LOG.error(_LE( - 'Exception handling resource: %s'), - ex_value, exc_info=exc_info) + LOG.error( + _LE('Exception handling resource: %s'), ex_value, exc_info=exc_info + ) raise Fault(webob.exc.HTTPBadRequest()) elif isinstance(ex_value, Fault): LOG.info(_LI("Fault thrown: %s"), ex_value) @@ -705,8 +709,7 @@ def __init__(self, controller, action_peek=None, **deserializers): default_deserializers.update(deserializers) self.default_deserializers = default_deserializers - self.default_serializers = dict(json=JSONDictSerializer, - csv=CSVSerializer) + self.default_serializers = dict(json=JSONDictSerializer, csv=CSVSerializer) self.action_peek = dict(json=action_peek_json) self.action_peek.update(action_peek or {}) @@ -832,8 +835,7 @@ def pre_process_extensions(self, extensions, request, action_args): # Run post-processing in the reverse order return None, reversed(post) - def post_process_extensions(self, extensions, resp_obj, request, - action_args): + def post_process_extensions(self, extensions, resp_obj, request, action_args): for ext in extensions: response = None if inspect.isgenerator(ext): @@ -851,8 +853,7 @@ def post_process_extensions(self, extensions, resp_obj, request, # Regular functions get post-processing... try: with ResourceExceptionHandler(): - response = ext(req=request, resp_obj=resp_obj, - **action_args) + response = ext(req=request, resp_obj=resp_obj, **action_args) except Fault as ex: response = ex @@ -866,9 +867,9 @@ def post_process_extensions(self, extensions, resp_obj, request, def __call__(self, request): """WSGI method that controls (de)serialization and method dispatch.""" - LOG.info(_LI("%(method)s %(url)s"), - {"method": request.method, - "url": request.url}) + LOG.info( + _LI("%(method)s %(url)s"), {"method": request.method, "url": request.url} + ) # Identify the action, its arguments, and the requested # content type @@ -882,17 +883,16 @@ def __call__(self, request): # function. If we try to audit __call__(), we can # run into troubles due to the @webob.dec.wsgify() # decorator. - return self._process_stack(request, action, action_args, - content_type, body, accept) + return self._process_stack( + request, action, action_args, content_type, body, accept + ) - def _process_stack(self, request, action, action_args, - content_type, body, accept): + def _process_stack(self, request, action, action_args, content_type, body, accept): """Implement the processing stack.""" # Get the implementing method try: - meth, extensions = self.get_method(request, action, - content_type, body) + meth, extensions = self.get_method(request, action, content_type, body) except (AttributeError, TypeError): return Fault(webob.exc.HTTPNotFound()) except KeyError as ex: @@ -920,13 +920,12 @@ def _process_stack(self, request, action, action_args, project_id = action_args.pop("project_id", None) context = request.environ.get('coriolis.context') - if (context and project_id and (project_id != context.project_id)): + if context and project_id and (project_id != context.project_id): msg = _("Malformed request url") return Fault(webob.exc.HTTPBadRequest(explanation=msg)) # Run pre-processing extensions - response, post = self.pre_process_extensions(extensions, - request, action_args) + response, post = self.pre_process_extensions(extensions, request, action_args) if not response: try: @@ -957,12 +956,12 @@ def _process_stack(self, request, action, action_args, resp_obj.preserialize(accept, self.default_serializers) # Process post-processing extensions - response = self.post_process_extensions(post, resp_obj, - request, action_args) + response = self.post_process_extensions( + post, resp_obj, request, action_args + ) if resp_obj and not response: - response = resp_obj.serialize(request, accept, - self.default_serializers) + response = resp_obj.serialize(request, accept, self.default_serializers) try: msg_dict = dict(url=request.url, status=response.status_int) @@ -986,10 +985,12 @@ def get_method(self, request, action, content_type, body): meth = getattr(self.controller, action) except AttributeError as e: with excutils.save_and_reraise_exception(e) as ctxt: - if (not self.wsgi_actions or action not in ['action', - 'create', - 'delete', - 'update']): + if not self.wsgi_actions or action not in [ + 'action', + 'create', + 'delete', + 'update', + ]: LOG.exception(_LE('Get method error.')) else: ctxt.reraise = False @@ -1005,8 +1006,10 @@ def get_method(self, request, action, content_type, body): action_name = action # Look up the action method - return (self.wsgi_actions[action_name], - self.wsgi_action_extensions.get(action_name, [])) + return ( + self.wsgi_actions[action_name], + self.wsgi_action_extensions.get(action_name, []), + ) def dispatch(self, method, request, action_args): """Dispatch a call to the action-specific method.""" @@ -1026,6 +1029,7 @@ def action(name): def decorator(func): func.wsgi_action = name return func + return decorator @@ -1086,8 +1090,7 @@ def __new__(mcs, name, bases, cls_dict): cls_dict['wsgi_actions'] = actions cls_dict['wsgi_extensions'] = extensions - return super(ControllerMetaclass, mcs).__new__(mcs, name, bases, - cls_dict) + return super(ControllerMetaclass, mcs).__new__(mcs, name, bases, cls_dict) @six.add_metaclass(ControllerMetaclass) @@ -1132,8 +1135,9 @@ def assert_valid_body(body, entity_name): # V2 api 'HTTPBadRequest' exception is getting raised. if not Controller.is_valid_body(body, entity_name): raise webob.exc.HTTPBadRequest( - explanation=_("Missing required element '%s' in " - "request body.") % entity_name) + explanation=_("Missing required element '%s' in request body.") + % entity_name + ) @staticmethod def validate_name_and_description(body): @@ -1142,22 +1146,23 @@ def validate_name_and_description(body): if isinstance(name, six.string_types): body['name'] = name.strip() try: - _check_string_length(body['name'], 'Name', - min_length=0, max_length=255) + _check_string_length(body['name'], 'Name', min_length=0, max_length=255) except exception.InvalidInput as error: raise webob.exc.HTTPBadRequest(explanation=error.msg) description = body.get('description') if description is not None: try: - _check_string_length(description, 'Description', - min_length=0, max_length=255) + _check_string_length( + description, 'Description', min_length=0, max_length=255 + ) except exception.InvalidInput as error: raise webob.exc.HTTPBadRequest(explanation=error.msg) @staticmethod - def validate_string_length(value, entity_name, min_length=0, - max_length=None, remove_whitespaces=False): + def validate_string_length( + value, entity_name, min_length=0, max_length=None, remove_whitespaces=False + ): """Check the length of specified string. :param value: the value of the string @@ -1170,9 +1175,9 @@ def validate_string_length(value, entity_name, min_length=0, if isinstance(value, six.string_types) and remove_whitespaces: value = value.strip() try: - _check_string_length(value, entity_name, - min_length=min_length, - max_length=max_length) + _check_string_length( + value, entity_name, min_length=min_length, max_length=max_length + ) except exception.InvalidInput as error: raise webob.exc.HTTPBadRequest(explanation=error.msg) @@ -1189,17 +1194,24 @@ def validate_integer(value, name, min_value=None, max_value=None): try: value = int(value) except (TypeError, ValueError, UnicodeEncodeError): - raise webob.exc.HTTPBadRequest(explanation=( - _('%s must be an integer.') % name)) + raise webob.exc.HTTPBadRequest( + explanation=(_('%s must be an integer.') % name) + ) if min_value is not None and value < min_value: raise webob.exc.HTTPBadRequest( - explanation=(_('%(value_name)s must be >= %(min_value)d') % - {'value_name': name, 'min_value': min_value})) + explanation=( + _('%(value_name)s must be >= %(min_value)d') + % {'value_name': name, 'min_value': min_value} + ) + ) if max_value is not None and value > max_value: raise webob.exc.HTTPBadRequest( - explanation=(_('%(value_name)s must be <= %(max_value)d') % - {'value_name': name, 'max_value': max_value})) + explanation=( + _('%(value_name)s must be <= %(max_value)d') + % {'value_name': name, 'max_value': max_value} + ) + ) return value @@ -1207,16 +1219,18 @@ def validate_integer(value, name, min_value=None, max_value=None): class Fault(webob.exc.HTTPException): """Wrap webob.exc.HTTPException to provide API friendly response.""" - _fault_names = {400: "badRequest", - 401: "unauthorized", - 403: "forbidden", - 404: "itemNotFound", - 405: "badMethod", - 409: "conflictingRequest", - 413: "overLimit", - 415: "badMediaType", - 501: "notImplemented", - 503: "serviceUnavailable"} + _fault_names = { + 400: "badRequest", + 401: "unauthorized", + 403: "forbidden", + 404: "itemNotFound", + 405: "badMethod", + 409: "conflictingRequest", + 413: "overLimit", + 415: "badMediaType", + 501: "notImplemented", + 503: "serviceUnavailable", + } def __init__(self, exception): """Create a Fault for the given webob.exc.exception.""" @@ -1235,7 +1249,9 @@ def __call__(self, req): 'error': { 'fault': fault_name, 'code': code, - 'message': i18n.translate(explanation, locale)}} + 'message': i18n.translate(explanation, locale), + } + } if code == 413: retry = self.wrapped_exc.headers.get('Retry-After', None) if retry: @@ -1278,13 +1294,17 @@ def _check_string_length(value, name, min_length=0, max_length=None): raise exception.InvalidInput(message=msg) if len(value) < min_length: - msg = _("%(name)s has a minimum character requirement of " - "%(min_length)s.") % {'name': name, 'min_length': min_length} + msg = _("%(name)s has a minimum character requirement of %(min_length)s.") % { + 'name': name, + 'min_length': min_length, + } raise exception.InvalidInput(message=msg) if max_length and len(value) > max_length: - msg = _("%(name)s has more than %(max_length)s " - "characters.") % {'name': name, 'max_length': max_length} + msg = _("%(name)s has more than %(max_length)s characters.") % { + 'name': name, + 'max_length': max_length, + } raise exception.InvalidInput(message=msg) @@ -1319,10 +1339,12 @@ def translate(msg): locale = request.best_match_language() return i18n.translate(msg, locale) - self.content['overLimitFault']['message'] = \ - translate(self.content['overLimitFault']['message']) - self.content['overLimitFault']['details'] = \ - translate(self.content['overLimitFault']['details']) + self.content['overLimitFault']['message'] = translate( + self.content['overLimitFault']['message'] + ) + self.content['overLimitFault']['details'] = translate( + self.content['overLimitFault']['details'] + ) serializer = { 'application/json': JSONDictSerializer(), diff --git a/coriolis/cache.py b/coriolis/cache.py index 94611303..62d061ff 100644 --- a/coriolis/cache.py +++ b/coriolis/cache.py @@ -1,11 +1,11 @@ # Copyright 2019 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis import exception - from oslo_cache import core as cache from oslo_config import cfg +from coriolis import exception + opts = [ cfg.BoolOpt('caching', default=False), cfg.IntOpt('cache_time', default=7200), @@ -20,8 +20,6 @@ def get_cache_decorator(provider): if type(provider) is not str or provider == "": - raise exception.CoriolisException( - "Invalid provider name") - MEMOIZE = cache.get_memoization_decorator( - CONF, cache_region, provider) + raise exception.CoriolisException("Invalid provider name") + MEMOIZE = cache.get_memoization_decorator(CONF, cache_region, provider) return MEMOIZE diff --git a/coriolis/cmd/api.py b/coriolis/cmd/api.py index cb3040f2..0fc670b0 100644 --- a/coriolis/cmd/api.py +++ b/coriolis/cmd/api.py @@ -8,13 +8,16 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import service -from coriolis import utils +from coriolis import service, utils api_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running')] + 'worker_count', + min=1, + default=processutils.get_worker_count(), + help='Number of processes in which the service will be running', + ) +] CONF = cfg.CONF CONF.register_opts(api_opts, 'api') @@ -30,10 +33,8 @@ def main(): gmr_opts.set_defaults(CONF) gmr.TextGuruMeditation.setup_autorun(version="1.0.0", conf=CONF) - server = service.WSGIService( - 'coriolis-api', worker_count=worker_count) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + server = service.WSGIService('coriolis-api', worker_count=worker_count) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/conductor.py b/coriolis/cmd/conductor.py index 0bdb537c..e151d154 100644 --- a/coriolis/cmd/conductor.py +++ b/coriolis/cmd/conductor.py @@ -8,15 +8,17 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts +from coriolis import constants, service, utils from coriolis.conductor.rpc import server as rpc_server -from coriolis import constants -from coriolis import service -from coriolis import utils conductor_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running')] + 'worker_count', + min=1, + default=processutils.get_worker_count(), + help='Number of processes in which the service will be running', + ) +] CONF = cfg.CONF CONF.register_opts(conductor_opts, 'conductor') @@ -36,9 +38,10 @@ def main(): server = service.MessagingService( constants.CONDUCTOR_MAIN_MESSAGING_TOPIC, [rpc_server.ConductorServerEndpoint()], - rpc_server.VERSION, worker_count=worker_count) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=worker_count, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/db_sync.py b/coriolis/cmd/db_sync.py index d904cf18..16bb7c17 100644 --- a/coriolis/cmd/db_sync.py +++ b/coriolis/cmd/db_sync.py @@ -5,15 +5,14 @@ from oslo_config import cfg -from coriolis.db import api as db_api from coriolis import utils +from coriolis.db import api as db_api CONF = cfg.CONF def main(): - CONF(sys.argv[1:], project='coriolis', - version="1.0.0") + CONF(sys.argv[1:], project='coriolis', version="1.0.0") utils.setup_logging() db_api.db_sync(db_api.get_engine()) diff --git a/coriolis/cmd/deployer_manager.py b/coriolis/cmd/deployer_manager.py index d9d20005..baba6a96 100644 --- a/coriolis/cmd/deployer_manager.py +++ b/coriolis/cmd/deployer_manager.py @@ -7,15 +7,17 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import constants +from coriolis import constants, service, utils from coriolis.deployer_manager.rpc import server as rpc_server -from coriolis import service -from coriolis import utils deployer_manager_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=1, - help="Number of processes in which the service will be running")] + 'worker_count', + min=1, + default=1, + help="Number of processes in which the service will be running", + ) +] CONF = cfg.CONF CONF.register_opts(deployer_manager_opts, 'deployer_manager') @@ -30,9 +32,10 @@ def main(): server = service.MessagingService( constants.DEPLOYER_MANAGER_MAIN_MESSAGING_TOPIC, [rpc_server.DeployerManagerServerEndpoint()], - rpc_server.VERSION, worker_count=CONF.deployer_manager.worker_count) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=CONF.deployer_manager.worker_count, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/minion_manager.py b/coriolis/cmd/minion_manager.py index e480757d..0aeb1d3e 100644 --- a/coriolis/cmd/minion_manager.py +++ b/coriolis/cmd/minion_manager.py @@ -7,23 +7,24 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import constants +from coriolis import constants, service, utils from coriolis.minion_manager.rpc import server as rpc_server -from coriolis import service -from coriolis import utils minion_manager_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=1, - help='Number of processes in which the service will be running')] + 'worker_count', + min=1, + default=1, + help='Number of processes in which the service will be running', + ) +] CONF = cfg.CONF CONF.register_opts(minion_manager_opts, 'minion_manager') def main(): - CONF(sys.argv[1:], project='coriolis', - version="1.0.0") + CONF(sys.argv[1:], project='coriolis', version="1.0.0") utils.setup_logging() gmr_opts.set_defaults(CONF) @@ -32,9 +33,10 @@ def main(): server = service.MessagingService( constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC, [rpc_server.MinionManagerServerEndpoint()], - rpc_server.VERSION, worker_count=CONF.minion_manager.worker_count) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=CONF.minion_manager.worker_count, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/scheduler.py b/coriolis/cmd/scheduler.py index a047629c..308773ce 100644 --- a/coriolis/cmd/scheduler.py +++ b/coriolis/cmd/scheduler.py @@ -7,23 +7,24 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import constants +from coriolis import constants, service, utils from coriolis.scheduler.rpc import server as rpc_server -from coriolis import service -from coriolis import utils scheduler_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=1, - help='Number of processes in which the service will be running')] + 'worker_count', + min=1, + default=1, + help='Number of processes in which the service will be running', + ) +] CONF = cfg.CONF CONF.register_opts(scheduler_opts, 'scheduler') def main(): - CONF(sys.argv[1:], project='coriolis', - version="1.0.0") + CONF(sys.argv[1:], project='coriolis', version="1.0.0") utils.setup_logging() gmr_opts.set_defaults(CONF) @@ -32,9 +33,10 @@ def main(): server = service.MessagingService( constants.SCHEDULER_MAIN_MESSAGING_TOPIC, [rpc_server.SchedulerServerEndpoint()], - rpc_server.VERSION, worker_count=CONF.scheduler.worker_count) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=CONF.scheduler.worker_count, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/transfer_cron.py b/coriolis/cmd/transfer_cron.py index 59e34adb..8eb0055d 100644 --- a/coriolis/cmd/transfer_cron.py +++ b/coriolis/cmd/transfer_cron.py @@ -7,17 +7,14 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import constants -from coriolis import service +from coriolis import constants, service, utils from coriolis.transfer_cron.rpc import server as rpc_server -from coriolis import utils CONF = cfg.CONF def main(): - CONF(sys.argv[1:], project='coriolis', - version="1.0.0") + CONF(sys.argv[1:], project='coriolis', version="1.0.0") utils.setup_logging() gmr_opts.set_defaults(CONF) @@ -26,9 +23,10 @@ def main(): server = service.MessagingService( constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC, [rpc_server.TransferCronServerEndpoint()], - rpc_server.VERSION, worker_count=1) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=1, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/cmd/worker.py b/coriolis/cmd/worker.py index 4c23c0da..46243759 100644 --- a/coriolis/cmd/worker.py +++ b/coriolis/cmd/worker.py @@ -8,15 +8,17 @@ from oslo_reports import guru_meditation_report as gmr from oslo_reports import opts as gmr_opts -from coriolis import constants -from coriolis import service -from coriolis import utils +from coriolis import constants, service, utils from coriolis.worker.rpc import server as rpc_server worker_opts = [ cfg.IntOpt( - 'worker_count', min=1, default=processutils.get_worker_count(), - help='Number of processes in which the service will be running')] + 'worker_count', + min=1, + default=processutils.get_worker_count(), + help='Number of processes in which the service will be running', + ) +] CONF = cfg.CONF CONF.register_opts(worker_opts, 'worker') @@ -35,9 +37,11 @@ def main(): server = service.MessagingService( constants.WORKER_MAIN_MESSAGING_TOPIC, [rpc_server.WorkerServerEndpoint()], - rpc_server.VERSION, worker_count=worker_count, init_rpc=False) - launcher = service.service.launch( - CONF, server, workers=server.get_workers_count()) + rpc_server.VERSION, + worker_count=worker_count, + init_rpc=False, + ) + launcher = service.service.launch(CONF, server, workers=server.get_workers_count()) launcher.wait() diff --git a/coriolis/conductor/rpc/client.py b/coriolis/conductor/rpc/client.py index 0e71d088..e3adb799 100644 --- a/coriolis/conductor/rpc/client.py +++ b/coriolis/conductor/rpc/client.py @@ -1,21 +1,20 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log as logging -import oslo_messaging as messaging -from coriolis import constants -from coriolis import events -from coriolis import rpc +from coriolis import constants, events, rpc VERSION = "1.0" LOG = logging.getLogger(__name__) conductor_opts = [ - cfg.IntOpt("conductor_rpc_timeout", - help="Number of seconds until RPC calls to the " - "conductor timeout.") + cfg.IntOpt( + "conductor_rpc_timeout", + help="Number of seconds until RPC calls to the conductor timeout.", + ) ] CONF = cfg.CONF @@ -23,133 +22,164 @@ class ConductorClient(rpc.BaseRPCClient): - def __init__(self, timeout=None, - topic=constants.CONDUCTOR_MAIN_MESSAGING_TOPIC): + def __init__(self, timeout=None, topic=constants.CONDUCTOR_MAIN_MESSAGING_TOPIC): target = messaging.Target(topic=topic, version=VERSION) if timeout is None: timeout = CONF.conductor.conductor_rpc_timeout - super(ConductorClient, self).__init__( - target, timeout=timeout) + super(ConductorClient, self).__init__(target, timeout=timeout) - def create_endpoint(self, ctxt, name, endpoint_type, description, - connection_info, mapped_regions): + def create_endpoint( + self, ctxt, name, endpoint_type, description, connection_info, mapped_regions + ): return self._call( - ctxt, 'create_endpoint', name=name, endpoint_type=endpoint_type, - description=description, connection_info=connection_info, - mapped_regions=mapped_regions) + ctxt, + 'create_endpoint', + name=name, + endpoint_type=endpoint_type, + description=description, + connection_info=connection_info, + mapped_regions=mapped_regions, + ) def update_endpoint(self, ctxt, endpoint_id, updated_values): return self._call( - ctxt, 'update_endpoint', + ctxt, + 'update_endpoint', endpoint_id=endpoint_id, - updated_values=updated_values) + updated_values=updated_values, + ) def get_endpoints(self, ctxt): - return self._call( - ctxt, 'get_endpoints') + return self._call(ctxt, 'get_endpoints') def get_endpoint(self, ctxt, endpoint_id): - return self._call( - ctxt, 'get_endpoint', endpoint_id=endpoint_id) + return self._call(ctxt, 'get_endpoint', endpoint_id=endpoint_id) def delete_endpoint(self, ctxt, endpoint_id): - return self._call( - ctxt, 'delete_endpoint', endpoint_id=endpoint_id) - - def get_endpoint_instances(self, ctxt, endpoint_id, source_environment, - marker=None, limit=None, - instance_name_pattern=None, refresh=False): - return self._call( - ctxt, 'get_endpoint_instances', + return self._call(ctxt, 'delete_endpoint', endpoint_id=endpoint_id) + + def get_endpoint_instances( + self, + ctxt, + endpoint_id, + source_environment, + marker=None, + limit=None, + instance_name_pattern=None, + refresh=False, + ): + return self._call( + ctxt, + 'get_endpoint_instances', endpoint_id=endpoint_id, source_environment=source_environment, marker=marker, limit=limit, instance_name_pattern=instance_name_pattern, - refresh=refresh) + refresh=refresh, + ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name): + self, ctxt, endpoint_id, source_environment, instance_name + ): return self._call( - ctxt, 'get_endpoint_instance', + ctxt, + 'get_endpoint_instance', endpoint_id=endpoint_id, source_environment=source_environment, - instance_name=instance_name) + instance_name=instance_name, + ) - def get_endpoint_source_options( - self, ctxt, endpoint_id, env, option_names): + def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): return self._call( - ctxt, 'get_endpoint_source_options', + ctxt, + 'get_endpoint_source_options', endpoint_id=endpoint_id, - env=env, option_names=option_names) + env=env, + option_names=option_names, + ) - def get_endpoint_destination_options( - self, ctxt, endpoint_id, env, option_names): + def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names): return self._call( - ctxt, 'get_endpoint_destination_options', + ctxt, + 'get_endpoint_destination_options', endpoint_id=endpoint_id, - env=env, option_names=option_names) + env=env, + option_names=option_names, + ) def get_endpoint_networks(self, ctxt, endpoint_id, env): return self._call( - ctxt, 'get_endpoint_networks', - endpoint_id=endpoint_id, - env=env) + ctxt, 'get_endpoint_networks', endpoint_id=endpoint_id, env=env + ) def get_endpoint_storage(self, ctxt, endpoint_id, env): return self._call( - ctxt, 'get_endpoint_storage', - endpoint_id=endpoint_id, - env=env) + ctxt, 'get_endpoint_storage', endpoint_id=endpoint_id, env=env + ) - def get_endpoint_inventory_csv( - self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): return self._call( - ctxt, 'get_endpoint_inventory_csv', + ctxt, + 'get_endpoint_inventory_csv', endpoint_id=endpoint_id, - source_environment=source_environment) + source_environment=source_environment, + ) def validate_endpoint_connection(self, ctxt, endpoint_id): - return self._call( - ctxt, 'validate_endpoint_connection', - endpoint_id=endpoint_id) + return self._call(ctxt, 'validate_endpoint_connection', endpoint_id=endpoint_id) - def validate_endpoint_target_environment( - self, ctxt, endpoint_id, target_env): + def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): return self._call( - ctxt, 'validate_endpoint_target_environment', - endpoint_id=endpoint_id, target_env=target_env) + ctxt, + 'validate_endpoint_target_environment', + endpoint_id=endpoint_id, + target_env=target_env, + ) - def validate_endpoint_source_environment( - self, ctxt, endpoint_id, source_env): + def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): return self._call( - ctxt, 'validate_endpoint_source_environment', - endpoint_id=endpoint_id, source_env=source_env) + ctxt, + 'validate_endpoint_source_environment', + endpoint_id=endpoint_id, + source_env=source_env, + ) def get_available_providers(self, ctxt): - return self._call( - ctxt, 'get_available_providers') + return self._call(ctxt, 'get_available_providers') def get_provider_schemas(self, ctxt, platform_name, provider_type): return self._call( - ctxt, 'get_provider_schemas', + ctxt, + 'get_provider_schemas', platform_name=platform_name, - provider_type=provider_type) + provider_type=provider_type, + ) - def execute_transfer_tasks(self, ctxt, transfer_id, - shutdown_instances=False, auto_deploy=False): + def execute_transfer_tasks( + self, ctxt, transfer_id, shutdown_instances=False, auto_deploy=False + ): return self._call( - ctxt, 'execute_transfer_tasks', transfer_id=transfer_id, - shutdown_instances=shutdown_instances, auto_deploy=auto_deploy) + ctxt, + 'execute_transfer_tasks', + transfer_id=transfer_id, + shutdown_instances=shutdown_instances, + auto_deploy=auto_deploy, + ) - def get_transfer_tasks_executions(self, ctxt, transfer_id, - include_tasks=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None): - return self._call( - ctxt, 'get_transfer_tasks_executions', + def get_transfer_tasks_executions( + self, + ctxt, + transfer_id, + include_tasks=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): + return self._call( + ctxt, + 'get_transfer_tasks_executions', transfer_id=transfer_id, include_tasks=include_tasks, marker=marker, @@ -158,43 +188,64 @@ def get_transfer_tasks_executions(self, ctxt, transfer_id, sort_dirs=sort_dirs, ) - def get_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, - include_task_info=False): + def get_transfer_tasks_execution( + self, ctxt, transfer_id, execution_id, include_task_info=False + ): return self._call( - ctxt, 'get_transfer_tasks_execution', transfer_id=transfer_id, - execution_id=execution_id, include_task_info=include_task_info) + ctxt, + 'get_transfer_tasks_execution', + transfer_id=transfer_id, + execution_id=execution_id, + include_task_info=include_task_info, + ) def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): return self._call( - ctxt, 'delete_transfer_tasks_execution', transfer_id=transfer_id, - execution_id=execution_id) + ctxt, + 'delete_transfer_tasks_execution', + transfer_id=transfer_id, + execution_id=execution_id, + ) - def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, - force): + def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force): return self._call( - ctxt, 'cancel_transfer_tasks_execution', transfer_id=transfer_id, - execution_id=execution_id, force=force) + ctxt, + 'cancel_transfer_tasks_execution', + transfer_id=transfer_id, + execution_id=execution_id, + force=force, + ) - def create_instances_transfer(self, ctxt, - transfer_scenario, - origin_endpoint_id, - destination_endpoint_id, - origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, destination_environment, - instances, network_map, storage_mappings, - notes=None, user_scripts=None, - clone_disks=True, skip_os_morphing=False): - return self._call( - ctxt, 'create_instances_transfer', + def create_instances_transfer( + self, + ctxt, + transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes=None, + user_scripts=None, + clone_disks=True, + skip_os_morphing=False, + ): + return self._call( + ctxt, + 'create_instances_transfer', transfer_scenario=transfer_scenario, origin_endpoint_id=origin_endpoint_id, destination_endpoint_id=destination_endpoint_id, origin_minion_pool_id=origin_minion_pool_id, destination_minion_pool_id=destination_minion_pool_id, instance_osmorphing_minion_pool_mappings=( - instance_osmorphing_minion_pool_mappings), + instance_osmorphing_minion_pool_mappings + ), destination_environment=destination_environment, instances=instances, notes=notes, @@ -203,14 +254,22 @@ def create_instances_transfer(self, ctxt, source_environment=source_environment, user_scripts=user_scripts, clone_disks=clone_disks, - skip_os_morphing=skip_os_morphing) + skip_os_morphing=skip_os_morphing, + ) - def get_transfers(self, ctxt, include_tasks_executions=False, - include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): - return self._call( - ctxt, 'get_transfers', + def get_transfers( + self, + ctxt, + include_tasks_executions=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): + return self._call( + ctxt, + 'get_transfers', include_tasks_executions=include_tasks_executions, include_task_info=include_task_info, marker=marker, @@ -221,23 +280,32 @@ def get_transfers(self, ctxt, include_tasks_executions=False, def get_transfer(self, ctxt, transfer_id, include_task_info=False): return self._call( - ctxt, 'get_transfer', transfer_id=transfer_id, - include_task_info=include_task_info) + ctxt, + 'get_transfer', + transfer_id=transfer_id, + include_task_info=include_task_info, + ) def delete_transfer(self, ctxt, transfer_id): - self._call( - ctxt, 'delete_transfer', transfer_id=transfer_id) + self._call(ctxt, 'delete_transfer', transfer_id=transfer_id) def delete_transfer_disks(self, ctxt, transfer_id): - return self._call( - ctxt, 'delete_transfer_disks', transfer_id=transfer_id) - - def get_deployments(self, ctxt, include_tasks=False, - include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): - return self._call( - ctxt, 'get_deployments', include_tasks=include_tasks, + return self._call(ctxt, 'delete_transfer_disks', transfer_id=transfer_id) + + def get_deployments( + self, + ctxt, + include_tasks=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): + return self._call( + ctxt, + 'get_deployments', + include_tasks=include_tasks, include_task_info=include_task_info, marker=marker, limit=limit, @@ -247,142 +315,192 @@ def get_deployments(self, ctxt, include_tasks=False, def get_deployment(self, ctxt, deployment_id, include_task_info=False): return self._call( - ctxt, 'get_deployment', deployment_id=deployment_id, - include_task_info=include_task_info) + ctxt, + 'get_deployment', + deployment_id=deployment_id, + include_task_info=include_task_info, + ) - def confirm_deployer_completed( - self, ctxt, deployment_id, force=False): + def confirm_deployer_completed(self, ctxt, deployment_id, force=False): return self._cast( - ctxt, 'confirm_deployer_completed', deployment_id=deployment_id, - force=force) + ctxt, 'confirm_deployer_completed', deployment_id=deployment_id, force=force + ) - def report_deployer_failure( - self, ctxt, deployemnt_id, deployer_error_details): + def report_deployer_failure(self, ctxt, deployemnt_id, deployer_error_details): return self._cast( - ctxt, 'report_deployer_failure', deployment_id=deployemnt_id, - deployer_error_details=deployer_error_details) + ctxt, + 'report_deployer_failure', + deployment_id=deployemnt_id, + deployer_error_details=deployer_error_details, + ) def deploy_transfer_instances( - self, ctxt, transfer_id, force, wait_for_execution=None, - instance_osmorphing_minion_pool_mappings=None, clone_disks=None, - skip_os_morphing=None, user_scripts=None, trust_id=None): - return self._call( - ctxt, 'deploy_transfer_instances', transfer_id=transfer_id, + self, + ctxt, + transfer_id, + force, + wait_for_execution=None, + instance_osmorphing_minion_pool_mappings=None, + clone_disks=None, + skip_os_morphing=None, + user_scripts=None, + trust_id=None, + ): + return self._call( + ctxt, + 'deploy_transfer_instances', + transfer_id=transfer_id, wait_for_execution=wait_for_execution, instance_osmorphing_minion_pool_mappings=( - instance_osmorphing_minion_pool_mappings), - clone_disks=clone_disks, force=force, + instance_osmorphing_minion_pool_mappings + ), + clone_disks=clone_disks, + force=force, skip_os_morphing=skip_os_morphing, - user_scripts=user_scripts, trust_id=trust_id) + user_scripts=user_scripts, + trust_id=trust_id, + ) def delete_deployment(self, ctxt, deployment_id): - self._call( - ctxt, 'delete_deployment', deployment_id=deployment_id) + self._call(ctxt, 'delete_deployment', deployment_id=deployment_id) def cancel_deployment(self, ctxt, deployment_id, force): - self._call( - ctxt, 'cancel_deployment', deployment_id=deployment_id, - force=force) + self._call(ctxt, 'cancel_deployment', deployment_id=deployment_id, force=force) def set_task_host(self, ctxt, task_id, host): - self._call( - ctxt, 'set_task_host', task_id=task_id, host=host) + self._call(ctxt, 'set_task_host', task_id=task_id, host=host) def set_task_process(self, ctxt, task_id, process_id): - self._call( - ctxt, 'set_task_process', task_id=task_id, process_id=process_id) + self._call(ctxt, 'set_task_process', task_id=task_id, process_id=process_id) def task_completed(self, ctxt, task_id, task_result): - self._call( - ctxt, 'task_completed', task_id=task_id, task_result=task_result) + self._call(ctxt, 'task_completed', task_id=task_id, task_result=task_result) def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): self._call( - ctxt, 'confirm_task_cancellation', task_id=task_id, - cancellation_details=cancellation_details) + ctxt, + 'confirm_task_cancellation', + task_id=task_id, + cancellation_details=cancellation_details, + ) def set_task_error(self, ctxt, task_id, exception_details): self._call( - ctxt, 'set_task_error', task_id=task_id, - exception_details=exception_details) + ctxt, 'set_task_error', task_id=task_id, exception_details=exception_details + ) def add_task_event(self, ctxt, task_id, level, message): - self._cast(ctxt, 'add_task_event', task_id=task_id, - level=level, message=message) + self._cast( + ctxt, 'add_task_event', task_id=task_id, level=level, message=message + ) def add_task_progress_update( - self, ctxt, task_id, message, initial_step=0, total_steps=0, - return_event=False): + self, ctxt, task_id, message, initial_step=0, total_steps=0, return_event=False + ): operation = self._cast if return_event: operation = self._call return operation( - ctxt, 'add_task_progress_update', task_id=task_id, - message=message, initial_step=initial_step, - total_steps=total_steps) + ctxt, + 'add_task_progress_update', + task_id=task_id, + message=message, + initial_step=initial_step, + total_steps=total_steps, + ) def update_task_progress_update( - self, ctxt, task_id, progress_update_index, new_current_step, - new_total_steps=None, new_message=None, sync=False): + self, + ctxt, + task_id, + progress_update_index, + new_current_step, + new_total_steps=None, + new_message=None, + sync=False, + ): # Use _call on the last step to make sure the DB is updated # and the UI is showing the correct progress. # Intermediate steps use cast to avoid blocking on every tick. if sync: return self._call( - ctxt, 'update_task_progress_update', task_id=task_id, + ctxt, + 'update_task_progress_update', + task_id=task_id, progress_update_index=progress_update_index, new_current_step=new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + new_total_steps=new_total_steps, + new_message=new_message, + ) return self._cast( - ctxt, 'update_task_progress_update', task_id=task_id, + ctxt, + 'update_task_progress_update', + task_id=task_id, progress_update_index=progress_update_index, new_current_step=new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + new_total_steps=new_total_steps, + new_message=new_message, + ) - def create_transfer_schedule(self, ctxt, transfer_id, - schedule, enabled, exp_date, - shutdown_instance, auto_deploy): - return self._call( - ctxt, 'create_transfer_schedule', + def create_transfer_schedule( + self, + ctxt, + transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ): + return self._call( + ctxt, + 'create_transfer_schedule', transfer_id=transfer_id, schedule=schedule, enabled=enabled, exp_date=exp_date, shutdown_instance=shutdown_instance, - auto_deploy=auto_deploy) + auto_deploy=auto_deploy, + ) - def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, - updated_values): + def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_values): return self._call( - ctxt, 'update_transfer_schedule', + ctxt, + 'update_transfer_schedule', transfer_id=transfer_id, schedule_id=schedule_id, - updated_values=updated_values) + updated_values=updated_values, + ) def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): return self._call( - ctxt, 'delete_transfer_schedule', + ctxt, + 'delete_transfer_schedule', transfer_id=transfer_id, - schedule_id=schedule_id) + schedule_id=schedule_id, + ) def get_transfer_schedules(self, ctxt, transfer_id=None, expired=True): return self._call( - ctxt, 'get_transfer_schedules', - transfer_id=transfer_id, expired=expired) + ctxt, 'get_transfer_schedules', transfer_id=transfer_id, expired=expired + ) - def get_transfer_schedule(self, ctxt, transfer_id, - schedule_id, expired=True): + def get_transfer_schedule(self, ctxt, transfer_id, schedule_id, expired=True): return self._call( - ctxt, 'get_transfer_schedule', + ctxt, + 'get_transfer_schedule', transfer_id=transfer_id, schedule_id=schedule_id, - expired=expired) + expired=expired, + ) def update_transfer(self, ctxt, transfer_id, updated_properties): return self._call( - ctxt, 'update_transfer', + ctxt, + 'update_transfer', transfer_id=transfer_id, - updated_properties=updated_properties) + updated_properties=updated_properties, + ) def get_diagnostics(self, ctxt): return self._call(ctxt, 'get_diagnostics') @@ -390,91 +508,113 @@ def get_diagnostics(self, ctxt): def get_all_diagnostics(self, ctxt): return self._call(ctxt, 'get_all_diagnostics') - def create_region( - self, ctxt, region_name, description="", enabled=True): + def create_region(self, ctxt, region_name, description="", enabled=True): return self._call( - ctxt, 'create_region', + ctxt, + 'create_region', region_name=region_name, description=description, - enabled=enabled) + enabled=enabled, + ) def get_regions(self, ctxt): return self._call(ctxt, 'get_regions') def get_region(self, ctxt, region_id): - return self._call( - ctxt, 'get_region', region_id=region_id) + return self._call(ctxt, 'get_region', region_id=region_id) def update_region(self, ctxt, region_id, updated_values): return self._call( - ctxt, 'update_region', - region_id=region_id, - updated_values=updated_values) + ctxt, 'update_region', region_id=region_id, updated_values=updated_values + ) def delete_region(self, ctxt, region_id): - return self._call( - ctxt, 'delete_region', region_id=region_id) + return self._call(ctxt, 'delete_region', region_id=region_id) def register_service( - self, ctxt, host, binary, topic, enabled, mapped_regions, - providers=None, specs=None): - return self._call( - ctxt, 'register_service', host=host, binary=binary, - topic=topic, enabled=enabled, mapped_regions=mapped_regions, - providers=providers, specs=specs) + self, + ctxt, + host, + binary, + topic, + enabled, + mapped_regions, + providers=None, + specs=None, + ): + return self._call( + ctxt, + 'register_service', + host=host, + binary=binary, + topic=topic, + enabled=enabled, + mapped_regions=mapped_regions, + providers=providers, + specs=specs, + ) def check_service_registered(self, ctxt, host, binary, topic): return self._call( - ctxt, 'check_service_registered', host=host, binary=binary, - topic=topic) + ctxt, 'check_service_registered', host=host, binary=binary, topic=topic + ) def refresh_service_status(self, ctxt, service_id): - return self._call( - ctxt, 'refresh_service_status', service_id=service_id) + return self._call(ctxt, 'refresh_service_status', service_id=service_id) def get_services(self, ctxt): return self._call(ctxt, 'get_services') def get_service(self, ctxt, service_id): - return self._call( - ctxt, 'get_service', service_id=service_id) + return self._call(ctxt, 'get_service', service_id=service_id) def update_service(self, ctxt, service_id, updated_values): return self._call( - ctxt, 'update_service', service_id=service_id, - updated_values=updated_values) + ctxt, 'update_service', service_id=service_id, updated_values=updated_values + ) def delete_service(self, ctxt, service_id): - return self._call( - ctxt, 'delete_service', service_id=service_id) + return self._call(ctxt, 'delete_service', service_id=service_id) def confirm_transfer_minions_allocation( - self, ctxt, transfer_id, minion_machine_allocations): + self, ctxt, transfer_id, minion_machine_allocations + ): self._call( - ctxt, 'confirm_transfer_minions_allocation', + ctxt, + 'confirm_transfer_minions_allocation', transfer_id=transfer_id, - minion_machine_allocations=minion_machine_allocations) + minion_machine_allocations=minion_machine_allocations, + ) def report_transfer_minions_allocation_error( - self, ctxt, transfer_id, minion_allocation_error_details): + self, ctxt, transfer_id, minion_allocation_error_details + ): self._call( - ctxt, 'report_transfer_minions_allocation_error', + ctxt, + 'report_transfer_minions_allocation_error', transfer_id=transfer_id, - minion_allocation_error_details=minion_allocation_error_details) + minion_allocation_error_details=minion_allocation_error_details, + ) def confirm_deployment_minions_allocation( - self, ctxt, deployment_id, minion_machine_allocations): + self, ctxt, deployment_id, minion_machine_allocations + ): self._call( - ctxt, 'confirm_deployment_minions_allocation', + ctxt, + 'confirm_deployment_minions_allocation', deployment_id=deployment_id, - minion_machine_allocations=minion_machine_allocations) + minion_machine_allocations=minion_machine_allocations, + ) def report_deployment_minions_allocation_error( - self, ctxt, deployment_id, minion_allocation_error_details): + self, ctxt, deployment_id, minion_allocation_error_details + ): self._call( - ctxt, 'report_deployment_minions_allocation_error', + ctxt, + 'report_deployment_minions_allocation_error', deployment_id=deployment_id, - minion_allocation_error_details=minion_allocation_error_details) + minion_allocation_error_details=minion_allocation_error_details, + ) class ConductorTaskRpcEventHandler(events.BaseEventHandler): @@ -497,25 +637,47 @@ def get_progress_update_identifier(cls, progress_update): return progress_update['index'] def add_progress_update( - self, message, initial_step=0, total_steps=0, return_event=False): + self, message, initial_step=0, total_steps=0, return_event=False + ): LOG.info( "Sending progress update for task '%s' to conductor: %s", - self._task_id, message) + self._task_id, + message, + ) return self._rpc_conductor_client.add_task_progress_update( - self._ctxt, self._task_id, message, initial_step=initial_step, - total_steps=total_steps, return_event=return_event) + self._ctxt, + self._task_id, + message, + initial_step=initial_step, + total_steps=total_steps, + return_event=return_event, + ) def update_progress_update( - self, update_identifier, new_current_step, - new_total_steps=None, new_message=None, sync=False): + self, + update_identifier, + new_current_step, + new_total_steps=None, + new_message=None, + sync=False, + ): LOG.info( "Updating progress update '%s' for task '%s' with new step %s", - update_identifier, self._task_id, new_current_step) + update_identifier, + self._task_id, + new_current_step, + ) self._rpc_conductor_client.update_task_progress_update( - self._ctxt, self._task_id, update_identifier, new_current_step, - new_total_steps=new_total_steps, new_message=new_message, - sync=sync) + self._ctxt, + self._task_id, + update_identifier, + new_current_step, + new_total_steps=new_total_steps, + new_message=new_message, + sync=sync, + ) def add_event(self, message, level=constants.TASK_EVENT_INFO): self._rpc_conductor_client.add_task_event( - self._ctxt, self._task_id, level, message) + self._ctxt, self._task_id, level, message + ) diff --git a/coriolis/conductor/rpc/server.py b/coriolis/conductor/rpc/server.py index 98bec5e1..5b0329dd 100644 --- a/coriolis/conductor/rpc/server.py +++ b/coriolis/conductor/rpc/server.py @@ -10,20 +10,15 @@ from oslo_config import cfg from oslo_log import log as logging -from coriolis import constants -from coriolis import context +from coriolis import constants, context, exception, keystone, schemas, utils from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models from coriolis.deployer_manager.rpc import client as rpc_deployer_manager_client -from coriolis import exception -from coriolis import keystone from coriolis.licensing import client as licensing_client from coriolis.minion_manager.rpc import client as rpc_minion_manager_client from coriolis.scheduler.rpc import client as rpc_scheduler_client -from coriolis import schemas from coriolis.tasks import factory as tasks_factory from coriolis.transfer_cron.rpc import client as rpc_cron_client -from coriolis import utils from coriolis.worker.rpc import client as rpc_worker_client VERSION = "1.0" @@ -32,11 +27,13 @@ CONDUCTOR_OPTS = [ - cfg.BoolOpt("debug_os_morphing_errors", - default=False, - help="If set, any OSMorphing task which errors out will have " - "all of its following tasks unscheduled so as to allow " - "for live debugging of the OSMorphing setup.") + cfg.BoolOpt( + "debug_os_morphing_errors", + default=False, + help="If set, any OSMorphing task which errors out will have " + "all of its following tasks unscheduled so as to allow " + "for live debugging of the OSMorphing setup.", + ) ] CONF = cfg.CONF @@ -44,13 +41,12 @@ TASK_DEADLOCK_ERROR_MESSAGE = ( "A fatal deadlock has occurred. Further debugging is required. " - "Please review the Conductor logs and contact support for assistance.") + "Please review the Conductor logs and contact support for assistance." +) SCENARIO_TYPE_TO_LICENSING_RESERVATION_MAP = { - constants.TRANSFER_SCENARIO_REPLICA: - licensing_client.RESERVATION_TYPE_REPLICA, - constants.TRANSFER_SCENARIO_LIVE_MIGRATION: - licensing_client.RESERVATION_TYPE_MIGRATION + constants.TRANSFER_SCENARIO_REPLICA: licensing_client.RESERVATION_TYPE_REPLICA, + constants.TRANSFER_SCENARIO_LIVE_MIGRATION: licensing_client.RESERVATION_TYPE_MIGRATION, } @@ -58,11 +54,13 @@ def endpoint_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, endpoint_id, *args, **kwargs): @lockutils.synchronized( - constants.ENDPOINT_LOCK_NAME_FORMAT % endpoint_id, - external=True) + constants.ENDPOINT_LOCK_NAME_FORMAT % endpoint_id, external=True + ) def inner(): return func(self, ctxt, endpoint_id, *args, **kwargs) + return inner() + return wrapper @@ -70,11 +68,13 @@ def transfer_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, transfer_id, *args, **kwargs): @lockutils.synchronized( - constants.TRANSFER_LOCK_NAME_FORMAT % transfer_id, - external=True) + constants.TRANSFER_LOCK_NAME_FORMAT % transfer_id, external=True + ) def inner(): return func(self, ctxt, transfer_id, *args, **kwargs) + return inner() + return wrapper @@ -82,11 +82,13 @@ def schedule_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, transfer_id, schedule_id, *args, **kwargs): @lockutils.synchronized( - constants.SCHEDULE_LOCK_NAME_FORMAT % schedule_id, - external=True) + constants.SCHEDULE_LOCK_NAME_FORMAT % schedule_id, external=True + ) def inner(): return func(self, ctxt, transfer_id, schedule_id, *args, **kwargs) + return inner() + return wrapper @@ -94,11 +96,13 @@ def task_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, task_id, *args, **kwargs): @lockutils.synchronized( - constants.TASK_LOCK_NAME_FORMAT % task_id, - external=True) + constants.TASK_LOCK_NAME_FORMAT % task_id, external=True + ) def inner(): return func(self, ctxt, task_id, *args, **kwargs) + return inner() + return wrapper @@ -108,14 +112,16 @@ def wrapper(self, ctxt, task_id, *args, **kwargs): task = db_api.get_task(ctxt, task_id) @lockutils.synchronized( - constants.EXECUTION_LOCK_NAME_FORMAT % task.execution_id, - external=True) + constants.EXECUTION_LOCK_NAME_FORMAT % task.execution_id, external=True + ) @lockutils.synchronized( - constants.TASK_LOCK_NAME_FORMAT % task_id, - external=True) + constants.TASK_LOCK_NAME_FORMAT % task_id, external=True + ) def inner(): return func(self, ctxt, task_id, *args, **kwargs) + return inner() + return wrapper @@ -123,11 +129,13 @@ def deployment_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, deployment_id, *args, **kwargs): @lockutils.synchronized( - constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, - external=True) + constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, external=True + ) def inner(): return func(self, ctxt, deployment_id, *args, **kwargs) + return inner() + return wrapper @@ -135,11 +143,13 @@ def tasks_execution_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, transfer_id, execution_id, *args, **kwargs): @lockutils.synchronized( - constants.EXECUTION_LOCK_NAME_FORMAT % execution_id, - external=True) + constants.EXECUTION_LOCK_NAME_FORMAT % execution_id, external=True + ) def inner(): return func(self, ctxt, transfer_id, execution_id, *args, **kwargs) + return inner() + return wrapper @@ -147,11 +157,13 @@ def region_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, region_id, *args, **kwargs): @lockutils.synchronized( - constants.REGION_LOCK_NAME_FORMAT % region_id, - external=True) + constants.REGION_LOCK_NAME_FORMAT % region_id, external=True + ) def inner(): return func(self, ctxt, region_id, *args, **kwargs) + return inner() + return wrapper @@ -159,11 +171,13 @@ def service_synchronized(func): @functools.wraps(func) def wrapper(self, ctxt, service_id, *args, **kwargs): @lockutils.synchronized( - constants.SERVICE_LOCK_NAME_FORMAT % service_id, - external=True) + constants.SERVICE_LOCK_NAME_FORMAT % service_id, external=True + ) def inner(): return func(self, ctxt, service_id, *args, **kwargs) + return inner() + return wrapper @@ -184,36 +198,35 @@ def __init__(self): @property def _worker_client(self): if not self._worker_client_instance: - self._worker_client_instance = ( - rpc_worker_client.WorkerClient()) + self._worker_client_instance = rpc_worker_client.WorkerClient() return self._worker_client_instance @property def _scheduler_client(self): if not self._scheduler_client_instance: - self._scheduler_client_instance = ( - rpc_scheduler_client.SchedulerClient()) + self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance @property def _transfer_cron_client(self): if not self._transfer_cron_client_instance: - self._transfer_cron_client_instance = ( - rpc_cron_client.TransferCronClient()) + self._transfer_cron_client_instance = rpc_cron_client.TransferCronClient() return self._transfer_cron_client_instance @property def _minion_manager_client(self): if not self._minion_manager_client_instance: self._minion_manager_client_instance = ( - rpc_minion_manager_client.MinionManagerClient()) + rpc_minion_manager_client.MinionManagerClient() + ) return self._minion_manager_client_instance @property def _deployer_manager_client(self): if not self._deployer_manager_client_instance: self._deployer_manager_client_instance = ( - rpc_deployer_manager_client.DeployerManagerClient()) + rpc_deployer_manager_client.DeployerManagerClient() + ) return self._deployer_manager_client_instance def get_all_diagnostics(self, ctxt): @@ -221,67 +234,86 @@ def get_all_diagnostics(self, ctxt): "conductor": self, "transfer_cron": self._transfer_cron_client, "minion_manager": self._minion_manager_client, - "scheduler": self._scheduler_client} + "scheduler": self._scheduler_client, + } worker_services = [] try: # this will return all registered worker services: - worker_services = self._scheduler_client.get_workers_for_specs( - ctxt) - client_objects.update({ - "worker_%s" % wrk['host']: ( - rpc_worker_client.WorkerClient.from_service_definition( - wrk, timeout=10)) - for wrk in worker_services}) + worker_services = self._scheduler_client.get_workers_for_specs(ctxt) + client_objects.update( + { + "worker_%s" % wrk['host']: ( + rpc_worker_client.WorkerClient.from_service_definition( + wrk, timeout=10 + ) + ) + for wrk in worker_services + } + ) except Exception: LOG.warn( "Exception occurred while listing worker services for " "diagnostics fetching. Exception was: %s", - utils.get_exception_details()) + utils.get_exception_details(), + ) diagnostics = [] - for (service_name, service_client) in client_objects.items(): + for service_name, service_client in client_objects.items(): try: diagnostics.append(service_client.get_diagnostics(ctxt)) except Exception: LOG.warn( "Exception occurred while fetching diagnostics for service" " '%s'. Exception was: %s", - service_name, utils.get_exception_details()) - - for worker_service in self._scheduler_client.get_workers_for_specs( - ctxt): - worker_rpc = ( - rpc_worker_client.WorkerClient.from_service_definition( - worker_service)) + service_name, + utils.get_exception_details(), + ) + + for worker_service in self._scheduler_client.get_workers_for_specs(ctxt): + worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( + worker_service + ) try: diagnostics.append(worker_rpc.get_diagnostics(ctxt)) except Exception: LOG.warn( "Exception occurred while fetching diagnostics for " "worker service '%s'. Error was: %s", - worker_service['host'], utils.get_exception_details()) + worker_service['host'], + utils.get_exception_details(), + ) return diagnostics def _get_worker_service_rpc_for_specs( - self, ctxt, provider_requirements=None, region_sets=None, - enabled=True, random_choice=False, raise_on_no_matches=True): + self, + ctxt, + provider_requirements=None, + region_sets=None, + enabled=True, + random_choice=False, + raise_on_no_matches=True, + ): selected_service = self._scheduler_client.get_worker_service_for_specs( - ctxt, provider_requirements=provider_requirements, - region_sets=region_sets, enabled=enabled, + ctxt, + provider_requirements=provider_requirements, + region_sets=region_sets, + enabled=enabled, random_choice=random_choice, - raise_on_no_matches=raise_on_no_matches) + raise_on_no_matches=raise_on_no_matches, + ) service = db_api.get_service(ctxt, selected_service["id"]) - return rpc_worker_client.WorkerClient.from_service_definition( - service) + return rpc_worker_client.WorkerClient.from_service_definition(service) def _check_delete_reservation_for_transfer(self, transfer_action): action_id = transfer_action.base_id if not self._licensing_client: LOG.warn( "Licensing client not instantiated. Skipping deletion of " - "reservation for transfer action '%s'", action_id) + "reservation for transfer action '%s'", + action_id, + ) return reservation_id = transfer_action.reservation_id @@ -292,34 +324,46 @@ def _check_delete_reservation_for_transfer(self, transfer_action): LOG.warn( "Failed to delete reservation with ID '%s' for transfer " "action with ID '%s'. Skipping. Exception\n%s", - reservation_id, action_id, utils.get_exception_details()) + reservation_id, + action_id, + utils.get_exception_details(), + ) def _create_reservation_for_transfer(self, transfer): action_id = transfer.base_id scenario = transfer.scenario reservation_type = SCENARIO_TYPE_TO_LICENSING_RESERVATION_MAP.get( - scenario, None) + scenario, None + ) if not reservation_type: raise exception.LicensingException( message="Could not determine reservation type for transfer " - f"'{action_id}' with scenario '{transfer.scenario}'.") # noqa + f"'{action_id}' with scenario '{transfer.scenario}'." + ) # noqa if not self._licensing_client: LOG.warn( "Licensing client not instantiated. Skipping creation of " - "reservation for transfer action '%s'", action_id) + "reservation for transfer action '%s'", + action_id, + ) return ninstances = len(transfer.instances) LOG.debug( "Attempting to create '%s' reservation for %d instances for " "transfer action with ID '%s'.", - reservation_type, ninstances, action_id) + reservation_type, + ninstances, + action_id, + ) reservation = self._licensing_client.add_reservation( - reservation_type, ninstances) + reservation_type, ninstances + ) LOG.info( f"Sucessfully created licensing reservation for transfer " - f"with ID '{action_id}' with properties: {reservation}") + f"with ID '{action_id}' with properties: {reservation}" + ) transfer.reservation_id = reservation['id'] return reservation @@ -329,26 +373,27 @@ def _get_licensing_reservation_for_action(self, transfer_action): if not self._licensing_client: LOG.warn( f"Licensing client not instantiated. Skipping getting " - f"reservation for transfer action '{action_id}'") + f"reservation for transfer action '{action_id}'" + ) return None reservation_id = transfer_action.reservation_id if not reservation_id: - LOG.warn( - f"No reservation_id set on transfer action '{action_id}'") + LOG.warn(f"No reservation_id set on transfer action '{action_id}'") return None return self._licensing_client.get_reservation(reservation_id) def _check_mark_reservation_fulfilled( - self, transfer_action, must_unfulfilled=False): + self, transfer_action, must_unfulfilled=False + ): action_id = transfer_action.id - reservation = self._get_licensing_reservation_for_action( - transfer_action) + reservation = self._get_licensing_reservation_for_action(transfer_action) if not reservation: LOG.info( f"No licensing reservation found for transfer action " - f"'{action_id}'. Skipping marking fulfilled.") + f"'{action_id}'. Skipping marking fulfilled." + ) return reservation_id = reservation['id'] @@ -360,53 +405,66 @@ def _check_mark_reservation_fulfilled( "already exists and has been marked as fulfilled " "within the licensing server. Please create a new " "transfer operation in order to obtain a new " - "reservation.") + "reservation." + ) LOG.debug( f"Reservation with ID '{reservation_id}' for transfer " f"transfer action '{action_id}' was already marked as " - f"fulfilled") + f"fulfilled" + ) else: self._licensing_client.mark_reservation_fulfilled(reservation_id) LOG.debug( f"Successfully marked reservation with ID '{reservation_id}' " - f"for transfer action '{action_id}' as fulfilled") + f"for transfer action '{action_id}' as fulfilled" + ) def _check_reservation_for_transfer(self, transfer): scenario = transfer.scenario reservation_type = SCENARIO_TYPE_TO_LICENSING_RESERVATION_MAP.get( - scenario, None) + scenario, None + ) if not reservation_type: raise exception.LicensingException( message="Could not determine reservation type for transfer " - f"'{transfer.id}' with scenario " - f"'{transfer.scenario}'.") + f"'{transfer.id}' with scenario " + f"'{transfer.scenario}'." + ) action_id = transfer.base_id if not self._licensing_client: LOG.warn( "Licensing client not instantiated. Skipping checking of " - "reservation for transfer action '%s'", action_id) + "reservation for transfer action '%s'", + action_id, + ) return reservation_id = transfer.reservation_id if reservation_id: LOG.debug( - "Attempting to check reservation with ID '%s' for transfer " - "action '%s'", reservation_id, action_id) + "Attempting to check reservation with ID '%s' for transfer action '%s'", + reservation_id, + action_id, + ) try: - reservation = self._licensing_client.get_reservation( - reservation_id) + reservation = self._licensing_client.get_reservation(reservation_id) fulfilled_at = reservation.get("fulfilled_at", None) if scenario == constants.TRANSFER_SCENARIO_LIVE_MIGRATION and ( - fulfilled_at): + fulfilled_at + ): raise exception.MigrationLicenceFulfilledException( - action_id=transfer.id, reservation_id=reservation_id, - fulfilled_at=fulfilled_at) + action_id=transfer.id, + reservation_id=reservation_id, + fulfilled_at=fulfilled_at, + ) transfer.reservation_id = ( - self._licensing_client.check_refresh_reservation( - reservation_id)['id']) + self._licensing_client.check_refresh_reservation(reservation_id)[ + 'id' + ] + ) except Exception as ex: exc_code = getattr(ex, 'code', None) if exc_code in [404, 409]: @@ -416,26 +474,38 @@ def _check_reservation_for_transfer(self, transfer): "check the existing reservation '%s' for action " "'%s'. Attempting to create a new reservation. " "Trace was: %s", - reservation_id, action_id, - utils.get_exception_details()) + reservation_id, + action_id, + utils.get_exception_details(), + ) elif exc_code == 404: LOG.debug( "Could not find previous reservation with ID '%s' " "for action '%s'. Attempting to create a new " "reservation. Trace was: %s", - reservation_id, action_id, - utils.get_exception_details()) + reservation_id, + action_id, + utils.get_exception_details(), + ) self._create_reservation_for_transfer(transfer) else: raise ex else: LOG.info( f"Transfer action '{action_id}' has no reservation ID set, " - f"attempting to create a new one for it") + f"attempting to create a new one for it" + ) self._create_reservation_for_transfer(transfer) - def create_endpoint(self, ctxt, name, endpoint_type, description, - connection_info, mapped_regions=None): + def create_endpoint( + self, + ctxt, + name, + endpoint_type, + description, + connection_info, + mapped_regions=None, + ): endpoint = models.Endpoint() endpoint.id = str(uuid.uuid4()) endpoint.name = name @@ -450,14 +520,16 @@ def create_endpoint(self, ctxt, name, endpoint_type, description, if mapped_regions: try: db_api.update_endpoint( - ctxt, endpoint.id, { - "mapped_regions": mapped_regions}) + ctxt, endpoint.id, {"mapped_regions": mapped_regions} + ) except Exception: LOG.warn( "Error adding region mappings during new endpoint " "creation (name: %s), cleaning up endpoint and all " "created mappings for regions: %s", - endpoint.name, mapped_regions) + endpoint.name, + mapped_regions, + ) db_api.delete_endpoint(ctxt, endpoint.id) raise @@ -467,7 +539,9 @@ def create_endpoint(self, ctxt, name, endpoint_type, description, def update_endpoint(self, ctxt, endpoint_id, updated_values): LOG.info( "Attempting to update endpoint '%s' with payload: %s", - endpoint_id, updated_values) + endpoint_id, + updated_values, + ) db_api.update_endpoint(ctxt, endpoint_id, updated_values) LOG.info("Endpoint updated: %s", endpoint_id) return db_api.get_endpoint(ctxt, endpoint_id) @@ -479,167 +553,214 @@ def get_endpoints(self, ctxt): def get_endpoint(self, ctxt, endpoint_id): endpoint = db_api.get_endpoint(ctxt, endpoint_id) if not endpoint: - raise exception.NotFound( - "Endpoint with ID '%s' not found." % endpoint_id) + raise exception.NotFound("Endpoint with ID '%s' not found." % endpoint_id) return endpoint @endpoint_synchronized def delete_endpoint(self, ctxt, endpoint_id): - q_transfers_count = db_api.get_endpoint_transfers_count( - ctxt, endpoint_id) + q_transfers_count = db_api.get_endpoint_transfers_count(ctxt, endpoint_id) if q_transfers_count != 0: - raise exception.NotAuthorized("%s transfers would be orphaned!" % - q_transfers_count) + raise exception.NotAuthorized( + "%s transfers would be orphaned!" % q_transfers_count + ) db_api.delete_endpoint(ctxt, endpoint_id) - def get_endpoint_instances(self, ctxt, endpoint_id, source_environment, - marker, limit, instance_name_pattern, - refresh=False): + def get_endpoint_instances( + self, + ctxt, + endpoint_id, + source_environment, + marker, + limit, + instance_name_pattern, + refresh=False, + ): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_INSTANCES]}) + endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_INSTANCES] + }, + ) return worker_rpc.get_endpoint_instances( - ctxt, endpoint.type, endpoint.connection_info, - source_environment, marker, limit, instance_name_pattern, - refresh=refresh) + ctxt, + endpoint.type, + endpoint.connection_info, + source_environment, + marker, + limit, + instance_name_pattern, + refresh=refresh, + ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name): + self, ctxt, endpoint_id, source_environment, instance_name + ): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_INSTANCES]}) + endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_INSTANCES] + }, + ) return worker_rpc.get_endpoint_instance( - ctxt, endpoint.type, endpoint.connection_info, - source_environment, instance_name) + ctxt, + endpoint.type, + endpoint.connection_info, + source_environment, + instance_name, + ) - def get_endpoint_source_options( - self, ctxt, endpoint_id, env, option_names): + def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [ - constants.PROVIDER_TYPE_SOURCE_ENDPOINT_OPTIONS]}) + endpoint.type: [constants.PROVIDER_TYPE_SOURCE_ENDPOINT_OPTIONS] + }, + ) return worker_rpc.get_endpoint_source_options( - ctxt, endpoint.type, endpoint.connection_info, env, option_names) + ctxt, endpoint.type, endpoint.connection_info, env, option_names + ) - def get_endpoint_destination_options( - self, ctxt, endpoint_id, env, option_names): + def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [ - constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS]}) + endpoint.type: [constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS] + }, + ) return worker_rpc.get_endpoint_destination_options( - ctxt, endpoint.type, endpoint.connection_info, env, option_names) + ctxt, endpoint.type, endpoint.connection_info, env, option_names + ) def get_endpoint_networks(self, ctxt, endpoint_id, env): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_NETWORKS]}) + endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_NETWORKS] + }, + ) return worker_rpc.get_endpoint_networks( - ctxt, endpoint.type, endpoint.connection_info, env) + ctxt, endpoint.type, endpoint.connection_info, env + ) def get_endpoint_storage(self, ctxt, endpoint_id, env): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_STORAGE]}) + endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_STORAGE] + }, + ) return worker_rpc.get_endpoint_storage( - ctxt, endpoint.type, endpoint.connection_info, env) + ctxt, endpoint.type, endpoint.connection_info, env + ) - def get_endpoint_inventory_csv( - self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], provider_requirements={ - endpoint.type: [ - constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT]}) + endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT] + }, + ) return worker_rpc.get_endpoint_inventory_csv( - ctxt, endpoint.type, endpoint.connection_info, - source_environment) + ctxt, endpoint.type, endpoint.connection_info, source_environment + ) def validate_endpoint_connection(self, ctxt, endpoint_id): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], - provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}) + provider_requirements={endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}, + ) return worker_rpc.validate_endpoint_connection( - ctxt, endpoint.type, endpoint.connection_info) + ctxt, endpoint.type, endpoint.connection_info + ) - def validate_endpoint_target_environment( - self, ctxt, endpoint_id, target_env): + def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], - provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}) + provider_requirements={endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}, + ) return worker_rpc.validate_endpoint_target_environment( - ctxt, endpoint.type, target_env) + ctxt, endpoint.type, target_env + ) - def validate_endpoint_source_environment( - self, ctxt, endpoint_id, source_env): + def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( - ctxt, enabled=True, + ctxt, + enabled=True, region_sets=[[reg.id for reg in endpoint.mapped_regions]], - provider_requirements={ - endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}) + provider_requirements={endpoint.type: [constants.PROVIDER_TYPE_ENDPOINT]}, + ) return worker_rpc.validate_endpoint_source_environment( - ctxt, endpoint.type, source_env) + ctxt, endpoint.type, source_env + ) def get_available_providers(self, ctxt): worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - self._scheduler_client.get_any_worker_service(ctxt)) + self._scheduler_client.get_any_worker_service(ctxt) + ) return worker_rpc.get_available_providers(ctxt) def get_provider_schemas(self, ctxt, platform_name, provider_type): # TODO(aznashwan): merge or version/namespace schemas for each worker? worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - self._scheduler_client.get_any_worker_service(ctxt)) - return worker_rpc.get_provider_schemas( - ctxt, platform_name, provider_type) + self._scheduler_client.get_any_worker_service(ctxt) + ) + return worker_rpc.get_provider_schemas(ctxt, platform_name, provider_type) @staticmethod - def _create_task(instance, task_type, execution, depends_on=None, - on_error=False, on_error_only=False): - """ Creates a task with the given properties. + def _create_task( + instance, + task_type, + execution, + depends_on=None, + on_error=False, + on_error_only=False, + ): + """Creates a task with the given properties. NOTE: for on_error and on_error_only tasks, the parent dependencies who are the ones which require cleanup should also be included! @@ -668,11 +789,12 @@ def _create_task(instance, task_type, execution, depends_on=None, # scheduled tasks count as scheduled: elif depends_on: for task_id in depends_on: - if [t + if [ + t for t in task.execution.tasks - if t.id == task_id and ( - (t.status != ( - constants.TASK_STATUS_ON_ERROR_ONLY)))]: + if t.id == task_id + and (t.status != (constants.TASK_STATUS_ON_ERROR_ONLY)) + ]: task.status = constants.TASK_STATUS_SCHEDULED break # on_error tasks with no deps are automatically scheduled: @@ -686,7 +808,7 @@ def _get_task_origin(self, ctxt, action): return { "connection_info": endpoint.connection_info, "type": endpoint.type, - "source_environment": action.source_environment + "source_environment": action.source_environment, } def _get_task_destination(self, ctxt, action): @@ -694,38 +816,58 @@ def _get_task_destination(self, ctxt, action): return { "connection_info": endpoint.connection_info, "type": endpoint.type, - "target_environment": action.destination_environment + "target_environment": action.destination_environment, } def _get_worker_service_rpc_for_task( - self, ctxt, task, origin_endpoint, destination_endpoint, - retry_count=5, retry_period=2, random_choice=True): + self, + ctxt, + task, + origin_endpoint, + destination_endpoint, + retry_count=5, + retry_period=2, + random_choice=True, + ): worker_service = None try: - worker_service = ( - self._scheduler_client.get_worker_service_for_task( - ctxt, {"id": task.id, "task_type": task.task_type}, - origin_endpoint, destination_endpoint, - retry_count=retry_count, retry_period=retry_period, - random_choice=random_choice)) + worker_service = self._scheduler_client.get_worker_service_for_task( + ctxt, + {"id": task.id, "task_type": task.task_type}, + origin_endpoint, + destination_endpoint, + retry_count=retry_count, + retry_period=retry_period, + random_choice=random_choice, + ) except Exception as ex: LOG.debug( "Failed to get worker service for task '%s'. Updating status " "to unscheduleable. Error trace was: %s", - task.id, utils.get_exception_details()) + task.id, + utils.get_exception_details(), + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_FAILED_TO_SCHEDULE, - exception_details=str(ex)) + ctxt, + task.id, + constants.TASK_STATUS_FAILED_TO_SCHEDULE, + exception_details=str(ex), + ) raise - return rpc_worker_client.WorkerClient.from_service_definition( - worker_service) + return rpc_worker_client.WorkerClient.from_service_definition(worker_service) def _begin_tasks( - self, ctxt, action, execution, task_info_override=None, - scheduling_retry_count=5, scheduling_retry_period=2, - delete_trust_id=True): - """ Starts all non-error-only tasks which have no depencies. """ + self, + ctxt, + action, + execution, + task_info_override=None, + scheduling_retry_count=5, + scheduling_retry_period=2, + delete_trust_id=True, + ): + """Starts all non-error-only tasks which have no depencies.""" if not ctxt.trust_id: keystone.create_trust(ctxt) ctxt.delete_trust_id = delete_trust_id @@ -736,25 +878,27 @@ def _begin_tasks( origin = self._get_task_origin(ctxt, action) destination = self._get_task_destination(ctxt, action) - origin_endpoint = db_api.get_endpoint( - ctxt, action.origin_endpoint_id) - destination_endpoint = db_api.get_endpoint( - ctxt, action.destination_endpoint_id) + origin_endpoint = db_api.get_endpoint(ctxt, action.origin_endpoint_id) + destination_endpoint = db_api.get_endpoint(ctxt, action.destination_endpoint_id) newly_started_tasks = [] for task in execution.tasks: - if (not task.depends_on and ( - task.status == constants.TASK_STATUS_SCHEDULED)): + if not task.depends_on and (task.status == constants.TASK_STATUS_SCHEDULED): LOG.info( "Starting dependency-less task '%s' for execution '%s'", - task.id, execution.id) - db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_PENDING) + task.id, + execution.id, + ) + db_api.set_task_status(ctxt, task.id, constants.TASK_STATUS_PENDING) try: worker_rpc = self._get_worker_service_rpc_for_task( - ctxt, task, origin_endpoint, destination_endpoint, + ctxt, + task, + origin_endpoint, + destination_endpoint, retry_count=scheduling_retry_count, - retry_period=scheduling_retry_period) + retry_period=scheduling_retry_period, + ) worker_rpc.begin_task( ctxt, task_id=task.id, @@ -762,112 +906,128 @@ def _begin_tasks( origin=origin, destination=destination, instance=task.instance, - task_info=task_info.get(task.instance, {})) + task_info=task_info.get(task.instance, {}), + ) except Exception: LOG.warn( "Error occured while starting new task '%s'. " "Cancelling execution '%s'. Error was: %s", - task.id, execution.id, utils.get_exception_details()) - self._cancel_tasks_execution( - ctxt, execution, requery=True) + task.id, + execution.id, + utils.get_exception_details(), + ) + self._cancel_tasks_execution(ctxt, execution, requery=True) raise newly_started_tasks.append(task.id) if newly_started_tasks: LOG.info( "Started the following tasks for Execution '%s': %s", - execution.id, newly_started_tasks) + execution.id, + newly_started_tasks, + ) self._set_tasks_execution_status( - ctxt, execution, constants.TASK_STATUS_RUNNING) + ctxt, execution, constants.TASK_STATUS_RUNNING + ) else: # NOTE: this should never happen if _check_execution_tasks_sanity # was called before this method: raise exception.InvalidActionTasksExecutionState( - "No tasks were started at the beginning of execution '%s'" % ( - execution.id)) + "No tasks were started at the beginning of execution '%s'" + % (execution.id) + ) - def _check_execution_tasks_sanity( - self, execution, initial_task_info): - """ Checks whether the given execution's tasks are: + def _check_execution_tasks_sanity(self, execution, initial_task_info): + """Checks whether the given execution's tasks are: - properly odered and not set to deadlock off the bat - properly manipulate the task_info in the right order """ - all_instances_in_tasks = { - t.instance for t in execution.tasks} + all_instances_in_tasks = {t.instance for t in execution.tasks} instances_tasks_mapping = { - instance: [ - t for t in execution.tasks if t.instance == instance] - for instance in all_instances_in_tasks} + instance: [t for t in execution.tasks if t.instance == instance] + for instance in all_instances_in_tasks + } def _check_task_cls_param_requirements(task, instance_task_info_keys): task_cls = tasks_factory.get_task_runner_class(task.task_type) missing_params = [ - p for p in task_cls.get_required_task_info_properties() - if p not in instance_task_info_keys] + p + for p in task_cls.get_required_task_info_properties() + if p not in instance_task_info_keys + ] if missing_params: raise exception.TaskParametersException( "The following task parameters for instance '%s' " "are missing from the task_info for task '%s' of " - "type '%s': %s" % ( - task.instance, task.id, task.task_type, - missing_params)) + "type '%s': %s" + % (task.instance, task.id, task.task_type, missing_params) + ) return task_cls.get_returned_task_info_properties() for instance, instance_tasks in instances_tasks_mapping.items(): - task_info_keys = set(initial_task_info.get( - instance, {}).keys()) + task_info_keys = set(initial_task_info.get(instance, {}).keys()) # mapping between the ID and associated object of processed tasks: processed_tasks = {} - tasks_to_process = { - t.id: t for t in instance_tasks} + tasks_to_process = {t.id: t for t in instance_tasks} while tasks_to_process: queued_tasks = [] # gather all tasks which will be queued to run in parallel: for task in tasks_to_process.values(): if task.status in ( - constants.TASK_STATUS_SCHEDULED, - constants.TASK_STATUS_ON_ERROR_ONLY): + constants.TASK_STATUS_SCHEDULED, + constants.TASK_STATUS_ON_ERROR_ONLY, + ): if not task.depends_on: queued_tasks.append(task) else: missing_deps = [ dep_id for dep_id in task.depends_on - if dep_id not in tasks_to_process and ( - dep_id not in processed_tasks)] + if dep_id not in tasks_to_process + and (dep_id not in processed_tasks) + ] if missing_deps: raise exception.TaskDependencyException( "Task '%s' (type '%s') for instance '%s' " "has non-existent tasks referenced as " - "dependencies: %s" % ( - task.id, task.task_type, - instance, missing_deps)) + "dependencies: %s" + % (task.id, task.task_type, instance, missing_deps) + ) if all( - [dep_id in processed_tasks - for dep_id in task.depends_on]): + [ + dep_id in processed_tasks + for dep_id in task.depends_on + ] + ): queued_tasks.append(task) else: raise exception.InvalidTaskState( "Invalid initial state '%s' for task '%s' " - "of type '%s'." % ( - task.status, task.id, task.task_type)) + "of type '%s'." % (task.status, task.id, task.task_type) + ) # check if nothing was left queued: if not queued_tasks: remaining_tasks_deps_map = { (tid, t.task_type): t.depends_on - for tid, t in tasks_to_process.items()} + for tid, t in tasks_to_process.items() + } processed_tasks_type_map = { - tid: t.task_type - for tid, t in processed_tasks.items()} + tid: t.task_type for tid, t in processed_tasks.items() + } raise exception.ExecutionDeadlockException( "Execution '%s' (type '%s') is bound to be deadlocked:" " there are leftover tasks for instance '%s' which " "will never get queued. Already processed tasks are: " - "%s. Tasks left: %s" % ( - execution.id, execution.type, instance, - processed_tasks_type_map, remaining_tasks_deps_map - )) + "%s. Tasks left: %s" + % ( + execution.id, + execution.type, + instance, + processed_tasks_type_map, + remaining_tasks_deps_map, + ) + ) # mapping for task_info fields modified by each task: modified_fields_by_queued_tasks = {} @@ -875,56 +1035,65 @@ def _check_task_cls_param_requirements(task, instance_task_info_keys): # register what they return/modify: for task in queued_tasks: for new_field in _check_task_cls_param_requirements( - task, task_info_keys): + task, task_info_keys + ): if new_field not in modified_fields_by_queued_tasks: - modified_fields_by_queued_tasks[new_field] = [ - task] + modified_fields_by_queued_tasks[new_field] = [task] else: - modified_fields_by_queued_tasks[new_field].append( - task) + modified_fields_by_queued_tasks[new_field].append(task) # check if any queued tasks would manipulate the same fields: conflicting_fields = { new_field: [t.task_type for t in tasks] - for new_field, tasks in ( - modified_fields_by_queued_tasks.items()) - if len(tasks) > 1} + for new_field, tasks in (modified_fields_by_queued_tasks.items()) + if len(tasks) > 1 + } if conflicting_fields: raise exception.TaskFieldsConflict( "There are fields which will encounter a state " "conflict following the parallelized execution of " "tasks for execution '%s' (type '%s') for instance " - "'%s'. Conflicting fields and tasks will be: : %s" % ( - execution.id, execution.type, instance, - conflicting_fields)) + "'%s'. Conflicting fields and tasks will be: : %s" + % (execution.id, execution.type, instance, conflicting_fields) + ) # register queued tasks as processed before continuing: for task in queued_tasks: processed_tasks[task.id] = task tasks_to_process.pop(task.id) # update current state fields at this point: - task_info_keys = task_info_keys.union(set( - modified_fields_by_queued_tasks.keys())) + task_info_keys = task_info_keys.union( + set(modified_fields_by_queued_tasks.keys()) + ) LOG.debug( "Successfully processed following tasks for instance '%s' " "for execution %s (type '%s') for any state conflict " - "checks: %s", instance, execution.id, execution.type, - [(t.id, t.task_type) for t in queued_tasks]) + "checks: %s", + instance, + execution.id, + execution.type, + [(t.id, t.task_type) for t in queued_tasks], + ) LOG.debug( "Successfully checked all tasks for instance '%s' as part of " "execution '%s' (type '%s') for any state conflicts: %s", - instance, execution.id, execution.type, - [(t.id, t.task_type) for t in instance_tasks]) + instance, + execution.id, + execution.type, + [(t.id, t.task_type) for t in instance_tasks], + ) LOG.debug( "Successfully checked all tasks for execution '%s' (type '%s') " "for ordering or state conflicts.", - execution.id, execution.type) + execution.id, + execution.type, + ) @transfer_synchronized - def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, - auto_deploy=False): - transfer = self._get_transfer( - ctxt, transfer_id, include_task_info=True) + def execute_transfer_tasks( + self, ctxt, transfer_id, shutdown_instances, auto_deploy=False + ): + transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) self._check_transfer_running_executions(ctxt, transfer) self._check_minion_pools_for_action(ctxt, transfer) self._check_reservation_for_transfer(transfer) @@ -953,9 +1122,12 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, transfer.info[instance]['volumes_info'] = [] # NOTE: we update all of the param values before triggering an # execution to ensure that the latest parameters are used: - transfer.info[instance].update({ - "source_environment": transfer.source_environment, - "target_environment": dest_env}) + transfer.info[instance].update( + { + "source_environment": transfer.source_environment, + "target_environment": dest_env, + } + ) # TODO(aznashwan): have these passed separately to the relevant # provider methods (they're currently passed directly inside # dest-env by the API service when accepting the call) @@ -963,20 +1135,19 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, # "storage_mappings": storage_mappings, validate_transfer_source_inputs_task = self._create_task( - instance, - constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS, - execution) + instance, constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS, execution + ) get_instance_info_task = self._create_task( - instance, - constants.TASK_TYPE_GET_INSTANCE_INFO, - execution) + instance, constants.TASK_TYPE_GET_INSTANCE_INFO, execution + ) validate_transfer_destination_inputs_task = self._create_task( instance, constants.TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS, execution, - depends_on=[get_instance_info_task.id]) + depends_on=[get_instance_info_task.id], + ) disk_deployment_depends_on = [] validate_origin_minion_task = None @@ -985,22 +1156,27 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, # _check_execution_tasks_sanity call but # will be populated later when the pool # allocations actually happen: - transfer.info[instance].update({ - "origin_minion_machine_id": None, - "origin_minion_provider_properties": None, - "origin_minion_connection_info": None}) + transfer.info[instance].update( + { + "origin_minion_machine_id": None, + "origin_minion_provider_properties": None, + "origin_minion_connection_info": None, + } + ) validate_origin_minion_task = self._create_task( instance, constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY, # noqa: E501 execution, depends_on=[ get_instance_info_task.id, - validate_transfer_source_inputs_task.id]) - disk_deployment_depends_on.append( - validate_origin_minion_task.id) + validate_transfer_source_inputs_task.id, + ], + ) + disk_deployment_depends_on.append(validate_origin_minion_task.id) else: disk_deployment_depends_on.append( - validate_transfer_source_inputs_task.id) + validate_transfer_source_inputs_task.id + ) validate_destination_minion_task = None if transfer.destination_minion_pool_id: @@ -1008,26 +1184,32 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, # _check_execution_tasks_sanity call but # will be populated later when the pool # allocations actually happen: - transfer.info[instance].update({ - "destination_minion_machine_id": None, - "destination_minion_provider_properties": None, - "destination_minion_connection_info": None, - "destination_minion_backup_writer_connection_info": None}) + transfer.info[instance].update( + { + "destination_minion_machine_id": None, + "destination_minion_provider_properties": None, + "destination_minion_connection_info": None, + "destination_minion_backup_writer_connection_info": None, + } + ) validate_destination_minion_task = self._create_task( instance, constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY, # noqa: E501 execution, - depends_on=[ - validate_transfer_destination_inputs_task.id]) - disk_deployment_depends_on.append( - validate_destination_minion_task.id) + depends_on=[validate_transfer_destination_inputs_task.id], + ) + disk_deployment_depends_on.append(validate_destination_minion_task.id) else: disk_deployment_depends_on.append( - validate_transfer_destination_inputs_task.id) + validate_transfer_destination_inputs_task.id + ) deploy_transfer_disks_task = self._create_task( - instance, constants.TASK_TYPE_DEPLOY_TRANSFER_DISKS, - execution, depends_on=disk_deployment_depends_on) + instance, + constants.TASK_TYPE_DEPLOY_TRANSFER_DISKS, + execution, + depends_on=disk_deployment_depends_on, + ) shutdown_deps = [] deploy_transfer_source_resources_task = None @@ -1035,8 +1217,9 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, deploy_transfer_source_resources_task = self._create_task( instance, constants.TASK_TYPE_DEPLOY_TRANSFER_SOURCE_RESOURCES, - execution, depends_on=[ - deploy_transfer_disks_task.id]) + execution, + depends_on=[deploy_transfer_disks_task.id], + ) shutdown_deps.append(deploy_transfer_source_resources_task) attach_destination_minion_disks_task = None @@ -1044,27 +1227,37 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, if transfer.destination_minion_pool_id: ttyp = constants.TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION attach_destination_minion_disks_task = self._create_task( - instance, ttyp, execution, depends_on=[ - deploy_transfer_disks_task.id]) + instance, + ttyp, + execution, + depends_on=[deploy_transfer_disks_task.id], + ) shutdown_deps.append(attach_destination_minion_disks_task) else: deploy_transfer_target_resources_task = self._create_task( instance, constants.TASK_TYPE_DEPLOY_TRANSFER_TARGET_RESOURCES, - execution, depends_on=[ - deploy_transfer_disks_task.id]) + execution, + depends_on=[deploy_transfer_disks_task.id], + ) shutdown_deps.append(deploy_transfer_target_resources_task) depends_on = [t.id for t in shutdown_deps] if shutdown_instances: shutdown_instance_task = self._create_task( - instance, constants.TASK_TYPE_SHUTDOWN_INSTANCE, - execution, depends_on=depends_on) + instance, + constants.TASK_TYPE_SHUTDOWN_INSTANCE, + execution, + depends_on=depends_on, + ) depends_on = [shutdown_instance_task.id] replicate_disks_task = self._create_task( - instance, constants.TASK_TYPE_REPLICATE_DISKS, - execution, depends_on=depends_on) + instance, + constants.TASK_TYPE_REPLICATE_DISKS, + execution, + depends_on=depends_on, + ) if transfer.origin_minion_pool_id: self._create_task( @@ -1073,8 +1266,10 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, execution, depends_on=[ validate_origin_minion_task.id, - replicate_disks_task.id], - on_error=True) + replicate_disks_task.id, + ], + on_error=True, + ) else: self._create_task( instance, @@ -1082,8 +1277,10 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, execution, depends_on=[ deploy_transfer_source_resources_task.id, - replicate_disks_task.id], - on_error=True) + replicate_disks_task.id, + ], + on_error=True, + ) if transfer.destination_minion_pool_id: detach_volumes_from_minion_task = self._create_task( @@ -1092,8 +1289,10 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, execution, depends_on=[ attach_destination_minion_disks_task.id, - replicate_disks_task.id], - on_error=True) + replicate_disks_task.id, + ], + on_error=True, + ) self._create_task( instance, @@ -1101,40 +1300,48 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, execution, depends_on=[ validate_destination_minion_task.id, - detach_volumes_from_minion_task.id], - on_error=True) + detach_volumes_from_minion_task.id, + ], + on_error=True, + ) else: self._create_task( instance, constants.TASK_TYPE_DELETE_TRANSFER_TARGET_RESOURCES, - execution, depends_on=[ + execution, + depends_on=[ deploy_transfer_target_resources_task.id, - replicate_disks_task.id], - on_error=True) + replicate_disks_task.id, + ], + on_error=True, + ) self._check_execution_tasks_sanity(execution, transfer.info) # update the action info for all of the Transfers: for instance in execution.action.instances: db_api.update_transfer_action_info_for_instance( - ctxt, transfer.id, instance, transfer.info[instance]) + ctxt, transfer.id, instance, transfer.info[instance] + ) # add new execution to DB: db_api.add_transfer_tasks_execution(ctxt, execution) LOG.info("Transfer tasks execution added to DB: %s", execution.id) - uses_minion_pools = any([ - transfer.origin_minion_pool_id, - transfer.destination_minion_pool_id]) + uses_minion_pools = any( + [transfer.origin_minion_pool_id, transfer.destination_minion_pool_id] + ) if uses_minion_pools: self._minion_manager_client.allocate_minion_machines_for_transfer( - ctxt, transfer) + ctxt, transfer + ) self._set_tasks_execution_status( - ctxt, execution, - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + ctxt, execution, constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ) else: self._begin_tasks( - ctxt, transfer, execution, delete_trust_id=not auto_deploy) + ctxt, transfer, execution, delete_trust_id=not auto_deploy + ) if auto_deploy: deployment_options = { @@ -1144,100 +1351,125 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances, "force": False, } self._deployer_manager_client.execute_auto_deployment( - ctxt, transfer.id, execution.id, **deployment_options) + ctxt, transfer.id, execution.id, **deployment_options + ) return self._get_transfer_tasks_execution( - ctxt, transfer_id, execution.id, to_dict=True) + ctxt, transfer_id, execution.id, to_dict=True + ) @transfer_synchronized - def get_transfer_tasks_executions(self, ctxt, transfer_id, - include_tasks=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None): + def get_transfer_tasks_executions( + self, + ctxt, + transfer_id, + include_tasks=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return db_api.get_transfer_tasks_executions( - ctxt, transfer_id, include_tasks, + ctxt, + transfer_id, + include_tasks, include_task_info=include_task_info, marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, - to_dict=True) + to_dict=True, + ) @tasks_execution_synchronized - def get_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, - include_task_info=False): + def get_transfer_tasks_execution( + self, ctxt, transfer_id, execution_id, include_task_info=False + ): return self._get_transfer_tasks_execution( - ctxt, transfer_id, execution_id, - include_task_info=include_task_info, to_dict=True) + ctxt, + transfer_id, + execution_id, + include_task_info=include_task_info, + to_dict=True, + ) @tasks_execution_synchronized def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): - execution = self._get_transfer_tasks_execution( - ctxt, transfer_id, execution_id) + execution = self._get_transfer_tasks_execution(ctxt, transfer_id, execution_id) if execution.status in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidActionTasksExecutionState( "Cannot delete execution '%s' for Transfer '%s' as it is " - "currently in '%s' state." % ( - execution_id, transfer_id, execution.status)) + "currently in '%s' state." + % (execution_id, transfer_id, execution.status) + ) db_api.delete_transfer_tasks_execution(ctxt, execution_id) @tasks_execution_synchronized - def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, - force): - execution = self._get_transfer_tasks_execution( - ctxt, transfer_id, execution_id) + def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force): + execution = self._get_transfer_tasks_execution(ctxt, transfer_id, execution_id) if execution.status not in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidTransferState( - "Transfer '%s' has no running execution to cancel." % ( - transfer_id)) - if execution.status == constants.EXECUTION_STATUS_CANCELLING and ( - not force): + "Transfer '%s' has no running execution to cancel." % (transfer_id) + ) + if execution.status == constants.EXECUTION_STATUS_CANCELLING and (not force): raise exception.InvalidTransferState( "Transfer '%s' is already being cancelled. Please use the " - "force option if you'd like to force-cancel it." % ( - transfer_id)) + "force option if you'd like to force-cancel it." % (transfer_id) + ) self._cancel_tasks_execution(ctxt, execution, force=force) @staticmethod - def _get_transfer_tasks_execution(ctxt, transfer_id, execution_id, - include_task_info=False, to_dict=False): + def _get_transfer_tasks_execution( + ctxt, transfer_id, execution_id, include_task_info=False, to_dict=False + ): execution = db_api.get_transfer_tasks_execution( - ctxt, transfer_id, execution_id, - include_task_info=include_task_info, to_dict=to_dict) + ctxt, + transfer_id, + execution_id, + include_task_info=include_task_info, + to_dict=to_dict, + ) if not execution: raise exception.NotFound( - "Execution with ID '%s' for Transfer '%s' not found." % ( - execution_id, transfer_id)) + "Execution with ID '%s' for Transfer '%s' not found." + % (execution_id, transfer_id) + ) return execution @staticmethod - def get_transfers(ctxt, include_tasks_executions=False, - include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): + def get_transfers( + ctxt, + include_tasks_executions=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return db_api.get_transfers( - ctxt, include_tasks_executions=include_tasks_executions, + ctxt, + include_tasks_executions=include_tasks_executions, include_task_info=include_task_info, marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, - to_dict=True) + to_dict=True, + ) @transfer_synchronized def get_transfer(self, ctxt, transfer_id, include_task_info=False): transfer = self._get_transfer( - ctxt, transfer_id, - include_task_info=include_task_info, to_dict=True) + ctxt, transfer_id, include_task_info=include_task_info, to_dict=True + ) if include_task_info and transfer.get('info'): for instance in transfer['instances']: if instance in transfer['info']: transfer['info'][instance] = utils.sanitize_task_info( - transfer['info'][instance]) + transfer['info'][instance] + ) return transfer @@ -1250,8 +1482,7 @@ def delete_transfer(self, ctxt, transfer_id): @transfer_synchronized def delete_transfer_disks(self, ctxt, transfer_id): - transfer = self._get_transfer( - ctxt, transfer_id, include_task_info=True) + transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) self._check_transfer_running_executions(ctxt, transfer) execution = models.TasksExecution() @@ -1262,22 +1493,28 @@ def delete_transfer_disks(self, ctxt, transfer_id): has_tasks = False for instance in transfer.instances: - if (instance in transfer.info and ( - transfer.info[instance].get('volumes_info'))): + if instance in transfer.info and ( + transfer.info[instance].get('volumes_info') + ): source_del_task = self._create_task( instance, constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS, - execution) + execution, + ) self._create_task( - instance, constants.TASK_TYPE_DELETE_TRANSFER_DISKS, - execution, depends_on=[source_del_task.id]) + instance, + constants.TASK_TYPE_DELETE_TRANSFER_DISKS, + execution, + depends_on=[source_del_task.id], + ) has_tasks = True if not has_tasks: raise exception.InvalidTransferState( "Transfer '%s' does not have volumes information for any " "instances. Ensure that the transfer has been executed " - "successfully priorly" % transfer_id) + "successfully priorly" % transfer_id + ) # ensure we're passing the updated target-env options on the # parent Transfer itself in case of a Transfer update: @@ -1285,21 +1522,20 @@ def delete_transfer_disks(self, ctxt, transfer_id): dest_env['network_map'] = transfer.network_map dest_env['storage_mappings'] = transfer.storage_mappings for instance in transfer.instances: - transfer.info[instance].update({ - "target_environment": dest_env}) + transfer.info[instance].update({"target_environment": dest_env}) self._check_execution_tasks_sanity(execution, transfer.info) # update the action info for all of the Transfers' instances: for instance in transfer.instances: db_api.update_transfer_action_info_for_instance( - ctxt, transfer.id, instance, transfer.info[instance]) + ctxt, transfer.id, instance, transfer.info[instance] + ) db_api.add_transfer_tasks_execution(ctxt, execution) LOG.info("Transfer tasks execution created: %s", execution.id) self._begin_tasks(ctxt, transfer, execution) - return self.get_transfer_tasks_execution( - ctxt, transfer_id, execution.id) + return self.get_transfer_tasks_execution(ctxt, transfer_id, execution.id) @staticmethod def _check_endpoints(ctxt, origin_endpoint, destination_endpoint): @@ -1308,34 +1544,43 @@ def _check_endpoints(ctxt, origin_endpoint, destination_endpoint): "The origin and destination endpoints cannot be the same. " "If you need to perform operations across two areas of " "the same platform (ex: migrating across public cloud regions)" - ", please create two separate endpoints.") + ", please create two separate endpoints." + ) # TODO(alexpilotti): check Barbican secrets content as well - if (origin_endpoint.connection_info == ( - destination_endpoint.connection_info)): + if origin_endpoint.connection_info == (destination_endpoint.connection_info): raise exception.SameDestination() - def create_instances_transfer(self, ctxt, transfer_scenario, - origin_endpoint_id, - destination_endpoint_id, - origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, - destination_environment, instances, - network_map, storage_mappings, notes=None, - user_scripts=None, clone_disks=True, - skip_os_morphing=False): + def create_instances_transfer( + self, + ctxt, + transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes=None, + user_scripts=None, + clone_disks=True, + skip_os_morphing=False, + ): supported_scenarios = [ constants.TRANSFER_SCENARIO_REPLICA, - constants.TRANSFER_SCENARIO_LIVE_MIGRATION] + constants.TRANSFER_SCENARIO_LIVE_MIGRATION, + ] if transfer_scenario not in supported_scenarios: raise exception.InvalidInput( message=f"Unsupported Transfer scenario '{transfer_scenario}'." - f" Must be one of: {supported_scenarios}") # noqa + f" Must be one of: {supported_scenarios}" + ) # noqa origin_endpoint = self.get_endpoint(ctxt, origin_endpoint_id) - destination_endpoint = self.get_endpoint( - ctxt, destination_endpoint_id) + destination_endpoint = self.get_endpoint(ctxt, destination_endpoint_id) self._check_endpoints(ctxt, origin_endpoint, destination_endpoint) transfer = models.Transfer() @@ -1351,13 +1596,13 @@ def create_instances_transfer(self, ctxt, transfer_scenario, transfer.last_execution_status = constants.EXECUTION_STATUS_UNEXECUTED transfer.instances = instances transfer.executions = [] - transfer.info = {instance: { - 'volumes_info': []} for instance in instances} + transfer.info = {instance: {'volumes_info': []} for instance in instances} transfer.notes = notes transfer.network_map = network_map transfer.storage_mappings = storage_mappings transfer.instance_osmorphing_minion_pool_mappings = ( - instance_osmorphing_minion_pool_mappings) + instance_osmorphing_minion_pool_mappings + ) transfer.user_scripts = user_scripts or {} transfer.clone_disks = clone_disks transfer.skip_os_morphing = skip_os_morphing @@ -1370,54 +1615,63 @@ def create_instances_transfer(self, ctxt, transfer_scenario, LOG.info("Transfer created: %s", transfer.id) return self.get_transfer(ctxt, transfer.id) - def _get_transfer(self, ctxt, transfer_id, include_task_info=False, - to_dict=False): + def _get_transfer(self, ctxt, transfer_id, include_task_info=False, to_dict=False): transfer = db_api.get_transfer( - ctxt, transfer_id, include_task_info=include_task_info, - to_dict=to_dict) + ctxt, transfer_id, include_task_info=include_task_info, to_dict=to_dict + ) if not transfer: - raise exception.NotFound( - "Transfer with ID '%s' not found." % transfer_id) + raise exception.NotFound("Transfer with ID '%s' not found." % transfer_id) return transfer @staticmethod - def get_deployments(ctxt, include_tasks, include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): + def get_deployments( + ctxt, + include_tasks, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return db_api.get_deployments( - ctxt, include_tasks, + ctxt, + include_tasks, include_task_info=include_task_info, marker=marker, limit=limit, sort_keys=sort_keys, sort_dirs=sort_dirs, - to_dict=True) + to_dict=True, + ) @deployment_synchronized def get_deployment(self, ctxt, deployment_id, include_task_info=False): return self._get_deployment( - ctxt, deployment_id, include_task_info=include_task_info, - to_dict=True) + ctxt, deployment_id, include_task_info=include_task_info, to_dict=True + ) def _check_running_transfer_deployments(self, ctxt, transfer_id): deployments = db_api.get_transfer_deployments(ctxt, transfer_id) for d in deployments: execution = self._get_execution_for_deployment( - ctxt, d, raise_on_executions_empty=False) - if execution and execution.status in ( - constants.ACTIVE_EXECUTION_STATUSES): + ctxt, d, raise_on_executions_empty=False + ) + if execution and execution.status in (constants.ACTIVE_EXECUTION_STATUSES): raise exception.InvalidTransferState( - "Transfer '%s' is currently being deployed" % transfer_id) + "Transfer '%s' is currently being deployed" % transfer_id + ) @staticmethod def _check_running_executions(action): running_executions = [ - e.id for e in action.executions - if e.status in constants.ACTIVE_EXECUTION_STATUSES] + e.id + for e in action.executions + if e.status in constants.ACTIVE_EXECUTION_STATUSES + ] if running_executions: raise exception.InvalidActionTasksExecutionState( - "Another tasks execution is in progress: %s" % ( - running_executions)) + "Another tasks execution is in progress: %s" % (running_executions) + ) def _check_transfer_running_executions(self, ctxt, transfer): self._check_running_executions(transfer) @@ -1426,24 +1680,29 @@ def _check_transfer_running_executions(self, ctxt, transfer): @staticmethod def _check_valid_transfer_tasks_execution(transfer, force=False): sorted_executions = sorted( - transfer.executions, key=lambda e: e.number, reverse=True) + transfer.executions, key=lambda e: e.number, reverse=True + ) if not sorted_executions: raise exception.InvalidTransferState( - "The Transfer has never been executed.") - - if not [e for e in sorted_executions - if e.type == constants.EXECUTION_TYPE_TRANSFER_EXECUTION and ( - e.status == constants.EXECUTION_STATUS_COMPLETED)]: + "The Transfer has never been executed." + ) + + if not [ + e + for e in sorted_executions + if e.type == constants.EXECUTION_TYPE_TRANSFER_EXECUTION + and (e.status == constants.EXECUTION_STATUS_COMPLETED) + ]: if not force: raise exception.InvalidTransferState( "A transfer must have been executed successfully at least " - "once in order to be deployed") + "once in order to be deployed" + ) def _get_provider_types(self, ctxt, endpoint): provider_types = self.get_available_providers(ctxt).get(endpoint.type) if provider_types is None: - raise exception.NotFound( - "No provider found for: %s" % endpoint.type) + raise exception.NotFound("No provider found for: %s" % endpoint.type) return provider_types["types"] def _validate_deployment_inputs(self, ctxt, deployment, transfer, force): @@ -1455,13 +1714,15 @@ def _validate_deployment_inputs(self, ctxt, deployment, transfer, force): "The transfer doesn't contain volumes information " f"for instance: {instance}. If transferred disks are " "deleted, the transfer needs to be executed anew " - "before a deployment can occur") + "before a deployment can occur" + ) self._check_minion_pools_for_action(ctxt, deployment) self._check_reservation_for_transfer(transfer) def _execute_deployment(self, ctxt, deployment, force): transfer = self._get_transfer( - ctxt, deployment.transfer_id, include_task_info=True) + ctxt, deployment.transfer_id, include_task_info=True + ) skip_os_morphing = deployment.skip_os_morphing clone_disks = deployment.clone_disks user_scripts = deployment.user_scripts @@ -1469,10 +1730,10 @@ def _execute_deployment(self, ctxt, deployment, force): if deployment.deployer_id: self._validate_deployment_inputs(ctxt, deployment, transfer, force) deployment.info = transfer.info - destination_endpoint = self.get_endpoint( - ctxt, transfer.destination_endpoint_id) + destination_endpoint = self.get_endpoint(ctxt, transfer.destination_endpoint_id) destination_provider_types = self._get_provider_types( - ctxt, destination_endpoint) + ctxt, destination_endpoint + ) execution = models.TasksExecution() execution.action = deployment @@ -1494,9 +1755,12 @@ def _execute_deployment(self, ctxt, deployment, force): # execution to ensure that the params on the Transfer are used # in case there was a failed Transfer update (where the new values # could be in the `.info` field instead of the old ones) - deployment.info[instance].update({ - "source_environment": deployment.source_environment, - "target_environment": deployment.destination_environment}) + deployment.info[instance].update( + { + "source_environment": deployment.source_environment, + "target_environment": deployment.destination_environment, + } + ) # TODO(aznashwan): have these passed separately to the relevant # provider methods (they're currently passed directly inside # dest-env by the API service when accepting the call) @@ -1504,136 +1768,163 @@ def _execute_deployment(self, ctxt, deployment, force): # "storage_mappings": storage_mappings, validate_transfer_deployment_inputs_task = self._create_task( - instance, - constants.TASK_TYPE_VALIDATE_DEPLOYMENT_INPUTS, - execution) + instance, constants.TASK_TYPE_VALIDATE_DEPLOYMENT_INPUTS, execution + ) validate_osmorphing_minion_task = None last_validation_task = validate_transfer_deployment_inputs_task if not skip_os_morphing and instance in ( - deployment.instance_osmorphing_minion_pool_mappings): + deployment.instance_osmorphing_minion_pool_mappings + ): # NOTE: these values are required for the # _check_execution_tasks_sanity call but # will be populated later when the pool # allocations actually happen: - deployment.info[instance].update({ - "osmorphing_minion_machine_id": None, - "osmorphing_minion_provider_properties": None, - "osmorphing_minion_connection_info": None}) + deployment.info[instance].update( + { + "osmorphing_minion_machine_id": None, + "osmorphing_minion_provider_properties": None, + "osmorphing_minion_connection_info": None, + } + ) validate_osmorphing_minion_task = self._create_task( instance, constants.TASK_TYPE_VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY, # noqa: E501 - execution, depends_on=[ - validate_transfer_deployment_inputs_task.id]) + execution, + depends_on=[validate_transfer_deployment_inputs_task.id], + ) last_validation_task = validate_osmorphing_minion_task create_snapshot_task = self._create_task( - instance, constants.TASK_TYPE_CREATE_TRANSFER_DISK_SNAPSHOTS, - execution, depends_on=[ - last_validation_task.id]) + instance, + constants.TASK_TYPE_CREATE_TRANSFER_DISK_SNAPSHOTS, + execution, + depends_on=[last_validation_task.id], + ) deploy_transfer_task = self._create_task( instance, constants.TASK_TYPE_DEPLOY_INSTANCE_RESOURCES, execution, - depends_on=[create_snapshot_task.id]) + depends_on=[create_snapshot_task.id], + ) depends_on = [deploy_transfer_task.id] if not skip_os_morphing: task_deploy_os_morphing_resources = None attach_osmorphing_minion_volumes_task = None last_osmorphing_resources_deployment_task = None - if instance in ( - deployment.instance_osmorphing_minion_pool_mappings): + if instance in (deployment.instance_osmorphing_minion_pool_mappings): osmorphing_vol_attachment_deps = [ - validate_osmorphing_minion_task.id] + validate_osmorphing_minion_task.id + ] osmorphing_vol_attachment_deps.extend(depends_on) attach_osmorphing_minion_volumes_task = self._create_task( instance, constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION, # noqa: E501 - execution, depends_on=osmorphing_vol_attachment_deps) + execution, + depends_on=osmorphing_vol_attachment_deps, + ) last_osmorphing_resources_deployment_task = ( - attach_osmorphing_minion_volumes_task) + attach_osmorphing_minion_volumes_task + ) collect_osmorphing_info_task = self._create_task( instance, constants.TASK_TYPE_COLLECT_OSMORPHING_INFO, execution, - depends_on=[attach_osmorphing_minion_volumes_task.id]) + depends_on=[attach_osmorphing_minion_volumes_task.id], + ) last_osmorphing_resources_deployment_task = ( - collect_osmorphing_info_task) + collect_osmorphing_info_task + ) else: task_deploy_os_morphing_resources = self._create_task( instance, constants.TASK_TYPE_DEPLOY_OS_MORPHING_RESOURCES, - execution, depends_on=depends_on) + execution, + depends_on=depends_on, + ) last_osmorphing_resources_deployment_task = ( - task_deploy_os_morphing_resources) + task_deploy_os_morphing_resources + ) task_osmorphing = self._create_task( - instance, constants.TASK_TYPE_OS_MORPHING, - execution, depends_on=[ - last_osmorphing_resources_deployment_task.id]) + instance, + constants.TASK_TYPE_OS_MORPHING, + execution, + depends_on=[last_osmorphing_resources_deployment_task.id], + ) depends_on = [task_osmorphing.id] - if instance in ( - deployment.instance_osmorphing_minion_pool_mappings): + if instance in (deployment.instance_osmorphing_minion_pool_mappings): detach_osmorphing_minion_volumes_task = self._create_task( instance, constants.TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION, # noqa: E501 - execution, depends_on=[ + execution, + depends_on=[ attach_osmorphing_minion_volumes_task.id, - task_osmorphing.id], - on_error=True) + task_osmorphing.id, + ], + on_error=True, + ) release_osmorphing_minion_task = self._create_task( instance, constants.TASK_TYPE_RELEASE_OSMORPHING_MINION, - execution, depends_on=[ + execution, + depends_on=[ validate_osmorphing_minion_task.id, - detach_osmorphing_minion_volumes_task.id], - on_error=True) + detach_osmorphing_minion_volumes_task.id, + ], + on_error=True, + ) depends_on.append(release_osmorphing_minion_task.id) else: task_delete_os_morphing_resources = self._create_task( instance, constants.TASK_TYPE_DELETE_OS_MORPHING_RESOURCES, - execution, depends_on=[ + execution, + depends_on=[ task_deploy_os_morphing_resources.id, - task_osmorphing.id], - on_error=True) + task_osmorphing.id, + ], + on_error=True, + ) depends_on.append(task_delete_os_morphing_resources.id) - if (constants.PROVIDER_TYPE_INSTANCE_FLAVOR in - destination_provider_types): + if constants.PROVIDER_TYPE_INSTANCE_FLAVOR in destination_provider_types: get_optimal_flavor_task = self._create_task( - instance, constants.TASK_TYPE_GET_OPTIMAL_FLAVOR, - execution, depends_on=depends_on) + instance, + constants.TASK_TYPE_GET_OPTIMAL_FLAVOR, + execution, + depends_on=depends_on, + ) depends_on = [get_optimal_flavor_task.id] finalize_deployment_task = self._create_task( instance, constants.TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT, execution, - depends_on=depends_on) + depends_on=depends_on, + ) self._create_task( instance, constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS, - execution, depends_on=[ - create_snapshot_task.id, - finalize_deployment_task.id], - on_error=clone_disks) + execution, + depends_on=[create_snapshot_task.id, finalize_deployment_task.id], + on_error=clone_disks, + ) cleanup_deployment_task = self._create_task( instance, constants.TASK_TYPE_CLEANUP_FAILED_INSTANCE_DEPLOYMENT, execution, - depends_on=[ - deploy_transfer_task.id, - finalize_deployment_task.id], - on_error_only=True) + depends_on=[deploy_transfer_task.id, finalize_deployment_task.id], + on_error_only=True, + ) if not clone_disks: self._create_task( @@ -1641,52 +1932,62 @@ def _execute_deployment(self, ctxt, deployment, force): constants.TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS, execution, depends_on=[cleanup_deployment_task.id], - on_error=True) + on_error=True, + ) self._check_execution_tasks_sanity(execution, deployment.info) db_api.add_transfer_tasks_execution(ctxt, execution) if not skip_os_morphing and ( - deployment.instance_osmorphing_minion_pool_mappings): + deployment.instance_osmorphing_minion_pool_mappings + ): # NOTE: we lock on the deployment ID to ensure the minion # allocation confirmations don't come in too early: with lockutils.lock( - constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment.id, - external=True): - (self._minion_manager_client - .allocate_minion_machines_for_deployment( - ctxt, deployment, include_transfer_minions=False, - include_osmorphing_minions=True)) + constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment.id, external=True + ): + ( + self._minion_manager_client.allocate_minion_machines_for_deployment( + ctxt, + deployment, + include_transfer_minions=False, + include_osmorphing_minions=True, + ) + ) self._set_tasks_execution_status( - ctxt, execution, - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + ctxt, + execution, + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS, + ) else: self._begin_tasks(ctxt, deployment, execution) - def confirm_deployer_completed( - self, ctxt, deployment_id, force=False): + def confirm_deployer_completed(self, ctxt, deployment_id, force=False): try: with lockutils.lock( - constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, - external=True): + constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, external=True + ): deployment = self._get_deployment( - ctxt, deployment_id, include_task_info=True) + ctxt, deployment_id, include_task_info=True + ) with lockutils.lock( - constants.TRANSFER_LOCK_NAME_FORMAT % - deployment.transfer_id, external=True): + constants.TRANSFER_LOCK_NAME_FORMAT % deployment.transfer_id, + external=True, + ): self._execute_deployment(ctxt, deployment, force) except BaseException: LOG.error( f"Something went wrong while attempting to launch pending " f"deployment '{deployment_id}. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) db_api.set_action_last_execution_status( - ctxt, deployment_id, constants.EXECUTION_STATUS_ERROR) + ctxt, deployment_id, constants.EXECUTION_STATUS_ERROR + ) raise @deployment_synchronized - def report_deployer_failure( - self, ctxt, deployment_id, deployer_error_details): + def report_deployer_failure(self, ctxt, deployment_id, deployer_error_details): error_status = constants.EXECUTION_STATUS_ERROR expected_status = constants.EXECUTION_STATUS_PENDING try: @@ -1696,50 +1997,65 @@ def report_deployer_failure( raise exception.InvalidDeploymentState( f"Deployment is in '{deployment.last_execution_status}' " f"status instead of the expected '{expected_status}' to " - "have deployers fail for it.") + "have deployers fail for it." + ) LOG.warn( "Error occurred while waiting for deployer to finish. Setting " f"'{error_status}' status to Deployment '{deployment_id}'. " - f"Error was: {deployer_error_details}") + f"Error was: {deployer_error_details}" + ) except BaseException: LOG.error( f"Something went wrong while attempting to report deployer " f"error for deployment {deployment_id}. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) raise finally: - db_api.set_action_last_execution_status( - ctxt, deployment_id, error_status) + db_api.set_action_last_execution_status(ctxt, deployment_id, error_status) def _normalize_user_scripts(self, user_scripts, instances): - """ Removes instance user_scripts if said instance is not one of the - selected instances for the replica/migration """ + """Removes instance user_scripts if said instance is not one of the + selected instances for the replica/migration""" if user_scripts is None: user_scripts = {} for instance in list(user_scripts.get('instances', {}).keys()): if instance not in instances: - LOG.warn("Removing provided instance '%s' from user_scripts " - "body because it's not included in one of the " - "selected instances for this replica/migration: %s", - instance, instances) + LOG.warn( + "Removing provided instance '%s' from user_scripts " + "body because it's not included in one of the " + "selected instances for this replica/migration: %s", + instance, + instances, + ) user_scripts['instances'].pop(instance, None) continue user_scripts['instances'][instance] = ( - user_scripts['instances'][instance].replace('\r\n', '\n'). - replace('\n\r', '\n')) + user_scripts['instances'][instance] + .replace('\r\n', '\n') + .replace('\n\r', '\n') + ) linux_scripts = user_scripts.get('global', {}).get('linux') if linux_scripts: - user_scripts['global']['linux'] = ( - linux_scripts.replace('\r\n', '\n').replace('\n\r', '\n')) + user_scripts['global']['linux'] = linux_scripts.replace( + '\r\n', '\n' + ).replace('\n\r', '\n') return user_scripts @transfer_synchronized def deploy_transfer_instances( - self, ctxt, transfer_id, force=False, wait_for_execution=None, - clone_disks=None, instance_osmorphing_minion_pool_mappings=None, - skip_os_morphing=None, user_scripts=None, trust_id=None): - transfer = self._get_transfer( - ctxt, transfer_id, include_task_info=True) + self, + ctxt, + transfer_id, + force=False, + wait_for_execution=None, + clone_disks=None, + instance_osmorphing_minion_pool_mappings=None, + skip_os_morphing=None, + user_scripts=None, + trust_id=None, + ): + transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) if user_scripts is None: user_scripts = transfer.user_scripts @@ -1783,10 +2099,12 @@ def deploy_transfer_instances( deployment.origin_minion_pool_id = None deployment.destination_minion_pool_id = None deployment.instance_osmorphing_minion_pool_mappings = ( - transfer.instance_osmorphing_minion_pool_mappings) + transfer.instance_osmorphing_minion_pool_mappings + ) if instance_osmorphing_minion_pool_mappings: deployment.instance_osmorphing_minion_pool_mappings.update( - instance_osmorphing_minion_pool_mappings) + instance_osmorphing_minion_pool_mappings + ) if not wait_for_execution: self._validate_deployment_inputs(ctxt, deployment, transfer, force) @@ -1805,75 +2123,88 @@ def _get_instance_scripts(self, user_scripts, instance): "instances": {}, } if user_scripts: - instance_script = user_scripts.get( - "instances", {}).get(instance) + instance_script = user_scripts.get("instances", {}).get(instance) if instance_script: ret["instances"][instance] = instance_script return ret def _deallocate_minion_machines_for_action(self, ctxt, action): - return ( - self._minion_manager_client.deallocate_minion_machines_for_action( - ctxt, action.base_id) + return self._minion_manager_client.deallocate_minion_machines_for_action( + ctxt, action.base_id ) def _check_minion_pools_for_action(self, ctxt, action): self._minion_manager_client.validate_minion_pool_selections_for_action( - ctxt, action) + ctxt, action + ) LOG.debug( - "Successfully checked minion pool selection for action '%s'", - action.base_id) + "Successfully checked minion pool selection for action '%s'", action.base_id + ) def _update_task_info_for_minion_allocations( - self, ctxt, action, minion_machine_allocations): + self, ctxt, action, minion_machine_allocations + ): for instance in action.instances: - instance_minion_machines = minion_machine_allocations.get( - instance, {}) - instance_origin_minion = instance_minion_machines.get( - 'origin_minion') + instance_minion_machines = minion_machine_allocations.get(instance, {}) + instance_origin_minion = instance_minion_machines.get('origin_minion') if instance_origin_minion: - action.info[instance].update({ - "origin_minion_machine_id": instance_origin_minion['id'], - "origin_minion_provider_properties": ( - instance_origin_minion['provider_properties']), - "origin_minion_connection_info": ( - instance_origin_minion['connection_info'])}) - - instance_dst_minion = instance_minion_machines.get( - 'destination_minion') + action.info[instance].update( + { + "origin_minion_machine_id": instance_origin_minion['id'], + "origin_minion_provider_properties": ( + instance_origin_minion['provider_properties'] + ), + "origin_minion_connection_info": ( + instance_origin_minion['connection_info'] + ), + } + ) + + instance_dst_minion = instance_minion_machines.get('destination_minion') if instance_dst_minion: instance_dst_minion_dict = { "destination_minion_machine_id": instance_dst_minion['id'], "destination_minion_provider_properties": ( - instance_dst_minion['provider_properties']), + instance_dst_minion['provider_properties'] + ), "destination_minion_connection_info": ( - instance_dst_minion['connection_info']), + instance_dst_minion['connection_info'] + ), "destination_minion_backup_writer_connection_info": ( - instance_dst_minion['backup_writer_connection_info'])} + instance_dst_minion['backup_writer_connection_info'] + ), + } action.info[instance].update(instance_dst_minion_dict) - osmorph_min = instance_minion_machines.get( - 'osmorphing_minion') + osmorph_min = instance_minion_machines.get('osmorphing_minion') if osmorph_min: - action.info[instance].update({ - "osmorphing_minion_machine_id": osmorph_min['id'], - "osmorphing_minion_provider_properties": ( - osmorph_min['provider_properties']), - "osmorphing_minion_connection_info": ( - osmorph_min['connection_info'])}) + action.info[instance].update( + { + "osmorphing_minion_machine_id": osmorph_min['id'], + "osmorphing_minion_provider_properties": ( + osmorph_min['provider_properties'] + ), + "osmorphing_minion_connection_info": ( + osmorph_min['connection_info'] + ), + } + ) # update the action info for all of the instances: for instance in minion_machine_allocations: if instance not in action.instances: LOG.warn( "Got minion pool allocations for machine(s) which are not " - "part of action '%s'. Ignoring: %s", action.id, - minion_machine_allocations[instance]) + "part of action '%s'. Ignoring: %s", + action.id, + minion_machine_allocations[instance], + ) continue db_api.update_transfer_action_info_for_instance( - ctxt, action.id, instance, action.info[instance]) + ctxt, action.id, instance, action.info[instance] + ) def _get_last_execution_for_transfer(self, ctxt, transfer, requery=False): if requery: @@ -1882,142 +2213,160 @@ def _get_last_execution_for_transfer(self, ctxt, transfer, requery=False): if not transfer.executions: raise exception.InvalidTransferState( "Transfer with ID '%s' has no existing Trasnfer " - "executions." % transfer.id) - last_transfer_execution = sorted( - transfer.executions, key=lambda e: e.number)[-1] + "executions." % transfer.id + ) + last_transfer_execution = sorted(transfer.executions, key=lambda e: e.number)[ + -1 + ] return last_transfer_execution - def _get_execution_for_deployment(self, ctxt, deployment, requery=False, - raise_on_executions_empty=True): + def _get_execution_for_deployment( + self, ctxt, deployment, requery=False, raise_on_executions_empty=True + ): if requery: deployment = self._get_deployment(ctxt, deployment.id) if not deployment.executions: if raise_on_executions_empty: raise exception.InvalidDeploymentState( - "Deployment with ID '%s' has no existing executions." % ( - deployment.id)) + "Deployment with ID '%s' has no existing executions." + % (deployment.id) + ) return None if len(deployment.executions) > 1: raise exception.InvalidDeploymentState( "Deployment with ID '%s' has more than one execution:" - " %s" % (deployment.id, [e.id for e in deployment.executions])) + " %s" % (deployment.id, [e.id for e in deployment.executions]) + ) return deployment.executions[0] @transfer_synchronized def confirm_transfer_minions_allocation( - self, ctxt, transfer_id, minion_machine_allocations): - transfer = self._get_transfer( - ctxt, transfer_id, include_task_info=True) + self, ctxt, transfer_id, minion_machine_allocations + ): + transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) - awaiting_minions_status = ( - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if transfer.last_execution_status != awaiting_minions_status: raise exception.InvalidTransferState( "Transfer is in '%s' status instead of the expected '%s' to " - "have minion machines allocated for it." % ( - transfer.last_execution_status, awaiting_minions_status)) + "have minion machines allocated for it." + % (transfer.last_execution_status, awaiting_minions_status) + ) last_transfer_execution = self._get_last_execution_for_transfer( - ctxt, transfer, requery=False) + ctxt, transfer, requery=False + ) self._update_task_info_for_minion_allocations( - ctxt, transfer, minion_machine_allocations) + ctxt, transfer, minion_machine_allocations + ) last_transfer_execution = db_api.get_transfer_tasks_execution( - ctxt, transfer.id, last_transfer_execution.id) - self._begin_tasks( - ctxt, transfer, last_transfer_execution) + ctxt, transfer.id, last_transfer_execution.id + ) + self._begin_tasks(ctxt, transfer, last_transfer_execution) @transfer_synchronized def report_transfer_minions_allocation_error( - self, ctxt, transfer_id, minion_allocation_error_details): + self, ctxt, transfer_id, minion_allocation_error_details + ): transfer = self._get_transfer(ctxt, transfer_id) - awaiting_minions_status = ( - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if transfer.last_execution_status != awaiting_minions_status: raise exception.InvalidTransferState( "Transfer is in '%s' status instead of the expected '%s' to " - "have minion machines allocations fail for it." % ( - transfer.last_execution_status, awaiting_minions_status)) + "have minion machines allocations fail for it." + % (transfer.last_execution_status, awaiting_minions_status) + ) last_transfer_execution = self._get_last_execution_for_transfer( - ctxt, transfer, requery=False) + ctxt, transfer, requery=False + ) LOG.warn( "Error occurred while allocating minion machines for Transfer " "'%s'. Cancelling the current Transfer Execution ('%s'). " "Error was: %s", - transfer_id, last_transfer_execution.id, - minion_allocation_error_details) - self._cancel_tasks_execution( - ctxt, last_transfer_execution, requery=True) + transfer_id, + last_transfer_execution.id, + minion_allocation_error_details, + ) + self._cancel_tasks_execution(ctxt, last_transfer_execution, requery=True) self._set_tasks_execution_status( - ctxt, last_transfer_execution, - constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS) + ctxt, + last_transfer_execution, + constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS, + ) @deployment_synchronized def confirm_deployment_minions_allocation( - self, ctxt, deployment_id, minion_machine_allocations): - deployment = self._get_deployment( - ctxt, deployment_id, include_task_info=True) + self, ctxt, deployment_id, minion_machine_allocations + ): + deployment = self._get_deployment(ctxt, deployment_id, include_task_info=True) - awaiting_minions_status = ( - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if deployment.last_execution_status != awaiting_minions_status: raise exception.InvalidDeploymentState( "Deployment is in '%s' status instead of the expected '%s' to " - "have minion machines allocated for it." % ( - deployment.last_execution_status, awaiting_minions_status)) + "have minion machines allocated for it." + % (deployment.last_execution_status, awaiting_minions_status) + ) - execution = self._get_execution_for_deployment( - ctxt, deployment, requery=False) + execution = self._get_execution_for_deployment(ctxt, deployment, requery=False) self._update_task_info_for_minion_allocations( - ctxt, deployment, minion_machine_allocations) + ctxt, deployment, minion_machine_allocations + ) self._begin_tasks(ctxt, deployment, execution) @deployment_synchronized def report_deployment_minions_allocation_error( - self, ctxt, deployment_id, minion_allocation_error_details): + self, ctxt, deployment_id, minion_allocation_error_details + ): deployment = self._get_deployment(ctxt, deployment_id) - awaiting_minions_status = ( - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if deployment.last_execution_status != awaiting_minions_status: raise exception.InvalidDeploymentState( "Deployment is in '%s' status instead of the expected '%s' to " - "have minion machines allocations fail for it." % ( - deployment.last_execution_status, awaiting_minions_status)) + "have minion machines allocations fail for it." + % (deployment.last_execution_status, awaiting_minions_status) + ) - execution = self._get_execution_for_deployment( - ctxt, deployment, requery=False) + execution = self._get_execution_for_deployment(ctxt, deployment, requery=False) LOG.warn( "Error occured while allocating minion machines for " "Deployment '%s'. Cancelling the current Execution ('%s'). " "Error was: %s", - deployment_id, execution.id, minion_allocation_error_details) - self._cancel_tasks_execution( - ctxt, execution, requery=True) + deployment_id, + execution.id, + minion_allocation_error_details, + ) + self._cancel_tasks_execution(ctxt, execution, requery=True) self._set_tasks_execution_status( - ctxt, execution, - constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS) + ctxt, execution, constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + ) - def _get_deployment(self, ctxt, deployment_id, include_task_info=False, - to_dict=False): + def _get_deployment( + self, ctxt, deployment_id, include_task_info=False, to_dict=False + ): deployment = db_api.get_deployment( - ctxt, deployment_id, include_task_info=include_task_info, - to_dict=to_dict) + ctxt, deployment_id, include_task_info=include_task_info, to_dict=to_dict + ) if not deployment: raise exception.NotFound( - "Deployment with ID '%s' not found." % deployment_id) + "Deployment with ID '%s' not found." % deployment_id + ) return deployment def _delete_deployment(self, ctxt, deployment_id): deployment = self._get_deployment(ctxt, deployment_id) execution = self._get_execution_for_deployment( - ctxt, deployment, raise_on_executions_empty=False) + ctxt, deployment, raise_on_executions_empty=False + ) if execution: if execution.status in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidDeploymentState( "Cannot delete Deployment '%s' as it is currently in " - "'%s' state." % (deployment_id, execution.status)) + "'%s' state." % (deployment_id, execution.status) + ) db_api.delete_deployment(ctxt, deployment_id) @deployment_synchronized @@ -2029,31 +2378,35 @@ def _cancel_deployment(self, ctxt, deployment_id, force): if len(deployment.executions) != 1: raise exception.InvalidDeploymentState( "Deployment '%s' has in improper number of tasks " - "executions: %d" % (deployment_id, len(deployment.executions))) + "executions: %d" % (deployment_id, len(deployment.executions)) + ) execution = self._get_execution_for_deployment( - ctxt, deployment, raise_on_executions_empty=False) + ctxt, deployment, raise_on_executions_empty=False + ) if execution: if execution.status not in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidDeploymentState( - "Deployment '%s' is not currently running" % deployment_id) + "Deployment '%s' is not currently running" % deployment_id + ) if execution.status == constants.EXECUTION_STATUS_CANCELLING and ( - not force): + not force + ): raise exception.InvalidDeploymentState( "Deployment '%s' is already being cancelled. Please use " - "the force option if you'd like to force-cancel it.") + "the force option if you'd like to force-cancel it." + ) with lockutils.lock( - constants.EXECUTION_LOCK_NAME_FORMAT % execution.id, - external=True): + constants.EXECUTION_LOCK_NAME_FORMAT % execution.id, external=True + ): self._cancel_tasks_execution(ctxt, execution, force=force) @deployment_synchronized def cancel_deployment(self, ctxt, deployment_id, force): self._cancel_deployment(ctxt, deployment_id, force) - def _cancel_tasks_execution( - self, ctxt, execution, requery=True, force=False): - """ Cancels a running Execution by: + def _cancel_tasks_execution(self, ctxt, execution, requery=True, force=False): + """Cancels a running Execution by: - telling workers to kill any already running non-on-error tasks - cancelling any non-on-error tasks which are pending - making all on-error-only tasks as scheduled @@ -2066,52 +2419,61 @@ def _cancel_tasks_execution( if execution.status == constants.EXECUTION_STATUS_RUNNING: LOG.info( "Cancelling tasks execution %s. Current status before " - "cancellation is '%s'", execution.id, execution.status) + "cancellation is '%s'", + execution.id, + execution.status, + ) # mark execution as cancelling: self._set_tasks_execution_status( - ctxt, execution, constants.EXECUTION_STATUS_CANCELLING) - elif execution.status == constants.EXECUTION_STATUS_CANCELLING and ( - not force): + ctxt, execution, constants.EXECUTION_STATUS_CANCELLING + ) + elif execution.status == constants.EXECUTION_STATUS_CANCELLING and (not force): LOG.info( "Execution '%s' is already in CANCELLING status and no " "force flag was provided, skipping re-cancellation.", - execution.id) - self._advance_execution_state( - ctxt, execution, requery=not requery) + execution.id, + ) + self._advance_execution_state(ctxt, execution, requery=not requery) return elif execution.status in constants.FINALIZED_TASK_STATUSES: LOG.info( "Execution '%s' is in a finalized status '%s'. " - "Skipping re-cancellation.", execution.id, execution.status) + "Skipping re-cancellation.", + execution.id, + execution.status, + ) return # iterate through and kill/cancel any non-error # tasks which are running/pending: for task in sorted(execution.tasks, key=lambda t: t.index): - # if force is provided, force-cancel tasks directly: if force and task.status in itertools.chain( - constants.ACTIVE_TASK_STATUSES, - [constants.TASK_STATUS_FAILED_TO_CANCEL]): + constants.ACTIVE_TASK_STATUSES, [constants.TASK_STATUS_FAILED_TO_CANCEL] + ): LOG.warn( "Task '%s' is in %s state, but forcibly setting to " "'%s' as part of cancellation of execution '%s' " "because the 'force' flag was provided.", - task.id, task.status, + task.id, + task.status, constants.TASK_STATUS_FORCE_CANCELED, - execution.id) + execution.id, + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_FORCE_CANCELED, + ctxt, + task.id, + constants.TASK_STATUS_FORCE_CANCELED, exception_details=( "This task was force-canceled at user request. " "Its state prior to its cancellation was '%s'. " "Its error details prior to its cancellation " - "were: %s" % ( - task.status, task.exception_details))) + "were: %s" % (task.status, task.exception_details) + ), + ) continue - if task.status == constants.TASK_STATUS_SCHEDULED and ( - not task.depends_on): + if task.status == constants.TASK_STATUS_SCHEDULED and (not task.depends_on): # any SCHEDULED tasks with no dependencies are automatically # unscheduled. The chain-reaction of unscheduling all # subsequent child tasks will be handled in @@ -2119,16 +2481,24 @@ def _cancel_tasks_execution( LOG.debug( "Setting currently '%s' task '%s' to '%s' as part of " "cancellation of execution '%s'.", - task.status, task.id, constants.TASK_STATUS_UNSCHEDULED, - execution.id) + task.status, + task.id, + constants.TASK_STATUS_UNSCHEDULED, + execution.id, + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_UNSCHEDULED, + ctxt, + task.id, + constants.TASK_STATUS_UNSCHEDULED, exception_details=( "This task was unscheduled during the cancellation " - "of the parent tasks execution.")) + "of the parent tasks execution." + ), + ) elif task.status in ( - constants.TASK_STATUS_PENDING, - constants.TASK_STATUS_STARTING): + constants.TASK_STATUS_PENDING, + constants.TASK_STATUS_STARTING, + ): # any PENDING/STARTING tasks means that they did not have a # host assigned to them yet, and presuming the host does not # start executing the task until it marks itself as the runner, @@ -2136,48 +2506,68 @@ def _cancel_tasks_execution( LOG.debug( "Setting currently '%s' task '%s' to '%s' as part of the " "cancellation of execution '%s'", - task.status, task.id, - constants.TASK_STATUS_UNSCHEDULED, execution.id) + task.status, + task.id, + constants.TASK_STATUS_UNSCHEDULED, + execution.id, + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_UNSCHEDULED, + ctxt, + task.id, + constants.TASK_STATUS_UNSCHEDULED, exception_details=( "This task was already pending execution but was " "unscheduled during the cancellation of the parent " - "tasks execution.")) + "tasks execution." + ), + ) elif task.status in ( - constants.TASK_STATUS_RUNNING, - constants.TASK_STATUS_FAILED_TO_CANCEL): + constants.TASK_STATUS_RUNNING, + constants.TASK_STATUS_FAILED_TO_CANCEL, + ): # cancel any currently running non-error tasks: if not task.on_error: LOG.debug( "Sending cancellation request for %s non-error task " "'%s' as part of cancellation of execution '%s'", - task.status, task.id, execution.id) + task.status, + task.id, + execution.id, + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_CANCELLING) + ctxt, task.id, constants.TASK_STATUS_CANCELLING + ) try: worker_rpc = rpc_worker_client.WorkerClient( # NOTE: we intetionally lowball the timeout for the # cancellation call to prevent the conductor from # hanging an excessive amount of time: - host=task.host, timeout=10) - worker_rpc.cancel_task( - ctxt, task.id, task.process_id, force) + host=task.host, + timeout=10, + ) + worker_rpc.cancel_task(ctxt, task.id, task.process_id, force) except (Exception, KeyboardInterrupt): msg = ( "Failed to send cancellation request for task '%s'" "to worker host '%s' as part of cancellation of " "execution '%s'. Marking task as '%s' for now and " - "awaiting any eventual worker replies later." % ( - task.id, task.host, execution.id, - constants.TASK_STATUS_FAILED_TO_CANCEL)) + "awaiting any eventual worker replies later." + % ( + task.id, + task.host, + execution.id, + constants.TASK_STATUS_FAILED_TO_CANCEL, + ) + ) LOG.error( - "%s. Exception was: %s", msg, - utils.get_exception_details()) + "%s. Exception was: %s", msg, utils.get_exception_details() + ) db_api.set_task_status( - ctxt, task.id, + ctxt, + task.id, constants.TASK_STATUS_FAILED_TO_CANCEL, - exception_details=msg) + exception_details=msg, + ) # let any on-error tasks run to completion but mark # them as CANCELLING_AFTER_COMPLETION so they will @@ -2186,44 +2576,57 @@ def _cancel_tasks_execution( LOG.debug( "Marking %s on-error task %s as %s as part of " "cancellation of execution %s", - task.status, task.id, + task.status, + task.id, constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION, - execution.id) + execution.id, + ) db_api.set_task_status( - ctxt, task.id, + ctxt, + task.id, constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION, exception_details=( "Task will be marked as cancelled after completion" - " as it is a cleanup task.")) + " as it is a cleanup task." + ), + ) elif task.status == constants.TASK_STATUS_ON_ERROR_ONLY: # mark all on-error-only tasks as scheduled: LOG.debug( "Marking on-error-only task '%s' as scheduled following " "cancellation of execution '%s'", - task.id, execution.id) - db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_SCHEDULED) + task.id, + execution.id, + ) + db_api.set_task_status(ctxt, task.id, constants.TASK_STATUS_SCHEDULED) else: LOG.debug( "No action currently taken with respect to task '%s' " "(status '%s', on_error=%s) during cancellation of " "execution '%s'", - task.id, task.status, task.on_error, execution.id) + task.id, + task.status, + task.on_error, + execution.id, + ) - started_tasks = self._advance_execution_state( - ctxt, execution, requery=True) + started_tasks = self._advance_execution_state(ctxt, execution, requery=True) if started_tasks: LOG.info( "The following tasks were started after state advancement " "of execution '%s' after cancellation request: %s", - execution.id, started_tasks) + execution.id, + started_tasks, + ) else: LOG.debug( "No new tasks were started for execution '%s' following " - "state advancement after cancellation.", execution.id) + "state advancement after cancellation.", + execution.id, + ) def _update_reservation_fulfillment_for_execution(self, ctxt, execution): - """ Updates the reservation fulfillment status for the parent + """Updates the reservation fulfillment status for the parent transfer action of the given execution based on its type. Replica transfers are marked as fulfilled as soon as a Transfer @@ -2232,19 +2635,23 @@ def _update_reservation_fulfillment_for_execution(self, ctxt, execution): are deployed for the first (and only) time. """ if execution.type not in ( - constants.EXECUTION_TYPE_TRANSFER_EXECUTION, - constants.EXECUTION_TYPE_DEPLOYMENT): + constants.EXECUTION_TYPE_TRANSFER_EXECUTION, + constants.EXECUTION_TYPE_DEPLOYMENT, + ): LOG.debug( f"Skipping setting reservation fulfillment for execution " - f"'{execution.id}' of type '{execution.type}'.") + f"'{execution.id}' of type '{execution.type}'." + ) return if execution.type not in ( - constants.EXECUTION_TYPE_TRANSFER_EXECUTION, - constants.EXECUTION_TYPE_DEPLOYMENT): + constants.EXECUTION_TYPE_TRANSFER_EXECUTION, + constants.EXECUTION_TYPE_DEPLOYMENT, + ): LOG.debug( f"Skipping setting transfer fulfillment for execution " - f"'{execution.id}' of type '{execution.type}'.") + f"'{execution.id}' of type '{execution.type}'." + ) return transfer_action = execution.action @@ -2253,40 +2660,49 @@ def _update_reservation_fulfillment_for_execution(self, ctxt, execution): deployment = self._get_deployment(ctxt, transfer_id) transfer_id = deployment.transfer_id transfer_action = self._get_transfer( - ctxt, transfer_id, include_task_info=False) + ctxt, transfer_id, include_task_info=False + ) else: transfer_action = self._get_transfer( - ctxt, execution.action_id, include_task_info=False) + ctxt, execution.action_id, include_task_info=False + ) scenario = transfer_action.scenario if scenario == constants.TRANSFER_SCENARIO_REPLICA and ( - execution.type == constants.EXECUTION_TYPE_TRANSFER_EXECUTION): + execution.type == constants.EXECUTION_TYPE_TRANSFER_EXECUTION + ): self._check_mark_reservation_fulfilled( - transfer_action, must_unfulfilled=False) + transfer_action, must_unfulfilled=False + ) elif scenario == constants.TRANSFER_SCENARIO_LIVE_MIGRATION and ( - execution.type == constants.EXECUTION_TYPE_DEPLOYMENT): + execution.type == constants.EXECUTION_TYPE_DEPLOYMENT + ): self._check_mark_reservation_fulfilled( - transfer_action, must_unfulfilled=False) + transfer_action, must_unfulfilled=False + ) else: LOG.debug( f"Skipping setting transfer fulfillment for execution " f"'{execution.id}' of type '{execution.type}' on parent" f"action {transfer_id} of scenario type " - f"{transfer_action.scenario}.") + f"{transfer_action.scenario}." + ) - def _set_tasks_execution_status( - self, ctxt, execution, new_execution_status): + def _set_tasks_execution_status(self, ctxt, execution, new_execution_status): previous_execution_status = execution.status LOG.info( "Tasks execution %(id)s (action %(action)s) status updated " "from %(old_status)s to %(new_status)s", - {"id": execution.id, "new_status": new_execution_status, - "action": execution.action_id, - "old_status": previous_execution_status}) + { + "id": execution.id, + "new_status": new_execution_status, + "action": execution.action_id, + "old_status": previous_execution_status, + }, + ) if new_execution_status == constants.EXECUTION_STATUS_COMPLETED: - self._update_reservation_fulfillment_for_execution( - ctxt, execution) + self._update_reservation_fulfillment_for_execution(ctxt, execution) if new_execution_status in constants.FINALIZED_EXECUTION_STATUSES: # NOTE(aznashwan): because the taskflow flows within the minion @@ -2296,20 +2712,24 @@ def _set_tasks_execution_status( # the minion manager attempts to confirm their allocation # (which the conductor shall refuse) if previous_execution_status == ( - constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS): + constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ): LOG.warn( "Execution '%s' was in '%s' status at time of transition " "to '%s' status. NOT requesting minion machine " - "deallocation from the minion manager." % ( - execution.id, previous_execution_status, - new_execution_status)) + "deallocation from the minion manager." + % (execution.id, previous_execution_status, new_execution_status) + ) else: LOG.debug( "Attempting to deallocate minion machines for finalized " "Execution '%s' of type '%s' (action '%s') following " "its transition into finalized status '%s'", - execution.id, execution.type, execution.action_id, - new_execution_status) + execution.id, + execution.type, + execution.action_id, + new_execution_status, + ) action = db_api.get_action(ctxt, execution.action_id) self._deallocate_minion_machines_for_action(ctxt, action) @@ -2317,24 +2737,31 @@ def _set_tasks_execution_status( LOG.debug( "Deleting Keystone trust following status change " "for Execution '%s' (action '%s') from '%s' to '%s'", - execution.id, execution.action_id, - previous_execution_status, new_execution_status) + execution.id, + execution.action_id, + previous_execution_status, + new_execution_status, + ) keystone.delete_trust(ctxt) else: LOG.debug( "Not deallocating minion machines for Execution '%s' " "of type '%s' (action '%s') yet as it is still in an " "active Execution status (%s)", - execution.id, execution.type, execution.action_id, - new_execution_status) + execution.id, + execution.type, + execution.action_id, + new_execution_status, + ) execution = db_api.set_execution_status( - ctxt, execution.id, new_execution_status) + ctxt, execution.id, new_execution_status + ) @parent_tasks_execution_synchronized def set_task_host(self, ctxt, task_id, host): - """ Saves the ID of the worker host which has accepted - the task to the DB and marks the task as STARTING. """ + """Saves the ID of the worker host which has accepted + the task to the DB and marks the task as STARTING.""" task = db_api.get_task(ctxt, task_id) new_status = constants.TASK_STATUS_STARTING exception_details = None @@ -2345,63 +2772,75 @@ def set_task_host(self, ctxt, task_id, host): LOG.warn( "Non-error task '%s' was in '%s' status although it should" " not have been. Setting a task host for it anyway.", - task.id, task.status) + task.id, + task.status, + ) LOG.debug( "Task '%s' is in %s status, so it will be allowed to " "have a host set for it and run to completion.", - task.id, task.status) + task.id, + task.status, + ) new_status = constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION exception_details = ( "This is a cleanup task so it will be allowed to run to " - "completion despite user-cancellation.") + "completion despite user-cancellation." + ) elif task.status != constants.TASK_STATUS_PENDING: raise exception.InvalidTaskState( "Task with ID '%s' is in '%s' status instead of the " - "expected '%s' required for it to have a task host set." % ( - task_id, task.status, constants.TASK_STATUS_PENDING)) - LOG.info( - "Setting host for task with ID '%s' to '%s'", task_id, host) + "expected '%s' required for it to have a task host set." + % (task_id, task.status, constants.TASK_STATUS_PENDING) + ) + LOG.info("Setting host for task with ID '%s' to '%s'", task_id, host) db_api.set_task_host_properties(ctxt, task_id, host=host) db_api.set_task_status( - ctxt, task_id, new_status, - exception_details=exception_details) - LOG.info( - "Successfully set host for task with ID '%s' to '%s'", - task_id, host) + ctxt, task_id, new_status, exception_details=exception_details + ) + LOG.info("Successfully set host for task with ID '%s' to '%s'", task_id, host) @parent_tasks_execution_synchronized def set_task_process(self, ctxt, task_id, process_id): - """ Sets the ID of the Worker-side process for the given task, - and marks the task as actually 'RUNNING'. """ + """Sets the ID of the Worker-side process for the given task, + and marks the task as actually 'RUNNING'.""" task = db_api.get_task(ctxt, task_id) if not task.host: raise exception.InvalidTaskState( "Task with ID '%s' (current status '%s') has no host set " - "for it. Cannot set host process." % ( - task_id, task.status)) + "for it. Cannot set host process." % (task_id, task.status) + ) acceptable_statuses = [ constants.TASK_STATUS_STARTING, - constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION] + constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION, + ] if task.status not in acceptable_statuses: raise exception.InvalidTaskState( "Task with ID '%s' is in '%s' status instead of the " "expected statuses (%s) required for it to have a task " - "process set." % ( - task_id, task.status, acceptable_statuses)) + "process set." % (task_id, task.status, acceptable_statuses) + ) LOG.info( "Setting process '%s' (host %s) for task '%s' and transitioning " - "it from status '%s' to '%s'", process_id, task.host, task_id, - task.status, constants.TASK_STATUS_RUNNING) + "it from status '%s' to '%s'", + process_id, + task.host, + task_id, + task.status, + constants.TASK_STATUS_RUNNING, + ) db_api.set_task_host_properties(ctxt, task_id, process_id=process_id) db_api.set_task_status(ctxt, task_id, constants.TASK_STATUS_RUNNING) LOG.info( "Successfully set task process for task with ID '%s' to '%s'", - task_id, process_id) + task_id, + process_id, + ) def _check_clean_execution_deadlock( - self, ctxt, execution, task_statuses=None, requery=True): - """ Checks whether an execution is deadlocked. + self, ctxt, execution, task_statuses=None, requery=True + ): + """Checks whether an execution is deadlocked. Deadlocked executions have no currently running/pending tasks but some remaining scheduled tasks. If this occurs, all pending/error-only tasks are marked @@ -2419,37 +2858,44 @@ def _check_clean_execution_deadlock( determined_state = constants.EXECUTION_STATUS_RUNNING status_vals = task_statuses.values() if constants.TASK_STATUS_SCHEDULED in status_vals and not ( - any([stat in status_vals - for stat in constants.ACTIVE_TASK_STATUSES])): + any([stat in status_vals for stat in constants.ACTIVE_TASK_STATUSES]) + ): LOG.warn( - "Execution '%s' is deadlocked. Cleaning up now. " - "Task statuses are: %s", - execution.id, task_statuses) + "Execution '%s' is deadlocked. Cleaning up now. Task statuses are: %s", + execution.id, + task_statuses, + ) for task_id, stat in task_statuses.items(): if stat in ( - constants.TASK_STATUS_SCHEDULED, - constants.TASK_STATUS_ON_ERROR_ONLY): + constants.TASK_STATUS_SCHEDULED, + constants.TASK_STATUS_ON_ERROR_ONLY, + ): LOG.warn( - "Marking deadlocked task '%s' as that (current " - "state: %s)", task_id, stat) + "Marking deadlocked task '%s' as that (current state: %s)", + task_id, + stat, + ) db_api.set_task_status( - ctxt, task_id, + ctxt, + task_id, constants.TASK_STATUS_CANCELED_FROM_DEADLOCK, - exception_details=TASK_DEADLOCK_ERROR_MESSAGE) - LOG.warn( - "Marking deadlocked execution '%s' as DEADLOCKED", - execution.id) + exception_details=TASK_DEADLOCK_ERROR_MESSAGE, + ) + LOG.warn("Marking deadlocked execution '%s' as DEADLOCKED", execution.id) self._set_tasks_execution_status( - ctxt, execution, constants.EXECUTION_STATUS_DEADLOCKED) + ctxt, execution, constants.EXECUTION_STATUS_DEADLOCKED + ) LOG.error( "Execution '%s' is deadlocked. Cleanup has been performed. " "Task statuses at time of deadlock were: %s", - execution.id, task_statuses) + execution.id, + task_statuses, + ) determined_state = constants.EXECUTION_STATUS_DEADLOCKED return determined_state def _get_execution_status(self, ctxt, execution, requery=False): - """ Returns the global status of an execution. + """Returns the global status of an execution. RUNNING - at least one task is RUNNING, STARTING, PENDING or CANCELLING COMPLETED - all non-error-only tasks are COMPLETED CANCELED - no more RUNNING/PENDING/SCHEDULED tasks but some CANCELED @@ -2472,12 +2918,14 @@ def _get_execution_status(self, ctxt, execution, requery=False): if task.status in constants.CANCELED_TASK_STATUSES: is_canceled = True if task.status in ( - constants.TASK_STATUS_ERROR, - constants.TASK_STATUS_FAILED_TO_SCHEDULE): + constants.TASK_STATUS_ERROR, + constants.TASK_STATUS_FAILED_TO_SCHEDULE, + ): is_errord = True if task.status in ( - constants.TASK_STATUS_CANCELLING, - constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION): + constants.TASK_STATUS_CANCELLING, + constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION, + ): is_cancelling = True if task.status == constants.TASK_STATUS_SCHEDULED: has_scheduled_tasks = True @@ -2504,12 +2952,14 @@ def _get_execution_status(self, ctxt, execution, requery=False): LOG.debug( "Overall status for Execution '%s' determined to be '%s'." "Task statuses at time of decision: %s", - execution.id, status, task_stat_map) + execution.id, + status, + task_stat_map, + ) return status - def _advance_execution_state( - self, ctxt, execution, requery=True, instance=None): - """ Advances the state of the execution by starting/refreshing + def _advance_execution_state(self, ctxt, execution, requery=True, instance=None): + """Advances the state of the execution by starting/refreshing the state of all child tasks. If the execution has finalized (either completed or error'd), updates its state to the finalized one. @@ -2536,40 +2986,44 @@ def _advance_execution_state( "Execution state advancement called on Execution '%s' which " "is not in an active status in the DB (it's currently '%s'). " "Double-checking for deadlock and returning early.", - execution.id, execution.status) + execution.id, + execution.status, + ) if self._check_clean_execution_deadlock( - ctxt, execution, task_statuses=None, - requery=not requery) == ( - constants.EXECUTION_STATUS_DEADLOCKED): + ctxt, execution, task_statuses=None, requery=not requery + ) == (constants.EXECUTION_STATUS_DEADLOCKED): LOG.error( "Execution '%s' deadlocked even before Transfer state " "advancement . Cleanup has been perfomed. Returning.", - execution.id) + execution.id, + ) return [] tasks_to_process = execution.tasks if instance: tasks_to_process = [ - task for task in execution.tasks - if task.instance == instance] + task for task in execution.tasks if task.instance == instance + ] if not tasks_to_process: raise exception.InvalidActionTasksExecutionState( "State advancement requested for execution '%s' for " - "instance '%s', which has no tasks defined for it." % ( - execution.id, instance)) + "instance '%s', which has no tasks defined for it." + % (execution.id, instance) + ) LOG.debug( "State of execution '%s' before state advancement is: %s", - execution.id, execution.status) + execution.id, + execution.status, + ) origin = self._get_task_origin(ctxt, execution.action) destination = self._get_task_destination(ctxt, execution.action) - action = db_api.get_action( - ctxt, execution.action_id, include_task_info=True) - origin_endpoint = db_api.get_endpoint( - ctxt, execution.action.origin_endpoint_id) + action = db_api.get_action(ctxt, execution.action_id, include_task_info=True) + origin_endpoint = db_api.get_endpoint(ctxt, execution.action.origin_endpoint_id) destination_endpoint = db_api.get_endpoint( - ctxt, execution.action.destination_endpoint_id) + ctxt, execution.action.destination_endpoint_id + ) started_tasks = [] @@ -2579,17 +3033,24 @@ def _start_task(task): LOG.error( "No info present for instance '%s' in action '%s' for task" " '%s' (type '%s') of execution '%s' (type '%s'). " - "Defaulting to empty dict." % - (task.instance, action.id, task.id, task.task_type, - execution.id, execution.type)) + "Defaulting to empty dict." + % ( + task.instance, + action.id, + task.id, + task.task_type, + execution.id, + execution.type, + ) + ) task_info = {} else: task_info = action.info[task.instance] - db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_PENDING) + db_api.set_task_status(ctxt, task.id, constants.TASK_STATUS_PENDING) try: worker_rpc = self._get_worker_service_rpc_for_task( - ctxt, task, origin_endpoint, destination_endpoint) + ctxt, task, origin_endpoint, destination_endpoint + ) worker_rpc.begin_task( ctxt, task_id=task.id, @@ -2597,20 +3058,26 @@ def _start_task(task): origin=origin, destination=destination, instance=task.instance, - task_info=task_info) + task_info=task_info, + ) LOG.debug( "Successfully started task with ID '%s' (type '%s') " - "for execution '%s'", task.id, task.task_type, - execution.id) + "for execution '%s'", + task.id, + task.task_type, + execution.id, + ) started_tasks.append(task.id) return constants.TASK_STATUS_PENDING except Exception: LOG.warn( "Error occured while starting new task '%s'. " "Cancelling execution '%s'. Error was: %s", - task.id, execution.id, utils.get_exception_details()) - self._cancel_tasks_execution( - ctxt, execution, requery=True) + task.id, + execution.id, + utils.get_exception_details(), + ) + self._cancel_tasks_execution(ctxt, execution, requery=True) raise # aggregate all tasks and statuses: @@ -2631,59 +3098,73 @@ def _start_task(task): LOG.debug( "All task statuses before execution '%s' lifecycle iteration " "(for tasks of instance '%s'): %s", - execution.id, instance, task_statuses) + execution.id, + instance, + task_statuses, + ) # NOTE: the tasks are saved in a random order in the DB, which # complicates the processing logic so we just pre-sort: for task in sorted(tasks_to_process, key=lambda t: t.index): - if task_statuses[task.id] == constants.TASK_STATUS_SCHEDULED: - # immediately start depency-less tasks (on-error or otherwise) if not task_deps[task.id]: - LOG.info( - "Starting depency-less task '%s'", task.id) + LOG.info("Starting depency-less task '%s'", task.id) task_statuses[task.id] = _start_task(task) continue parent_task_statuses = { - dep_id: task_statuses[dep_id] - for dep_id in task_deps[task.id]} + dep_id: task_statuses[dep_id] for dep_id in task_deps[task.id] + } # immediately unschedule tasks (on-error or otherwise) # if all of their parent tasks got un-scheduled: - if task_deps[task.id] and all([ + if task_deps[task.id] and all( + [ dep_stat == constants.TASK_STATUS_UNSCHEDULED - for dep_stat in parent_task_statuses.values()]): + for dep_stat in parent_task_statuses.values() + ] + ): LOG.info( "Unscheduling task '%s' as all parent " "tasks got unscheduled: %s", - task.id, parent_task_statuses) + task.id, + parent_task_statuses, + ) db_api.set_task_status( - ctxt, task.id, constants.TASK_STATUS_UNSCHEDULED, + ctxt, + task.id, + constants.TASK_STATUS_UNSCHEDULED, exception_details=( - "Unscheduled due to the unscheduling of all " - "parent tasks.")) + "Unscheduled due to the unscheduling of all parent tasks." + ), + ) task_statuses[task.id] = constants.TASK_STATUS_UNSCHEDULED continue # check all parents have finalized: - if all([ + if all( + [ dep_stat in constants.FINALIZED_TASK_STATUSES - for dep_stat in parent_task_statuses.values()]): - + for dep_stat in parent_task_statuses.values() + ] + ): # handle non-error tasks: if task.id not in on_error_tasks: # start non-error tasks whose parents have # all completed successfully: - if all([ + if all( + [ dep_stat == constants.TASK_STATUS_COMPLETED - for dep_stat in ( - parent_task_statuses.values())]): + for dep_stat in (parent_task_statuses.values()) + ] + ): LOG.info( "Starting task '%s' as all dependencies have " "completed successfully: %s", - task.id, parent_task_statuses) + task.id, + parent_task_statuses, + ) task_statuses[task.id] = _start_task(task) else: # it means one/more parents error'd/unscheduled @@ -2691,133 +3172,166 @@ def _start_task(task): LOG.info( "Unscheduling plain task '%s' as not all " "parent tasks completed successfully: %s", - task.id, parent_task_statuses) + task.id, + parent_task_statuses, + ) db_api.set_task_status( - ctxt, task.id, + ctxt, + task.id, constants.TASK_STATUS_UNSCHEDULED, exception_details=( "Unscheduled due to some parent tasks not " - "having completed successfully.")) - task_statuses[task.id] = ( - constants.TASK_STATUS_UNSCHEDULED) + "having completed successfully." + ), + ) + task_statuses[task.id] = constants.TASK_STATUS_UNSCHEDULED # handle on-error tasks: else: non_error_parents = { dep_id: task_statuses[dep_id] for dep_id in parent_task_statuses.keys() - if dep_id not in on_error_tasks} + if dep_id not in on_error_tasks + } # if there are no non-error parents whatsoever, it # means that one or more of the parent on-error tasks # [should] have a parent non-error task which failed: if not non_error_parents and ( - constants.TASK_STATUS_COMPLETED in ( - parent_task_statuses.values())): + constants.TASK_STATUS_COMPLETED + in (parent_task_statuses.values()) + ): LOG.info( "Starting on-error task '%s' as all parent " "tasks have been finalized and there are " "no non-error parents to directly depend on, " "but one or more on-error tasks have completed" - " successfully: %s", task.id, - parent_task_statuses) + " successfully: %s", + task.id, + parent_task_statuses, + ) task_statuses[task.id] = _start_task(task) # start on-error tasks only if at least one non-error # parent task has completed successfully: elif constants.TASK_STATUS_COMPLETED in ( - non_error_parents.values()): + non_error_parents.values() + ): LOG.info( "Starting on-error task '%s' as all parent " "tasks have been finalized and at least one " "non-error parent (%s) was completed: %s", - task.id, list(non_error_parents.keys()), - parent_task_statuses) + task.id, + list(non_error_parents.keys()), + parent_task_statuses, + ) task_statuses[task.id] = _start_task(task) else: LOG.info( "Unscheduling on-error task '%s' as none of " "its parent non-error tasks (%s) have " "completed successfully: %s", - task.id, list(non_error_parents.keys()), - parent_task_statuses) + task.id, + list(non_error_parents.keys()), + parent_task_statuses, + ) db_api.set_task_status( - ctxt, task.id, + ctxt, + task.id, constants.TASK_STATUS_UNSCHEDULED, exception_details=( "Unscheduled due to no non-error parent " - "tasks having completed successfully.")) - task_statuses[task.id] = ( - constants.TASK_STATUS_UNSCHEDULED) + "tasks having completed successfully." + ), + ) + task_statuses[task.id] = constants.TASK_STATUS_UNSCHEDULED else: LOG.debug( "No lifecycle decision was taken with respect to task " "%s of execution %s as not all parent tasks have " "reached a terminal state: %s", - task.id, execution.id, parent_task_statuses) + task.id, + execution.id, + parent_task_statuses, + ) else: LOG.debug( "No lifecycle decision to make for task '%s' of execution " "'%s' as it is not in a position to be scheduled: %s", - task.id, execution.id, task_statuses[task.id]) + task.id, + execution.id, + task_statuses[task.id], + ) if started_tasks: LOG.debug( "Started the following tasks for execution '%s': %s", - execution.id, started_tasks) + execution.id, + started_tasks, + ) else: # check for deadlock: if self._check_clean_execution_deadlock( - ctxt, execution, task_statuses=task_statuses) == ( - constants.EXECUTION_STATUS_DEADLOCKED): + ctxt, execution, task_statuses=task_statuses + ) == (constants.EXECUTION_STATUS_DEADLOCKED): LOG.error( "Execution '%s' deadlocked after Transfer state " "advancement. Cleanup has been performed. " - "Returning early.", execution.id) + "Returning early.", + execution.id, + ) return [] - LOG.debug( - "No new tasks were started for execution '%s'", execution.id) + LOG.debug("No new tasks were started for execution '%s'", execution.id) # check if execution status has changed: latest_execution_status = self._get_execution_status( - ctxt, execution, requery=True) + ctxt, execution, requery=True + ) if latest_execution_status != execution.status: LOG.info( "Execution '%s' transitioned from status %s to %s " "following the updated task statuses: %s", - execution.id, execution.status, - latest_execution_status, task_statuses) - self._set_tasks_execution_status( - ctxt, execution, latest_execution_status) + execution.id, + execution.status, + latest_execution_status, + task_statuses, + ) + self._set_tasks_execution_status(ctxt, execution, latest_execution_status) else: LOG.debug( "Execution '%s' has remained in status '%s' following " "state advancement. task statuses are: %s", - execution.id, latest_execution_status, task_statuses) + execution.id, + latest_execution_status, + task_statuses, + ) return started_tasks @staticmethod - def _update_transfer_volumes_info(ctxt, transfer_id, instance, - updated_task_info): - """ WARN: the lock for the Transfer must be pre-acquired. """ + def _update_transfer_volumes_info(ctxt, transfer_id, instance, updated_task_info): + """WARN: the lock for the Transfer must be pre-acquired.""" db_api.update_transfer_action_info_for_instance( - ctxt, transfer_id, instance, - updated_task_info) + ctxt, transfer_id, instance, updated_task_info + ) def _update_volumes_info_for_deployment_parent_transfer( - self, ctxt, deployment_id, instance, updated_task_info): + self, ctxt, deployment_id, instance, updated_task_info + ): deployment = db_api.get_deployment(ctxt, deployment_id) transfer_id = deployment.transfer_id with lockutils.lock( - constants.TRANSFER_LOCK_NAME_FORMAT % transfer_id, - external=True): + constants.TRANSFER_LOCK_NAME_FORMAT % transfer_id, external=True + ): LOG.debug( "Updating volume_info in transfer due to snapshot " - "restore during deployment. transfer id: %s", transfer_id) + "restore during deployment. transfer id: %s", + transfer_id, + ) self._update_transfer_volumes_info( - ctxt, transfer_id, instance, updated_task_info) + ctxt, transfer_id, instance, updated_task_info + ) def _handle_post_task_actions(self, ctxt, task, execution, task_info): task_type = task.task_type @@ -2833,7 +3347,6 @@ def _check_other_tasks_running(execution, current_task): return still_running if task_type == constants.TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS: - # When restoring a snapshot in some import providers (OpenStack), # a new volume_id is generated. This needs to be updated in the # Transfer instance as well. @@ -2842,21 +3355,28 @@ def _check_other_tasks_running(execution, current_task): LOG.warn( "No volumes_info was provided by task '%s' (type '%s') " "after completion. NOT updating parent action '%s'", - task.id, task_type, execution.action_id) + task.id, + task_type, + execution.action_id, + ) else: LOG.debug( "Updating volumes_info for instance '%s' in parent action " "'%s' following completion of task '%s' (type '%s'): %s", - task.instance, execution.action_id, task.id, task_type, - utils.sanitize_task_info( - {'volumes_info': volumes_info})) + task.instance, + execution.action_id, + task.id, + task_type, + utils.sanitize_task_info({'volumes_info': volumes_info}), + ) self._update_volumes_info_for_deployment_parent_transfer( - ctxt, execution.action_id, task.instance, - {"volumes_info": volumes_info}) - - elif task_type == ( - constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS): + ctxt, + execution.action_id, + task.instance, + {"volumes_info": volumes_info}, + ) + elif task_type == (constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS): if not task_info.get("clone_disks"): # The deployment completed. If the transfer is executed again, # new volumes need to be created in place of the deployed @@ -2865,11 +3385,14 @@ def _check_other_tasks_running(execution, current_task): "Unsetting 'volumes_info' for instance '%s' in Transfer " "'%s' after completion of Transfer task '%s' " "(type '%s') with clone_disks=False.", - task.instance, execution.action_id, task.id, - task_type) + task.instance, + execution.action_id, + task.id, + task_type, + ) self._update_volumes_info_for_deployment_parent_transfer( - ctxt, execution.action_id, task.instance, - {"volumes_info": []}) + ctxt, execution.action_id, task.instance, {"volumes_info": []} + ) elif task_type == constants.TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT: # set 'transfer_result' in the 'base_transfer_action' @@ -2878,35 +3401,46 @@ def _check_other_tasks_running(execution, current_task): transfer_result = task_info.get("transfer_result") try: schemas.validate_value( - transfer_result, - schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + transfer_result, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA + ) LOG.debug( "Setting result for transfer action '%s': %s", - execution.action_id, transfer_result) + execution.action_id, + transfer_result, + ) db_api.set_transfer_action_result( - ctxt, execution.action_id, task.instance, - transfer_result) + ctxt, execution.action_id, task.instance, transfer_result + ) except exception.SchemaValidationException: LOG.warn( "Could not validate transfer result '%s' against the " "VM export info schema. NOT saving value in Database. " "Exception details: %s", - transfer_result, utils.get_exception_details()) + transfer_result, + utils.get_exception_details(), + ) else: LOG.debug( "No 'transfer_result' was returned for task type '%s' " - "for transfer action '%s'", task_type, execution.action_id) + "for transfer action '%s'", + task_type, + execution.action_id, + ) elif task_type in ( - constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, - constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER): + constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, + constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER, + ): # NOTE: remember to update the `volumes_info`: # NOTE: considering this method is only called with a lock on the # `execution.action_id` (in a Transfer update tasks' case that's # the ID of the Transfer itself) we can safely call # `_update_transfer_volumes_info` below: self._update_transfer_volumes_info( - ctxt, execution.action_id, task.instance, - {"volumes_info": task_info.get("volumes_info", [])}) + ctxt, + execution.action_id, + task.instance, + {"volumes_info": task_info.get("volumes_info", [])}, + ) if task_type == constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER: # check if this was the last task in the update execution: @@ -2919,106 +3453,131 @@ def _check_other_tasks_running(execution, current_task): "All tasks of the '%s' Transfer update procedure have " "completed successfully. Setting the updated " "parameter values on the parent Transfer itself.", - execution.action_id) + execution.action_id, + ) # NOTE: considering all the instances of the Transfer get # the same params, it doesn't matter which instance's # update task finishes last: - db_api.update_transfer( - ctxt, execution.action_id, task_info) + db_api.update_transfer(ctxt, execution.action_id, task_info) elif task_type in ( - constants.TASK_TYPE_ATTACH_VOLUMES_TO_SOURCE_MINION, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION): - + constants.TASK_TYPE_ATTACH_VOLUMES_TO_SOURCE_MINION, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION, + ): updated_values = { - "provider_properties": task_info[ - "origin_minion_provider_properties"]} + "provider_properties": task_info["origin_minion_provider_properties"] + } LOG.debug( "Updating minion provider properties of minion machine '%s' " "following the completion of task '%s' (type '%s') to %s", task_info['origin_minion_machine_id'], - task.id, task_type, updated_values) + task.id, + task_type, + updated_values, + ) db_api.update_minion_machine( - ctxt, task_info['origin_minion_machine_id'], updated_values) + ctxt, task_info['origin_minion_machine_id'], updated_values + ) elif task_type in ( - constants.TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_DESTINATION_MINION): - + constants.TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_DESTINATION_MINION, + ): updated_values = { "provider_properties": task_info[ - "destination_minion_provider_properties"]} + "destination_minion_provider_properties" + ] + } LOG.debug( "Updating minion provider properties of minion machine '%s' " "following the completion of task '%s' (type '%s') to %s", task_info['destination_minion_machine_id'], - task.id, task_type, updated_values) + task.id, + task_type, + updated_values, + ) db_api.update_minion_machine( - ctxt, task_info['destination_minion_machine_id'], - updated_values) + ctxt, task_info['destination_minion_machine_id'], updated_values + ) elif task_type in ( - constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION): - + constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION, + ): updated_values = { "provider_properties": task_info[ - "osmorphing_minion_provider_properties"]} + "osmorphing_minion_provider_properties" + ] + } LOG.debug( "Updating minion provider properties of minion machine '%s' " "following the completion of task '%s' (type '%s') to %s", task_info['osmorphing_minion_machine_id'], - task.id, task_type, updated_values) + task.id, + task_type, + updated_values, + ) db_api.update_minion_machine( - ctxt, task_info['osmorphing_minion_machine_id'], - updated_values) + ctxt, task_info['osmorphing_minion_machine_id'], updated_values + ) elif task_type == constants.TASK_TYPE_RELEASE_SOURCE_MINION: - LOG.debug( "Releasing source minion '%s' following the completion of " "task with ID '%s' (type '%s')", task_info['origin_minion_machine_id'], - task.id, task_type) + task.id, + task_type, + ) self._minion_manager_client.deallocate_minion_machine( - ctxt, task_info['origin_minion_machine_id']) + ctxt, task_info['origin_minion_machine_id'] + ) elif task_type == constants.TASK_TYPE_RELEASE_DESTINATION_MINION: - if task_info['destination_minion_machine_id'] != task_info.get( - "osmorphing_minion_machine_id"): + "osmorphing_minion_machine_id" + ): LOG.debug( "Releasing destination minion '%s' following the " "completion of task with ID '%s' (type '%s')", task_info['destination_minion_machine_id'], - task.id, task_type) + task.id, + task_type, + ) self._minion_manager_client.deallocate_minion_machine( - ctxt, task_info['destination_minion_machine_id']) + ctxt, task_info['destination_minion_machine_id'] + ) else: LOG.debug( "NOT releasing destination minion with ID '%s' following " "the completion of task with ID '%s' (type '%s') as it " "also to be used as the OSMorphing minion.", task_info['destination_minion_machine_id'], - task.id, task_type) + task.id, + task_type, + ) elif task_type == constants.TASK_TYPE_RELEASE_OSMORPHING_MINION: - LOG.debug( "Releasing OSMorphing minion '%s' following the completion of " "task with ID '%s' (type '%s')", task_info['osmorphing_minion_machine_id'], - task.id, task_type) + task.id, + task_type, + ) self._minion_manager_client.deallocate_minion_machine( - ctxt, task_info['osmorphing_minion_machine_id']) + ctxt, task_info['osmorphing_minion_machine_id'] + ) else: LOG.debug( "No post-task actions required for task '%s' of type '%s'", - task.id, task_type) + task.id, + task_type, + ) @parent_tasks_execution_synchronized def task_completed(self, ctxt, task_id, task_result): @@ -3030,18 +3589,26 @@ def task_completed(self, ctxt, task_id, task_result): LOG.warn( "Non-error task '%s' was marked as %s although it should " "not have. It was run to completion anyway.", - task.id, task.status) + task.id, + task.status, + ) LOG.info( "On-error task '%s' which was '%s' has just completed " "successfully. Marking it as '%s' as a final status, " "but processing its result as if it completed successfully.", - task_id, task.status, - constants.TASK_STATUS_CANCELED_AFTER_COMPLETION) + task_id, + task.status, + constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, + ) db_api.set_task_status( - ctxt, task_id, constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, + ctxt, + task_id, + constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, exception_details=( "This is a cleanup task so it was allowed to run to " - "completion after user-cancellation.")) + "completion after user-cancellation." + ), + ) elif task.status == constants.TASK_STATUS_CANCELLING: LOG.error( "Received confirmation that presumably cancelling task '%s' " @@ -3052,34 +3619,44 @@ def task_completed(self, ctxt, task_id, task_result): "Please check the worker logs for more details. " "Marking as %s and processing its result as if it completed " "successfully.", - task.id, task.status, task.host, - constants.TASK_STATUS_CANCELED_AFTER_COMPLETION) + task.id, + task.status, + task.host, + constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, + ) db_api.set_task_status( - ctxt, task_id, + ctxt, + task_id, constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, exception_details=( "The worker host for this task ('%s') has either failed " "at cancelling it or the cancellation request arrived " "after it was already completed so this task was run to " "completion. Please review the worker logs for " - "more relevant details." % ( - task.host))) + "more relevant details." % (task.host) + ), + ) elif task.status == constants.TASK_STATUS_FAILED_TO_CANCEL: LOG.error( "Received confirmation '%s' task '%s' has presumably just " "completed successfully. Marking as '%s' and processing its " "result as if it had completed normally.", - task.status, task.id, - constants.TASK_STATUS_CANCELED_AFTER_COMPLETION) + task.status, + task.id, + constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, + ) db_api.set_task_status( - ctxt, task_id, constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, + ctxt, + task_id, + constants.TASK_STATUS_CANCELED_AFTER_COMPLETION, exception_details=( "The worker host for this task ('%s') had not either not " "accepted task cancellation request when it was asked to " "or had failed to receive the request, so this task was " "run to completion. Please review the worker logs for " - "more relevant details." % ( - task.host))) + "more relevant details." % (task.host) + ), + ) elif task.status in constants.FINALIZED_TASK_STATUSES: LOG.error( "Received confirmation that presumably finalized task '%s' " @@ -3088,70 +3665,82 @@ def task_completed(self, ctxt, task_id, task_result): "that there is an inconsistency with the task scheduling. " "Check the rest of the logs for further details. " "The results of this task will NOT be processed.", - task.id, task.status, task.host) + task.id, + task.status, + task.host, + ) return else: if task.status != constants.TASK_STATUS_RUNNING: LOG.warn( "Just-completed task '%s' was in '%s' state instead of " "the expected '%s' state. Marking as '%s' anyway.", - task_id, task.status, constants.TASK_STATUS_RUNNING, - constants.TASK_STATUS_COMPLETED) - db_api.set_task_status( - ctxt, task_id, constants.TASK_STATUS_COMPLETED) + task_id, + task.status, + constants.TASK_STATUS_RUNNING, + constants.TASK_STATUS_COMPLETED, + ) + db_api.set_task_status(ctxt, task_id, constants.TASK_STATUS_COMPLETED) execution = db_api.get_tasks_execution(ctxt, task.execution_id) with lockutils.lock( - constants.EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP[ - execution.type] % execution.action_id, - external=True): + constants.EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP[execution.type] + % execution.action_id, + external=True, + ): action_id = execution.action_id - action = db_api.get_action( - ctxt, action_id, include_task_info=True) + action = db_api.get_action(ctxt, action_id, include_task_info=True) updated_task_info = None if task_result: LOG.info( "Setting task %(task_id)s (type %(task_type)s)result for " "instance %(instance)s into action %(action_id)s info: " - "%(task_result)s", { + "%(task_result)s", + { "task_id": task_id, "instance": task.instance, "task_type": task.task_type, "action_id": action_id, - "task_result": utils.sanitize_task_info( - task_result)}) - updated_task_info = ( - db_api.update_transfer_action_info_for_instance( - ctxt, action_id, task.instance, task_result)) + "task_result": utils.sanitize_task_info(task_result), + }, + ) + updated_task_info = db_api.update_transfer_action_info_for_instance( + ctxt, action_id, task.instance, task_result + ) else: - action = db_api.get_action( - ctxt, action_id, include_task_info=True) + action = db_api.get_action(ctxt, action_id, include_task_info=True) updated_task_info = action.info[task.instance] LOG.info( "Task '%s' for instance '%s' of transfer action '%s' " "has completed successfuly but has not returned " - "any result.", task.id, task.instance, action_id) + "any result.", + task.id, + task.instance, + action_id, + ) # NOTE: refresh the execution just in case: execution = db_api.get_tasks_execution(ctxt, task.execution_id) - self._handle_post_task_actions( - ctxt, task, execution, updated_task_info) + self._handle_post_task_actions(ctxt, task, execution, updated_task_info) newly_started_tasks = self._advance_execution_state( - ctxt, execution, instance=task.instance, requery=False) + ctxt, execution, instance=task.instance, requery=False + ) if newly_started_tasks: LOG.info( "The following tasks were started for execution '%s' " "following the completion of task '%s' for instance %s: " - "%s" % ( - execution.id, task.id, task.instance, - newly_started_tasks)) + "%s" % (execution.id, task.id, task.instance, newly_started_tasks) + ) else: LOG.debug( "No new tasks started for execution '%s' for instance " "'%s' following the successful completion of task '%s'.", - execution.id, task.instance, task.id) + execution.id, + task.instance, + task.id, + ) def _cancel_execution_for_osmorphing_debugging(self, ctxt, execution): # go through all scheduled tasks and cancel them: @@ -3162,93 +3751,126 @@ def _cancel_execution_for_osmorphing_debugging(self, ctxt, execution): if subtask.status in constants.ACTIVE_TASK_STATUSES: raise exception.CoriolisException( "Task %s is still in an active state (%s) although " - "it should not!" % (subtask.id, subtask.status)) + "it should not!" % (subtask.id, subtask.status) + ) if subtask.status in [ - constants.TASK_STATUS_SCHEDULED, - constants.TASK_STATUS_ON_ERROR_ONLY]: + constants.TASK_STATUS_SCHEDULED, + constants.TASK_STATUS_ON_ERROR_ONLY, + ]: msg = ( "This task was unscheduled for debugging an error in the " - "OSMorphing process.") + "OSMorphing process." + ) if subtask.on_error: msg = ( "%s Please note that any cleanup operations this task " "should have included will need to performed manually " - "once the debugging process has been completed." % - (msg)) + "once the debugging process has been completed." % (msg) + ) db_api.set_task_status( - ctxt, subtask.id, + ctxt, + subtask.id, constants.TASK_STATUS_CANCELED_FOR_DEBUGGING, - exception_details=msg) + exception_details=msg, + ) @parent_tasks_execution_synchronized def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): LOG.info( "Received confirmation of cancellation for task '%s': %s", - task_id, cancellation_details) + task_id, + cancellation_details, + ) task = db_api.get_task(ctxt, task_id) final_status = constants.TASK_STATUS_CANCELED exception_details = ( "This task was user-cancelled. Additional cancellation " - "info from worker service: '%s'" % cancellation_details) + "info from worker service: '%s'" % cancellation_details + ) if task.status == constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION: LOG.error( "Received cancellation confirmation for task '%s' which was " "in '%s' state. This likely means that a double-cancellation " "occurred. Marking task as '%s' either way.", - task.id, task.status, final_status) + task.id, + task.status, + final_status, + ) elif task.status == constants.TASK_STATUS_FORCE_CANCELED: # it means a force cancel has been issued before the # confirmation that the task was canceled came in: LOG.warn( "Only just received cancellation confirmation for " "force-canceled task '%s'. Leaving marked as " - "force-cancelled.", task.id) + "force-cancelled.", + task.id, + ) final_status = constants.TASK_STATUS_FORCE_CANCELED exception_details = ( "This task was force-cancelled. Confirmation of its " "cancellation did eventually come in. Additional details on " - "the cancellation: %s" % cancellation_details) + "the cancellation: %s" % cancellation_details + ) elif task.status == constants.TASK_STATUS_FAILED_TO_CANCEL: final_status = constants.TASK_STATUS_CANCELED LOG.warn( "Only just received cancellation confirmation for task '%s' " "despite it being marked as '%s'. Marking as '%s' anyway. " "Error reported by worker was: %s", - task.id, task.status, final_status, exception_details) + task.id, + task.status, + final_status, + exception_details, + ) exception_details = ( "The worker service for this task (%s) had either failed to " "cancel this task when it was initially asked to or did not " "receive the cancellation request, but it did eventually " "confirm the task's cancellation with the following " - "details: %s" % (task.host, cancellation_details)) + "details: %s" % (task.host, cancellation_details) + ) elif task.status in constants.FINALIZED_TASK_STATUSES: LOG.warn( "Received confirmation of cancellation for already finalized " "task '%s' (status '%s') from host '%s'. NOT modifying " - "its status.", task.id, task.status, task.host) + "its status.", + task.id, + task.status, + task.host, + ) final_status = task.status elif task.status != constants.TASK_STATUS_CANCELLING: LOG.warn( "Received confirmation of cancellation for non-CANCELLING " "task '%s' (status '%s'). Marking as '%s' anyway.", - task.id, task.status, final_status) + task.id, + task.status, + final_status, + ) db_api.set_task_status( - ctxt, task.id, final_status, - exception_details=exception_details) + ctxt, task.id, final_status, exception_details=exception_details + ) if final_status == task.status: LOG.debug( "NOT altering state of finalized task '%s' ('%s') following " "confirmation of cancellation. Updating its exception " - "details though: %s", task.id, task.status, exception_details) + "details though: %s", + task.id, + task.status, + exception_details, + ) else: LOG.info( "Transitioning canceled task '%s' from '%s' to '%s' following " "confirmation of its cancellation.", - task.id, task.status, final_status) + task.id, + task.status, + final_status, + ) execution = db_api.get_tasks_execution(ctxt, task.execution_id) self._advance_execution_state(ctxt, execution, requery=False) @@ -3256,7 +3878,8 @@ def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): def set_task_error(self, ctxt, task_id, exception_details): LOG.error( "Received error confirmation for task: %(task_id)s - %(ex)s", - {"task_id": task_id, "ex": exception_details}) + {"task_id": task_id, "ex": exception_details}, + ) task = db_api.get_task(ctxt, task_id) @@ -3269,56 +3892,78 @@ def set_task_error(self, ctxt, task_id, exception_details): LOG.warn( "Non-error '%s' was in '%s' status although it should " "never have been marked as such. Marking as '%s' anyway.", - task.id, task.status, final_status) + task.id, + task.status, + final_status, + ) else: LOG.warn( "On-error task '%s' which was in '%s' status ended up " "error-ing. Marking as '%s'", - task.id, task.status, final_status) + task.id, + task.status, + final_status, + ) exception_details = ( "This is a cleanup task so was allowed to complete " "following user-cancellation, but encountered an " - "error: %s" % exception_details) + "error: %s" % exception_details + ) elif task.status == constants.TASK_STATUS_FORCE_CANCELED: # it means a force cancel has been issued priorly but # the task has error'd anyway: LOG.warn( "Only just received error confirmation for force-cancelled " - "task '%s'. Leaving marked as force-cancelled.", task.id) + "task '%s'. Leaving marked as force-cancelled.", + task.id, + ) final_status = constants.TASK_STATUS_FORCE_CANCELED exception_details = ( "This task was force-cancelled but ended up errorring anyway. " - "The error details were: '%s'" % exception_details) + "The error details were: '%s'" % exception_details + ) elif task.status == constants.TASK_STATUS_FAILED_TO_CANCEL: final_status = constants.TASK_STATUS_CANCELED LOG.warn( "Only just received error confirmation for task '%s' despite " "it being marked as '%s'. Marking as '%s' anyway. Error " "reported by worker was: %s", - task.id, task.status, final_status, exception_details) + task.id, + task.status, + final_status, + exception_details, + ) exception_details = ( "The worker service for this task (%s) has failed to cancel " "this task when it was asked to, and the task eventually " - "exited with an error. The error details were: %s" % ( - task.host, exception_details)) + "exited with an error. The error details were: %s" + % (task.host, exception_details) + ) elif task.status in constants.FINALIZED_TASK_STATUSES: LOG.error( "Error confirmation on task '%s' arrived despite it being " "in a terminal state ('%s'). This should never happen and " "indicates an issue with its scheduling/handling. Error " - "was: %s", task.id, task.status, exception_details) + "was: %s", + task.id, + task.status, + exception_details, + ) exception_details = ( "Error confirmation came in for this task despite it having " "already been marked as %s. Please notify support of this " "occurence and share the Conductor and Worker logs. " - "Error message in confirmation was: %s" % ( - task.status, exception_details)) + "Error message in confirmation was: %s" + % (task.status, exception_details) + ) LOG.debug( "Transitioning errored task '%s' from '%s' to '%s'", - task.id, task.status, final_status) - db_api.set_task_status( - ctxt, task_id, final_status, exception_details) + task.id, + task.status, + final_status, + ) + db_api.set_task_status(ctxt, task_id, final_status, exception_details) task = db_api.get_task(ctxt, task_id) execution = db_api.get_tasks_execution(ctxt, task.execution_id) @@ -3326,45 +3971,61 @@ def set_task_error(self, ctxt, task_id, exception_details): action_id = execution.action_id action = db_api.get_action(ctxt, action_id, include_task_info=True) with lockutils.lock( - constants.EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP[ - execution.type] % action_id, - external=True): + constants.EXECUTION_TYPE_TO_ACTION_LOCK_NAME_FORMAT_MAP[execution.type] + % action_id, + external=True, + ): if task.task_type == constants.TASK_TYPE_OS_MORPHING and ( - CONF.conductor.debug_os_morphing_errors): + CONF.conductor.debug_os_morphing_errors + ): LOG.debug( "Attempting to cancel execution '%s' of action '%s' " - "for OSMorphing debugging.", execution.id, action_id) + "for OSMorphing debugging.", + execution.id, + action_id, + ) # NOTE: the OSMorphing task always runs by itself so no # further tasks should be running, but we double-check here: running = [ - t for t in execution.tasks + t + for t in execution.tasks if t.status in constants.ACTIVE_TASK_STATUSES - and t.task_type != constants.TASK_TYPE_OS_MORPHING] + and t.task_type != constants.TASK_TYPE_OS_MORPHING + ] if not running: - self._cancel_execution_for_osmorphing_debugging( - ctxt, execution) + self._cancel_execution_for_osmorphing_debugging(ctxt, execution) db_api.set_task_status( - ctxt, task_id, final_status, + ctxt, + task_id, + final_status, exception_details=( "An error has occured during OSMorphing. Cleanup " "will not be performed for debugging reasons. " "Please review the Conductor logs for the debug " - "connection info. Original error was: %s" % ( - exception_details))) + "connection info. Original error was: %s" + % (exception_details) + ), + ) LOG.warn( "All subtasks for Deployment '%s' have been cancelled " "to allow for OSMorphing debugging. The connection " "info for the worker VM is: %s", - action_id, action.info.get(task.instance, {}).get( - 'osmorphing_connection_info', {})) + action_id, + action.info.get(task.instance, {}).get( + 'osmorphing_connection_info', {} + ), + ) self._set_tasks_execution_status( - ctxt, execution, - constants.EXECUTION_STATUS_CANCELED_FOR_DEBUGGING) + ctxt, + execution, + constants.EXECUTION_STATUS_CANCELED_FOR_DEBUGGING, + ) else: LOG.warn( "Some tasks are running in parallel with the " "OSMorphing task, a debug setup cannot be safely " - "achieved. Proceeding with cleanup tasks as usual.") + "achieved. Proceeding with cleanup tasks as usual." + ) self._cancel_tasks_execution(ctxt, execution) else: self._cancel_tasks_execution(ctxt, execution) @@ -3377,13 +4038,15 @@ def add_task_event(self, ctxt, task_id, level, message): raise exception.InvalidTaskState( "Task with ID '%s' is in a non-running state ('%s') but it " "has received a task event from its task host ('%s'). " - "Refusing task event. The event string was: %s" % ( - task.id, task.status, task.host, message)) + "Refusing task event. The event string was: %s" + % (task.id, task.status, task.host, message) + ) db_api.add_task_event(ctxt, task_id, level, message) @task_synchronized def add_task_progress_update( - self, ctxt, task_id, message, initial_step=0, total_steps=0): + self, ctxt, task_id, message, initial_step=0, total_steps=0 + ): LOG.info("Adding task progress update: %s", task_id) task = db_api.get_task(ctxt, task_id) if task.status not in constants.ACTIVE_TASK_STATUSES: @@ -3391,36 +4054,59 @@ def add_task_progress_update( "Task with ID '%s' is in a non-running state ('%s') but it " "has received a progress update from its task host ('%s'). " "Refusing progress update. The progress update string " - "was: %s" % ( - task.id, task.status, task.host, message)) + "was: %s" % (task.id, task.status, task.host, message) + ) return db_api.add_task_progress_update( - ctxt, task_id, message, initial_step=initial_step, - total_steps=total_steps) + ctxt, task_id, message, initial_step=initial_step, total_steps=total_steps + ) @task_synchronized def update_task_progress_update( - self, ctxt, task_id, progress_update_index, - new_current_step, new_total_steps=None, new_message=None): + self, + ctxt, + task_id, + progress_update_index, + new_current_step, + new_total_steps=None, + new_message=None, + ): LOG.info( "Updating progress update with index '%s' for task %s: %s", - progress_update_index, task_id, new_current_step) + progress_update_index, + task_id, + new_current_step, + ) db_api.update_task_progress_update( - ctxt, task_id, progress_update_index, new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + ctxt, + task_id, + progress_update_index, + new_current_step, + new_total_steps=new_total_steps, + new_message=new_message, + ) @staticmethod def _get_transfer_schedule(ctxt, transfer_id, schedule_id, expired=True): schedule = db_api.get_transfer_schedule( - ctxt, transfer_id, schedule_id, expired=expired) + ctxt, transfer_id, schedule_id, expired=expired + ) if not schedule: raise exception.NotFound( - "Schedule with ID '%s' for Transfer '%s' not found." % ( - schedule_id, transfer_id)) + "Schedule with ID '%s' for Transfer '%s' not found." + % (schedule_id, transfer_id) + ) return schedule - def create_transfer_schedule(self, ctxt, transfer_id, - schedule, enabled, exp_date, - shutdown_instance, auto_deploy): + def create_transfer_schedule( + self, + ctxt, + transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ): keystone.create_trust(ctxt) transfer = self._get_transfer(ctxt, transfer_id) transfer_schedule = models.TransferSchedule() @@ -3435,71 +4121,82 @@ def create_transfer_schedule(self, ctxt, transfer_id, transfer_schedule.trust_id = ctxt.trust_id db_api.add_transfer_schedule( - ctxt, transfer_schedule, - lambda ctxt, sched: self._transfer_cron_client.register( - ctxt, sched)) - return self.get_transfer_schedule( - ctxt, transfer_id, transfer_schedule.id) + ctxt, + transfer_schedule, + lambda ctxt, sched: self._transfer_cron_client.register(ctxt, sched), + ) + return self.get_transfer_schedule(ctxt, transfer_id, transfer_schedule.id) @schedule_synchronized - def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, - updated_values): + def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_values): db_api.update_transfer_schedule( - ctxt, transfer_id, schedule_id, updated_values, None, - lambda ctxt, sched: self._transfer_cron_client.register( - ctxt, sched)) + ctxt, + transfer_id, + schedule_id, + updated_values, + None, + lambda ctxt, sched: self._transfer_cron_client.register(ctxt, sched), + ) return self._get_transfer_schedule(ctxt, transfer_id, schedule_id) def _cleanup_schedule_resources(self, ctxt, schedule): self._transfer_cron_client.unregister(ctxt, schedule) if schedule.trust_id: - tmp_trust = context.get_admin_context( - trust_id=schedule.trust_id) + tmp_trust = context.get_admin_context(trust_id=schedule.trust_id) try: keystone.delete_trust(tmp_trust) except Exception: LOG.warning( f"Failed to delete schedule's trust_id. Trust might " f"already be deleted. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) @schedule_synchronized def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): transfer = self._get_transfer(ctxt, transfer_id) transfer_status = transfer.last_execution_status - valid_statuses = list(itertools.chain( - constants.FINALIZED_EXECUTION_STATUSES, - [constants.EXECUTION_STATUS_UNEXECUTED])) + valid_statuses = list( + itertools.chain( + constants.FINALIZED_EXECUTION_STATUSES, + [constants.EXECUTION_STATUS_UNEXECUTED], + ) + ) if transfer_status not in valid_statuses: raise exception.InvalidTransferState( 'Transfer Schedule cannot be deleted while the Transfer is in ' - '%s state. Please wait for the Transfer execution to finish' % - (transfer_status)) + '%s state. Please wait for the Transfer execution to finish' + % (transfer_status) + ) db_api.delete_transfer_schedule( - ctxt, transfer_id, schedule_id, None, - lambda ctxt, sched: self._cleanup_schedule_resources( - ctxt, sched)) + ctxt, + transfer_id, + schedule_id, + None, + lambda ctxt, sched: self._cleanup_schedule_resources(ctxt, sched), + ) @transfer_synchronized def get_transfer_schedules(self, ctxt, transfer_id=None, expired=True): return db_api.get_transfer_schedules( - ctxt, transfer_id=transfer_id, expired=expired) + ctxt, transfer_id=transfer_id, expired=expired + ) @schedule_synchronized - def get_transfer_schedule(self, ctxt, transfer_id, - schedule_id, expired=True): + def get_transfer_schedule(self, ctxt, transfer_id, schedule_id, expired=True): return self._get_transfer_schedule( - ctxt, transfer_id, schedule_id, expired=expired) + ctxt, transfer_id, schedule_id, expired=expired + ) @transfer_synchronized - def update_transfer( - self, ctxt, transfer_id, updated_properties): - transfer = self._get_transfer( - ctxt, transfer_id, include_task_info=True) + def update_transfer(self, ctxt, transfer_id, updated_properties): + transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) minion_pool_fields = [ - "origin_minion_pool_id", "destination_minion_pool_id", - "instance_osmorphing_minion_pool_mappings"] + "origin_minion_pool_id", + "destination_minion_pool_id", + "instance_osmorphing_minion_pool_mappings", + ] if any([mpf in updated_properties for mpf in minion_pool_fields]): # NOTE: this is just a dummy Transfer model to use for validation: dummy = models.Transfer() @@ -3508,12 +4205,14 @@ def update_transfer( dummy.origin_endpoint_id = transfer.origin_endpoint_id dummy.destination_endpoint_id = transfer.destination_endpoint_id dummy.origin_minion_pool_id = updated_properties.get( - 'origin_minion_pool_id') + 'origin_minion_pool_id' + ) dummy.destination_minion_pool_id = updated_properties.get( - 'destination_minion_pool_id') - dummy.instance_osmorphing_minion_pool_mappings = ( - updated_properties.get( - 'instance_osmorphing_minion_pool_mappings')) + 'destination_minion_pool_id' + ) + dummy.instance_osmorphing_minion_pool_mappings = updated_properties.get( + 'instance_osmorphing_minion_pool_mappings' + ) self._check_minion_pools_for_action(ctxt, dummy) self._check_transfer_running_executions(ctxt, transfer) @@ -3528,10 +4227,11 @@ def update_transfer( for instance in transfer.instances: LOG.debug( - "Pre-transfer-update task_info for instance '%s' of Transfer " - "'%s': %s", instance, transfer_id, - utils.sanitize_task_info( - transfer.info[instance])) + "Pre-transfer-update task_info for instance '%s' of Transfer '%s': %s", + instance, + transfer_id, + utils.sanitize_task_info(transfer.info[instance]), + ) # NOTE: "circular assignment" would lead to a `None` value # so we must operate on a copy: @@ -3545,67 +4245,76 @@ def update_transfer( # The actual values on the Transfer object itself will be set # during _handle_post_task_actions once the final destination-side # update task will be completed. - inst_info_copy.update({ - key: updated_properties[key] - for key in updated_properties - if key != "destination_environment"}) + inst_info_copy.update( + { + key: updated_properties[key] + for key in updated_properties + if key != "destination_environment" + } + ) # NOTE: the API service labels the target-env as the # "destination_environment": if "destination_environment" in updated_properties: inst_info_copy["target_environment"] = updated_properties[ - "destination_environment"] + "destination_environment" + ] transfer.info[instance] = inst_info_copy LOG.debug( "Updated task_info for instance '%s' of Transfer " "'%s' which will be verified during update procedure: %s", - instance, transfer_id, utils.sanitize_task_info( - transfer.info[instance])) + instance, + transfer_id, + utils.sanitize_task_info(transfer.info[instance]), + ) get_instance_info_task = self._create_task( - instance, constants.TASK_TYPE_GET_INSTANCE_INFO, - execution) + instance, constants.TASK_TYPE_GET_INSTANCE_INFO, execution + ) update_source_transfer_task = self._create_task( - instance, constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, - execution) + instance, constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, execution + ) self._create_task( - instance, constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER, + instance, + constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER, execution, depends_on=[ get_instance_info_task.id, # NOTE: the dest-side update task must be done after # the source-side one as both can potentially modify # the 'volumes_info' together: - update_source_transfer_task.id]) + update_source_transfer_task.id, + ], + ) self._check_execution_tasks_sanity(execution, transfer.info) # update the action info for all of the instances in the Transfer: for instance in execution.action.instances: db_api.update_transfer_action_info_for_instance( - ctxt, transfer.id, instance, transfer.info[instance]) + ctxt, transfer.id, instance, transfer.info[instance] + ) db_api.add_transfer_tasks_execution(ctxt, execution) - LOG.debug("Execution for Transfer update tasks created: %s", - execution.id) + LOG.debug("Execution for Transfer update tasks created: %s", execution.id) self._begin_tasks(ctxt, transfer, execution) - return self.get_transfer_tasks_execution( - ctxt, transfer_id, execution.id) + return self.get_transfer_tasks_execution(ctxt, transfer_id, execution.id) def get_diagnostics(self, ctxt): diagnostics = utils.get_diagnostics_info() if self._licensing_client: diagnostics['licensing_status'] = ( - self._licensing_client.get_licence_status()) + self._licensing_client.get_licence_status() + ) diagnostics['licences'] = self._licensing_client.get_licences() - diagnostics['reservations'] = ( - self._licensing_client.get_reservations()) + diagnostics['reservations'] = self._licensing_client.get_reservations() else: LOG.debug( "Licensing client not instantiated. Cannot add licensing " - "information to diagnostics.") + "information to diagnostics." + ) return diagnostics def create_region(self, ctxt, region_name, description="", enabled=True): @@ -3624,15 +4333,16 @@ def get_regions(self, ctxt): def get_region(self, ctxt, region_id): region = db_api.get_region(ctxt, region_id) if not region: - raise exception.NotFound( - "Region with ID '%s' not found." % region_id) + raise exception.NotFound("Region with ID '%s' not found." % region_id) return region @region_synchronized def update_region(self, ctxt, region_id, updated_values): LOG.info( "Attempting to update region '%s' with payload: %s", - region_id, updated_values) + region_id, + updated_values, + ) db_api.update_region(ctxt, region_id, updated_values) LOG.info("Region '%s' successfully updated", region_id) return db_api.get_region(ctxt, region_id) @@ -3644,14 +4354,23 @@ def delete_region(self, ctxt, region_id): db_api.delete_region(ctxt, region_id) def register_service( - self, ctxt, host, binary, topic, enabled, mapped_regions=None, - providers=None, specs=None): + self, + ctxt, + host, + binary, + topic, + enabled, + mapped_regions=None, + providers=None, + specs=None, + ): service = db_api.find_service(ctxt, host, binary, topic=topic) if service: raise exception.Conflict( "A Service with the specified parameters (host %s, binary %s, " - "topic %s) has already been registered under ID: %s" % ( - host, binary, topic, service.id)) + "topic %s) has already been registered under ID: %s" + % (host, binary, topic, service.id) + ) service = models.Service() service.id = str(uuid.uuid4()) @@ -3662,8 +4381,7 @@ def register_service( service.status = constants.SERVICE_STATUS_UP if None in (providers, specs): - worker_rpc = ( - rpc_worker_client.WorkerClient(host=service.host)) + worker_rpc = rpc_worker_client.WorkerClient(host=service.host) status = worker_rpc.get_service_status(ctxt) service.providers = status["providers"] @@ -3674,21 +4392,22 @@ def register_service( # create the service: db_api.add_service(ctxt, service) - LOG.debug( - "Added new service to DB: %s", service.id) + LOG.debug("Added new service to DB: %s", service.id) # add region associations: if mapped_regions: try: db_api.update_service( - ctxt, service.id, { - "mapped_regions": mapped_regions}) + ctxt, service.id, {"mapped_regions": mapped_regions} + ) except Exception: LOG.warn( "Error adding region mappings during new service " "registration (host: %s), cleaning up endpoint and " "all created mappings for regions: %s", - service.host, mapped_regions) + service.host, + mapped_regions, + ) db_api.delete_service(ctxt, service.id) raise @@ -3696,16 +4415,14 @@ def register_service( def check_service_registered(self, ctxt, host, binary, topic): props = "host='%s', binary='%s', topic='%s'" % (host, binary, topic) - LOG.debug( - "Checking for existence of service with properties: %s", props) + LOG.debug("Checking for existence of service with properties: %s", props) service = db_api.find_service(ctxt, host, binary, topic=topic) if service: - LOG.debug( - "Found service '%s' for properties %s", service.id, props) + LOG.debug("Found service '%s' for properties %s", service.id, props) else: LOG.debug( - "Could not find any service with the specified " - "properties: %s", props) + "Could not find any service with the specified properties: %s", props + ) return service @service_synchronized @@ -3717,7 +4434,8 @@ def refresh_service_status(self, ctxt, service_id): updated_values = { "providers": status["providers"], "specs": status["specs"], - "status": constants.SERVICE_STATUS_UP} + "status": constants.SERVICE_STATUS_UP, + } db_api.update_service(ctxt, service_id, updated_values) LOG.debug("Successfully refreshed status of service '%s'", service_id) return db_api.get_service(ctxt, service_id) @@ -3729,15 +4447,16 @@ def get_services(self, ctxt): def get_service(self, ctxt, service_id): service = db_api.get_service(ctxt, service_id) if not service: - raise exception.NotFound( - "Service with ID '%s' not found." % service_id) + raise exception.NotFound("Service with ID '%s' not found." % service_id) return service @service_synchronized def update_service(self, ctxt, service_id, updated_values): LOG.info( "Attempting to update service '%s' with payload: %s", - service_id, updated_values) + service_id, + updated_values, + ) db_api.update_service(ctxt, service_id, updated_values) LOG.info("Successfully updated service '%s'", service_id) return db_api.get_service(ctxt, service_id) diff --git a/coriolis/conductor/rpc/utils.py b/coriolis/conductor/rpc/utils.py index c4c622f8..52e121f3 100644 --- a/coriolis/conductor/rpc/utils.py +++ b/coriolis/conductor/rpc/utils.py @@ -7,14 +7,22 @@ from coriolis import utils - LOG = logging.getLogger(__name__) def check_create_registration_for_service( - conductor_rpc, request_context, host, binary, topic, enabled=False, - mapped_regions=None, providers=None, specs=None, retry_period=30): - """ Checks with the conductor whether or not a service has already been + conductor_rpc, + request_context, + host, + binary, + topic, + enabled=False, + mapped_regions=None, + providers=None, + specs=None, + retry_period=30, +): + """Checks with the conductor whether or not a service has already been registered for this host and topic and creates one if not. If the service is already registered, directs the conductor to refresh the service status. @@ -25,30 +33,45 @@ def check_create_registration_for_service( # check is service already exists: LOG.info( "Checking with conductor if service with following porperties " - "was already registered: %s", props) + "was already registered: %s", + props, + ) worker_service = conductor_rpc.check_service_registered( - request_context, host, binary, topic) + request_context, host, binary, topic + ) if worker_service: LOG.info( "A service with properties %s has already been registered " "under ID '%s'. Updating existing registration.", - props, worker_service['id']) + props, + worker_service['id'], + ) worker_service = conductor_rpc.update_service( - request_context, worker_service['id'], updated_values={ - "providers": providers, - "specs": specs}) + request_context, + worker_service['id'], + updated_values={"providers": providers, "specs": specs}, + ) else: LOG.debug( - "Attempting to register new service with properties: %s", - props) + "Attempting to register new service with properties: %s", props + ) worker_service = conductor_rpc.register_service( - request_context, host, binary, topic, enabled, - mapped_regions=mapped_regions, providers=providers, - specs=specs) + request_context, + host, + binary, + topic, + enabled, + mapped_regions=mapped_regions, + providers=providers, + specs=specs, + ) return worker_service except Exception: LOG.warn( "Failed to register service with specs %s. Retrying again in " - "%d seconds. Error was: %s", props, retry_period, - utils.get_exception_details()) + "%d seconds. Error was: %s", + props, + retry_period, + utils.get_exception_details(), + ) time.sleep(retry_period) diff --git a/coriolis/conf.py b/coriolis/conf.py index edd9eeed..e2dfbcf4 100644 --- a/coriolis/conf.py +++ b/coriolis/conf.py @@ -6,9 +6,11 @@ def init_common_opts(): opts = [ - cfg.IntOpt('default_requests_timeout', - default=60, - help='Number of seconds for HTTP request timeouts.'), + cfg.IntOpt( + 'default_requests_timeout', + default=60, + help='Number of seconds for HTTP request timeouts.', + ), ] cfg.CONF.register_opts(opts) diff --git a/coriolis/constants.py b/coriolis/constants.py index 270bf47d..75d480c5 100644 --- a/coriolis/constants.py +++ b/coriolis/constants.py @@ -21,7 +21,7 @@ ACTIVE_EXECUTION_STATUSES = [ EXECUTION_STATUS_RUNNING, EXECUTION_STATUS_CANCELLING, - EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS, ] FINALIZED_EXECUTION_STATUSES = [ @@ -30,7 +30,7 @@ EXECUTION_STATUS_ERROR, EXECUTION_STATUS_CANCELED_FOR_DEBUGGING, EXECUTION_STATUS_DEADLOCKED, - EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS, ] TASK_STATUS_SCHEDULED = "SCHEDULED" @@ -56,7 +56,7 @@ TASK_STATUS_STARTING, TASK_STATUS_RUNNING, TASK_STATUS_CANCELLING, - TASK_STATUS_CANCELLING_AFTER_COMPLETION + TASK_STATUS_CANCELLING_AFTER_COMPLETION, ] CANCELED_TASK_STATUSES = [ @@ -67,7 +67,7 @@ TASK_STATUS_CANCELED_FOR_DEBUGGING, TASK_STATUS_CANCELED_FROM_DEADLOCK, TASK_STATUS_FAILED_TO_SCHEDULE, - TASK_STATUS_FAILED_TO_CANCEL + TASK_STATUS_FAILED_TO_CANCEL, ] FINALIZED_TASK_STATUSES = [ @@ -80,12 +80,11 @@ TASK_STATUS_CANCELED_FROM_DEADLOCK, TASK_STATUS_CANCELED_AFTER_COMPLETION, TASK_STATUS_FAILED_TO_SCHEDULE, - TASK_STATUS_FAILED_TO_CANCEL + TASK_STATUS_FAILED_TO_CANCEL, ] TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT = "FINALIZE_INSTANCE_DEPLOYMENT" -TASK_TYPE_CLEANUP_FAILED_INSTANCE_DEPLOYMENT = ( - "CLEANUP_FAILED_INSTANCE_DEPLOYMENT") +TASK_TYPE_CLEANUP_FAILED_INSTANCE_DEPLOYMENT = "CLEANUP_FAILED_INSTANCE_DEPLOYMENT" TASK_TYPE_DEPLOY_OS_MORPHING_RESOURCES = "DEPLOY_OS_MORPHING_RESOURCES" TASK_TYPE_OS_MORPHING = "OS_MORPHING" @@ -94,7 +93,8 @@ TASK_TYPE_GET_INSTANCE_INFO = "GET_INSTANCE_INFO" TASK_TYPE_DEPLOY_TRANSFER_DISKS = "DEPLOY_TRANSFER_DISKS" TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS = ( - "DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS") + "DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS" +) TASK_TYPE_DELETE_TRANSFER_DISKS = "DELETE_TRANSFER_DISKS" TASK_TYPE_REPLICATE_DISKS = "REPLICATE_DISKS" TASK_TYPE_DEPLOY_TRANSFER_SOURCE_RESOURCES = "DEPLOY_TRANSFER_SOURCE_RESOURCES" @@ -105,51 +105,55 @@ TASK_TYPE_DEPLOY_INSTANCE_RESOURCES = "DEPLOY_INSTANCE_RESOURCES" TASK_TYPE_CREATE_TRANSFER_DISK_SNAPSHOTS = "CREATE_TRANSFER_DISK_SNAPSHOTS" TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS = ( - "DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS") + "DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS" +) TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS = "RESTORE_TRANSFER_DISK_SNAPSHOTS" TASK_TYPE_GET_OPTIMAL_FLAVOR = "GET_OPTIMAL_FLAVOR" TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS = "VALIDATE_TRANSFER_SOURCE_INPUTS" -TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS = ( - "VALIDATE_TRANSFER_DESTINATION_INPUTS") +TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS = "VALIDATE_TRANSFER_DESTINATION_INPUTS" TASK_TYPE_VALIDATE_DEPLOYMENT_INPUTS = "VALIDATE_DEPLOYMENT_INPUTS" TASK_TYPE_UPDATE_SOURCE_TRANSFER = "UPDATE_SOURCE_TRANSFER" TASK_TYPE_UPDATE_DESTINATION_TRANSFER = "UPDATE_DESTINATION_TRANSFER" TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS = ( - "VALIDATE_SOURCE_MINION_POOL_ENVIRONMENT_OPTIONS") + "VALIDATE_SOURCE_MINION_POOL_ENVIRONMENT_OPTIONS" +) TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS = ( - "VALIDATE_DESTINATION_MINION_POOL_ENVIRONMENT_OPTIONS") + "VALIDATE_DESTINATION_MINION_POOL_ENVIRONMENT_OPTIONS" +) TASK_TYPE_CREATE_SOURCE_MINION_MACHINE = "CREATE_SOURCE_MINION_MACHINE" -TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE = ( - "CREATE_DESTINATION_MINION_MACHINE") +TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE = "CREATE_DESTINATION_MINION_MACHINE" TASK_TYPE_DELETE_SOURCE_MINION_MACHINE = "DELETE_SOURCE_MINION_MACHINE" -TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE = ( - "DELETE_DESTINATION_MINION_MACHINE") -TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES = ( - "SET_UP_SOURCE_POOL_SHARED_RESOURCES") +TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE = "DELETE_DESTINATION_MINION_MACHINE" +TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES = "SET_UP_SOURCE_POOL_SHARED_RESOURCES" TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES = ( - "SET_UP_DESTINATION_POOL_SHARED_RESOURCES") + "SET_UP_DESTINATION_POOL_SHARED_RESOURCES" +) TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES = ( - "TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES") + "TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES" +) TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES = ( - "TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES") + "TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES" +) TASK_TYPE_ATTACH_VOLUMES_TO_SOURCE_MINION = "ATTACH_VOLUMES_TO_SOURCE_MINION" -TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION = ( - "DETACH_VOLUMES_FROM_SOURCE_MINION") -TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION = ( - "ATTACH_VOLUMES_TO_DESTINATION_MINION") +TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION = "DETACH_VOLUMES_FROM_SOURCE_MINION" +TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION = "ATTACH_VOLUMES_TO_DESTINATION_MINION" TASK_TYPE_DETACH_VOLUMES_FROM_DESTINATION_MINION = ( - "DETACH_VOLUMES_FROM_DESTINATION_MINION") -TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION = ( - "ATTACH_VOLUMES_TO_OSMORPHING_MINION") + "DETACH_VOLUMES_FROM_DESTINATION_MINION" +) +TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION = "ATTACH_VOLUMES_TO_OSMORPHING_MINION" TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION = ( - "DETACH_VOLUMES_FROM_OSMORPHING_MINION") + "DETACH_VOLUMES_FROM_OSMORPHING_MINION" +) TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY = ( - "VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY") + "VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY" +) TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY = ( - "VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY") + "VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY" +) TASK_TYPE_VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY = ( - "VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY") + "VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY" +) TASK_TYPE_RELEASE_SOURCE_MINION = "RELEASE_SOURCE_MINION" TASK_TYPE_RELEASE_DESTINATION_MINION = "RELEASE_DESTINATION_MINION" TASK_TYPE_RELEASE_OSMORPHING_MINION = "RELEASE_OSMORPHING_MINION" @@ -178,7 +182,7 @@ TASK_TYPE_POWER_ON_SOURCE_MINION, TASK_TYPE_POWER_OFF_SOURCE_MINION, TASK_TYPE_POWER_ON_DESTINATION_MINION, - TASK_TYPE_POWER_OFF_DESTINATION_MINION + TASK_TYPE_POWER_OFF_DESTINATION_MINION, ] TASK_PLATFORM_SOURCE = "source" @@ -250,18 +254,20 @@ DEFAULT_OS_TYPE = OS_TYPE_LINUX -VALID_OS_TYPES = [OS_TYPE_BSD, OS_TYPE_LINUX, - OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS] +VALID_OS_TYPES = [ + OS_TYPE_BSD, + OS_TYPE_LINUX, + OS_TYPE_OS_X, + OS_TYPE_SOLARIS, + OS_TYPE_WINDOWS, +] TMP_DIRS_KEY = "__tmp_dirs" COMPRESSION_FORMAT_GZIP = "gzip" COMPRESSION_FORMAT_ZLIB = "zlib" -VALID_COMPRESSION_FORMATS = [ - COMPRESSION_FORMAT_GZIP, - COMPRESSION_FORMAT_ZLIB -] +VALID_COMPRESSION_FORMATS = [COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_ZLIB] TRANSFER_ACTION_TYPE_DEPLOYMENT = "deployment" TRANSFER_ACTION_TYPE_TRANSFER = "transfer" @@ -273,12 +279,13 @@ EXECUTION_TYPE_MINION_POOL_MAINTENANCE = "minion_pool_maintenance" EXECUTION_TYPE_MINION_POOL_UPDATE = "minion_pool_update" EXECUTION_TYPE_MINION_POOL_SET_UP_SHARED_RESOURCES = ( - "minion_pool_set_up_shared_resources") + "minion_pool_set_up_shared_resources" +) EXECUTION_TYPE_MINION_POOL_TEAR_DOWN_SHARED_RESOURCES = ( - "minion_pool_tear_down_shared_resources") + "minion_pool_tear_down_shared_resources" +) EXECUTION_TYPE_MINION_POOL_ALLOCATE_MINIONS = "minion_pool_allocate_minions" -EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS = ( - "minion_pool_deallocate_minions") +EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS = "minion_pool_deallocate_minions" TASK_LOCK_NAME_FORMAT = "task-%s" TASKFLOW_LOCK_NAME_FORMAT = "taskflow-%s" @@ -299,12 +306,12 @@ EXECUTION_TYPE_TRANSFER_DISKS_DELETE: TRANSFER_LOCK_NAME_FORMAT, EXECUTION_TYPE_MINION_POOL_MAINTENANCE: MINION_POOL_LOCK_NAME_FORMAT, EXECUTION_TYPE_MINION_POOL_UPDATE: MINION_POOL_LOCK_NAME_FORMAT, - EXECUTION_TYPE_MINION_POOL_SET_UP_SHARED_RESOURCES: ( - MINION_POOL_LOCK_NAME_FORMAT), + EXECUTION_TYPE_MINION_POOL_SET_UP_SHARED_RESOURCES: (MINION_POOL_LOCK_NAME_FORMAT), EXECUTION_TYPE_MINION_POOL_TEAR_DOWN_SHARED_RESOURCES: ( - MINION_POOL_LOCK_NAME_FORMAT), + MINION_POOL_LOCK_NAME_FORMAT + ), EXECUTION_TYPE_MINION_POOL_ALLOCATE_MINIONS: MINION_POOL_LOCK_NAME_FORMAT, - EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS: MINION_POOL_LOCK_NAME_FORMAT + EXECUTION_TYPE_MINION_POOL_DEALLOCATE_MINIONS: MINION_POOL_LOCK_NAME_FORMAT, } SERVICE_STATUS_UP = "UP" @@ -329,8 +336,7 @@ MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES = "ALLOCATING_SHARED_RESOURCES" MINION_POOL_STATUS_ALLOCATING_MACHINES = "ALLOCATING_MACHINES" MINION_POOL_STATUS_DEALLOCATING_MACHINES = "DEALLOCATING_MACHINES" -MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES = ( - "DEALLOCATING_SHARED_RESOURCES") +MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES = "DEALLOCATING_SHARED_RESOURCES" MINION_POOL_STATUS_ALLOCATED = "ALLOCATED" MINION_POOL_STATUS_POOL_MAINTENANCE = "IN_MAINTENANCE" @@ -339,10 +345,10 @@ MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES, MINION_POOL_STATUS_ALLOCATING_MACHINES, MINION_POOL_STATUS_DEALLOCATING_MACHINES, - MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES] + MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES, +] -MINION_MACHINE_IDENTIFIER_FORMAT = ( - "coriolis-pool-%(pool_id)s-minion-%(minion_id)s") +MINION_MACHINE_IDENTIFIER_FORMAT = "coriolis-pool-%(pool_id)s-minion-%(minion_id)s" MINION_MACHINE_STATUS_UNINITIALIZED = "UNINITIALIZED" MINION_MACHINE_STATUS_HEALTHCHECKING = "HEALTHCHECKING" MINION_MACHINE_STATUS_ALLOCATING = "ALLOCATING" diff --git a/coriolis/context.py b/coriolis/context.py index a2a4dc49..2c6ed69f 100644 --- a/coriolis/context.py +++ b/coriolis/context.py @@ -7,30 +7,50 @@ from oslo_db.sqlalchemy import enginefacade from oslo_utils import timeutils -from coriolis import exception -from coriolis import policy +from coriolis import exception, policy @enginefacade.transaction_context_provider class RequestContext(context.RequestContext): - def __init__(self, user, project_id, is_admin=None, - roles=None, project_name=None, remote_address=None, - timestamp=None, request_id=None, auth_token=None, - overwrite=True, domain_name=None, domain_id=None, - user_domain_name=None, user_domain_id=None, - project_domain_name=None, project_domain_id=None, - show_deleted=None, trust_id=None, - delete_trust_id=False, **kwargs): + def __init__( + self, + user, + project_id, + is_admin=None, + roles=None, + project_name=None, + remote_address=None, + timestamp=None, + request_id=None, + auth_token=None, + overwrite=True, + domain_name=None, + domain_id=None, + user_domain_name=None, + user_domain_id=None, + project_domain_name=None, + project_domain_id=None, + show_deleted=None, + trust_id=None, + delete_trust_id=False, + **kwargs, + ): - super( - RequestContext, self).__init__( - auth_token=auth_token, user=user, project_id=project_id, - domain_name=domain_name, domain_id=domain_id, - user_domain_name=user_domain_name, user_domain_id=user_domain_id, + super(RequestContext, self).__init__( + auth_token=auth_token, + user=user, + project_id=project_id, + domain_name=domain_name, + domain_id=domain_id, + user_domain_name=user_domain_name, + user_domain_id=user_domain_id, project_domain_name=(project_domain_name), project_domain_id=(project_domain_id), - is_admin=is_admin, show_deleted=show_deleted, - request_id=request_id, overwrite=overwrite) + is_admin=is_admin, + show_deleted=show_deleted, + request_id=request_id, + overwrite=overwrite, + ) self.roles = roles or [] self.project_name = project_name self.remote_address = remote_address @@ -73,11 +93,10 @@ def to_policy_values(self): return policy def can(self, action, target=None, fatal=True): - """ Validates policies allow the requested action to be + """Validates policies allow the requested action to be perfomed in the given context, and raises otherwise. """ - default_target = { - 'project_id': self.project_id, 'user_id': self.user_id} + default_target = {'project_id': self.project_id, 'user_id': self.user_id} if target is None: target = default_target else: @@ -95,6 +114,4 @@ def can(self, action, target=None, fatal=True): def get_admin_context(trust_id=None): - return RequestContext( - user=None, project_id=None, is_admin=True, - trust_id=trust_id) + return RequestContext(user=None, project_id=None, is_admin=True, trust_id=trust_id) diff --git a/coriolis/cron/cron.py b/coriolis/cron/cron.py index 5444e335..7f6ec0c4 100644 --- a/coriolis/cron/cron.py +++ b/coriolis/cron/cron.py @@ -4,14 +4,11 @@ import threading import time +import schedule from oslo_log import log from oslo_utils import timeutils -import schedule - -from coriolis import exception -from coriolis import schemas -from coriolis import utils +from coriolis import exception, schemas, utils LOG = log.getLogger(__name__) @@ -19,10 +16,19 @@ class CronJob(object): - - def __init__(self, name, description, schedule, enabled, - expires, on_success, on_error, - job_callable, *args, **kw): + def __init__( + self, + name, + description, + schedule, + enabled, + expires, + on_success, + on_error, + job_callable, + *args, + **kw, + ): # param: name: string: unique ID that describes this job # param: description: string: a short description of the job # param: schedule: dict: cron job schedule. This is of the form: @@ -71,8 +77,7 @@ def __init__(self, name, description, schedule, enabled, self._enabled = enabled if expires: if not isinstance(expires, datetime.datetime): - raise exception.CoriolisException( - "Invalid expires") + raise exception.CoriolisException("Invalid expires") self._expires = expires def _compare(self, pairs): @@ -105,12 +110,10 @@ def should_run(self, dt): LOG.debug('Job %s is not enabled', self.name) return False - fields = ('year', 'month', 'dom', 'hour', - 'minute', 'second', 'dow') + fields = ('year', 'month', 'dom', 'hour', 'minute', 'second', 'dow') dt_fields = dict(zip(fields, dt.timetuple())) - pairs = [(dt_fields[i], self.schedule.get(i)) - for i in SCHEDULE_FIELDS] + pairs = [(dt_fields[i], self.schedule.get(i)) for i in SCHEDULE_FIELDS] compared = self._compare(pairs) return False not in compared @@ -136,14 +139,16 @@ def start(self, status_queue=None): LOG.exception(callback_err) self._send_status( status_queue, - {"result": result, - "description": self._description, - "name": self.name, - "error_info": exc_info}) + { + "result": result, + "description": self._description, + "name": self.name, + "error_info": exc_info, + }, + ) class Cron(object): - def __init__(self): self._queue = queue.Queue(maxsize=1000) self._should_stop = False @@ -167,13 +172,14 @@ def unregister(self, name): del self._jobs[name] def unregister_jobs_with_prefix(self, prefix): - jobs = [ - job for job in self._jobs - if job.startswith(prefix)] + jobs = [job for job in self._jobs if job.startswith(prefix)] if jobs: LOG.debug( "Unregistering the following cron jobs based on " - "the requested prefix ('%s'): %s", prefix, jobs) + "the requested prefix ('%s'): %s", + prefix, + jobs, + ) with self._semaphore: for job in jobs: del self._jobs[job] @@ -186,8 +192,11 @@ def _check_jobs(self): now = timeutils.utcnow() if job_nr: for job in jobs: - LOG.debug('Checking job %s with schedule: %s', jobs[job].name, - jobs[job].schedule) + LOG.debug( + 'Checking job %s with schedule: %s', + jobs[job].name, + jobs[job].schedule, + ) if jobs[job].should_run(now): LOG.debug("Spawning job %s" % job) utils.start_thread(jobs[job].start, args=[self._queue]) @@ -195,14 +204,15 @@ def _check_jobs(self): done = timeutils.utcnow() delta = done - now - LOG.debug("Spawned %(jobs)d jobs in %(seconds)d seconds" % { - "seconds": delta.seconds, - "jobs": spawned}) + LOG.debug( + "Spawned %(jobs)d jobs in %(seconds)d seconds" + % {"seconds": delta.seconds, "jobs": spawned} + ) def _loop(self): while not self._should_stop: schedule.run_pending() - time.sleep(.2) + time.sleep(0.2) def _result_loop(self): while not self._should_stop: @@ -214,12 +224,13 @@ def _result_loop(self): # the logs table...or do something much more meaningful if error: LOG.error( - "Job %(job_desc)s exited with error: %(job_err)r" % - {"job_desc": desc, "job_err": error}) + "Job %(job_desc)s exited with error: %(job_err)r" + % {"job_desc": desc, "job_err": error} + ) if result: - LOG.info("Job %(desc)s returned: %(ret)r" % { - "desc": desc, - "ret": result}) + LOG.info( + "Job %(desc)s returned: %(ret)r" % {"desc": desc, "ret": result} + ) def _janitor(self): # remove expired jobs from memory. The check for expired diff --git a/coriolis/data_transfer.py b/coriolis/data_transfer.py index d1c573ce..b59dff3f 100644 --- a/coriolis/data_transfer.py +++ b/coriolis/data_transfer.py @@ -6,24 +6,24 @@ import stat import struct import zlib +from urllib import parse import requests import requests_unixsocket - from oslo_config import cfg from oslo_log import log as logging -from urllib import parse -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception compressor_opts = [ - cfg.StrOpt('compressor_address', - default=None, - help='Compressor address. If set, all gzip/zlib compression ' - 'will be done through this service. This value can be ' - 'either a unix socket path (/var/run/compressor.sock ' - 'or an IP:PORT.'), + cfg.StrOpt( + 'compressor_address', + default=None, + help='Compressor address. If set, all gzip/zlib compression ' + 'will be done through this service. This value can be ' + 'either a unix socket path (/var/run/compressor.sock ' + 'or an IP:PORT.', + ), ] CONF = cfg.CONF @@ -46,22 +46,26 @@ def _get_session_and_address(): if os.path.exists(CONF.compressor_address): mode = os.stat(CONF.compressor_address).st_mode if stat.S_ISSOCK(mode): - return (requests_unixsocket.Session(), - "http+unix://%s/" % parse.quote_plus( - CONF.compressor_address)) + return ( + requests_unixsocket.Session(), + "http+unix://%s/" % parse.quote_plus(CONF.compressor_address), + ) else: raise exception.CoriolisException( - "compressor_address is not a valid unix socket") + "compressor_address is not a valid unix socket" + ) else: raise exception.CoriolisException( - "compressor_address is not a valid unix socket") + "compressor_address is not a valid unix socket" + ) return (requests.Session(), "http://%s/" % CONF.compressor_address) def compression_proxy(content, fmt): if fmt not in constants.VALID_COMPRESSION_FORMATS: raise exception.CoriolisException( - "Invalid compression format requested: %s" % fmt) + "Invalid compression format requested: %s" % fmt + ) data = content sess, url = _get_session_and_address() if None in (sess, url): @@ -71,13 +75,13 @@ def compression_proxy(content, fmt): headers = { "X-Compression-Format": fmt, } - ret = sess.post(url, data=data, headers=headers, - timeout=CONF.default_requests_timeout) + ret = sess.post( + url, data=data, headers=headers, timeout=CONF.default_requests_timeout + ) ret.raise_for_status() compressed_data = ret.content except Exception as err: - LOG.exception( - "failed to compress using coriolis-compressor: %s" % err) + LOG.exception("failed to compress using coriolis-compressor: %s" % err) LOG.info("falling back to built-in compressor") compressed_data = _COMPRESS_FUNC[fmt](content) finally: @@ -86,8 +90,7 @@ def compression_proxy(content, fmt): data_len = len(compressed_data) data_len_inflated = len(data) compression_saving = 100.0 * (1 - float(data_len) / data_len_inflated) - LOG.debug("Compression space saving: {:.02f}%".format( - compression_saving)) + LOG.debug("Compression space saving: {:.02f}%".format(compression_saving)) if data_len >= data_len_inflated: # No advantage in sending the compressed data @@ -99,16 +102,15 @@ def compression_proxy(content, fmt): def encode_data(msg_id, path, offset, content, compress=True): - inflated_content = (path.encode() + b'\0' + - struct.pack(" now)) + or_( + models.TransferSchedule.expiration_date == null(), + models.TransferSchedule.expiration_date > now, + ) + ) return sched_filter @@ -140,22 +143,21 @@ def _soft_delete_aware_query(context, *args, **kwargs): @enginefacade.reader def get_endpoints(context): q = _soft_delete_aware_query(context, models.Endpoint).options( - orm.joinedload('mapped_regions')) + orm.joinedload('mapped_regions') + ) if is_user_context(context): - q = q.filter( - models.Endpoint.project_id == context.project_id) + q = q.filter(models.Endpoint.project_id == context.project_id) return q.filter().all() @enginefacade.reader def get_endpoint(context, endpoint_id): q = _soft_delete_aware_query(context, models.Endpoint).options( - orm.joinedload('mapped_regions')) + orm.joinedload('mapped_regions') + ) if is_user_context(context): - q = q.filter( - models.Endpoint.project_id == context.project_id) - return q.filter( - models.Endpoint.id == endpoint_id).first() + q = q.filter(models.Endpoint.project_id == context.project_id) + return q.filter(models.Endpoint.id == endpoint_id).first() @enginefacade.writer @@ -174,22 +176,26 @@ def update_endpoint(context, endpoint_id, updated_values): if not isinstance(updated_values, dict): raise exception.InvalidInput( "Update payload for endpoints must be a dict. Got the following " - "(type: %s): %s" % (type(updated_values), updated_values)) + "(type: %s): %s" % (type(updated_values), updated_values) + ) def _try_unmap_regions(region_ids): for region_to_unmap in region_ids: try: LOG.debug( "Attempting to unmap region '%s' from endpoint '%s'", - region_to_unmap, endpoint_id) - delete_endpoint_region_mapping( - context, endpoint_id, region_to_unmap) + region_to_unmap, + endpoint_id, + ) + delete_endpoint_region_mapping(context, endpoint_id, region_to_unmap) except Exception: LOG.warn( "Exception occurred while attempting to unmap region '%s' " "from endpoint '%s'. Ignoring. Error was: %s", - region_to_unmap, endpoint_id, - utils.get_exception_details()) + region_to_unmap, + endpoint_id, + utils.get_exception_details(), + ) newly_mapped_regions = [] regions_to_unmap = [] @@ -202,24 +208,28 @@ def _try_unmap_regions(region_ids): if not region: raise exception.NotFound( "Could not find region with ID '%s' for associating " - "with endpoint '%s' during update process." % ( - region_id, endpoint_id)) + "with endpoint '%s' during update process." + % (region_id, endpoint_id) + ) # get all existing mappings: existing_region_mappings = [ mapping.region_id - for mapping in get_region_mappings_for_endpoint( - context, endpoint_id)] + for mapping in get_region_mappings_for_endpoint(context, endpoint_id) + ] # check and add new mappings: - to_map = set( - desired_region_mappings).difference(set(existing_region_mappings)) - regions_to_unmap = set( - existing_region_mappings).difference(set(desired_region_mappings)) + to_map = set(desired_region_mappings).difference(set(existing_region_mappings)) + regions_to_unmap = set(existing_region_mappings).difference( + set(desired_region_mappings) + ) LOG.debug( "Remapping regions for endpoint '%s' from %s to %s", - endpoint_id, existing_region_mappings, desired_region_mappings) + endpoint_id, + existing_region_mappings, + desired_region_mappings, + ) region_id = None try: @@ -233,27 +243,36 @@ def _try_unmap_regions(region_ids): LOG.warn( "Exception occurred while adding region mapping for '%s' to " "endpoint '%s'. Cleaning up created mappings (%s). Error was: " - "%s", region_id, endpoint_id, newly_mapped_regions, - utils.get_exception_details()) + "%s", + region_id, + endpoint_id, + newly_mapped_regions, + utils.get_exception_details(), + ) _try_unmap_regions(newly_mapped_regions) raise updateable_fields = ["name", "description", "connection_info"] try: - _update_sqlalchemy_object_fields( - endpoint, updateable_fields, updated_values) + _update_sqlalchemy_object_fields(endpoint, updateable_fields, updated_values) except Exception: LOG.warn( "Exception occurred while updating fields of endpoint '%s'. " - "Cleaning ""up created mappings (%s). Error was: %s", - endpoint_id, newly_mapped_regions, utils.get_exception_details()) + "Cleaning " + "up created mappings (%s). Error was: %s", + endpoint_id, + newly_mapped_regions, + utils.get_exception_details(), + ) _try_unmap_regions(newly_mapped_regions) raise # remove all of the old region mappings: LOG.debug( "Unmapping the following regions during update of endpoint '%s': %s", - endpoint_id, regions_to_unmap) + endpoint_id, + regions_to_unmap, + ) _try_unmap_regions(regions_to_unmap) @@ -263,8 +282,11 @@ def delete_endpoint(context, endpoint_id): args = {"id": endpoint_id} if is_user_context(context): args["project_id"] = context.project_id - count = _soft_delete_aware_query(context, models.Endpoint).filter_by( - **args).soft_delete() + count = ( + _soft_delete_aware_query(context, models.Endpoint) + .filter_by(**args) + .soft_delete() + ) if count == 0: raise exception.NotFound("0 Endpoint entries were soft deleted") # NOTE(aznashwan): many-to-many tables with soft deletion on either end of @@ -275,13 +297,17 @@ def delete_endpoint(context, endpoint_id): @enginefacade.reader -def get_transfer_tasks_executions(context, transfer_id, include_tasks=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys: list[str] | None = None, - sort_dirs: list[str] | None = None, - to_dict=False): +def get_transfer_tasks_executions( + context, + transfer_id, + include_tasks=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys: list[str] | None = None, + sort_dirs: list[str] | None = None, + to_dict=False, +): q = _soft_delete_aware_query(context, models.TasksExecution) q = q.join(models.Transfer) if include_task_info: @@ -299,12 +325,13 @@ def get_transfer_tasks_executions(context, transfer_id, include_tasks=False, ) if marker: try: - marker = get_transfer_tasks_execution( - context, transfer_id, marker) + marker = get_transfer_tasks_execution(context, transfer_id, marker) except exception.NotFound: raise exception.MarkerNotFound(marker=marker) q = sqlalchemy_utils.paginate_query( - q, models.TasksExecution, limit, + q, + models.TasksExecution, + limit, sort_keys=sort_keys, sort_dirs=sort_dirs, marker=marker, @@ -317,10 +344,10 @@ def get_transfer_tasks_executions(context, transfer_id, include_tasks=False, @enginefacade.reader -def get_transfer_tasks_execution(context, transfer_id, execution_id, - include_task_info=False, to_dict=False): - q = _soft_delete_aware_query(context, models.TasksExecution).join( - models.Transfer) +def get_transfer_tasks_execution( + context, transfer_id, execution_id, include_task_info=False, to_dict=False +): + q = _soft_delete_aware_query(context, models.TasksExecution).join(models.Transfer) if include_task_info: q = q.options(orm.joinedload('action').undefer('info')) q = _get_tasks_with_details_options(q) @@ -328,8 +355,8 @@ def get_transfer_tasks_execution(context, transfer_id, execution_id, q = q.filter(models.Transfer.project_id == context.project_id) db_result = q.filter( - models.Transfer.id == transfer_id, - models.TasksExecution.id == execution_id).first() + models.Transfer.id == transfer_id, models.TasksExecution.id == execution_id + ).first() if to_dict and db_result is not None: return db_result.to_dict() return db_result @@ -342,12 +369,12 @@ def add_transfer_tasks_execution(context, execution): raise exception.NotAuthorized() # include deleted records - max_number = _model_query( - context, - func.max( - models.TasksExecution.number)).filter( - models.TasksExecution.action_id == ( - execution.action.id)).first()[0] or 0 + max_number = ( + _model_query(context, func.max(models.TasksExecution.number)) + .filter(models.TasksExecution.action_id == (execution.action.id)) + .first()[0] + or 0 + ) execution.number = max_number + 1 _session(context).add(execution) @@ -356,10 +383,14 @@ def add_transfer_tasks_execution(context, execution): @enginefacade.writer def delete_transfer_tasks_execution(context, execution_id): q = _soft_delete_aware_query(context, models.TasksExecution).filter( - models.TasksExecution.id == execution_id) + models.TasksExecution.id == execution_id + ) if is_user_context(context): - if not q.join(models.Transfer).filter( - models.Transfer.project_id == context.project_id).first(): + if ( + not q.join(models.Transfer) + .filter(models.Transfer.project_id == context.project_id) + .first() + ): raise exception.NotAuthorized() count = q.soft_delete() if count == 0: @@ -369,22 +400,28 @@ def delete_transfer_tasks_execution(context, execution_id): @enginefacade.reader def get_transfer_schedules(context, transfer_id=None, expired=True): sched_filter = _get_transfer_schedules_filter( - context, transfer_id=transfer_id, expired=expired) + context, transfer_id=transfer_id, expired=expired + ) return sched_filter.all() @enginefacade.reader def get_transfer_schedule(context, transfer_id, schedule_id, expired=True): sched_filter = _get_transfer_schedules_filter( - context, transfer_id=transfer_id, schedule_id=schedule_id, - expired=expired) + context, transfer_id=transfer_id, schedule_id=schedule_id, expired=expired + ) return sched_filter.first() @enginefacade.writer -def update_transfer_schedule(context, transfer_id, schedule_id, - updated_values, pre_update_callable=None, - post_update_callable=None): +def update_transfer_schedule( + context, + transfer_id, + schedule_id, + updated_values, + pre_update_callable=None, + post_update_callable=None, +): # NOTE(gsamfira): we need to refactor the DB layer a bit to allow # two-phase transactions or at least allow running these functions # inside a single transaction block. @@ -392,8 +429,12 @@ def update_transfer_schedule(context, transfer_id, schedule_id, if pre_update_callable: pre_update_callable(schedule=schedule) updateable_attributes = [ - "schedule", "expiration_date", "enabled", "shutdown_instance", - "auto_deploy"] + "schedule", + "expiration_date", + "enabled", + "shutdown_instance", + "auto_deploy", + ] for val in updateable_attributes: if val in updated_values: setattr(schedule, val, updated_values[val]) @@ -405,23 +446,30 @@ def update_transfer_schedule(context, transfer_id, schedule_id, @enginefacade.writer -def delete_transfer_schedule(context, transfer_id, - schedule_id, pre_delete_callable=None, - post_delete_callable=None): +def delete_transfer_schedule( + context, + transfer_id, + schedule_id, + pre_delete_callable=None, + post_delete_callable=None, +): # NOTE(gsamfira): we need to refactor the DB layer a bit to allow # two-phase transactions or at least allow running these functions # inside a single transaction block. q = _soft_delete_aware_query(context, models.TransferSchedule).filter( models.TransferSchedule.id == schedule_id, - models.TransferSchedule.transfer_id == transfer_id) + models.TransferSchedule.transfer_id == transfer_id, + ) schedule = q.first() if not schedule: - raise exception.NotFound( - "No such schedule") + raise exception.NotFound("No such schedule") if is_user_context(context): - if not q.join(models.Transfer).filter( - models.Transfer.project_id == context.project_id).first(): + if ( + not q.join(models.Transfer) + .filter(models.Transfer.project_id == context.project_id) + .first() + ): raise exception.NotAuthorized() if pre_delete_callable: pre_delete_callable(context, schedule) @@ -450,15 +498,17 @@ def _get_transfer_with_tasks_executions_options(q): @enginefacade.reader -def get_transfers(context, - transfer_scenario=None, - include_tasks_executions=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys: list[str] | None = None, - sort_dirs: list[str] | None = None, - to_dict=False): +def get_transfers( + context, + transfer_scenario=None, + include_tasks_executions=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys: list[str] | None = None, + sort_dirs: list[str] | None = None, + to_dict=False, +): q = _soft_delete_aware_query(context, models.Transfer) if include_tasks_executions: q = _get_transfer_with_tasks_executions_options(q) @@ -468,8 +518,7 @@ def get_transfers(context, if transfer_scenario: q = q.filter(models.Transfer.scenario == transfer_scenario) if is_user_context(context): - q = q.filter( - models.Transfer.project_id == context.project_id) + q = q.filter(models.Transfer.project_id == context.project_id) sort_keys, sort_dirs = process_sort_params( sort_keys, @@ -481,7 +530,9 @@ def get_transfers(context, except exception.NotFound: raise exception.MarkerNotFound(marker=marker) q = sqlalchemy_utils.paginate_query( - q, models.Transfer, limit, + q, + models.Transfer, + limit, sort_keys=sort_keys, sort_dirs=sort_dirs, marker=marker, @@ -492,29 +543,27 @@ def get_transfers(context, return [ i.to_dict( include_task_info=include_task_info, - include_executions=include_tasks_executions) - for i in db_result] + include_executions=include_tasks_executions, + ) + for i in db_result + ] return db_result @enginefacade.reader -def get_transfer(context, transfer_id, - transfer_scenario=None, - include_task_info=False, - to_dict=False): +def get_transfer( + context, transfer_id, transfer_scenario=None, include_task_info=False, to_dict=False +): q = _soft_delete_aware_query(context, models.Transfer) q = _get_transfer_with_tasks_executions_options(q) if include_task_info: q = q.options(orm.undefer('info')) if transfer_scenario: - q = q.filter( - models.Transfer.scenario == transfer_scenario) + q = q.filter(models.Transfer.scenario == transfer_scenario) if is_user_context(context): - q = q.filter( - models.Transfer.project_id == context.project_id) + q = q.filter(models.Transfer.project_id == context.project_id) - transfer = q.filter( - models.Transfer.id == transfer_id).first() + transfer = q.filter(models.Transfer.id == transfer_id).first() if to_dict and transfer is not None: return transfer.to_dict(include_task_info=include_task_info) @@ -522,8 +571,7 @@ def get_transfer(context, transfer_id, @enginefacade.reader -def get_endpoint_transfers_count( - context, endpoint_id, transfer_scenario=None): +def get_endpoint_transfers_count(context, endpoint_id, transfer_scenario=None): scenario_filter_kwargs = {} if transfer_scenario: @@ -531,13 +579,19 @@ def get_endpoint_transfers_count( origin_args = {'origin_endpoint_id': endpoint_id} origin_args.update(scenario_filter_kwargs) - q_origin_count = _soft_delete_aware_query( - context, models.Transfer).filter_by(**origin_args).count() + q_origin_count = ( + _soft_delete_aware_query(context, models.Transfer) + .filter_by(**origin_args) + .count() + ) destination_args = {'destination_endpoint_id': endpoint_id} destination_args.update(scenario_filter_kwargs) - q_destination_count = _soft_delete_aware_query( - context, models.Transfer).filter_by(**destination_args).count() + q_destination_count = ( + _soft_delete_aware_query(context, models.Transfer) + .filter_by(**destination_args) + .count() + ) return q_origin_count + q_destination_count @@ -554,13 +608,13 @@ def _delete_transfer_action(context, cls, id): args = {"base_id": id} if is_user_context(context): args["project_id"] = context.project_id - count = _soft_delete_aware_query(context, cls).filter_by( - **args).soft_delete() + count = _soft_delete_aware_query(context, cls).filter_by(**args).soft_delete() if count == 0: raise exception.NotFound("0 entries were soft deleted") _soft_delete_aware_query(context, models.TasksExecution).filter_by( - action_id=id).soft_delete() + action_id=id + ).soft_delete() @enginefacade.writer @@ -574,21 +628,21 @@ def get_transfer_deployments(context, transfer_id): q = q.join("transfer") q = q.options(orm.joinedload("executions")) if is_user_context(context): - q = q.filter( - models.Deployment.project_id == context.project_id) - return q.filter( - models.Transfer.id == transfer_id).all() + q = q.filter(models.Deployment.project_id == context.project_id) + return q.filter(models.Transfer.id == transfer_id).all() @enginefacade.reader -def get_deployments(context, - include_tasks=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys: list[str] | None = None, - sort_dirs: list[str] | None = None, - to_dict=False): +def get_deployments( + context, + include_tasks=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys: list[str] | None = None, + sort_dirs: list[str] | None = None, + to_dict=False, +): q = _soft_delete_aware_query(context, models.Deployment) if include_tasks: q = _get_deployment_task_query_options(q) @@ -610,7 +664,9 @@ def get_deployments(context, except exception.NotFound: raise exception.MarkerNotFound(marker=marker) q = sqlalchemy_utils.paginate_query( - q, models.Deployment, limit, + q, + models.Deployment, + limit, sort_keys=sort_keys, sort_dirs=sort_dirs, marker=marker, @@ -618,36 +674,35 @@ def get_deployments(context, result = q.all() if to_dict: - return [i.to_dict( - include_task_info=include_task_info, - include_tasks=include_tasks) for i in result] + return [ + i.to_dict(include_task_info=include_task_info, include_tasks=include_tasks) + for i in result + ] return result def _get_tasks_with_details_options(query): - return query.options( - orm.joinedload("action")).options( - orm.joinedload("tasks"). - joinedload("progress_updates")).options( - orm.joinedload("tasks"). - joinedload("events")) + return ( + query.options(orm.joinedload("action")) + .options(orm.joinedload("tasks").joinedload("progress_updates")) + .options(orm.joinedload("tasks").joinedload("events")) + ) def _get_deployment_task_query_options(query): - return query.options( - orm.joinedload("executions"). - joinedload("tasks"). - joinedload("progress_updates")).options( - orm.joinedload("executions"). - joinedload("tasks"). - joinedload("events")).options( - orm.joinedload("executions"). - joinedload("action")) + return ( + query.options( + orm.joinedload("executions") + .joinedload("tasks") + .joinedload("progress_updates") + ) + .options(orm.joinedload("executions").joinedload("tasks").joinedload("events")) + .options(orm.joinedload("executions").joinedload("action")) + ) @enginefacade.reader -def get_deployment(context, deployment_id, include_task_info=False, - to_dict=False): +def get_deployment(context, deployment_id, include_task_info=False, to_dict=False): q = _soft_delete_aware_query(context, models.Deployment) q = _get_deployment_task_query_options(q) if include_task_info: @@ -665,8 +720,7 @@ def get_deployment(context, deployment_id, include_task_info=False, @enginefacade.writer def add_deployment(context, deployment): deployment.user_id = context.user or deployment.transfer.user_id - deployment.project_id = ( - context.project_id or deployment.transfer.project_id) + deployment.project_id = context.project_id or deployment.transfer.project_id _session(context).add(deployment) @@ -676,55 +730,50 @@ def delete_deployment(context, deployment_id): @enginefacade.writer -def set_execution_status( - context, execution_id, status, update_action_status=True): - execution = _soft_delete_aware_query( - context, models.TasksExecution).join( - models.TasksExecution.action) +def set_execution_status(context, execution_id, status, update_action_status=True): + execution = _soft_delete_aware_query(context, models.TasksExecution).join( + models.TasksExecution.action + ) if is_user_context(context): execution = execution.filter( - models.BaseTransferAction.project_id == context.project_id) - execution = execution.filter( - models.TasksExecution.id == execution_id).first() + models.BaseTransferAction.project_id == context.project_id + ) + execution = execution.filter(models.TasksExecution.id == execution_id).first() if not execution: - raise exception.NotFound( - "Tasks execution not found: %s" % execution_id) + raise exception.NotFound("Tasks execution not found: %s" % execution_id) execution.status = status if update_action_status: - set_action_last_execution_status( - context, execution.action_id, status) + set_action_last_execution_status(context, execution.action_id, status) return execution @enginefacade.reader def get_action(context, action_id, include_task_info=False): - action = _soft_delete_aware_query( - context, models.BaseTransferAction) + action = _soft_delete_aware_query(context, models.BaseTransferAction) if include_task_info: action = action.options(orm.undefer('info')) if is_user_context(context): action = action.filter( - models.BaseTransferAction.project_id == context.project_id) - action = action.filter( - models.BaseTransferAction.base_id == action_id).first() + models.BaseTransferAction.project_id == context.project_id + ) + action = action.filter(models.BaseTransferAction.base_id == action_id).first() if not action: - raise exception.NotFound( - "Transfer action not found: %s" % action_id) + raise exception.NotFound("Transfer action not found: %s" % action_id) return action @enginefacade.writer -def set_action_last_execution_status( - context, action_id, last_execution_status): +def set_action_last_execution_status(context, action_id, last_execution_status): action = get_action(context, action_id) action.last_execution_status = last_execution_status @enginefacade.writer def update_transfer_action_info_for_instance( - context, action_id, instance, new_instance_info): - """ Updates the info for the given action with the provided dict. + context, action_id, instance, new_instance_info +): + """Updates the info for the given action with the provided dict. Returns the updated value. Sub-fields of the dict already in the info will get overwritten entirely! """ @@ -733,7 +782,9 @@ def update_transfer_action_info_for_instance( LOG.debug( "No new info provided for action '%s' and instance '%s'. " "Nothing to update in the DB.", - action_id, instance) + action_id, + instance, + ) return action.info.get(instance, {}) # Copy is needed, otherwise sqlalchemy won't save the changes @@ -749,13 +800,19 @@ def update_transfer_action_info_for_instance( LOG.debug( "Overwriting the values of the following keys for info of " "instance '%s' of action with ID '%s': %s", - instance, action_id, overwritten_keys) + instance, + action_id, + overwritten_keys, + ) newly_added_keys = new_keys.difference(old_keys) if newly_added_keys: LOG.debug( "The following new keys will be added for info of instance " "'%s' in action with ID '%s': %s", - instance, action_id, newly_added_keys) + instance, + action_id, + newly_added_keys, + ) instance_info_old_copy = instance_info_old.copy() instance_info_old_copy.update(new_instance_info) @@ -767,7 +824,7 @@ def update_transfer_action_info_for_instance( @enginefacade.writer def set_transfer_action_result(context, action_id, instance, result): - """ Adds the result for the given 'instance' in the 'transfer_result' + """Adds the result for the given 'instance' in the 'transfer_result' JSON in the 'base_transfer_action' table. """ action = get_action(context, action_id) @@ -788,19 +845,15 @@ def get_tasks_execution(context, execution_id): q = q.options(orm.joinedload("action")) q = q.options(orm.joinedload("tasks")) if is_user_context(context): - q = q.filter( - models.BaseTransferAction.project_id == context.project_id) - execution = q.filter( - models.TasksExecution.id == execution_id).first() + q = q.filter(models.BaseTransferAction.project_id == context.project_id) + execution = q.filter(models.TasksExecution.id == execution_id).first() if not execution: - raise exception.NotFound( - "Tasks execution not found: %s" % execution_id) + raise exception.NotFound("Tasks execution not found: %s" % execution_id) return execution def _get_task(context, task_id): - task = _soft_delete_aware_query(context, models.Task).filter_by( - id=task_id).first() + task = _soft_delete_aware_query(context, models.Task).filter_by(id=task_id).first() if not task: raise exception.NotFound("Task not found: %s" % task_id) return task @@ -845,41 +898,45 @@ def add_task_event(context, task_id, level, message): @enginefacade.reader def _get_last_task_event(context, task_id): - q = _soft_delete_aware_query( - context, models.TaskEvent) - last_event = q.filter( - models.TaskEvent.task_id == task_id).order_by( - models.TaskEvent.index.desc()).first() + q = _soft_delete_aware_query(context, models.TaskEvent) + last_event = ( + q.filter(models.TaskEvent.task_id == task_id) + .order_by(models.TaskEvent.index.desc()) + .first() + ) return last_event @enginefacade.reader def _get_last_task_progress_update(context, task_id): - q = _soft_delete_aware_query( - context, models.TaskProgressUpdate) - last_update = q.filter( - models.TaskProgressUpdate.task_id == task_id).order_by( - models.TaskProgressUpdate.index.desc()).first() + q = _soft_delete_aware_query(context, models.TaskProgressUpdate) + last_update = ( + q.filter(models.TaskProgressUpdate.task_id == task_id) + .order_by(models.TaskProgressUpdate.index.desc()) + .first() + ) return last_update @enginefacade.reader def _get_last_minion_pool_event(context, pool_id): - q = _soft_delete_aware_query( - context, models.MinionPoolEvent) - last_event = q.filter( - models.MinionPoolEvent.pool_id == pool_id).order_by( - models.MinionPoolEvent.index.desc()).first() + q = _soft_delete_aware_query(context, models.MinionPoolEvent) + last_event = ( + q.filter(models.MinionPoolEvent.pool_id == pool_id) + .order_by(models.MinionPoolEvent.index.desc()) + .first() + ) return last_event @enginefacade.reader def _get_last_minion_pool_progress_update(context, pool_id): - q = _soft_delete_aware_query( - context, models.MinionPoolProgressUpdate) - last_event = q.filter( - models.MinionPoolProgressUpdate.pool_id == pool_id).order_by( - models.MinionPoolProgressUpdate.index.desc()).first() + q = _soft_delete_aware_query(context, models.MinionPoolProgressUpdate) + last_event = ( + q.filter(models.MinionPoolProgressUpdate.pool_id == pool_id) + .order_by(models.MinionPoolProgressUpdate.index.desc()) + .first() + ) return last_event @@ -904,12 +961,14 @@ def _get_minion_pool_progress_update(context, pool_id, index): q = _soft_delete_aware_query(context, models.MinionPoolProgressUpdate) return q.filter( models.MinionPoolProgressUpdate.pool_id == pool_id, - models.MinionPoolProgressUpdate.index == index).first() + models.MinionPoolProgressUpdate.index == index, + ).first() @enginefacade.writer def add_minion_pool_progress_update( - context, pool_id, message, initial_step=0, total_steps=0): + context, pool_id, message, initial_step=0, total_steps=0 +): pool_progress_update = models.MinionPoolProgressUpdate() pool_progress_update.id = str(uuid.uuid4()) pool_progress_update.pool_id = pool_id @@ -917,8 +976,7 @@ def add_minion_pool_progress_update( pool_progress_update.total_steps = total_steps pool_progress_update.message = message pool_progress_update.index = 0 - last_progress_update = _get_last_minion_pool_progress_update( - context, pool_id) + last_progress_update = _get_last_minion_pool_progress_update(context, pool_id) if last_progress_update: pool_progress_update.index = last_progress_update.index + 1 @@ -928,14 +986,21 @@ def add_minion_pool_progress_update( @enginefacade.writer def update_minion_pool_progress_update( - context, pool_id, update_index, new_current_step, - new_total_steps=None, new_message=None): + context, + pool_id, + update_index, + new_current_step, + new_total_steps=None, + new_message=None, +): pool_progress_update = _get_minion_pool_progress_update( - context, pool_id, update_index) + context, pool_id, update_index + ) if not pool_progress_update: raise exception.NotFound( "Could not find progress update for minion pool with ID '%s' and " - "index %s in the DB for updating." % (pool_id, update_index)) + "index %s in the DB for updating." % (pool_id, update_index) + ) pool_progress_update.current_step = new_current_step if new_total_steps is not None: @@ -949,12 +1014,12 @@ def _get_progress_update(context, task_id, index): q = _soft_delete_aware_query(context, models.TaskProgressUpdate) return q.filter( models.TaskProgressUpdate.task_id == task_id, - models.TaskProgressUpdate.index == index).first() + models.TaskProgressUpdate.index == index, + ).first() @enginefacade.writer -def add_task_progress_update( - context, task_id, message, initial_step=0, total_steps=0): +def add_task_progress_update(context, task_id, message, initial_step=0, total_steps=0): task_progress_update = models.TaskProgressUpdate() task_event_id = str(uuid.uuid4()) task_progress_update.id = task_event_id @@ -966,8 +1031,9 @@ def add_task_progress_update( LOG.warn( f"Progress message for task '{task_id}' with ID '{task_event_id}'" f"is too long. Truncating before insertion. " - f"Original message was: '{message}'") - message = f"{message[:max_msg_len-len('...')]}..." # noqa + f"Original message was: '{message}'" + ) + message = f"{message[: max_msg_len - len('...')]}..." # noqa task_progress_update.message = message task_progress_update.index = 0 @@ -981,14 +1047,19 @@ def add_task_progress_update( @enginefacade.writer def update_task_progress_update( - context, task_id, update_index, new_current_step, - new_total_steps=None, new_message=None): - task_progress_update = _get_progress_update( - context, task_id, update_index) + context, + task_id, + update_index, + new_current_step, + new_total_steps=None, + new_message=None, +): + task_progress_update = _get_progress_update(context, task_id, update_index) if not task_progress_update: raise exception.NotFound( "Could not find progress update for task with ID '%s' and " - "index %s in the DB for updating." % (task_id, update_index)) + "index %s in the DB for updating." % (task_id, update_index) + ) # Quick progress updates may be processed out of order. # We're trying to mitigate this by checking the current step. @@ -1001,8 +1072,11 @@ def update_task_progress_update( LOG.debug( "Ignoring out-of-order progress update for task '%s' " "index %s: new_current_step=%s is behind stored current_step=%s", - task_id, update_index, new_current_step, - task_progress_update.current_step) + task_id, + update_index, + new_current_step, + task_progress_update.current_step, + ) return task_progress_update.current_step = new_current_step @@ -1015,8 +1089,9 @@ def update_task_progress_update( LOG.warn( f"Progress message for task '{task_id}' with ID " f"'{task_event_id}' is too long. Truncating before insertion." - f" Original message was: '{new_message}'") - new_message = f"{new_message[:max_msg_len-len('...')]}..." # noqa + f" Original message was: '{new_message}'" + ) + new_message = f"{new_message[: max_msg_len - len('...')]}..." # noqa task_progress_update.message = new_message @@ -1026,33 +1101,40 @@ def update_transfer(context, transfer_id, updated_values): if not transfer: raise exception.NotFound("Transfer not found") - mapped_info_fields = { - 'destination_environment': 'target_environment'} + mapped_info_fields = {'destination_environment': 'target_environment'} updateable_fields = [ - "source_environment", "destination_environment", "notes", - "network_map", "storage_mappings", - "origin_minion_pool_id", "destination_minion_pool_id", - "instance_osmorphing_minion_pool_mappings", "clone_disks", - "skip_os_morphing"] + "source_environment", + "destination_environment", + "notes", + "network_map", + "storage_mappings", + "origin_minion_pool_id", + "destination_minion_pool_id", + "instance_osmorphing_minion_pool_mappings", + "clone_disks", + "skip_os_morphing", + ] for field in updateable_fields: if mapped_info_fields.get(field, field) in updated_values: LOG.debug( "Updating the '%s' field of Transfer '%s' to: '%s'", - field, transfer_id, updated_values[ - mapped_info_fields.get(field, field)]) + field, + transfer_id, + updated_values[mapped_info_fields.get(field, field)], + ) setattr( - transfer, field, - updated_values[mapped_info_fields.get(field, field)]) + transfer, field, updated_values[mapped_info_fields.get(field, field)] + ) - non_updateable_fields = set( - updated_values.keys()).difference({ - mapped_info_fields.get(field, field) - for field in updateable_fields}) + non_updateable_fields = set(updated_values.keys()).difference( + {mapped_info_fields.get(field, field) for field in updateable_fields} + ) if non_updateable_fields: LOG.warn( "The following Transfer fields can NOT be updated: %s", - non_updateable_fields) + non_updateable_fields, + ) # the oslo_db library uses this method for both the `created_at` and # `updated_at` fields @@ -1077,30 +1159,29 @@ def get_region(context, region_id): q = _soft_delete_aware_query(context, models.Region) q = q.options(orm.joinedload('mapped_endpoints')) q = q.options(orm.joinedload('mapped_services')) - return q.filter( - models.Region.id == region_id).first() + return q.filter(models.Region.id == region_id).first() @enginefacade.writer def update_region(context, region_id, updated_values): if not region_id: - raise exception.InvalidInput( - "No region ID specified for updating.") + raise exception.InvalidInput("No region ID specified for updating.") region = get_region(context, region_id) if not region: - raise exception.NotFound( - "Region with ID '%s' does not exist." % region_id) + raise exception.NotFound("Region with ID '%s' does not exist." % region_id) updateable_fields = ["name", "description", "enabled"] - _update_sqlalchemy_object_fields( - region, updateable_fields, updated_values) + _update_sqlalchemy_object_fields(region, updateable_fields, updated_values) @enginefacade.writer def delete_region(context, region_id): region = get_region(context, region_id) - count = _soft_delete_aware_query(context, models.Region).filter_by( - id=region_id).soft_delete() + count = ( + _soft_delete_aware_query(context, models.Region) + .filter_by(id=region_id) + .soft_delete() + ) if count == 0: raise exception.NotFound("0 region entries were soft deleted") # NOTE(aznashwan): many-to-many tables with soft deletion on either end of @@ -1120,8 +1201,9 @@ def add_endpoint_region_mapping(context, endpoint_region_mapping): if None in [region_id, endpoint_id]: raise exception.InvalidInput( "Provided endpoint region mapping params for the region ID " - "('%s') and the endpoint ID ('%s') must both be non-null." % ( - region_id, endpoint_id)) + "('%s') and the endpoint ID ('%s') must both be non-null." + % (region_id, endpoint_id) + ) _session(context).add(endpoint_region_mapping) @@ -1129,10 +1211,8 @@ def add_endpoint_region_mapping(context, endpoint_region_mapping): @enginefacade.reader def get_endpoint_region_mapping(context, endpoint_id, region_id): q = _soft_delete_aware_query(context, models.EndpointRegionMapping) - q = q.filter( - models.EndpointRegionMapping.region_id == region_id) - q = q.filter( - models.EndpointRegionMapping.endpoint_id == endpoint_id) + q = q.filter(models.EndpointRegionMapping.region_id == region_id) + q = q.filter(models.EndpointRegionMapping.endpoint_id == endpoint_id) return q.all() @@ -1143,28 +1223,30 @@ def delete_endpoint_region_mapping(context, endpoint_id, region_id): # supporting soft deletion from the sqlalchemy layer wihout # writing join condictions, so we hard-`delete()` instead of # `soft_delete()` util we find a better option: - count = _soft_delete_aware_query( - context, models.EndpointRegionMapping).filter_by( - **args).delete() + count = ( + _soft_delete_aware_query(context, models.EndpointRegionMapping) + .filter_by(**args) + .delete() + ) if count == 0: raise exception.NotFound( - "There is no mapping between endpoint '%s' and region '%s'." % ( - endpoint_id, region_id)) + "There is no mapping between endpoint '%s' and region '%s'." + % (endpoint_id, region_id) + ) LOG.debug( "Deleted mapping between endpoint '%s' and region '%s' from DB", - endpoint_id, region_id) + endpoint_id, + region_id, + ) @enginefacade.reader -def get_region_mappings_for_endpoint( - context, endpoint_id, enabled_regions_only=False): +def get_region_mappings_for_endpoint(context, endpoint_id, enabled_regions_only=False): q = _soft_delete_aware_query(context, models.EndpointRegionMapping) q = q.join(models.Region) - q = q.filter( - models.EndpointRegionMapping.endpoint_id == endpoint_id) + q = q.filter(models.EndpointRegionMapping.endpoint_id == endpoint_id) if enabled_regions_only: - q = q.filter( - models.Region.enabled == True) # noqa: E712 + q = q.filter(models.Region.enabled == True) # noqa: E712 return q.all() @@ -1172,8 +1254,7 @@ def get_region_mappings_for_endpoint( def get_mapped_endpoints_for_region(context, region_id): q = _soft_delete_aware_query(context, models.Endpoint) q = q.join(models.EndpointRegionMapping) - q = q.filter( - models.EndpointRegionMapping.endpoint_id == region_id) + q = q.filter(models.EndpointRegionMapping.endpoint_id == region_id) return q.all() @@ -1185,16 +1266,17 @@ def add_service(context, service): @enginefacade.reader def get_services(context): q = _soft_delete_aware_query(context, models.Service).options( - orm.joinedload('mapped_regions')) + orm.joinedload('mapped_regions') + ) return q.all() @enginefacade.reader def get_service(context, service_id): q = _soft_delete_aware_query(context, models.Service).options( - orm.joinedload('mapped_regions')) - return q.filter( - models.Service.id == service_id).first() + orm.joinedload('mapped_regions') + ) + return q.filter(models.Service.id == service_id).first() @enginefacade.reader @@ -1202,40 +1284,45 @@ def find_service(context, host, binary, topic=None): args = {"host": host, "binary": binary} if topic: args["topic"] = topic - q = _soft_delete_aware_query(context, models.Service).options( - orm.joinedload('mapped_regions')).filter_by(**args) + q = ( + _soft_delete_aware_query(context, models.Service) + .options(orm.joinedload('mapped_regions')) + .filter_by(**args) + ) return q.first() @enginefacade.writer def update_service(context, service_id, updated_values): if not service_id: - raise exception.InvalidInput( - "No service ID specified for updating.") + raise exception.InvalidInput("No service ID specified for updating.") service = get_service(context, service_id) if not service: - raise exception.NotFound( - "Service with ID '%s' does not exist." % service_id) + raise exception.NotFound("Service with ID '%s' does not exist." % service_id) if not isinstance(updated_values, dict): raise exception.InvalidInput( "Update payload for services must be a dict. Got the following " - "(type: %s): %s" % (type(updated_values), updated_values)) + "(type: %s): %s" % (type(updated_values), updated_values) + ) def _try_unmap_regions(region_ids): for region_to_unmap in region_ids: try: LOG.debug( "Attempting to unmap region '%s' from service '%s'", - region_to_unmap, service_id) - delete_service_region_mapping( - context, service_id, region_to_unmap) + region_to_unmap, + service_id, + ) + delete_service_region_mapping(context, service_id, region_to_unmap) except Exception: LOG.warn( "Exception occurred while attempting to unmap region '%s' " "from service '%s'. Ignoring. Error was: %s", - region_to_unmap, service_id, - utils.get_exception_details()) + region_to_unmap, + service_id, + utils.get_exception_details(), + ) newly_mapped_regions = [] regions_to_unmap = [] @@ -1248,24 +1335,27 @@ def _try_unmap_regions(region_ids): if not region: raise exception.NotFound( "Could not find region with ID '%s' for associating " - "with serce '%s' during update process." % ( - region_id, service_id)) + "with serce '%s' during update process." % (region_id, service_id) + ) # get all existing mappings: existing_region_mappings = [ mapping.region_id - for mapping in get_region_mappings_for_service( - context, service_id)] + for mapping in get_region_mappings_for_service(context, service_id) + ] # check and add new mappings: - to_map = set( - desired_region_mappings).difference(set(existing_region_mappings)) - regions_to_unmap = set( - existing_region_mappings).difference(set(desired_region_mappings)) + to_map = set(desired_region_mappings).difference(set(existing_region_mappings)) + regions_to_unmap = set(existing_region_mappings).difference( + set(desired_region_mappings) + ) LOG.debug( "Remapping regions for service '%s' from %s to %s", - service_id, existing_region_mappings, desired_region_mappings) + service_id, + existing_region_mappings, + desired_region_mappings, + ) region_id = None try: @@ -1279,35 +1369,47 @@ def _try_unmap_regions(region_ids): LOG.warn( "Exception occurred while adding region mapping for '%s' to " "service '%s'. Cleaning up created mappings (%s). Error was: " - "%s", region_id, service_id, newly_mapped_regions, - utils.get_exception_details()) + "%s", + region_id, + service_id, + newly_mapped_regions, + utils.get_exception_details(), + ) _try_unmap_regions(newly_mapped_regions) raise updateable_fields = ["enabled", "status", "providers", "specs"] try: - _update_sqlalchemy_object_fields( - service, updateable_fields, updated_values) + _update_sqlalchemy_object_fields(service, updateable_fields, updated_values) except Exception: LOG.warn( "Exception occurred while updating fields of service '%s'. " - "Cleaning ""up created mappings (%s). Error was: %s", - service_id, newly_mapped_regions, utils.get_exception_details()) + "Cleaning " + "up created mappings (%s). Error was: %s", + service_id, + newly_mapped_regions, + utils.get_exception_details(), + ) _try_unmap_regions(newly_mapped_regions) raise # remove all of the old region mappings: LOG.debug( "Unmapping the following regions during update of service '%s': %s", - service_id, regions_to_unmap) + service_id, + regions_to_unmap, + ) _try_unmap_regions(regions_to_unmap) @enginefacade.writer def delete_service(context, service_id): service = get_service(context, service_id) - count = _soft_delete_aware_query(context, models.Service).filter_by( - id=service_id).soft_delete() + count = ( + _soft_delete_aware_query(context, models.Service) + .filter_by(id=service_id) + .soft_delete() + ) if count == 0: raise exception.NotFound("0 service entries were soft deleted") # NOTE(aznashwan): many-to-many tables with soft deletion on either end of @@ -1325,8 +1427,9 @@ def add_service_region_mapping(context, service_region_mapping): if None in [region_id, service_id]: raise exception.InvalidInput( "Provided service region mapping params for the region ID " - "('%s') and the service ID ('%s') must both be non-null." % ( - region_id, service_id)) + "('%s') and the service ID ('%s') must both be non-null." + % (region_id, service_id) + ) _session(context).add(service_region_mapping) @@ -1334,10 +1437,8 @@ def add_service_region_mapping(context, service_region_mapping): @enginefacade.reader def get_service_region_mapping(context, service_id, region_id): q = _soft_delete_aware_query(context, models.ServiceRegionMapping) - q = q.filter( - models.ServiceRegionMapping.region_id == region_id) - q = q.filter( - models.ServiceRegionMapping.service_id == service_id) + q = q.filter(models.ServiceRegionMapping.region_id == region_id) + q = q.filter(models.ServiceRegionMapping.service_id == service_id) return q.all() @@ -1348,25 +1449,25 @@ def delete_service_region_mapping(context, service_id, region_id): # supporting soft deletion from the sqlalchemy layer wihout # writing join condictions, so we hard-`delete()` instead of # `soft_delete()` util we find a better option: - count = _soft_delete_aware_query( - context, models.ServiceRegionMapping).filter_by( - **args).delete() + count = ( + _soft_delete_aware_query(context, models.ServiceRegionMapping) + .filter_by(**args) + .delete() + ) if count == 0: raise exception.NotFound( - "There is no mapping between service '%s' and region '%s'." % ( - service_id, region_id)) + "There is no mapping between service '%s' and region '%s'." + % (service_id, region_id) + ) @enginefacade.reader -def get_region_mappings_for_service( - context, service_id, enabled_regions_only=False): +def get_region_mappings_for_service(context, service_id, enabled_regions_only=False): q = _soft_delete_aware_query(context, models.ServiceRegionMapping) q = q.join(models.Region) - q = q.filter( - models.ServiceRegionMapping.service_id == service_id) + q = q.filter(models.ServiceRegionMapping.service_id == service_id) if enabled_regions_only: - q = q.filter( - models.Region.enabled == True) # noqa: E712 + q = q.filter(models.Region.enabled == True) # noqa: E712 return q.all() @@ -1374,8 +1475,7 @@ def get_region_mappings_for_service( def get_mapped_services_for_region(context, region_id): q = _soft_delete_aware_query(context, models.Service) q = q.join(models.ServiceRegionMapping) - q = q.filter( - models.ServiceRegionMapping.service_id == region_id) + q = q.filter(models.ServiceRegionMapping.service_id == region_id) return q.all() @@ -1397,34 +1497,36 @@ def add_minion_machine(context, minion_machine): def get_minion_machines(context, allocated_action_id=None): q = _soft_delete_aware_query(context, models.MinionMachine) if allocated_action_id: - q = q.filter( - models.MinionMachine.allocated_action == allocated_action_id) + q = q.filter(models.MinionMachine.allocated_action == allocated_action_id) return q.all() @enginefacade.reader def get_minion_machine(context, minion_machine_id): q = _soft_delete_aware_query(context, models.MinionMachine) - return q.filter( - models.MinionMachine.id == minion_machine_id).first() + return q.filter(models.MinionMachine.id == minion_machine_id).first() @enginefacade.writer def update_minion_machine(context, minion_machine_id, updated_values): if not minion_machine_id: - raise exception.InvalidInput( - "No minion_machine ID specified for updating.") + raise exception.InvalidInput("No minion_machine ID specified for updating.") minion_machine = get_minion_machine(context, minion_machine_id) if not minion_machine: raise exception.NotFound( - "MinionMachine with ID '%s' does not exist." % minion_machine_id) + "MinionMachine with ID '%s' does not exist." % minion_machine_id + ) updateable_fields = [ - "connection_info", "provider_properties", "allocation_status", - "backup_writer_connection_info", "allocated_action", - "last_used_at", "power_status"] - _update_sqlalchemy_object_fields( - minion_machine, updateable_fields, updated_values) + "connection_info", + "provider_properties", + "allocation_status", + "backup_writer_connection_info", + "allocated_action", + "last_used_at", + "power_status", + ] + _update_sqlalchemy_object_fields(minion_machine, updateable_fields, updated_values) @enginefacade.writer @@ -1432,37 +1534,51 @@ def set_minion_machine_allocation_status(context, minion_machine_id, status): machine = get_minion_machine(context, minion_machine_id) if not machine: raise exception.NotFound( - "Minion machine with ID '%s' not found" % minion_machine_id) + "Minion machine with ID '%s' not found" % minion_machine_id + ) LOG.debug( "Transitioning minion machine '%s' (pool '%s') from status '%s' to " "'%s' in the DB", - minion_machine_id, machine.pool_id, machine.allocation_status, status) + minion_machine_id, + machine.pool_id, + machine.allocation_status, + status, + ) machine.allocation_status = status setattr(machine, 'updated_at', timeutils.utcnow()) @enginefacade.writer def set_minion_machines_allocation_statuses( - context, minion_machine_ids, action_id, allocation_status, - refresh_allocation_time=True): + context, + minion_machine_ids, + action_id, + allocation_status, + refresh_allocation_time=True, +): machines = get_minion_machines(context) - existing_machine_id_mappings = { - machine.id: machine for machine in machines} + existing_machine_id_mappings = {machine.id: machine for machine in machines} missing = [ - mid for mid in minion_machine_ids - if mid not in existing_machine_id_mappings] + mid for mid in minion_machine_ids if mid not in existing_machine_id_mappings + ] if missing: raise exception.NotFound( - "The following minion machines could not be found: %s" % ( - missing)) + "The following minion machines could not be found: %s" % (missing) + ) for machine_id in minion_machine_ids: machine = existing_machine_id_mappings[machine_id] LOG.debug( "Changing allocation status in DB for minion machine '%s' " - "from '%s' to '%s' and allocated action from '%s' to '%s'" % ( - machine.id, machine.allocation_status, allocation_status, - machine.allocated_action, action_id)) + "from '%s' to '%s' and allocated action from '%s' to '%s'" + % ( + machine.id, + machine.allocation_status, + allocation_status, + machine.allocated_action, + action_id, + ) + ) machine.allocated_action = action_id if refresh_allocation_time: machine.last_used_at = timeutils.utcnow() @@ -1473,8 +1589,11 @@ def set_minion_machines_allocation_statuses( def delete_minion_machine(context, minion_machine_id): # TODO(aznashwan): update models to be soft-delete-aware to # avoid needing to hard-delete here: - count = _soft_delete_aware_query(context, models.MinionMachine).filter_by( - id=minion_machine_id).delete() + count = ( + _soft_delete_aware_query(context, models.MinionMachine) + .filter_by(id=minion_machine_id) + .delete() + ) if count == 0: raise exception.NotFound("0 MinionMachine entries were soft deleted") @@ -1491,16 +1610,23 @@ def delete_minion_pool(context, minion_pool_id): args = {"id": minion_pool_id} if is_user_context(context): args["project_id"] = context.project_id - count = _soft_delete_aware_query(context, models.MinionPool).filter_by( - **args).soft_delete() + count = ( + _soft_delete_aware_query(context, models.MinionPool) + .filter_by(**args) + .soft_delete() + ) if count == 0: raise exception.NotFound("0 entries were soft deleted") @enginefacade.reader def get_minion_pool( - context, minion_pool_id, include_machines=False, - include_events=False, include_progress_updates=False): + context, + minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, +): q = _soft_delete_aware_query(context, models.MinionPool) if include_machines: q = q.options(orm.selectinload('minion_machines')) @@ -1509,21 +1635,22 @@ def get_minion_pool( if include_progress_updates: q = q.options(orm.selectinload('progress_updates')) if is_user_context(context): - q = q.filter( - models.MinionPool.project_id == context.project_id) - return q.filter( - models.MinionPool.id == minion_pool_id).first() + q = q.filter(models.MinionPool.project_id == context.project_id) + return q.filter(models.MinionPool.id == minion_pool_id).first() @enginefacade.reader def get_minion_pools( - context, include_machines=False, include_events=False, - include_progress_updates=False, to_dict=True): + context, + include_machines=False, + include_events=False, + include_progress_updates=False, + to_dict=True, +): q = _soft_delete_aware_query(context, models.MinionPool) q = q.filter() if is_user_context(context): - q = q.filter( - models.MinionPool.project_id == context.project_id) + q = q.filter(models.MinionPool.project_id == context.project_id) if include_machines: q = q.options(orm.joinedload('minion_machines')) if include_events: @@ -1536,50 +1663,61 @@ def get_minion_pools( i.to_dict( include_machines=include_machines, include_events=include_events, - include_progress_updates=include_progress_updates) - for i in db_result] + include_progress_updates=include_progress_updates, + ) + for i in db_result + ] return db_result @enginefacade.writer def set_minion_pool_status(context, minion_pool_id, status): - pool = get_minion_pool( - context, minion_pool_id, include_machines=False) + pool = get_minion_pool(context, minion_pool_id, include_machines=False) if not pool: - raise exception.NotFound( - "Minion pool '%s' not found" % minion_pool_id) + raise exception.NotFound("Minion pool '%s' not found" % minion_pool_id) LOG.debug( "Transitioning minion pool '%s' from status '%s' to '%s' in DB", - minion_pool_id, pool.status, status) + minion_pool_id, + pool.status, + status, + ) pool.status = status setattr(pool, 'updated_at', timeutils.utcnow()) @enginefacade.writer def update_minion_pool(context, minion_pool_id, updated_values): - lifecycle = get_minion_pool( - context, minion_pool_id, include_machines=False) + lifecycle = get_minion_pool(context, minion_pool_id, include_machines=False) if not lifecycle: - raise exception.NotFound( - "Minion pool '%s' not found" % minion_pool_id) + raise exception.NotFound("Minion pool '%s' not found" % minion_pool_id) updateable_fields = [ - "minimum_minions", "maximum_minions", "minion_max_idle_time", - "minion_retention_strategy", "environment_options", - "shared_resources", "notes", "name", "os_type"] + "minimum_minions", + "maximum_minions", + "minion_max_idle_time", + "minion_retention_strategy", + "environment_options", + "shared_resources", + "notes", + "name", + "os_type", + ] for field in updateable_fields: if field in updated_values: LOG.debug( "Updating the '%s' field of Minion Pool '%s' to: '%s'", - field, minion_pool_id, updated_values[field]) + field, + minion_pool_id, + updated_values[field], + ) setattr(lifecycle, field, updated_values[field]) - non_updateable_fields = set( - updated_values.keys()).difference(updateable_fields) + non_updateable_fields = set(updated_values.keys()).difference(updateable_fields) if non_updateable_fields: LOG.warn( "The following Minion Pool fields can NOT be updated: %s", - non_updateable_fields) + non_updateable_fields, + ) # the oslo_db library uses this method for both the `created_at` and # `updated_at` fields @@ -1637,8 +1775,7 @@ def process_sort_params( # Verify sort direction for sort_dir in sort_dirs: if sort_dir not in ('asc', 'desc'): - msg = (f"Unknown sort direction: {sort_dir}, " - "must be 'desc' or 'asc'.") + msg = f"Unknown sort direction: {sort_dir}, must be 'desc' or 'asc'." raise exception.InvalidInput(reason=msg) result_dirs.append(sort_dir) else: diff --git a/coriolis/db/sqlalchemy/api.py b/coriolis/db/sqlalchemy/api.py index 82297226..3f05df7a 100644 --- a/coriolis/db/sqlalchemy/api.py +++ b/coriolis/db/sqlalchemy/api.py @@ -7,8 +7,8 @@ from oslo_db import options as db_options from oslo_db.sqlalchemy import session as db_session -from coriolis.db.sqlalchemy import migration from coriolis import exception +from coriolis.db.sqlalchemy import migration from coriolis.i18n import _ CONF = cfg.CONF @@ -40,8 +40,7 @@ def get_backend(): def db_sync(engine, version=None): """Migrate the database to `version` or the most recent version.""" if version is not None and int(version) < db_version(engine): - raise exception.CoriolisException( - _("Cannot migrate to lower schema version.")) + raise exception.CoriolisException(_("Cannot migrate to lower schema version.")) return migration.db_sync(engine, version=version) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py b/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py index 71b6c534..60698c7c 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py @@ -11,132 +11,166 @@ def upgrade(migrate_engine): meta.bind = migrate_engine base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, - sqlalchemy.Column("base_id", sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'base_transfer_action', + meta, + sqlalchemy.Column( + "base_id", + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column("user_id", sqlalchemy.String(255), nullable=False), - sqlalchemy.Column("project_id", sqlalchemy.String(255), - nullable=False), + sqlalchemy.Column("project_id", sqlalchemy.String(255), nullable=False), sqlalchemy.Column("origin", sqlalchemy.Text, nullable=False), - sqlalchemy.Column("destination", sqlalchemy.Text, - nullable=False), + sqlalchemy.Column("destination", sqlalchemy.Text, nullable=False), sqlalchemy.Column("instances", sqlalchemy.Text, nullable=False), sqlalchemy.Column("type", sqlalchemy.String(50), nullable=False), sqlalchemy.Column("info", sqlalchemy.Text, nullable=False), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) migration = sqlalchemy.Table( - 'migration', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), - sqlalchemy.Column("replica_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'replica.id'), nullable=True), + 'migration', + meta, + sqlalchemy.Column( + "id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + primary_key=True, + ), + sqlalchemy.Column( + "replica_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('replica.id'), + nullable=True, + ), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) task = sqlalchemy.Table( - 'task', meta, sqlalchemy.Column( - 'id', sqlalchemy.String(36), - primary_key=True, default=lambda: str(uuid.uuid4())), + 'task', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column( - "execution_id", sqlalchemy.String(36), + "execution_id", + sqlalchemy.String(36), sqlalchemy.ForeignKey('tasks_execution.id'), - nullable=False), - sqlalchemy.Column( - "instance", sqlalchemy.String(1024), - nullable=False), - sqlalchemy.Column( - "host", sqlalchemy.String(1024), - nullable=True), - sqlalchemy.Column( - "process_id", sqlalchemy.Integer, nullable=True), - sqlalchemy.Column( - "status", sqlalchemy.String(100), - nullable=False), - sqlalchemy.Column( - "task_type", sqlalchemy.String(100), - nullable=False), - sqlalchemy.Column( - "exception_details", sqlalchemy.Text, nullable=True), + nullable=False, + ), + sqlalchemy.Column("instance", sqlalchemy.String(1024), nullable=False), + sqlalchemy.Column("host", sqlalchemy.String(1024), nullable=True), + sqlalchemy.Column("process_id", sqlalchemy.Integer, nullable=True), + sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), + sqlalchemy.Column("task_type", sqlalchemy.String(100), nullable=False), + sqlalchemy.Column("exception_details", sqlalchemy.Text, nullable=True), sqlalchemy.Column("depends_on", sqlalchemy.Text, nullable=True), sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True), - mysql_engine='InnoDB', mysql_charset='utf8') + mysql_engine='InnoDB', + mysql_charset='utf8', + ) tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'tasks_execution', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("action_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - nullable=False), + sqlalchemy.Column( + "action_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + nullable=False, + ), sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'task_progress_update', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("task_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('task.id'), - nullable=False), + sqlalchemy.Column( + "task_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('task.id'), + nullable=False, + ), sqlalchemy.Column("current_step", sqlalchemy.Integer, nullable=False), sqlalchemy.Column("total_steps", sqlalchemy.Integer, nullable=True), sqlalchemy.Column("message", sqlalchemy.String(1024), nullable=True), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) task_events = sqlalchemy.Table( - 'task_event', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'task_event', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("task_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('task.id'), - nullable=False), + sqlalchemy.Column( + "task_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('task.id'), + nullable=False, + ), sqlalchemy.Column("level", sqlalchemy.String(50), nullable=False), sqlalchemy.Column("message", sqlalchemy.String(1024), nullable=False), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) replica = sqlalchemy.Table( - 'replica', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), + 'replica', + meta, + sqlalchemy.Column( + "id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + primary_key=True, + ), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) tables = ( diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py b/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py index 4320a119..f729a909 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py @@ -11,27 +11,29 @@ def upgrade(migrate_engine): meta.bind = migrate_engine endpoint = sqlalchemy.Table( - 'endpoint', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'endpoint', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column("user_id", sqlalchemy.String(255), nullable=False), - sqlalchemy.Column("project_id", sqlalchemy.String(255), - nullable=False), + sqlalchemy.Column("project_id", sqlalchemy.String(255), nullable=False), sqlalchemy.Column("connection_info", sqlalchemy.Text, nullable=False), sqlalchemy.Column("type", sqlalchemy.String(255), nullable=False), sqlalchemy.Column("name", sqlalchemy.String(255), nullable=False), sqlalchemy.Column("description", sqlalchemy.Text), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) - tables = ( - endpoint, - ) + tables = (endpoint,) for index, table in enumerate(tables): try: @@ -42,23 +44,29 @@ def upgrade(migrate_engine): meta.drop_all(tables=tables[:index]) raise - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) # NOTE(alexpilotti) delete all records in base_transfer_action # before performing this migration origin_endpoint_id = sqlalchemy.Column( - "origin_endpoint_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False) + "origin_endpoint_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('endpoint.id'), + nullable=False, + ) base_transfer_action.create_column(origin_endpoint_id) destination_endpoint_id = sqlalchemy.Column( - "destination_endpoint_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False) + "destination_endpoint_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('endpoint.id'), + nullable=False, + ) base_transfer_action.create_column(destination_endpoint_id) destination_environment = sqlalchemy.Column( - "destination_environment", sqlalchemy.Text, nullable=True) + "destination_environment", sqlalchemy.Text, nullable=True + ) base_transfer_action.create_column(destination_environment) base_transfer_action.drop_column("origin") diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py b/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py index 40556f25..71d5ac44 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py @@ -8,10 +8,8 @@ def upgrade(migrate_engine): meta = sqlalchemy.MetaData() meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) - notes = sqlalchemy.Column( - "notes", sqlalchemy.Text, nullable=True) + notes = sqlalchemy.Column("notes", sqlalchemy.Text, nullable=True) base_transfer_action.create_column(notes) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py b/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py index 8ae2fa44..cee3fa51 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py @@ -7,34 +7,39 @@ def upgrade(migrate_engine): meta = sqlalchemy.MetaData() meta.bind = migrate_engine - sqlalchemy.Table( - 'replica', meta, autoload=True) + sqlalchemy.Table('replica', meta, autoload=True) replica_schedules = sqlalchemy.Table( - 'replica_schedules', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + 'replica_schedules', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("replica_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'replica.id'), nullable=False), + sqlalchemy.Column( + "replica_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('replica.id'), + nullable=False, + ), sqlalchemy.Column("schedule", sqlalchemy.String(255), nullable=False), sqlalchemy.Column("expiration_date", sqlalchemy.DateTime), - sqlalchemy.Column("enabled", sqlalchemy.Boolean, - default=True, nullable=False), - sqlalchemy.Column("shutdown_instance", sqlalchemy.Boolean, - default=False, nullable=False), + sqlalchemy.Column("enabled", sqlalchemy.Boolean, default=True, nullable=False), + sqlalchemy.Column( + "shutdown_instance", sqlalchemy.Boolean, default=False, nullable=False + ), sqlalchemy.Column('trust_id', sqlalchemy.String(36)), mysql_engine='InnoDB', - mysql_charset='utf8' + mysql_charset='utf8', ) - tables = ( - replica_schedules, - ) + tables = (replica_schedules,) for index, table in enumerate(tables): try: diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py b/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py index 7805fd4a..8cfb4699 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py @@ -9,9 +9,9 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'transfer_result' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) transfer_result = sqlalchemy.Column( - "transfer_result", sqlalchemy.Text, nullable=True) + "transfer_result", sqlalchemy.Text, nullable=True + ) base_transfer_action.create_column(transfer_result) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py b/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py index 460b5f43..7a5fff50 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py @@ -9,9 +9,7 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'network_map' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) - network_map = sqlalchemy.Column( - "network_map", sqlalchemy.Text, nullable=True) + network_map = sqlalchemy.Column("network_map", sqlalchemy.Text, nullable=True) base_transfer_action.create_column(network_map) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py b/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py index c9996329..ffb2e583 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py @@ -9,9 +9,9 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'storage_mappings' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) storage_mappings = sqlalchemy.Column( - "storage_mappings", sqlalchemy.Text, nullable=True) + "storage_mappings", sqlalchemy.Text, nullable=True + ) base_transfer_action.create_column(storage_mappings) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py b/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py index c04096d6..8c883106 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py @@ -9,9 +9,9 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'source_environment' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) source_environment = sqlalchemy.Column( - "source_environment", sqlalchemy.Text, nullable=True) + "source_environment", sqlalchemy.Text, nullable=True + ) base_transfer_action.create_column(source_environment) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py b/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py index 1814a815..2d6ff01a 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py @@ -6,7 +6,6 @@ def upgrade(migrate_engine): meta = sqlalchemy.MetaData() meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) base_transfer_action.c.info.alter(type=types.LargeBinary(4294967295)) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py b/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py index 0bd370d1..0e2dea6e 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py @@ -9,9 +9,9 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'reservation_id' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) + base_transfer_action = sqlalchemy.Table('base_transfer_action', meta, autoload=True) reservation_id = sqlalchemy.Column( - "reservation_id", sqlalchemy.String(36), nullable=True) + "reservation_id", sqlalchemy.String(36), nullable=True + ) base_transfer_action.create_column(reservation_id) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py b/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py index 44e83543..f525b430 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py @@ -9,9 +9,7 @@ def upgrade(migrate_engine): meta.bind = migrate_engine # add 'type' column to 'tasks_execution': - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) + tasks_execution = sqlalchemy.Table('tasks_execution', meta, autoload=True) - execution_type = sqlalchemy.Column( - "type", sqlalchemy.String(20)) + execution_type = sqlalchemy.Column("type", sqlalchemy.String(20)) tasks_execution.create_column(execution_type) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py b/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py index d278324c..708157e6 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py @@ -5,15 +5,14 @@ def upgrade(migrate_engine): meta = sqlalchemy.MetaData() meta.bind = migrate_engine - migration = sqlalchemy.Table( - 'migration', meta, autoload=True) + migration = sqlalchemy.Table('migration', meta, autoload=True) shutdown_instances = sqlalchemy.Column( - "shutdown_instances", sqlalchemy.Boolean, - nullable=False, default=False) + "shutdown_instances", sqlalchemy.Boolean, nullable=False, default=False + ) migration.create_column(shutdown_instances) replication_count = sqlalchemy.Column( - "replication_count", sqlalchemy.Integer, default=0, - nullable=False) + "replication_count", sqlalchemy.Integer, default=0, nullable=False + ) migration.create_column(replication_count) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py b/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py index fb841d7d..1103a9c8 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py @@ -7,7 +7,6 @@ def upgrade(migrate_engine): task = sqlalchemy.Table('task', meta, autoload=True) - index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) + index = sqlalchemy.Column("index", sqlalchemy.Integer, default=0, nullable=False) task.create_column(index) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py b/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py index 4682a974..e5732289 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py @@ -10,8 +10,7 @@ def upgrade(migrate_engine): meta = sqlalchemy.MetaData() meta.bind = migrate_engine - sqlalchemy.Table( - 'endpoint', meta, autoload=True) + sqlalchemy.Table('endpoint', meta, autoload=True) tables = [] @@ -20,20 +19,25 @@ def upgrade(migrate_engine): sqlalchemy.Table( 'region', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('name', sqlalchemy.String(255), nullable=False), sqlalchemy.Column( - 'description', sqlalchemy.String(1024), nullable=True), + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + sqlalchemy.Column('name', sqlalchemy.String(255), nullable=False), + sqlalchemy.Column('description', sqlalchemy.String(1024), nullable=True), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column( - 'enabled', sqlalchemy.Boolean, nullable=True, - default=lambda: False), + 'enabled', sqlalchemy.Boolean, nullable=True, default=lambda: False + ), mysql_engine='InnoDB', - mysql_charset='utf8')) + mysql_charset='utf8', + ) + ) # declare endpoint-region-mapping table: tables.append( @@ -44,23 +48,28 @@ def upgrade(migrate_engine): 'id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column( 'endpoint_id', sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), - nullable=False), + nullable=False, + ), sqlalchemy.Column( 'region_id', sqlalchemy.String(36), sqlalchemy.ForeignKey('region.id'), - nullable=False), + nullable=False, + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), mysql_engine='InnoDB', - mysql_charset='utf8')) + mysql_charset='utf8', + ) + ) # declare service table: tables.append( @@ -71,29 +80,30 @@ def upgrade(migrate_engine): 'id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column( - 'enabled', sqlalchemy.Boolean, nullable=True, - default=lambda: False), - sqlalchemy.Column( - 'host', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column( - 'binary', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column( - 'topic', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column( - 'status', sqlalchemy.String(255), nullable=False, - default=lambda: "UNKNOWN"), - sqlalchemy.Column( - 'providers', sqlalchemy.Text(), nullable=False), - sqlalchemy.Column( - 'specs', sqlalchemy.Text(), nullable=False), + default=lambda: str(uuid.uuid4()), + ), + sqlalchemy.Column( + 'enabled', sqlalchemy.Boolean, nullable=True, default=lambda: False + ), + sqlalchemy.Column('host', sqlalchemy.String(255), nullable=False), + sqlalchemy.Column('binary', sqlalchemy.String(255), nullable=False), + sqlalchemy.Column('topic', sqlalchemy.String(255), nullable=False), + sqlalchemy.Column( + 'status', + sqlalchemy.String(255), + nullable=False, + default=lambda: "UNKNOWN", + ), + sqlalchemy.Column('providers', sqlalchemy.Text(), nullable=False), + sqlalchemy.Column('specs', sqlalchemy.Text(), nullable=False), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), mysql_engine='InnoDB', - mysql_charset='utf8')) + mysql_charset='utf8', + ) + ) # declare service-region mappings table: tables.append( @@ -104,23 +114,28 @@ def upgrade(migrate_engine): 'id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column( 'service_id', sqlalchemy.String(36), sqlalchemy.ForeignKey('service.id'), - nullable=False), + nullable=False, + ), sqlalchemy.Column( 'region_id', sqlalchemy.String(36), sqlalchemy.ForeignKey('region.id'), - nullable=False), + nullable=False, + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), mysql_engine='InnoDB', - mysql_charset='utf8')) + mysql_charset='utf8', + ) + ) for index, table in enumerate(tables): try: diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py b/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py index 27c3a8b1..37dca1b8 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py @@ -10,11 +10,17 @@ def upgrade(migrate_engine): # add 'last_execution_status' column to 'base_transfer_action': base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, + 'base_transfer_action', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) last_execution_status = sqlalchemy.Column( - "last_execution_status", sqlalchemy.String(255), - default=lambda: "UNEXECUTED", nullable=False) + "last_execution_status", + sqlalchemy.String(255), + default=lambda: "UNEXECUTED", + nullable=False, + ) base_transfer_action.create_column(last_execution_status) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py b/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py index 6c48212b..66912903 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py @@ -11,13 +11,15 @@ def upgrade(migrate_engine): meta.bind = migrate_engine base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, + 'base_transfer_action', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) # extend tasks execution 'type' column: - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) + tasks_execution = sqlalchemy.Table('tasks_execution', meta, autoload=True) tasks_execution.c.type.alter(type=sqlalchemy.String(255)) tables = [] @@ -25,152 +27,187 @@ def upgrade(migrate_engine): # add table for pool lifecycles: tables.append( sqlalchemy.Table( - 'minion_pool', meta, sqlalchemy.Column( - "id", sqlalchemy.String(36), + 'minion_pool', + meta, + sqlalchemy.Column( + "id", + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), - primary_key=True), + primary_key=True, + ), sqlalchemy.Column("notes", sqlalchemy.Text, nullable=True), + sqlalchemy.Column("user_id", sqlalchemy.String(255), nullable=False), + sqlalchemy.Column("project_id", sqlalchemy.String(255), nullable=False), sqlalchemy.Column( - "user_id", sqlalchemy.String(255), - nullable=False), - sqlalchemy.Column( - "project_id", sqlalchemy.String(255), - nullable=False), - sqlalchemy.Column( - "maintenance_trust_id", sqlalchemy.String(255), - nullable=True), + "maintenance_trust_id", sqlalchemy.String(255), nullable=True + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), + sqlalchemy.Column("name", sqlalchemy.String(255), nullable=False), sqlalchemy.Column( - "name", sqlalchemy.String(255), - nullable=False), - sqlalchemy.Column( - "endpoint_id", sqlalchemy.String(36), + "endpoint_id", + sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), - nullable=False), - sqlalchemy.Column( - "environment_options", sqlalchemy.Text, nullable=False), - sqlalchemy.Column( - "os_type", sqlalchemy.String(255), - nullable=False), - sqlalchemy.Column( - "platform", sqlalchemy.String(255), - nullable=True), - sqlalchemy.Column( - "status", sqlalchemy.String(255), - nullable=False, default=lambda: "UNKNOWN"), - sqlalchemy.Column( - "shared_resources", sqlalchemy.Text, nullable=True), - sqlalchemy.Column( - 'minimum_minions', sqlalchemy.Integer, nullable=False), - sqlalchemy.Column( - 'maximum_minions', sqlalchemy.Integer, nullable=False), - sqlalchemy.Column( - 'minion_max_idle_time', sqlalchemy.Integer, nullable=False), - sqlalchemy.Column( - 'minion_retention_strategy', sqlalchemy.String(255), - nullable=False), + nullable=False, + ), + sqlalchemy.Column("environment_options", sqlalchemy.Text, nullable=False), + sqlalchemy.Column("os_type", sqlalchemy.String(255), nullable=False), + sqlalchemy.Column("platform", sqlalchemy.String(255), nullable=True), + sqlalchemy.Column( + "status", + sqlalchemy.String(255), + nullable=False, + default=lambda: "UNKNOWN", + ), + sqlalchemy.Column("shared_resources", sqlalchemy.Text, nullable=True), + sqlalchemy.Column('minimum_minions', sqlalchemy.Integer, nullable=False), + sqlalchemy.Column('maximum_minions', sqlalchemy.Integer, nullable=False), + sqlalchemy.Column( + 'minion_max_idle_time', sqlalchemy.Integer, nullable=False + ), + sqlalchemy.Column( + 'minion_retention_strategy', sqlalchemy.String(255), nullable=False + ), mysql_engine="InnoDB", - mysql_charset="utf8")) + mysql_charset="utf8", + ) + ) # declare minion machine table: tables.append( sqlalchemy.Table( 'minion_machine', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column( - "user_id", sqlalchemy.String(255), nullable=False), sqlalchemy.Column( - "project_id", sqlalchemy.String(255), nullable=False), + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + sqlalchemy.Column("user_id", sqlalchemy.String(255), nullable=False), + sqlalchemy.Column("project_id", sqlalchemy.String(255), nullable=False), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column( - 'pool_id', sqlalchemy.String(36), + 'pool_id', + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False), - sqlalchemy.Column( - 'allocated_action', sqlalchemy.String(36), nullable=True), - sqlalchemy.Column( - 'last_used_at', sqlalchemy.DateTime, nullable=True), - sqlalchemy.Column( - 'allocation_status', sqlalchemy.String(255), nullable=False, - default=lambda: "UNINITIALIZED"), - sqlalchemy.Column( - 'power_status', sqlalchemy.String(255), nullable=False, - default=lambda: "UNINITIALIZED"), + nullable=False, + ), + sqlalchemy.Column('allocated_action', sqlalchemy.String(36), nullable=True), + sqlalchemy.Column('last_used_at', sqlalchemy.DateTime, nullable=True), + sqlalchemy.Column( + 'allocation_status', + sqlalchemy.String(255), + nullable=False, + default=lambda: "UNINITIALIZED", + ), + sqlalchemy.Column( + 'power_status', + sqlalchemy.String(255), + nullable=False, + default=lambda: "UNINITIALIZED", + ), sqlalchemy.Column('connection_info', sqlalchemy.Text), sqlalchemy.Column( - 'backup_writer_connection_info', sqlalchemy.Text, - nullable=True), - sqlalchemy.Column( - 'provider_properties', sqlalchemy.Text, - nullable=True), + 'backup_writer_connection_info', sqlalchemy.Text, nullable=True + ), + sqlalchemy.Column('provider_properties', sqlalchemy.Text, nullable=True), mysql_engine="InnoDB", - mysql_charset="utf8")) - - tables.append(sqlalchemy.Table( - 'minion_pool_event', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column('index', sqlalchemy.Integer, default=0), - sqlalchemy.Column("pool_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False), - sqlalchemy.Column("level", sqlalchemy.String(50), nullable=False), - sqlalchemy.Column("message", sqlalchemy.Text, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8')) + mysql_charset="utf8", + ) + ) tables.append( sqlalchemy.Table( - 'minion_pool_progress_update', meta, sqlalchemy.Column( - 'id', sqlalchemy.String(36), - primary_key=True, default=lambda: str(uuid.uuid4())), + 'minion_pool_event', + meta, + sqlalchemy.Column( + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), sqlalchemy.Column('created_at', sqlalchemy.DateTime), sqlalchemy.Column('updated_at', sqlalchemy.DateTime), sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('index', sqlalchemy.Integer, default=0), sqlalchemy.Column('deleted', sqlalchemy.String(36)), + sqlalchemy.Column('index', sqlalchemy.Integer, default=0), sqlalchemy.Column( - "pool_id", sqlalchemy.String(36), + "pool_id", + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False), - sqlalchemy.Column( - "current_step", sqlalchemy.BigInteger, nullable=False), + nullable=False, + ), + sqlalchemy.Column("level", sqlalchemy.String(50), nullable=False), + sqlalchemy.Column("message", sqlalchemy.Text, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + ) + + tables.append( + sqlalchemy.Table( + 'minion_pool_progress_update', + meta, sqlalchemy.Column( - "total_steps", sqlalchemy.BigInteger, nullable=True), + 'id', + sqlalchemy.String(36), + primary_key=True, + default=lambda: str(uuid.uuid4()), + ), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('index', sqlalchemy.Integer, default=0), + sqlalchemy.Column('deleted', sqlalchemy.String(36)), sqlalchemy.Column( - "message", sqlalchemy.Text, nullable=True), - mysql_engine='InnoDB', mysql_charset='utf8')) + "pool_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('minion_pool.id'), + nullable=False, + ), + sqlalchemy.Column("current_step", sqlalchemy.BigInteger, nullable=False), + sqlalchemy.Column("total_steps", sqlalchemy.BigInteger, nullable=True), + sqlalchemy.Column("message", sqlalchemy.Text, nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + ) # add the pool option properties for the transfer: origin_minion_pool_id = sqlalchemy.Column( - "origin_minion_pool_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + "origin_minion_pool_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('minion_pool.id'), + nullable=True, + ) destination_minion_pool_id = sqlalchemy.Column( - "destination_minion_pool_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + "destination_minion_pool_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('minion_pool.id'), + nullable=True, + ) instance_osmorphing_minion_pool_mappings = sqlalchemy.Column( - "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text, - nullable=False, default='{}') + "instance_osmorphing_minion_pool_mappings", + sqlalchemy.Text, + nullable=False, + default='{}', + ) created_columns = [] try: for index, table in enumerate(tables): table.create() for col in [ - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings]: + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + ]: base_transfer_action.create_column(col) created_columns.append(col) except Exception: diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py b/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py index f860fd72..8ca70941 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py @@ -10,10 +10,12 @@ def upgrade(migrate_engine): # add 'user_scripts' column to 'base_transfer_action': base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, + 'base_transfer_action', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) - user_scripts = sqlalchemy.Column( - "user_scripts", sqlalchemy.Text, nullable=True) + user_scripts = sqlalchemy.Column("user_scripts", sqlalchemy.Text, nullable=True) base_transfer.create_column(user_scripts) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py b/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py index 6e610146..2bbf6181 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py @@ -7,15 +7,20 @@ def upgrade(migrate_engine): task_event = sqlalchemy.Table('task_event', meta, autoload=True) event_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) + "index", sqlalchemy.Integer, default=0, nullable=False + ) task_event.create_column(event_index) task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, autoload=True, + 'task_progress_update', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) progress_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) + "index", sqlalchemy.Integer, default=0, nullable=False + ) task_progress_update.create_column(progress_index) task_progress_update.c.current_step.alter(type=sqlalchemy.BigInteger) task_progress_update.c.total_steps.alter(type=sqlalchemy.BigInteger) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py b/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py index edd16a5d..b6f86c71 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py @@ -9,12 +9,11 @@ def upgrade(migrate_engine): meta.bind = migrate_engine replica = sqlalchemy.Table( - 'replica', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") + 'replica', meta, autoload=True, mysql_engine="InnoDB", mysql_charset="utf8" + ) replica_scenario = sqlalchemy.Column( - "scenario", sqlalchemy.String(255), nullable=False, - default="replica") + "scenario", sqlalchemy.String(255), nullable=False, default="replica" + ) replica.create_column(replica_scenario) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py b/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py index a6bed3b3..bce8b528 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py @@ -9,24 +9,30 @@ def upgrade(migrate_engine): replica.rename('transfer') deployment = sqlalchemy.Table( - 'deployment', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), - sqlalchemy.Column("transfer_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('transfer.id'), - nullable=False), + 'deployment', + meta, + sqlalchemy.Column( + "id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + primary_key=True, + ), + sqlalchemy.Column( + "transfer_id", + sqlalchemy.String(36), + sqlalchemy.ForeignKey('transfer.id'), + nullable=False, + ), mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) try: deployment.create() except Exception: deployment.drop() raise - replica_schedule = sqlalchemy.Table( - 'replica_schedules', meta, autoload=True) + replica_schedule = sqlalchemy.Table('replica_schedules', meta, autoload=True) replica_schedule.rename('transfer_schedules') replica_schedule.c.replica_id.alter(name='transfer_id') @@ -35,16 +41,21 @@ def upgrade(migrate_engine): # Had to resort to using raw SQL statements. with migrate_engine.connect() as conn: conn.execute( - 'UPDATE base_transfer_action SET type = "transfer" ' - 'WHERE type = "replica";') - conn.execute('UPDATE tasks_execution SET type = "transfer_execution"' - 'WHERE type = "replica_execution"') + 'UPDATE base_transfer_action SET type = "transfer" WHERE type = "replica";' + ) + conn.execute( + 'UPDATE tasks_execution SET type = "transfer_execution"' + 'WHERE type = "replica_execution"' + ) conn.execute( 'UPDATE tasks_execution SET type = "transfer_disks_delete"' - 'WHERE type = "replica_disks_delete"') + 'WHERE type = "replica_disks_delete"' + ) conn.execute( 'UPDATE tasks_execution SET type = "deployment"' - 'WHERE type = "replica_deploy"') + 'WHERE type = "replica_deploy"' + ) conn.execute( 'UPDATE tasks_execution SET type = "transfer_update"' - 'WHERE type = "replica_update"') + 'WHERE type = "replica_update"' + ) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py b/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py index 95a5b168..6f4686f4 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py @@ -6,12 +6,17 @@ def upgrade(migrate_engine): meta.bind = migrate_engine base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, + 'base_transfer_action', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) clone_disks = sqlalchemy.Column( - "clone_disks", sqlalchemy.Boolean, nullable=False, default=True) + "clone_disks", sqlalchemy.Boolean, nullable=False, default=True + ) base_transfer.create_column(clone_disks) skip_os_morphing = sqlalchemy.Column( - "skip_os_morphing", sqlalchemy.Boolean, nullable=False, default=False) + "skip_os_morphing", sqlalchemy.Boolean, nullable=False, default=False + ) base_transfer.create_column(skip_os_morphing) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py b/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py index 63a7cfb0..994102d8 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py @@ -6,9 +6,13 @@ def upgrade(migrate_engine): meta.bind = migrate_engine transfer_schedule = sqlalchemy.Table( - 'transfer_schedules', meta, autoload=True, + 'transfer_schedules', + meta, + autoload=True, mysql_engine="InnoDB", - mysql_charset="utf8") + mysql_charset="utf8", + ) auto_deploy = sqlalchemy.Column( - 'auto_deploy', sqlalchemy.Boolean, nullable=False, default=False) + 'auto_deploy', sqlalchemy.Boolean, nullable=False, default=False + ) transfer_schedule.create_column(auto_deploy) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py b/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py index 37729f55..8214ad8b 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py +++ b/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py @@ -6,13 +6,10 @@ def upgrade(migrate_engine): meta.bind = migrate_engine deployment = sqlalchemy.Table( - 'deployment', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - deployer_id = sqlalchemy.Column( - 'deployer_id', sqlalchemy.String(36), nullable=True) - trust_id = sqlalchemy.Column( - 'trust_id', sqlalchemy.String(255), nullable=True) + 'deployment', meta, autoload=True, mysql_engine="InnoDB", mysql_charset="utf8" + ) + deployer_id = sqlalchemy.Column('deployer_id', sqlalchemy.String(36), nullable=True) + trust_id = sqlalchemy.Column('trust_id', sqlalchemy.String(255), nullable=True) created_columns = [] try: deployment.create_column(deployer_id) diff --git a/coriolis/db/sqlalchemy/migration.py b/coriolis/db/sqlalchemy/migration.py index f05ceea1..fae39855 100644 --- a/coriolis/db/sqlalchemy/migration.py +++ b/coriolis/db/sqlalchemy/migration.py @@ -9,19 +9,15 @@ def db_sync(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_sync(engine, path, version, - init_version=INIT_VERSION) + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'migrate_repo') + return oslo_migration.db_sync(engine, path, version, init_version=INIT_VERSION) def db_version(engine): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'migrate_repo') return oslo_migration.db_version(engine, path, INIT_VERSION) def db_version_control(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'migrate_repo') return oslo_migration.db_version_control(engine, path, version) diff --git a/coriolis/db/sqlalchemy/models.py b/coriolis/db/sqlalchemy/models.py index d4377999..861d84b7 100644 --- a/coriolis/db/sqlalchemy/models.py +++ b/coriolis/db/sqlalchemy/models.py @@ -3,11 +3,10 @@ import uuid -from oslo_db.sqlalchemy import models import sqlalchemy +from oslo_db.sqlalchemy import models +from sqlalchemy import orm, schema from sqlalchemy.ext import declarative -from sqlalchemy import orm -from sqlalchemy import schema from coriolis import constants from coriolis.db.sqlalchemy import types @@ -17,21 +16,20 @@ MAX_EVENT_MESSAGE_LENGHT = 1024 -class TaskEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin, - models.ModelBase): - +class TaskEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase): __tablename__ = 'task_event' - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) - task_id = sqlalchemy.Column(sqlalchemy.String(36), - sqlalchemy.ForeignKey('task.id'), - nullable=False) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) + task_id = sqlalchemy.Column( + sqlalchemy.String(36), sqlalchemy.ForeignKey('task.id'), nullable=False + ) level = sqlalchemy.Column(sqlalchemy.String(20), nullable=False) index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) message = sqlalchemy.Column( - sqlalchemy.String(MAX_EVENT_MESSAGE_LENGHT), nullable=False) + sqlalchemy.String(MAX_EVENT_MESSAGE_LENGHT), nullable=False + ) def to_dict(self): result = { @@ -48,16 +46,17 @@ def to_dict(self): return result -class MinionPoolEvent(BASE, models.TimestampMixin, models.SoftDeleteMixin, - models.ModelBase): +class MinionPoolEvent( + BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase +): __tablename__ = 'minion_pool_event' - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) - pool_id = sqlalchemy.Column(sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) + pool_id = sqlalchemy.Column( + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=False + ) level = sqlalchemy.Column(sqlalchemy.String(20), nullable=False) index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) message = sqlalchemy.Column(sqlalchemy.Text, nullable=False) @@ -77,18 +76,18 @@ def to_dict(self): return result -class TaskProgressUpdate(BASE, models.TimestampMixin, models.SoftDeleteMixin, - models.ModelBase): +class TaskProgressUpdate( + BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase +): __tablename__ = 'task_progress_update' - __table_args__ = ( - schema.UniqueConstraint("task_id", "index", "deleted"),) + __table_args__ = (schema.UniqueConstraint("task_id", "index", "deleted"),) - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) - task_id = sqlalchemy.Column(sqlalchemy.String(36), - sqlalchemy.ForeignKey('task.id'), - nullable=False) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) + task_id = sqlalchemy.Column( + sqlalchemy.String(36), sqlalchemy.ForeignKey('task.id'), nullable=False + ) index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) current_step = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False) @@ -112,17 +111,17 @@ def to_dict(self): class MinionPoolProgressUpdate( - BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase): + BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase +): __tablename__ = 'minion_pool_progress_update' - __table_args__ = ( - schema.UniqueConstraint("pool_id", "index", "deleted"),) - - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) - pool_id = sqlalchemy.Column(sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False) + __table_args__ = (schema.UniqueConstraint("pool_id", "index", "deleted"),) + + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) + pool_id = sqlalchemy.Column( + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=False + ) index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) current_step = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False) total_steps = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=True) @@ -144,16 +143,17 @@ def to_dict(self): return result -class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin, - models.ModelBase): +class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase): __tablename__ = 'task' - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) execution_id = sqlalchemy.Column( sqlalchemy.String(36), - sqlalchemy.ForeignKey('tasks_execution.id'), nullable=False) + sqlalchemy.ForeignKey('tasks_execution.id'), + nullable=False, + ) instance = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False) host = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True) process_id = sqlalchemy.Column(sqlalchemy.Integer, nullable=True) @@ -164,15 +164,19 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin, index = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) on_error = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False) # TODO(alexpilotti): Add soft delete filter - events = orm.relationship(TaskEvent, cascade="all,delete", - backref=orm.backref('task'), - order_by=TaskEvent.index) + events = orm.relationship( + TaskEvent, + cascade="all,delete", + backref=orm.backref('task'), + order_by=TaskEvent.index, + ) # TODO(alexpilotti): Add soft delete filter - progress_updates = orm.relationship(TaskProgressUpdate, - cascade="all,delete", - backref=orm.backref('task'), - order_by=( - TaskProgressUpdate.index)) + progress_updates = orm.relationship( + TaskProgressUpdate, + cascade="all,delete", + backref=orm.backref('task'), + order_by=(TaskProgressUpdate.index), + ) def to_dict(self): result = { @@ -199,24 +203,27 @@ def to_dict(self): result["events"].append(evt.to_dict()) for pgu in self.progress_updates: - result["progress_updates"].append( - pgu.to_dict()) + result["progress_updates"].append(pgu.to_dict()) return result -class TasksExecution(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class TasksExecution( + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = 'tasks_execution' - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) action_id = sqlalchemy.Column( sqlalchemy.String(36), - sqlalchemy.ForeignKey('base_transfer_action.base_id'), nullable=False) + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + nullable=False, + ) # TODO(alexpilotti): Add soft delete filter - tasks = orm.relationship(Task, cascade="all,delete", - backref=orm.backref('execution')) + tasks = orm.relationship( + Task, cascade="all,delete", backref=orm.backref('execution') + ) status = sqlalchemy.Column(sqlalchemy.String(100), nullable=False) number = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) type = sqlalchemy.Column(sqlalchemy.String(255)) @@ -239,52 +246,59 @@ def to_dict(self): return result -class BaseTransferAction(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class BaseTransferAction( + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = 'base_transfer_action' - base_id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + base_id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) destination_environment = sqlalchemy.Column(types.Json, nullable=True) type = sqlalchemy.Column(sqlalchemy.String(50)) - executions = orm.relationship(TasksExecution, cascade="all,delete", - backref=orm.backref('action'), - primaryjoin="and_(BaseTransferAction." - "base_id==TasksExecution.action_id, " - "TasksExecution.deleted=='0')") + executions = orm.relationship( + TasksExecution, + cascade="all,delete", + backref=orm.backref('action'), + primaryjoin="and_(BaseTransferAction." + "base_id==TasksExecution.action_id, " + "TasksExecution.deleted=='0')", + ) instances = sqlalchemy.Column(types.List, nullable=False) last_execution_status = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False, - default=lambda: constants.EXECUTION_STATUS_UNEXECUTED) + sqlalchemy.String(255), + nullable=False, + default=lambda: constants.EXECUTION_STATUS_UNEXECUTED, + ) reservation_id = sqlalchemy.Column(sqlalchemy.String(36), nullable=True) info = orm.deferred(sqlalchemy.Column(types.Bson, nullable=False)) notes = sqlalchemy.Column(sqlalchemy.Text, nullable=True) origin_endpoint_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False + ) destination_endpoint_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False + ) transfer_result = sqlalchemy.Column(types.Json, nullable=True) network_map = sqlalchemy.Column(types.Json, nullable=True) storage_mappings = sqlalchemy.Column(types.Json, nullable=True) source_environment = sqlalchemy.Column(types.Json, nullable=True) origin_minion_pool_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True + ) destination_minion_pool_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True + ) instance_osmorphing_minion_pool_mappings = sqlalchemy.Column( - types.Json, nullable=False, default=lambda: {}) + types.Json, nullable=False, default=lambda: {} + ) user_scripts = sqlalchemy.Column(types.Json, nullable=True) - clone_disks = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=True) + clone_disks = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True) skip_os_morphing = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=False) + sqlalchemy.Boolean, nullable=False, default=False + ) __mapper_args__ = { 'polymorphic_identity': 'base_transfer_action', @@ -315,8 +329,7 @@ def to_dict(self, include_task_info=True, include_executions=True): "deleted": self.deleted, "origin_minion_pool_id": self.origin_minion_pool_id, "destination_minion_pool_id": self.destination_minion_pool_id, - "instance_osmorphing_minion_pool_mappings": - self.instance_osmorphing_minion_pool_mappings, + "instance_osmorphing_minion_pool_mappings": self.instance_osmorphing_minion_pool_mappings, "user_scripts": self.user_scripts, "clone_disks": self.clone_disks, "skip_os_morphing": self.skip_os_morphing, @@ -334,11 +347,14 @@ class Transfer(BaseTransferAction): id = sqlalchemy.Column( sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), primary_key=True) + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + primary_key=True, + ) scenario = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False, - default=constants.TRANSFER_SCENARIO_REPLICA) + sqlalchemy.String(255), + nullable=False, + default=constants.TRANSFER_SCENARIO_REPLICA, + ) __mapper_args__ = { 'polymorphic_identity': 'transfer', @@ -346,11 +362,9 @@ class Transfer(BaseTransferAction): def to_dict(self, include_task_info=True, include_executions=True): base = super(Transfer, self).to_dict( - include_task_info=include_task_info, - include_executions=include_executions) - base.update({ - "id": self.id, - "scenario": self.scenario}) + include_task_info=include_task_info, include_executions=include_executions + ) + base.update({"id": self.id, "scenario": self.scenario}) return base @@ -359,14 +373,15 @@ class Deployment(BaseTransferAction): id = sqlalchemy.Column( sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), primary_key=True) + sqlalchemy.ForeignKey('base_transfer_action.base_id'), + primary_key=True, + ) transfer_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('transfer.id'), nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('transfer.id'), nullable=False + ) transfer = orm.relationship( - Transfer, backref=orm.backref("deployments"), - foreign_keys=[transfer_id]) + Transfer, backref=orm.backref("deployments"), foreign_keys=[transfer_id] + ) deployer_id = sqlalchemy.Column(sqlalchemy.String(36), nullable=True) trust_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) @@ -376,160 +391,155 @@ class Deployment(BaseTransferAction): def to_dict(self, include_task_info=True, include_tasks=True): base = super(Deployment, self).to_dict( - include_task_info=include_task_info, - include_executions=include_tasks) - - base.update({ - "id": self.id, - "transfer_id": self.transfer_id, - "transfer_scenario_type": self.transfer.scenario, - "deployer_id": self.deployer_id, - "trust_id": self.trust_id, - }) + include_task_info=include_task_info, include_executions=include_tasks + ) + + base.update( + { + "id": self.id, + "transfer_id": self.transfer_id, + "transfer_scenario_type": self.transfer.scenario, + "deployer_id": self.deployer_id, + "trust_id": self.trust_id, + } + ) return base class ServiceRegionMapping( - BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = "service_region_mapping" id = sqlalchemy.Column( sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), nullable=False, - primary_key=True) + primary_key=True, + ) service_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('service.id'), - nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('service.id'), nullable=False + ) region_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('region.id'), - nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('region.id'), nullable=False + ) -class Service(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class Service(BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): __tablename__ = "service" __table_args__ = ( - schema.UniqueConstraint("host", "topic", "deleted", - name="uniq_services0host0topic0deleted"), - schema.UniqueConstraint("host", "binary", "deleted", - name="uniq_services0host0binary0deleted")) + schema.UniqueConstraint( + "host", "topic", "deleted", name="uniq_services0host0topic0deleted" + ), + schema.UniqueConstraint( + "host", "binary", "deleted", name="uniq_services0host0binary0deleted" + ), + ) id = sqlalchemy.Column( - sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), - primary_key=True) - - host = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) - binary = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) - topic = sqlalchemy.Column( - sqlalchemy.String(255), nullable=True, default=None) + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) + + host = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + binary = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + topic = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, default=None) enabled = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=lambda: False) + sqlalchemy.Boolean, nullable=False, default=lambda: False + ) status = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False, - default=lambda: constants.SERVICE_STATUS_UNKNOWN) + sqlalchemy.String(255), + nullable=False, + default=lambda: constants.SERVICE_STATUS_UNKNOWN, + ) providers = sqlalchemy.Column(types.Json(), nullable=True) specs = sqlalchemy.Column(types.Json(), nullable=True) mapped_regions = orm.relationship( - 'Region', back_populates='mapped_services', - secondary="service_region_mapping") + 'Region', back_populates='mapped_services', secondary="service_region_mapping" + ) class EndpointRegionMapping( - BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = "endpoint_region_mapping" id = sqlalchemy.Column( sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), nullable=False, - primary_key=True) + primary_key=True, + ) endpoint_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), - nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False + ) region_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('region.id'), - nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('region.id'), nullable=False + ) -class Region( - BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): +class Region(BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): __tablename__ = "region" id = sqlalchemy.Column( sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), nullable=False, - primary_key=True) + primary_key=True, + ) - name = sqlalchemy.Column( - sqlalchemy.String(255), - nullable=False) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) - description = sqlalchemy.Column( - sqlalchemy.String(1024), - nullable=True) + description = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True) enabled = sqlalchemy.Column( - sqlalchemy.Boolean, - default=lambda: False, - nullable=False) + sqlalchemy.Boolean, default=lambda: False, nullable=False + ) mapped_endpoints = orm.relationship( - 'Endpoint', back_populates='mapped_regions', - secondary="endpoint_region_mapping") + 'Endpoint', back_populates='mapped_regions', secondary="endpoint_region_mapping" + ) mapped_services = orm.relationship( - 'Service', back_populates='mapped_regions', - secondary="service_region_mapping") + 'Service', back_populates='mapped_regions', secondary="service_region_mapping" + ) -class MinionMachine(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class MinionMachine( + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = "minion_machine" - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) pool_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('minion_pool.id'), - nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=False + ) allocation_status = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False, - default=lambda: constants.MINION_MACHINE_STATUS_UNINITIALIZED) + sqlalchemy.String(255), + nullable=False, + default=lambda: constants.MINION_MACHINE_STATUS_UNINITIALIZED, + ) - allocated_action = sqlalchemy.Column( - sqlalchemy.String(36), nullable=True) + allocated_action = sqlalchemy.Column(sqlalchemy.String(36), nullable=True) - power_status = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) + power_status = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) - last_used_at = sqlalchemy.Column( - sqlalchemy.types.DateTime, nullable=True) + last_used_at = sqlalchemy.Column(sqlalchemy.types.DateTime, nullable=True) - connection_info = sqlalchemy.Column( - types.Json, nullable=True) + connection_info = sqlalchemy.Column(types.Json, nullable=True) - backup_writer_connection_info = sqlalchemy.Column( - types.Json, nullable=True) + backup_writer_connection_info = sqlalchemy.Column(types.Json, nullable=True) - provider_properties = sqlalchemy.Column( - types.Json, nullable=True) + provider_properties = sqlalchemy.Column(types.Json, nullable=True) def to_dict(self): result = { @@ -546,68 +556,63 @@ def to_dict(self): "connection_info": self.connection_info, "allocated_action": self.allocated_action, "last_used_at": self.last_used_at, - "backup_writer_connection_info": ( - self.backup_writer_connection_info), - "provider_properties": self.provider_properties + "backup_writer_connection_info": (self.backup_writer_connection_info), + "provider_properties": self.provider_properties, } return result -class MinionPool( - BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class MinionPool(BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): __tablename__ = 'minion_pool' - id = sqlalchemy.Column( - sqlalchemy.String(36), - primary_key=True) + id = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True) user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) - maintenance_trust_id = sqlalchemy.Column( - sqlalchemy.String(255), nullable=True) + maintenance_trust_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) - name = sqlalchemy.Column( - sqlalchemy.String(255), - nullable=False) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) notes = sqlalchemy.Column(sqlalchemy.Text, nullable=True) endpoint_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('endpoint.id'), nullable=False) - os_type = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) - platform = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False + ) + os_type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + platform = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) environment_options = sqlalchemy.Column(types.Json, nullable=True) status = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False, - default=lambda: constants.MINION_POOL_STATUS_UNKNOWN) - shared_resources = sqlalchemy.Column( - types.Json, nullable=True) - minimum_minions = sqlalchemy.Column( - sqlalchemy.Integer, nullable=False) - maximum_minions = sqlalchemy.Column( - sqlalchemy.Integer, nullable=False) - minion_max_idle_time = sqlalchemy.Column( - sqlalchemy.Integer, nullable=False) + sqlalchemy.String(255), + nullable=False, + default=lambda: constants.MINION_POOL_STATUS_UNKNOWN, + ) + shared_resources = sqlalchemy.Column(types.Json, nullable=True) + minimum_minions = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) + maximum_minions = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) + minion_max_idle_time = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) minion_retention_strategy = sqlalchemy.Column( - sqlalchemy.String(255), nullable=False) + sqlalchemy.String(255), nullable=False + ) minion_machines = orm.relationship( - MinionMachine, backref=orm.backref('minion_pool'), + MinionMachine, + backref=orm.backref('minion_pool'), primaryjoin="and_(MinionMachine.pool_id==MinionPool.id, " - "MinionMachine.deleted=='0')") - events = orm.relationship(MinionPoolEvent, cascade="all,delete", - backref=orm.backref('minion_pool'), - order_by=MinionPoolEvent.index) - progress_updates = orm.relationship(MinionPoolProgressUpdate, - cascade="all,delete", - backref=orm.backref('minion_pool'), - order_by=( - MinionPoolProgressUpdate.index)) + "MinionMachine.deleted=='0')", + ) + events = orm.relationship( + MinionPoolEvent, + cascade="all,delete", + backref=orm.backref('minion_pool'), + order_by=MinionPoolEvent.index, + ) + progress_updates = orm.relationship( + MinionPoolProgressUpdate, + cascade="all,delete", + backref=orm.backref('minion_pool'), + order_by=(MinionPoolProgressUpdate.index), + ) def to_dict( - self, include_machines=True, include_events=True, - include_progress_updates=True): + self, include_machines=True, include_events=True, include_progress_updates=True + ): base = { "id": self.id, "name": self.name, @@ -626,27 +631,26 @@ def to_dict( "minimum_minions": self.minimum_minions, "maximum_minions": self.maximum_minions, "minion_max_idle_time": self.minion_max_idle_time, - "minion_retention_strategy": self.minion_retention_strategy} + "minion_retention_strategy": self.minion_retention_strategy, + } base["minion_machines"] = [] if include_machines: base["minion_machines"] = [ - machine.to_dict() for machine in self.minion_machines] + machine.to_dict() for machine in self.minion_machines + ] if include_events: - base["events"] = [ - ev.to_dict() for ev in self.events] + base["events"] = [ev.to_dict() for ev in self.events] if include_progress_updates: - base["progress_updates"] = [ - pu.to_dict() for pu in self.progress_updates] + base["progress_updates"] = [pu.to_dict() for pu in self.progress_updates] return base -class Endpoint(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class Endpoint(BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin): __tablename__ = 'endpoint' - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) project_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) connection_info = sqlalchemy.Column(types.Json, nullable=False) @@ -654,41 +658,49 @@ class Endpoint(BASE, models.TimestampMixin, models.ModelBase, name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) description = sqlalchemy.Column(sqlalchemy.String(1024), nullable=True) origin_actions = orm.relationship( - BaseTransferAction, backref=orm.backref('origin_endpoint'), + BaseTransferAction, + backref=orm.backref('origin_endpoint'), primaryjoin="and_(BaseTransferAction.origin_endpoint_id==Endpoint.id, " - "BaseTransferAction.deleted=='0')") + "BaseTransferAction.deleted=='0')", + ) destination_actions = orm.relationship( - BaseTransferAction, backref=orm.backref('destination_endpoint'), + BaseTransferAction, + backref=orm.backref('destination_endpoint'), primaryjoin="and_(BaseTransferAction.destination_endpoint_id==" - "Endpoint.id, BaseTransferAction.deleted=='0')") + "Endpoint.id, BaseTransferAction.deleted=='0')", + ) minion_pools = orm.relationship( - MinionPool, backref=orm.backref('endpoint'), + MinionPool, + backref=orm.backref('endpoint'), primaryjoin="and_(MinionPool.endpoint_id==" - "Endpoint.id, MinionPool.deleted=='0')") + "Endpoint.id, MinionPool.deleted=='0')", + ) mapped_regions = orm.relationship( - 'Region', back_populates='mapped_endpoints', - secondary="endpoint_region_mapping") + 'Region', back_populates='mapped_endpoints', secondary="endpoint_region_mapping" + ) -class TransferSchedule(BASE, models.TimestampMixin, models.ModelBase, - models.SoftDeleteMixin): +class TransferSchedule( + BASE, models.TimestampMixin, models.ModelBase, models.SoftDeleteMixin +): __tablename__ = "transfer_schedules" - id = sqlalchemy.Column(sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), - primary_key=True) + id = sqlalchemy.Column( + sqlalchemy.String(36), default=lambda: str(uuid.uuid4()), primary_key=True + ) transfer_id = sqlalchemy.Column( - sqlalchemy.String(36), - sqlalchemy.ForeignKey('transfer.id'), nullable=False) + sqlalchemy.String(36), sqlalchemy.ForeignKey('transfer.id'), nullable=False + ) transfer = orm.relationship( - Transfer, backref=orm.backref("schedules"), foreign_keys=[transfer_id]) + Transfer, backref=orm.backref("schedules"), foreign_keys=[transfer_id] + ) schedule = sqlalchemy.Column(types.Json, nullable=False) - expiration_date = sqlalchemy.Column( - sqlalchemy.types.DateTime, nullable=True) + expiration_date = sqlalchemy.Column(sqlalchemy.types.DateTime, nullable=True) enabled = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=lambda: False) + sqlalchemy.Boolean, nullable=False, default=lambda: False + ) shutdown_instance = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=False) - auto_deploy = sqlalchemy.Column( - sqlalchemy.Boolean, nullable=False, default=False) + sqlalchemy.Boolean, nullable=False, default=False + ) + auto_deploy = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False) trust_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) diff --git a/coriolis/db/sqlalchemy/types.py b/coriolis/db/sqlalchemy/types.py index 2c150c0b..2d3fcc60 100644 --- a/coriolis/db/sqlalchemy/types.py +++ b/coriolis/db/sqlalchemy/types.py @@ -17,8 +17,8 @@ import zlib from oslo_serialization import jsonutils -from sqlalchemy.dialects import mysql from sqlalchemy import types +from sqlalchemy.dialects import mysql class LongText(types.TypeDecorator): @@ -42,7 +42,6 @@ def load_dialect_impl(self, dialect): class Json(LongText): - def process_bind_param(self, value, dialect): return jsonutils.dumps(value) @@ -53,10 +52,8 @@ def process_result_value(self, value, dialect): class Bson(Blob): - def process_bind_param(self, value, dialect): - return zlib.compress( - jsonutils.dumps(value).encode('utf-8')) + return zlib.compress(jsonutils.dumps(value).encode('utf-8')) def process_result_value(self, value, dialect): if value is None: diff --git a/coriolis/deployer_manager/rpc/client.py b/coriolis/deployer_manager/rpc/client.py index 23294200..267c07c3 100644 --- a/coriolis/deployer_manager/rpc/client.py +++ b/coriolis/deployer_manager/rpc/client.py @@ -1,36 +1,37 @@ # Copyright 2025 Cloudbase Solutions Srl # All Rights Reserved. -from oslo_config import cfg import oslo_messaging as messaging +from oslo_config import cfg -from coriolis import constants -from coriolis import rpc +from coriolis import constants, rpc VERSION = "1.0" deployer_manager_opts = [ cfg.IntOpt( 'deployer_manager_rpc_timeout', - help="Number of seconds until RPC calls to the deployer manager " - "timeout.")] + help="Number of seconds until RPC calls to the deployer manager timeout.", + ) +] CONF = cfg.CONF CONF.register_opts(deployer_manager_opts, 'deployer_manager') class DeployerManagerClient(rpc.BaseRPCClient): - - def __init__( - self, timeout=None): + def __init__(self, timeout=None): target = messaging.Target( - topic=constants.DEPLOYER_MANAGER_MAIN_MESSAGING_TOPIC, - version=VERSION) + topic=constants.DEPLOYER_MANAGER_MAIN_MESSAGING_TOPIC, version=VERSION + ) if timeout is None: timeout = CONF.deployer_manager.deployer_manager_rpc_timeout super(DeployerManagerClient, self).__init__(target, timeout=timeout) - def execute_auto_deployment( - self, ctxt, transfer_id, deployer_id, **kwargs): + def execute_auto_deployment(self, ctxt, transfer_id, deployer_id, **kwargs): self._cast( - ctxt, 'execute_auto_deployment', transfer_id=transfer_id, - deployer_id=deployer_id, **kwargs) + ctxt, + 'execute_auto_deployment', + transfer_id=transfer_id, + deployer_id=deployer_id, + **kwargs, + ) diff --git a/coriolis/deployer_manager/rpc/server.py b/coriolis/deployer_manager/rpc/server.py index 21f59696..fd0b65ec 100644 --- a/coriolis/deployer_manager/rpc/server.py +++ b/coriolis/deployer_manager/rpc/server.py @@ -6,12 +6,8 @@ from oslo_config import cfg from oslo_log import log as logging +from coriolis import constants, context, exception, keystone, utils from coriolis.conductor.rpc import client as rpc_conductor_client -from coriolis import constants -from coriolis import context -from coriolis import exception -from coriolis import keystone -from coriolis import utils CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -20,7 +16,6 @@ class DeployerManagerServerEndpoint: - def __init__(self): self._admin_ctx = context.get_admin_context() self._conductor_client_instance = None @@ -29,35 +24,36 @@ def __init__(self): @property def _rpc_conductor_client(self): if not getattr(self, '_conductor_client_instance', None): - self._conductor_client_instance = ( - rpc_conductor_client.ConductorClient()) + self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance def _wait_for_pending_deployment_status_change(self, deployment_id): pending = constants.EXECUTION_STATUS_PENDING LOG.info( - f"Waiting for deployment '{deployment_id}' to be out of " - f"'{pending}' status") + f"Waiting for deployment '{deployment_id}' to be out of '{pending}' status" + ) i = 0 max_retries = 60 while i < max_retries: deployment = self._rpc_conductor_client.get_deployment( - self._admin_ctx, deployment_id) + self._admin_ctx, deployment_id + ) deployment_status = deployment['last_execution_status'] if deployment_status != pending: LOG.info( f"Deployment '{deployment_id}' changed into " - f"'{deployment_status}' and not {pending} anymore.") + f"'{deployment_status}' and not {pending} anymore." + ) return - LOG.info( - f"Deployment '{deployment_id}' is still '{deployment_status}'") + LOG.info(f"Deployment '{deployment_id}' is still '{deployment_status}'") i += 1 time.sleep(1) raise exception.InvalidDeploymentState( f"Timed out waiting for deployment '{deployment_id}' to be out of " - f"'{pending}' status.") + f"'{pending}' status." + ) def _check_deployer_status(self, deployment_id): active_statuses = [ @@ -75,73 +71,83 @@ def _check_deployer_status(self, deployment_id): ] try: deployment = self._rpc_conductor_client.get_deployment( - self._admin_ctx, deployment_id) + self._admin_ctx, deployment_id + ) deployer_id = deployment.get('deployer_id') transfer_id = deployment.get('transfer_id') if not deployer_id: raise exception.InvalidDeploymentState( f"Deployment '{deployment['id']}' is in {PENDING_STATUS} " - f"status, without any deployer execution registered.") + f"status, without any deployer execution registered." + ) deployer_execution = ( self._rpc_conductor_client.get_transfer_tasks_execution( - self._admin_ctx, transfer_id, deployer_id)) - LOG.debug( - f"Waiting for deployer '{deployer_id}' to complete.") + self._admin_ctx, transfer_id, deployer_id + ) + ) + LOG.debug(f"Waiting for deployer '{deployer_id}' to complete.") ex_status = deployer_execution['status'] LOG.debug(f"Deployer '{deployer_id}' status is {ex_status}") if ex_status in active_statuses: return elif ex_status == constants.EXECUTION_STATUS_COMPLETED: - LOG.debug( - f"Confirming deployer '{deployer_id}' completed.") - admin_ctx = context.get_admin_context( - trust_id=deployment['trust_id']) + LOG.debug(f"Confirming deployer '{deployer_id}' completed.") + admin_ctx = context.get_admin_context(trust_id=deployment['trust_id']) admin_ctx.delete_trust_id = True self._rpc_conductor_client.confirm_deployer_completed( - admin_ctx, deployment['id'], force=False) - return self._wait_for_pending_deployment_status_change( - deployment_id) + admin_ctx, deployment['id'], force=False + ) + return self._wait_for_pending_deployment_status_change(deployment_id) else: if ex_status in error_statuses: raise exception.InvalidTransferState( f"Got status '{ex_status}' for execution with ID " - f"'{deployer_id}'. Deployment cannot occur.") + f"'{deployer_id}'. Deployment cannot occur." + ) else: raise exception.InvalidTransferState( f"Deployer with ID '{deployer_id}' is in invalid " - f"state '{ex_status}'. Deployment cannot occur.") + f"state '{ex_status}'. Deployment cannot occur." + ) except BaseException as ex: LOG.error( f"Reporting deployer failure for deployment " f"'{deployment_id}'. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) self._rpc_conductor_client.report_deployer_failure( - self._admin_ctx, deployment_id, str(ex)) + self._admin_ctx, deployment_id, str(ex) + ) def _loop(self): while True: try: deployments = self._rpc_conductor_client.get_deployments( - self._admin_ctx, include_tasks=False, - include_task_info=False) + self._admin_ctx, include_tasks=False, include_task_info=False + ) for d in deployments: if d['last_execution_status'] == PENDING_STATUS: self._check_deployer_status(d['id']) except Exception: LOG.warning( f"Deployer manager failed to list pending deployments. " - f"Error was: {utils.get_exception_details()}") + f"Error was: {utils.get_exception_details()}" + ) time.sleep(10) def _init_loop(self): utils.start_thread(self._loop) - def execute_auto_deployment( - self, ctxt, transfer_id, deployer_id, **kwargs): + def execute_auto_deployment(self, ctxt, transfer_id, deployer_id, **kwargs): LOG.debug( f"Creating deployment for deployer ID '{deployer_id}' of transfer " - f"'{transfer_id}'") + f"'{transfer_id}'" + ) keystone.create_trust(ctxt) self._rpc_conductor_client.deploy_transfer_instances( - ctxt, transfer_id, wait_for_execution=deployer_id, - trust_id=ctxt.trust_id, **kwargs) + ctxt, + transfer_id, + wait_for_execution=deployer_id, + trust_id=ctxt.trust_id, + **kwargs, + ) diff --git a/coriolis/deployments/api.py b/coriolis/deployments/api.py index 695665fd..0db88884 100644 --- a/coriolis/deployments/api.py +++ b/coriolis/deployments/api.py @@ -8,16 +8,27 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def deploy_transfer_instances(self, ctxt, transfer_id, - instance_osmorphing_minion_pool_mappings, - clone_disks=False, force=False, - skip_os_morphing=False, user_scripts=None): + def deploy_transfer_instances( + self, + ctxt, + transfer_id, + instance_osmorphing_minion_pool_mappings, + clone_disks=False, + force=False, + skip_os_morphing=False, + user_scripts=None, + ): return self._rpc_client.deploy_transfer_instances( - ctxt, transfer_id, instance_osmorphing_minion_pool_mappings=( - instance_osmorphing_minion_pool_mappings), - clone_disks=clone_disks, force=force, + ctxt, + transfer_id, + instance_osmorphing_minion_pool_mappings=( + instance_osmorphing_minion_pool_mappings + ), + clone_disks=clone_disks, + force=force, skip_os_morphing=skip_os_morphing, - user_scripts=user_scripts) + user_scripts=user_scripts, + ) def delete(self, ctxt, deployment_id): self._rpc_client.delete_deployment(ctxt, deployment_id) @@ -25,15 +36,27 @@ def delete(self, ctxt, deployment_id): def cancel(self, ctxt, deployment_id, force): self._rpc_client.cancel_deployment(ctxt, deployment_id, force) - def get_deployments(self, ctxt, include_tasks=False, - include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): + def get_deployments( + self, + ctxt, + include_tasks=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return self._rpc_client.get_deployments( - ctxt, include_tasks, include_task_info=include_task_info, - marker=marker, limit=limit, - sort_keys=sort_keys, sort_dirs=sort_dirs) + ctxt, + include_tasks, + include_task_info=include_task_info, + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + ) def get_deployment(self, ctxt, deployment_id, include_task_info=False): return self._rpc_client.get_deployment( - ctxt, deployment_id, include_task_info=include_task_info) + ctxt, deployment_id, include_task_info=include_task_info + ) diff --git a/coriolis/diagnostics/api.py b/coriolis/diagnostics/api.py index aaa865a8..a6e70384 100644 --- a/coriolis/diagnostics/api.py +++ b/coriolis/diagnostics/api.py @@ -1,9 +1,9 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. +from coriolis import utils from coriolis.conductor.rpc import client as conductor_rpc from coriolis.transfer_cron.rpc import client as cron_rpc -from coriolis import utils from coriolis.worker.rpc import client as worker_rpc diff --git a/coriolis/endpoint_options/api.py b/coriolis/endpoint_options/api.py index 21fe1842..1828a0ef 100644 --- a/coriolis/endpoint_options/api.py +++ b/coriolis/endpoint_options/api.py @@ -8,27 +8,34 @@ class API(object): def __init__(self): self._rpc_minion_manager_client = ( - rpc_minion_manager_client.MinionManagerClient()) + rpc_minion_manager_client.MinionManagerClient() + ) self._rpc_conductor_client = rpc_conductor_client.ConductorClient() def get_endpoint_source_options( - self, ctxt, endpoint_id, env=None, option_names=None): + self, ctxt, endpoint_id, env=None, option_names=None + ): return self._rpc_conductor_client.get_endpoint_source_options( - ctxt, endpoint_id, env, option_names) + ctxt, endpoint_id, env, option_names + ) def get_endpoint_destination_options( - self, ctxt, endpoint_id, env=None, option_names=None): + self, ctxt, endpoint_id, env=None, option_names=None + ): return self._rpc_conductor_client.get_endpoint_destination_options( - ctxt, endpoint_id, env, option_names) + ctxt, endpoint_id, env, option_names + ) def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env=None, option_names=None): - return (self._rpc_minion_manager_client. - get_endpoint_source_minion_pool_options)( - ctxt, endpoint_id, env, option_names) + self, ctxt, endpoint_id, env=None, option_names=None + ): + return ( + self._rpc_minion_manager_client.get_endpoint_source_minion_pool_options + )(ctxt, endpoint_id, env, option_names) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env=None, option_names=None): - return (self._rpc_minion_manager_client. - get_endpoint_destination_minion_pool_options)( - ctxt, endpoint_id, env, option_names) + self, ctxt, endpoint_id, env=None, option_names=None + ): + return ( + self._rpc_minion_manager_client.get_endpoint_destination_minion_pool_options + )(ctxt, endpoint_id, env, option_names) diff --git a/coriolis/endpoint_resources/api.py b/coriolis/endpoint_resources/api.py index 59b7c182..94d952ea 100644 --- a/coriolis/endpoint_resources/api.py +++ b/coriolis/endpoint_resources/api.py @@ -8,27 +8,40 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def get_endpoint_instances(self, ctxt, endpoint_id, source_environment, - marker=None, limit=None, - instance_name_pattern=None, refresh=False): + def get_endpoint_instances( + self, + ctxt, + endpoint_id, + source_environment, + marker=None, + limit=None, + instance_name_pattern=None, + refresh=False, + ): return self._rpc_client.get_endpoint_instances( - ctxt, endpoint_id, source_environment, marker, - limit, instance_name_pattern, refresh=refresh) + ctxt, + endpoint_id, + source_environment, + marker, + limit, + instance_name_pattern, + refresh=refresh, + ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name): + self, ctxt, endpoint_id, source_environment, instance_name + ): return self._rpc_client.get_endpoint_instance( - ctxt, endpoint_id, source_environment, instance_name) + ctxt, endpoint_id, source_environment, instance_name + ) def get_endpoint_networks(self, ctxt, endpoint_id, env): - return self._rpc_client.get_endpoint_networks( - ctxt, endpoint_id, env) + return self._rpc_client.get_endpoint_networks(ctxt, endpoint_id, env) def get_endpoint_storage(self, ctxt, endpoint_id, env): - return self._rpc_client.get_endpoint_storage( - ctxt, endpoint_id, env) + return self._rpc_client.get_endpoint_storage(ctxt, endpoint_id, env) - def get_endpoint_inventory_csv( - self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): return self._rpc_client.get_endpoint_inventory_csv( - ctxt, endpoint_id, source_environment) + ctxt, endpoint_id, source_environment + ) diff --git a/coriolis/endpoints/api.py b/coriolis/endpoints/api.py index 887abbd2..7b5b0533 100644 --- a/coriolis/endpoints/api.py +++ b/coriolis/endpoints/api.py @@ -1,26 +1,27 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from coriolis import utils from coriolis.conductor.rpc import client as rpc_conductor_client from coriolis.minion_manager.rpc import client as rpc_minion_manager_client -from coriolis import utils class API(object): def __init__(self): self._rpc_conductor_client = rpc_conductor_client.ConductorClient() self._rpc_minion_manager_client = ( - rpc_minion_manager_client.MinionManagerClient()) + rpc_minion_manager_client.MinionManagerClient() + ) - def create(self, ctxt, name, endpoint_type, description, - connection_info, mapped_regions): + def create( + self, ctxt, name, endpoint_type, description, connection_info, mapped_regions + ): return self._rpc_conductor_client.create_endpoint( - ctxt, name, endpoint_type, description, connection_info, - mapped_regions) + ctxt, name, endpoint_type, description, connection_info, mapped_regions + ) def update(self, ctxt, endpoint_id, properties): - return self._rpc_conductor_client.update_endpoint( - ctxt, endpoint_id, properties) + return self._rpc_conductor_client.update_endpoint(ctxt, endpoint_id, properties) def delete(self, ctxt, endpoint_id): self._rpc_conductor_client.delete_endpoint(ctxt, endpoint_id) @@ -33,29 +34,33 @@ def get_endpoint(self, ctxt, endpoint_id): def validate_connection(self, ctxt, endpoint_id): return self._rpc_conductor_client.validate_endpoint_connection( - ctxt, endpoint_id) + ctxt, endpoint_id + ) @utils.bad_request_on_error("Invalid destination environment: %s") def validate_target_environment(self, ctxt, endpoint_id, target_env): return self._rpc_conductor_client.validate_endpoint_target_environment( - ctxt, endpoint_id, target_env) + ctxt, endpoint_id, target_env + ) @utils.bad_request_on_error("Invalid source environment: %s") def validate_source_environment(self, ctxt, endpoint_id, source_env): return self._rpc_conductor_client.validate_endpoint_source_environment( - ctxt, endpoint_id, source_env) + ctxt, endpoint_id, source_env + ) @utils.bad_request_on_error("Invalid source minion pool environment: %s") def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): - return (self._rpc_minion_manager_client. - validate_endpoint_source_minion_pool_options)( - ctxt, endpoint_id, pool_environment) + self, ctxt, endpoint_id, pool_environment + ): + return ( + self._rpc_minion_manager_client.validate_endpoint_source_minion_pool_options + )(ctxt, endpoint_id, pool_environment) - @utils.bad_request_on_error( - "Invalid destination minion pool environment: %s") + @utils.bad_request_on_error("Invalid destination minion pool environment: %s") def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): - return (self._rpc_minion_manager_client. - validate_endpoint_destination_minion_pool_options)( - ctxt, endpoint_id, pool_environment) + self, ctxt, endpoint_id, pool_environment + ): + return ( + self._rpc_minion_manager_client.validate_endpoint_destination_minion_pool_options + )(ctxt, endpoint_id, pool_environment) diff --git a/coriolis/events.py b/coriolis/events.py index f9257d21..93e6de51 100644 --- a/coriolis/events.py +++ b/coriolis/events.py @@ -10,15 +10,14 @@ from coriolis import constants - LOG = logging.getLogger(__name__) _PercStepData = collections.namedtuple( - "_PercStepData", "progress_update_id last_perc last_value total_steps") + "_PercStepData", "progress_update_id last_perc last_value total_steps" +) class EventManager(object, with_metaclass(abc.ABCMeta)): - def __init__(self, event_handler): self._event_handler = event_handler self._perc_steps = {} @@ -28,15 +27,14 @@ def _call_event_handler(self, method_name, *args, **kwargs): method_obj = getattr(self._event_handler, str(method_name), None) if not method_obj: raise AttributeError( - "No method named '%s' for event handler of type '%s'." % ( - method_name, type(self._event_handler))) + "No method named '%s' for event handler of type '%s'." + % (method_name, type(self._event_handler)) + ) return method_obj(*args, **kwargs) def add_percentage_step(self, message, total_steps, initial_step=0): if total_steps < 0: - LOG.warn( - "Max percentage value was negative (%s). Reset to 0", - total_steps) + LOG.warn("Max percentage value was negative (%s). Reset to 0", total_steps) total_steps = 0 if total_steps == 0: LOG.warn("Max percentage value set to 0 (zero)") @@ -44,31 +42,37 @@ def add_percentage_step(self, message, total_steps, initial_step=0): if initial_step > total_steps: raise ValueError( "Provided percent step initial value '%s' is larger than the " - "maximum value '%s'" % (initial_step, total_steps)) + "maximum value '%s'" % (initial_step, total_steps) + ) progress_update = self._call_event_handler( - 'add_progress_update', message, initial_step=initial_step, - total_steps=total_steps, return_event=True) - progress_update_id = ( - self._call_event_handler( - 'get_progress_update_identifier', progress_update)) + 'add_progress_update', + message, + initial_step=initial_step, + total_steps=total_steps, + return_event=True, + ) + progress_update_id = self._call_event_handler( + 'get_progress_update_identifier', progress_update + ) perc = 0 if initial_step > 0 and total_steps > 0: perc = int(initial_step * 100 // total_steps) self._perc_steps[progress_update_id] = _PercStepData( - progress_update_id, perc, initial_step, total_steps) + progress_update_id, perc, initial_step, total_steps + ) return self._perc_steps[progress_update_id] def set_percentage_step(self, step, new_current_step): - perc_step = self._perc_steps.get( - step.progress_update_id, None) + perc_step = self._perc_steps.get(step.progress_update_id, None) if perc_step is None: return if perc_step.last_value > new_current_step: - LOG.warn("rollback for perc update %s not allowed" % - step.progress_update_id) + LOG.warn( + "rollback for perc update %s not allowed" % step.progress_update_id + ) return perc = 0 @@ -77,53 +81,57 @@ def set_percentage_step(self, step, new_current_step): if self._call_event_handler and perc > perc_step.last_perc: sync_final = ( - perc_step.total_steps > 0 and - new_current_step >= perc_step.total_steps) + perc_step.total_steps > 0 and new_current_step >= perc_step.total_steps + ) self._call_event_handler( - 'update_progress_update', step.progress_update_id, - new_current_step, sync=sync_final) + 'update_progress_update', + step.progress_update_id, + new_current_step, + sync=sync_final, + ) perc_id = copy.copy(step.progress_update_id) total_steps = perc_step.total_steps del self._perc_steps[step.progress_update_id] del perc_step - self._perc_steps[perc_id] = _PercStepData( - perc_id, perc, 0, total_steps) + self._perc_steps[perc_id] = _PercStepData(perc_id, perc, 0, total_steps) def progress_update(self, message): - self._call_event_handler( - 'add_progress_update', message, return_event=False) + self._call_event_handler('add_progress_update', message, return_event=False) def info(self, message): - self._call_event_handler( - 'add_event', message, level=constants.TASK_EVENT_INFO) + self._call_event_handler('add_event', message, level=constants.TASK_EVENT_INFO) def warn(self, message): self._call_event_handler( - 'add_event', message, level=constants.TASK_EVENT_WARNING) + 'add_event', message, level=constants.TASK_EVENT_WARNING + ) def error(self, message): - self._call_event_handler( - 'add_event', message, level=constants.TASK_EVENT_ERROR) + self._call_event_handler('add_event', message, level=constants.TASK_EVENT_ERROR) class BaseEventHandler(object, with_metaclass(abc.ABCMeta)): - @abc.abstractmethod def add_progress_update( - self, message, initial_step=0, total_steps=0, - return_event=False): + self, message, initial_step=0, total_steps=0, return_event=False + ): pass @abc.abstractmethod def update_progress_update( - self, update_identifier, new_current_step, - new_total_steps=None, new_message=None, sync=False): + self, + update_identifier, + new_current_step, + new_total_steps=None, + new_message=None, + sync=False, + ): pass @classmethod @abc.abstractmethod def get_progress_update_identifier(cls, progress_update): - """ Returns the identifier for a given progress update. """ + """Returns the identifier for a given progress update.""" pass @abc.abstractmethod diff --git a/coriolis/exception.py b/coriolis/exception.py index 73f38488..2dc0e439 100644 --- a/coriolis/exception.py +++ b/coriolis/exception.py @@ -16,26 +16,23 @@ import sys +import six +import webob.exc from oslo_config import cfg from oslo_log import log as logging from oslo_versionedobjects import exception as obj_exc -import six -import webob.exc -from webob.util import status_generic_reasons -from webob.util import status_reasons +from webob.util import status_generic_reasons, status_reasons -from coriolis.i18n import _, _LE # noqa +from coriolis.i18n import _LE, _ # noqa LOG = logging.getLogger(__name__) CONF = cfg.CONF -TASK_ALREADY_CANCELLING_EXCEPTION_FMT = ( - "Task %(task_id)s is in CANCELLING status.") +TASK_ALREADY_CANCELLING_EXCEPTION_FMT = "Task %(task_id)s is in CANCELLING status." class ConvertedException(webob.exc.WSGIHTTPException): - def __init__(self, code=500, title="", explanation=""): self.code = code # There is a strict rule about constructing status line for HTTP: @@ -69,6 +66,7 @@ class CoriolisException(Exception): with the keyword arguments provided to the constructor. """ + message = _("An unknown exception occurred.") code = 500 headers = {} @@ -97,8 +95,9 @@ def __init__(self, message=None, **kwargs): # log the issue and the kwargs LOG.exception(_LE('Exception in string format operation')) for name, value in kwargs.items(): - LOG.error(_LE("%(name)s: %(value)s"), - {'name': name, 'value': value}) + LOG.error( + _LE("%(name)s: %(value)s"), {'name': name, 'value': value} + ) if CONF.fatal_exception_format_errors: six.reraise(*exc_info) # at least get the core message out if something happened @@ -209,18 +208,15 @@ class InvalidAuthKey(Invalid): class InvalidConfigurationValue(Invalid): - message = _('Value "%(value)s" is not valid for ' - 'configuration option "%(option)s"') + message = _('Value "%(value)s" is not valid for configuration option "%(option)s"') class InvalidTaskState(Invalid): - message = _( - 'Task "%(task_id)s" in in an invalid state: %(task_state)s') + message = _('Task "%(task_id)s" in in an invalid state: %(task_state)s') class InvalidMinionPoolState(Invalid): - message = _( - 'Minion pool "%(pool_id)s" in in an invalid state: %(pool_state)s') + message = _('Minion pool "%(pool_id)s" in in an invalid state: %(pool_state)s') class TaskIsCancelling(InvalidTaskState): @@ -260,9 +256,7 @@ class TaskFieldsConflict(CoriolisException): class TaskDependencyException(CoriolisException): - message = _( - "Execution task has non-existent tasks referenced as dependencies." - ) + message = _("Execution task has non-existent tasks referenced as dependencies.") class ServiceUnavailable(Invalid): @@ -290,9 +284,7 @@ class NotFound(CoriolisException): class MarkerNotFound(NotFound): - message = _( - "Could not find database record " - "identified by marker: %(marker)s") + message = _("Could not find database record identified by marker: %(marker)s") class RegionNotFound(NotFound): @@ -307,7 +299,8 @@ class OSMorphingToolsNotFound(NotFound): 'Suggestions include performing any needed OSMorphing steps manually ' 'within the source VM and then re-syncing with the "Skip OS Morphing" ' 'option enabled to bypass this stage, or contacting Cloudbase support ' - 'for further assistance.') + 'for further assistance.' + ) class OSDetectToolsNotFound(NotFound): @@ -319,7 +312,8 @@ class OSDetectToolsNotFound(NotFound): 'Suggestions include performing any needed OSMorphing steps manually ' 'within the source VM and then re-syncing with the "Skip OS Morphing" ' 'option enabled to bypass this stage, or contacting Cloudbase support ' - 'for further assistance.') + 'for further assistance.' + ) class FileNotFound(NotFound): @@ -339,8 +333,7 @@ class DiskStorageMappingNotFound(NotFound): class StorageBackendNotFound(NotFound): - message = _( - 'Storage backend with name "%(storage_name)s" could not be found.') + message = _('Storage backend with name "%(storage_name)s" could not be found.') class ImageNotFound(NotFound): @@ -432,6 +425,7 @@ class QEMUException(Exception): if six.PY2: + class ConnectionRefusedError(OSError): pass else: @@ -442,41 +436,44 @@ class UnrecognizedWorkerInitSystem(CoriolisException): message = _( "Could not determine init system for temporary worker VM. The image " "used for the worker VM must use systemd as an init system for " - "Coriolis to be able to use it for data Replication.") + "Coriolis to be able to use it for data Replication." + ) class NoRegionError(CoriolisException): safe = True code = 503 message = _( - "No Coriolis region is avaialable to process this request at this " - "time.") + "No Coriolis region is avaialable to process this request at this time." + ) class NoSuitableRegionError(NoRegionError): message = _( "No Coriolis Region(s) fitting the criteria of the required operation " - "could be found.") + "could be found." + ) class NoServiceError(CoriolisException): safe = True code = 503 - message = _( - "No service is avaialable to process this request at this time.") + message = _("No service is avaialable to process this request at this time.") class NoWorkerServiceError(NoServiceError): message = _( "No Coriolis Worker Service(s) were found. Please ensure that " "at least one or Coriolis Worker Service(s) are registered " - "within the Coriolis installation.") + "within the Coriolis installation." + ) class NoSuitableWorkerServiceError(NoServiceError): message = _( "No suitable Coriolis Worker service was found which fits the " - "criteria for the required operation.") + "criteria for the required operation." + ) class OSMorphingException(CoriolisException): @@ -496,13 +493,15 @@ class FailedPackageInstallationException(PackageManagerOperationException): "additional repositories within the source machine which contain the " "packages Coriolis requires, or attempt to manually install the " "packages on the source machine and then migrate the VM using Coriolis" - " with the OSMorphing process disabled. Error was: %(error)s") + " with the OSMorphing process disabled. Error was: %(error)s" + ) class FailedPackageUninstallationException(PackageManagerOperationException): message = ( "Failed to remove unwanted packages (%(package_names)s) through " - "%(package_manager)s. Error was: %(error)s") + "%(package_manager)s. Error was: %(error)s" + ) class MinionMachineCommandTimeout(CoriolisException): @@ -523,7 +522,8 @@ class OSMorphingSSHOperationTimeout(OSMorphingOperationTimeout): "Coriolis may have encountered connection issues to the minion machine" " or the command execution time exceeds the timeout set. Try extending" " the timeout by editing the 'default_osmorphing_operation_timeout' " - "in Coriolis' static configuration file.") + "in Coriolis' static configuration file." + ) class OSMorphingWinRMOperationTimeout(OSMorphingOperationTimeout): @@ -532,19 +532,22 @@ class OSMorphingWinRMOperationTimeout(OSMorphingOperationTimeout): "Coriolis may have encountered connection issues to the minion machine" " or the command execution time exceeds the timeout set. Try extending" " the timeout by editing the 'default_osmorphing_operation_timeout' " - "in Coriolis' static configuration file.") + "in Coriolis' static configuration file." + ) class ChecksumAlgorithmMismatch(CoriolisException): message = ( "Checksum algorithm mismatch for disk '%(disk)s': " - "source=%(source_alg)s, destination=%(dest_alg)s") + "source=%(source_alg)s, destination=%(dest_alg)s" + ) class ChecksumMismatch(CoriolisException): message = ( "Checksum mismatch for disk '%(disk)s': " - "source=%(source_checksum)s, destination=%(dest_checksum)s") + "source=%(source_checksum)s, destination=%(dest_checksum)s" + ) class MigrationLicenceFulfilledException(Invalid): @@ -552,4 +555,5 @@ class MigrationLicenceFulfilledException(Invalid): "The Live Migration operation with ID '%(action_id)s' (licensing " "reservation '%(reservation_id)s') has already been fulfilled on " "%(fulfilled_at)s. Please create a new Live Migration operation to " - "create a new licensing reservation.") + "create a new licensing reservation." + ) diff --git a/coriolis/keystone.py b/coriolis/keystone.py index 90d42e92..9f008525 100644 --- a/coriolis/keystone.py +++ b/coriolis/keystone.py @@ -11,21 +11,27 @@ from coriolis import exception opts = [ - cfg.StrOpt('auth_url', - default=None, - help='Default auth URL to be used when not specified in the' - ' migration\'s connection info.'), - cfg.StrOpt('cafile', - default=None, - help='The CA file used to validate openstack service' - ' API endpoints.'), - cfg.IntOpt('identity_api_version', - min=2, max=3, - default=2, - help='Default Keystone API version.'), - cfg.BoolOpt('allow_untrusted', - default=False, - help='Allow untrusted SSL/TLS certificates.'), + cfg.StrOpt( + 'auth_url', + default=None, + help='Default auth URL to be used when not specified in the' + ' migration\'s connection info.', + ), + cfg.StrOpt( + 'cafile', + default=None, + help='The CA file used to validate openstack service API endpoints.', + ), + cfg.IntOpt( + 'identity_api_version', + min=2, + max=3, + default=2, + help='Default Keystone API version.', + ), + cfg.BoolOpt( + 'allow_untrusted', default=False, help='Allow untrusted SSL/TLS certificates.' + ), ] CONF = cfg.CONF @@ -34,12 +40,16 @@ LOG = logging.getLogger(__name__) TRUSTEE_CONF_GROUP = 'trustee' -loading.register_auth_conf_options(CONF, TRUSTEE_CONF_GROUP, ) +loading.register_auth_conf_options( + CONF, + TRUSTEE_CONF_GROUP, +) def _get_trusts_auth_plugin(trust_id=None): return loading.load_auth_from_conf_options( - CONF, TRUSTEE_CONF_GROUP, trust_id=trust_id) + CONF, TRUSTEE_CONF_GROUP, trust_id=trust_id + ) def _get_verify_option(): @@ -65,9 +75,9 @@ def create_trust(ctxt): auth_url=trusts_auth_plugin.auth_url, token=ctxt.auth_token, project_name=ctxt.project_name, - project_domain_name=ctxt.project_domain_name) - session = ks_session.Session( - auth=auth, verify=_get_verify_option()) + project_domain_name=ctxt.project_domain_name, + ) + session = ks_session.Session(auth=auth, verify=_get_verify_option()) try: trustee_user_id = trusts_auth_plugin.get_user_id(session) @@ -83,17 +93,23 @@ def create_trust(ctxt): "Granting Keystone trust. Trustor: %(trustor_user_id)s, trustee:" " %(trustee_user_id)s, project: %(trustor_proj_id)s, roles:" " %(roles)s", - {"trustor_user_id": trustor_user_id, - "trustee_user_id": trustee_user_id, - "trustor_proj_id": trustor_proj_id, "roles": roles}) + { + "trustor_user_id": trustor_user_id, + "trustee_user_id": trustee_user_id, + "trustor_proj_id": trustor_proj_id, + "roles": roles, + }, + ) # Trusts are not supported before Keystone v3 client = kc_v3.Client(session=session) - trust = client.trusts.create(trustor_user=trustor_user_id, - trustee_user=trustee_user_id, - project=trustor_proj_id, - impersonation=True, - role_names=roles) + trust = client.trusts.create( + trustor_user=trustor_user_id, + trustee_user=trustee_user_id, + project=trustor_proj_id, + impersonation=True, + role_names=roles, + ) LOG.debug("Trust id: %s" % trust.id) ctxt.trust_id = trust.id @@ -103,8 +119,7 @@ def delete_trust(ctxt): LOG.debug("Deleting trust id: %s", ctxt.trust_id) auth = _get_trusts_auth_plugin(ctxt.trust_id) - session = ks_session.Session( - auth=auth, verify=_get_verify_option()) + session = ks_session.Session(auth=auth, verify=_get_verify_option()) client = kc_v3.Client(session=session) try: client.trusts.delete(ctxt.trust_id) @@ -126,9 +141,7 @@ def create_keystone_session(ctxt, connection_info={}): auth = _get_trusts_auth_plugin(ctxt.trust_id) else: plugin_name = "token" - plugin_args = { - "token": ctxt.auth_token - } + plugin_args = {"token": ctxt.auth_token} else: plugin_name = "password" password = connection_info.get("password") @@ -145,53 +158,63 @@ def create_keystone_session(ctxt, connection_info={}): raise exception.CoriolisException( '"auth_url" not provided in "connection_info" and option ' '"auth_url" in group "[openstack_migration_provider]" ' - 'not set') + 'not set' + ) - plugin_args.update({ - "auth_url": auth_url, - "project_name": project_name, - }) + plugin_args.update( + { + "auth_url": auth_url, + "project_name": project_name, + } + ) keystone_version = connection_info.get( - "identity_api_version", CONF.keystone.identity_api_version) + "identity_api_version", CONF.keystone.identity_api_version + ) if keystone_version == 3: plugin_name = "v3" + plugin_name project_domain_name = connection_info.get( - "project_domain_name", ctxt.project_domain_name) + "project_domain_name", ctxt.project_domain_name + ) # NOTE: only set the kwarg if proper argument is provided: if project_domain_name: plugin_args["project_domain_name"] = project_domain_name project_domain_id = connection_info.get( - "project_domain_id", ctxt.project_domain_id) + "project_domain_id", ctxt.project_domain_id + ) if project_domain_id: plugin_args["project_domain_id"] = project_domain_id if not project_domain_name and not project_domain_id: raise exception.CoriolisException( "Either 'project_domain_name' or 'project_domain_id' is " - "required for Keystone v3 Auth.") + "required for Keystone v3 Auth." + ) # NOTE: The v3token plugin does not allow the user_domain_name # or user_domain_id options, while the v3password plugin # requires at least any of these. if plugin_name != "v3token": user_domain_name = connection_info.get( - "user_domain_name", ctxt.user_domain_name) + "user_domain_name", ctxt.user_domain_name + ) if user_domain_name: plugin_args["user_domain_name"] = user_domain_name user_domain_id = connection_info.get( - "user_domain_id", ctxt.user_domain_id) + "user_domain_id", ctxt.user_domain_id + ) if user_domain_id: plugin_args["user_domain_id"] = user_domain_id if not user_domain_name and not user_domain_id: raise exception.CoriolisException( "Either 'user_domain_name' or 'user_domain_id' is " - "required for Keystone v3 Auth.") + "required for Keystone v3 Auth." + ) loader = loading.get_plugin_loader(plugin_name) auth = loader.load_from_options(**plugin_args) diff --git a/coriolis/licensing/client.py b/coriolis/licensing/client.py index 967ce8eb..15851ef8 100644 --- a/coriolis/licensing/client.py +++ b/coriolis/licensing/client.py @@ -5,12 +5,10 @@ import os import requests - -from coriolis import exception -from coriolis import utils from oslo_config import cfg from oslo_log import log as logging +from coriolis import exception, utils LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -20,17 +18,17 @@ class LicensingClient(object): - """ Class for accessing the Coriolis licensing server API. """ + """Class for accessing the Coriolis licensing server API.""" def __init__(self, base_url, appliance_id=None, allow_untrusted=False): - """ :param base_url: URL for the API service, including scheme """ + """:param base_url: URL for the API service, including scheme""" self._base_url = base_url.rstrip('/') self._verify = not allow_untrusted self._appliance_id = appliance_id @classmethod def from_env(cls): - """ Retuns a `LicensingClient` object instatiated using the + """Retuns a `LicensingClient` object instatiated using the following env vars: LICENSING_SERVER_BASE_URL="https://10.7.2.3:37667/v1" LICENSING_SERVER_ALLOW_UNTRUSTED="" @@ -41,33 +39,36 @@ def from_env(cls): if base_url in ["", None, "None", "null"]: LOG.warn( "No 'LICENSING_SERVER_BASE_URL' env var present. Cannot " - "instantiate licensing client.") + "instantiate licensing client." + ) return None - allow_untrusted = os.environ.get( - "LICENSING_SERVER_ALLOW_UNTRUSTED", None) is not None - client = cls( - base_url, appliance_id=None, allow_untrusted=allow_untrusted) + allow_untrusted = ( + os.environ.get("LICENSING_SERVER_ALLOW_UNTRUSTED", None) is not None + ) + client = cls(base_url, appliance_id=None, allow_untrusted=allow_untrusted) appliance_ids = client.get_appliances() if not appliance_ids: client._appliance_id = client.create_appliance().get("id") elif len(appliance_ids) == 1: client._appliance_id = appliance_ids[0].get('id') else: - raise exception.CoriolisException( - 'More than one appliance IDs found.') + raise exception.CoriolisException('More than one appliance IDs found.') client.get_licence_status() return client def _get_url_for_resource(self, resource): - """ Provides full URL for subresource. + """Provides full URL for subresource. Ex: "licences" -> "http://$host:$port/v1/licences" """ return "%s/%s" % (self._base_url, resource.strip('/')) def _get_url_for_appliance_resource(self, resource): return "%s/appliances/%s/%s" % ( - self._base_url, self._appliance_id, resource.strip('/')) + self._base_url, + self._appliance_id, + resource.strip('/'), + ) def _raise_response_error(self, resp): error = None @@ -77,19 +78,24 @@ def _raise_response_error(self, resp): LOG.debug( "Exception occured during error extraction from licensing " "response: '%s'\nException:\n%s", - resp.text, utils.get_exception_details()) + resp.text, + utils.get_exception_details(), + ) if error and all([x in error for x in ['code', 'message']]): - raise exception.Conflict( - message=error['message'], - code=int(error['code'])) + raise exception.Conflict(message=error['message'], code=int(error['code'])) else: resp.raise_for_status() @utils.retry_on_error() def _do_req( - self, method_name, resource, body=None, - response_key=None, raw_response=False, - appliance_scoped=True): + self, + method_name, + resource, + body=None, + response_key=None, + raw_response=False, + appliance_scoped=True, + ): method = getattr(requests, method_name.lower(), None) if not method: raise ValueError("No such HTTP method '%s'" % method_name) @@ -98,8 +104,7 @@ def _do_req( if appliance_scoped: url = self._get_url_for_appliance_resource(resource) - kwargs = {"verify": self._verify, - "timeout": CONF.default_requests_timeout} + kwargs = {"verify": self._verify, "timeout": CONF.default_requests_timeout} if body: if not isinstance(body, (str, bytes)): body = json.dumps(body) @@ -107,7 +112,10 @@ def _do_req( LOG.debug( "Making '%s' call to licensing server at '%s' with body: %s", - method_name, url, kwargs.get('data')) + method_name, + url, + kwargs.get('data'), + ) resp = method(url, **kwargs) if raw_response: @@ -120,117 +128,135 @@ def _do_req( if response_key: if response_key not in resp_data: raise ValueError( - "No response key '%s' in response body: %s" % ( - response_key, resp_data)) + "No response key '%s' in response body: %s" + % (response_key, resp_data) + ) resp_data = resp_data[response_key] return resp_data def _get(self, resource, response_key=None, appliance_scoped=True): - return self._do_req("GET", resource, response_key=response_key, - appliance_scoped=appliance_scoped) + return self._do_req( + "GET", + resource, + response_key=response_key, + appliance_scoped=appliance_scoped, + ) def _post(self, resource, body, response_key=None, appliance_scoped=True): return self._do_req( - "POST", resource, body=body, + "POST", + resource, + body=body, response_key=response_key, - appliance_scoped=appliance_scoped) + appliance_scoped=appliance_scoped, + ) def _put(self, resource, body, response_key=None, appliance_scoped=True): return self._do_req( - "PUT", resource, body=body, + "PUT", + resource, + body=body, response_key=response_key, - appliance_scoped=appliance_scoped) + appliance_scoped=appliance_scoped, + ) - def _delete(self, resource, body, response_key=None, - appliance_scoped=True): + def _delete(self, resource, body, response_key=None, appliance_scoped=True): return self._do_req( - "DELETE", resource, body=body, + "DELETE", + resource, + body=body, response_key=response_key, - appliance_scoped=appliance_scoped) + appliance_scoped=appliance_scoped, + ) def get_appliances(self): - """ Lists all appliances on the Licensing server. """ + """Lists all appliances on the Licensing server.""" return self._get( - "/appliances", - response_key="appliances", - appliance_scoped=False) + "/appliances", response_key="appliances", appliance_scoped=False + ) def get_appliance(self): - """ Get the appliance corresponding to this client instance. """ + """Get the appliance corresponding to this client instance.""" return self._get( - "/appliances/%s" % self._appliance_id, response_key="appliance", - appliance_scoped=False) + "/appliances/%s" % self._appliance_id, + response_key="appliance", + appliance_scoped=False, + ) def create_appliance(self): return self._post( - "/appliances", body=None, response_key="appliance", - appliance_scoped=False) + "/appliances", body=None, response_key="appliance", appliance_scoped=False + ) def get_licence_status(self): - """ Gets licence status for appliance. """ + """Gets licence status for appliance.""" return self._get("/status", "appliance_licence_status") def get_licences(self): - """ Lists all installed licences. """ + """Lists all installed licences.""" return self._get("/licences", response_key="licences") def add_licence(self, licence_data): - """ Sends request to add licence (in .PEM format). """ + """Sends request to add licence (in .PEM format).""" return self._post("/licences", licence_data) def add_reservation(self, reservation_type, num_vms): - """ Creates a reservation of the given type. """ - allowed_values = [ - RESERVATION_TYPE_MIGRATION, RESERVATION_TYPE_REPLICA] + """Creates a reservation of the given type.""" + allowed_values = [RESERVATION_TYPE_MIGRATION, RESERVATION_TYPE_REPLICA] if reservation_type not in allowed_values: - raise ValueError( - "Reservation type must be one of %s" % allowed_values) + raise ValueError("Reservation type must be one of %s" % allowed_values) return self._post( - "/reservations", { - "type": reservation_type, "count": num_vms}, - response_key="reservation") + "/reservations", + {"type": reservation_type, "count": num_vms}, + response_key="reservation", + ) def add_migrations_reservation(self, num_vms): - """ Creates a reservation for the given number of VM Migrations. """ + """Creates a reservation for the given number of VM Migrations.""" return self.add_reservation(RESERVATION_TYPE_MIGRATION, num_vms) def add_replicas_reservation(self, num_vms): - """ Creates a reservation for the given number of VM Replicas. """ + """Creates a reservation for the given number of VM Replicas.""" return self.add_reservation(RESERVATION_TYPE_REPLICA, num_vms) def get_reservations(self): - """ Lists all existing reservations. """ + """Lists all existing reservations.""" return self._get("/reservations", response_key="reservations") def get_reservation(self, reservation_id): - """ Gets a reservation with the given ID. """ + """Gets a reservation with the given ID.""" return self._get( - "/reservations/%s" % reservation_id, response_key="reservation") + "/reservations/%s" % reservation_id, response_key="reservation" + ) def check_refresh_reservation(self, reservation_id): - """ Checks the reservation with the given ID. """ + """Checks the reservation with the given ID.""" return self._post( - "/reservations/%s/refresh" % reservation_id, None, - response_key="reservation") + "/reservations/%s/refresh" % reservation_id, + None, + response_key="reservation", + ) def mark_reservation_fulfilled(self, reservation_id): - """ Marks the given reservation as fulfilled. """ + """Marks the given reservation as fulfilled.""" return self._post( - "/reservations/%s/fulfill" % reservation_id, None, - response_key="reservation") + "/reservations/%s/fulfill" % reservation_id, + None, + response_key="reservation", + ) def delete_reservation(self, reservation_id, raise_on_404=False): - """ Deletes a reservation by its ID. + """Deletes a reservation by its ID. Unless `raise_on_404` is set, ignores not found reservations. """ resp = self._do_req( - "delete", "/reservations/%s" % reservation_id, raw_response=True) + "delete", "/reservations/%s" % reservation_id, raw_response=True + ) if not resp.ok: if resp.status_code == 404: if raise_on_404: self._raise_response_error(resp) - LOG.warn( - "Got 404 when deleting reservation '%s'", reservation_id) + LOG.warn("Got 404 when deleting reservation '%s'", reservation_id) else: self._raise_response_error(resp) diff --git a/coriolis/minion_manager/rpc/client.py b/coriolis/minion_manager/rpc/client.py index e1ca325b..8b66c285 100644 --- a/coriolis/minion_manager/rpc/client.py +++ b/coriolis/minion_manager/rpc/client.py @@ -1,22 +1,20 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log as logging -import oslo_messaging as messaging - -from coriolis import constants -from coriolis import events -from coriolis import rpc +from coriolis import constants, events, rpc VERSION = "1.0" LOG = logging.getLogger(__name__) MINION_MANAGER_OPTS = [ - cfg.IntOpt("minion_mananger_rpc_timeout", - help="Number of seconds until RPC calls to the " - "minion manager timeout.") + cfg.IntOpt( + "minion_mananger_rpc_timeout", + help="Number of seconds until RPC calls to the minion manager timeout.", + ) ] CONF = cfg.CONF @@ -24,155 +22,212 @@ class MinionManagerClient(rpc.BaseRPCClient): - def __init__(self, timeout=None): - target = messaging.Target( - topic='coriolis_minion_manager', version=VERSION) + target = messaging.Target(topic='coriolis_minion_manager', version=VERSION) if timeout is None: timeout = CONF.minion_manager.minion_mananger_rpc_timeout - super(MinionManagerClient, self).__init__( - target, timeout=timeout) + super(MinionManagerClient, self).__init__(target, timeout=timeout) def add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, - total_steps=0, return_event=False): + self, + ctxt, + minion_pool_id, + message, + initial_step=0, + total_steps=0, + return_event=False, + ): operation = self._cast if return_event: operation = self._call return operation( - ctxt, 'add_minion_pool_progress_update', - minion_pool_id=minion_pool_id, message=message, - initial_step=initial_step, total_steps=total_steps) + ctxt, + 'add_minion_pool_progress_update', + minion_pool_id=minion_pool_id, + message=message, + initial_step=initial_step, + total_steps=total_steps, + ) def update_minion_pool_progress_update( - self, ctxt, minion_pool_id, progress_update_index, - new_current_step, new_total_steps=None, new_message=None): + self, + ctxt, + minion_pool_id, + progress_update_index, + new_current_step, + new_total_steps=None, + new_message=None, + ): self._cast( - ctxt, 'update_minion_pool_progress_update', + ctxt, + 'update_minion_pool_progress_update', minion_pool_id=minion_pool_id, progress_update_index=progress_update_index, new_current_step=new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + new_total_steps=new_total_steps, + new_message=new_message, + ) def add_minion_pool_event(self, ctxt, minion_pool_id, level, message): return self._cast( - ctxt, 'add_minion_pool_event', minion_pool_id=minion_pool_id, - level=level, message=message) + ctxt, + 'add_minion_pool_event', + minion_pool_id=minion_pool_id, + level=level, + message=message, + ) def get_diagnostics(self, ctxt): return self._call(ctxt, 'get_diagnostics') def validate_minion_pool_selections_for_action(self, ctxt, action): return self._call( - ctxt, 'validate_minion_pool_selections_for_action', - action=action) + ctxt, 'validate_minion_pool_selections_for_action', action=action + ) - def allocate_minion_machines_for_transfer( - self, ctxt, transfer): + def allocate_minion_machines_for_transfer(self, ctxt, transfer): return self._cast( - ctxt, 'allocate_minion_machines_for_transfer', transfer=transfer) + ctxt, 'allocate_minion_machines_for_transfer', transfer=transfer + ) def allocate_minion_machines_for_deployment( - self, ctxt, deployment, include_transfer_minions=True, - include_osmorphing_minions=True): + self, + ctxt, + deployment, + include_transfer_minions=True, + include_osmorphing_minions=True, + ): return self._cast( - ctxt, 'allocate_minion_machines_for_deployment', + ctxt, + 'allocate_minion_machines_for_deployment', deployment=deployment, include_transfer_minions=include_transfer_minions, - include_osmorphing_minions=include_osmorphing_minions) + include_osmorphing_minions=include_osmorphing_minions, + ) def deallocate_minion_machine(self, ctxt, minion_machine_id): return self._cast( - ctxt, 'deallocate_minion_machine', - minion_machine_id=minion_machine_id) + ctxt, 'deallocate_minion_machine', minion_machine_id=minion_machine_id + ) def deallocate_minion_machines_for_action(self, ctxt, action_id): return self._cast( - ctxt, 'deallocate_minion_machines_for_action', - action_id=action_id) + ctxt, 'deallocate_minion_machines_for_action', action_id=action_id + ) def create_minion_pool( - self, ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=None, - skip_allocation=False): + self, + ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=None, + skip_allocation=False, + ): return self._call( - ctxt, 'create_minion_pool', name=name, endpoint_id=endpoint_id, - pool_platform=pool_platform, pool_os_type=pool_os_type, + ctxt, + 'create_minion_pool', + name=name, + endpoint_id=endpoint_id, + pool_platform=pool_platform, + pool_os_type=pool_os_type, environment_options=environment_options, minimum_minions=minimum_minions, maximum_minions=maximum_minions, minion_max_idle_time=minion_max_idle_time, minion_retention_strategy=minion_retention_strategy, - notes=notes, skip_allocation=skip_allocation) + notes=notes, + skip_allocation=skip_allocation, + ) def set_up_shared_minion_pool_resources(self, ctxt, minion_pool_id): return self._call( - ctxt, "set_up_shared_minion_pool_resources", - minion_pool_id=minion_pool_id) + ctxt, "set_up_shared_minion_pool_resources", minion_pool_id=minion_pool_id + ) - def tear_down_shared_minion_pool_resources( - self, ctxt, minion_pool_id, force=False): + def tear_down_shared_minion_pool_resources(self, ctxt, minion_pool_id, force=False): return self._call( - ctxt, "tear_down_shared_minion_pool_resources", - minion_pool_id=minion_pool_id, force=force) + ctxt, + "tear_down_shared_minion_pool_resources", + minion_pool_id=minion_pool_id, + force=force, + ) def allocate_minion_pool(self, ctxt, minion_pool_id): - return self._call( - ctxt, "allocate_minion_pool", - minion_pool_id=minion_pool_id) + return self._call(ctxt, "allocate_minion_pool", minion_pool_id=minion_pool_id) def refresh_minion_pool(self, ctxt, minion_pool_id): - return self._call( - ctxt, "refresh_minion_pool", - minion_pool_id=minion_pool_id) + return self._call(ctxt, "refresh_minion_pool", minion_pool_id=minion_pool_id) - def deallocate_minion_pool( - self, ctxt, minion_pool_id, force=False): + def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): return self._call( - ctxt, "deallocate_minion_pool", - minion_pool_id=minion_pool_id, - force=force) + ctxt, "deallocate_minion_pool", minion_pool_id=minion_pool_id, force=force + ) def get_minion_pools(self, ctxt): return self._call(ctxt, 'get_minion_pools') def get_minion_pool(self, ctxt, minion_pool_id): - return self._call( - ctxt, 'get_minion_pool', minion_pool_id=minion_pool_id) + return self._call(ctxt, 'get_minion_pool', minion_pool_id=minion_pool_id) def update_minion_pool(self, ctxt, minion_pool_id, updated_values): return self._call( - ctxt, 'update_minion_pool', - minion_pool_id=minion_pool_id, updated_values=updated_values) + ctxt, + 'update_minion_pool', + minion_pool_id=minion_pool_id, + updated_values=updated_values, + ) def delete_minion_pool(self, ctxt, minion_pool_id): - return self._call( - ctxt, 'delete_minion_pool', minion_pool_id=minion_pool_id) + return self._call(ctxt, 'delete_minion_pool', minion_pool_id=minion_pool_id) def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env, option_names): + self, ctxt, endpoint_id, env, option_names + ): return self._call( - ctxt, 'get_endpoint_source_minion_pool_options', - endpoint_id=endpoint_id, env=env, option_names=option_names) + ctxt, + 'get_endpoint_source_minion_pool_options', + endpoint_id=endpoint_id, + env=env, + option_names=option_names, + ) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env, option_names): + self, ctxt, endpoint_id, env, option_names + ): return self._call( - ctxt, 'get_endpoint_destination_minion_pool_options', - endpoint_id=endpoint_id, env=env, option_names=option_names) + ctxt, + 'get_endpoint_destination_minion_pool_options', + endpoint_id=endpoint_id, + env=env, + option_names=option_names, + ) def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): + self, ctxt, endpoint_id, pool_environment + ): return self._call( - ctxt, 'validate_endpoint_source_minion_pool_options', - endpoint_id=endpoint_id, pool_environment=pool_environment) + ctxt, + 'validate_endpoint_source_minion_pool_options', + endpoint_id=endpoint_id, + pool_environment=pool_environment, + ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): + self, ctxt, endpoint_id, pool_environment + ): return self._call( - ctxt, 'validate_endpoint_destination_minion_pool_options', - endpoint_id=endpoint_id, pool_environment=pool_environment) + ctxt, + 'validate_endpoint_destination_minion_pool_options', + endpoint_id=endpoint_id, + pool_environment=pool_environment, + ) class MinionManagerPoolRpcEventHandler(events.BaseEventHandler): @@ -195,24 +250,46 @@ def get_progress_update_identifier(self, progress_update): return progress_update['index'] def add_progress_update( - self, message, initial_step=0, total_steps=0, return_event=False): + self, message, initial_step=0, total_steps=0, return_event=False + ): LOG.info( "Sending progress update for pool '%s' to minion manager : %s", - self._pool_id, message) + self._pool_id, + message, + ) return self._rpc_minion_manager_client.add_minion_pool_progress_update( - self._ctxt, self._pool_id, message, initial_step=initial_step, - total_steps=total_steps, return_event=return_event) + self._ctxt, + self._pool_id, + message, + initial_step=initial_step, + total_steps=total_steps, + return_event=return_event, + ) def update_progress_update( - self, update_identifier, new_current_step, - new_total_steps=None, new_message=None, sync=False): + self, + update_identifier, + new_current_step, + new_total_steps=None, + new_message=None, + sync=False, + ): LOG.info( "Updating progress update '%s' for pool '%s' with new step %s", - update_identifier, self._pool_id, new_current_step) + update_identifier, + self._pool_id, + new_current_step, + ) self._rpc_minion_manager_client.update_minion_pool_progress_update( - self._ctxt, self._pool_id, update_identifier, new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + self._ctxt, + self._pool_id, + update_identifier, + new_current_step, + new_total_steps=new_total_steps, + new_message=new_message, + ) def add_event(self, message, level=constants.TASK_EVENT_INFO): self._rpc_minion_manager_client.add_minion_pool_event( - self._ctxt, self._pool_id, level, message) + self._ctxt, self._pool_id, level, message + ) diff --git a/coriolis/minion_manager/rpc/server.py b/coriolis/minion_manager/rpc/server.py index dd3899c6..265dfc33 100644 --- a/coriolis/minion_manager/rpc/server.py +++ b/coriolis/minion_manager/rpc/server.py @@ -9,25 +9,19 @@ from oslo_log import log as logging from oslo_utils import timeutils from taskflow import deciders as taskflow_deciders -from taskflow.patterns import graph_flow -from taskflow.patterns import linear_flow -from taskflow.patterns import unordered_flow +from taskflow.patterns import graph_flow, linear_flow, unordered_flow +from coriolis import constants, context, exception, keystone, utils from coriolis.conductor.rpc import client as rpc_conductor_client -from coriolis import constants -from coriolis import context from coriolis.cron import cron from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models -from coriolis import exception -from coriolis import keystone from coriolis.minion_manager.rpc import client as rpc_minion_manager_client from coriolis.minion_manager.rpc import tasks as minion_mgr_tasks from coriolis.minion_manager.rpc import utils as minion_manager_utils from coriolis.scheduler.rpc import client as rpc_scheduler_client from coriolis.taskflow import runner as taskflow_runner from coriolis.taskflow import utils as taskflow_utils -from coriolis import utils from coriolis.worker.rpc import client as rpc_worker_client VERSION = "1.0" @@ -39,7 +33,9 @@ "minion_pool_default_refresh_period_minutes", default=10, help="Number of minutes in which to refresh minion pools." - "Set to 0 to completely disable automatic refreshing.")] + "Set to 0 to completely disable automatic refreshing.", + ) +] CONF = cfg.CONF CONF.register_opts(MINION_MANAGER_OPTS, 'minion_manager') @@ -47,21 +43,23 @@ MINION_POOL_REFRESH_JOB_PREFIX_FORMAT = "pool-%s-refresh" MINION_POOL_REFRESH_CRON_JOB_NAME_FORMAT = "pool-%s-refresh-minute-%d" MINION_POOL_REFRESH_CRON_JOB_DESCRIPTION_FORMAT = ( - "Regularly scheduled refresh job for minion pool '%s' on minute %d.") + "Regularly scheduled refresh job for minion pool '%s' on minute %d." +) def _trigger_pool_refresh(ctxt, minion_manager_client, minion_pool_id): try: - minion_manager_client.refresh_minion_pool( - ctxt, minion_pool_id) + minion_manager_client.refresh_minion_pool(ctxt, minion_pool_id) except exception.InvalidMinionPoolState as ex: LOG.warn( "Minion Pool '%s' is in an invalid state for having a refresh run." - " Skipping for now. Error was: %s", minion_pool_id, str(ex)) + " Skipping for now. Error was: %s", + minion_pool_id, + str(ex), + ) class MinionManagerServerEndpoint(object): - def __init__(self): self._admin_ctxt = context.get_admin_context() self._scheduler_client_instance = None @@ -79,13 +77,17 @@ def __init__(self): "jobs for automatic pool refreshing. Automatic refreshing will" " not be perfomed until the issue is fixed and the service is " "restarted. Exception details were: %s", - utils.get_exception_details()) + utils.get_exception_details(), + ) def _init_pools_refresh_cron_jobs(self): minion_pools = db_api.get_minion_pools( - self._admin_ctxt, include_machines=False, - include_progress_updates=False, include_events=False, - to_dict=False) + self._admin_ctxt, + include_machines=False, + include_progress_updates=False, + include_events=False, + to_dict=False, + ) for minion_pool in minion_pools: active_pool_statuses = [constants.MINION_POOL_STATUS_ALLOCATED] @@ -93,7 +95,9 @@ def _init_pools_refresh_cron_jobs(self): LOG.debug( "Not setting any refresh schedules for minion pool '%s' " "as it is in an inactive status '%s'.", - minion_pool.id, minion_pool.status) + minion_pool.id, + minion_pool.status, + ) continue if not minion_pool.maintenance_trust_id: @@ -101,68 +105,85 @@ def _init_pools_refresh_cron_jobs(self): "Minion Pool with ID '%s' had no maintenance trust " "ID associated with it. Cannot set up automatic " "refreshing during startup. Skipping.", - minion_pool.id) + minion_pool.id, + ) continue LOG.debug( "Adding refresh schedule for minion pool '%s' as part of " - "server startup.", minion_pool.id) + "server startup.", + minion_pool.id, + ) try: self._register_refresh_jobs_for_minion_pool(minion_pool) except Exception: LOG.warn( "An Exception occurred while setting up automatic " "refreshing for minion pool with ID '%s'. Error was: %s", - minion_pool.id, utils.get_exception_details()) + minion_pool.id, + utils.get_exception_details(), + ) - def _register_refresh_jobs_for_minion_pool( - self, minion_pool, period_minutes=None): + def _register_refresh_jobs_for_minion_pool(self, minion_pool, period_minutes=None): if period_minutes is None: period_minutes = ( - CONF.minion_manager.minion_pool_default_refresh_period_minutes) + CONF.minion_manager.minion_pool_default_refresh_period_minutes + ) if period_minutes < 0: LOG.warn( - "Got negative pool refresh period %s. Defaulting to 0.", - period_minutes) + "Got negative pool refresh period %s. Defaulting to 0.", period_minutes + ) period_minutes = 0 if period_minutes == 0: LOG.info( "Minon pool refresh period is set to zero. Not setting up " "any automatic refresh jobs for Minion Pool '%s'.", - minion_pool.id) + minion_pool.id, + ) return if period_minutes > 60: LOG.warn( "Selected pool refresh period_minutes is greater than 60, " - "defaulting to 10. Original value was: %s", period_minutes) + "defaulting to 10. Original value was: %s", + period_minutes, + ) period_minutes = 10 - admin_ctxt = context.get_admin_context( - minion_pool.maintenance_trust_id) - description = ( - "Scheduled refresh job for minion pool '%s'" % minion_pool.id) + admin_ctxt = context.get_admin_context(minion_pool.maintenance_trust_id) + description = "Scheduled refresh job for minion pool '%s'" % minion_pool.id # NOTE: we need to generate hourly schedules for each minute in # the hour we would like the refresh to be triggered: for minute in [ - period_minutes * i for i in range( - math.ceil(60 / period_minutes))]: - name = MINION_POOL_REFRESH_CRON_JOB_NAME_FORMAT % ( - minion_pool.id, minute) + period_minutes * i for i in range(math.ceil(60 / period_minutes)) + ]: + name = MINION_POOL_REFRESH_CRON_JOB_NAME_FORMAT % (minion_pool.id, minute) description = MINION_POOL_REFRESH_CRON_JOB_DESCRIPTION_FORMAT % ( - minion_pool.id, minute) + minion_pool.id, + minute, + ) self._cron.register( cron.CronJob( - name, description, {"minute": minute}, True, None, None, - None, _trigger_pool_refresh, admin_ctxt, - self._rpc_minion_manager_client, minion_pool.id)) + name, + description, + {"minute": minute}, + True, + None, + None, + None, + _trigger_pool_refresh, + admin_ctxt, + self._rpc_minion_manager_client, + minion_pool.id, + ) + ) def _unregister_refresh_jobs_for_minion_pool( - self, minion_pool, raise_on_error=True): - job_prefix = MINION_POOL_REFRESH_JOB_PREFIX_FORMAT % ( - minion_pool.id) + self, minion_pool, raise_on_error=True + ): + job_prefix = MINION_POOL_REFRESH_JOB_PREFIX_FORMAT % (minion_pool.id) try: self._cron.unregister_jobs_with_prefix(job_prefix) except Exception: @@ -171,15 +192,17 @@ def _unregister_refresh_jobs_for_minion_pool( "Exception occurred while unregistering minion pool " "refresh cron jobs for pool with ID '%s'. " "Exception was: %s", - minion_pool.id, utils.get_exception_details()) + minion_pool.id, + utils.get_exception_details(), + ) else: raise @property def _taskflow_runner(self): return taskflow_runner.TaskFlowRunner( - constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC, - max_workers=25) + constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC, max_workers=25 + ) # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated # oslo_messaging clients as the underlying thread queues will @@ -189,108 +212,118 @@ def _taskflow_runner(self): @property def _rpc_worker_client(self): if not getattr(self, '_worker_client_instance', None): - self._worker_client_instance = ( - rpc_worker_client.WorkerClient()) + self._worker_client_instance = rpc_worker_client.WorkerClient() return self._worker_client_instance @property def _rpc_scheduler_client(self): if not getattr(self, '_scheduler_client_instance', None): - self._scheduler_client_instance = ( - rpc_scheduler_client.SchedulerClient()) + self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance @property def _rpc_conductor_client(self): if not getattr(self, '_conductor_client_instance', None): - self._conductor_client_instance = ( - rpc_conductor_client.ConductorClient()) + self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance @property def _rpc_minion_manager_client(self): if not getattr(self, '_minion_manager_client_instance', None): self._minion_manager_client_instance = ( - rpc_minion_manager_client.MinionManagerClient()) + rpc_minion_manager_client.MinionManagerClient() + ) return self._minion_manager_client_instance def get_diagnostics(self, ctxt): return utils.get_diagnostics_info() def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env, option_names): + self, ctxt, endpoint_id, env, option_names + ): endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) - worker_service = ( - self._rpc_scheduler_client.get_worker_service_for_specs( - ctxt, enabled=True, - region_sets=[[r['id'] for r in endpoint['mapped_regions']]], - provider_requirements={ - endpoint['type']: [ - constants.PROVIDER_TYPE_SOURCE_MINION_POOL]}) + worker_service = self._rpc_scheduler_client.get_worker_service_for_specs( + ctxt, + enabled=True, + region_sets=[[r['id'] for r in endpoint['mapped_regions']]], + provider_requirements={ + endpoint['type']: [constants.PROVIDER_TYPE_SOURCE_MINION_POOL] + }, ) worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - worker_service) + worker_service + ) return worker_rpc.get_endpoint_source_minion_pool_options( - ctxt, endpoint['type'], endpoint['connection_info'], env, - option_names) + ctxt, endpoint['type'], endpoint['connection_info'], env, option_names + ) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env, option_names): + self, ctxt, endpoint_id, env, option_names + ): endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) - worker_service = ( - self._rpc_scheduler_client.get_worker_service_for_specs( - ctxt, enabled=True, - region_sets=[[r['id'] for r in endpoint['mapped_regions']]], - provider_requirements={ - endpoint['type']: [ - constants.PROVIDER_TYPE_DESTINATION_MINION_POOL]})) + worker_service = self._rpc_scheduler_client.get_worker_service_for_specs( + ctxt, + enabled=True, + region_sets=[[r['id'] for r in endpoint['mapped_regions']]], + provider_requirements={ + endpoint['type']: [constants.PROVIDER_TYPE_DESTINATION_MINION_POOL] + }, + ) worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - worker_service) + worker_service + ) return worker_rpc.get_endpoint_destination_minion_pool_options( - ctxt, endpoint['type'], endpoint['connection_info'], env, - option_names) + ctxt, endpoint['type'], endpoint['connection_info'], env, option_names + ) def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): + self, ctxt, endpoint_id, pool_environment + ): endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) - worker_service = ( - self._rpc_scheduler_client.get_worker_service_for_specs( - ctxt, enabled=True, - region_sets=[[r['id'] for r in endpoint['mapped_regions']]], - provider_requirements={ - endpoint['type']: [ - constants.PROVIDER_TYPE_SOURCE_MINION_POOL]})) + worker_service = self._rpc_scheduler_client.get_worker_service_for_specs( + ctxt, + enabled=True, + region_sets=[[r['id'] for r in endpoint['mapped_regions']]], + provider_requirements={ + endpoint['type']: [constants.PROVIDER_TYPE_SOURCE_MINION_POOL] + }, + ) worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - worker_service) + worker_service + ) return worker_rpc.validate_endpoint_source_minion_pool_options( - ctxt, endpoint['type'], pool_environment) + ctxt, endpoint['type'], pool_environment + ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment): + self, ctxt, endpoint_id, pool_environment + ): endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) - worker_service = ( - self._rpc_scheduler_client.get_worker_service_for_specs( - ctxt, enabled=True, - region_sets=[[r['id'] for r in endpoint['mapped_regions']]], - provider_requirements={ - endpoint['type']: [ - constants.PROVIDER_TYPE_DESTINATION_MINION_POOL]})) + worker_service = self._rpc_scheduler_client.get_worker_service_for_specs( + ctxt, + enabled=True, + region_sets=[[r['id'] for r in endpoint['mapped_regions']]], + provider_requirements={ + endpoint['type']: [constants.PROVIDER_TYPE_DESTINATION_MINION_POOL] + }, + ) worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( - worker_service) + worker_service + ) return worker_rpc.validate_endpoint_destination_minion_pool_options( - ctxt, endpoint['type'], pool_environment) + ctxt, endpoint['type'], pool_environment + ) def _add_minion_pool_event(self, ctxt, minion_pool_id, level, message): - LOG.info( - "Minion pool event for pool %s: %s", minion_pool_id, message) + LOG.info("Minion pool event for pool %s: %s", minion_pool_id, message) pool = db_api.get_minion_pool(ctxt, minion_pool_id) db_api.add_minion_pool_event(ctxt, pool.id, level, message) @@ -299,92 +332,135 @@ def add_minion_pool_event(self, ctxt, minion_pool_id, level, message): self._add_minion_pool_event(ctxt, minion_pool_id, level, message) def _add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, - total_steps=0): - LOG.info( - "Adding pool progress update for %s: %s", minion_pool_id, message) + self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0 + ): + LOG.info("Adding pool progress update for %s: %s", minion_pool_id, message) db_api.add_minion_pool_progress_update( - ctxt, minion_pool_id, message, initial_step=initial_step, - total_steps=total_steps) + ctxt, + minion_pool_id, + message, + initial_step=initial_step, + total_steps=total_steps, + ) @minion_manager_utils.minion_pool_synchronized_op def add_minion_pool_progress_update( - self, ctxt, minion_pool_id, message, initial_step=0, - total_steps=0): + self, ctxt, minion_pool_id, message, initial_step=0, total_steps=0 + ): self._add_minion_pool_progress_update( - ctxt, minion_pool_id, message, initial_step=initial_step, - total_steps=total_steps) + ctxt, + minion_pool_id, + message, + initial_step=initial_step, + total_steps=total_steps, + ) @minion_manager_utils.minion_pool_synchronized_op def update_minion_pool_progress_update( - self, ctxt, minion_pool_id, progress_update_index, - new_current_step, new_total_steps=None, new_message=None): + self, + ctxt, + minion_pool_id, + progress_update_index, + new_current_step, + new_total_steps=None, + new_message=None, + ): LOG.info( "Updating minion pool '%s' progress update '%s': %s", - minion_pool_id, progress_update_index, new_current_step) + minion_pool_id, + progress_update_index, + new_current_step, + ) db_api.update_minion_pool_progress_update( - ctxt, minion_pool_id, progress_update_index, new_current_step, - new_total_steps=new_total_steps, new_message=new_message) + ctxt, + minion_pool_id, + progress_update_index, + new_current_step, + new_total_steps=new_total_steps, + new_message=new_message, + ) def _check_keys_for_action_dict( - self, action, required_action_properties, operation=None): + self, action, required_action_properties, operation=None + ): if not isinstance(action, dict): raise exception.InvalidInput( - "Action must be a dict, got '%s': %s" % ( - type(action), action)) - missing = [ - prop for prop in required_action_properties - if prop not in action] + "Action must be a dict, got '%s': %s" % (type(action), action) + ) + missing = [prop for prop in required_action_properties if prop not in action] if missing: raise exception.InvalidInput( "Missing the following required action properties for " - "%s: %s. Got %s" % ( - operation, missing, action)) + "%s: %s. Got %s" % (operation, missing, action) + ) def validate_minion_pool_selections_for_action(self, ctxt, action): - """ Validates the minion pool selections for a given action. """ + """Validates the minion pool selections for a given action.""" required_action_properties = [ - 'id', 'origin_endpoint_id', 'destination_endpoint_id', - 'origin_minion_pool_id', 'destination_minion_pool_id', - 'instance_osmorphing_minion_pool_mappings', 'instances'] + 'id', + 'origin_endpoint_id', + 'destination_endpoint_id', + 'origin_minion_pool_id', + 'destination_minion_pool_id', + 'instance_osmorphing_minion_pool_mappings', + 'instances', + ] self._check_keys_for_action_dict( - action, required_action_properties, - operation="minion pool selection validation") + action, + required_action_properties, + operation="minion pool selection validation", + ) minion_pools = { pool.id: pool # NOTE: we can just load all the pools in one go to # avoid extraneous DB queries: for pool in db_api.get_minion_pools( - ctxt, include_machines=False, include_events=False, - include_progress_updates=False, to_dict=False)} + ctxt, + include_machines=False, + include_events=False, + include_progress_updates=False, + to_dict=False, + ) + } def _get_pool(pool_id): pool = minion_pools.get(pool_id) if not pool: raise exception.NotFound( - "Could not find minion pool with ID '%s'." % pool_id) + "Could not find minion pool with ID '%s'." % pool_id + ) return pool - def _check_pool_minion_count( - minion_pool, instances, minion_pool_type=""): + def _check_pool_minion_count(minion_pool, instances, minion_pool_type=""): desired_minion_count = len(instances) if minion_pool.status != constants.MINION_POOL_STATUS_ALLOCATED: raise exception.InvalidMinionPoolState( "Minion Pool '%s' is an invalid state ('%s') to be " "used as a %s pool for action '%s'. The pool must be " - "in '%s' status." % ( - minion_pool.id, minion_pool.status, - minion_pool_type.lower(), action['id'], - constants.MINION_POOL_STATUS_ALLOCATED)) + "in '%s' status." + % ( + minion_pool.id, + minion_pool.status, + minion_pool_type.lower(), + action['id'], + constants.MINION_POOL_STATUS_ALLOCATED, + ) + ) if desired_minion_count > minion_pool.maximum_minions: msg = ( "Minion Pool with ID '%s' has a lower maximum minion " "count (%d) than the requested number of minions " "(%d) to handle all of the instances of action '%s': " - "%s" % ( - minion_pool.id, minion_pool.maximum_minions, - desired_minion_count, action['id'], instances)) + "%s" + % ( + minion_pool.id, + minion_pool.maximum_minions, + desired_minion_count, + action['id'], + instances, + ) + ) if minion_pool_type: msg = "%s %s" % (minion_pool_type, msg) raise exception.InvalidMinionPoolSelection(msg) @@ -397,172 +473,216 @@ def _check_pool_minion_count( raise exception.InvalidMinionPoolSelection( "The selected origin minion pool ('%s') belongs to a " "different Coriolis endpoint ('%s') than the requested " - "origin endpoint ('%s')" % ( + "origin endpoint ('%s')" + % ( action['origin_minion_pool_id'], origin_pool.endpoint_id, - action['origin_endpoint_id'])) + action['origin_endpoint_id'], + ) + ) if origin_pool.platform != constants.PROVIDER_PLATFORM_SOURCE: raise exception.InvalidMinionPoolSelection( "The selected origin minion pool ('%s') is configured as a" " '%s' pool. The pool must be of type %s to be used for " - "data exports." % ( + "data exports." + % ( action['origin_minion_pool_id'], origin_pool.platform, - constants.PROVIDER_PLATFORM_SOURCE)) + constants.PROVIDER_PLATFORM_SOURCE, + ) + ) if origin_pool.os_type != constants.OS_TYPE_LINUX: raise exception.InvalidMinionPoolSelection( "The selected origin minion pool ('%s') is of OS type '%s'" " instead of the Linux OS type required for a source " - "transfer minion pool." % ( - action['origin_minion_pool_id'], - origin_pool.os_type)) - _check_pool_minion_count( - origin_pool, instances, minion_pool_type="Source") + "transfer minion pool." + % (action['origin_minion_pool_id'], origin_pool.os_type) + ) + _check_pool_minion_count(origin_pool, instances, minion_pool_type="Source") LOG.debug( "Successfully validated compatibility of origin minion pool " - "'%s' for use with action '%s'." % ( - action['origin_minion_pool_id'], action['id'])) + "'%s' for use with action '%s'." + % (action['origin_minion_pool_id'], action['id']) + ) # check destination pool: if action['destination_minion_pool_id']: destination_pool = _get_pool(action['destination_minion_pool_id']) - if destination_pool.endpoint_id != ( - action['destination_endpoint_id']): + if destination_pool.endpoint_id != (action['destination_endpoint_id']): raise exception.InvalidMinionPoolSelection( "The selected destination minion pool ('%s') belongs to a " "different Coriolis endpoint ('%s') than the requested " - "destination endpoint ('%s')" % ( + "destination endpoint ('%s')" + % ( action['destination_minion_pool_id'], destination_pool.endpoint_id, - action['destination_endpoint_id'])) - if destination_pool.platform != ( - constants.PROVIDER_PLATFORM_DESTINATION): + action['destination_endpoint_id'], + ) + ) + if destination_pool.platform != (constants.PROVIDER_PLATFORM_DESTINATION): raise exception.InvalidMinionPoolSelection( "The selected destination minion pool ('%s') is configured" " as a '%s'. The pool must be of type %s to be used for " - "data imports." % ( + "data imports." + % ( action['destination_minion_pool_id'], destination_pool.platform, - constants.PROVIDER_PLATFORM_DESTINATION)) + constants.PROVIDER_PLATFORM_DESTINATION, + ) + ) if destination_pool.os_type != constants.OS_TYPE_LINUX: raise exception.InvalidMinionPoolSelection( "The selected destination minion pool ('%s') is of OS type" " '%s' instead of the Linux OS type required for a source " - "transfer minion pool." % ( - action['destination_minion_pool_id'], - destination_pool.os_type)) + "transfer minion pool." + % (action['destination_minion_pool_id'], destination_pool.os_type) + ) _check_pool_minion_count( - destination_pool, instances, - minion_pool_type="Destination") + destination_pool, instances, minion_pool_type="Destination" + ) LOG.debug( "Successfully validated compatibility of destination minion " - "pool '%s' for use with action '%s'." % ( - action['origin_minion_pool_id'], action['id'])) + "pool '%s' for use with action '%s'." + % (action['origin_minion_pool_id'], action['id']) + ) # check OSMorphing pool(s): instance_osmorphing_minion_pool_mappings = action.get( - 'instance_osmorphing_minion_pool_mappings') + 'instance_osmorphing_minion_pool_mappings' + ) if instance_osmorphing_minion_pool_mappings: osmorphing_pool_mappings = {} - for (instance_id, pool_id) in ( - instance_osmorphing_minion_pool_mappings).items(): + for instance_id, pool_id in ( + instance_osmorphing_minion_pool_mappings + ).items(): if instance_id not in instances: LOG.warn( "Ignoring OSMorphing pool validation for instance with" " ID '%s' (mapped pool '%s') as it is not part of " "action '%s's declared instances: %s", - instance_id, pool_id, action['id'], instances) + instance_id, + pool_id, + action['id'], + instances, + ) continue if pool_id not in osmorphing_pool_mappings: osmorphing_pool_mappings[pool_id] = [instance_id] else: osmorphing_pool_mappings[pool_id].append(instance_id) - for (pool_id, instances) in osmorphing_pool_mappings.items(): + for pool_id, instances in osmorphing_pool_mappings.items(): osmorphing_pool = _get_pool(pool_id) - if osmorphing_pool.endpoint_id != ( - action['destination_endpoint_id']): + if osmorphing_pool.endpoint_id != (action['destination_endpoint_id']): raise exception.InvalidMinionPoolSelection( "The selected OSMorphing minion pool for instances %s" " ('%s') belongs to a different Coriolis endpoint " - "('%s') than the destination endpoint ('%s')" % ( - instances, pool_id, + "('%s') than the destination endpoint ('%s')" + % ( + instances, + pool_id, osmorphing_pool.endpoint_id, - action['destination_endpoint_id'])) + action['destination_endpoint_id'], + ) + ) if osmorphing_pool.platform != ( - constants.PROVIDER_PLATFORM_DESTINATION): + constants.PROVIDER_PLATFORM_DESTINATION + ): raise exception.InvalidMinionPoolSelection( "The selected OSMorphing minion pool for instances %s " "('%s') is configured as a '%s' pool. The pool must " - "be of type %s to be used for OSMorphing." % ( - instances, pool_id, + "be of type %s to be used for OSMorphing." + % ( + instances, + pool_id, osmorphing_pool.platform, - constants.PROVIDER_PLATFORM_DESTINATION)) + constants.PROVIDER_PLATFORM_DESTINATION, + ) + ) _check_pool_minion_count( - osmorphing_pool, instances, - minion_pool_type="OSMorphing") + osmorphing_pool, instances, minion_pool_type="OSMorphing" + ) LOG.debug( "Successfully validated compatibility of destination " "minion pool '%s' for use as OSMorphing minion for " - "instances %s during action '%s'." % ( - pool_id, instances, action['id'])) + "instances %s during action '%s'." + % (pool_id, instances, action['id']) + ) LOG.debug( "Successfully validated minion pool selections for action '%s' " - "with properties: %s", action['id'], action) + "with properties: %s", + action['id'], + action, + ) - def allocate_minion_machines_for_transfer( - self, ctxt, transfer): + def allocate_minion_machines_for_transfer(self, ctxt, transfer): try: self._run_machine_allocation_subflow_for_action( - ctxt, transfer, + ctxt, + transfer, constants.TRANSFER_ACTION_TYPE_TRANSFER, include_transfer_minions=True, - include_osmorphing_minions=False) + include_osmorphing_minions=False, + ) except Exception as ex: LOG.warn( "Error occurred while allocating minion machines for " "Transfer with ID '%s'. Removing all allocations. " - "Error was: %s" % ( - transfer['id'], utils.get_exception_details())) + "Error was: %s" % (transfer['id'], utils.get_exception_details()) + ) self._cleanup_machines_with_statuses_for_action( - ctxt, transfer['id'], - [constants.MINION_MACHINE_STATUS_UNINITIALIZED]) - self.deallocate_minion_machines_for_action( - ctxt, transfer['id']) - (self._rpc_conductor_client - .report_transfer_minions_allocation_error( - ctxt, transfer['id'], str(ex))) + ctxt, transfer['id'], [constants.MINION_MACHINE_STATUS_UNINITIALIZED] + ) + self.deallocate_minion_machines_for_action(ctxt, transfer['id']) + ( + self._rpc_conductor_client.report_transfer_minions_allocation_error( + ctxt, transfer['id'], str(ex) + ) + ) raise def allocate_minion_machines_for_deployment( - self, ctxt, deployment, include_transfer_minions=True, - include_osmorphing_minions=True): + self, + ctxt, + deployment, + include_transfer_minions=True, + include_osmorphing_minions=True, + ): try: self._run_machine_allocation_subflow_for_action( - ctxt, deployment, + ctxt, + deployment, constants.TRANSFER_ACTION_TYPE_DEPLOYMENT, include_transfer_minions=include_transfer_minions, - include_osmorphing_minions=include_osmorphing_minions) + include_osmorphing_minions=include_osmorphing_minions, + ) except Exception as ex: LOG.warn( "Error occurred while allocating minion machines for " "Deployment with ID '%s'. Removing all allocations. " - "Error was: %s" % ( - deployment['id'], utils.get_exception_details())) + "Error was: %s" % (deployment['id'], utils.get_exception_details()) + ) self._cleanup_machines_with_statuses_for_action( - ctxt, deployment['id'], - [constants.MINION_MACHINE_STATUS_UNINITIALIZED]) - self.deallocate_minion_machines_for_action( - ctxt, deployment['id']) - (self._rpc_conductor_client - .report_deployment_minions_allocation_error( - ctxt, deployment['id'], str(ex))) + ctxt, deployment['id'], [constants.MINION_MACHINE_STATUS_UNINITIALIZED] + ) + self.deallocate_minion_machines_for_action(ctxt, deployment['id']) + ( + self._rpc_conductor_client.report_deployment_minions_allocation_error( + ctxt, deployment['id'], str(ex) + ) + ) raise def _make_minion_machine_allocation_subflow_for_action( - self, ctxt, minion_pool, action_id, action_instances, - subflow_name, inject_for_tasks=None): - """ Creates a subflow for allocating minion machines from the + self, + ctxt, + minion_pool, + action_id, + action_instances, + subflow_name, + inject_for_tasks=None, + ): + """Creates a subflow for allocating minion machines from the provided minion pool to the given action (one for each instance) Returns a mapping between the action's instaces' IDs and the minion @@ -574,15 +694,18 @@ def _make_minion_machine_allocation_subflow_for_action( "": ""}} """ currently_available_machines = [ - machine for machine in minion_pool.minion_machines - if machine.allocation_status - == constants.MINION_MACHINE_STATUS_AVAILABLE] - extra_available_machine_slots = ( - minion_pool.maximum_minions - len(minion_pool.minion_machines)) + machine + for machine in minion_pool.minion_machines + if machine.allocation_status == constants.MINION_MACHINE_STATUS_AVAILABLE + ] + extra_available_machine_slots = minion_pool.maximum_minions - len( + minion_pool.minion_machines + ) num_instances = len(action_instances) num_currently_available_machines = len(currently_available_machines) - if (num_instances > (len(currently_available_machines) + ( - extra_available_machine_slots))): + if num_instances > ( + len(currently_available_machines) + (extra_available_machine_slots) + ): raise exception.InvalidMinionPoolState( "Minion pool '%s' is unable to accommodate the requested " "number of machines (%s) for transfer action '%s', as it only " @@ -590,10 +713,15 @@ def _make_minion_machine_allocation_subflow_for_action( "further %d until the maximum is reached. Please either " "increase the number of maximum machines for the pool " "or wait for other minions to become available before " - "retrying." % ( - minion_pool.id, num_instances, action_id, + "retrying." + % ( + minion_pool.id, + num_instances, + action_id, num_currently_available_machines, - extra_available_machine_slots)) + extra_available_machine_slots, + ) + ) def _select_machine(minion_pool, exclude=None): selected_machine = None @@ -604,17 +732,24 @@ def _select_machine(minion_pool, exclude=None): for machine in minion_pool.minion_machines: if exclude and machine.id in exclude: LOG.debug( - "Excluding minion machine '%s' from search for use " - "action '%s'", machine.id, action_id) + "Excluding minion machine '%s' from search for use action '%s'", + machine.id, + action_id, + ) continue - if (machine.allocation_status != - constants.MINION_MACHINE_STATUS_AVAILABLE): + if ( + machine.allocation_status + != constants.MINION_MACHINE_STATUS_AVAILABLE + ): LOG.debug( "Minion machine with ID '%s' is in status '%s' " "instead of the expected '%s'. Skipping for use " "with action '%s'.", - machine.id, machine.allocation_status, - constants.MINION_MACHINE_STATUS_AVAILABLE, action_id) + machine.id, + machine.allocation_status, + constants.MINION_MACHINE_STATUS_AVAILABLE, + action_id, + ) continue selected_machine = machine break @@ -625,15 +760,16 @@ def _select_machine(minion_pool, exclude=None): machine_db_entries_to_add = [] existing_machines_to_allocate = {} for instance in action_instances: - if instance in instance_minion_allocations: raise exception.InvalidInput( "Instance with identifier '%s' passed twice for " "minion machine allocation from pool '%s' for action " - "'%s'. Full instances list was: %s" % - (instance, minion_pool.id, action_id, action_instances)) + "'%s'. Full instances list was: %s" + % (instance, minion_pool.id, action_id, action_instances) + ) minion_machine = _select_machine( - minion_pool, exclude=instance_minion_allocations.values()) + minion_pool, exclude=instance_minion_allocations.values() + ) if minion_machine: # take note of the machine and setup a healthcheck: instance_minion_allocations[instance] = minion_machine.id @@ -641,40 +777,56 @@ def _select_machine(minion_pool, exclude=None): LOG.debug( "Allocating pre-existing machine '%s' from pool '%s' for " "use with action with ID '%s'.", - minion_machine.id, minion_pool.id, action_id) + minion_machine.id, + minion_pool.id, + action_id, + ) allocation_subflow.add( self._get_healtchcheck_flow_for_minion_machine( - minion_pool, minion_machine, + minion_pool, + minion_machine, allocate_to_action=action_id, power_on_machine=True, inject_for_tasks=inject_for_tasks, machine_status_on_success=( - constants.MINION_MACHINE_STATUS_IN_USE))) + constants.MINION_MACHINE_STATUS_IN_USE + ), + ) + ) else: # add task which creates the new machine: new_machine_id = str(uuid.uuid4()) LOG.debug( "New minion machine with ID '%s' will be created for " "minion pool '%s' for use with action '%s'.", - new_machine_id, minion_pool.id, action_id) + new_machine_id, + minion_pool.id, + action_id, + ) new_minion_machine = models.MinionMachine() new_minion_machine.id = new_machine_id new_minion_machine.pool_id = minion_pool.id new_minion_machine.allocation_status = ( - constants.MINION_MACHINE_STATUS_UNINITIALIZED) + constants.MINION_MACHINE_STATUS_UNINITIALIZED + ) new_minion_machine.power_status = ( - constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED) + constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED + ) new_minion_machine.allocated_action = action_id machine_db_entries_to_add.append(new_minion_machine) instance_minion_allocations[instance] = new_machine_id allocation_subflow.add( minion_mgr_tasks.AllocateMinionMachineTask( - minion_pool.id, new_machine_id, minion_pool.platform, + minion_pool.id, + new_machine_id, + minion_pool.platform, allocate_to_action=action_id, raise_on_cleanup_failure=False, - inject=inject_for_tasks)) + inject=inject_for_tasks, + ) + ) new_machine_db_entries_added = [] try: @@ -684,16 +836,25 @@ def _select_machine(minion_pool, exclude=None): "Marking the following pre-existing minion machines " "from pool '%s' of action '%s' for each instance as " "allocated with the DB: %s", - minion_pool.id, action_id, existing_machines_to_allocate) + minion_pool.id, + action_id, + existing_machines_to_allocate, + ) db_api.set_minion_machines_allocation_statuses( - ctxt, list(existing_machines_to_allocate.keys()), - action_id, constants.MINION_MACHINE_STATUS_RESERVED, - refresh_allocation_time=True) + ctxt, + list(existing_machines_to_allocate.keys()), + action_id, + constants.MINION_MACHINE_STATUS_RESERVED, + refresh_allocation_time=True, + ) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, "The following pre-existing minion machines will be " - "allocated to transfer action '%s': %s" % ( - action_id, list(existing_machines_to_allocate.keys()))) + "allocated to transfer action '%s': %s" + % (action_id, list(existing_machines_to_allocate.keys())), + ) # add any new machine entries to the DB: if machine_db_entries_to_add: @@ -701,22 +862,30 @@ def _select_machine(minion_pool, exclude=None): LOG.info( "Adding new minion machine with ID '%s' to the DB " "for pool '%s' for use with action '%s'.", - new_machine_id, minion_pool.id, action_id) + new_machine_id, + minion_pool.id, + action_id, + ) db_api.add_minion_machine(ctxt, new_machine) new_machine_db_entries_added.append(new_machine.id) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, "The following new minion machines will be created for use" - " in transfer action '%s': %s" % ( - action_id, [m.id for m in machine_db_entries_to_add])) + " in transfer action '%s': %s" + % (action_id, [m.id for m in machine_db_entries_to_add]), + ) except Exception: LOG.warn( "Exception occurred while adding new minion machine entries to" " the DB for pool '%s' for use with action '%s'. Clearing " "any DB entries added so far (%s). Error was: %s", - minion_pool.id, action_id, + minion_pool.id, + action_id, new_machine_db_entries_added, - utils.get_exception_details()) + utils.get_exception_details(), + ) try: LOG.debug( "Reverting the following pre-existing minion machines from" @@ -725,18 +894,25 @@ def _select_machine(minion_pool, exclude=None): minion_pool.id, constants.MINION_MACHINE_STATUS_AVAILABLE, action_id, - list(existing_machines_to_allocate.keys())) + list(existing_machines_to_allocate.keys()), + ) db_api.set_minion_machines_allocation_statuses( - ctxt, list(existing_machines_to_allocate.keys()), - None, constants.MINION_MACHINE_STATUS_AVAILABLE, - refresh_allocation_time=False) + ctxt, + list(existing_machines_to_allocate.keys()), + None, + constants.MINION_MACHINE_STATUS_AVAILABLE, + refresh_allocation_time=False, + ) except Exception: LOG.warn( "Failed to deallocate the following machines from pool " "'%s' following allocation error for action '%s': %s. " "Error trace was: %s", - minion_pool.id, action_id, existing_machines_to_allocate, - utils.get_exception_details()) + minion_pool.id, + action_id, + existing_machines_to_allocate, + utils.get_exception_details(), + ) for new_machine in new_machine_db_entries_added: try: db_api.delete_minion_machine(ctxt, new_machine) @@ -744,36 +920,48 @@ def _select_machine(minion_pool, exclude=None): LOG.warn( "Error occurred while removing minion machine entry " "'%s' from the DB. This may leave the pool in an " - "inconsistent state. Error trace was: %s" % ( - new_machine, utils.get_exception_details())) + "inconsistent state. Error trace was: %s" + % (new_machine, utils.get_exception_details()) + ) continue raise LOG.debug( "The following minion machine allocation from pool '%s' were or " "will be made for action '%s': %s", - minion_pool.id, action_id, instance_minion_allocations) + minion_pool.id, + action_id, + instance_minion_allocations, + ) return { "flow": allocation_subflow, - "action_instance_minion_allocation_mappings": ( - instance_minion_allocations)} + "action_instance_minion_allocation_mappings": (instance_minion_allocations), + } def _run_machine_allocation_subflow_for_action( - self, ctxt, action, action_type, include_transfer_minions=True, - include_osmorphing_minions=True): - """ Defines and starts a taskflow subflow for allocating minion + self, + ctxt, + action, + action_type, + include_transfer_minions=True, + include_osmorphing_minions=True, + ): + """Defines and starts a taskflow subflow for allocating minion machines for the given action. If there are no more minion machines available, upscaling will occur. Also adds to the DB/marks as allocated any minion machines on the spot. """ required_action_properties = [ - 'id', 'instances', 'origin_minion_pool_id', + 'id', + 'instances', + 'origin_minion_pool_id', 'destination_minion_pool_id', - 'instance_osmorphing_minion_pool_mappings'] + 'instance_osmorphing_minion_pool_mappings', + ] self._check_keys_for_action_dict( - action, required_action_properties, - operation="minion machine selection") + action, required_action_properties, operation="minion machine selection" + ) allocation_flow_name_format = None machines_allocation_subflow_name_format = None @@ -782,213 +970,266 @@ def _run_machine_allocation_subflow_for_action( allocation_confirmation_reporting_task_class = None if action_type == constants.TRANSFER_ACTION_TYPE_DEPLOYMENT: allocation_flow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT)) + minion_mgr_tasks.MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT + ) allocation_failure_reporting_task_class = ( - minion_mgr_tasks.ReportMinionAllocationFailureForMigrationTask) + minion_mgr_tasks.ReportMinionAllocationFailureForMigrationTask + ) allocation_confirmation_reporting_task_class = ( - minion_mgr_tasks.ConfirmMinionAllocationForMigrationTask) + minion_mgr_tasks.ConfirmMinionAllocationForMigrationTask + ) machines_allocation_subflow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_MIGRATION_ALLOCATION_SUBFLOW_NAME_FORMAT)) - machine_action_allocation_subflow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_ALLOCATE_MACHINES_FOR_MIGRATION_SUBFLOW_NAME_FORMAT)) # noqa: E501 + minion_mgr_tasks.MINION_POOL_MIGRATION_ALLOCATION_SUBFLOW_NAME_FORMAT + ) + machine_action_allocation_subflow_name_format = minion_mgr_tasks.MINION_POOL_ALLOCATE_MACHINES_FOR_MIGRATION_SUBFLOW_NAME_FORMAT # noqa: E501 elif action_type == constants.TRANSFER_ACTION_TYPE_TRANSFER: allocation_flow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT)) + minion_mgr_tasks.MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT + ) allocation_failure_reporting_task_class = ( - minion_mgr_tasks.ReportMinionAllocationFailureForReplicaTask) + minion_mgr_tasks.ReportMinionAllocationFailureForReplicaTask + ) allocation_confirmation_reporting_task_class = ( - minion_mgr_tasks.ConfirmMinionAllocationForReplicaTask) + minion_mgr_tasks.ConfirmMinionAllocationForReplicaTask + ) machines_allocation_subflow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_REPLICA_ALLOCATION_SUBFLOW_NAME_FORMAT)) - machine_action_allocation_subflow_name_format = ( - (minion_mgr_tasks. - MINION_POOL_ALLOCATE_MACHINES_FOR_REPLICA_SUBFLOW_NAME_FORMAT)) # noqa: E501 + minion_mgr_tasks.MINION_POOL_REPLICA_ALLOCATION_SUBFLOW_NAME_FORMAT + ) + machine_action_allocation_subflow_name_format = minion_mgr_tasks.MINION_POOL_ALLOCATE_MACHINES_FOR_REPLICA_SUBFLOW_NAME_FORMAT # noqa: E501 else: raise exception.InvalidInput( - "Unknown transfer action type '%s'" % action_type) + "Unknown transfer action type '%s'" % action_type + ) # define main flow: - main_allocation_flow_name = ( - allocation_flow_name_format % action['id']) + main_allocation_flow_name = allocation_flow_name_format % action['id'] main_allocation_flow = linear_flow.Flow(main_allocation_flow_name) instance_machine_allocations = { - instance: {} for instance in action['instances']} + instance: {} for instance in action['instances'] + } # add allocation failure reporting task: - main_allocation_flow.add( - allocation_failure_reporting_task_class( - action['id'])) + main_allocation_flow.add(allocation_failure_reporting_task_class(action['id'])) # define subflow for all the pool minions allocations: machines_subflow = unordered_flow.Flow( - machines_allocation_subflow_name_format % action['id']) + machines_allocation_subflow_name_format % action['id'] + ) pools_used = [] - def _check_pool_allocation_status( - minion_pool, minion_pool_type): + def _check_pool_allocation_status(minion_pool, minion_pool_type): if minion_pool.status != constants.MINION_POOL_STATUS_ALLOCATED: raise exception.InvalidMinionPoolState( "Minion Pool '%s' is an invalid state ('%s') to be " "used as a %s pool for action '%s'. The pool must be " - "in '%s' status." % ( - minion_pool.id, minion_pool.status, - minion_pool_type.lower(), action['id'], - constants.MINION_POOL_STATUS_ALLOCATED)) + "in '%s' status." + % ( + minion_pool.id, + minion_pool.status, + minion_pool_type.lower(), + action['id'], + constants.MINION_POOL_STATUS_ALLOCATED, + ) + ) # add subflow for origin pool: if include_transfer_minions and action['origin_minion_pool_id']: pools_used.append(action['origin_minion_pool_id']) with minion_manager_utils.get_minion_pool_lock( - action['origin_minion_pool_id'], external=True): + action['origin_minion_pool_id'], external=True + ): # fetch pool, origin endpoint, and initial store: minion_pool = self._get_minion_pool( - ctxt, action['origin_minion_pool_id'], - include_machines=True, include_events=False, - include_progress_updates=False) + ctxt, + action['origin_minion_pool_id'], + include_machines=True, + include_events=False, + include_progress_updates=False, + ) _check_pool_allocation_status(minion_pool, "source") endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) + ctxt, minion_pool.endpoint_id + ) origin_pool_store = self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict) + ctxt, minion_pool, endpoint_dict + ) # add subflow for machine allocations from origin pool: - subflow_name = ( - machine_action_allocation_subflow_name_format % ( - minion_pool.id, action['id'])) + subflow_name = machine_action_allocation_subflow_name_format % ( + minion_pool.id, + action['id'], + ) # NOTE: required to avoid internal taskflow conflicts subflow_name = "origin-%s" % subflow_name allocations_subflow_result = ( self._make_minion_machine_allocation_subflow_for_action( - ctxt, minion_pool, action['id'], action['instances'], - subflow_name, inject_for_tasks=origin_pool_store)) + ctxt, + minion_pool, + action['id'], + action['instances'], + subflow_name, + inject_for_tasks=origin_pool_store, + ) + ) machines_subflow.add(allocations_subflow_result['flow']) # register each instances' origin minion: source_machine_allocations = allocations_subflow_result[ - 'action_instance_minion_allocation_mappings'] - for (action_instance_id, allocated_minion_id) in ( - source_machine_allocations.items()): - instance_machine_allocations[ - action_instance_id]['origin_minion_id'] = ( - allocated_minion_id) + 'action_instance_minion_allocation_mappings' + ] + for ( + action_instance_id, + allocated_minion_id, + ) in source_machine_allocations.items(): + instance_machine_allocations[action_instance_id][ + 'origin_minion_id' + ] = allocated_minion_id # add subflow for destination pool: if include_transfer_minions and action['destination_minion_pool_id']: pools_used.append(action['destination_minion_pool_id']) with minion_manager_utils.get_minion_pool_lock( - action['destination_minion_pool_id'], external=True): + action['destination_minion_pool_id'], external=True + ): # fetch pool, destination endpoint, and initial store: minion_pool = self._get_minion_pool( - ctxt, action['destination_minion_pool_id'], - include_machines=True, include_events=False, - include_progress_updates=False) + ctxt, + action['destination_minion_pool_id'], + include_machines=True, + include_events=False, + include_progress_updates=False, + ) _check_pool_allocation_status(minion_pool, "destination") endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) - destination_pool_store = ( - self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict)) + ctxt, minion_pool.endpoint_id + ) + destination_pool_store = self._get_pool_initial_taskflow_store_base( + ctxt, minion_pool, endpoint_dict + ) # add subflow for machine allocations from destination pool: - subflow_name = ( - machine_action_allocation_subflow_name_format % ( - minion_pool.id, action['id'])) + subflow_name = machine_action_allocation_subflow_name_format % ( + minion_pool.id, + action['id'], + ) # NOTE: required to avoid internal taskflow conflicts subflow_name = "destination-%s" % subflow_name allocations_subflow_result = ( self._make_minion_machine_allocation_subflow_for_action( - ctxt, minion_pool, action['id'], action['instances'], + ctxt, + minion_pool, + action['id'], + action['instances'], subflow_name, - inject_for_tasks=destination_pool_store)) + inject_for_tasks=destination_pool_store, + ) + ) machines_subflow.add(allocations_subflow_result['flow']) destination_machine_allocations = allocations_subflow_result[ - 'action_instance_minion_allocation_mappings'] + 'action_instance_minion_allocation_mappings' + ] # register each instances' destination minion: - for (action_instance_id, allocated_minion_id) in ( - destination_machine_allocations.items()): - instance_machine_allocations[ - action_instance_id]['destination_minion_id'] = ( - allocated_minion_id) + for ( + action_instance_id, + allocated_minion_id, + ) in destination_machine_allocations.items(): + instance_machine_allocations[action_instance_id][ + 'destination_minion_id' + ] = allocated_minion_id # add subflow for OSMorphing minions: osmorphing_pool_instance_mappings = {} - for (action_instance_id, mapped_pool_id) in action[ - 'instance_osmorphing_minion_pool_mappings'].items(): + for action_instance_id, mapped_pool_id in action[ + 'instance_osmorphing_minion_pool_mappings' + ].items(): if mapped_pool_id not in osmorphing_pool_instance_mappings: - osmorphing_pool_instance_mappings[ - mapped_pool_id] = [action_instance_id] + osmorphing_pool_instance_mappings[mapped_pool_id] = [action_instance_id] else: osmorphing_pool_instance_mappings[mapped_pool_id].append( - action_instance_id) + action_instance_id + ) if include_osmorphing_minions and osmorphing_pool_instance_mappings: - for (osmorphing_pool_id, action_instance_ids) in ( - osmorphing_pool_instance_mappings.items()): + for ( + osmorphing_pool_id, + action_instance_ids, + ) in osmorphing_pool_instance_mappings.items(): # if the destination pool was selected as an OSMorphing pool # for any instances, we simply re-use all of the destination # minions for said instances: if action['destination_minion_pool_id'] and ( - include_osmorphing_minions and ( - osmorphing_pool_id == ( - action['destination_minion_pool_id']) and ( - include_transfer_minions))): + include_osmorphing_minions + and ( + osmorphing_pool_id == (action['destination_minion_pool_id']) + and (include_transfer_minions) + ) + ): LOG.debug( "Reusing destination minion pool with ID '%s' for the " "following instances which had it selected as an " "OSMorphing pool for action '%s': %s", - osmorphing_pool_id, action['id'], - action_instance_ids) + osmorphing_pool_id, + action['id'], + action_instance_ids, + ) for instance in action_instance_ids: - instance_machine_allocations[ - instance]['osmorphing_minion_id'] = ( - instance_machine_allocations[ - instance]['destination_minion_id']) + instance_machine_allocations[instance][ + 'osmorphing_minion_id' + ] = instance_machine_allocations[instance][ + 'destination_minion_id' + ] continue with minion_manager_utils.get_minion_pool_lock( - osmorphing_pool_id, external=True): + osmorphing_pool_id, external=True + ): pools_used.append(osmorphing_pool_id) # fetch pool, destination endpoint, and initial store: minion_pool = self._get_minion_pool( - ctxt, osmorphing_pool_id, - include_machines=True, include_events=False, - include_progress_updates=False) - _check_pool_allocation_status( - minion_pool, "OSMorphing") + ctxt, + osmorphing_pool_id, + include_machines=True, + include_events=False, + include_progress_updates=False, + ) + _check_pool_allocation_status(minion_pool, "OSMorphing") endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) - osmorphing_pool_store = ( - self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict)) + ctxt, minion_pool.endpoint_id + ) + osmorphing_pool_store = self._get_pool_initial_taskflow_store_base( + ctxt, minion_pool, endpoint_dict + ) # add subflow for machine allocations from osmorphing pool: - subflow_name = ( - machine_action_allocation_subflow_name_format % ( - minion_pool.id, action['id'])) + subflow_name = machine_action_allocation_subflow_name_format % ( + minion_pool.id, + action['id'], + ) # NOTE: required to avoid internal taskflow conflicts subflow_name = "osmorphing-%s" % subflow_name allocations_subflow_result = ( self._make_minion_machine_allocation_subflow_for_action( # noqa: E501 - ctxt, minion_pool, action['id'], + ctxt, + minion_pool, + action['id'], action_instance_ids, subflow_name, - inject_for_tasks=osmorphing_pool_store)) + inject_for_tasks=osmorphing_pool_store, + ) + ) machines_subflow.add(allocations_subflow_result['flow']) # register each instances' osmorphing minion: - osmorphing_machine_allocations = ( - allocations_subflow_result[ - 'action_instance_minion_allocation_mappings']) - for (action_instance_id, allocated_minion_id) in ( - osmorphing_machine_allocations.items()): - instance_machine_allocations[ - action_instance_id]['osmorphing_minion_id'] = ( - allocated_minion_id) + osmorphing_machine_allocations = allocations_subflow_result[ + 'action_instance_minion_allocation_mappings' + ] + for ( + action_instance_id, + allocated_minion_id, + ) in osmorphing_machine_allocations.items(): + instance_machine_allocations[action_instance_id][ + 'osmorphing_minion_id' + ] = allocated_minion_id # add the machines subflow to the main flow: main_allocation_flow.add(machines_subflow) @@ -997,43 +1238,53 @@ def _check_pool_allocation_status( # to the conductor at the end of the flow: main_allocation_flow.add( allocation_confirmation_reporting_task_class( - action['id'], instance_machine_allocations)) + action['id'], instance_machine_allocations + ) + ) LOG.info( "Starting main minion allocation flow '%s' for with ID '%s'. " - "The minion allocations will be: %s" % ( - main_allocation_flow_name, action['id'], - instance_machine_allocations)) + "The minion allocations will be: %s" + % (main_allocation_flow_name, action['id'], instance_machine_allocations) + ) try: self._taskflow_runner.run_flow_in_background( - main_allocation_flow, store={"context": ctxt}) + main_allocation_flow, store={"context": ctxt} + ) except Exception as ex: minion_pool_id = None try: for minion_pool_id in pools_used: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_ERROR, + ctxt, + minion_pool.id, + constants.TASK_EVENT_ERROR, "A fatal exception occurred while attempting to start " "the task flow for allocating machines for %s '%s'. " "Forced deallocation and reallocation may be required." " Please review the minion manager logs for additional" - " details. Error was: %s" % ( - action_type, action['id'], str(ex))) + " details. Error was: %s" + % (action_type, action['id'], str(ex)), + ) except Exception: LOG.warn( "Failed to add minion pool error event for pool '%s' " "during allocation of machines for %s '%s'. Ignoring. " "Exception was: %s", - minion_pool_id, action_type, action['id'], - utils.get_exception_details()) + minion_pool_id, + action_type, + action['id'], + utils.get_exception_details(), + ) raise return main_allocation_flow def _cleanup_machines_with_statuses_for_action( - self, ctxt, action_id, targeted_statuses, exclude_pools=None): - """ Deletes all minion machines which are marked with the given + self, ctxt, action_id, targeted_statuses, exclude_pools=None + ): + """Deletes all minion machines which are marked with the given from the DB. """ if exclude_pools is None: @@ -1041,8 +1292,8 @@ def _cleanup_machines_with_statuses_for_action( machines = db_api.get_minion_machines(ctxt, action_id) if not machines: LOG.debug( - "No minion machines allocated to action '%s'. Returning.", - action_id) + "No minion machines allocated to action '%s'. Returning.", action_id + ) return pool_machine_mappings = {} @@ -1051,15 +1302,20 @@ def _cleanup_machines_with_statuses_for_action( LOG.debug( "Skipping deletion of machine '%s' from pool '%s' as " "its status (%s) is not one of the targeted statuses (%s)", - machine.id, machine.pool_id, machine.allocation_status, - targeted_statuses) + machine.id, + machine.pool_id, + machine.allocation_status, + targeted_statuses, + ) continue if machine.pool_id in exclude_pools: LOG.debug( "Skipping deletion of machine '%s' (status '%s') from " "whitelisted pool '%s'", - machine.id, machine.allocation_status, - machine.pool_id) + machine.id, + machine.allocation_status, + machine.pool_id, + ) continue if machine.pool_id not in pool_machine_mappings: @@ -1067,62 +1323,77 @@ def _cleanup_machines_with_statuses_for_action( else: pool_machine_mappings[machine.pool_id].append(machine) - for (pool_id, machines) in pool_machine_mappings.items(): - with (minion_manager_utils. - get_minion_pool_lock(pool_id, external=True)): + for pool_id, machines in pool_machine_mappings.items(): + with minion_manager_utils.get_minion_pool_lock(pool_id, external=True): for machine in machines: LOG.debug( "Deleting machine with ID '%s' " "(pool '%s', status '%s') " - "from the DB.", machine.id, - pool_id, machine.allocation_status) + "from the DB.", + machine.id, + pool_id, + machine.allocation_status, + ) db_api.delete_minion_machine(ctxt, machine.id) def deallocate_minion_machine(self, ctxt, minion_machine_id): - minion_machine = db_api.get_minion_machine( - ctxt, minion_machine_id) + minion_machine = db_api.get_minion_machine(ctxt, minion_machine_id) if not minion_machine: LOG.warn( "Could not find minion machine with ID '%s' for deallocation. " "Presuming it was deleted and returning early", - minion_machine_id) + minion_machine_id, + ) return machine_allocated_status = constants.MINION_MACHINE_STATUS_IN_USE with minion_manager_utils.get_minion_pool_lock( - minion_machine.pool_id, external=True): - if (minion_machine.allocation_status != machine_allocated_status - or not minion_machine.allocated_action): + minion_machine.pool_id, external=True + ): + if ( + minion_machine.allocation_status != machine_allocated_status + or not minion_machine.allocated_action + ): LOG.warn( "Minion machine '%s' was either in an improper status (%s)" ", or did not have an associated action ('%s') for " "deallocation request. Marking as available anyway.", - minion_machine.id, minion_machine.allocation_status, - minion_machine.allocated_action) + minion_machine.id, + minion_machine.allocation_status, + minion_machine.allocated_action, + ) LOG.debug( "Attempting to deallocate minion pool machine '%s' " "(currently allocated to action '%s' with status '%s')", - minion_machine.id, minion_machine.allocated_action, - minion_machine.allocation_status) + minion_machine.id, + minion_machine.allocated_action, + minion_machine.allocation_status, + ) db_api.update_minion_machine( - ctxt, minion_machine.id, { + ctxt, + minion_machine.id, + { "allocation_status": constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 - "allocated_action": None}) + "allocated_action": None, + }, + ) LOG.debug( - "Successfully deallocated minion machine with '%s'.", - minion_machine.id) + "Successfully deallocated minion machine with '%s'.", minion_machine.id + ) def deallocate_minion_machines_for_action(self, ctxt, action_id): allocated_minion_machines = db_api.get_minion_machines( - ctxt, allocated_action_id=action_id) + ctxt, allocated_action_id=action_id + ) if not allocated_minion_machines: LOG.debug( "No minion machines seem to have been used for action with " "base_id '%s'. Skipping minion machine deallocation.", - action_id) + action_id, + ) return # categorise machine objects by pool: @@ -1133,9 +1404,8 @@ def deallocate_minion_machines_for_action(self, ctxt, action_id): pool_machine_mappings[machine.pool_id].append(machine) # iterate over each pool and its machines allocated to this action: - for (pool_id, pool_machines) in pool_machine_mappings.items(): - with minion_manager_utils.get_minion_pool_lock( - pool_id, external=True): + for pool_id, pool_machines in pool_machine_mappings.items(): + with minion_manager_utils.get_minion_pool_lock(pool_id, external=True): machine_ids_to_deallocate = [] # NOTE: this is a workaround in case some crash/restart happens # in the minion-manager service while new machine DB entries @@ -1143,159 +1413,213 @@ def deallocate_minion_machines_for_action(self, ctxt, action_id): # reached for them to ever get out of 'UNINITIALIZED' status: for machine in pool_machines: if machine.allocation_status == ( - constants.MINION_MACHINE_STATUS_UNINITIALIZED): + constants.MINION_MACHINE_STATUS_UNINITIALIZED + ): LOG.warn( "Found minion machine '%s' in pool '%s' which " "is in '%s' status. Removing from the DB " - "entirely." % - (machine.id, pool_id, machine.allocation_status)) - db_api.delete_minion_machine( - ctxt, machine.id) + "entirely." + % (machine.id, pool_id, machine.allocation_status) + ) + db_api.delete_minion_machine(ctxt, machine.id) LOG.info( "Successfully deleted minion machine entry '%s' " - "from pool '%s' from the DB.", machine.id, - pool_id) + "from pool '%s' from the DB.", + machine.id, + pool_id, + ) continue LOG.debug( "Going to mark minion machine '%s' (current status " "'%s') of pool '%s' as available following machine " - "deallocation request for action '%s'.", machine.id, - machine.allocation_status, pool_id, action_id) + "deallocation request for action '%s'.", + machine.id, + machine.allocation_status, + pool_id, + action_id, + ) machine_ids_to_deallocate.append(machine.id) LOG.info( "Marking minion machines '%s' from pool '%s' for " "as available after having been allocated to action '%s'.", - machine_ids_to_deallocate, pool_id, action_id) + machine_ids_to_deallocate, + pool_id, + action_id, + ) db_api.set_minion_machines_allocation_statuses( - ctxt, machine_ids_to_deallocate, None, + ctxt, + machine_ids_to_deallocate, + None, constants.MINION_MACHINE_STATUS_AVAILABLE, - refresh_allocation_time=False) + refresh_allocation_time=False, + ) LOG.debug( "Successfully released all minion machines associated " - "with action with base_id '%s'.", action_id) + "with action with base_id '%s'.", + action_id, + ) def _get_healtchcheck_flow_for_minion_machine( - self, minion_pool, minion_machine, allocate_to_action=None, - machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 - power_on_machine=True, inject_for_tasks=None): - """ Returns a taskflow graph flow with a healtcheck task - and redeployment subflow on error. """ + self, + minion_pool, + minion_machine, + allocate_to_action=None, + machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 + power_on_machine=True, + inject_for_tasks=None, + ): + """Returns a taskflow graph flow with a healtcheck task + and redeployment subflow on error.""" # define healthcheck subflow for each machine: machine_healthcheck_subflow = graph_flow.Flow( - (minion_mgr_tasks. - MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT) % ( - minion_pool.id, minion_machine.id)) + (minion_mgr_tasks.MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT) + % (minion_pool.id, minion_machine.id) + ) # add healtcheck task to healthcheck subflow: - machine_healthcheck_task = ( - minion_mgr_tasks.HealthcheckMinionMachineTask( - minion_pool.id, minion_machine.id, minion_pool.platform, - machine_status_on_success=machine_status_on_success, - inject=inject_for_tasks, - # we prevent a raise here as the healthcheck subflow - # will take care of redeploying the instance later: - fail_on_error=False)) + machine_healthcheck_task = minion_mgr_tasks.HealthcheckMinionMachineTask( + minion_pool.id, + minion_machine.id, + minion_pool.platform, + machine_status_on_success=machine_status_on_success, + inject=inject_for_tasks, + # we prevent a raise here as the healthcheck subflow + # will take care of redeploying the instance later: + fail_on_error=False, + ) machine_healthcheck_subflow.add(machine_healthcheck_task) # optionally add minion machine power on task: if power_on_machine: if minion_machine.power_status == ( - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF): + constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF + ): power_on_task = minion_mgr_tasks.PowerOnMinionMachineTask( - minion_pool.id, minion_machine.id, minion_pool.platform, + minion_pool.id, + minion_machine.id, + minion_pool.platform, inject=inject_for_tasks, # we prevent a raise here as the healthcheck subflow # will take care of redeploying the instance later: - fail_on_error=False) + fail_on_error=False, + ) machine_healthcheck_subflow.add( power_on_task, # NOTE: this is required to not have taskflow attempt # (and fail) to automatically link the above Healthcheck # task to the power on task based on inputs/outputs alone: - resolve_existing=False) + resolve_existing=False, + ) machine_healthcheck_subflow.link( - power_on_task, machine_healthcheck_task, + power_on_task, + machine_healthcheck_task, # NOTE: taskflow gets confused when a task in the graph # flow is linked to two others with no decider for each # so we have to add a dummy decider which will always # greenlight the rest of the execution here: decider=taskflow_utils.DummyDecider(allow=True), - decider_depth=taskflow_deciders.Depth.FLOW) + decider_depth=taskflow_deciders.Depth.FLOW, + ) else: LOG.debug( "Minion Machine with ID '%s' of pool '%s' is in power " "state '%s' during healtchcheck subflow definition. " "Not adding any power on task for it.", - minion_machine.id, minion_machine.pool_id, - minion_machine.power_status) + minion_machine.id, + minion_machine.pool_id, + minion_machine.power_status, + ) # define reallocation subflow: machine_reallocation_subflow = linear_flow.Flow( - (minion_mgr_tasks. - MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT) % ( - minion_pool.id, minion_machine.id)) + (minion_mgr_tasks.MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT) + % (minion_pool.id, minion_machine.id) + ) machine_reallocation_subflow.add( minion_mgr_tasks.DeallocateMinionMachineTask( - minion_pool.id, minion_machine.id, minion_pool.platform, - inject=inject_for_tasks)) + minion_pool.id, + minion_machine.id, + minion_pool.platform, + inject=inject_for_tasks, + ) + ) machine_reallocation_subflow.add( minion_mgr_tasks.AllocateMinionMachineTask( - minion_pool.id, minion_machine.id, minion_pool.platform, + minion_pool.id, + minion_machine.id, + minion_pool.platform, allocate_to_action=allocate_to_action, - inject=inject_for_tasks)) + inject=inject_for_tasks, + ) + ) machine_healthcheck_subflow.add( machine_reallocation_subflow, # NOTE: this is required to not have taskflow attempt (and fail) # to automatically link the above Healthcheck task to the # new subflow based on inputs/outputs alone: - resolve_existing=False) + resolve_existing=False, + ) # link reallocation subflow to healthcheck task: machine_healthcheck_subflow.link( - machine_healthcheck_task, machine_reallocation_subflow, + machine_healthcheck_task, + machine_reallocation_subflow, # NOTE: this is required to prevent any parent flows from skipping: decider_depth=taskflow_deciders.Depth.FLOW, decider=minion_mgr_tasks.MinionMachineHealtchcheckDecider( - minion_pool.id, minion_machine.id, - on_successful_healthcheck=False)) + minion_pool.id, minion_machine.id, on_successful_healthcheck=False + ), + ) return machine_healthcheck_subflow - def _get_minion_pool_refresh_flow( - self, ctxt, minion_pool, requery=True): + def _get_minion_pool_refresh_flow(self, ctxt, minion_pool, requery=True): if requery: minion_pool = self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_progress_updates=False, include_events=False) + ctxt, + minion_pool.id, + include_machines=True, + include_progress_updates=False, + include_events=False, + ) # determine how many machines could be feasibily downscaled: machine_statuses = { machine.id: machine.allocation_status - for machine in minion_pool.minion_machines} + for machine in minion_pool.minion_machines + } ignorable_machine_statuses = [ constants.MINION_MACHINE_STATUS_DEALLOCATING, constants.MINION_MACHINE_STATUS_POWERING_OFF, constants.MINION_MACHINE_STATUS_ERROR, constants.MINION_MACHINE_STATUS_POWER_ERROR, - constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING] - max_minions_to_deallocate = ( - len([ - mid for mid in machine_statuses - if machine_statuses[mid] not in ignorable_machine_statuses]) - ( # noqa: E501 - minion_pool.minimum_minions)) + constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, + ] + max_minions_to_deallocate = len( + [ + mid + for mid in machine_statuses + if machine_statuses[mid] not in ignorable_machine_statuses + ] + ) - ( # noqa: E501 + minion_pool.minimum_minions + ) LOG.debug( "Determined minion pool '%s' machine deallocation number to be %d " "(pool minimum is '%d') based on current machines stauses: %s", - minion_pool.id, max_minions_to_deallocate, - minion_pool.minimum_minions, machine_statuses) + minion_pool.id, + max_minions_to_deallocate, + minion_pool.minimum_minions, + machine_statuses, + ) # define refresh flow and process all relevant machines: pool_refresh_flow = unordered_flow.Flow( - minion_mgr_tasks.MINION_POOL_REFRESH_FLOW_NAME_FORMAT % ( - minion_pool.id)) + minion_mgr_tasks.MINION_POOL_REFRESH_FLOW_NAME_FORMAT % (minion_pool.id) + ) now = timeutils.utcnow() machines_to_deallocate = [] machines_to_healthcheck = [] @@ -1308,125 +1632,163 @@ def _get_minion_pool_refresh_flow( # sublow redeploy it after the healthcheck fails: constants.MINION_MACHINE_STATUS_ERROR, constants.MINION_MACHINE_STATUS_POWER_ERROR, - constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING] + constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, + ] for machine in minion_pool.minion_machines: - if machine.allocation_status not in ( - healthcheckable_machine_statuses): + if machine.allocation_status not in (healthcheckable_machine_statuses): skipped_machines[machine.id] = ( - machine.allocation_status, machine.power_status) + machine.allocation_status, + machine.power_status, + ) continue minion_expired = True if machine.last_used_at: - expiry_time = ( - machine.last_used_at + datetime.timedelta( - seconds=minion_pool.minion_max_idle_time)) + expiry_time = machine.last_used_at + datetime.timedelta( + seconds=minion_pool.minion_max_idle_time + ) minion_expired = expiry_time <= now # deallocate the machine if it is expired: if max_minions_to_deallocate > 0 and minion_expired: if minion_pool.minion_retention_strategy == ( - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF): # noqa: E501 + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF + ): # noqa: E501 if machine.power_status in ( - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF, - constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF): # noqa: E501 + constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF, + constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF, + ): # noqa: E501 LOG.debug( "Skipping powering off minion machine '%s' of pool" " '%s' as it is already in powered off state.", - machine.id, minion_pool.id) + machine.id, + minion_pool.id, + ) # NOTE: we count this machine out of the downscaling: - max_minions_to_deallocate = ( - max_minions_to_deallocate - 1) + max_minions_to_deallocate = max_minions_to_deallocate - 1 continue LOG.debug( "Minion machine '%s' of pool '%s' will be powered off " "as part of the pool refresh process (current " "deallocation count %d excluding the current machine)", - machine.id, minion_pool.id, max_minions_to_deallocate) + machine.id, + minion_pool.id, + max_minions_to_deallocate, + ) pool_refresh_flow.add( minion_mgr_tasks.PowerOffMinionMachineTask( - minion_pool.id, machine.id, minion_pool.platform, + minion_pool.id, + machine.id, + minion_pool.platform, fail_on_error=False, status_once_powered_off=( - constants.MINION_MACHINE_STATUS_AVAILABLE))) + constants.MINION_MACHINE_STATUS_AVAILABLE + ), + ) + ) elif minion_pool.minion_retention_strategy == ( - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE): # noqa: E501 + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE + ): # noqa: E501 pool_refresh_flow.add( minion_mgr_tasks.DeallocateMinionMachineTask( - minion_pool.id, machine.id, - minion_pool.platform)) + minion_pool.id, machine.id, minion_pool.platform + ) + ) else: raise exception.InvalidMinionPoolState( "Unknown minion pool retention strategy '%s' for pool " - "'%s'" % ( - minion_pool.minion_retention_strategy, - minion_pool.id)) + "'%s'" % (minion_pool.minion_retention_strategy, minion_pool.id) + ) max_minions_to_deallocate = max_minions_to_deallocate - 1 machines_to_deallocate.append(machine.id) # else, perform a healthcheck on the machine if it is powered on: elif machine.power_status == ( - constants.MINION_MACHINE_POWER_STATUS_POWERED_ON): + constants.MINION_MACHINE_POWER_STATUS_POWERED_ON + ): pool_refresh_flow.add( self._get_healtchcheck_flow_for_minion_machine( - minion_pool, machine, allocate_to_action=None, + minion_pool, + machine, + allocate_to_action=None, machine_status_on_success=( - constants.MINION_MACHINE_STATUS_AVAILABLE))) + constants.MINION_MACHINE_STATUS_AVAILABLE + ), + ) + ) machines_to_healthcheck.append(machine.id) else: skipped_machines[machine.id] = ( - machine.allocation_status, machine.power_status) + machine.allocation_status, + machine.power_status, + ) # update DB entried for all machines and emit relevant events: if skipped_machines: base_msg = ( "The following minion machines were skipped during the " "refreshing of the minion pool as they were in other " - "statuses than the serviceable ones: %s") - LOG.debug( - "[Pool '%s'] %s: %s", - minion_pool.id, base_msg, skipped_machines) + "statuses than the serviceable ones: %s" + ) + LOG.debug("[Pool '%s'] %s: %s", minion_pool.id, base_msg, skipped_machines) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - base_msg % list(skipped_machines.keys())) + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + base_msg % list(skipped_machines.keys()), + ) if machines_to_deallocate: deallocation_action = "deallocated" status_for_deallocated_machines = ( - constants.MINION_MACHINE_STATUS_DEALLOCATING) + constants.MINION_MACHINE_STATUS_DEALLOCATING + ) if minion_pool.minion_retention_strategy == ( - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF): + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_POWEROFF + ): deallocation_action = "powered off" status_for_deallocated_machines = ( - constants.MINION_MACHINE_STATUS_POWERING_OFF) + constants.MINION_MACHINE_STATUS_POWERING_OFF + ) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, "The following minion machines will be %s as part " - "of the refreshing of the minion pool: %s" % ( - deallocation_action, machines_to_deallocate)) + "of the refreshing of the minion pool: %s" + % (deallocation_action, machines_to_deallocate), + ) for machine in machines_to_deallocate: db_api.set_minion_machine_allocation_status( - ctxt, machine, status_for_deallocated_machines) + ctxt, machine, status_for_deallocated_machines + ) else: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "No minion machines require deallocation during pool refresh") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "No minion machines require deallocation during pool refresh", + ) if machines_to_healthcheck: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, "The following minion machines will be healthchecked as part " - "of the refreshing of the minion pool: %s" % ( - machines_to_healthcheck)) + "of the refreshing of the minion pool: %s" % (machines_to_healthcheck), + ) for machine in machines_to_healthcheck: db_api.set_minion_machine_allocation_status( - ctxt, machine, - constants.MINION_MACHINE_STATUS_HEALTHCHECKING) + ctxt, machine, constants.MINION_MACHINE_STATUS_HEALTHCHECKING + ) else: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "No minion machines require healthchecking during " - "pool refresh") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "No minion machines require healthchecking during pool refresh", + ) return pool_refresh_flow @@ -1434,84 +1796,114 @@ def _get_minion_pool_refresh_flow( def refresh_minion_pool(self, ctxt, minion_pool_id): LOG.info("Attempting to healthcheck Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_events=False, include_machines=True, - include_progress_updates=False) + ctxt, + minion_pool_id, + include_events=False, + include_machines=True, + include_progress_updates=False, + ) endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) - acceptable_allocation_statuses = [ - constants.MINION_POOL_STATUS_ALLOCATED] + ctxt, minion_pool.endpoint_id + ) + acceptable_allocation_statuses = [constants.MINION_POOL_STATUS_ALLOCATED] current_status = minion_pool.status if current_status not in acceptable_allocation_statuses: raise exception.InvalidMinionPoolState( "Minion machines for pool '%s' cannot be healthchecked as the " - "pool is in '%s' state instead of the expected %s." % ( - minion_pool_id, current_status, - acceptable_allocation_statuses)) + "pool is in '%s' state instead of the expected %s." + % (minion_pool_id, current_status, acceptable_allocation_statuses) + ) refresh_flow = self._get_minion_pool_refresh_flow( - ctxt, minion_pool, requery=False) + ctxt, minion_pool, requery=False + ) if not refresh_flow: msg = ( "There are no minion machine refresh operations to be " - "performed at this time") + "performed at this time" + ) db_api.add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, msg) + ctxt, minion_pool.id, constants.TASK_EVENT_INFO, msg + ) return self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool.id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) initial_store = self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict) - self._taskflow_runner.run_flow_in_background( - refresh_flow, store=initial_store) + ctxt, minion_pool, endpoint_dict + ) + self._taskflow_runner.run_flow_in_background(refresh_flow, store=initial_store) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Begun minion pool refreshing process") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Begun minion pool refreshing process", + ) return self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool.id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) def _get_minion_pool_allocation_flow(self, minion_pool): - """ Returns a taskflow.Flow object pertaining to all the tasks + """Returns a taskflow.Flow object pertaining to all the tasks required for allocating a minion pool (validation, shared resource setup, and actual minion creation) """ # create task flow: allocation_flow = linear_flow.Flow( - minion_mgr_tasks.MINION_POOL_ALLOCATION_FLOW_NAME_FORMAT % ( - minion_pool.id)) + minion_mgr_tasks.MINION_POOL_ALLOCATION_FLOW_NAME_FORMAT % (minion_pool.id) + ) # tansition pool to VALIDATING: - allocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, constants.MINION_POOL_STATUS_VALIDATING_INPUTS, - status_to_revert_to=constants.MINION_POOL_STATUS_ERROR)) + allocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, + constants.MINION_POOL_STATUS_VALIDATING_INPUTS, + status_to_revert_to=constants.MINION_POOL_STATUS_ERROR, + ) + ) # add pool options validation task: - allocation_flow.add(minion_mgr_tasks.ValidateMinionPoolOptionsTask( - # NOTE: we pass in the ID of the minion pool itself as both - # the task ID and the instance ID for tasks which are strictly - # pool-related. - minion_pool.id, - minion_pool.id, - minion_pool.platform)) + allocation_flow.add( + minion_mgr_tasks.ValidateMinionPoolOptionsTask( + # NOTE: we pass in the ID of the minion pool itself as both + # the task ID and the instance ID for tasks which are strictly + # pool-related. + minion_pool.id, + minion_pool.id, + minion_pool.platform, + ) + ) # transition pool to 'DEPLOYING_SHARED_RESOURCES': - allocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, - constants.MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES)) + allocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, constants.MINION_POOL_STATUS_ALLOCATING_SHARED_RESOURCES + ) + ) # add pool shared resources deployment task: allocation_flow.add( minion_mgr_tasks.AllocateSharedPoolResourcesTask( - minion_pool.id, minion_pool.id, minion_pool.platform, + minion_pool.id, + minion_pool.id, + minion_pool.platform, # NOTE: the shared resource deployment task will always get # run by itself so it is safe to have it override task_info: - provides='task_info')) + provides='task_info', + ) + ) # add subflow for deploying all of the minion machines: - fmt = ( - minion_mgr_tasks.MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT) + fmt = minion_mgr_tasks.MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT machines_flow = unordered_flow.Flow(fmt % minion_pool.id) pool_machine_ids = [] for _ in range(minion_pool.minimum_minions): @@ -1519,35 +1911,54 @@ def _get_minion_pool_allocation_flow(self, minion_pool): pool_machine_ids.append(machine_id) machines_flow.add( minion_mgr_tasks.AllocateMinionMachineTask( - minion_pool.id, machine_id, minion_pool.platform)) + minion_pool.id, machine_id, minion_pool.platform + ) + ) # NOTE: bool(flow) == False if the flow has no child flows/tasks: if machines_flow: - allocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, - constants.MINION_POOL_STATUS_ALLOCATING_MACHINES)) + allocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, constants.MINION_POOL_STATUS_ALLOCATING_MACHINES + ) + ) LOG.debug( "The following minion machine IDs will be created for " - "pool with ID '%s': %s" % (minion_pool.id, pool_machine_ids)) + "pool with ID '%s': %s" % (minion_pool.id, pool_machine_ids) + ) allocation_flow.add(machines_flow) else: LOG.debug( "No upfront minion machine deployments required for minion " - "pool with ID '%s'", minion_pool.id) + "pool with ID '%s'", + minion_pool.id, + ) # transition pool to ALLOCATED: - allocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, constants.MINION_POOL_STATUS_ALLOCATED)) + allocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, constants.MINION_POOL_STATUS_ALLOCATED + ) + ) return allocation_flow def create_minion_pool( - self, ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=None, - skip_allocation=False): - - endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, endpoint_id) + self, + ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=None, + skip_allocation=False, + ): + + endpoint_dict = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) minion_pool = models.MinionPool() minion_pool.id = str(uuid.uuid4()) minion_pool.name = name @@ -1568,49 +1979,58 @@ def create_minion_pool( minion_pool.maintenance_trust_id = ctxt.trust_id db_api.add_minion_pool(ctxt, minion_pool) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Successfully added minion pool to the DB") - self._register_refresh_jobs_for_minion_pool( - minion_pool) + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Successfully added minion pool to the DB", + ) + self._register_refresh_jobs_for_minion_pool(minion_pool) except Exception: if cleanup_trust: keystone.delete_trust(ctxt) raise if not skip_allocation: - allocation_flow = self._get_minion_pool_allocation_flow( - minion_pool) + allocation_flow = self._get_minion_pool_allocation_flow(minion_pool) # start the deployment flow: initial_store = self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict) + ctxt, minion_pool, endpoint_dict + ) try: self._taskflow_runner.run_flow_in_background( - allocation_flow, store=initial_store) + allocation_flow, store=initial_store + ) except Exception as ex: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_ERROR, + ctxt, + minion_pool.id, + constants.TASK_EVENT_ERROR, "A fatal exception occurred while attempting to start the " "task flow for allocating the minion pool. Forced " "deallocation and reallocation may be required. Please " "review the manager logs for additional details. " - "Error was: %s" % str(ex)) + "Error was: %s" % str(ex), + ) raise self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Begun minion pool allocation process") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Begun minion pool allocation process", + ) return self.get_minion_pool(ctxt, minion_pool.id) - def _get_pool_initial_taskflow_store_base( - self, ctxt, minion_pool, endpoint_dict): + def _get_pool_initial_taskflow_store_base(self, ctxt, minion_pool, endpoint_dict): # NOTE: considering pools are associated to strictly one endpoint, # we can duplicate the 'origin/destination': origin_info = { "id": endpoint_dict['id'], "connection_info": endpoint_dict['connection_info'], "mapped_regions": endpoint_dict['mapped_regions'], - "type": endpoint_dict['type']} + "type": endpoint_dict['type'], + } initial_store = { "context": ctxt, "origin": origin_info, @@ -1618,7 +2038,9 @@ def _get_pool_initial_taskflow_store_base( "task_info": { "pool_identifier": minion_pool.id, "pool_os_type": minion_pool.os_type, - "pool_environment_options": minion_pool.environment_options}} + "pool_environment_options": minion_pool.environment_options, + }, + } shared_resources = minion_pool.shared_resources if shared_resources is None: shared_resources = {} @@ -1626,275 +2048,365 @@ def _get_pool_initial_taskflow_store_base( return initial_store def _check_pool_machines_in_use( - self, ctxt, minion_pool, raise_if_in_use=False, requery=False): - """ Checks whether the given pool has any machines currently in-use. + self, ctxt, minion_pool, raise_if_in_use=False, requery=False + ): + """Checks whether the given pool has any machines currently in-use. Returns a list of the used machines if so, or an empty list of not. """ if requery: minion_pool = self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=False, include_progress_updates=False) + ctxt, + minion_pool.id, + include_machines=True, + include_events=False, + include_progress_updates=False, + ) unused_machine_states = [ constants.MINION_MACHINE_STATUS_AVAILABLE, constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, constants.MINION_MACHINE_STATUS_POWER_ERROR, - constants.MINION_MACHINE_STATUS_ERROR] + constants.MINION_MACHINE_STATUS_ERROR, + ] used_machines = { - mch for mch in minion_pool.minion_machines - if mch.allocation_status not in unused_machine_states} + mch + for mch in minion_pool.minion_machines + if mch.allocation_status not in unused_machine_states + } if used_machines and raise_if_in_use: mch_id = {mch.id: mch.allocation_status for mch in used_machines} raise exception.InvalidMinionPoolState( "Minion pool '%s' has one or more machines which are in an" - " active state: %s" % ( - minion_pool.id, - mch_id)) + " active state: %s" % (minion_pool.id, mch_id) + ) return used_machines @minion_manager_utils.minion_pool_synchronized_op def allocate_minion_pool(self, ctxt, minion_pool_id): LOG.info("Attempting to allocate Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_events=False, - include_machines=False, include_progress_updates=False) + ctxt, + minion_pool_id, + include_events=False, + include_machines=False, + include_progress_updates=False, + ) endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) - acceptable_allocation_statuses = [ - constants.MINION_POOL_STATUS_DEALLOCATED] + ctxt, minion_pool.endpoint_id + ) + acceptable_allocation_statuses = [constants.MINION_POOL_STATUS_DEALLOCATED] current_status = minion_pool.status if current_status not in acceptable_allocation_statuses: raise exception.InvalidMinionPoolState( "Minion machines for pool '%s' cannot be allocated as the pool" " is in '%s' state instead of the expected %s. Please " - "force-deallocate the pool and try again." % - (minion_pool_id, minion_pool.status, - acceptable_allocation_statuses)) + "force-deallocate the pool and try again." + % (minion_pool_id, minion_pool.status, acceptable_allocation_statuses) + ) allocation_flow = self._get_minion_pool_allocation_flow(minion_pool) initial_store = self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict) + ctxt, minion_pool, endpoint_dict + ) try: db_api.set_minion_pool_status( - ctxt, minion_pool_id, - constants.MINION_POOL_STATUS_POOL_MAINTENANCE) + ctxt, minion_pool_id, constants.MINION_POOL_STATUS_POOL_MAINTENANCE + ) self._taskflow_runner.run_flow_in_background( - allocation_flow, store=initial_store) - self._register_refresh_jobs_for_minion_pool( - minion_pool) + allocation_flow, store=initial_store + ) + self._register_refresh_jobs_for_minion_pool(minion_pool) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Begun minion pool allocation process") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Begun minion pool allocation process", + ) except Exception as ex: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_ERROR, + ctxt, + minion_pool.id, + constants.TASK_EVENT_ERROR, "A fatal exception occurred while attempting to start the " "task flow for allocating the minion pool. Forced " "deallocation and reallocation may be required. Please " "review the manager logs for additional details. " - "Error was: %s" % str(ex)) - db_api.set_minion_pool_status( - ctxt, minion_pool_id, current_status) + "Error was: %s" % str(ex), + ) + db_api.set_minion_pool_status(ctxt, minion_pool_id, current_status) raise return self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool.id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) - def _get_minion_pool_deallocation_flow( - self, minion_pool, raise_on_error=True): - """ Returns a taskflow.Flow object pertaining to all the tasks + def _get_minion_pool_deallocation_flow(self, minion_pool, raise_on_error=True): + """Returns a taskflow.Flow object pertaining to all the tasks required for deallocating a minion pool (machines and shared resources) """ # create task flow: deallocation_flow = linear_flow.Flow( - minion_mgr_tasks.MINION_POOL_DEALLOCATION_FLOW_NAME_FORMAT % ( - minion_pool.id)) + minion_mgr_tasks.MINION_POOL_DEALLOCATION_FLOW_NAME_FORMAT + % (minion_pool.id) + ) # add subflow for deallocating all of the minion machines: - fmt = ( - (minion_mgr_tasks. - MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT)) + fmt = minion_mgr_tasks.MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT machines_flow = unordered_flow.Flow(fmt % minion_pool.id) for machine in minion_pool.minion_machines: machines_flow.add( minion_mgr_tasks.DeallocateMinionMachineTask( - minion_pool.id, machine.id, minion_pool.platform, - raise_on_cleanup_failure=raise_on_error)) + minion_pool.id, + machine.id, + minion_pool.platform, + raise_on_cleanup_failure=raise_on_error, + ) + ) # NOTE: bool(flow) == False if the flow has no child flows/tasks: if machines_flow: # tansition pool to DEALLOCATING_MACHINES: - deallocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, - constants.MINION_POOL_STATUS_DEALLOCATING_MACHINES, - status_to_revert_to=constants.MINION_POOL_STATUS_ERROR)) + deallocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, + constants.MINION_POOL_STATUS_DEALLOCATING_MACHINES, + status_to_revert_to=constants.MINION_POOL_STATUS_ERROR, + ) + ) deallocation_flow.add(machines_flow) else: - LOG.debug( - "No machines for pool '%s' require deallocating.", - minion_pool.id) + LOG.debug("No machines for pool '%s' require deallocating.", minion_pool.id) # transition pool to DEALLOCATING_SHARED_RESOURCES: - deallocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, - constants.MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES, - status_to_revert_to=constants.MINION_POOL_STATUS_ERROR)) + deallocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, + constants.MINION_POOL_STATUS_DEALLOCATING_SHARED_RESOURCES, + status_to_revert_to=constants.MINION_POOL_STATUS_ERROR, + ) + ) # add pool shared resources deletion task: deallocation_flow.add( minion_mgr_tasks.DeallocateSharedPoolResourcesTask( - minion_pool.id, minion_pool.id, minion_pool.platform)) + minion_pool.id, minion_pool.id, minion_pool.platform + ) + ) # transition pool to DEALLOCATED: - deallocation_flow.add(minion_mgr_tasks.UpdateMinionPoolStatusTask( - minion_pool.id, constants.MINION_POOL_STATUS_DEALLOCATED)) + deallocation_flow.add( + minion_mgr_tasks.UpdateMinionPoolStatusTask( + minion_pool.id, constants.MINION_POOL_STATUS_DEALLOCATED + ) + ) return deallocation_flow - def _get_pool_deallocation_initial_store( - self, ctxt, minion_pool, endpoint_dict): + def _get_pool_deallocation_initial_store(self, ctxt, minion_pool, endpoint_dict): base = self._get_pool_initial_taskflow_store_base( - ctxt, minion_pool, endpoint_dict) + ctxt, minion_pool, endpoint_dict + ) if 'task_info' not in base: base['task_info'] = {} - base['task_info']['pool_shared_resources'] = ( - minion_pool.shared_resources) + base['task_info']['pool_shared_resources'] = minion_pool.shared_resources return base @minion_manager_utils.minion_pool_synchronized_op def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): LOG.info("Attempting to deallocate Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_events=False, include_machines=True, - include_progress_updates=False) + ctxt, + minion_pool_id, + include_events=False, + include_machines=True, + include_progress_updates=False, + ) current_status = minion_pool.status if current_status == constants.MINION_POOL_STATUS_DEALLOCATED: LOG.debug( "Deallocation requested on already deallocated pool '%s'. " - "Nothing to do so returning early.", minion_pool_id) + "Nothing to do so returning early.", + minion_pool_id, + ) return self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool.id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) acceptable_deallocation_statuses = [ constants.MINION_POOL_STATUS_ALLOCATED, - constants.MINION_POOL_STATUS_ERROR] + constants.MINION_POOL_STATUS_ERROR, + ] if current_status not in acceptable_deallocation_statuses: if not force: raise exception.InvalidMinionPoolState( "Minion pool '%s' cannot be deallocated as the pool" - " is in '%s' state instead of one of the expected %s" % ( - minion_pool_id, minion_pool.status, - acceptable_deallocation_statuses)) + " is in '%s' state instead of one of the expected %s" + % ( + minion_pool_id, + minion_pool.status, + acceptable_deallocation_statuses, + ) + ) else: LOG.warn( "Forcibly deallocating minion pool '%s' at user request.", - minion_pool_id) - self._check_pool_machines_in_use( - ctxt, minion_pool, raise_if_in_use=not force) + minion_pool_id, + ) + self._check_pool_machines_in_use(ctxt, minion_pool, raise_if_in_use=not force) endpoint_dict = self._rpc_conductor_client.get_endpoint( - ctxt, minion_pool.endpoint_id) + ctxt, minion_pool.endpoint_id + ) deallocation_flow = self._get_minion_pool_deallocation_flow( - minion_pool, raise_on_error=not force) + minion_pool, raise_on_error=not force + ) initial_store = self._get_pool_deallocation_initial_store( - ctxt, minion_pool, endpoint_dict) + ctxt, minion_pool, endpoint_dict + ) try: db_api.set_minion_pool_status( - ctxt, minion_pool_id, - constants.MINION_POOL_STATUS_POOL_MAINTENANCE) + ctxt, minion_pool_id, constants.MINION_POOL_STATUS_POOL_MAINTENANCE + ) self._taskflow_runner.run_flow_in_background( - deallocation_flow, store=initial_store) + deallocation_flow, store=initial_store + ) self._unregister_refresh_jobs_for_minion_pool( - minion_pool, raise_on_error=False) + minion_pool, raise_on_error=False + ) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Begun minion pool deallocation process") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Begun minion pool deallocation process", + ) except Exception as ex: self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_ERROR, + ctxt, + minion_pool.id, + constants.TASK_EVENT_ERROR, "A fatal exception occurred while attempting to start the " "task flow for deallocating the minion pool. Forced " "deallocation and reallocation may be required. Please " "review the manager logs for additional details. " - "Error was: %s" % str(ex)) - db_api.set_minion_pool_status( - ctxt, minion_pool_id, current_status) + "Error was: %s" % str(ex), + ) + db_api.set_minion_pool_status(ctxt, minion_pool_id, current_status) raise return self._get_minion_pool( - ctxt, minion_pool.id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool.id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) def get_minion_pools(self, ctxt, include_machines=True): return db_api.get_minion_pools( - ctxt, include_machines=include_machines, include_events=False, - include_progress_updates=False) + ctxt, + include_machines=include_machines, + include_events=False, + include_progress_updates=False, + ) def _get_minion_pool( - self, ctxt, minion_pool_id, include_machines=False, - include_events=False, include_progress_updates=False): + self, + ctxt, + minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ): minion_pool = db_api.get_minion_pool( - ctxt, minion_pool_id, include_machines=include_machines, + ctxt, + minion_pool_id, + include_machines=include_machines, include_events=include_events, - include_progress_updates=include_progress_updates) + include_progress_updates=include_progress_updates, + ) if not minion_pool: raise exception.NotFound( - "Minion pool with ID '%s' not found." % minion_pool_id) + "Minion pool with ID '%s' not found." % minion_pool_id + ) return minion_pool @minion_manager_utils.minion_pool_synchronized_op def get_minion_pool(self, ctxt, minion_pool_id): return self._get_minion_pool( - ctxt, minion_pool_id, include_machines=True, include_events=True, - include_progress_updates=True) + ctxt, + minion_pool_id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) @minion_manager_utils.minion_pool_synchronized_op def update_minion_pool(self, ctxt, minion_pool_id, updated_values): minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_machines=False) + ctxt, minion_pool_id, include_machines=False + ) if minion_pool.status != constants.MINION_POOL_STATUS_DEALLOCATED: raise exception.InvalidMinionPoolState( "Minion Pool '%s' cannot be updated as it is in '%s' status " "instead of the expected '%s'. Please ensure the pool machines" "have been deallocated and the pool's supporting resources " - "have been torn down before updating the pool." % ( - minion_pool_id, minion_pool.status, - constants.MINION_POOL_STATUS_DEALLOCATED)) + "have been torn down before updating the pool." + % ( + minion_pool_id, + minion_pool.status, + constants.MINION_POOL_STATUS_DEALLOCATED, + ) + ) LOG.info( "Attempting to update minion_pool '%s' with payload: %s", - minion_pool_id, updated_values) + minion_pool_id, + updated_values, + ) db_api.update_minion_pool(ctxt, minion_pool_id, updated_values) LOG.info("Minion Pool '%s' successfully updated", minion_pool_id) self._add_minion_pool_event( - ctxt, minion_pool.id, constants.TASK_EVENT_INFO, - "Successfully updated minion pool properties") + ctxt, + minion_pool.id, + constants.TASK_EVENT_INFO, + "Successfully updated minion pool properties", + ) return db_api.get_minion_pool( - ctxt, minion_pool_id, include_machines=True, - include_events=True, include_progress_updates=True) + ctxt, + minion_pool_id, + include_machines=True, + include_events=True, + include_progress_updates=True, + ) @minion_manager_utils.minion_pool_synchronized_op def delete_minion_pool(self, ctxt, minion_pool_id): - minion_pool = self._get_minion_pool( - ctxt, minion_pool_id, include_machines=True) + minion_pool = self._get_minion_pool(ctxt, minion_pool_id, include_machines=True) acceptable_deletion_statuses = [ constants.MINION_POOL_STATUS_DEALLOCATED, - constants.MINION_POOL_STATUS_ERROR] + constants.MINION_POOL_STATUS_ERROR, + ] if minion_pool.status not in acceptable_deletion_statuses: raise exception.InvalidMinionPoolState( "Minion Pool '%s' cannot be deleted as it is in '%s' status " "instead of one of the expected '%s'. Please ensure the pool " "machines have been deallocated and the pool's supporting " - "resources have been torn down before deleting the pool." % ( - minion_pool_id, minion_pool.status, - acceptable_deletion_statuses)) - self._unregister_refresh_jobs_for_minion_pool( - minion_pool, raise_on_error=False) + "resources have been torn down before deleting the pool." + % (minion_pool_id, minion_pool.status, acceptable_deletion_statuses) + ) + self._unregister_refresh_jobs_for_minion_pool(minion_pool, raise_on_error=False) LOG.info("Deleting minion pool with ID '%s'" % minion_pool_id) db_api.delete_minion_pool(ctxt, minion_pool_id) if minion_pool.maintenance_trust_id: maintenance_ctxt = context.get_admin_context( - minion_pool.maintenance_trust_id) + minion_pool.maintenance_trust_id + ) keystone.delete_trust(maintenance_ctxt) diff --git a/coriolis/minion_manager/rpc/tasks.py b/coriolis/minion_manager/rpc/tasks.py index e67c0f39..3801fefa 100644 --- a/coriolis/minion_manager/rpc/tasks.py +++ b/coriolis/minion_manager/rpc/tasks.py @@ -7,73 +7,67 @@ from oslo_log import log as logging from oslo_utils import timeutils +from taskflow.types import failure +from coriolis import constants, exception, utils from coriolis.conductor.rpc import client as rpc_conductor_client -from coriolis import constants from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models -from coriolis import exception from coriolis.minion_manager.rpc import client as rpc_minion_manager_client from coriolis.minion_manager.rpc import utils as minion_manager_utils from coriolis.taskflow import base as coriolis_taskflow_base -from coriolis import utils - -from taskflow.types import failure - LOG = logging.getLogger(__name__) -MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT = ( - "migration-%s-minions-allocation") -MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT = ( - "replica-%s-minions-allocation") +MINION_POOL_MIGRATION_ALLOCATION_FLOW_NAME_FORMAT = "migration-%s-minions-allocation" +MINION_POOL_REPLICA_ALLOCATION_FLOW_NAME_FORMAT = "replica-%s-minions-allocation" MINION_POOL_MIGRATION_ALLOCATION_SUBFLOW_NAME_FORMAT = ( - "migration-%s-minions-machines-allocation") + "migration-%s-minions-machines-allocation" +) MINION_POOL_REPLICA_ALLOCATION_SUBFLOW_NAME_FORMAT = ( - "replica-%s-minions-machines-allocation") + "replica-%s-minions-machines-allocation" +) MINION_POOL_ALLOCATION_FLOW_NAME_FORMAT = "pool-%s-allocation" MINION_POOL_DEALLOCATION_FLOW_NAME_FORMAT = "pool-%s-deallocation" MINION_POOL_REFRESH_FLOW_NAME_FORMAT = "pool-%s-refresh" MINION_POOL_VALIDATION_TASK_NAME_FORMAT = "pool-%s-validation" MINION_POOL_UPDATE_STATUS_TASK_NAME_FORMAT = "pool-%s-update-status-%s" -MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT = ( - "pool-%s-machine-%s-healthcheck") +MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT = "pool-%s-machine-%s-healthcheck" MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT = ( - "pool-%s-allocate-shared-resources") + "pool-%s-allocate-shared-resources" +) MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT = ( - "pool-%s-deallocate-shared-resources") -MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT = ( - "pool-%s-machines-allocation") -MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT = ( - "pool-%s-machines-deallocation") -MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT = ( - "pool-%s-machine-%s-healthcheck") -MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT = ( - "pool-%s-machine-%s-reallocation") + "pool-%s-deallocate-shared-resources" +) +MINION_POOL_ALLOCATE_MINIONS_SUBFLOW_NAME_FORMAT = "pool-%s-machines-allocation" +MINION_POOL_DEALLOCATE_MACHINES_SUBFLOW_NAME_FORMAT = "pool-%s-machines-deallocation" +MINION_POOL_HEALTHCHECK_MACHINE_SUBFLOW_NAME_FORMAT = "pool-%s-machine-%s-healthcheck" +MINION_POOL_REALLOCATE_MACHINE_SUBFLOW_NAME_FORMAT = "pool-%s-machine-%s-reallocation" MINION_POOL_ALLOCATE_MACHINES_FOR_REPLICA_SUBFLOW_NAME_FORMAT = ( - "pool-%s-allocate-replica-%s-machines") + "pool-%s-allocate-replica-%s-machines" +) MINION_POOL_ALLOCATE_MACHINES_FOR_MIGRATION_SUBFLOW_NAME_FORMAT = ( - "pool-%s-allocate-migration-%s-machines") -MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT = ( - "pool-%s-machine-%s-allocation") -MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT = ( - "pool-%s-machine-%s-deallocation") + "pool-%s-allocate-migration-%s-machines" +) +MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT = "pool-%s-machine-%s-allocation" +MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT = "pool-%s-machine-%s-deallocation" MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT = ( - "migration-%s-minion-allocation-confirmation") + "migration-%s-minion-allocation-confirmation" +) MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT = ( - "replica-%s-minion-allocation-confirmation") + "replica-%s-minion-allocation-confirmation" +) MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT = ( - "migration-%s-minion-allocation-failure") + "migration-%s-minion-allocation-failure" +) MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT = ( - "replica-%s-minion-allocation-failure") -MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT = ( - "pool-%s-machine-%s-power-on") -MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT = ( - "pool-%s-machine-%s-power-off") + "replica-%s-minion-allocation-failure" +) +MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT = "pool-%s-machine-%s-power-on" +MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT = "pool-%s-machine-%s-power-off" class MinionManagerTaskEventMixin(object): - # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated # oslo_messaging clients as the underlying thread queues will # be invalidated. Considering this class both serves from a "main @@ -82,137 +76,134 @@ class MinionManagerTaskEventMixin(object): @property def _conductor_client(self): if not getattr(self, '_conductor_client_instance', None): - self._conductor_client_instance = ( - rpc_conductor_client.ConductorClient()) + self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance @property def _minion_manager_client(self): if not getattr(self, '_minion_manager_client_instance', None): self._minion_manager_client_instance = ( - rpc_minion_manager_client.MinionManagerClient()) + rpc_minion_manager_client.MinionManagerClient() + ) return self._minion_manager_client_instance - def _add_minion_pool_event( - self, context, message, level=constants.TASK_EVENT_INFO): + def _add_minion_pool_event(self, context, message, level=constants.TASK_EVENT_INFO): LOG.debug("Minion pool '%s' event: %s", self._minion_pool_id, message) - db_api.add_minion_pool_event( - context, self._minion_pool_id, level, message) + db_api.add_minion_pool_event(context, self._minion_pool_id, level, message) - def _get_minion_machine( - self, context, minion_machine_id, - raise_if_not_found=False): + def _get_minion_machine(self, context, minion_machine_id, raise_if_not_found=False): machine = db_api.get_minion_machine(context, minion_machine_id) if not machine and raise_if_not_found: raise exception.NotFound( - "Could not find minion machine with ID '%s'" % ( - minion_machine_id)) + "Could not find minion machine with ID '%s'" % (minion_machine_id) + ) return machine def _set_minion_pool_status(self, ctxt, minion_pool_id, new_status): - with minion_manager_utils.get_minion_pool_lock( - minion_pool_id, external=True): + with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): db_api.set_minion_pool_status(ctxt, minion_pool_id, new_status) def _update_minion_machine( - self, ctxt, minion_pool_id, minion_machine_id, updated_values): - with minion_manager_utils.get_minion_pool_lock( - minion_pool_id, external=True): - db_api.update_minion_machine( - ctxt, minion_machine_id, updated_values) + self, ctxt, minion_pool_id, minion_machine_id, updated_values + ): + with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): + db_api.update_minion_machine(ctxt, minion_machine_id, updated_values) def _set_minion_machine_allocation_status( - self, ctxt, minion_pool_id, minion_machine_id, new_status): - with minion_manager_utils.get_minion_pool_lock( - minion_pool_id, external=True): + self, ctxt, minion_pool_id, minion_machine_id, new_status + ): + with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): db_api.set_minion_machine_allocation_status( - ctxt, minion_machine_id, new_status) + ctxt, minion_machine_id, new_status + ) def _set_minion_machine_power_status( - self, ctxt, minion_pool_id, minion_machine_id, new_status): + self, ctxt, minion_pool_id, minion_machine_id, new_status + ): self._update_minion_machine( - ctxt, minion_pool_id, minion_machine_id, - {"power_status": new_status}) + ctxt, minion_pool_id, minion_machine_id, {"power_status": new_status} + ) class _BaseReportMinionAllocationFailureForActionTask( - coriolis_taskflow_base.BaseCoriolisTaskflowTask, - MinionManagerTaskEventMixin): - """ Task with no operation on `execute()`, but whose `revert()` method + coriolis_taskflow_base.BaseCoriolisTaskflowTask, MinionManagerTaskEventMixin +): + """Task with no operation on `execute()`, but whose `revert()` method reports a minion allocation failure to the conductor for the afferent - transfer action. """ + transfer action.""" def __init__(self, action_id, **kwargs): self._action_id = action_id self._task_name = self._get_task_name(action_id) super(_BaseReportMinionAllocationFailureForActionTask, self).__init__( - name=self._task_name, **kwargs) + name=self._task_name, **kwargs + ) @abc.abstractmethod def _get_task_name(self, action_id): - raise NotImplementedError( - "No allocation failure task name provided") + raise NotImplementedError("No allocation failure task name provided") @abc.abstractmethod - def _report_machine_allocation_failure( - self, context, action_id, failure_str): - raise NotImplementedError( - "No allocation failure operation defined") + def _report_machine_allocation_failure(self, context, action_id, failure_str): + raise NotImplementedError("No allocation failure operation defined") def execute(self, context): - super( - _BaseReportMinionAllocationFailureForActionTask, self).execute() - LOG.debug( - "Nothing to execute for task '%s'", self._task_name) + super(_BaseReportMinionAllocationFailureForActionTask, self).execute() + LOG.debug("Nothing to execute for task '%s'", self._task_name) def revert(self, context, *args, **kwargs): - super( - _BaseReportMinionAllocationFailureForActionTask, self).revert( - *args, **kwargs) + super(_BaseReportMinionAllocationFailureForActionTask, self).revert( + *args, **kwargs + ) flow_failures = kwargs.get('flow_failures', {}) flow_failures_str = self._get_error_str_for_flow_failures( - flow_failures, full_tracebacks=False) + flow_failures, full_tracebacks=False + ) LOG.info( "Reporting minion allocation failure for action '%s': %s", - self._action_id, flow_failures_str) + self._action_id, + flow_failures_str, + ) self._minion_manager_client.deallocate_minion_machines_for_action( - context, self._action_id) + context, self._action_id + ) self._report_machine_allocation_failure( - context, self._action_id, flow_failures_str) + context, self._action_id, flow_failures_str + ) class ReportMinionAllocationFailureForMigrationTask( - _BaseReportMinionAllocationFailureForActionTask): - + _BaseReportMinionAllocationFailureForActionTask +): def _get_task_name(self, action_id): - return ( - MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT - % (action_id)) + return MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT % ( + action_id + ) - def _report_machine_allocation_failure( - self, context, action_id, failure_str): + def _report_machine_allocation_failure(self, context, action_id, failure_str): self._conductor_client.report_deployment_minions_allocation_error( - context, action_id, failure_str) + context, action_id, failure_str + ) class ReportMinionAllocationFailureForReplicaTask( - _BaseReportMinionAllocationFailureForActionTask): - + _BaseReportMinionAllocationFailureForActionTask +): def _get_task_name(self, action_id): - return ( - MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT - % (action_id)) + return MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT % ( + action_id + ) - def _report_machine_allocation_failure( - self, context, action_id, failure_str): + def _report_machine_allocation_failure(self, context, action_id, failure_str): self._conductor_client.report_transfer_minions_allocation_error( - context, action_id, failure_str) + context, action_id, failure_str + ) class _BaseConfirmMinionAllocationForActionTask( - coriolis_taskflow_base.BaseCoriolisTaskflowTask, - MinionManagerTaskEventMixin): - """ Task which confirms the minion machine allocations for the given action + coriolis_taskflow_base.BaseCoriolisTaskflowTask, MinionManagerTaskEventMixin +): + """Task which confirms the minion machine allocations for the given action to the conductor. """ @@ -229,258 +220,312 @@ def __init__(self, action_id, allocated_machine_id_mappings, **kwargs): self._task_name = self._get_task_name(action_id) self._allocated_machine_id_mappings = allocated_machine_id_mappings super(_BaseConfirmMinionAllocationForActionTask, self).__init__( - name=self._task_name, **kwargs) + name=self._task_name, **kwargs + ) @abc.abstractmethod def _get_action_label(self): raise NotImplementedError( - "No minion allocation confirmation task action label defined") + "No minion allocation confirmation task action label defined" + ) @abc.abstractmethod def _get_task_name(self, action_id): - raise NotImplementedError( - "No minion allocation confirmation task name defined") + raise NotImplementedError("No minion allocation confirmation task name defined") @abc.abstractmethod def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations): - raise NotImplementedError( - "No minion allocation confirmation operation defined") + self, context, action_id, machine_allocations + ): + raise NotImplementedError("No minion allocation confirmation operation defined") def execute(self, context): machines_cache = {} machine_allocations = {} def _check_minion_properties( - minion_machine, instance, minion_purpose="unknown"): + minion_machine, instance, minion_purpose="unknown" + ): if minion_machine.allocation_status != ( - constants.MINION_MACHINE_STATUS_IN_USE): + constants.MINION_MACHINE_STATUS_IN_USE + ): raise exception.InvalidMinionMachineState( "Minion machine with ID '%s' of pool '%s' is in '%s' " "status instead of the expected '%s' for it to be used " "as a '%s' minion for instance '%s' of transfer " - "action '%s'." % ( - minion_machine.id, minion_machine.pool_id, + "action '%s'." + % ( + minion_machine.id, + minion_machine.pool_id, minion_machine.allocation_status, constants.MINION_MACHINE_STATUS_IN_USE, - minion_purpose, instance, self._action_id)) + minion_purpose, + instance, + self._action_id, + ) + ) if minion_machine.allocated_action != self._action_id: raise exception.InvalidMinionMachineState( "Minion machine with ID '%s' of pool '%s' appears to be " "allocated to action with ID '%s' instead of the expected" " '%s' for it to be used as a '%s' minion for instance " - "'%s'." % ( - minion_machine.id, minion_machine.pool_id, - minion_machine.allocated_action, self._action_id, - minion_purpose, instance)) + "'%s'." + % ( + minion_machine.id, + minion_machine.pool_id, + minion_machine.allocated_action, + self._action_id, + minion_purpose, + instance, + ) + ) if minion_machine.power_status != ( - constants.MINION_MACHINE_POWER_STATUS_POWERED_ON): + constants.MINION_MACHINE_POWER_STATUS_POWERED_ON + ): raise exception.InvalidMinionMachineState( "Minion machine with ID '%s' of pool '%s' is in '%s' " "power status instead of the expected '%s' for it to be " "used as a '%s' minion for instance '%s' of transfer " - "action '%s'." % ( - minion_machine.id, minion_machine.pool_id, + "action '%s'." + % ( + minion_machine.id, + minion_machine.pool_id, minion_machine.power_status, constants.MINION_MACHINE_POWER_STATUS_POWERED_ON, - minion_purpose, instance, self._action_id)) + minion_purpose, + instance, + self._action_id, + ) + ) # TODO(aznashwan): add extra checks for conn info schemas here? required_props = { "provider_properties": minion_machine.provider_properties, - "connection_info": minion_machine.connection_info} + "connection_info": minion_machine.connection_info, + } if not all(required_props.values()): raise exception.InvalidMinionMachineState( "One or more required paroperties for minion machine '%s' " "(to be used as a '%s' minion for instance '%s' of action " - "'%s') were missing: %s" % ( - minion_machine.id, minion_purpose, instance, - self._action_id, required_props)) - - for (instance, allocated_machines_for_instance) in ( - self._allocated_machine_id_mappings.items()): - + "'%s') were missing: %s" + % ( + minion_machine.id, + minion_purpose, + instance, + self._action_id, + required_props, + ) + ) + + for ( + instance, + allocated_machines_for_instance, + ) in self._allocated_machine_id_mappings.items(): if not allocated_machines_for_instance: LOG.warn( "No machine allocations were provided for instance '%s' " "for action '%s'. Skipping. mappings were: %s", - instance, self._action_id, allocated_machines_for_instance) + instance, + self._action_id, + allocated_machines_for_instance, + ) continue if instance not in machine_allocations: machine_allocations[instance] = {} # check for and fetch the source minion: - origin_minion_id = allocated_machines_for_instance.get( - 'origin_minion_id') + origin_minion_id = allocated_machines_for_instance.get('origin_minion_id') if origin_minion_id: origin_minion_machine = machines_cache.get(origin_minion_id) if not origin_minion_machine: origin_minion_machine = self._get_minion_machine( - context, origin_minion_id, raise_if_not_found=True) + context, origin_minion_id, raise_if_not_found=True + ) machines_cache[origin_minion_id] = origin_minion_machine _check_minion_properties( - origin_minion_machine, instance, - minion_purpose="source") + origin_minion_machine, instance, minion_purpose="source" + ) machine_allocations[instance]['origin_minion'] = ( - origin_minion_machine.to_dict()) + origin_minion_machine.to_dict() + ) # check for and fetch the destination minion: destination_minion_id = allocated_machines_for_instance.get( - 'destination_minion_id') + 'destination_minion_id' + ) if destination_minion_id: - destination_minion_machine = machines_cache.get( - destination_minion_id) + destination_minion_machine = machines_cache.get(destination_minion_id) if not destination_minion_machine: destination_minion_machine = self._get_minion_machine( - context, destination_minion_id, - raise_if_not_found=True) + context, destination_minion_id, raise_if_not_found=True + ) _check_minion_properties( - destination_minion_machine, instance, - minion_purpose="destination") - machines_cache[destination_minion_id] = ( - destination_minion_machine) + destination_minion_machine, + instance, + minion_purpose="destination", + ) + machines_cache[destination_minion_id] = destination_minion_machine machine_allocations[instance]['destination_minion'] = ( - destination_minion_machine.to_dict()) + destination_minion_machine.to_dict() + ) # check for and fetch the OSMorphing minion: osmorphing_minion_id = allocated_machines_for_instance.get( - 'osmorphing_minion_id') + 'osmorphing_minion_id' + ) if osmorphing_minion_id: - osmorphing_minion_machine = machines_cache.get( - osmorphing_minion_id) + osmorphing_minion_machine = machines_cache.get(osmorphing_minion_id) if not osmorphing_minion_machine: osmorphing_minion_machine = self._get_minion_machine( - context, osmorphing_minion_id, raise_if_not_found=True) + context, osmorphing_minion_id, raise_if_not_found=True + ) _check_minion_properties( - osmorphing_minion_machine, instance, - minion_purpose="OSMorphing") - machines_cache[osmorphing_minion_id] = ( - osmorphing_minion_machine) + osmorphing_minion_machine, instance, minion_purpose="OSMorphing" + ) + machines_cache[osmorphing_minion_id] = osmorphing_minion_machine machine_allocations[instance]['osmorphing_minion'] = ( - osmorphing_minion_machine.to_dict()) + osmorphing_minion_machine.to_dict() + ) try: self._confirm_machine_allocation_for_action( - context, self._action_id, machine_allocations) + context, self._action_id, machine_allocations + ) except exception.NotFound as ex: msg = ( "The Conductor has refused minion machine allocations for " "%s with ID '%s' as it has purportedly been deleted." " Please check both the Conductor and Minion Manager " - "service logs for more details." % ( - self._get_action_label().lower().capitalize(), - self._action_id)) + "service logs for more details." + % (self._get_action_label().lower().capitalize(), self._action_id) + ) LOG.error( "%s. Allocations were: %s. Original trace was: %s", - msg, machine_allocations, utils.get_exception_details()) - raise exception.MinionMachineAllocationFailure( - msg) from ex - except ( - exception.InvalidDeploymentState, - exception.InvalidTransferState) as ex: + msg, + machine_allocations, + utils.get_exception_details(), + ) + raise exception.MinionMachineAllocationFailure(msg) from ex + except (exception.InvalidDeploymentState, exception.InvalidTransferState) as ex: msg = ( "The Conductor has refused minion machine allocations for " "%s with ID '%s' as it is purportedly in an invalid state " "to have minions allocated for it. It is possible " "that the transfer had been user-cancelled or had " "otherwise been halted. Please check both the Conductor " - "and Minion Manager service logs for more details." % ( - self._get_action_label().lower().capitalize(), - self._action_id)) + "and Minion Manager service logs for more details." + % (self._get_action_label().lower().capitalize(), self._action_id) + ) LOG.error( "%s. Allocations were: %s. Original trace was: %s", - msg, machine_allocations, utils.get_exception_details()) - raise exception.MinionMachineAllocationFailure( - msg) from ex + msg, + machine_allocations, + utils.get_exception_details(), + ) + raise exception.MinionMachineAllocationFailure(msg) from ex class ConfirmMinionAllocationForMigrationTask( - _BaseConfirmMinionAllocationForActionTask): - + _BaseConfirmMinionAllocationForActionTask +): def _get_action_label(self): return "migration" def _get_task_name(self, action_id): - return ( - MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT - % (action_id)) + return MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT % ( + action_id + ) def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations): + self, context, action_id, machine_allocations + ): self._conductor_client.confirm_deployment_minions_allocation( - context, action_id, machine_allocations) - + context, action_id, machine_allocations + ) -class ConfirmMinionAllocationForReplicaTask( - _BaseConfirmMinionAllocationForActionTask): +class ConfirmMinionAllocationForReplicaTask(_BaseConfirmMinionAllocationForActionTask): def _get_action_label(self): return "replica" def _get_task_name(self, action_id): - return ( - MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT - % (action_id)) + return MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT % ( + action_id + ) def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations): + self, context, action_id, machine_allocations + ): self._conductor_client.confirm_transfer_minions_allocation( - context, action_id, machine_allocations) + context, action_id, machine_allocations + ) class UpdateMinionPoolStatusTask( - coriolis_taskflow_base.BaseCoriolisTaskflowTask, - MinionManagerTaskEventMixin): - """ Task which updates the status of the given pool. + coriolis_taskflow_base.BaseCoriolisTaskflowTask, MinionManagerTaskEventMixin +): + """Task which updates the status of the given pool. Is capable of recording and reverting the state. """ default_provides = ["latest_status"] def __init__( - self, minion_pool_id, target_status, - status_to_revert_to=None, **kwargs): + self, minion_pool_id, target_status, status_to_revert_to=None, **kwargs + ): self._target_status = target_status self._minion_pool_id = minion_pool_id - self._task_name = (MINION_POOL_UPDATE_STATUS_TASK_NAME_FORMAT % ( - self._minion_pool_id, self._target_status)).lower() + self._task_name = ( + MINION_POOL_UPDATE_STATUS_TASK_NAME_FORMAT + % (self._minion_pool_id, self._target_status) + ).lower() self._previous_status = None self._status_to_revert_to = status_to_revert_to - super(UpdateMinionPoolStatusTask, self).__init__( - name=self._task_name, **kwargs) + super(UpdateMinionPoolStatusTask, self).__init__(name=self._task_name, **kwargs) def execute(self, context, *args): super(UpdateMinionPoolStatusTask, self).execute(*args) if not self._previous_status: minion_pool = db_api.get_minion_pool( - context, self._minion_pool_id, include_machines=False, - include_events=False, include_progress_updates=False) + context, + self._minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ) self._previous_status = minion_pool.status if self._previous_status == self._target_status: LOG.debug( "[Task '%s'] Minion pool '%s' already in status '%s'. " - "Nothing to do." % ( - self._task_name, self._minion_pool_id, - self._target_status)) + "Nothing to do." + % (self._task_name, self._minion_pool_id, self._target_status) + ) else: LOG.debug( "[Task '%s'] Transitioning minion pool '%s' from status '%s' " - "to '%s'." % ( - self._task_name, self._minion_pool_id, - self._previous_status, self._target_status)) + "to '%s'." + % ( + self._task_name, + self._minion_pool_id, + self._previous_status, + self._target_status, + ) + ) self._set_minion_pool_status( - context, self._minion_pool_id, self._target_status) + context, self._minion_pool_id, self._target_status + ) self._add_minion_pool_event( context, - "Pool status transitioned from '%s' to '%s'" % ( - self._previous_status, self._target_status)) + "Pool status transitioned from '%s' to '%s'" + % (self._previous_status, self._target_status), + ) return self._target_status @@ -488,12 +533,17 @@ def revert(self, context, *args, **kwargs): super(UpdateMinionPoolStatusTask, self).revert(*args, **kwargs) minion_pool = db_api.get_minion_pool( - context, self._minion_pool_id, include_machines=False, - include_events=False, include_progress_updates=False) + context, + self._minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ) if not minion_pool: LOG.debug( "[Task '%s'] Could not find pool with ID '%s' for status " - "reversion." % (self._task_name, self._minion_pool_id)) + "reversion." % (self._task_name, self._minion_pool_id) + ) return previous_status = self._previous_status @@ -501,39 +551,49 @@ def revert(self, context, *args, **kwargs): LOG.debug( "Forcibly reverting pool to status '%s' despite previous " "status being '%s'", - self._status_to_revert_to, self._previous_status) + self._status_to_revert_to, + self._previous_status, + ) previous_status = self._status_to_revert_to if minion_pool.status == previous_status: LOG.debug( "[Task '%s'] Minion pool '%s' is/was already reverted to " - "'%s'." % ( - self._task_name, self._minion_pool_id, - previous_status)) + "'%s'." % (self._task_name, self._minion_pool_id, previous_status) + ) else: if minion_pool.status != self._target_status: LOG.warn( "[Task %s] Minion pool '%s' is in status '%s', which is " "neither the previous status ('%s'), nor the newly-set " "status ('%s'). Reverting to '%s' anyway.", - self._task_name, self._minion_pool_id, minion_pool.status, - previous_status, self._target_status, previous_status) + self._task_name, + self._minion_pool_id, + minion_pool.status, + previous_status, + self._target_status, + previous_status, + ) LOG.debug( "[Task '%s'] Reverting pool '%s' status from '%s' to " - "'%s'" % ( - self._task_name, self._minion_pool_id, minion_pool.status, - previous_status)) - self._set_minion_pool_status( - context, self._minion_pool_id, previous_status) + "'%s'" + % ( + self._task_name, + self._minion_pool_id, + minion_pool.status, + previous_status, + ) + ) + self._set_minion_pool_status(context, self._minion_pool_id, previous_status) self._add_minion_pool_event( context, - "Pool status reverted from '%s' to '%s'" % ( - minion_pool.status, previous_status)) + "Pool status reverted from '%s' to '%s'" + % (minion_pool.status, previous_status), + ) class BaseMinionManangerTask( - coriolis_taskflow_base.BaseRunWorkerTask, - MinionManagerTaskEventMixin): - + coriolis_taskflow_base.BaseRunWorkerTask, MinionManagerTaskEventMixin +): """Base taskflow.Task implementation for Minion Mananger tasks. Acts as a simple adapter between minion-pool-specific params and the @@ -543,8 +603,8 @@ class BaseMinionManangerTask( default_provides = 'task_info' def __init__( - self, minion_pool_id, minion_machine_id, - main_task_runner_type, **kwargs): + self, minion_pool_id, minion_machine_id, main_task_runner_type, **kwargs + ): self._minion_pool_id = minion_pool_id self._minion_machine_id = minion_machine_id @@ -553,7 +613,11 @@ def __init__( # TODO(aznashwan): passing the minion pool ID as the task ID is # required to allow for the Minion pool event manager in the worker # service to know what pool to emit events for. - minion_pool_id, minion_machine_id, main_task_runner_type, **kwargs) + minion_pool_id, + minion_machine_id, + main_task_runner_type, + **kwargs, + ) @abc.abstractmethod def _get_task_name(self, minion_pool_id, minion_machine_id): @@ -562,12 +626,17 @@ def _get_task_name(self, minion_pool_id, minion_machine_id): def execute(self, context, origin, destination, task_info): LOG.info( "Starting minion pool task '%s' (runner type '%s')", - self._task_name, self._main_task_runner_type) + self._task_name, + self._main_task_runner_type, + ) res = super(BaseMinionManangerTask, self).execute( - context, origin, destination, task_info) + context, origin, destination, task_info + ) LOG.info( "Completed minion pool task '%s' (runner type '%s')", - self._task_name, self._main_task_runner_type) + self._task_name, + self._main_task_runner_type, + ) return res def revert(self, context, origin, destination, task_info, **kwargs): @@ -576,99 +645,108 @@ def revert(self, context, origin, destination, task_info, **kwargs): context, "Failure occurred for one or more operations on minion pool '%s'. " "Please check the logs for additional details. Error messages " - "were:\n%s" % ( + "were:\n%s" + % ( self._minion_pool_id, self._get_error_str_for_flow_failures( - flow_failures, full_tracebacks=False)), - level=constants.TASK_EVENT_ERROR) + flow_failures, full_tracebacks=False + ), + ), + level=constants.TASK_EVENT_ERROR, + ) super(BaseMinionManangerTask, self).revert( - context, origin, destination, task_info, **kwargs) + context, origin, destination, task_info, **kwargs + ) class ValidateMinionPoolOptionsTask(BaseMinionManangerTask): - - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - **kwargs): + def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): task_type = constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: - task_type = ( - constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS) + task_type = constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS super(ValidateMinionPoolOptionsTask, self).__init__( - minion_pool_id, minion_machine_id, task_type, **kwargs) + minion_pool_id, minion_machine_id, task_type, **kwargs + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_VALIDATION_TASK_NAME_FORMAT % minion_pool_id def execute(self, context, origin, destination, task_info): - self._add_minion_pool_event( - context, "Validating minion pool options") + self._add_minion_pool_event(context, "Validating minion pool options") _ = super(ValidateMinionPoolOptionsTask, self).execute( - context, origin, destination, task_info) + context, origin, destination, task_info + ) self._add_minion_pool_event( - context, "Successfully validated minion pool options") + context, "Successfully validated minion pool options" + ) def revert(self, context, origin, destination, task_info, **kwargs): LOG.debug("[%s] Nothing to revert for validation", self._task_name) super(ValidateMinionPoolOptionsTask, self).revert( - context, origin, destination, task_info, **kwargs) + context, origin, destination, task_info, **kwargs + ) class AllocateSharedPoolResourcesTask(BaseMinionManangerTask): - - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - **kwargs): + def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): resource_deployment_task_type = ( - constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES) + constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES + ) resource_cleanup_task_type = ( - constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES) + constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES + ) if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: resource_deployment_task_type = ( - constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES) + constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES + ) resource_cleanup_task_type = ( - constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES) # noqa: E501 + constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES + ) # noqa: E501 super(AllocateSharedPoolResourcesTask, self).__init__( - minion_pool_id, minion_machine_id, resource_deployment_task_type, - cleanup_task_runner_type=resource_cleanup_task_type, **kwargs) + minion_pool_id, + minion_machine_id, + resource_deployment_task_type, + cleanup_task_runner_type=resource_cleanup_task_type, + **kwargs, + ) def _get_task_name(self, minion_pool_id, minion_machine_id): - return MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % ( - minion_pool_id) + return MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % (minion_pool_id) def execute(self, context, origin, destination, task_info): with minion_manager_utils.get_minion_pool_lock( - self._minion_pool_id, external=True): - minion_pool = db_api.get_minion_pool( - context, self._minion_pool_id) + self._minion_pool_id, external=True + ): + minion_pool = db_api.get_minion_pool(context, self._minion_pool_id) if not minion_pool: raise exception.InvalidMinionPoolSelection( "[Task '%s'] Minion pool '%s' doesn't exist in the DB. " - "It cannot have shared resources deployed for it." % ( - self._task_name, self._minion_pool_id)) + "It cannot have shared resources deployed for it." + % (self._task_name, self._minion_pool_id) + ) if minion_pool.shared_resources: raise exception.InvalidMinionPoolState( "[Task '%s'] Minion pool already has shared resources " "defined for it. Cannot re-deploy shared resources. " - "DB entry is: %s" % ( - self._task_name, minion_pool.shared_resources)) + "DB entry is: %s" % (self._task_name, minion_pool.shared_resources) + ) - self._add_minion_pool_event( - context, "Deploying shared pool resources") + self._add_minion_pool_event(context, "Deploying shared pool resources") res = super(AllocateSharedPoolResourcesTask, self).execute( - context, origin, destination, task_info) + context, origin, destination, task_info + ) pool_shared_resources = res['pool_shared_resources'] - updated_values = { - "shared_resources": pool_shared_resources} + updated_values = {"shared_resources": pool_shared_resources} self._add_minion_pool_event( - context, "Successfully deployed shared pool resources") + context, "Successfully deployed shared pool resources" + ) with minion_manager_utils.get_minion_pool_lock( - self._minion_pool_id, external=True): - db_api.update_minion_pool( - context, self._minion_pool_id, updated_values) + self._minion_pool_id, external=True + ): + db_api.update_minion_pool(context, self._minion_pool_id, updated_values) task_info['pool_shared_resources'] = res['pool_shared_resources'] return task_info @@ -679,145 +757,181 @@ def revert(self, context, origin, destination, task_info, **kwargs): "[Task '%s'] Failed to find 'pool_shared_resources' in " "provided task_info from original execution of allocation " "task for pool '%s'. Defaulting to None.", - self._task_name, self._minion_pool_id) + self._task_name, + self._minion_pool_id, + ) task_info['pool_shared_resources'] = {} super(AllocateSharedPoolResourcesTask, self).revert( - context, origin, destination, task_info, **kwargs) + context, origin, destination, task_info, **kwargs + ) with minion_manager_utils.get_minion_pool_lock( - self._minion_pool_id, external=True): - updated_values = { - "pool_shared_resources": None} - db_api.update_minion_pool( - context, self._minion_pool_id, updated_values) + self._minion_pool_id, external=True + ): + updated_values = {"pool_shared_resources": None} + db_api.update_minion_pool(context, self._minion_pool_id, updated_values) class DeallocateSharedPoolResourcesTask(BaseMinionManangerTask): - - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - **kwargs): + def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): resource_deallocation_task = ( - constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES) + constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES + ) if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: resource_deallocation_task = ( - constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES) # noqa: E501 + constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES + ) # noqa: E501 super(DeallocateSharedPoolResourcesTask, self).__init__( - minion_pool_id, minion_machine_id, resource_deallocation_task, - **kwargs) + minion_pool_id, minion_machine_id, resource_deallocation_task, **kwargs + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % ( - minion_pool_id) + minion_pool_id + ) def execute(self, context, origin, destination, task_info): - self._add_minion_pool_event( - context, "Deallocating shared pool resources") + self._add_minion_pool_event(context, "Deallocating shared pool resources") if 'pool_shared_resources' not in task_info: raise exception.InvalidInput( "[Task '%s'] No 'pool_shared_resources' provided in the " - "task_info." % self._task_name) + "task_info." % self._task_name + ) execution_info = { - "pool_environment_options": task_info.get( - 'pool_environment_options', {}), - "pool_shared_resources": task_info['pool_shared_resources']} + "pool_environment_options": task_info.get('pool_environment_options', {}), + "pool_shared_resources": task_info['pool_shared_resources'], + } res = super(DeallocateSharedPoolResourcesTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) if res: LOG.warn( "[Task '%s'] Pool '%s' shared resource deallocation task " - "returned non-void values: %s" % ( - self._task_name, self._minion_pool_id, res)) - updated_values = { - "shared_resources": None} - db_api.update_minion_pool( - context, self._minion_pool_id, updated_values) + "returned non-void values: %s" + % (self._task_name, self._minion_pool_id, res) + ) + updated_values = {"shared_resources": None} + db_api.update_minion_pool(context, self._minion_pool_id, updated_values) self._add_minion_pool_event( - context, "Successfully deallocated shared pool resources") + context, "Successfully deallocated shared pool resources" + ) return task_info class AllocateMinionMachineTask(BaseMinionManangerTask): - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - raise_on_cleanup_failure=True, allocate_to_action=None, **kwargs): - resource_deployment_task_type = ( - constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE) - resource_cleanup_task_type = ( - constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE) + self, + minion_pool_id, + minion_machine_id, + minion_pool_type, + raise_on_cleanup_failure=True, + allocate_to_action=None, + **kwargs, + ): + resource_deployment_task_type = constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE + resource_cleanup_task_type = constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: resource_deployment_task_type = ( - constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE) + constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE + ) resource_cleanup_task_type = ( - constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE) + constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE + ) self._allocate_to_action = allocate_to_action self._raise_on_cleanup_failure = raise_on_cleanup_failure super(AllocateMinionMachineTask, self).__init__( - minion_pool_id, minion_machine_id, resource_deployment_task_type, + minion_pool_id, + minion_machine_id, + resource_deployment_task_type, cleanup_task_runner_type=resource_cleanup_task_type, - raise_on_cleanup_failure=raise_on_cleanup_failure, **kwargs) + raise_on_cleanup_failure=raise_on_cleanup_failure, + **kwargs, + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id) + minion_pool_id, + minion_machine_id, + ) def execute(self, context, origin, destination, task_info): minion_machine = self._get_minion_machine( - context, self._minion_machine_id, raise_if_not_found=False) + context, self._minion_machine_id, raise_if_not_found=False + ) if minion_machine: if minion_machine.allocation_status != ( - constants.MINION_MACHINE_STATUS_UNINITIALIZED): + constants.MINION_MACHINE_STATUS_UNINITIALIZED + ): raise exception.InvalidMinionMachineState( "Minion machine entry with ID '%s' already exists within " "the DB and it is in '%s' status instead of the expected " - "'%s' status. Existing machine's properties are: %s" % ( + "'%s' status. Existing machine's properties are: %s" + % ( self._minion_machine_id, minion_machine.allocation_status, constants.MINION_MACHINE_STATUS_UNINITIALIZED, - minion_machine.to_dict())) + minion_machine.to_dict(), + ) + ) if minion_machine.pool_id != self._minion_pool_id: raise exception.InvalidMinionMachineState( "Minion machine entry with ID '%s' already exists within " "the DB but it belongs to a different minion pool ('%s') " - "from the one requested by this task ('%s')." % ( - self._minion_machine_id, minion_machine.pool_id, - self._minion_pool_id)) + "from the one requested by this task ('%s')." + % ( + self._minion_machine_id, + minion_machine.pool_id, + self._minion_pool_id, + ) + ) if self._allocate_to_action and ( - minion_machine.allocated_action and ( - self._allocate_to_action != ( - minion_machine.allocated_action))): + minion_machine.allocated_action + and (self._allocate_to_action != (minion_machine.allocated_action)) + ): raise exception.InvalidMinionMachineState( "Minion machine entry with ID '%s' already exists in the " "DB but it is already allocated to a different action " - "('%s') from the one requested by the task ('%s')." % ( + "('%s') from the one requested by the task ('%s')." + % ( self._minion_machine_id, minion_machine.allocated_action, - self._allocate_to_action)) + self._allocate_to_action, + ) + ) LOG.info( "[Task '%s'] Found existing entry in DB for minion machine " "'%s'. Reusing that for deployment task.", - self._task_name, self._minion_machine_id) + self._task_name, + self._minion_machine_id, + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_ALLOCATING) + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_ALLOCATING, + ) else: minion_machine = models.MinionMachine() minion_machine.id = self._minion_machine_id minion_machine.pool_id = self._minion_pool_id minion_machine.allocation_status = ( - constants.MINION_MACHINE_STATUS_ALLOCATING) + constants.MINION_MACHINE_STATUS_ALLOCATING + ) minion_machine.power_status = ( - constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED) - log_msg = ( - "[Task '%s'] Adding new minion machine with ID '%s' " - "to the DB" % (self._task_name, self._minion_machine_id)) + constants.MINION_MACHINE_POWER_STATUS_UNINITIALIZED + ) + log_msg = "[Task '%s'] Adding new minion machine with ID '%s' to the DB" % ( + self._task_name, + self._minion_machine_id, + ) if self._allocate_to_action: minion_machine.allocated_action = self._allocate_to_action log_msg = "%s (allocated to action '%s')" % ( - log_msg, self._allocate_to_action) + log_msg, + self._allocate_to_action, + ) LOG.info(log_msg) db_api.add_minion_machine(context, minion_machine) @@ -825,31 +939,37 @@ def execute(self, context, origin, destination, task_info): "pool_environment_options": task_info["pool_environment_options"], "pool_identifier": task_info["pool_identifier"], "pool_shared_resources": task_info["pool_shared_resources"], - "pool_os_type": task_info["pool_os_type"]} + "pool_os_type": task_info["pool_os_type"], + } - event_message = ( - "Allocating minion machine with internal pool ID '%s'" % ( - self._minion_machine_id)) + event_message = "Allocating minion machine with internal pool ID '%s'" % ( + self._minion_machine_id + ) if self._allocate_to_action: - event_message = ( - "%s to be used for transfer action with ID '%s'" % ( - event_message, self._allocate_to_action)) - self._add_minion_pool_event( - context, event_message) + event_message = "%s to be used for transfer action with ID '%s'" % ( + event_message, + self._allocate_to_action, + ) + self._add_minion_pool_event(context, event_message) try: res = super(AllocateMinionMachineTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) except Exception: self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING) + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, + ) raise self._add_minion_pool_event( context, "Successfully allocated minion machine with internal pool " - "ID '%s'" % (self._minion_machine_id)) + "ID '%s'" % (self._minion_machine_id), + ) updated_values = { "last_used_at": timeutils.utcnow(), @@ -858,14 +978,15 @@ def execute(self, context, origin, destination, task_info): "connection_info": res['minion_connection_info'], "provider_properties": res['minion_provider_properties'], "backup_writer_connection_info": res[ - "minion_backup_writer_connection_info"]} + "minion_backup_writer_connection_info" + ], + } if self._allocate_to_action: updated_values["allocated_action"] = self._allocate_to_action - updated_values["allocation_status"] = ( - constants.MINION_MACHINE_STATUS_IN_USE) + updated_values["allocation_status"] = constants.MINION_MACHINE_STATUS_IN_USE self._update_minion_machine( - context, self._minion_pool_id, self._minion_machine_id, - updated_values) + context, self._minion_pool_id, self._minion_machine_id, updated_values + ) return task_info @@ -873,7 +994,8 @@ def revert(self, context, origin, destination, task_info, **kwargs): minion_provider_properties = None task_info_minion_provider_properties = task_info.get( - 'minion_provider_properties') + 'minion_provider_properties' + ) # check if the original result is a taskflow Failure object: original_result = kwargs.get('result', {}) @@ -883,64 +1005,87 @@ def revert(self, context, origin, destination, task_info, **kwargs): "(pool '%s') received a failure as the original result. " "Presuming the original execution failed and found the " "following 'machine_properties' key in task info: %s", - self._task_name, self._minion_machine_id, - self._minion_pool_id, task_info_minion_provider_properties) + self._task_name, + self._minion_machine_id, + self._minion_pool_id, + task_info_minion_provider_properties, + ) LOG.warn( "[Task '%s'] Allocation failed for machine '%s'. Error " "details were: %s", - self._task_name, self._minion_machine_id, - original_result.traceback_str) + self._task_name, + self._minion_machine_id, + original_result.traceback_str, + ) # else, if it's a dict, fetch it: elif isinstance(original_result, dict): minion_provider_properties = original_result.get( - 'minion_provider_properties', None) + 'minion_provider_properties', None + ) else: LOG.warn( "[Task '%s'] Allocation task reversion for machine '%s' " "of pool '%s' got an unexpected task result type (%s): %s", - self._task_name, self._minion_machine_id, - self._minion_pool_id, type(original_result), original_result) + self._task_name, + self._minion_machine_id, + self._minion_pool_id, + type(original_result), + original_result, + ) minion_provider_properties = None # default to any minion properties found in the task_info: task_info_minion_provider_properties = task_info.get( - 'minion_provider_properties') + 'minion_provider_properties' + ) if not minion_provider_properties: LOG.debug( "[Task '%s'] Reversion for Minion Machine '%s' (pool '%s')" " did not return any 'minion_provider_properties' after " "its initial execution. Defaulting to task_info value: %s", - self._task_name, self._minion_machine_id, + self._task_name, + self._minion_machine_id, self._minion_pool_id, - task_info_minion_provider_properties) + task_info_minion_provider_properties, + ) minion_provider_properties = task_info_minion_provider_properties # lastly, if the machine entry exists in the DB: with minion_manager_utils.get_minion_pool_lock( - self._minion_pool_id, external=True): - machine_db_entry = ( - db_api.get_minion_machine(context, self._minion_machine_id)) + self._minion_pool_id, external=True + ): + machine_db_entry = db_api.get_minion_machine( + context, self._minion_machine_id + ) if machine_db_entry: LOG.debug( "[Task %s] Removing minion machine entry with ID '%s' for " "minion pool '%s' from the DB as part of reversion of its " "allocation task. Machine properties at deletion time " - "were: %s", self._task_name, self._minion_machine_id, - self._minion_pool_id, machine_db_entry.to_dict()) + "were: %s", + self._task_name, + self._minion_machine_id, + self._minion_pool_id, + machine_db_entry.to_dict(), + ) if not minion_provider_properties and ( - machine_db_entry.provider_properties): - minion_provider_properties = ( - machine_db_entry.provider_properties) + machine_db_entry.provider_properties + ): + minion_provider_properties = machine_db_entry.provider_properties LOG.debug( "[Task '%s'] Using minion provider properties of " "minion machine with ID '%s' from DB entry during the " "reversion of its allocation task. DB props are: %s", - self._task_name, self._minion_machine_id, - minion_provider_properties) + self._task_name, + self._minion_machine_id, + minion_provider_properties, + ) LOG.debug( "[Task %s] Deleting minion machine with ID '%s' from the DB.", - self._task_name, self._minion_machine_id) + self._task_name, + self._minion_machine_id, + ) try: db_api.delete_minion_machine(context, self._minion_machine_id) except Exception: @@ -948,59 +1093,73 @@ def revert(self, context, origin, destination, task_info, **kwargs): "[Task '%s'] Failed to delete DB entry for minion machine " "'%s' following reversion of its allocation task. Error " "trace was: %s", - self._task_name, self._minion_machine_id, - utils.get_exception_details()) + self._task_name, + self._minion_machine_id, + utils.get_exception_details(), + ) if not minion_provider_properties: LOG.debug( "[Task '%s'] Reversion for Minion Machine '%s' (pool '%s') " "found no 'minion_provider_properties'. Presuming the machine " "never got created on the cloud and skiping any deletion task", - self._task_name, self._minion_machine_id, - self._minion_pool_id) + self._task_name, + self._minion_machine_id, + self._minion_pool_id, + ) else: cleanup_info = copy.deepcopy(task_info) - cleanup_info['minion_provider_properties'] = ( - minion_provider_properties) + cleanup_info['minion_provider_properties'] = minion_provider_properties try: super(AllocateMinionMachineTask, self).revert( - context, origin, destination, cleanup_info, **kwargs) + context, origin, destination, cleanup_info, **kwargs + ) except Exception: log_msg = ( "[Task '%s'] Exception occurred while attempting to " "revert deployment of minion machine with ID '%s' " - "for pool '%s'." % ( - self._task_name, self._minion_machine_id, - self._minion_pool_id)) + "for pool '%s'." + % (self._task_name, self._minion_machine_id, self._minion_pool_id) + ) if not self._raise_on_cleanup_failure: - log_msg = ( - "%s Ignoring exception." % log_msg) - log_msg = ( - "%s Exception details were: %s" % ( - log_msg, utils.get_exception_details)) + log_msg = "%s Ignoring exception." % log_msg + log_msg = "%s Exception details were: %s" % ( + log_msg, + utils.get_exception_details, + ) LOG.warn(log_msg) if self._raise_on_cleanup_failure: raise class DeallocateMinionMachineTask(BaseMinionManangerTask): - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - raise_on_cleanup_failure=True, **kwargs): - resource_deletion_task_type = ( - constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE) + self, + minion_pool_id, + minion_machine_id, + minion_pool_type, + raise_on_cleanup_failure=True, + **kwargs, + ): + resource_deletion_task_type = constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE self._raise_on_cleanup_failure = raise_on_cleanup_failure if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: resource_deletion_task_type = ( - constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE) + constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE + ) super(DeallocateMinionMachineTask, self).__init__( - minion_pool_id, minion_machine_id, resource_deletion_task_type, - raise_on_cleanup_failure=raise_on_cleanup_failure, **kwargs) + minion_pool_id, + minion_machine_id, + resource_deletion_task_type, + raise_on_cleanup_failure=raise_on_cleanup_failure, + **kwargs, + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id) + minion_pool_id, + minion_machine_id, + ) def execute(self, context, origin, destination, task_info): machine = self._get_minion_machine(context, self._minion_machine_id) @@ -1008,36 +1167,45 @@ def execute(self, context, origin, destination, task_info): LOG.info( "[Task '%s'] Could not find machine with ID '%s' in the DB. " "Presuming it was already deleted and returning early.", - self._task_name, self._minion_machine_id) + self._task_name, + self._minion_machine_id, + ) return task_info self._add_minion_pool_event( context, - "Deallocating minion machine with internal pool ID '%s'" % ( - self._minion_machine_id)) + "Deallocating minion machine with internal pool ID '%s'" + % (self._minion_machine_id), + ) if machine.provider_properties: self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_DEALLOCATING) - execution_info = { - "minion_provider_properties": machine.provider_properties} + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_DEALLOCATING, + ) + execution_info = {"minion_provider_properties": machine.provider_properties} try: _ = super(DeallocateMinionMachineTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) except Exception: base_msg = ( "Exception occured while deallocating minion machine '%s' " "There might be leftover instance resources requiring " - "manual cleanup" % self._minion_machine_id) + "manual cleanup" % self._minion_machine_id + ) LOG.warn( "[Task '%s'] %s. Error was: %s", - self._task_name, base_msg, utils.get_exception_details()) + self._task_name, + base_msg, + utils.get_exception_details(), + ) event_level = constants.TASK_EVENT_INFO if self._raise_on_cleanup_failure: event_level = constants.TASK_EVENT_ERROR - self._add_minion_pool_event( - context, base_msg, level=event_level) + self._add_minion_pool_event(context, base_msg, level=event_level) if self._raise_on_cleanup_failure: raise else: @@ -1045,62 +1213,73 @@ def execute(self, context, origin, destination, task_info): context, "Minion machine with ID '%s' had no provider properties set. " "Presuming it failed to deploy in the first place and simply " - "removing the machine's entry from the DB" % ( - self._minion_machine_id)) + "removing the machine's entry from the DB" % (self._minion_machine_id), + ) LOG.debug( "[Task '%s'] Deleting minion machine with ID '%s' from the DB", - self._task_name, self._minion_machine_id) + self._task_name, + self._minion_machine_id, + ) with minion_manager_utils.get_minion_pool_lock( - self._minion_pool_id, external=True): + self._minion_pool_id, external=True + ): db_api.delete_minion_machine(context, self._minion_machine_id) self._add_minion_pool_event( context, "Successfully deallocated minion machine with internal pool " - "ID '%s'" % (self._minion_machine_id)) + "ID '%s'" % (self._minion_machine_id), + ) return task_info class HealthcheckMinionMachineTask(BaseMinionManangerTask): - """ Task which healthchecks the given minion machine. """ + """Task which healthchecks the given minion machine.""" def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - fail_on_error=False, - machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 - **kwargs): + self, + minion_pool_id, + minion_machine_id, + minion_pool_type, + fail_on_error=False, + machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 + **kwargs, + ): self._fail_on_error = fail_on_error self._machine_status_on_success = machine_status_on_success - resource_healthcheck_task = ( - constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION) + resource_healthcheck_task = constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: resource_healthcheck_task = ( - constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION) + constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION + ) super(HealthcheckMinionMachineTask, self).__init__( - minion_pool_id, minion_machine_id, resource_healthcheck_task, - **kwargs) + minion_pool_id, minion_machine_id, resource_healthcheck_task, **kwargs + ) def execute(self, context, origin, destination, task_info): - res = { - "healthy": True, - "error": None} + res = {"healthy": True, "error": None} machine = self._get_minion_machine( - context, self._minion_machine_id, raise_if_not_found=False) + context, self._minion_machine_id, raise_if_not_found=False + ) if not machine: LOG.info( "[Task '%s'] Could not find machine with ID '%s' in the DB. " "Presuming it was already deleted so healthcheck failed.", - self._task_name, self._minion_machine_id) + self._task_name, + self._minion_machine_id, + ) base_msg = ( "Could not find minion machine DB entry with ID '%s' for " - "healthcheck." % self._minion_machine_id) + "healthcheck." % self._minion_machine_id + ) self._add_minion_pool_event( context, "%s Reporting healthcheck as failed" % base_msg, - level=constants.TASK_EVENT_WARNING) + level=constants.TASK_EVENT_WARNING, + ) if self._fail_on_error: raise exception.InvalidMinionMachineState(base_msg) @@ -1109,17 +1288,19 @@ def execute(self, context, origin, destination, task_info): machine_error_statuses = [ constants.MINION_MACHINE_STATUS_ERROR, constants.MINION_MACHINE_STATUS_POWER_ERROR, - constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING] + constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, + ] if machine.allocation_status in machine_error_statuses: - base_msg = ( - "Minion Machine with ID '%s' is marked as '%s' in the DB." % ( - self._minion_machine_id, machine.allocation_status)) - LOG.debug( - "[Task '%s'] %s" % (self._task_name, base_msg)) + base_msg = "Minion Machine with ID '%s' is marked as '%s' in the DB." % ( + self._minion_machine_id, + machine.allocation_status, + ) + LOG.debug("[Task '%s'] %s" % (self._task_name, base_msg)) self._add_minion_pool_event( context, "%s Reporting healthcheck as failed" % base_msg, - level=constants.TASK_EVENT_WARNING) + level=constants.TASK_EVENT_WARNING, + ) if self._fail_on_error: raise exception.InvalidMinionMachineState(base_msg) @@ -1127,172 +1308,207 @@ def execute(self, context, origin, destination, task_info): self._add_minion_pool_event( context, - "Healthchecking minion machine with internal pool ID '%s'" % ( - self._minion_machine_id)) + "Healthchecking minion machine with internal pool ID '%s'" + % (self._minion_machine_id), + ) execution_info = { "minion_provider_properties": machine.provider_properties, - "minion_connection_info": machine.connection_info} + "minion_connection_info": machine.connection_info, + } try: _ = super(HealthcheckMinionMachineTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) self._add_minion_pool_event( context, "Successfully healthchecked minion machine with internal " - "pool ID '%s'" % self._minion_machine_id) + "pool ID '%s'" % self._minion_machine_id, + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - self._machine_status_on_success) + context, + self._minion_pool_id, + self._minion_machine_id, + self._machine_status_on_success, + ) except Exception as ex: self._add_minion_pool_event( context, "Healthcheck for machine with internal pool ID '%s' has " "failed." % (self._minion_machine_id), - level=constants.TASK_EVENT_WARNING) + level=constants.TASK_EVENT_WARNING, + ) LOG.debug( "[Task '%s'] Healthcheck failed for machine '%s' of pool '%s'." - "Full trace was:\n%s", self._task_name, - self._minion_machine_id, self._minion_pool_id, - utils.get_exception_details()) + "Full trace was:\n%s", + self._task_name, + self._minion_machine_id, + self._minion_pool_id, + utils.get_exception_details(), + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_ERROR) + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_ERROR, + ) if not self._fail_on_error: - res = { - "healthy": False, - "error": str(ex)} + res = {"healthy": False, "error": str(ex)} else: raise return res def _get_task_name(self, minion_pool_id, minion_machine_id): - return self.get_healthcheck_task_name( - minion_pool_id, minion_machine_id) + return self.get_healthcheck_task_name(minion_pool_id, minion_machine_id) @classmethod def get_healthcheck_task_name(cls, minion_pool_id, minion_machine_id): return MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id) + minion_pool_id, + minion_machine_id, + ) class MinionMachineHealtchcheckDecider(object): - """ A callable to green/redlight further execution based on the result. """ + """A callable to green/redlight further execution based on the result.""" def __init__( - self, minion_pool_id, minion_machine_id, - on_successful_healthcheck=True): + self, minion_pool_id, minion_machine_id, on_successful_healthcheck=True + ): self._minion_pool_id = minion_pool_id self._minion_machine_id = minion_machine_id self._on_success = on_successful_healthcheck def __call__(self, history): - healthcheck_task_name = ( - HealthcheckMinionMachineTask.get_healthcheck_task_name( - self._minion_pool_id, self._minion_machine_id)) + healthcheck_task_name = HealthcheckMinionMachineTask.get_healthcheck_task_name( + self._minion_pool_id, self._minion_machine_id + ) if not history or healthcheck_task_name not in history: LOG.warn( "Could not find healthceck result for minion machine '%s' " "of pool '%s' (task name '%s'). NOT greenlighting futher " - "tasks.", self._minion_machine_id, self._minion_pool_id, - healthcheck_task_name) + "tasks.", + self._minion_machine_id, + self._minion_pool_id, + healthcheck_task_name, + ) return False healtcheck_result = history[healthcheck_task_name] if healtcheck_result.get('healthy'): LOG.debug( - "Healtcheck task '%s' confirmed worker health. Decider " - "returning %s", healthcheck_task_name, - self._on_success) + "Healtcheck task '%s' confirmed worker health. Decider returning %s", + healthcheck_task_name, + self._on_success, + ) return self._on_success else: LOG.debug( "Healtcheck task '%s' denied worker health. Decider " "returning %s. Error mesage was: %s", - healthcheck_task_name, not self._on_success, - healtcheck_result.get('error')) + healthcheck_task_name, + not self._on_success, + healtcheck_result.get('error'), + ) return not self._on_success class PowerOnMinionMachineTask(BaseMinionManangerTask): - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - fail_on_error=True, **kwargs): + self, + minion_pool_id, + minion_machine_id, + minion_pool_type, + fail_on_error=True, + **kwargs, + ): self._fail_on_error = fail_on_error - power_on_task_type = ( - constants.TASK_TYPE_POWER_ON_SOURCE_MINION) + power_on_task_type = constants.TASK_TYPE_POWER_ON_SOURCE_MINION if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: - power_on_task_type = ( - constants.TASK_TYPE_POWER_ON_DESTINATION_MINION) + power_on_task_type = constants.TASK_TYPE_POWER_ON_DESTINATION_MINION super(PowerOnMinionMachineTask, self).__init__( - minion_pool_id, minion_machine_id, power_on_task_type, - **kwargs) + minion_pool_id, minion_machine_id, power_on_task_type, **kwargs + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id) + minion_pool_id, + minion_machine_id, + ) def execute(self, context, origin, destination, task_info): machine = self._get_minion_machine( - context, self._minion_machine_id, raise_if_not_found=True) + context, self._minion_machine_id, raise_if_not_found=True + ) - if (machine.power_status == - constants.MINION_MACHINE_POWER_STATUS_POWERED_ON): + if machine.power_status == constants.MINION_MACHINE_POWER_STATUS_POWERED_ON: LOG.debug( "[Task '%s'] Minion machine with ID '%s' from pool '%s' is " - "already marked as powered on. Returning early." % ( - self._task_name, self._minion_machine_id, - self._minion_pool_id)) + "already marked as powered on. Returning early." + % (self._task_name, self._minion_machine_id, self._minion_pool_id) + ) return task_info - if (machine.power_status != - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF): + if machine.power_status != constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF: raise exception.InvalidMinionMachineState( "Minion machine with ID '%s' from pool '%s' is in '%s' state " "instead of the expected '%s' required for it to be powered " - "on." % ( - self._minion_machine_id, self._minion_pool_id, + "on." + % ( + self._minion_machine_id, + self._minion_pool_id, machine.power_status, - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF)) + constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF, + ) + ) - execution_info = { - "minion_provider_properties": machine.provider_properties} + execution_info = {"minion_provider_properties": machine.provider_properties} try: self._set_minion_machine_power_status( - context, self._minion_pool_id, + context, + self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_POWER_STATUS_POWERING_ON) + constants.MINION_MACHINE_POWER_STATUS_POWERING_ON, + ) _ = super(PowerOnMinionMachineTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) self._set_minion_machine_power_status( - context, self._minion_pool_id, + context, + self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_POWER_STATUS_POWERED_ON) + constants.MINION_MACHINE_POWER_STATUS_POWERED_ON, + ) self._add_minion_pool_event( context, "Successfully powered on minion machine with internal pool " - "ID '%s'" % self._minion_machine_id) + "ID '%s'" % self._minion_machine_id, + ) except Exception: base_msg = ( "[Task '%s'] Exception occurred while powering on minion " - "machine with ID '%s' of pool '%s'." % ( - self._task_name, self._minion_machine_id, - self._minion_pool_id)) + "machine with ID '%s' of pool '%s'." + % (self._task_name, self._minion_machine_id, self._minion_pool_id) + ) LOG.warn( - "%s Error details were: %s" % ( - base_msg, utils.get_exception_details())) + "%s Error details were: %s" % (base_msg, utils.get_exception_details()) + ) self._add_minion_pool_event( context, "Exception occurred while powering on minion machine with " "internal pool ID '%s'. The minion machine will be marked " - "as ERROR'd and automatically redeployed later." % ( - self._minion_machine_id), - level=constants.TASK_EVENT_ERROR) + "as ERROR'd and automatically redeployed later." + % (self._minion_machine_id), + level=constants.TASK_EVENT_ERROR, + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_POWER_ERROR) + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_POWER_ERROR, + ) if self._fail_on_error: raise exception.CoriolisException(base_msg) @@ -1300,79 +1516,94 @@ def execute(self, context, origin, destination, task_info): class PowerOffMinionMachineTask(BaseMinionManangerTask): - def __init__( - self, minion_pool_id, minion_machine_id, minion_pool_type, - fail_on_error=True, - status_once_powered_off=constants.MINION_MACHINE_STATUS_AVAILABLE, - **kwargs): + self, + minion_pool_id, + minion_machine_id, + minion_pool_type, + fail_on_error=True, + status_once_powered_off=constants.MINION_MACHINE_STATUS_AVAILABLE, + **kwargs, + ): self._fail_on_error = fail_on_error self._status_once_powered_off = status_once_powered_off - power_on_task_type = ( - constants.TASK_TYPE_POWER_OFF_SOURCE_MINION) + power_on_task_type = constants.TASK_TYPE_POWER_OFF_SOURCE_MINION if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: - power_on_task_type = ( - constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION) + power_on_task_type = constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION super(PowerOffMinionMachineTask, self).__init__( - minion_pool_id, minion_machine_id, power_on_task_type, - **kwargs) + minion_pool_id, minion_machine_id, power_on_task_type, **kwargs + ) def _get_task_name(self, minion_pool_id, minion_machine_id): return MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id) + minion_pool_id, + minion_machine_id, + ) def execute(self, context, origin, destination, task_info): machine = self._get_minion_machine( - context, self._minion_machine_id, raise_if_not_found=True) + context, self._minion_machine_id, raise_if_not_found=True + ) - if machine.power_status == ( - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF): + if machine.power_status == (constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF): LOG.debug( "[Task '%s'] Minion machine with ID '%s' from pool '%s' is " - "already marked as powered off. Returning early." % ( - self._task_name, self._minion_machine_id, - self._minion_pool_id)) + "already marked as powered off. Returning early." + % (self._task_name, self._minion_machine_id, self._minion_pool_id) + ) return task_info - execution_info = { - "minion_provider_properties": machine.provider_properties} + execution_info = {"minion_provider_properties": machine.provider_properties} try: self._set_minion_machine_power_status( - context, self._minion_pool_id, + context, + self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF) + constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF, + ) _ = super(PowerOffMinionMachineTask, self).execute( - context, origin, destination, execution_info) + context, origin, destination, execution_info + ) self._set_minion_machine_power_status( - context, self._minion_pool_id, + context, + self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF) + constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF, + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - self._status_once_powered_off) + context, + self._minion_pool_id, + self._minion_machine_id, + self._status_once_powered_off, + ) self._add_minion_pool_event( context, "Successfully powered off minion machine with internal pool " - "ID '%s'" % self._minion_machine_id) + "ID '%s'" % self._minion_machine_id, + ) except Exception: base_msg = ( "[Task '%s'] Exception occurred while powering off minion " - "machine with ID '%s' of pool '%s'." % ( - self._task_name, self._minion_machine_id, - self._minion_pool_id)) + "machine with ID '%s' of pool '%s'." + % (self._task_name, self._minion_machine_id, self._minion_pool_id) + ) self._add_minion_pool_event( context, "Exception occurred while powering off minion machine with " "internal pool ID '%s'. The minion machine will be marked " - "as ERROR'd and automatically redeployed later." % ( - self._minion_machine_id), - level=constants.TASK_EVENT_ERROR) + "as ERROR'd and automatically redeployed later." + % (self._minion_machine_id), + level=constants.TASK_EVENT_ERROR, + ) self._set_minion_machine_allocation_status( - context, self._minion_pool_id, self._minion_machine_id, - constants.MINION_MACHINE_STATUS_POWER_ERROR) + context, + self._minion_pool_id, + self._minion_machine_id, + constants.MINION_MACHINE_STATUS_POWER_ERROR, + ) LOG.warn( - "%s Error details were: %s" % ( - base_msg, utils.get_exception_details())) + "%s Error details were: %s" % (base_msg, utils.get_exception_details()) + ) if self._fail_on_error: raise exception.CoriolisException(base_msg) diff --git a/coriolis/minion_manager/rpc/utils.py b/coriolis/minion_manager/rpc/utils.py index d491d700..9ec0d282 100644 --- a/coriolis/minion_manager/rpc/utils.py +++ b/coriolis/minion_manager/rpc/utils.py @@ -10,19 +10,21 @@ def get_minion_pool_lock(minion_pool_id, external=True): return lockutils.lock( - constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id, - external=external) + constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id, external=external + ) def minion_pool_synchronized(minion_pool_id, func): @functools.wraps(func) def wrapper(*args, **kwargs): @lockutils.synchronized( - constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id, - external=True) + constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id, external=True + ) def inner(): return func(*args, **kwargs) + return inner() + return wrapper @@ -30,7 +32,9 @@ def minion_pool_synchronized_op(func): @functools.wraps(func) def wrapper(self, ctxt, minion_pool_id, *args, **kwargs): return minion_pool_synchronized(minion_pool_id, func)( - self, ctxt, minion_pool_id, *args, **kwargs) + self, ctxt, minion_pool_id, *args, **kwargs + ) + return wrapper @@ -38,10 +42,13 @@ def minion_machine_synchronized(minion_pool_id, minion_machine_id, func): @functools.wraps(func) def wrapper(*args, **kwargs): @lockutils.synchronized( - constants.MINION_MACHINE_LOCK_NAME_FORMAT % ( - minion_pool_id, minion_machine_id), - external=True) + constants.MINION_MACHINE_LOCK_NAME_FORMAT + % (minion_pool_id, minion_machine_id), + external=True, + ) def inner(): return func(*args, **kwargs) + return inner() + return wrapper diff --git a/coriolis/minion_pools/api.py b/coriolis/minion_pools/api.py index 4bf60de4..a007f333 100644 --- a/coriolis/minion_pools/api.py +++ b/coriolis/minion_pools/api.py @@ -9,19 +9,39 @@ def __init__(self): self._rpc_client = rpc_client.MinionManagerClient() def create( - self, ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=None, - skip_allocation=False): + self, + ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=None, + skip_allocation=False, + ): return self._rpc_client.create_minion_pool( - ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=notes, - skip_allocation=skip_allocation) + ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=notes, + skip_allocation=skip_allocation, + ) def update(self, ctxt, minion_pool_id, updated_values): return self._rpc_client.update_minion_pool( - ctxt, minion_pool_id, updated_values=updated_values) + ctxt, minion_pool_id, updated_values=updated_values + ) def delete(self, ctxt, minion_pool_id): self._rpc_client.delete_minion_pool(ctxt, minion_pool_id) @@ -33,13 +53,12 @@ def get_minion_pool(self, ctxt, minion_pool_id): return self._rpc_client.get_minion_pool(ctxt, minion_pool_id) def allocate_minion_pool(self, ctxt, minion_pool_id): - return self._rpc_client.allocate_minion_pool( - ctxt, minion_pool_id) + return self._rpc_client.allocate_minion_pool(ctxt, minion_pool_id) def refresh_minion_pool(self, ctxt, minion_pool_id): - return self._rpc_client.refresh_minion_pool( - ctxt, minion_pool_id) + return self._rpc_client.refresh_minion_pool(ctxt, minion_pool_id) def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): return self._rpc_client.deallocate_minion_pool( - ctxt, minion_pool_id, force=force) + ctxt, minion_pool_id, force=force + ) diff --git a/coriolis/osmorphing/amazon.py b/coriolis/osmorphing/amazon.py index d1cf72fa..541a84eb 100644 --- a/coriolis/osmorphing/amazon.py +++ b/coriolis/osmorphing/amazon.py @@ -1,21 +1,19 @@ # Copyright 2023 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.osmorphing.osdetect import amazon as amazon_detect from coriolis.osmorphing import redhat - +from coriolis.osmorphing.osdetect import amazon as amazon_detect AMAZON_DISTRO_NAME_IDENTIFIER = amazon_detect.AMAZON_DISTRO_NAME class BaseAmazonLinuxOSMorphingTools(redhat.BaseRedHatMorphingTools): - UEFI_GRUB_LOCATION = "/boot/efi/EFI/amzn" @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - AMAZON_DISTRO_NAME_IDENTIFIER): + if detected_os_info['distribution_name'] != (AMAZON_DISTRO_NAME_IDENTIFIER): return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=2) + detected_os_info['release_version'], minimum=2 + ) diff --git a/coriolis/osmorphing/base.py b/coriolis/osmorphing/base.py index e7a99f04..ff2a0b0e 100644 --- a/coriolis/osmorphing/base.py +++ b/coriolis/osmorphing/base.py @@ -7,13 +7,12 @@ import re import uuid +import yaml from oslo_log import log as logging from six import with_metaclass -import yaml -from coriolis import exception +from coriolis import exception, utils from coriolis.osmorphing.netpreserver import factory -from coriolis import utils GRUB2_SERIAL = "serial --word=8 --stop=1 --speed=%d --parity=%s --unit=0" LOG = logging.getLogger(__name__) @@ -22,19 +21,28 @@ # Required OS release fields which are expected from the OSDetect tools. # 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' schema: REQUIRED_DETECTED_OS_FIELDS = [ - "os_type", "distribution_name", "release_version", - "friendly_release_name"] + "os_type", + "distribution_name", + "release_version", + "friendly_release_name", +] DEFAULT_CLOUD_USER = "cloud-user" CLOUD_INIT_SERVICE_UNIT_NAME = "cloud-init" CLOUD_INIT_SERVICE_UNIT_NAME_FALLBACK = "cloud-init-main" class BaseOSMorphingTools(object, with_metaclass(abc.ABCMeta)): - def __init__( - self, conn, os_root_dir, os_root_device, hypervisor, - event_manager, detected_os_info, osmorphing_parameters, - operation_timeout): + self, + conn, + os_root_dir, + os_root_device, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout, + ): self.check_detected_os_info_parameters(detected_os_info) @@ -59,30 +67,32 @@ def get_required_detected_os_info_fields(cls): def check_detected_os_info_parameters(cls, detected_os_info): required_fields = cls.get_required_detected_os_info_fields() missing_os_info_fields = [ - field for field in required_fields - if field not in detected_os_info] + field for field in required_fields if field not in detected_os_info + ] if missing_os_info_fields: raise exception.InvalidDetectedOSParams( "There are parameters (%s) which are required by %s but " - "are missing from the detected OS info: %s" % ( - missing_os_info_fields, cls.__name__, detected_os_info)) + "are missing from the detected OS info: %s" + % (missing_os_info_fields, cls.__name__, detected_os_info) + ) extra_os_info_fields = [ - field for field in detected_os_info - if field not in required_fields] + field for field in detected_os_info if field not in required_fields + ] if extra_os_info_fields: raise exception.InvalidDetectedOSParams( "There were detected OS info parameters (%s) which were not " - "expected by %s: %s" % ( - extra_os_info_fields, cls.__name__, detected_os_info)) + "expected by %s: %s" + % (extra_os_info_fields, cls.__name__, detected_os_info) + ) return True @classmethod @abc.abstractmethod def check_os_supported(cls, detected_os_info): raise NotImplementedError( - "OS compatibility check not implemented for tools class %s" % ( - cls.__name__)) + "OS compatibility check not implemented for tools class %s" % (cls.__name__) + ) @abc.abstractmethod def get_installed_packages(self): @@ -129,15 +139,29 @@ def set_environment(self, environment): class BaseLinuxOSMorphingTools(BaseOSMorphingTools): - _packages = {} - def __init__(self, conn, os_root_dir, os_root_dev, hypervisor, - event_manager, detected_os_info, osmorphing_parameters, - operation_timeout=None): + def __init__( + self, + conn, + os_root_dir, + os_root_dev, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout=None, + ): super(BaseLinuxOSMorphingTools, self).__init__( - conn, os_root_dir, os_root_dev, hypervisor, event_manager, - detected_os_info, osmorphing_parameters, operation_timeout) + conn, + os_root_dir, + os_root_dev, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout, + ) self._ssh = conn @classmethod @@ -146,7 +170,7 @@ def get_required_detected_os_info_fields(cls): @classmethod def _version_supported_util(cls, version, minimum, maximum=None): - """ Parses version strings which are prefixed with a floating point and + """Parses version strings which are prefixed with a floating point and checks whether the value is between the provided minimum and maximum (excluding the maximum). If a check for specific version is desired, the provided minimum and @@ -158,59 +182,74 @@ def _version_supported_util(cls, version, minimum, maximum=None): if type(version) is not str: raise ValueError( - "Non-string version provided: %s (type %s)" % ( - version, type(version))) + "Non-string version provided: %s (type %s)" % (version, type(version)) + ) float_regex = "([0-9]+(\\.[0-9]+)?)" match = re.match(float_regex, version) if not match: - LOG.warn( - "Version string '%s' does not contain a float", version) + LOG.warn("Version string '%s' does not contain a float", version) return False version_float = None try: version_float = float(match.groups()[0]) except ValueError: - LOG.warn( - "Failed to parse float OS release '%s'", match.groups()[0]) + LOG.warn("Failed to parse float OS release '%s'", match.groups()[0]) return False LOG.debug( - "Extracted float version from string '%s' is: %s", - version, version_float) + "Extracted float version from string '%s' is: %s", version, version_float + ) if version_float < minimum: LOG.debug( - "Version '%s' smaller than the minimum of '%s' for " - "release: %s", version_float, minimum, version) + "Version '%s' smaller than the minimum of '%s' for release: %s", + version_float, + minimum, + version, + ) return False if maximum: if maximum == minimum and version_float == minimum: LOG.debug( "Version '%s' coincides exactly with the required " - "minimum/maximum version: %s", version_float, minimum) + "minimum/maximum version: %s", + version_float, + minimum, + ) return True if version_float >= maximum: LOG.debug( "Version '%s' is larger or equal to the maximum of '%s' " - "for release: %s", version_float, maximum, version) + "for release: %s", + version_float, + maximum, + version, + ) return False return True def get_packages(self): - k_add = [h for h in self._packages.keys() if - h is None or h == self._hypervisor] + k_add = [h for h in self._packages.keys() if h is None or h == self._hypervisor] - add = [p[0] for p in itertools.chain.from_iterable( - [l for k, l in self._packages.items() if k in k_add])] + add = [ + p[0] + for p in itertools.chain.from_iterable( + [pkgs for k, pkgs in self._packages.items() if k in k_add] + ) + ] # If the second value in the tuple is True, the package must # be uninstalled on export - remove = [p[0] for p in itertools.chain.from_iterable( - [l for k, l in self._packages.items()]) - if p[1]] + remove = [ + p[0] + for p in itertools.chain.from_iterable( + [pkgs for k, pkgs in self._packages.items()] + ) + if p[1] + ] return add, remove @@ -220,27 +259,24 @@ def run_user_script(self, user_script): script_path = "/tmp/coriolis_user_script" try: - utils.write_ssh_file( - self._conn, - script_path, - user_script) + utils.write_ssh_file(self._conn, script_path, user_script) except Exception as err: raise exception.CoriolisException( - "Failed to copy user script to target system.") from err + "Failed to copy user script to target system." + ) from err try: utils.exec_ssh_cmd( - self._conn, - "sudo chmod +x %s" % script_path, - get_pty=True) + self._conn, "sudo chmod +x %s" % script_path, get_pty=True + ) utils.exec_ssh_cmd( self._conn, 'sudo "%s" "%s"' % (script_path, self._os_root_dir), - get_pty=True) + get_pty=True, + ) except Exception as err: - raise exception.CoriolisException( - "Failed to run user script.") from err + raise exception.CoriolisException("Failed to run user script.") from err def pre_packages_install(self, package_names): self._copy_resolv_conf() @@ -282,22 +318,33 @@ def _exec_cmd(self, cmd, timeout=None): timeout = self._osmorphing_operation_timeout try: return utils.exec_ssh_cmd( - self._ssh, cmd, environment=self._environment, get_pty=True, - timeout=timeout) + self._ssh, + cmd, + environment=self._environment, + get_pty=True, + timeout=timeout, + ) except exception.MinionMachineCommandTimeout as ex: raise exception.OSMorphingSSHOperationTimeout( - cmd=cmd, timeout=timeout) from ex + cmd=cmd, timeout=timeout + ) from ex def _exec_cmd_chroot(self, cmd, timeout=None): if not timeout: timeout = self._osmorphing_operation_timeout try: return utils.exec_ssh_cmd_chroot( - self._ssh, self._os_root_dir, cmd, - environment=self._environment, get_pty=True, timeout=timeout) + self._ssh, + self._os_root_dir, + cmd, + environment=self._environment, + get_pty=True, + timeout=timeout, + ) except exception.MinionMachineCommandTimeout as ex: raise exception.OSMorphingSSHOperationTimeout( - cmd=cmd, timeout=timeout) from ex + cmd=cmd, timeout=timeout + ) from ex def _check_user_exists(self, username): try: @@ -312,8 +359,7 @@ def _write_file_sudo(self, chroot_path, content): self._write_file(tmp_file, content) self._exec_cmd_chroot("cp /%s /%s" % (tmp_file, chroot_path)) self._exec_cmd_chroot("rm /%s" % tmp_file) - utils.exec_ssh_cmd( - self._ssh, "sudo sync", self._environment, get_pty=True) + utils.exec_ssh_cmd(self._ssh, "sudo sync", self._environment, get_pty=True) def _enable_systemd_service(self, service_name): self._exec_cmd_chroot("systemctl enable %s.service" % service_name) @@ -322,8 +368,7 @@ def _disable_systemd_service(self, service_name): self._exec_cmd_chroot("systemctl disable %s.service" % service_name) def _disable_upstart_service(self, service_name): - self._exec_cmd_chroot( - "echo manual | tee /etc/init/%s.override" % service_name) + self._exec_cmd_chroot("echo manual | tee /etc/init/%s.override" % service_name) def _get_os_release(self): return self._read_config_file("etc/os-release", check_exists=True) @@ -332,7 +377,8 @@ def _read_config_file(self, chroot_path, check_exists=False): chroot_path = chroot_path.lstrip('/') full_path = os.path.join(self._os_root_dir, chroot_path) return utils.read_ssh_ini_config_file( - self._ssh, full_path, check_exists=check_exists) + self._ssh, full_path, check_exists=check_exists + ) def _read_config_file_sudo(self, chroot_path, check_exists=False): if self._test_path_chroot(chroot_path) is False: @@ -350,9 +396,11 @@ def _copy_resolv_conf(self): if self._test_path(resolv_conf): self._exec_cmd( - "sudo mv -f %s %s" % (resolv_conf_path, resolv_conf_path_old)) - self._exec_cmd('sudo cp -L --remove-destination /etc/resolv.conf %s' % - resolv_conf_path) + "sudo mv -f %s %s" % (resolv_conf_path, resolv_conf_path_old) + ) + self._exec_cmd( + 'sudo cp -L --remove-destination /etc/resolv.conf %s' % resolv_conf_path + ) def _restore_resolv_conf(self): resolv_conf = "etc/resolv.conf" @@ -362,10 +410,12 @@ def _restore_resolv_conf(self): if self._test_path(resolv_conf_old): self._exec_cmd( - "sudo mv -f %s %s" % (resolv_conf_path_old, resolv_conf_path)) + "sudo mv -f %s %s" % (resolv_conf_path_old, resolv_conf_path) + ) def _replace_fstab_entries_device_prefix( - self, current_prefix="/dev/sd", new_prefix="/dev/sd"): + self, current_prefix="/dev/sd", new_prefix="/dev/sd" + ): fstab_chroot_path = "etc/fstab" fstab_contents = self._read_file(fstab_chroot_path).decode() LOG.debug("Contents of /%s: %s", fstab_chroot_path, fstab_contents) @@ -377,24 +427,24 @@ def _replace_fstab_entries_device_prefix( if re.match(regex, line): found = True LOG.debug( - "Found FSTAB line starting with '%s': %s" % ( - current_prefix, line)) + "Found FSTAB line starting with '%s': %s" % (current_prefix, line) + ) fstab_contents_lines[i] = re.sub(regex, new_prefix, line) if found: self._event_manager.progress_update( "Replacing all /etc/fstab entries prefixed with " - "'%s' to '%s'" % (current_prefix, new_prefix)) + "'%s' to '%s'" % (current_prefix, new_prefix) + ) self._exec_cmd_chroot( - "mv -f /%s /%s.bak" % (fstab_chroot_path, fstab_chroot_path)) - self._write_file( - fstab_chroot_path, "\n".join(fstab_contents_lines)) + "mv -f /%s /%s.bak" % (fstab_chroot_path, fstab_chroot_path) + ) + self._write_file(fstab_chroot_path, "\n".join(fstab_contents_lines)) def _set_selinux_autorelabel(self): LOG.debug("setting autorelabel on /") try: - self._exec_cmd_chroot( - "touch /.autorelabel") + self._exec_cmd_chroot("touch /.autorelabel") except Exception as err: LOG.warning("Failed to set autorelabel: %r" % err) @@ -403,8 +453,7 @@ def _write_cloud_init_mods_config(self, cloud_cfg): cloud_config_path = f"{cloud_cfgs_dir}/99_coriolis.cfg" if not self._test_path(cloud_cfgs_dir): self._exec_cmd_chroot("mkdir -p %s" % cloud_cfgs_dir) - self._event_manager.progress_update( - "Customizing cloud-init configuration") + self._event_manager.progress_update("Customizing cloud-init configuration") new_cloud_cfg = yaml.dump(cloud_cfg, Dumper=yaml.SafeDumper) self._write_file_sudo(cloud_config_path, new_cloud_cfg) @@ -412,9 +461,11 @@ def _disable_installer_cloud_config(self): installer_config_path = "/etc/cloud/cloud.cfg.d/99-installer.cfg" if self._test_path(installer_config_path): self._event_manager.progress_update( - "Disabling installer-generated cloud-config") + "Disabling installer-generated cloud-config" + ) self._exec_cmd_chroot( - f"mv {installer_config_path} {installer_config_path}.bak") + f"mv {installer_config_path} {installer_config_path}.bak" + ) def _ensure_cloud_init_not_disabled(self): disabler_file = "/etc/cloud/cloud-init.disabled" @@ -424,10 +475,12 @@ def _ensure_cloud_init_not_disabled(self): self._exec_cmd_chroot(f"rm {disabler_file}") if self._test_path(system_conf_disabler): self._exec_cmd_chroot( - "sed -i '/cloud-init=disabled/d' %s" % system_conf_disabler) + "sed -i '/cloud-init=disabled/d' %s" % system_conf_disabler + ) if self._test_path(grub_conf_disabler): self._exec_cmd_chroot( - "sed -i '/cloud-init=disabled/d' %s" % grub_conf_disabler) + "sed -i '/cloud-init=disabled/d' %s" % grub_conf_disabler + ) self._execute_update_grub() def _reset_cloud_init_run(self): @@ -449,8 +502,11 @@ def _get_default_cloud_user(self): return DEFAULT_CLOUD_USER cloud_cfg_content = self._read_file_sudo(cloud_cfg_path) cloud_cfg = yaml.load(cloud_cfg_content, Loader=yaml.SafeLoader) - return cloud_cfg.get('system_info', {}).get('default_user', {}).get( - 'name', DEFAULT_CLOUD_USER) + return ( + cloud_cfg.get('system_info', {}) + .get('default_user', {}) + .get('name', DEFAULT_CLOUD_USER) + ) def _create_cloudinit_user(self): cloud_user = self._get_default_cloud_user() @@ -473,7 +529,7 @@ def _configure_cloud_init(self): cloud_cfg_user_retention = { 'disable_root': False, 'ssh_pwauth': True, - 'users': None + 'users': None, } cloud_cfg_mods.update(cloud_cfg_user_retention) else: @@ -495,11 +551,11 @@ def _configure_cloud_init(self): except exception.CoriolisException: LOG.warning( "Failed to enable service unit with name " - f"{CLOUD_INIT_SERVICE_UNIT_NAME}. Trying new name.") + f"{CLOUD_INIT_SERVICE_UNIT_NAME}. Trying new name." + ) # NOTE(dvincze): New versions of cloud-init (>26) got its # unit service name renamed. - self._enable_systemd_service( - CLOUD_INIT_SERVICE_UNIT_NAME_FALLBACK) + self._enable_systemd_service(CLOUD_INIT_SERVICE_UNIT_NAME_FALLBACK) def _test_path_chroot(self, path): # This method uses _exec_cmd_chroot() instead of SFTP stat() @@ -510,15 +566,15 @@ def _test_path_chroot(self, path): # ensures you always run as root. if path.startswith('/') is False: path = "/%s" % path - exists = self._exec_cmd_chroot( - '[ -f "%s" ] && echo 1 || echo 0' % path).rstrip('\n') + exists = self._exec_cmd_chroot('[ -f "%s" ] && echo 1 || echo 0' % path).rstrip( + '\n' + ) return exists == "1" def _read_file_sudo(self, chroot_path): if chroot_path.startswith("/") is False: chroot_path = "/%s" % chroot_path - contents = self._exec_cmd_chroot( - 'cat "%s"' % chroot_path) + contents = self._exec_cmd_chroot('cat "%s"' % chroot_path) return contents def _read_grub_config(self, config): @@ -540,8 +596,7 @@ def _get_grub_config_obj(self, grub_conf=None): if self._test_path_chroot(grub_conf) is False: raise IOError("could not find %s" % grub_conf) tmp_file = self._exec_cmd_chroot("mktemp").rstrip('\n') - self._exec_cmd_chroot( - "/bin/cp -fp %s %s" % (grub_conf, tmp_file)) + self._exec_cmd_chroot("/bin/cp -fp %s %s" % (grub_conf, tmp_file)) config_file = self._read_grub_config(tmp_file) config_obj = { "source": grub_conf, @@ -561,8 +616,7 @@ def _validate_grub_config_obj(self, config_obj): missing.append(key) if len(missing) > 0: - raise ValueError( - "Invalid configObj. Missing: %s" % ", ".join(missing)) + raise ValueError("Invalid configObj. Missing: %s" % ", ".join(missing)) def set_grub_value(self, option, value, config_obj, replace=True): self._validate_grub_config_obj(config_obj) @@ -571,7 +625,7 @@ def append_to_cfg(opt, val): cmd = "sed -ie '$a%(o)s=\"%(v)s\"' %(cfg)s" % { "o": opt, "v": val, - "cfg": config_obj["location"] + "cfg": config_obj["location"], } self._exec_cmd_chroot(cmd) @@ -579,7 +633,7 @@ def replace_in_cfg(opt, val): cmd = "sed -i 's|^%(o)s=.*|%(o)s=\"%(v)s\"|g' %(cfg)s" % { "o": opt, "v": val, - "cfg": config_obj["location"] + "cfg": config_obj["location"], } self._exec_cmd_chroot(cmd) @@ -592,16 +646,13 @@ def replace_in_cfg(opt, val): LOG.warning("TEMP CONFIG IS: %r" % cfg) def _set_grub2_cmdline(self, config_obj, options, clobber=False): - kernel_cmd_def = config_obj["contents"].get( - "GRUB_CMDLINE_LINUX_DEFAULT") - kernel_cmd = config_obj["contents"].get( - "GRUB_CMDLINE_LINUX") + kernel_cmd_def = config_obj["contents"].get("GRUB_CMDLINE_LINUX_DEFAULT") + kernel_cmd = config_obj["contents"].get("GRUB_CMDLINE_LINUX") replace = kernel_cmd is not None if clobber: opt = " ".join(options) - self.set_grub_value( - "GRUB_CMDLINE_LINUX", opt, config_obj, replace=replace) + self.set_grub_value("GRUB_CMDLINE_LINUX", opt, config_obj, replace=replace) return kernel_cmd_def = kernel_cmd_def or "" kernel_cmd = kernel_cmd or "" @@ -612,24 +663,29 @@ def _set_grub2_cmdline(self, config_obj, options, clobber=False): if len(to_add): kernel_cmd = "%s %s" % (kernel_cmd, " ".join(to_add)) self.set_grub_value( - "GRUB_CMDLINE_LINUX", kernel_cmd, config_obj, replace=replace) + "GRUB_CMDLINE_LINUX", kernel_cmd, config_obj, replace=replace + ) def _execute_update_grub(self): update_cmd = self.get_update_grub2_command() self._exec_cmd_chroot(update_cmd) - def _apply_grub2_config(self, config_obj, - execute_update_grub=True): + def _apply_grub2_config(self, config_obj, execute_update_grub=True): self._validate_grub_config_obj(config_obj) self._exec_cmd_chroot( - "mv -f %s %s" % ( - config_obj["location"], config_obj["source"])) + "mv -f %s %s" % (config_obj["location"], config_obj["source"]) + ) if execute_update_grub: self._execute_update_grub() - def _set_grub2_console_settings(self, consoles=None, speed=None, - parity=None, grub_conf=None, - execute_update_grub=True): + def _set_grub2_console_settings( + self, + consoles=None, + speed=None, + parity=None, + grub_conf=None, + execute_update_grub=True, + ): # This method updates the GRUB2 config file and adds serial console # support. # @@ -642,7 +698,8 @@ def _set_grub2_console_settings(self, consoles=None, speed=None, valid_parity = ["no", "odd", "even"] if parity and parity not in valid_parity: raise ValueError( - "Valid values for parity are: %s" % ", ".join(valid_parity)) + "Valid values for parity are: %s" % ", ".join(valid_parity) + ) speed = speed or 115200 parity = parity or "no" @@ -662,8 +719,7 @@ def _set_grub2_console_settings(self, consoles=None, speed=None, options.append(c) self._set_grub2_cmdline(config_obj, options) - self._apply_grub2_config( - config_obj, execute_update_grub) + self._apply_grub2_config(config_obj, execute_update_grub) def _add_net_udev_rules(self, net_ifaces_info): coriolis_udev_rules_file = "etc/udev/rules.d/99-coriolis-net.rules" @@ -678,11 +734,11 @@ def _setup_network_preservation(self, nics_info) -> None: LOG.info("Using network preserver class: %s", net_preserver_class) if not net_preserver_class: raise exception.CoriolisException( - "Could not find any valid static network configuration") + "Could not find any valid static network configuration" + ) netpreserver = net_preserver_class(self) netpreserver.parse_network() - LOG.info("Parsed network configuration: %s", - netpreserver.interface_info) + LOG.info("Parsed network configuration: %s", netpreserver.interface_info) for nic in nics_info: nic_mac = nic.get('mac_address') @@ -690,7 +746,8 @@ def _setup_network_preservation(self, nics_info) -> None: if not nic_mac: LOG.warning( "Parsed NIC '%s' info does not contain MAC address" - % nic.get('name')) + % nic.get('name') + ) continue matching_ifaces = dict() @@ -700,20 +757,27 @@ def _setup_network_preservation(self, nics_info) -> None: if mac_address and mac_address == nic_mac: LOG.info( "Found matching interface for NIC '%s' with MAC '%s'", - nic.get('name'), nic_mac) + nic.get('name'), + nic_mac, + ) matching_ifaces[iface] = nic_mac break if ip_addresses and nic_ips: if set(ip_addresses) & set(nic_ips): LOG.info( - "Found matching interface for NIC '%s' with MAC " - "'%s'", nic.get('name'), nic_mac) + "Found matching interface for NIC '%s' with MAC '%s'", + nic.get('name'), + nic_mac, + ) matching_ifaces[iface] = nic_mac break if not matching_ifaces: LOG.warning( "Could not find a matching guest interface for NIC '%s' " - "with MAC address '%s'", nic, nic_mac) + "with MAC address '%s'", + nic, + nic_mac, + ) net_ifaces_info.update(matching_ifaces) self._add_net_udev_rules(net_ifaces_info) diff --git a/coriolis/osmorphing/centos.py b/coriolis/osmorphing/centos.py index bd4d8ee4..cc9db494 100644 --- a/coriolis/osmorphing/centos.py +++ b/coriolis/osmorphing/centos.py @@ -2,23 +2,21 @@ # All Rights Reserved. -from coriolis.osmorphing.osdetect import centos as centos_detect from coriolis.osmorphing import redhat - +from coriolis.osmorphing.osdetect import centos as centos_detect CENTOS_DISTRO_IDENTIFIER = centos_detect.CENTOS_DISTRO_IDENTIFIER CENTOS_STREAM_DISTRO_IDENTIFIER = centos_detect.CENTOS_STREAM_DISTRO_IDENTIFIER class BaseCentOSMorphingTools(redhat.BaseRedHatMorphingTools): - UEFI_GRUB_LOCATION = "/boot/efi/EFI/centos" @classmethod def check_os_supported(cls, detected_os_info): - supported_oses = [ - CENTOS_STREAM_DISTRO_IDENTIFIER, CENTOS_DISTRO_IDENTIFIER] + supported_oses = [CENTOS_STREAM_DISTRO_IDENTIFIER, CENTOS_DISTRO_IDENTIFIER] if detected_os_info['distribution_name'] not in supported_oses: return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=6) + detected_os_info['release_version'], minimum=6 + ) diff --git a/coriolis/osmorphing/coreos.py b/coriolis/osmorphing/coreos.py index 4c9d88db..e63e2af3 100644 --- a/coriolis/osmorphing/coreos.py +++ b/coriolis/osmorphing/coreos.py @@ -6,11 +6,11 @@ class BaseCoreOSMorphingTools(base.BaseLinuxOSMorphingTools): - @classmethod def check_os_supported(cls, detected_os_info): if detected_os_info['distribution_name'] == ( - coreos_detect.COREOS_DISTRO_IDENTIFIER): + coreos_detect.COREOS_DISTRO_IDENTIFIER + ): return True return False diff --git a/coriolis/osmorphing/debian.py b/coriolis/osmorphing/debian.py index f713a819..63b49534 100644 --- a/coriolis/osmorphing/debian.py +++ b/coriolis/osmorphing/debian.py @@ -3,13 +3,12 @@ from io import StringIO -from oslo_log import log as logging import yaml +from oslo_log import log as logging -from coriolis import exception +from coriolis import exception, utils from coriolis.osmorphing import base from coriolis.osmorphing.osdetect import debian as debian_osdetect -from coriolis import utils DEBIAN_DISTRO_IDENTIFIER = debian_osdetect.DEBIAN_DISTRO_IDENTIFIER @@ -27,16 +26,15 @@ class BaseDebianMorphingTools(base.BaseLinuxOSMorphingTools): - netplan_base = "etc/netplan" @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - DEBIAN_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] != (DEBIAN_DISTRO_IDENTIFIER): return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=8) + detected_os_info['release_version'], minimum=8 + ) def disable_predictable_nic_names(self): grub_cfg = "etc/default/grub" @@ -46,16 +44,20 @@ def disable_predictable_nic_names(self): cfg = utils.Grub2ConfigEditor(contents) cfg.append_to_option( "GRUB_CMDLINE_LINUX_DEFAULT", - {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}) + {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}, + ) cfg.append_to_option( "GRUB_CMDLINE_LINUX_DEFAULT", - {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}) + {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}, + ) cfg.append_to_option( "GRUB_CMDLINE_LINUX", - {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}) + {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}, + ) cfg.append_to_option( "GRUB_CMDLINE_LINUX", - {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}) + {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}, + ) self._write_file_sudo("etc/default/grub", cfg.dump()) self._exec_cmd_chroot("/usr/sbin/update-grub") @@ -81,13 +83,8 @@ def _compose_netplan_cfg(self, nics_info): "network": { "version": 2, "ethernets": { - "lo": { - "match": { - "name": "lo" - }, - "addresses": ["127.0.0.1/8"] - } - } + "lo": {"match": {"name": "lo"}, "addresses": ["127.0.0.1/8"]} + }, } } for idx, _ in enumerate(nics_info): @@ -108,8 +105,7 @@ def set_net_config(self, nics_info, dhcp): ifaces_file = "etc/network/interfaces" contents = self._compose_interfaces_config(nics_info) if self._test_path(ifaces_file): - self._exec_cmd_chroot( - "cp %s %s.bak" % (ifaces_file, ifaces_file)) + self._exec_cmd_chroot("cp %s %s.bak" % (ifaces_file, ifaces_file)) self._write_file_sudo(ifaces_file, contents) if self._test_path(self.netplan_base): @@ -117,9 +113,7 @@ def set_net_config(self, nics_info, dhcp): for cnf in curr_files: if cnf.endswith(".yaml") or cnf.endswith(".yml"): pth = "%s/%s" % (self.netplan_base, cnf) - self._exec_cmd_chroot( - "mv %s %s.bak" % (pth, pth) - ) + self._exec_cmd_chroot("mv %s %s.bak" % (pth, pth)) new_cfg = self._compose_netplan_cfg(nics_info) cfg_name = "%s/coriolis_netplan.yaml" % self.netplan_base self._write_file_sudo(cfg_name, new_cfg) @@ -132,8 +126,7 @@ def get_installed_packages(self): self.installed_packages = [] def pre_packages_install(self, package_names): - super(BaseDebianMorphingTools, self).pre_packages_install( - package_names) + super(BaseDebianMorphingTools, self).pre_packages_install(package_names) try: if package_names: self._event_manager.progress_update("Updating packages list") @@ -152,12 +145,12 @@ def pre_packages_install(self, package_names): "source machine, or try to set up a mirror of said " "repositories which will be reachable from the temporary " "OSMorphing minion machine on the target platform. Original " - "error was: %s" % str(err)) from err + "error was: %s" % str(err) + ) from err def post_packages_install(self, package_names): self._configure_cloud_init() - super(BaseDebianMorphingTools, self).post_packages_install( - package_names) + super(BaseDebianMorphingTools, self).post_packages_install(package_names) def install_packages(self, package_names): try: @@ -165,20 +158,19 @@ def install_packages(self, package_names): # while the source machine was midway through installing or # configuring a package, forcing it to retain old conf files # which were previously modified on the source machine. - deb_reconfigure_cmd = ( - "dpkg --configure --force-confold -a") + deb_reconfigure_cmd = "dpkg --configure --force-confold -a" self._exec_cmd_chroot(deb_reconfigure_cmd) apt_get_cmd = ( '/bin/bash -c "DEBIAN_FRONTEND=noninteractive ' 'apt-get install %s -y ' - '-o Dpkg::Options::=\'--force-confdef\'"' % ( - " ".join(package_names))) + '-o Dpkg::Options::=\'--force-confdef\'"' % (" ".join(package_names)) + ) self._exec_cmd_chroot(apt_get_cmd) except Exception as err: raise exception.FailedPackageInstallationException( - package_names=package_names, package_manager='apt', - error=str(err)) from err + package_names=package_names, package_manager='apt', error=str(err) + ) from err def uninstall_packages(self, package_names): try: @@ -187,5 +179,5 @@ def uninstall_packages(self, package_names): self._exec_cmd_chroot(apt_get_cmd) except exception.CoriolisException as err: raise exception.FailedPackageUninstallationException( - package_names=package_names, package_manager='apt', - error=str(err)) from err + package_names=package_names, package_manager='apt', error=str(err) + ) from err diff --git a/coriolis/osmorphing/manager.py b/coriolis/osmorphing/manager.py index 50493b8a..e6b04929 100644 --- a/coriolis/osmorphing/manager.py +++ b/coriolis/osmorphing/manager.py @@ -6,33 +6,24 @@ from oslo_config import cfg from oslo_log import log as logging -from coriolis import events -from coriolis import exception +from coriolis import events, exception, schemas from coriolis.osmorphing import base as base_osmorphing from coriolis.osmorphing.osdetect import manager as osdetect_manager from coriolis.osmorphing.osmount import factory as osmount_factory -from coriolis import schemas - opts = [ - cfg.IntOpt('default_osmorphing_operation_timeout', - help='Number of seconds to wait for a pending SSH or WinRM ' - 'command before the socket times out.') + cfg.IntOpt( + 'default_osmorphing_operation_timeout', + help='Number of seconds to wait for a pending SSH or WinRM ' + 'command before the socket times out.', + ) ] proxy_opts = [ - cfg.StrOpt('url', - default=None, - help='Proxy URL.'), - cfg.StrOpt('username', - default=None, - help='Proxy username.'), - cfg.StrOpt('password', - default=None, - help='Proxy password.'), - cfg.ListOpt('no_proxy', - default=[], - help='List of proxy exclusions.'), + cfg.StrOpt('url', default=None, help='Proxy URL.'), + cfg.StrOpt('username', default=None, help='Proxy username.'), + cfg.StrOpt('password', default=None, help='Proxy password.'), + cfg.ListOpt('no_proxy', default=[], help='List of proxy exclusions.'), ] CONF = cfg.CONF @@ -52,53 +43,71 @@ def _get_proxy_settings(): def run_os_detect( - origin_provider, destination_provider, worker_connection, - os_type, os_root_dir, osmorphing_info, tools_environment={}): - custom_export_os_detect_tools = ( - origin_provider.get_custom_os_detect_tools( - os_type, osmorphing_info)) - custom_import_os_detect_tools = ( - destination_provider.get_custom_os_detect_tools( - os_type, osmorphing_info)) + origin_provider, + destination_provider, + worker_connection, + os_type, + os_root_dir, + osmorphing_info, + tools_environment={}, +): + custom_export_os_detect_tools = origin_provider.get_custom_os_detect_tools( + os_type, osmorphing_info + ) + custom_import_os_detect_tools = destination_provider.get_custom_os_detect_tools( + os_type, osmorphing_info + ) detected_info = osdetect_manager.detect_os( - worker_connection, os_type, os_root_dir, + worker_connection, + os_type, + os_root_dir, CONF.default_osmorphing_operation_timeout, tools_environment=tools_environment, custom_os_detect_tools=list( itertools.chain( - custom_export_os_detect_tools, - custom_import_os_detect_tools))) + custom_export_os_detect_tools, custom_import_os_detect_tools + ) + ), + ) schemas.validate_value( - detected_info, schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA) + detected_info, schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA + ) return detected_info def get_osmorphing_tools_class_for_provider( - provider, detected_os_info, os_type, osmorphing_info): - available_tools_cls = provider.get_os_morphing_tools( - os_type, osmorphing_info) + provider, detected_os_info, os_type, osmorphing_info +): + available_tools_cls = provider.get_os_morphing_tools(os_type, osmorphing_info) LOG.debug( "OSMorphing tools classes returned by provider '%s' for os_type '%s' " "and 'osmorphing_info' %s: %s", - type(provider), os_type, osmorphing_info, available_tools_cls) + type(provider), + os_type, + osmorphing_info, + available_tools_cls, + ) osmorphing_base_class = base_osmorphing.BaseOSMorphingTools for toolscls in available_tools_cls: if not issubclass(toolscls, osmorphing_base_class): raise exception.InvalidOSMorphingTools( "Provider class '%s' returned OSMorphing tools which are not " - "a subclass of '%s': %s" % ( - type(provider), osmorphing_base_class, toolscls)) + "a subclass of '%s': %s" + % (type(provider), osmorphing_base_class, toolscls) + ) detected_toolscls = None LOG.info( "Checking OSMorphing tools classes %s returned by provider '%s' " "for compatibility on detected OS info '%s'", [cls.__name__ for cls in available_tools_cls], - type(provider), detected_os_info) + type(provider), + detected_os_info, + ) for toolscls in available_tools_cls: try: toolscls.check_detected_os_info_parameters(detected_os_info) @@ -106,27 +115,39 @@ def get_osmorphing_tools_class_for_provider( LOG.warn( "OSMorphing tools class %s will be skipped as it is " "incompatbile with the detected OS info params %s. " - "Error was: %s" % ( - toolscls.__name__, detected_os_info, str(ex))) + "Error was: %s" % (toolscls.__name__, detected_os_info, str(ex)) + ) continue if toolscls.check_os_supported(detected_os_info): LOG.info( "Found compatible OSMorphing tools class '%s' from provider " "'%s' for detected OS info: %s", - toolscls.__name__, type(provider), detected_os_info) + toolscls.__name__, + type(provider), + detected_os_info, + ) detected_toolscls = toolscls break else: LOG.debug( "OSMorphing tools class '%s' is not compatible with detected " - "OS info: %s", toolscls.__name__, detected_os_info) + "OS info: %s", + toolscls.__name__, + detected_os_info, + ) return detected_toolscls -def morph_image(origin_provider, destination_provider, connection_info, - osmorphing_info, user_script, event_handler): +def morph_image( + origin_provider, + destination_provider, + connection_info, + osmorphing_info, + user_script, + event_handler, +): event_manager = events.EventManager(event_handler) os_type = osmorphing_info.get('os_type') @@ -134,8 +155,12 @@ def morph_image(origin_provider, destination_provider, connection_info, # instantiate and run OSMount tools: os_mount_tools = osmount_factory.get_os_mount_tools( - os_type, connection_info, event_manager, ignore_devices, - CONF.default_osmorphing_operation_timeout) + os_type, + connection_info, + event_manager, + ignore_devices, + CONF.default_osmorphing_operation_timeout, + ) proxy_settings = _get_proxy_settings() os_mount_tools.set_proxy(proxy_settings) @@ -148,7 +173,8 @@ def morph_image(origin_provider, destination_provider, connection_info, "Failed to set up the minion machine for OSMorphing. This may be " "due to an incompatibility between the OS image used for the " "OSMorphing minion machine and the VM undergoing OSMorphing. " - "Error was: %s" % str(err)) from err + "Error was: %s" % str(err) + ) from err event_manager.progress_update("Discovering and mounting OS partitions") os_root_dir, os_root_dev = os_mount_tools.mount_os() @@ -160,66 +186,86 @@ def morph_image(origin_provider, destination_provider, connection_info, environment = os_mount_tools.get_environment() detected_os_info = run_os_detect( - origin_provider, destination_provider, conn, - os_type, os_root_dir, osmorphing_info, - tools_environment=environment) + origin_provider, + destination_provider, + conn, + os_type, + os_root_dir, + osmorphing_info, + tools_environment=environment, + ) # TODO(aznashwan): # - export the source hypervisor type option in the VM's export info # - automatically detect the target hypervisor type from the worker VM - hypervisor_type = osmorphing_info.get( - 'hypervisor_type', None) + hypervisor_type = osmorphing_info.get('hypervisor_type', None) osmorphing_parameters = osmorphing_info.get('osmorphing_parameters', {}) export_os_morphing_tools = None try: export_tools_cls = get_osmorphing_tools_class_for_provider( - origin_provider, detected_os_info, os_type, osmorphing_info) + origin_provider, detected_os_info, os_type, osmorphing_info + ) if export_tools_cls: LOG.info( - "Instantiating OSMorphing tools class '%s' for export provider" - " '%s'", export_tools_cls.__name__, - type(origin_provider)) + "Instantiating OSMorphing tools class '%s' for export provider '%s'", + export_tools_cls.__name__, + type(origin_provider), + ) export_os_morphing_tools = export_tools_cls( - conn, os_root_dir, os_root_dev, hypervisor_type, - event_manager, detected_os_info, osmorphing_parameters, - CONF.default_osmorphing_operation_timeout) + conn, + os_root_dir, + os_root_dev, + hypervisor_type, + event_manager, + detected_os_info, + osmorphing_parameters, + CONF.default_osmorphing_operation_timeout, + ) export_os_morphing_tools.set_environment(environment) else: LOG.debug( - "No compatible OSMorphing tools class found for export " - "provider '%s'", type(origin_provider).__name__) + "No compatible OSMorphing tools class found for export provider '%s'", + type(origin_provider).__name__, + ) except exception.OSMorphingToolsNotFound: LOG.warn( - "No tools found for export provider of type: %s", - type(origin_provider)) + "No tools found for export provider of type: %s", type(origin_provider) + ) import_os_morphing_tools_cls = get_osmorphing_tools_class_for_provider( - destination_provider, detected_os_info, os_type, osmorphing_info) + destination_provider, detected_os_info, os_type, osmorphing_info + ) if not import_os_morphing_tools_cls: LOG.error( "No compatible OSMorphing tools found from import provider '%s' " "for the given detected OS info %s", type(destination_provider), - detected_os_info) + detected_os_info, + ) raise exception.OSMorphingToolsNotFound(os_type=os_type) import_os_morphing_tools = import_os_morphing_tools_cls( - conn, os_root_dir, os_root_dev, hypervisor_type, - event_manager, detected_os_info, osmorphing_parameters, - CONF.default_osmorphing_operation_timeout) + conn, + os_root_dir, + os_root_dev, + hypervisor_type, + event_manager, + detected_os_info, + osmorphing_parameters, + CONF.default_osmorphing_operation_timeout, + ) import_os_morphing_tools.set_environment(environment) if user_script: - event_manager.progress_update( - 'Running OS morphing user script') + event_manager.progress_update('Running OS morphing user script') import_os_morphing_tools.run_user_script(user_script) else: - event_manager.progress_update( - 'No OS morphing user script specified') + event_manager.progress_update('No OS morphing user script specified') event_manager.progress_update( - 'OS being migrated: %s' % detected_os_info['friendly_release_name']) + 'OS being migrated: %s' % detected_os_info['friendly_release_name'] + ) (packages_add, _) = import_os_morphing_tools.get_packages() @@ -233,7 +279,8 @@ def morph_image(origin_provider, destination_provider, connection_info, if packages_remove: event_manager.progress_update( - "Removing packages: %s" % str(packages_remove)) + "Removing packages: %s" % str(packages_remove) + ) export_os_morphing_tools.uninstall_packages(packages_remove) LOG.info("Post packages uninstall") @@ -242,7 +289,8 @@ def morph_image(origin_provider, destination_provider, connection_info, LOG.info("Checking for packages already installed") import_os_morphing_tools.get_installed_packages() packages_add = list( - set(packages_add) - set(import_os_morphing_tools.installed_packages)) + set(packages_add) - set(import_os_morphing_tools.installed_packages) + ) LOG.info("Pre packages install") import_os_morphing_tools.pre_packages_install(packages_add) @@ -253,11 +301,9 @@ def morph_image(origin_provider, destination_provider, connection_info, LOG.info("Pre packages") if packages_add: - event_manager.progress_update( - "Adding packages: %s" % str(packages_add)) + event_manager.progress_update("Adding packages: %s" % str(packages_add)) - import_os_morphing_tools.install_packages( - packages_add) + import_os_morphing_tools.install_packages(packages_add) LOG.info("Post packages install") import_os_morphing_tools.post_packages_install(packages_add) @@ -273,4 +319,5 @@ def morph_image(origin_provider, destination_provider, connection_info, "filesystem quiescing, crash-consistent backups, etc.) are " "enabled and available for the source machine. If none are " "available, please try migrating/replicating the source machine " - "while it is powered off. Error was: %s" % str(err)) from err + "while it is powered off. Error was: %s" % str(err) + ) from err diff --git a/coriolis/osmorphing/netpreserver/base.py b/coriolis/osmorphing/netpreserver/base.py index 06391d19..84867540 100644 --- a/coriolis/osmorphing/netpreserver/base.py +++ b/coriolis/osmorphing/netpreserver/base.py @@ -3,7 +3,6 @@ class BaseNetPreserver(object): - def __init__(self, osmorphing_tool): self.osmorphing_tool = osmorphing_tool self.interface_info = dict() diff --git a/coriolis/osmorphing/netpreserver/factory.py b/coriolis/osmorphing/netpreserver/factory.py index e0fc4e4d..02f2700b 100644 --- a/coriolis/osmorphing/netpreserver/factory.py +++ b/coriolis/osmorphing/netpreserver/factory.py @@ -1,15 +1,14 @@ # Copyright 2025 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis.osmorphing.netpreserver import ifcfg -from coriolis.osmorphing.netpreserver import interfaces -from coriolis.osmorphing.netpreserver import netplan -from coriolis.osmorphing.netpreserver import nmconnection +from coriolis.osmorphing.netpreserver import ifcfg, interfaces, netplan, nmconnection -NET_PRESERVERS = [netplan.NetplanNetPreserver, - nmconnection.NmconnectionNetPreserver, - ifcfg.IfcfgNetPreserver, - interfaces.InterfacesNetPreserver] +NET_PRESERVERS = [ + netplan.NetplanNetPreserver, + nmconnection.NmconnectionNetPreserver, + ifcfg.IfcfgNetPreserver, + interfaces.InterfacesNetPreserver, +] def get_net_preserver(osmorphing_tool): diff --git a/coriolis/osmorphing/netpreserver/ifcfg.py b/coriolis/osmorphing/netpreserver/ifcfg.py index ad2f01b8..a85819c0 100644 --- a/coriolis/osmorphing/netpreserver/ifcfg.py +++ b/coriolis/osmorphing/netpreserver/ifcfg.py @@ -12,7 +12,6 @@ class IfcfgNetPreserver(base.BaseNetPreserver): - def __init__(self, osmorphing_tool): super(IfcfgNetPreserver, self).__init__(osmorphing_tool) self.network_scripts_path = "etc/sysconfig/network-scripts" @@ -22,14 +21,16 @@ def check_net_preserver(self): ifcfg_files = self._get_net_config_files(self.network_scripts_path) if ifcfg_files: ifcfgs_ethernet = self._get_ifcfgs_by_type( - "Ethernet", self.network_scripts_path) + "Ethernet", self.network_scripts_path + ) if ifcfgs_ethernet: return True return False def parse_network(self): ifcfgs_ethernet = self._get_ifcfgs_by_type( - "Ethernet", self.network_scripts_path) + "Ethernet", self.network_scripts_path + ) if ifcfgs_ethernet: for ifcfg_file, ifcfg in ifcfgs_ethernet: name = ifcfg.get("DEVICE") @@ -40,13 +41,16 @@ def parse_network(self): ip_address = ifcfg.get("IPADDR") self.interface_info[name] = { "mac_address": mac_address, - "ip_addresses": [ip_address] if ip_address else [] + "ip_addresses": [ip_address] if ip_address else [], } def _get_net_config_files(self, network_scripts_path): dir_content = self.osmorphing_tool._list_dir(network_scripts_path) - return [os.path.join(network_scripts_path, f) for f in - dir_content if re.match("^ifcfg-(.*)", f)] + return [ + os.path.join(network_scripts_path, f) + for f in dir_content + if re.match("^ifcfg-(.*)", f) + ] def _get_ifcfgs_by_type(self, ifcfg_type, network_scripts_path): ifcfgs = [] diff --git a/coriolis/osmorphing/netpreserver/interfaces.py b/coriolis/osmorphing/netpreserver/interfaces.py index 19fbae98..6a8c44f2 100644 --- a/coriolis/osmorphing/netpreserver/interfaces.py +++ b/coriolis/osmorphing/netpreserver/interfaces.py @@ -11,15 +11,13 @@ class InterfacesNetPreserver(base.BaseNetPreserver): - def __init__(self, osmorphing_tool): - super(InterfacesNetPreserver, self).__init__(osmorphing_tool) - self.ifaces_file = "etc/network/interfaces" + super(InterfacesNetPreserver, self).__init__(osmorphing_tool) + self.ifaces_file = "etc/network/interfaces" def check_net_preserver(self): if self.osmorphing_tool._test_path(self.ifaces_file): - if self.osmorphing_tool._list_dir( - os.path.dirname(self.ifaces_file)): + if self.osmorphing_tool._list_dir(os.path.dirname(self.ifaces_file)): return True return False @@ -31,9 +29,9 @@ def _parse_iface_file(interface_file): curr_ip_address = None curr_iface = None interfaces_contents = self.osmorphing_tool._read_file_sudo( - interface_file).decode() - LOG.debug( - "Fetched %s contents: %s", interface_file, interfaces_contents) + interface_file + ).decode() + LOG.debug("Fetched %s contents: %s", interface_file, interfaces_contents) for line in interfaces_contents.splitlines(): if line.strip().startswith("iface"): words = line.split() @@ -41,26 +39,28 @@ def _parse_iface_file(interface_file): curr_iface = words[1] self.interface_info[curr_iface] = { "mac_address": "", - "ip_addresses": [] + "ip_addresses": [], } elif line.strip().startswith("hwaddress ether"): words = line.split() if len(words) > 2: if not curr_iface: - LOG.warn("Found MAC address %s does not belong to " - "any interface stanza. Skipping.", - words[2]) + LOG.warn( + "Found MAC address %s does not belong to " + "any interface stanza. Skipping.", + words[2], + ) continue - self.interface_info[curr_iface][ - "mac_address"] = words[2] + self.interface_info[curr_iface]["mac_address"] = words[2] elif line.strip().startswith("address"): words = line.split() if len(words) > 1: curr_ip_address = words[1].split('/')[0] if curr_iface: - self.interface_info[curr_iface][ - "ip_addresses"].append(curr_ip_address) + self.interface_info[curr_iface]["ip_addresses"].append( + curr_ip_address + ) elif line.strip().startswith("source"): words = line.split() @@ -68,7 +68,8 @@ def _parse_iface_file(interface_file): source_path = words[1] if self.osmorphing_tool._test_path(source_path): paths += self.osmorphing_tool._exec_cmd_chroot( - 'ls -1 %s' % source_path).splitlines() + 'ls -1 %s' % source_path + ).splitlines() while paths: _parse_iface_file(paths[0]) diff --git a/coriolis/osmorphing/netpreserver/netplan.py b/coriolis/osmorphing/netpreserver/netplan.py index 2dccb11c..e235b33f 100644 --- a/coriolis/osmorphing/netpreserver/netplan.py +++ b/coriolis/osmorphing/netpreserver/netplan.py @@ -2,17 +2,15 @@ # All Rights Reserved. import yaml - from oslo_log import log as logging -from coriolis.osmorphing.netpreserver import base from coriolis import utils +from coriolis.osmorphing.netpreserver import base LOG = logging.getLogger(__name__) class NetplanNetPreserver(base.BaseNetPreserver): - def __init__(self, osmorphing_tool): super(NetplanNetPreserver, self).__init__(osmorphing_tool) self.netplan_base = "etc/netplan" @@ -20,25 +18,28 @@ def __init__(self, osmorphing_tool): def check_net_preserver(self): if self.osmorphing_tool._test_path(self.netplan_base): - return any(f.endswith((".yaml", ".yml")) - for f in self.osmorphing_tool._list_dir( - self.netplan_base)) + return any( + f.endswith((".yaml", ".yml")) + for f in self.osmorphing_tool._list_dir(self.netplan_base) + ) return False def parse_network(self): - self.netplan_files = [f for f in self.osmorphing_tool._list_dir( - self.netplan_base) - if f.endswith(".yaml") or f.endswith(".yml")] + self.netplan_files = [ + f + for f in self.osmorphing_tool._list_dir(self.netplan_base) + if f.endswith(".yaml") or f.endswith(".yml") + ] for cfg in self.netplan_files: cfg_path = "%s/%s" % (self.netplan_base, cfg) try: - contents = yaml.safe_load(self.osmorphing_tool._read_file_sudo( - cfg_path)) + contents = yaml.safe_load( + self.osmorphing_tool._read_file_sudo(cfg_path) + ) LOG.info("Parsing netplan configuration '%s'", cfg_path) ifaces = contents.get('network', {}).get('ethernets', {}) for iface, net_cfg in ifaces.items(): - self.interface_info[iface] = {"mac_address": "", - "ip_addresses": []} + self.interface_info[iface] = {"mac_address": "", "ip_addresses": []} mac_address = net_cfg.get('match', {}).get('macaddress') if mac_address: self.interface_info[iface]["mac_address"] = mac_address @@ -46,28 +47,36 @@ def parse_network(self): LOG.warn( "Could not find MAC address or IP addresses for " "interface '%s' in netplan configuration '%s'", - iface, cfg_path) + iface, + cfg_path, + ) for ip in net_cfg.get('addresses', []): if isinstance(ip, dict): ip_keys = list(ip.keys()) if not ip_keys: LOG.warning( - "Found empty IP address object entry. " - "Skipping") + "Found empty IP address object entry. Skipping" + ) continue - ip_obj_addrs = [ - addr.split('/')[0] for addr in ip_keys] + ip_obj_addrs = [addr.split('/')[0] for addr in ip_keys] self.interface_info[iface]["ip_addresses"].extend( - ip_obj_addrs) + ip_obj_addrs + ) else: self.interface_info[iface]["ip_addresses"].append( - ip.split('/')[0]) + ip.split('/')[0] + ) if not self.interface_info[iface]["ip_addresses"]: LOG.warning( "No IP addresses found for interface '%s' in netplan" - "config '%s'", iface, cfg_path) + "config '%s'", + iface, + cfg_path, + ) except yaml.YAMLError: LOG.warn( - "Could not parse netplan configuration '%s'. Invalid YAML " - "file: %s", cfg_path, utils.get_exception_details()) + "Could not parse netplan configuration '%s'. Invalid YAML file: %s", + cfg_path, + utils.get_exception_details(), + ) diff --git a/coriolis/osmorphing/netpreserver/nmconnection.py b/coriolis/osmorphing/netpreserver/nmconnection.py index b371a319..d3355c03 100644 --- a/coriolis/osmorphing/netpreserver/nmconnection.py +++ b/coriolis/osmorphing/netpreserver/nmconnection.py @@ -12,56 +12,58 @@ class NmconnectionNetPreserver(base.BaseNetPreserver): - def __init__(self, osmorphing_tool): super(NmconnectionNetPreserver, self).__init__(osmorphing_tool) self.nmconnection_file = "etc/NetworkManager/system-connections" def check_net_preserver(self): if self.osmorphing_tool._test_path(self.nmconnection_file): - nmconnection_files = self._get_nmconnection_files( - self.nmconnection_file) + nmconnection_files = self._get_nmconnection_files(self.nmconnection_file) if nmconnection_files: nmconnection_ethernet = self._get_keyfiles_by_type( - "ethernet", self.nmconnection_file) + "ethernet", self.nmconnection_file + ) if nmconnection_ethernet: return True return False def parse_network(self): nmconnection_ethernet = self._get_keyfiles_by_type( - "ethernet", self.nmconnection_file) + "ethernet", self.nmconnection_file + ) if nmconnection_ethernet: for nmconn_file, nmconn in nmconnection_ethernet: name = nmconn.get("interface-name", nmconn.get("id")) if not name: - name = re.match(r"^.*/(.*)\.nmconnection$", - nmconn_file).groups()[0] + name = re.match(r"^.*/(.*)\.nmconnection$", nmconn_file).groups()[0] name = name.replace(" ", "_") mac_address = nmconn.get("mac-address") self.interface_info[name] = { "mac_address": mac_address, - "ip_addresses": [] + "ip_addresses": [], } for key, value in nmconn.items(): if key.lower().startswith("address") and value: for addr in value.split(','): addr = addr.strip() ip = addr.strip().split('/')[0] - self.interface_info[name][ - "ip_addresses"].append(ip) - if not mac_address and not self.interface_info[ - name]["ip_addresses"]: + self.interface_info[name]["ip_addresses"].append(ip) + if not mac_address and not self.interface_info[name]["ip_addresses"]: LOG.warning( "Could not find MAC address or IP addresses for " "interface '%s' in nmconnection configuration " - "'%s'", name, nmconn_file) + "'%s'", + name, + nmconn_file, + ) def _get_nmconnection_files(self, network_scripts_path): dir_content = self.osmorphing_tool._list_dir(network_scripts_path) - return [os.path.join(network_scripts_path, f) - for f in dir_content if re.match( - r"^(.*\.nmconnection)$", f)] + return [ + os.path.join(network_scripts_path, f) + for f in dir_content + if re.match(r"^(.*\.nmconnection)$", f) + ] def _get_keyfiles_by_type(self, nmconnection_type, network_scripts_path): keyfiles = [] diff --git a/coriolis/osmorphing/openwrt.py b/coriolis/osmorphing/openwrt.py index a098c9cd..11e57904 100644 --- a/coriolis/osmorphing/openwrt.py +++ b/coriolis/osmorphing/openwrt.py @@ -4,16 +4,13 @@ from coriolis.osmorphing import base from coriolis.osmorphing.osdetect import openwrt as openwrt_detect - OPENWRT_DISTRO_IDENTIFIER = openwrt_detect.OPENWRT_DISTRO_IDENTIFIER class BaseOpenWRTMorphingTools(base.BaseLinuxOSMorphingTools): - @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] == ( - OPENWRT_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] == (OPENWRT_DISTRO_IDENTIFIER): return True return False diff --git a/coriolis/osmorphing/oracle.py b/coriolis/osmorphing/oracle.py index 81f51475..252d7334 100644 --- a/coriolis/osmorphing/oracle.py +++ b/coriolis/osmorphing/oracle.py @@ -3,51 +3,55 @@ import uuid -from coriolis.osmorphing.osdetect import oracle as oracle_detect from coriolis.osmorphing import redhat - +from coriolis.osmorphing.osdetect import oracle as oracle_detect ORACLE_DISTRO_IDENTIFIER = oracle_detect.ORACLE_DISTRO_IDENTIFIER class BaseOracleMorphingTools(redhat.BaseRedHatMorphingTools): - @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - ORACLE_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] != (ORACLE_DISTRO_IDENTIFIER): return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=6) + detected_os_info['release_version'], minimum=6 + ) def _get_oracle_repos(self): repos = [] major_version = int(self._version.split(".")[0]) uekr_version = int(major_version) - 2 if major_version < 8: - repo_file_path = ( - '/etc/yum.repos.d/%s.repo' % str(uuid.uuid4())) + repo_file_path = '/etc/yum.repos.d/%s.repo' % str(uuid.uuid4()) self._exec_cmd_chroot( "curl -L http://public-yum.oracle.com/public-yum-ol%s.repo " - "-o %s" % (major_version, repo_file_path)) + "-o %s" % (major_version, repo_file_path) + ) # During OSMorphing, we temporarily enable needed package repos, # so we make sure we disable all downloaded repos here. self._exec_cmd_chroot( - 'sed -i "s/^enabled=1$/enabled=0/g" %s' % repo_file_path) - - repos_to_enable = ["ol%s_software_collections" % major_version, - "ol%s_addons" % major_version, - "ol%s_UEKR" % major_version, - "ol%s_latest" % major_version] + 'sed -i "s/^enabled=1$/enabled=0/g" %s' % repo_file_path + ) + + repos_to_enable = [ + "ol%s_software_collections" % major_version, + "ol%s_addons" % major_version, + "ol%s_UEKR" % major_version, + "ol%s_latest" % major_version, + ] repos = self._find_yum_repos(repos_to_enable) else: self._yum_install( ['oraclelinux-release-el%s' % major_version], - self._find_yum_repos(['ol%s_baseos_latest' % major_version])) - repos_to_enable = ["ol%s_baseos_latest" % major_version, - "ol%s_appstream" % major_version, - "ol%d_addons" % major_version, - "ol%s_UEKR%s" % (major_version, uekr_version)] + self._find_yum_repos(['ol%s_baseos_latest' % major_version]), + ) + repos_to_enable = [ + "ol%s_baseos_latest" % major_version, + "ol%s_appstream" % major_version, + "ol%d_addons" % major_version, + "ol%s_UEKR%s" % (major_version, uekr_version), + ] repos = self._find_yum_repos(repos_to_enable) return repos diff --git a/coriolis/osmorphing/osdetect/amazon.py b/coriolis/osmorphing/osdetect/amazon.py index 887e2f17..338ce8bd 100644 --- a/coriolis/osmorphing/osdetect/amazon.py +++ b/coriolis/osmorphing/osdetect/amazon.py @@ -4,13 +4,11 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - AMAZON_DISTRO_IDENTIFIER = "amzn" AMAZON_DISTRO_NAME = "Amazon Linux" class AmazonLinuxOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} os_release = self._get_os_release() @@ -23,5 +21,6 @@ def detect_os(self): "os_type": constants.OS_TYPE_LINUX, "distribution_name": AMAZON_DISTRO_NAME, "release_version": version, - "friendly_release_name": friendly_name} + "friendly_release_name": friendly_name, + } return info diff --git a/coriolis/osmorphing/osdetect/base.py b/coriolis/osmorphing/osdetect/base.py index 68d3c3b7..539e7747 100644 --- a/coriolis/osmorphing/osdetect/base.py +++ b/coriolis/osmorphing/osdetect/base.py @@ -7,18 +7,19 @@ from six import with_metaclass -from coriolis import exception -from coriolis import utils +from coriolis import exception, utils # Required OS release fields to be returned as declared in the # 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' schema: REQUIRED_DETECTED_OS_FIELDS = [ - "os_type", "distribution_name", "release_version", - "friendly_release_name"] + "os_type", + "distribution_name", + "release_version", + "friendly_release_name", +] class BaseOSDetectTools(object, with_metaclass(abc.ABCMeta)): - def __init__(self, conn, os_root_dir, operation_timeout): self._conn = conn self._os_root_dir = os_root_dir @@ -29,18 +30,19 @@ def __init__(self, conn, os_root_dir, operation_timeout): @abc.abstractmethod def returned_detected_os_info_fields(cls): raise NotImplementedError( - "No returned OS info fields by class '%s'" % cls.__name__) + "No returned OS info fields by class '%s'" % cls.__name__ + ) @abc.abstractmethod def detect_os(self): - """ Attempts to detect the mounted OS and return all relevant + """Attempts to detect the mounted OS and return all relevant release info as a dict. Must conform to the 'schemas.CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA' """ raise NotImplementedError( - "`detect_os` not implemented for OS detection tools '%s'" % ( - type(self))) + "`detect_os` not implemented for OS detection tools '%s'" % (type(self)) + ) def set_environment(self, environment): self._environment = environment @@ -55,7 +57,6 @@ def set_environment(self, environment): class BaseLinuxOSDetectTools(BaseOSDetectTools): - @classmethod def returned_detected_os_info_fields(cls): return REQUIRED_DETECTED_OS_FIELDS @@ -67,11 +68,11 @@ def _read_file(self, chroot_path): def _read_config_file(self, chroot_path, check_exists=False): full_path = os.path.join(self._os_root_dir, chroot_path) return utils.read_ssh_ini_config_file( - self._conn, full_path, check_exists=check_exists) + self._conn, full_path, check_exists=check_exists + ) def _get_os_release(self): - return self._read_config_file( - "etc/os-release", check_exists=True) + return self._read_config_file("etc/os-release", check_exists=True) def _test_path(self, chroot_path): full_path = os.path.join(self._os_root_dir, chroot_path) @@ -82,19 +83,30 @@ def _exec_cmd(self, cmd, timeout=None): timeout = self._osdetect_operation_timeout try: return utils.exec_ssh_cmd( - self._conn, cmd, environment=self._environment, get_pty=True, - timeout=timeout) + self._conn, + cmd, + environment=self._environment, + get_pty=True, + timeout=timeout, + ) except exception.MinionMachineCommandTimeout as ex: raise exception.OSMorphingSSHOperationTimeout( - cmd=cmd, timeout=timeout) from ex + cmd=cmd, timeout=timeout + ) from ex def _exec_cmd_chroot(self, cmd, timeout=None): if not timeout: timeout = self._osdetect_operation_timeout try: return utils.exec_ssh_cmd_chroot( - self._conn, self._os_root_dir, cmd, - environment=self._environment, get_pty=True, timeout=timeout) + self._conn, + self._os_root_dir, + cmd, + environment=self._environment, + get_pty=True, + timeout=timeout, + ) except exception.MinionMachineCommandTimeout as ex: raise exception.OSMorphingSSHOperationTimeout( - cmd=cmd, timeout=timeout) from ex + cmd=cmd, timeout=timeout + ) from ex diff --git a/coriolis/osmorphing/osdetect/centos.py b/coriolis/osmorphing/osdetect/centos.py index 4b0ae019..5ea53305 100644 --- a/coriolis/osmorphing/osdetect/centos.py +++ b/coriolis/osmorphing/osdetect/centos.py @@ -3,10 +3,10 @@ import re -from coriolis import constants -from coriolis.osmorphing.osdetect import base from oslo_log import log as logging +from coriolis import constants +from coriolis.osmorphing.osdetect import base LOG = logging.getLogger(__name__) CENTOS_DISTRO_IDENTIFIER = "CentOS" @@ -15,23 +15,25 @@ class CentOSOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} redhat_release_path = "etc/redhat-release" if self._test_path(redhat_release_path): - release_info = self._read_file( - redhat_release_path).decode().splitlines() + release_info = self._read_file(redhat_release_path).decode().splitlines() if release_info: - m = re.match(r"^(.*) release ([0-9](\.[0-9])*)( \(.*\))?.*$", - release_info[0].strip()) + m = re.match( + r"^(.*) release ([0-9](\.[0-9])*)( \(.*\))?.*$", + release_info[0].strip(), + ) if m: distro, version, _, _ = m.groups() - if (CENTOS_DISTRO_IDENTIFIER not in distro and - ALMA_IDENTIFIER not in distro): + if ( + CENTOS_DISTRO_IDENTIFIER not in distro + and ALMA_IDENTIFIER not in distro + ): LOG.debug( - "Distro does not appear to be a CentOS or Alma: " - f"{distro}") + f"Distro does not appear to be a CentOS or Alma: {distro}" + ) return {} distribution_name = CENTOS_DISTRO_IDENTIFIER @@ -41,6 +43,7 @@ def detect_os(self): "os_type": constants.OS_TYPE_LINUX, "distribution_name": distribution_name, "release_version": version, - "friendly_release_name": "%s Version %s" % ( - distribution_name, version)} + "friendly_release_name": "%s Version %s" + % (distribution_name, version), + } return info diff --git a/coriolis/osmorphing/osdetect/coreos.py b/coriolis/osmorphing/osdetect/coreos.py index afb25e8c..95ac3b34 100644 --- a/coriolis/osmorphing/osdetect/coreos.py +++ b/coriolis/osmorphing/osdetect/coreos.py @@ -4,12 +4,10 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - COREOS_DISTRO_IDENTIFIER = "CoreOS" class CoreOSOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} os_release = self._get_os_release() @@ -20,5 +18,6 @@ def detect_os(self): "os_type": constants.OS_TYPE_LINUX, "distribution_name": COREOS_DISTRO_IDENTIFIER, "release_version": version, - "friendly_release_name": "CoreOS Linux %s" % version} + "friendly_release_name": "CoreOS Linux %s" % version, + } return info diff --git a/coriolis/osmorphing/osdetect/debian.py b/coriolis/osmorphing/osdetect/debian.py index f14c7385..f9f97c18 100644 --- a/coriolis/osmorphing/osdetect/debian.py +++ b/coriolis/osmorphing/osdetect/debian.py @@ -4,17 +4,16 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - DEBIAN_DISTRO_IDENTIFIER = "Debian" class DebianOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): release = None base_info = { "os_type": constants.OS_TYPE_LINUX, - "distribution_name": DEBIAN_DISTRO_IDENTIFIER} + "distribution_name": DEBIAN_DISTRO_IDENTIFIER, + } lsb_release_path = "etc/lsb-release" debian_version_path = "etc/debian_version" if self._test_path(lsb_release_path): @@ -23,8 +22,9 @@ def detect_os(self): if dist_id == 'Debian': release = config.get('DISTRIB_RELEASE') elif self._test_path(debian_version_path): - deb_release_info = self._read_file( - debian_version_path).decode().splitlines() + deb_release_info = ( + self._read_file(debian_version_path).decode().splitlines() + ) if deb_release_info: release = deb_release_info[0] @@ -32,7 +32,6 @@ def detect_os(self): return {} base_info['release_version'] = release - base_info['friendly_release_name'] = ( - "Debian Linux %s" % release) + base_info['friendly_release_name'] = "Debian Linux %s" % release return base_info diff --git a/coriolis/osmorphing/osdetect/manager.py b/coriolis/osmorphing/osdetect/manager.py index cebf7f47..d5f33b26 100644 --- a/coriolis/osmorphing/osdetect/manager.py +++ b/coriolis/osmorphing/osdetect/manager.py @@ -3,20 +3,21 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import exception -from coriolis.osmorphing.osdetect import amazon -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import centos -from coriolis.osmorphing.osdetect import coreos -from coriolis.osmorphing.osdetect import debian -from coriolis.osmorphing.osdetect import openwrt -from coriolis.osmorphing.osdetect import oracle -from coriolis.osmorphing.osdetect import redhat -from coriolis.osmorphing.osdetect import rocky -from coriolis.osmorphing.osdetect import suse -from coriolis.osmorphing.osdetect import ubuntu -from coriolis.osmorphing.osdetect import windows +from coriolis import constants, exception +from coriolis.osmorphing.osdetect import ( + amazon, + base, + centos, + coreos, + debian, + openwrt, + oracle, + redhat, + rocky, + suse, + ubuntu, + windows, +) LOG = logging.getLogger(__name__) @@ -31,7 +32,7 @@ redhat.RedHatOSDetectTools, rocky.RockyLinuxOSDetectTools, suse.SUSEOSDetectTools, - ubuntu.UbuntuOSDetectTools + ubuntu.UbuntuOSDetectTools, ] WINDOWS_OS_DETECTION_TOOLS = [windows.WindowsOSDetectTools] @@ -40,20 +41,27 @@ def _check_custom_os_detect_tools(custom_os_detect_tools): if not isinstance(custom_os_detect_tools, list): raise exception.InvalidCustomOSDetectTools( - "Custom OS detect tools must be a list, got '%s': %s" % ( - type(custom_os_detect_tools), custom_os_detect_tools)) + "Custom OS detect tools must be a list, got '%s': %s" + % (type(custom_os_detect_tools), custom_os_detect_tools) + ) for detect_tool in custom_os_detect_tools: if not isinstance(detect_tool, base.BaseOSDetectTools): raise exception.InvalidCustomOSDetectTools( "Custom OS tools are of an invalid type which does not " - "extend base.BaseOSDetectTools: %s" % type(detect_tool)) + "extend base.BaseOSDetectTools: %s" % type(detect_tool) + ) return True def detect_os( - conn, os_type, os_root_dir, operation_timeout, tools_environment=None, - custom_os_detect_tools=None): - """ Iterates through all of the OS detection tools until one successfully + conn, + os_type, + os_root_dir, + operation_timeout, + tools_environment=None, + custom_os_detect_tools=None, +): + """Iterates through all of the OS detection tools until one successfully identifies the OS/release and returns the release info from it. param custom_os_detect_tools: list: list of classes which inherit from @@ -80,13 +88,14 @@ def detect_os( for detectcls in detect_tools_classes: tools = detectcls(conn, os_root_dir, operation_timeout) tools.set_environment(tools_environment) - LOG.debug( - "Running OS detection tools class: %s" % detectcls.__name__) + LOG.debug("Running OS detection tools class: %s" % detectcls.__name__) detected_info = tools.detect_os() if detected_info: LOG.info( - "Successfully detected OS using tools '%s'. Returned " - "OS info is: %s", detectcls.__name__, detected_info) + "Successfully detected OS using tools '%s'. Returned OS info is: %s", + detectcls.__name__, + detected_info, + ) break if not detected_info: @@ -94,24 +103,27 @@ def detect_os( if not isinstance(detected_info, dict): raise exception.InvalidDetectedOSParams( - "OS detect tools '%s' returned a non-dict value: %s" % ( - type(tools), detected_info)) + "OS detect tools '%s' returned a non-dict value: %s" + % (type(tools), detected_info) + ) declared_returns = tools.returned_detected_os_info_fields() missing_returns = [ - field for field in declared_returns if field not in detected_info] + field for field in declared_returns if field not in detected_info + ] if missing_returns: raise exception.InvalidDetectedOSParams( "OS detect tools class '%s' has failed to return some required " - "fields (%s) in the detected OS info: %s" % ( - type(tools), declared_returns, detected_info)) + "fields (%s) in the detected OS info: %s" + % (type(tools), declared_returns, detected_info) + ) - extra_returns = [ - field for field in detected_info if field not in declared_returns] + extra_returns = [field for field in detected_info if field not in declared_returns] if extra_returns: raise exception.InvalidDetectedOSParams( "OS detect tools class '%s' has returned fields (%s) which it had " - "declared it would return (%s) in info: %s" % ( - type(tools), extra_returns, declared_returns, detected_info)) + "declared it would return (%s) in info: %s" + % (type(tools), extra_returns, declared_returns, detected_info) + ) return detected_info diff --git a/coriolis/osmorphing/osdetect/openwrt.py b/coriolis/osmorphing/osdetect/openwrt.py index 9cb8fc80..89182cb9 100644 --- a/coriolis/osmorphing/osdetect/openwrt.py +++ b/coriolis/osmorphing/osdetect/openwrt.py @@ -4,16 +4,15 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - OPENWRT_DISTRO_IDENTIFIER = "OpenWRT" class OpenWRTOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} openwrt_release = self._read_config_file( - "etc/openwrt_release", check_exists=True) + "etc/openwrt_release", check_exists=True + ) distrib_id = openwrt_release.get("DISTRIB_ID") if distrib_id == "OpenWrt": name = openwrt_release.get("DISTRIB_DESCRIPTION", distrib_id) @@ -22,6 +21,6 @@ def detect_os(self): "os_type": constants.OS_TYPE_LINUX, "distribution_name": OPENWRT_DISTRO_IDENTIFIER, "release_version": version, - "friendly_release_name": "%s Version %s" % ( - name, version)} + "friendly_release_name": "%s Version %s" % (name, version), + } return info diff --git a/coriolis/osmorphing/osdetect/oracle.py b/coriolis/osmorphing/osdetect/oracle.py index c52ea3d5..dcd7106a 100644 --- a/coriolis/osmorphing/osdetect/oracle.py +++ b/coriolis/osmorphing/osdetect/oracle.py @@ -6,27 +6,23 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - ORACLE_DISTRO_IDENTIFIER = "Oracle Linux" class OracleOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} oracle_release_path = "etc/oracle-release" if self._test_path(oracle_release_path): - release_info = self._read_file( - oracle_release_path).decode().splitlines() + release_info = self._read_file(oracle_release_path).decode().splitlines() if release_info: - m = re.match(r"^(.*) release ([0-9].*)$", - release_info[0].strip()) + m = re.match(r"^(.*) release ([0-9].*)$", release_info[0].strip()) if m: distro, version = m.groups() info = { "os_type": constants.OS_TYPE_LINUX, "distribution_name": ORACLE_DISTRO_IDENTIFIER, "release_version": version, - "friendly_release_name": "%s Version %s" % ( - distro, version)} + "friendly_release_name": "%s Version %s" % (distro, version), + } return info diff --git a/coriolis/osmorphing/osdetect/redhat.py b/coriolis/osmorphing/osdetect/redhat.py index 90f12a35..ce808af4 100644 --- a/coriolis/osmorphing/osdetect/redhat.py +++ b/coriolis/osmorphing/osdetect/redhat.py @@ -8,33 +8,31 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - LOG = logging.getLogger(__name__) RED_HAT_DISTRO_IDENTIFIER = "Red Hat Enterprise Linux" class RedHatOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} redhat_release_path = "etc/redhat-release" if self._test_path(redhat_release_path): - release_info = self._read_file( - redhat_release_path).decode().splitlines() + release_info = self._read_file(redhat_release_path).decode().splitlines() if release_info: - m = re.match(r"^(.*) release ([0-9].*) \((.*)\).*$", - release_info[0].strip()) + m = re.match( + r"^(.*) release ([0-9].*) \((.*)\).*$", release_info[0].strip() + ) if m: distro, version, _ = m.groups() if RED_HAT_DISTRO_IDENTIFIER not in distro: - LOG.debug( - "Distro does not appear to be a RHEL: %s", distro) + LOG.debug("Distro does not appear to be a RHEL: %s", distro) return {} info = { "os_type": constants.OS_TYPE_LINUX, "distribution_name": RED_HAT_DISTRO_IDENTIFIER, "release_version": version, - "friendly_release_name": "%s Version %s" % ( - RED_HAT_DISTRO_IDENTIFIER, version)} + "friendly_release_name": "%s Version %s" + % (RED_HAT_DISTRO_IDENTIFIER, version), + } return info diff --git a/coriolis/osmorphing/osdetect/rocky.py b/coriolis/osmorphing/osdetect/rocky.py index 60f10fd9..38b58407 100644 --- a/coriolis/osmorphing/osdetect/rocky.py +++ b/coriolis/osmorphing/osdetect/rocky.py @@ -3,38 +3,39 @@ import re -from coriolis import constants -from coriolis.osmorphing.osdetect import base from oslo_log import log as logging +from coriolis import constants +from coriolis.osmorphing.osdetect import base LOG = logging.getLogger(__name__) ROCKY_LINUX_DISTRO_IDENTIFIER = "Rocky Linux" class RockyLinuxOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} redhat_release_path = "etc/redhat-release" if self._test_path(redhat_release_path): - release_info = self._read_file( - redhat_release_path).decode().splitlines() + release_info = self._read_file(redhat_release_path).decode().splitlines() if release_info: - m = re.match(r"^(.*) release ([0-9](\.[0-9])*)( \(.*\))?.*$", - release_info[0].strip()) + m = re.match( + r"^(.*) release ([0-9](\.[0-9])*)( \(.*\))?.*$", + release_info[0].strip(), + ) if m: distro, version, _, _ = m.groups() if ROCKY_LINUX_DISTRO_IDENTIFIER not in distro: LOG.debug( - "Distro does not appear to be a Rocky Linux: %s", - distro) + "Distro does not appear to be a Rocky Linux: %s", distro + ) return {} info = { "os_type": constants.OS_TYPE_LINUX, "distribution_name": ROCKY_LINUX_DISTRO_IDENTIFIER, "release_version": version, - "friendly_release_name": "%s Version %s" % ( - ROCKY_LINUX_DISTRO_IDENTIFIER, version)} + "friendly_release_name": "%s Version %s" + % (ROCKY_LINUX_DISTRO_IDENTIFIER, version), + } return info diff --git a/coriolis/osmorphing/osdetect/suse.py b/coriolis/osmorphing/osdetect/suse.py index cb5ac11b..ba5b6b77 100644 --- a/coriolis/osmorphing/osdetect/suse.py +++ b/coriolis/osmorphing/osdetect/suse.py @@ -6,7 +6,6 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - DETECTED_SUSE_RELEASE_FIELD_NAME = "suse_release_name" SLES_DISTRO_IDENTIFIER = "SLES" OPENSUSE_DISTRO_IDENTIFIER = "openSUSE" @@ -14,11 +13,9 @@ class SUSEOSDetectTools(base.BaseLinuxOSDetectTools): - @classmethod def returned_detected_os_info_fields(cls): - common_fields = super( - SUSEOSDetectTools, cls).returned_detected_os_info_fields() + common_fields = super(SUSEOSDetectTools, cls).returned_detected_os_info_fields() fields = copy.deepcopy(common_fields) fields.append(DETECTED_SUSE_RELEASE_FIELD_NAME) return fields @@ -36,13 +33,11 @@ def detect_os(self): distro_name = SLES_DISTRO_IDENTIFIER elif name.lower().startswith("opensuse"): distro_name = OPENSUSE_DISTRO_IDENTIFIER - version = os_release.get( - "VERSION_ID", constants.OS_TYPE_UNKNOWN) + version = os_release.get("VERSION_ID", constants.OS_TYPE_UNKNOWN) if 'tumbleweed' in name.lower(): version = OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER else: - friendly_release_name = "%s %s" % ( - distro_name, version) + friendly_release_name = "%s %s" % (distro_name, version) if distro_name: info = { @@ -50,7 +45,8 @@ def detect_os(self): "distribution_name": distro_name, DETECTED_SUSE_RELEASE_FIELD_NAME: name, "release_version": version, - "friendly_release_name": friendly_release_name} + "friendly_release_name": friendly_release_name, + } # NOTE: should be redundant as all SUSEs which have a # SuSE-release but no os-release have been deprecated diff --git a/coriolis/osmorphing/osdetect/ubuntu.py b/coriolis/osmorphing/osdetect/ubuntu.py index 3dbe09ba..c9747e3a 100644 --- a/coriolis/osmorphing/osdetect/ubuntu.py +++ b/coriolis/osmorphing/osdetect/ubuntu.py @@ -4,12 +4,10 @@ from coriolis import constants from coriolis.osmorphing.osdetect import base - UBUNTU_DISTRO_IDENTIFIER = "Ubuntu" class UbuntuOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): info = {} config = self._read_config_file("etc/lsb-release", check_exists=True) @@ -20,5 +18,6 @@ def detect_os(self): "os_type": constants.OS_TYPE_LINUX, "distribution_name": UBUNTU_DISTRO_IDENTIFIER, "release_version": release, - "friendly_release_name": "Ubuntu %s" % release} + "friendly_release_name": "Ubuntu %s" % release, + } return info diff --git a/coriolis/osmorphing/osdetect/windows.py b/coriolis/osmorphing/osdetect/windows.py index 9653b4d4..0ee5221b 100644 --- a/coriolis/osmorphing/osdetect/windows.py +++ b/coriolis/osmorphing/osdetect/windows.py @@ -5,15 +5,12 @@ import os import re import uuid - from distutils import version + from oslo_log import log as logging -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, utils from coriolis.osmorphing.osdetect import base -from coriolis import utils - WINDOWS_SERVER_IDENTIFIER = "Server" WINDOWS_CLIENT_IDENTIFIER = "Client" @@ -22,17 +19,18 @@ LOG = logging.getLogger(__name__) REQUIRED_DETECTED_WINDOWS_OS_FIELDS = [ - "version_number", "edition_id", "installation_type", "product_name"] + "version_number", + "edition_id", + "installation_type", + "product_name", +] class WindowsOSDetectTools(base.BaseOSDetectTools): - @classmethod def returned_detected_os_info_fields(cls): - base_fields = copy.deepcopy( - base.REQUIRED_DETECTED_OS_FIELDS) - base_fields.extend( - REQUIRED_DETECTED_WINDOWS_OS_FIELDS) + base_fields = copy.deepcopy(base.REQUIRED_DETECTED_OS_FIELDS) + base_fields.extend(REQUIRED_DETECTED_WINDOWS_OS_FIELDS) return base_fields def _load_registry_hive(self, subkey, path): @@ -51,43 +49,55 @@ def _get_image_version_info(self): self._load_registry_hive( "HKLM\\%s" % key_name, - "%sWindows\\System32\\config\\SOFTWARE" % self._os_root_dir) + "%sWindows\\System32\\config\\SOFTWARE" % self._os_root_dir, + ) try: version_info_str = self._conn.exec_ps_command( "Get-ItemProperty " "'HKLM:\\%s\\Microsoft\\Windows NT\\CurrentVersion' " "| select CurrentVersion, CurrentMajorVersionNumber, " "CurrentMinorVersionNumber, CurrentBuildNumber, " - "InstallationType, ProductName, EditionID | FL" % - key_name).replace(self._conn.EOL, os.linesep) + "InstallationType, ProductName, EditionID | FL" % key_name + ).replace(self._conn.EOL, os.linesep) finally: self._unload_registry_hive("HKLM\\%s" % key_name) version_info = {} - for n in ["CurrentVersion", "CurrentMajorVersionNumber", - "CurrentMinorVersionNumber", "CurrentBuildNumber", - "InstallationType", "ProductName", "EditionID"]: + for n in [ + "CurrentVersion", + "CurrentMajorVersionNumber", + "CurrentMinorVersionNumber", + "CurrentBuildNumber", + "InstallationType", + "ProductName", + "EditionID", + ]: version_info[n] = self._get_ps_fl_value(version_info_str, n) - if (not version_info["CurrentMajorVersionNumber"] and - not version_info["CurrentVersion"]): - raise exception.CoriolisException( - "Cannot find Windows version info") + if ( + not version_info["CurrentMajorVersionNumber"] + and not version_info["CurrentVersion"] + ): + raise exception.CoriolisException("Cannot find Windows version info") if version_info["CurrentMajorVersionNumber"]: version_str = "%s.%s.%s" % ( version_info["CurrentMajorVersionNumber"], version_info["CurrentMinorVersionNumber"], - version_info["CurrentBuildNumber"]) + version_info["CurrentBuildNumber"], + ) else: version_str = "%s.%s" % ( version_info["CurrentVersion"], - version_info["CurrentBuildNumber"]) + version_info["CurrentBuildNumber"], + ) - return (version.LooseVersion(version_str), - version_info["EditionID"], - version_info["InstallationType"], - version_info["ProductName"]) + return ( + version.LooseVersion(version_str), + version_info["EditionID"], + version_info["InstallationType"], + version_info["ProductName"], + ) def detect_os(self): version_number = None @@ -95,21 +105,24 @@ def detect_os(self): installation_type = None product_name = None try: - (version_number, - edition_id, - installation_type, - product_name) = self._get_image_version_info() + (version_number, edition_id, installation_type, product_name) = ( + self._get_image_version_info() + ) except exception.CoriolisException: LOG.debug( "Exception during Windows OS detection: %s", - utils.get_exception_details()) + utils.get_exception_details(), + ) raise LOG.debug( "Successfully identified Windows release as: Version no.: %s; " "Edition id: %s; Install type: %s; Name: %s", - version_number, edition_id, - installation_type, product_name) + version_number, + edition_id, + installation_type, + product_name, + ) typ = WINDOWS_CLIENT_IDENTIFIER if 'server' in edition_id.lower(): @@ -123,4 +136,5 @@ def detect_os(self): "os_type": constants.OS_TYPE_WINDOWS, "distribution_name": typ, "release_version": product_name, - "friendly_release_name": "Windows %s" % product_name} + "friendly_release_name": "Windows %s" % product_name, + } diff --git a/coriolis/osmorphing/osmount/base.py b/coriolis/osmorphing/osmount/base.py index b7a1a585..6bc0c9b4 100644 --- a/coriolis/osmorphing/osmount/base.py +++ b/coriolis/osmorphing/osmount/base.py @@ -13,8 +13,7 @@ from oslo_log import log as logging from six import with_metaclass -from coriolis import exception -from coriolis import utils +from coriolis import exception, utils LOG = logging.getLogger(__name__) @@ -22,9 +21,9 @@ class BaseOSMountTools(object, with_metaclass(abc.ABCMeta)): - - def __init__(self, connection_info, event_manager, ignore_devices, - operation_timeout): + def __init__( + self, connection_info, event_manager, ignore_devices, operation_timeout + ): self._event_manager = event_manager self._ignore_devices = ignore_devices self._environment = {} @@ -75,14 +74,15 @@ def _connect(self): LOG.info( "Waiting for SSH connectivity on OSMorphing host: %(ip)s:%(port)s", - {"ip": ip, "port": port}) + {"ip": ip, "port": port}, + ) utils.wait_for_port_connectivity(ip, port) self._event_manager.progress_update( - "Connecting through SSH to OSMorphing host on: %(ip)s:%(port)s" % - ({"ip": ip, "port": port})) - ssh = utils.connect_ssh( - ip, port, username, pkey=pkey, password=password) + "Connecting through SSH to OSMorphing host on: %(ip)s:%(port)s" + % ({"ip": ip, "port": port}) + ) + ssh = utils.connect_ssh(ip, port, username, pkey=pkey, password=password) ssh.set_log_channel("paramiko.morpher.%s.%s" % (ip, port)) self._ssh = ssh @@ -98,8 +98,8 @@ def _allow_ssh_env_vars(self): except exception.CoriolisException: # Newer Ubuntu distros renamed this service to `ssh` LOG.warning( - "Could not restart service sshd. Attempting to restart ssh " - "service") + "Could not restart service sshd. Attempting to restart ssh service" + ) utils.restart_service(self._ssh, 'ssh') return True @@ -107,11 +107,13 @@ def _exec_cmd(self, cmd, timeout=None): if not timeout: timeout = self._osmount_operation_timeout try: - return utils.exec_ssh_cmd(self._ssh, cmd, self._environment, - get_pty=True, timeout=timeout) + return utils.exec_ssh_cmd( + self._ssh, cmd, self._environment, get_pty=True, timeout=timeout + ) except exception.MinionMachineCommandTimeout as ex: raise exception.OSMorphingSSHOperationTimeout( - cmd=cmd, timeout=timeout) from ex + cmd=cmd, timeout=timeout + ) from ex def get_connection(self): return self._ssh @@ -128,25 +130,25 @@ def _get_pvs(self): line = line.strip().split(":") if len(line) >= 2: if pvs.get(line[1]) is None: - pvs[line[1]] = [line[0], ] + pvs[line[1]] = [ + line[0], + ] else: pvs[line[1]].append(line[0]) else: - LOG.warn( - "Ignoring improper `pvdisplay` output entry: %s" % line) + LOG.warn("Ignoring improper `pvdisplay` output entry: %s" % line) LOG.debug("Physical Volume attributes: %s", pvs) return pvs def _get_vgs(self): - """ Returns a dict of the form: { + """Returns a dict of the form: { "VG UUID": { "name": "", "pvs": ["list", "of", "PV", "names", "in", "VG"], } } """ - vgs_cmd = ( - "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :") + vgs_cmd = "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :" out = self._exec_cmd(vgs_cmd).splitlines() LOG.debug("Output of %s command: %s", vgs_cmd, out) vgs_uuid_map = {} @@ -168,13 +170,12 @@ def _get_vgs(self): LOG.debug( "VG with name '%s' already detected. Renaming VG " "with UUID '%s' to '%s' to avoid conflicts", - vg_name, vg_uuid, new_name) - self._exec_cmd("sudo vgrename %s %s" % - (vg_uuid, new_name)) - vgs_uuid_map[vg_uuid] = { - "name": new_name, - "pvs": [pv_name] - } + vg_name, + vg_uuid, + new_name, + ) + self._exec_cmd("sudo vgrename %s %s" % (vg_uuid, new_name)) + vgs_uuid_map[vg_uuid] = {"name": new_name, "pvs": [pv_name]} else: LOG.warning("Ignoring improper `vgs` output entry: %s", line) LOG.debug("Volume groups: %s", vgs_uuid_map) @@ -190,7 +191,8 @@ def _check_vgs(self): "the OSMorphing from proceeding further. Please ensure that " "the source VM's LVM configuration is correct and the VM is " "able to use all LVM volumes on the source. Error occured " - "while checking the consistency of all LVM VGs: %s" % str(ex)) + "while checking the consistency of all LVM VGs: %s" % str(ex) + ) def _get_vgnames(self): vg_names = [] @@ -199,14 +201,15 @@ def _get_vgnames(self): for vgscan_out_line in vgscan_out_lines: m = re.match( r'\s*Found volume group "(.*)" using metadata type lvm2', - vgscan_out_line) + vgscan_out_line, + ) if m: vg_names.append(m.groups()[0]) LOG.debug("Volume group names: %s", vg_names) return vg_names def _get_lv_paths(self): - """ Returns list with paths of available LVM volumes. """ + """Returns list with paths of available LVM volumes.""" lvm_paths = [] out = self._exec_cmd("sudo lvdisplay -c").strip() if out: @@ -221,9 +224,13 @@ def _get_lv_paths(self): return lvm_paths def _check_mount_fstab_partitions( - self, os_root_dir, skip_mounts=["/", "/boot"], - skip_filesystems=["swap"], mountable_lvm_devs=None): - """ Reads the contents of /etc/fstab from the VM's root directory and + self, + os_root_dir, + skip_mounts=["/", "/boot"], + skip_filesystems=["swap"], + mountable_lvm_devs=None, + ): + """Reads the contents of /etc/fstab from the VM's root directory and tries to mount all clearly identified (by UUID or path) filesystems. Returns the list of the new directories which were mounted. param: skip_mounts: list(str()): list of directories (inside the @@ -238,15 +245,17 @@ def _check_mount_fstab_partitions( if not utils.test_ssh_path(self._ssh, etc_fstab_path): LOG.warn( "etc/fstab file not found in '%s'. Cannot mount non-root dirs", - os_root_dir) + os_root_dir, + ) return [] etc_fstab_raw = utils.read_ssh_file(self._ssh, etc_fstab_path) etc_fstab = etc_fstab_raw.decode('utf-8') LOG.debug( - "Mounting non-root partitions from fstab file: %s" % ( - base64.b64encode(etc_fstab_raw))) + "Mounting non-root partitions from fstab file: %s" + % (base64.b64encode(etc_fstab_raw)) + ) # dictionary of the form {"mountpoint": # {"device": "", "filesystem": "", "options": ""}} @@ -254,13 +263,13 @@ def _check_mount_fstab_partitions( # fstab entry format: # fstab_entry_regex = ( - "^(\s*([^#\s]+)\s+(\S+)\s+(\S+)\s+(\S+)(\s+(\d)(\s+(\d))?)?\s*)$") + "^(\s*([^#\s]+)\s+(\S+)\s+(\S+)\s+(\S+)(\s+(\d)(\s+(\d))?)?\s*)$" + ) for line in etc_fstab.splitlines(): match = re.match(fstab_entry_regex, line) if not match: - LOG.warn( - "Skipping unparseable /etc/fstab line: '%s'", line) + LOG.warn("Skipping unparseable /etc/fstab line: '%s'", line) continue device = match.group(2) @@ -268,31 +277,33 @@ def _check_mount_fstab_partitions( if not mountpoint.startswith('/'): LOG.warning( - f"Skipping mountpoint that is not a valid directory: " - f"{mountpoint}") + f"Skipping mountpoint that is not a valid directory: {mountpoint}" + ) continue if mountpoint in mounts: raise exception.CoriolisException( "Mountpoint '%s' appears to be mounted twice in " - "'/etc/fstab'" % mountpoint) + "'/etc/fstab'" % mountpoint + ) mounts[mountpoint] = { "device": device, "filesystem": match.group(4), - "options": match.group(5)} + "options": match.group(5), + } # NOTE: we sort the mountpoints based on length to ensure # they get mounted in the correct order: mounts = collections.OrderedDict( - (mountpoint, mounts[mountpoint]) - for mountpoint in sorted(mounts, key=len)) + (mountpoint, mounts[mountpoint]) for mountpoint in sorted(mounts, key=len) + ) # regex for supported fstab device references: fs_uuid_entry_regex = "^((UUID=|/dev/disk/by-uuid/)(.+))$" if not mountable_lvm_devs: mountable_lvm_devs = [] device_paths = self._get_device_file_paths(mountable_lvm_devs) - for (mountpoint, details) in mounts.items(): + for mountpoint, details in mounts.items(): device = details['device'] fs_uuid_match = re.match(fs_uuid_entry_regex, device) if fs_uuid_match: @@ -300,36 +311,37 @@ def _check_mount_fstab_partitions( else: device_file_path = self._get_symlink_target(device) if device not in mountable_lvm_devs and ( - device_file_path not in device_paths): + device_file_path not in device_paths + ): LOG.warn( "Found fstab entry for dir %s which references device " "%s. Only LVM volumes or devices referenced by UUID=* " "or /dev/disk/by-uuid/* notation are supported. " "Devicemapper paths for LVM volumes are also " - "supported. Skipping mounting directory." % - (mountpoint, device)) + "supported. Skipping mounting directory." % (mountpoint, device) + ) continue if mountpoint in skip_mounts: - LOG.debug( - "Skipping undesired mount: %s: %s", mountpoint, details) + LOG.debug("Skipping undesired mount: %s: %s", mountpoint, details) continue if details["filesystem"] in skip_filesystems: LOG.debug( - "Skipping mounting undesired FS for device %s: %s", - device, details) + "Skipping mounting undesired FS for device %s: %s", device, details + ) continue - LOG.debug("Attempting to mount fstab device: %s: %s", - device, details) + LOG.debug("Attempting to mount fstab device: %s: %s", device, details) # NOTE: `mountpoint` should always be an absolute path: chroot_mountpoint = "%s%s" % (os_root_dir, mountpoint) if not utils.test_ssh_path(self._ssh, device): - LOG.warn( - "Device path %s not found, skipping mount.", device) + LOG.warn("Device path %s not found, skipping mount.", device) continue mountcmd = "sudo mount -t %s -o %s %s '%s'" % ( - details["filesystem"], details["options"], - device, chroot_mountpoint) + details["filesystem"], + details["options"], + device, + chroot_mountpoint, + ) try: self._exec_cmd(mountcmd) new_mountpoints.append(chroot_mountpoint) @@ -337,12 +349,15 @@ def _check_mount_fstab_partitions( LOG.warn( "Failed to run fstab filesystem mount command: '%s'. " "Skipping mount. Error details: %s", - mountcmd, utils.get_exception_details()) + mountcmd, + utils.get_exception_details(), + ) if new_mountpoints: LOG.info( - "The following new /etc/fstab entries were successfully " - "mounted: %s", new_mountpoints) + "The following new /etc/fstab entries were successfully mounted: %s", + new_mountpoints, + ) return new_mountpoints @@ -352,13 +367,15 @@ def _get_symlink_target(self, symlink): target = self._exec_cmd('readlink -en %s' % symlink) LOG.debug("readlink %s returned: %s" % (symlink, target)) except Exception: - LOG.warn('Target not found for symlink: %s. Original link path ' - 'will be returned' % symlink) + LOG.warn( + 'Target not found for symlink: %s. Original link path ' + 'will be returned' % symlink + ) return target def _get_device_file_paths(self, symlink_list): - """ Reads a list of symlink paths, such as `/dev/GROUP/VOLUME0` or + """Reads a list of symlink paths, such as `/dev/GROUP/VOLUME0` or `/dev/mapper/GROUP-VOLUME0` and returns a list of respective links' target device file paths, such as `/dev/dm0` """ @@ -380,19 +397,20 @@ def _get_mounted_devices(self): if colls[0].startswith("/dev"): dev_name = self._get_symlink_target(colls[0]) if not utils.test_ssh_path(self._ssh, dev_name): - LOG.warn( - "Device name %s not found, skipping mount.", dev_name) + LOG.warn("Device name %s not found, skipping mount.", dev_name) continue ret.append(dev_name) mounted_device_numbers.append( - self._exec_cmd(dev_nmb_cmd % dev_name).rstrip()) + self._exec_cmd(dev_nmb_cmd % dev_name).rstrip() + ) block_devs = self._exec_cmd("ls -al /dev | grep ^b").splitlines() for dev_line in block_devs: dev = dev_line.split() major_minor = "%s:%s" % ( dev[MAJOR_COLUMN_INDEX].rstrip(','), - dev[MAJOR_COLUMN_INDEX + 1]) + dev[MAJOR_COLUMN_INDEX + 1], + ) if major_minor in mounted_device_numbers: dev_path = "/dev/%s" % dev[-1] @@ -420,21 +438,21 @@ def _get_volume_block_devices(self): volume_devs = self._exec_cmd("lsblk -lnao KNAME").splitlines() LOG.debug("All block devices: %s", str(volume_devs)) - volume_devs = ["/dev/%s" % d for d in volume_devs if - not re.match(r"^.*\d+$", d)] + volume_devs = [ + "/dev/%s" % d for d in volume_devs if not re.match(r"^.*\d+$", d) + ] LOG.debug("Ignoring block devices: %s", self._ignore_devices) - volume_devs = [d for d in volume_devs if d - not in self._ignore_devices] + volume_devs = [d for d in volume_devs if d not in self._ignore_devices] LOG.info("Volume block devices: %s", volume_devs) return volume_devs - def _find_dev_with_contents(self, devices, all_files=None, - one_of_files=None): + def _find_dev_with_contents(self, devices, all_files=None, one_of_files=None): if all_files and one_of_files: raise exception.CoriolisException( - "all_files and one_of_files are mutually exclusive") + "all_files and one_of_files are mutually exclusive" + ) dev_name = None for dev_path in devices: dirs = None @@ -461,16 +479,15 @@ def _find_dev_with_contents(self, devices, all_files=None, except Exception: self._event_manager.progress_update( - "Failed to mount and scan device '%s'" % dev_path) + "Failed to mount and scan device '%s'" % dev_path + ) LOG.warn( "Failed to mount and scan device '%s':\n%s", - dev_path, utils.get_exception_details()) - utils.ignore_exceptions(self._exec_cmd)( - "sudo umount %s" % tmp_dir - ) - utils.ignore_exceptions(self._exec_cmd)( - "sudo rmdir %s" % tmp_dir + dev_path, + utils.get_exception_details(), ) + utils.ignore_exceptions(self._exec_cmd)("sudo umount %s" % tmp_dir) + utils.ignore_exceptions(self._exec_cmd)("sudo rmdir %s" % tmp_dir) tmp_dir = None continue finally: @@ -482,8 +499,7 @@ def _find_dev_with_contents(self, devices, all_files=None, def _find_and_mount_root(self, devices): files = ["etc", "bin", "sbin", "boot"] os_root_dir = None - os_root_device = self._find_dev_with_contents( - devices, all_files=files) + os_root_device = self._find_dev_with_contents(devices, all_files=files) if os_root_device is None: raise exception.OperatingSystemNotFound( @@ -497,7 +513,8 @@ def _find_and_mount_root(self, devices): "declared using '/dev/disk/by-uuid/' or 'UUID=' notation. " "If all else fails, please retry while using an OSMorphing " "minion machine image which is the same OS release as the VM " - "being migrated.") + "being migrated." + ) try: tmp_dir = self._exec_cmd('mktemp -d').splitlines()[0] @@ -515,29 +532,27 @@ def _find_and_mount_root(self, devices): "'/dev/disk/by-uuid/' or 'UUID=' notation. If all else fails, " "please retry while using an OSMorphing minion machine image " "which is the same OS release as the VM being migrated. Error " - "was: %s" % (os_root_device, str(ex))) + "was: %s" % (os_root_device, str(ex)) + ) LOG.error(ex) LOG.warn( "Failed to mount root device '%s':\n%s", - os_root_device, utils.get_exception_details()) - utils.ignore_exceptions(self._exec_cmd)( - "sudo umount %s" % tmp_dir - ) - utils.ignore_exceptions(self._exec_cmd)( - "sudo rmdir %s" % tmp_dir + os_root_device, + utils.get_exception_details(), ) + utils.ignore_exceptions(self._exec_cmd)("sudo umount %s" % tmp_dir) + utils.ignore_exceptions(self._exec_cmd)("sudo rmdir %s" % tmp_dir) raise for directory in ['proc', 'sys', 'dev', 'run']: mount_dir = os.path.join(os_root_dir, directory) if not utils.test_ssh_path(self._ssh, mount_dir): - LOG.info( - "No '%s' directory in mounted OS. Skipping mount.", - directory) + LOG.info("No '%s' directory in mounted OS. Skipping mount.", directory) continue self._exec_cmd( - 'sudo mount -o bind /%(dir)s/ %(mount_dir)s' % - {'dir': directory, 'mount_dir': mount_dir}) + 'sudo mount -o bind /%(dir)s/ %(mount_dir)s' + % {'dir': directory, 'mount_dir': mount_dir} + ) if os_root_device in devices: devices.remove(os_root_device) @@ -551,8 +566,7 @@ def mount_os(self): volume_devs = self._get_volume_block_devices() for volume_dev in volume_devs: self._exec_cmd("sudo partx -v -a %s || true" % volume_dev) - dev_paths += self._exec_cmd( - "sudo ls -1 %s*" % volume_dev).splitlines() + dev_paths += self._exec_cmd("sudo ls -1 %s*" % volume_dev).splitlines() LOG.debug("All simple devices to scan: %s", dev_paths) lvm_dev_paths = [] @@ -566,12 +580,11 @@ def mount_os(self): self._exec_cmd("sudo vgchange --refresh") dev_vg_path = f"/dev/{vg_props['name']}" if not utils.test_ssh_path(self._ssh, dev_vg_path): - LOG.warning( - "Volume Group '%s' not found. Skipping.", - dev_vg_path) + LOG.warning("Volume Group '%s' not found. Skipping.", dev_vg_path) continue dev_paths_for_group = self._exec_cmd( - f"sudo ls -1 /dev/{vg_props['name']}/*").splitlines() + f"sudo ls -1 /dev/{vg_props['name']}/*" + ).splitlines() lvm_dev_paths.extend(dev_paths_for_group) LOG.debug("All LVM vols to scan: %s", lvm_dev_paths) @@ -587,7 +600,9 @@ def mount_os(self): LOG.debug( "Device '%s' (target '%s') is already mounted, assuming it" "belongs to the worker VM so it will be skipped", - dev_path, dev_target) + dev_path, + dev_target, + ) continue fs_type = self._exec_cmd( "sudo blkid -o value -s TYPE %s || true" % dev_path @@ -610,26 +625,24 @@ def mount_os(self): "etc.) are enabled and available for the source " "machine. If none are available, please try " "migrating/replicating the source machine while it" - " is powered off. Error was: %s" % str(err)) + " is powered off. Error was: %s" % str(err) + ) LOG.error(err) dev_paths_to_mount.append(dev_path) - os_root_dir, os_root_device = self._find_and_mount_root( - dev_paths_to_mount) + os_root_dir, os_root_device = self._find_and_mount_root(dev_paths_to_mount) grub_dirs = ["grub", "grub2"] os_boot_device = self._find_dev_with_contents( - dev_paths_to_mount, one_of_files=grub_dirs) + dev_paths_to_mount, one_of_files=grub_dirs + ) if os_boot_device: LOG.debug("Mounting boot device '%s'", os_boot_device) - self._exec_cmd( - 'sudo mount %s "%s/boot"' % ( - os_boot_device, os_root_dir)) + self._exec_cmd('sudo mount %s "%s/boot"' % (os_boot_device, os_root_dir)) lvm_devs = list(set(self._get_lv_paths()) - set(mounted_devs)) - self._check_mount_fstab_partitions( - os_root_dir, mountable_lvm_devs=lvm_devs) + self._check_mount_fstab_partitions(os_root_dir, mountable_lvm_devs=lvm_devs) return os_root_dir, os_root_device @@ -649,10 +662,8 @@ def dismount_os(self, root_dir): self._exec_cmd('sudo umount %s' % d) dev_fs = "%s/%s" % (root_dir.rstrip('/'), "dev") - self._exec_cmd('mountpoint -q %s && sudo umount %s' % - (dev_fs, dev_fs)) - self._exec_cmd( - 'mountpoint -q %s && sudo umount %s' % (root_dir, root_dir)) + self._exec_cmd('mountpoint -q %s && sudo umount %s' % (dev_fs, dev_fs)) + self._exec_cmd('mountpoint -q %s && sudo umount %s' % (root_dir, root_dir)) def set_proxy(self, proxy_settings): url = proxy_settings.get('url') diff --git a/coriolis/osmorphing/osmount/factory.py b/coriolis/osmorphing/osmount/factory.py index 890eb771..ba4c3a25 100644 --- a/coriolis/osmorphing/osmount/factory.py +++ b/coriolis/osmorphing/osmount/factory.py @@ -5,30 +5,29 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import exception -from coriolis.osmorphing.osmount import redhat -from coriolis.osmorphing.osmount import suse -from coriolis.osmorphing.osmount import ubuntu -from coriolis.osmorphing.osmount import windows +from coriolis import constants, exception +from coriolis.osmorphing.osmount import redhat, suse, ubuntu, windows LOG = logging.getLogger(__name__) -def get_os_mount_tools(os_type, connection_info, event_manager, - ignore_devices, operation_timeout): - os_mount_tools = {constants.OS_TYPE_LINUX: [ubuntu.UbuntuOSMountTools, - redhat.RedHatOSMountTools, - suse.SUSEOSMountTools], - constants.OS_TYPE_WINDOWS: [windows.WindowsMountTools]} +def get_os_mount_tools( + os_type, connection_info, event_manager, ignore_devices, operation_timeout +): + os_mount_tools = { + constants.OS_TYPE_LINUX: [ + ubuntu.UbuntuOSMountTools, + redhat.RedHatOSMountTools, + suse.SUSEOSMountTools, + ], + constants.OS_TYPE_WINDOWS: [windows.WindowsMountTools], + } if os_type and os_type not in os_mount_tools: raise exception.CoriolisException("Unsupported OS type: %s" % os_type) - for cls in os_mount_tools.get(os_type, - itertools.chain(*os_mount_tools.values())): - tools = cls(connection_info, event_manager, ignore_devices, - operation_timeout) + for cls in os_mount_tools.get(os_type, itertools.chain(*os_mount_tools.values())): + tools = cls(connection_info, event_manager, ignore_devices, operation_timeout) LOG.debug("Testing OS mount tools: %s", cls.__name__) if tools.check_os(): return tools diff --git a/coriolis/osmorphing/osmount/redhat.py b/coriolis/osmorphing/osmount/redhat.py index 942430e9..2ac396f9 100644 --- a/coriolis/osmorphing/osmount/redhat.py +++ b/coriolis/osmorphing/osmount/redhat.py @@ -3,8 +3,8 @@ from oslo_log import log as logging -from coriolis.osmorphing.osmount import base from coriolis import utils +from coriolis.osmorphing.osmount import base LOG = logging.getLogger(__name__) @@ -14,8 +14,14 @@ def check_os(self): # make sure the package redhat-lsb-core is installed os_info = utils.get_linux_os_info(self._ssh) if os_info and os_info[0] in [ - 'RedHatEnterpriseServer', 'CentOS', 'OracleServer', - 'rhel', 'centos', 'ol', 'rocky']: + 'RedHatEnterpriseServer', + 'CentOS', + 'OracleServer', + 'rhel', + 'centos', + 'ol', + 'rocky', + ]: return True def setup(self): diff --git a/coriolis/osmorphing/osmount/suse.py b/coriolis/osmorphing/osmount/suse.py index 134376a0..31d4fcb8 100644 --- a/coriolis/osmorphing/osmount/suse.py +++ b/coriolis/osmorphing/osmount/suse.py @@ -3,14 +3,12 @@ from oslo_log import log as logging -from coriolis import exception +from coriolis import exception, utils from coriolis.osmorphing.osmount import base -from coriolis import utils LOG = logging.getLogger(__name__) -SUSE_DISTRO_IDENTIFIERS = [ - 'sles', 'opensuse-leap', 'opensuse-tumbleweed', 'opensuse'] +SUSE_DISTRO_IDENTIFIERS = ['sles', 'opensuse-leap', 'opensuse-tumbleweed', 'opensuse'] SSHD_CONFIG_PATH = "/etc/ssh/sshd_config" USR_SSHD_CONFIG_PATH = "/usr/etc/ssh/sshd_config" @@ -24,22 +22,22 @@ def check_os(self): def _allow_ssh_env_vars(self): if not utils.test_ssh_path(self._ssh, SSHD_CONFIG_PATH): - self._exec_cmd( - "sudo cp %s %s" % (USR_SSHD_CONFIG_PATH, SSHD_CONFIG_PATH)) - self._exec_cmd( - 'sudo sed -i -e "\\$aAcceptEnv *" %s' % SSHD_CONFIG_PATH) + self._exec_cmd("sudo cp %s %s" % (USR_SSHD_CONFIG_PATH, SSHD_CONFIG_PATH)) + self._exec_cmd('sudo sed -i -e "\\$aAcceptEnv *" %s' % SSHD_CONFIG_PATH) try: utils.restart_service(self._ssh, "sshd") except exception.CoriolisException: LOG.warning( "Could not restart sshd service. The SSH connection " - "may have been reset during the restart.") + "may have been reset during the restart." + ) return True def setup(self): super(SUSEOSMountTools, self).setup() - retry_ssh_cmd = utils.retry_on_error( - max_attempts=10, sleep_seconds=30)(self._exec_cmd) + retry_ssh_cmd = utils.retry_on_error(max_attempts=10, sleep_seconds=30)( + self._exec_cmd + ) retry_ssh_cmd("sudo -E zypper --non-interactive install lvm2 psmisc") self._exec_cmd("sudo modprobe dm-mod") self._exec_cmd("sudo rm -f /etc/lvm/devices/system.devices") diff --git a/coriolis/osmorphing/osmount/ubuntu.py b/coriolis/osmorphing/osmount/ubuntu.py index ad7b3ec6..ddd9b383 100644 --- a/coriolis/osmorphing/osmount/ubuntu.py +++ b/coriolis/osmorphing/osmount/ubuntu.py @@ -3,8 +3,8 @@ from oslo_log import log as logging -from coriolis.osmorphing.osmount import base from coriolis import utils +from coriolis.osmorphing.osmount import base LOG = logging.getLogger(__name__) @@ -24,15 +24,16 @@ def setup(self): # another package list refresh is happening. # Apart from relying on possibly not-yet-installed tools like `fuser`, # or checking every /proc/*/fd ourselves, we simply retry it: - retry_ssh_cmd = utils.retry_on_error( - max_attempts=10, sleep_seconds=30)(self._exec_cmd) + retry_ssh_cmd = utils.retry_on_error(max_attempts=10, sleep_seconds=30)( + self._exec_cmd + ) retry_ssh_cmd("sudo -E apt-get update -y") # NOTE(aznashwan): in case an unattended upgrade is already happening # and is at the package installation stage (in which case the # /var/lib/dpkg/* locks will be held), we pass a 10-minute timeout: self._exec_cmd( - "sudo -E apt-get -o DPkg::Lock::Timeout=600 " - "install lvm2 psmisc -y") + "sudo -E apt-get -o DPkg::Lock::Timeout=600 install lvm2 psmisc -y" + ) self._exec_cmd("sudo modprobe dm-mod") diff --git a/coriolis/osmorphing/osmount/windows.py b/coriolis/osmorphing/osmount/windows.py index 66375e6e..bb4e44b4 100644 --- a/coriolis/osmorphing/osmount/windows.py +++ b/coriolis/osmorphing/osmount/windows.py @@ -6,10 +6,8 @@ from oslo_log import log as logging -from coriolis import exception +from coriolis import exception, utils, wsman from coriolis.osmorphing.osmount import base -from coriolis import utils -from coriolis import wsman LOG = logging.getLogger(__name__) @@ -21,11 +19,12 @@ def _connect(self): host = connection_info["ip"] port = connection_info.get("port", 5986) self._event_manager.progress_update( - "Connecting to WinRM host: %(host)s:%(port)s" % - {"host": host, "port": port}) + "Connecting to WinRM host: %(host)s:%(port)s" % {"host": host, "port": port} + ) self._conn = wsman.WSManConnection.from_connection_info( - connection_info, self._osmount_operation_timeout) + connection_info, self._osmount_operation_timeout + ) def get_connection(self): return self._conn @@ -33,7 +32,8 @@ def get_connection(self): def check_os(self): try: version_info = self._conn.exec_ps_command( - "(get-ciminstance Win32_OperatingSystem).Caption") + "(get-ciminstance Win32_OperatingSystem).Caption" + ) LOG.debug("Windows version: %s", version_info) return True except exception.NotAuthorized: @@ -41,8 +41,8 @@ def check_os(self): raise except exception.CoriolisException: LOG.debug( - "Failed Windows OSMount OS check: %s", - utils.get_exception_details()) + "Failed Windows OSMount OS check: %s", utils.get_exception_details() + ) pass return False @@ -62,9 +62,13 @@ def _run_diskpart_script(self, script): return self._conn.exec_ps_command("diskpart.exe /s '%s'" % filepath) def _service_disks_with_status( - self, status, service_script_with_id_fmt, skip_on_error=False, - logmsg_fmt="Operating on disk with index '%s'", - disk_ids_to_skip=None): + self, + status, + service_script_with_id_fmt, + skip_on_error=False, + logmsg_fmt="Operating on disk with index '%s'", + disk_ids_to_skip=None, + ): """Executes given service script on detected disks. Uses diskpart.exe to detect all disks with the given 'status', and @@ -83,13 +87,17 @@ def _service_disks_with_status( disk_list = self._run_diskpart_script(disk_list_script) servicable_disk_ids = [ m.group(1) - for m - in - [re.match(search_disk_entry_re, d) - for d in disk_list.split("\r\n")] if m is not None] + for m in [ + re.match(search_disk_entry_re, d) for d in disk_list.split("\r\n") + ] + if m is not None + ] LOG.debug( "Servicing disks with status '%s' (%s) from disk list: %s", - status, servicable_disk_ids, disk_list) + status, + servicable_disk_ids, + disk_list, + ) for disk_id in servicable_disk_ids: if disk_id in disk_ids_to_skip: LOG.warn('Skipping disk with ID: %s', disk_id) @@ -108,8 +116,8 @@ def _service_disks_with_status( LOG.warn( "Exception ocurred while servicing disk '%s' " "with status '%s'.Skipping running script '%s'" - ". Error message: %s" % - (disk_id, status, script, ex)) + ". Error message: %s" % (disk_id, status, script, ex) + ) else: raise break @@ -120,10 +128,13 @@ def _set_foreign_disks_rw_mode(self): # group must be R/W in order to import it (importing one will # trigger the importing of all of them) set_rw_foreign_disk_script_fmt = ( - "SELECT DISK %s\r\nATTRIBUTES DISK CLEAR READONLY\r\nEXIT") + "SELECT DISK %s\r\nATTRIBUTES DISK CLEAR READONLY\r\nEXIT" + ) self._service_disks_with_status( - "Foreign", set_rw_foreign_disk_script_fmt, - logmsg_fmt="Clearing R/O flag on foreign disk with ID '%s'.") + "Foreign", + set_rw_foreign_disk_script_fmt, + logmsg_fmt="Clearing R/O flag on foreign disk with ID '%s'.", + ) def _import_foreign_disks(self): """Imports foreign disks. @@ -134,44 +145,47 @@ def _import_foreign_disks(self): # NOTE: foreign disks are not exposed via the APIs the PowerShell # disk cmdlets use, thus any disk which is foreign is likely # still RO, which is why we must change the RO attribute as well: - import_disk_script_fmt = ( - "SELECT DISK %s\r\nIMPORT\r\nEXIT") + import_disk_script_fmt = "SELECT DISK %s\r\nIMPORT\r\nEXIT" self._service_disks_with_status( - "Foreign", import_disk_script_fmt, - logmsg_fmt="Importing foreign disk with ID '%s'.") + "Foreign", + import_disk_script_fmt, + logmsg_fmt="Importing foreign disk with ID '%s'.", + ) def _bring_disks_online(self, disk_nums=None): if disk_nums is None: disk_nums = self._conn.exec_ps_command( - "(Get-Disk | Where-Object { $_.IsOffline -eq $True }" - ").Number").splitlines() + "(Get-Disk | Where-Object { $_.IsOffline -eq $True }).Number" + ).splitlines() for disk_num in disk_nums: try: - self._conn.exec_ps_command( - f"Set-Disk -IsOffline $False {disk_num}") + self._conn.exec_ps_command(f"Set-Disk -IsOffline $False {disk_num}") except exception.CoriolisException: LOG.warning( f"Failed setting disk {disk_num} online. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) def _set_basic_disks_rw_mode(self): read_only_disk_nums = self._conn.exec_ps_command( - "(Get-Disk | Where-Object { $_.IsReadOnly -eq $True }).Number") + "(Get-Disk | Where-Object { $_.IsReadOnly -eq $True }).Number" + ) for disk_num in read_only_disk_nums.splitlines(): try: - self._conn.exec_ps_command( - f"Set-Disk -IsReadOnly $False {disk_num}") + self._conn.exec_ps_command(f"Set-Disk -IsReadOnly $False {disk_num}") except exception.CoriolisException: LOG.warning( f"Failed setting disk {disk_num} RW flag. Error was: " - f"{utils.get_exception_details()}") + f"{utils.get_exception_details()}" + ) def _get_system_drive(self): return self._conn.exec_ps_command("$env:SystemDrive") def _get_fs_roots(self, fail_if_empty=False): drives = self._conn.exec_ps_command( - "(get-psdrive -PSProvider FileSystem).Root").split(self._conn.EOL) + "(get-psdrive -PSProvider FileSystem).Root" + ).split(self._conn.EOL) if len(drives) == 0 and fail_if_empty: raise exception.CoriolisException("No filesystems found") return drives @@ -179,16 +193,17 @@ def _get_fs_roots(self, fail_if_empty=False): def _bring_nonboot_disks_offline(self, disk_nums=None): if disk_nums is None: disk_nums = self._conn.exec_ps_command( - "(Get-Disk | Where-Object { $_.IsBoot -eq $False }" - ").Number").splitlines() + "(Get-Disk | Where-Object { $_.IsBoot -eq $False }).Number" + ).splitlines() for disk_num in disk_nums: try: - self._conn.exec_ps_command( - "Set-Disk -IsOffline $True %s" % disk_num) + self._conn.exec_ps_command("Set-Disk -IsOffline $True %s" % disk_num) except exception.CoriolisException: LOG.warning( "Failed setting disk %s offline. Error was: %s", - disk_num, utils.get_exception_details()) + disk_num, + utils.get_exception_details(), + ) def _rebring_disks_online(self, disk_nums=None): self._bring_nonboot_disks_offline(disk_nums=disk_nums) @@ -199,7 +214,8 @@ def _set_volumes_drive_letter(self): partitions = self._conn.exec_ps_command( 'Get-Partition | Where-Object { $_.Type -eq "Basic" -and ' '$_.NoDefaultDriveLetter -eq $True } | Select-Object -Property ' - 'DiskNumber,PartitionNumber') + 'DiskNumber,PartitionNumber' + ) if partitions: LOG.debug(f"Partitions without default drive letter: {partitions}") for part in partitions.splitlines(): @@ -214,13 +230,15 @@ def _set_volumes_drive_letter(self): try: self._conn.exec_ps_command( f'Set-Partition -NoDefaultDriveLetter $False ' - f'-DiskNumber {disk_num} -PartitionNumber {part_num}') + f'-DiskNumber {disk_num} -PartitionNumber {part_num}' + ) disk_nums.append(disk_num) except exception.CoriolisException: LOG.warning( f"Failed setting default drive letter on partition " f"number '{part_num}' of disk number '{disk_num}'. " - f"Error was: {utils.get_exception_details()}") + f"Error was: {utils.get_exception_details()}" + ) self._rebring_disks_online(disk_nums=disk_nums) def mount_os(self): @@ -228,7 +246,8 @@ def mount_os(self): self._bring_disks_online() self._set_volumes_drive_letter() fs_roots = utils.retry_on_error(sleep_seconds=5)(self._get_fs_roots)( - fail_if_empty=True) + fail_if_empty=True + ) system_drive = self._get_system_drive() for fs_root in [r for r in fs_roots if not r[:-1] == system_drive]: diff --git a/coriolis/osmorphing/redhat.py b/coriolis/osmorphing/redhat.py index b7ac1449..fd67da81 100644 --- a/coriolis/osmorphing/redhat.py +++ b/coriolis/osmorphing/redhat.py @@ -7,11 +7,10 @@ from oslo_log import log as logging -from coriolis import exception +from coriolis import exception, utils from coriolis.osmorphing import base from coriolis.osmorphing.osdetect import centos as centos_detect from coriolis.osmorphing.osdetect import redhat as redhat_detect -from coriolis import utils RED_HAT_DISTRO_IDENTIFIER = redhat_detect.RED_HAT_DISTRO_IDENTIFIER @@ -46,19 +45,33 @@ class BaseRedHatMorphingTools(base.BaseLinuxOSMorphingTools): @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - RED_HAT_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] != (RED_HAT_DISTRO_IDENTIFIER): return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=6) - - def __init__(self, conn, os_root_dir, os_root_dev, - hypervisor, event_manager, detected_os_info, - osmorphing_parameters, operation_timeout=None): - super( - BaseRedHatMorphingTools, self).__init__( - conn, os_root_dir, os_root_dev, hypervisor, event_manager, - detected_os_info, osmorphing_parameters, operation_timeout) + detected_os_info['release_version'], minimum=6 + ) + + def __init__( + self, + conn, + os_root_dir, + os_root_dev, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout=None, + ): + super(BaseRedHatMorphingTools, self).__init__( + conn, + os_root_dir, + os_root_dev, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout, + ) def disable_predictable_nic_names(self): cmd = 'grubby --update-kernel=ALL --args="%s"' @@ -87,8 +100,8 @@ def _get_grub2_cfg_location(self): if self._test_path_chroot(uefi_cfg): return uefi_cfg raise Exception( - "could not determine grub location." - " boot partition not mounted?") + "could not determine grub location. boot partition not mounted?" + ) def _has_systemd(self): try: @@ -115,8 +128,7 @@ def _set_dhcp_net_config(self, ifcfgs_ethernet): self._write_config_file(ifcfg_file, ifcfg) network_cfg_file = "etc/sysconfig/network" - network_cfg = self._read_config_file(network_cfg_file, - check_exists=True) + network_cfg = self._read_config_file(network_cfg_file, check_exists=True) if "GATEWAY" in network_cfg: del network_cfg["GATEWAY"] self._write_config_file(network_cfg_file, network_cfg) @@ -126,48 +138,48 @@ def _write_nic_configs(self, nics_info): dev_name = "eth%d" % idx cfg_path = "etc/sysconfig/network-scripts/ifcfg-%s" % dev_name if self._test_path(cfg_path): - self._exec_cmd_chroot( - "cp %s %s.bak" % (cfg_path, cfg_path) - ) + self._exec_cmd_chroot("cp %s %s.bak" % (cfg_path, cfg_path)) self._write_file_sudo( cfg_path, - IFCFG_TEMPLATE % { + IFCFG_TEMPLATE + % { "device_name": dev_name, - }) + }, + ) def _comment_keys_from_ifcfg_files( - self, keys, interfaces=None, backup_file_suffix=".bak"): - """ Comments the provided list of keys from all 'ifcfg-*' files. + self, keys, interfaces=None, backup_file_suffix=".bak" + ): + """Comments the provided list of keys from all 'ifcfg-*' files. Optinally skips ifcfg files for interfaces not listed in 'interfaces'. """ if not interfaces: interfaces = [] - scripts_dir = os.path.join( - self._os_root_dir, "etc/sysconfig/network-scripts") + scripts_dir = os.path.join(self._os_root_dir, "etc/sysconfig/network-scripts") all_ifcfg_files = utils.list_ssh_dir(self._ssh, scripts_dir) regex = "^(ifcfg-[a-z0-9]+)$" for ifcfgf in all_ifcfg_files: if not re.match(regex, ifcfgf): - LOG.debug( - "Skipping ifcfg file with unknown filename '%s'." % - ifcfgf) + LOG.debug("Skipping ifcfg file with unknown filename '%s'." % ifcfgf) continue if interfaces and not any([i in ifcfgf for i in interfaces]): LOG.debug( "Skipping ifcfg file '%s' as it's not for one of the " - "requested interfaces (%s)", ifcfgf, interfaces) + "requested interfaces (%s)", + ifcfgf, + interfaces, + ) continue fullpath = os.path.join(scripts_dir, ifcfgf) for key in keys: self._exec_cmd( - "sudo sed -i%s -E -e 's/^(%s=.*)$/# \\1/g' %s" % ( - backup_file_suffix, key, fullpath)) - LOG.debug( - "Commented all %s references from '%s'" % ( - keys, fullpath)) + "sudo sed -i%s -E -e 's/^(%s=.*)$/# \\1/g' %s" + % (backup_file_suffix, key, fullpath) + ) + LOG.debug("Commented all %s references from '%s'" % (keys, fullpath)) def set_net_config(self, nics_info, dhcp): if dhcp: @@ -191,12 +203,13 @@ def _yum_install(self, package_names, enable_repos=[]): try: yum_cmd = 'yum install %s -y%s' % ( " ".join(package_names), - "".join([" --enablerepo=%s" % r for r in enable_repos])) + "".join([" --enablerepo=%s" % r for r in enable_repos]), + ) self._exec_cmd_chroot(yum_cmd) except exception.CoriolisException as err: raise exception.FailedPackageInstallationException( - package_names=package_names, package_manager='yum', - error=str(err)) from err + package_names=package_names, package_manager='yum', error=str(err) + ) from err def _yum_uninstall(self, package_names): try: @@ -205,8 +218,8 @@ def _yum_uninstall(self, package_names): self._exec_cmd_chroot(yum_cmd) except exception.CoriolisException as err: raise exception.FailedPackageUninstallationException( - package_names=package_names, package_manager='yum', - error=str(err)) from err + package_names=package_names, package_manager='yum', error=str(err) + ) from err def _yum_clean_all(self): self._exec_cmd_chroot("yum clean all") @@ -233,16 +246,14 @@ def _find_yum_repos(self, repos_to_enable=[]): reposdir_path = 'etc/yum.repos.d' - repofiles = [ - f for f in self._list_dir(reposdir_path) if f.endswith('.repo')] + repofiles = [f for f in self._list_dir(reposdir_path) if f.endswith('.repo')] installed_repos = [] for file in repofiles: path = os.path.join(reposdir_path, file) try: content = self._read_file_sudo(path) except Exception as e: - LOG.warning( - "Could not read yum repository file %s: %s", path, e) + LOG.warning("Could not read yum repository file %s: %s", path, e) continue for line in content.splitlines(): m = re.match(r'^\[(.+)\]$', line) @@ -257,7 +268,9 @@ def _find_yum_repos(self, repos_to_enable=[]): else: LOG.warn( "Could not find yum repository while searching for " - "repositories to enable: %s.", repo) + "repositories to enable: %s.", + repo, + ) return found_repos @@ -265,8 +278,7 @@ def _get_repos_to_enable(self): return [] def pre_packages_install(self, package_names): - super(BaseRedHatMorphingTools, self).pre_packages_install( - package_names) + super(BaseRedHatMorphingTools, self).pre_packages_install(package_names) self._yum_clean_all() if 'grubby' not in self.installed_packages: self._yum_install(['grubby']) @@ -276,8 +288,7 @@ def pre_packages_install(self, package_names): def post_packages_install(self, package_names): self._configure_cloud_init() self._run_dracut() - super(BaseRedHatMorphingTools, self).post_packages_install( - package_names) + super(BaseRedHatMorphingTools, self).post_packages_install(package_names) def install_packages(self, package_names): enable_repos = self._get_repos_to_enable() @@ -291,8 +302,7 @@ def _run_dracut(self): def _set_network_nozeroconf_config(self): network_cfg_file = "etc/sysconfig/network" - network_cfg = self._read_config_file(network_cfg_file, - check_exists=True) + network_cfg = self._read_config_file(network_cfg_file, check_exists=True) network_cfg["NOZEROCONF"] = "yes" self._write_config_file(network_cfg_file, network_cfg) @@ -301,5 +311,4 @@ def _write_config_file(self, chroot_path, config_data): self._write_file_sudo(chroot_path, content) def _get_config_file_content(self, config): - return "%s\n" % "\n".join( - ['%s="%s"' % (k, v) for k, v in config.items()]) + return "%s\n" % "\n".join(['%s="%s"' % (k, v) for k, v in config.items()]) diff --git a/coriolis/osmorphing/rocky.py b/coriolis/osmorphing/rocky.py index 00a6384c..224501d0 100644 --- a/coriolis/osmorphing/rocky.py +++ b/coriolis/osmorphing/rocky.py @@ -4,18 +4,16 @@ from coriolis.osmorphing import centos from coriolis.osmorphing.osdetect import rocky as rocky_osdetect - ROCKY_LINUX_DISTRO_IDENTIFIER = rocky_osdetect.ROCKY_LINUX_DISTRO_IDENTIFIER class BaseRockyLinuxMorphingTools(centos.BaseCentOSMorphingTools): - UEFI_GRUB_LOCATION = "/boot/efi/EFI/rocky" @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - ROCKY_LINUX_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] != (ROCKY_LINUX_DISTRO_IDENTIFIER): return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=8) + detected_os_info['release_version'], minimum=8 + ) diff --git a/coriolis/osmorphing/suse.py b/coriolis/osmorphing/suse.py index 6c4fd8a6..f1b3d3af 100644 --- a/coriolis/osmorphing/suse.py +++ b/coriolis/osmorphing/suse.py @@ -8,10 +8,9 @@ from oslo_log import log as logging -from coriolis import exception +from coriolis import exception, utils from coriolis.osmorphing import base from coriolis.osmorphing.osdetect import suse as suse_detect -from coriolis import utils LOG = logging.getLogger(__name__) @@ -19,23 +18,26 @@ SLES_DISTRO_IDENTIFIER = suse_detect.SLES_DISTRO_IDENTIFIER OPENSUSE_DISTRO_IDENTIFIER = suse_detect.OPENSUSE_DISTRO_IDENTIFIER OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER = ( - suse_detect.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER) + suse_detect.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER +) CLOUD_TOOLS_REPO_URI_FORMAT = ( - "https://download.opensuse.org/repositories/Cloud:/Tools/%s%s") + "https://download.opensuse.org/repositories/Cloud:/Tools/%s%s" +) CLOUD_TOOLS_REPO_URI_VERSION_ONLY_FORMAT = ( - "https://download.opensuse.org/repositories/Cloud:/Tools/%s/") + "https://download.opensuse.org/repositories/Cloud:/Tools/%s/" +) CLOUD_TOOLS_NEW_URL_MINIMUM_VERSION = 16 class BaseSUSEMorphingTools(base.BaseLinuxOSMorphingTools): - BIOS_GRUB_LOCATION = "/boot/grub2" UEFI_GRUB_LOCATION = "/boot/efi/EFI/suse" @classmethod def get_required_detected_os_info_fields(cls): common_fields = super( - BaseSUSEMorphingTools, cls).get_required_detected_os_info_fields() + BaseSUSEMorphingTools, cls + ).get_required_detected_os_info_fields() fields = copy.deepcopy(common_fields) fields.append(DETECTED_SUSE_RELEASE_FIELD_NAME) return fields @@ -43,8 +45,7 @@ def get_required_detected_os_info_fields(cls): @classmethod def check_os_supported(cls, detected_os_info): distro = detected_os_info['distribution_name'] - if distro not in ( - SLES_DISTRO_IDENTIFIER, OPENSUSE_DISTRO_IDENTIFIER): + if distro not in (SLES_DISTRO_IDENTIFIER, OPENSUSE_DISTRO_IDENTIFIER): return False version = detected_os_info['release_version'] @@ -52,11 +53,9 @@ def check_os_supported(cls, detected_os_info): if version == OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER: return True else: - return cls._version_supported_util( - version, minimum=15) + return cls._version_supported_util(version, minimum=15) elif distro == SLES_DISTRO_IDENTIFIER: - return cls._version_supported_util( - version, minimum=12) + return cls._version_supported_util(version, minimum=12) return False @@ -91,8 +90,8 @@ def _get_grub2_cfg_location(self): if self._test_path_chroot(bios_cfg): return bios_cfg raise Exception( - "could not determine grub location." - " boot partition not mounted?") + "could not determine grub location. boot partition not mounted?" + ) def _run_dracut(self): self._exec_cmd_chroot("dracut --regenerate-all -f") @@ -103,21 +102,22 @@ def _run_mkinitrd(self): # NOTE: on SLES<12, the `mkinitrd` executable # must be run with no arguments: self._exec_cmd_chroot("mkinitrd") - self._event_manager.progress_update( - "Successfully rebuilt initrds") + self._event_manager.progress_update("Successfully rebuilt initrds") except Exception: # NOTE: old version of `mkinitrd` can error out due to # incompatibilities with sysfs/procfs: self._event_manager.progress_update( - "Error occurred while rebuilding initrds, skipping") + "Error occurred while rebuilding initrds, skipping" + ) LOG.warn( - "Exception occured while rebuilding SLES initrds:\n%s" % ( - utils.get_exception_details())) + "Exception occured while rebuilding SLES initrds:\n%s" + % (utils.get_exception_details()) + ) def _rebuild_initrds(self): if self._version_supported_util( - self._detected_os_info['release_version'], - minimum=0, maximum=12): + self._detected_os_info['release_version'], minimum=0, maximum=12 + ): self._run_mkinitrd() else: self._run_dracut() @@ -135,62 +135,69 @@ def post_packages_install(self, package_names): super(BaseSUSEMorphingTools, self).post_packages_install(package_names) def _enable_sles_module(self, module): - available_modules = self._exec_cmd_chroot( - "SUSEConnect --list-extensions") + available_modules = self._exec_cmd_chroot("SUSEConnect --list-extensions") module_match = re.search("%s.*" % module, available_modules) try: module_path = module_match.group(0) - self._event_manager.progress_update( - "Enabling module: %s" % module_path) + self._event_manager.progress_update("Enabling module: %s" % module_path) conf = "/etc/zypp/zypp.conf" self._exec_cmd_chroot("cp %s %s.tmp" % (conf, conf)) self._exec_cmd_chroot( - "sed -i -e 's/^gpgcheck.*//g' -e '$ a\gpgcheck = off' %s" % ( - conf)) + "sed -i -e 's/^gpgcheck.*//g' -e '$ a\gpgcheck = off' %s" % (conf) + ) self._exec_cmd_chroot("SUSEConnect -p %s" % module_path) self._exec_cmd_chroot("mv -f %s.tmp %s" % (conf, conf)) - self._exec_cmd_chroot( - "zypper --non-interactive --no-gpg-checks refresh") + self._exec_cmd_chroot("zypper --non-interactive --no-gpg-checks refresh") except Exception as err: raise exception.CoriolisException( "Failed to activate SLES module: %s. Please check whether the " "SUSE system registration is still valid on the source VM " - "and retry. Review logs for more details. Error was: %s" % - (module, str(err))) from err + "and retry. Review logs for more details. Error was: %s" + % (module, str(err)) + ) from err def _add_cloud_tools_repo(self): if self._version == OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER: repo = CLOUD_TOOLS_REPO_URI_FORMAT % ( - self._detected_os_info[ - DETECTED_SUSE_RELEASE_FIELD_NAME].replace(" ", "_"), - "") + self._detected_os_info[DETECTED_SUSE_RELEASE_FIELD_NAME].replace( + " ", "_" + ), + "", + ) elif self._version_supported_util( - self._version, minimum=CLOUD_TOOLS_NEW_URL_MINIMUM_VERSION): + self._version, minimum=CLOUD_TOOLS_NEW_URL_MINIMUM_VERSION + ): repo = CLOUD_TOOLS_REPO_URI_VERSION_ONLY_FORMAT % self._version else: repo = CLOUD_TOOLS_REPO_URI_FORMAT % ( - self._detected_os_info[ - DETECTED_SUSE_RELEASE_FIELD_NAME].replace(" ", "_"), - "_%s" % self._version) + self._detected_os_info[DETECTED_SUSE_RELEASE_FIELD_NAME].replace( + " ", "_" + ), + "_%s" % self._version, + ) try: self._add_repo(repo, 'Cloud-Tools') except Exception: LOG.warning( "Failed to add Cloud-Tools repo '%s'. If custom " "repositories are configured on the target, this may be " - "safely ignored. Error was: %s", repo, - utils.get_exception_details()) + "safely ignored. Error was: %s", + repo, + utils.get_exception_details(), + ) self._event_manager.progress_update( "Warning: failed to add Cloud-Tools repo '%s'. If the " "required packages are available in already-configured " "repositories, the migration may still succeed. If not, " "please ensure the worker has internet access or " - "appropriate custom repositories are set up." % repo) + "appropriate custom repositories are set up." % repo + ) def _get_repos(self): repos = {} repos_list = self._exec_cmd_chroot( - "zypper repos -u | awk -F '|' '/^\s[0-9]+/ {print $2 $7}'") + "zypper repos -u | awk -F '|' '/^\s[0-9]+/ {print $2 $7}'" + ) for repo in repos_list.splitlines(): alias, uri = repo.strip().split() repos[alias] = uri @@ -202,30 +209,36 @@ def _add_repo(self, uri, alias): if repos.get(alias): if repos[alias] == uri: LOG.debug( - 'Repo with alias %s already exists and has the same ' - 'URI. Enabling', alias) - self._event_manager.progress_update( - "Enabling repository: %s" % alias) + 'Repo with alias %s already exists and has the same URI. Enabling', + alias, + ) + self._event_manager.progress_update("Enabling repository: %s" % alias) self._exec_cmd_chroot( - 'zypper --non-interactive modifyrepo -e %s' % alias) + 'zypper --non-interactive modifyrepo -e %s' % alias + ) self._exec_cmd_chroot( - "zypper --non-interactive --no-gpg-checks refresh") + "zypper --non-interactive --no-gpg-checks refresh" + ) return else: - LOG.debug('Repo with alias %s already exists, but has a ' - 'different URI. Renaming alias', alias) + LOG.debug( + 'Repo with alias %s already exists, but has a ' + 'different URI. Renaming alias', + alias, + ) alias = "%s%s" % (alias, str(uuid.uuid4())) self._event_manager.progress_update("Adding repository: %s" % alias) try: self._exec_cmd_chroot( - "zypper --non-interactive addrepo -f %s %s" % (uri, alias)) - self._exec_cmd_chroot( - "zypper --non-interactive --no-gpg-checks refresh") + "zypper --non-interactive addrepo -f %s %s" % (uri, alias) + ) + self._exec_cmd_chroot("zypper --non-interactive --no-gpg-checks refresh") except Exception as err: raise exception.CoriolisException( "Failed to add %s repo: %s. Please review logs" - " for more details." % (alias, uri)) from err + " for more details." % (alias, uri) + ) from err def install_packages(self, package_names): try: @@ -234,17 +247,19 @@ def install_packages(self, package_names): ) except exception.CoriolisException as err: raise exception.FailedPackageInstallationException( - package_names=package_names, package_manager='zypper', - error=str(err)) from err + package_names=package_names, package_manager='zypper', error=str(err) + ) from err def uninstall_packages(self, package_names): try: self._exec_cmd_chroot( - 'zypper --non-interactive remove %s' % - " ".join(package_names)) + 'zypper --non-interactive remove %s' % " ".join(package_names) + ) except Exception: self._event_manager.progress_update( - "Error occured while uninstalling packages. Ignoring") + "Error occured while uninstalling packages. Ignoring" + ) LOG.warn( - "Error occured while uninstalling packages. Ignoring. " - "Exception:\n%s", utils.get_exception_details()) + "Error occured while uninstalling packages. Ignoring. Exception:\n%s", + utils.get_exception_details(), + ) diff --git a/coriolis/osmorphing/ubuntu.py b/coriolis/osmorphing/ubuntu.py index b310793c..9110946d 100644 --- a/coriolis/osmorphing/ubuntu.py +++ b/coriolis/osmorphing/ubuntu.py @@ -3,25 +3,22 @@ import copy import io -import yaml +import yaml from oslo_log import log as logging from coriolis.osmorphing import debian from coriolis.osmorphing.osdetect import ubuntu as ubuntu_osdetect - LOG = logging.getLogger(__name__) UBUNTU_DISTRO_IDENTIFIER = ubuntu_osdetect.UBUNTU_DISTRO_IDENTIFIER class BaseUbuntuMorphingTools(debian.BaseDebianMorphingTools): - @classmethod def check_os_supported(cls, detected_os_info): - if detected_os_info['distribution_name'] != ( - UBUNTU_DISTRO_IDENTIFIER): + if detected_os_info['distribution_name'] != (UBUNTU_DISTRO_IDENTIFIER): return False version, subversion = detected_os_info['release_version'].split('.', 1) @@ -29,15 +26,18 @@ def check_os_supported(cls, detected_os_info): LOG.warning( "Detected Ubuntu release version " f"'{detected_os_info['release_version']}' is not an LTS one. " - "Coriolis only supports morphing Ubuntu LTS versions.") + "Coriolis only supports morphing Ubuntu LTS versions." + ) return False return cls._version_supported_util( - detected_os_info['release_version'], minimum=12.04) + detected_os_info['release_version'], minimum=12.04 + ) def _set_netplan_ethernet_configs( - self, nics_info, dhcp=False, iface_name_prefix=None): - """ Updates any Ethernet interface configurations in /etc/netplan/* + self, nics_info, dhcp=False, iface_name_prefix=None + ): + """Updates any Ethernet interface configurations in /etc/netplan/* If an 'iface_name_prefix' is given, replaces all interface names consitently throughout all netplan files. @@ -47,63 +47,62 @@ def _set_netplan_ethernet_configs( processed_interfaces = {} for conffile in self._list_dir("etc/netplan"): if not conffile.endswith(".yaml"): - LOG.debug( - "Skipping file '%s' because of missing extension", - conffile) + LOG.debug("Skipping file '%s' because of missing extension", conffile) continue # back existing file up: config_path_chroot = "etc/netplan/%s" % conffile config_path = "%s/%s" % (self._os_root_dir, config_path_chroot) config_backup_path = "%s.bak" % config_path - self._exec_cmd("sudo cp '%s' '%s'" % ( - config_path, config_backup_path)) + self._exec_cmd("sudo cp '%s' '%s'" % (config_path, config_backup_path)) config_contents = self._read_file(config_path_chroot) config_data = yaml.load( - io.StringIO(config_contents.decode("utf-8")), - Loader=yaml.SafeLoader) + io.StringIO(config_contents.decode("utf-8")), Loader=yaml.SafeLoader + ) config_network_data = config_data.get("network") if config_network_data is None: LOG.debug( - "Missing network config in file '%s'. Skipping" % ( - config_path_chroot)) + "Missing network config in file '%s'. Skipping" + % (config_path_chroot) + ) continue config_version = config_network_data.get("version") if not config_version or int(config_version) != 2: LOG.debug( - "Skipping incompatible config version '%s' in file %s" % ( - config_version, config_path_chroot)) + "Skipping incompatible config version '%s' in file %s" + % (config_version, config_path_chroot) + ) continue ethernet_configurations = config_network_data.get("ethernets") if not ethernet_configurations: - LOG.debug( - "No Ethernet configurations in file %s", - config_path_chroot) + LOG.debug("No Ethernet configurations in file %s", config_path_chroot) new_ethernet_configurations = {} set_dhcp = bool(dhcp) for iface_name in ethernet_configurations.keys(): new_iface_name = iface_name - if iface_name_prefix and not iface_name.startswith( - iface_name_prefix): + if iface_name_prefix and not iface_name.startswith(iface_name_prefix): if iface_name in processed_interfaces.keys(): new_iface_name = processed_interfaces[iface_name] LOG.info( "Already processed config for interface '%s'. " "Using previously mapped name (%s)", - iface_name, new_iface_name) + iface_name, + new_iface_name, + ) else: - new_iface_name = "%s%d" % ( - iface_name_prefix, interface_index) + new_iface_name = "%s%d" % (iface_name_prefix, interface_index) interface_index = interface_index + 1 LOG.debug( "Renamed interface '%s' to '%s' in '%s'", - iface_name, new_iface_name, config_path_chroot) - new_config = copy.deepcopy( - ethernet_configurations[iface_name]) + iface_name, + new_iface_name, + config_path_chroot, + ) + new_config = copy.deepcopy(ethernet_configurations[iface_name]) if set_dhcp: new_config["dhcp4"] = True new_config["dhcp6"] = True @@ -113,12 +112,14 @@ def _set_netplan_ethernet_configs( "'%s' in file '%s'.", {iface_name: ethernet_configurations[iface_name]}, {new_iface_name: new_config}, - config_path_chroot) + config_path_chroot, + ) new_ethernet_configurations[new_iface_name] = new_config processed_interfaces[iface_name] = new_iface_name config_data["network"]["ethernets"] = new_ethernet_configurations LOG.info( - "Writing following configuration to '%s': %s" % ( - config_path_chroot, config_data)) + "Writing following configuration to '%s': %s" + % (config_path_chroot, config_data) + ) self._write_file_sudo(config_path_chroot, yaml.dump(config_data)) diff --git a/coriolis/osmorphing/windows.py b/coriolis/osmorphing/windows.py index 0149d4e7..d09a6cc3 100644 --- a/coriolis/osmorphing/windows.py +++ b/coriolis/osmorphing/windows.py @@ -11,11 +11,9 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, utils from coriolis.osmorphing import base from coriolis.osmorphing.osdetect import windows as windows_osdetect -from coriolis import utils LOG = logging.getLogger(__name__) @@ -29,53 +27,44 @@ SERVICES_PATH_FORMAT = "HKLM:\\%s\\ControlSet001\\Services" SERVICE_PATH_FORMAT = "HKLM:\\%s\\ControlSet001\\Services\\%s" RUN_PATH_FORMAT = "HKLM:\\%s\\\Microsoft\\Windows\\CurrentVersion\\Run" -UNINSTALL_PATH_FORMAT = \ - "HKLM:\\%s\\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\*" +UNINSTALL_PATH_FORMAT = "HKLM:\\%s\\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\*" CLOUDBASEINIT_SERVICE_NAME = "cloudbase-init" CLOUDBASE_INIT_DEFAULT_PLUGINS = [ 'cloudbaseinit.plugins.common.mtu.MTUPlugin', - 'cloudbaseinit.plugins.windows.ntpclient' - '.NTPClientPlugin', - 'cloudbaseinit.plugins.common.sethostname' - '.SetHostNamePlugin', - 'cloudbaseinit.plugins.windows.createuser' - '.CreateUserPlugin', - 'cloudbaseinit.plugins.common.networkconfig' - '.NetworkConfigPlugin', - 'cloudbaseinit.plugins.windows.licensing' - '.WindowsLicensingPlugin', - 'cloudbaseinit.plugins.common.sshpublickeys' - '.SetUserSSHPublicKeysPlugin', - 'cloudbaseinit.plugins.windows.extendvolumes' - '.ExtendVolumesPlugin', + 'cloudbaseinit.plugins.windows.ntpclient.NTPClientPlugin', + 'cloudbaseinit.plugins.common.sethostname.SetHostNamePlugin', + 'cloudbaseinit.plugins.windows.createuser.CreateUserPlugin', + 'cloudbaseinit.plugins.common.networkconfig.NetworkConfigPlugin', + 'cloudbaseinit.plugins.windows.licensing.WindowsLicensingPlugin', + 'cloudbaseinit.plugins.common.sshpublickeys.SetUserSSHPublicKeysPlugin', + 'cloudbaseinit.plugins.windows.extendvolumes.ExtendVolumesPlugin', 'cloudbaseinit.plugins.common.userdata.UserDataPlugin', - 'cloudbaseinit.plugins.common.setuserpassword.' - 'SetUserPasswordPlugin', - 'cloudbaseinit.plugins.windows.winrmlistener.' - 'ConfigWinRMListenerPlugin', + 'cloudbaseinit.plugins.common.setuserpassword.SetUserPasswordPlugin', + 'cloudbaseinit.plugins.windows.winrmlistener.ConfigWinRMListenerPlugin', 'cloudbaseinit.plugins.windows.winrmcertificateauth.' 'ConfigWinRMCertificateAuthPlugin', - 'cloudbaseinit.plugins.common.localscripts' - '.LocalScriptsPlugin', + 'cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin', ] CLOUDBASE_INIT_DEFAULT_METADATA_SVCS = [ 'cloudbaseinit.metadata.services.httpservice.HttpService', - 'cloudbaseinit.metadata.services' - '.configdrive.ConfigDriveService', + 'cloudbaseinit.metadata.services.configdrive.ConfigDriveService', 'cloudbaseinit.metadata.services.ec2service.EC2Service', - 'cloudbaseinit.metadata.services' - '.maasservice.MaaSHttpService', + 'cloudbaseinit.metadata.services.maasservice.MaaSHttpService', 'cloudbaseinit.metadata.services.cloudstack.CloudStack', - 'cloudbaseinit.metadata.services' - '.opennebulaservice.OpenNebulaService', + 'cloudbaseinit.metadata.services.opennebulaservice.OpenNebulaService', ] REQUIRED_DETECTED_WINDOWS_OS_FIELDS = [ - "version_number", "edition_id", "installation_type", "product_name"] + "version_number", + "edition_id", + "installation_type", + "product_name", +] INTERFACES_PATH_FORMAT = ( - "HKLM:\\%s\\ControlSet001\\Services\\Tcpip\\Parameters\\Interfaces") + "HKLM:\\%s\\ControlSet001\\Services\\Tcpip\\Parameters\\Interfaces" +) STATIC_IP_SCRIPT_TEMPLATE = """ $ErrorActionPreference = "Stop" @@ -188,7 +177,6 @@ class BaseWindowsMorphingTools(base.BaseOSMorphingTools): - @classmethod def get_required_detected_os_info_fields(cls): base_fields = copy.deepcopy(base.REQUIRED_DETECTED_OS_FIELDS) @@ -203,13 +191,26 @@ def check_os_supported(cls, detected_os_info): return False def __init__( - self, conn, os_root_dir, os_root_device, hypervisor, - event_manager, detected_os_info, osmorphing_parameters, - operation_timeout=None): + self, + conn, + os_root_dir, + os_root_device, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout=None, + ): super(BaseWindowsMorphingTools, self).__init__( - conn, os_root_dir, os_root_device, hypervisor, - event_manager, detected_os_info, osmorphing_parameters, - operation_timeout) + conn, + os_root_dir, + os_root_device, + hypervisor, + event_manager, + detected_os_info, + osmorphing_parameters, + operation_timeout, + ) self._version_number = detected_os_info['version_number'] self._edition_id = detected_os_info['edition_id'] @@ -218,26 +219,25 @@ def __init__( def _get_worker_os_drive_path(self): return self._conn.exec_ps_command( - "(Get-WmiObject Win32_OperatingSystem).SystemDrive") + "(Get-WmiObject Win32_OperatingSystem).SystemDrive" + ) def _get_dism_path(self): - return "%s\\Windows\\System32\\dism.exe" % ( - self._get_worker_os_drive_path()) + return "%s\\Windows\\System32\\dism.exe" % (self._get_worker_os_drive_path()) def _get_sid(self): sid = self._conn.exec_ps_command( "(New-Object System.Security.Principal.NTAccount($ENV:USERNAME))." - "Translate([System.Security.Principal.SecurityIdentifier]).Value") + "Translate([System.Security.Principal.SecurityIdentifier]).Value" + ) LOG.debug("Current user's SID: %s", sid) return sid def _grant_permissions(self, path, user, perm="(OI)(CI)F"): - self._conn.exec_command( - "icacls.exe", [path, "/grant", "%s:%s" % (user, perm)]) + self._conn.exec_command("icacls.exe", [path, "/grant", "%s:%s" % (user, perm)]) def _revoke_permissions(self, path, user): - self._conn.exec_command( - "icacls.exe", [path, "/remove", user]) + self._conn.exec_command("icacls.exe", [path, "/remove", user]) def _load_registry_hive(self, subkey, path): self._conn.exec_command("reg.exe", ["load", subkey, path]) @@ -256,40 +256,49 @@ def _add_dism_driver(self, driver_path): try: return self._conn.exec_command( dism_path, - ["/add-driver", "/image:%s" % self._os_root_dir, - "/driver:\"%s\"" % driver_path, "/recurse", - "/forceunsigned"]) + [ + "/add-driver", + "/image:%s" % self._os_root_dir, + "/driver:\"%s\"" % driver_path, + "/recurse", + "/forceunsigned", + ], + ) except Exception as ex: dism_log_path = "%s\\Windows\\Logs\\DISM\\dism.log" % ( - self._get_worker_os_drive_path()) + self._get_worker_os_drive_path() + ) if self._conn.test_path(dism_log_path): dism_log_contents = self._conn.exec_ps_command( - "Get-Content %s" % dism_log_path) + "Get-Content %s" % dism_log_path + ) LOG.error( "Error occured whilst adding driver '%s' through DISM. " "Contents of '%s': %s", - driver_path, dism_log_path, dism_log_contents) + driver_path, + dism_log_path, + dism_log_contents, + ) else: - LOG.warn( - "Could not find DISM error logs for failure:'%s'", - str(ex)) + LOG.warn("Could not find DISM error logs for failure:'%s'", str(ex)) raise def _mount_disk_image(self, path): LOG.info("Mounting disk image: %s" % path) return self._conn.exec_ps_command( - "(Mount-DiskImage '%s' -PassThru | Get-Volume).DriveLetter" % - path) + "(Mount-DiskImage '%s' -PassThru | Get-Volume).DriveLetter" % path + ) def _dismount_disk_image(self, path): LOG.info("Unmounting disk image: %s" % path) - self._conn.exec_ps_command("Dismount-DiskImage '%s'" % path, - ignore_stdout=True) + self._conn.exec_ps_command("Dismount-DiskImage '%s'" % path, ignore_stdout=True) @utils.retry_on_error() def _expand_archive(self, path, destination, overwrite=True): - LOG.info("Expanding archive \"%(path)s\" in \"%(destination)s\"", - {"path": path, "destination": destination}) + LOG.info( + "Expanding archive \"%(path)s\" in \"%(destination)s\"", + {"path": path, "destination": destination}, + ) if self._conn.test_path(destination): LOG.info("Destination folder %s already exists" % destination) @@ -297,33 +306,37 @@ def _expand_archive(self, path, destination, overwrite=True): if destination.endswith(":\\") or ":\\Windows" in destination: LOG.warn( "Not removing target directory, as it is either the " - "root directory or is within the Windows directory") + "root directory or is within the Windows directory" + ) else: - self._conn.exec_ps_command( - "rm -recurse -force %s" % destination) + self._conn.exec_ps_command("rm -recurse -force %s" % destination) self._conn.exec_ps_command( "Expand-Archive -LiteralPath '%(path)s' " - "-DestinationPath '%(destination)s' -Force" % - {"path": path, "destination": destination}, - ignore_stdout=True) + "-DestinationPath '%(destination)s' -Force" + % {"path": path, "destination": destination}, + ignore_stdout=True, + ) def _set_service_start_mode(self, key_name, service_name, start_mode): - LOG.info("Setting service start mode: %(service_name)s, " - "%(start_mode)s", {"service_name": service_name, - "start_mode": start_mode}) + LOG.info( + "Setting service start mode: %(service_name)s, %(start_mode)s", + {"service_name": service_name, "start_mode": start_mode}, + ) registry_path = SERVICE_PATH_FORMAT % (key_name, service_name) self._conn.exec_ps_command( "Set-ItemProperty -Path '%(path)s' -Name 'Start' -Value " - "%(start_mode)s" % - {"path": registry_path, "start_mode": start_mode}) + "%(start_mode)s" % {"path": registry_path, "start_mode": start_mode} + ) def _set_services_start_mode(self, key_name, service_names, start_mode): - LOG.info("Setting service start mode: %(service_names)s, " - "%(start_mode)s", {"service_names": service_names, - "start_mode": start_mode}) - registry_paths = [SERVICE_PATH_FORMAT % (key_name, service) - for service in service_names] + LOG.info( + "Setting service start mode: %(service_names)s, %(start_mode)s", + {"service_names": service_names, "start_mode": start_mode}, + ) + registry_paths = [ + SERVICE_PATH_FORMAT % (key_name, service) for service in service_names + ] paths_string = ", ".join(f"'{path}'" for path in registry_paths) self._conn.exec_ps_command( f""" @@ -332,13 +345,20 @@ def _set_services_start_mode(self, key_name, service_names, start_mode): Set-ItemProperty -Path $_ -Name 'Start' -Value {start_mode} }} """, # noqa - ignore_stdout=True) + ignore_stdout=True, + ) - def _create_service(self, key_name, service_name, image_path, - display_name, description, - start_mode=SERVICE_START_AUTO, - service_account="LocalSystem", - depends_on=[]): + def _create_service( + self, + key_name, + service_name, + image_path, + display_name, + description, + start_mode=SERVICE_START_AUTO, + service_account="LocalSystem", + depends_on=[], + ): LOG.info("Creating service: %s", service_name) registry_path = SERVICE_PATH_FORMAT % (key_name, service_name) depends_on_ps = "@(%s)" % (",".join(["'%s'" % v for v in depends_on])) @@ -361,12 +381,18 @@ def _create_service(self, key_name, service_name, image_path, "New-ItemProperty -Path '%(path)s' -Name 'Type' -Value " "16 -Type DWord -Force;" "New-ItemProperty -Path '%(path)s' -Name 'ErrorControl' -Value " - "0 -Type DWord -Force" % - {"path": registry_path, "image_path": image_path, - "display_name": display_name, "description": description, - "depends_on": depends_on_ps, "service_account": service_account, - "start_mode": start_mode}, - ignore_stdout=True) + "0 -Type DWord -Force" + % { + "path": registry_path, + "image_path": image_path, + "display_name": display_name, + "description": description, + "depends_on": depends_on_ps, + "service_account": service_account, + "start_mode": start_mode, + }, + ignore_stdout=True, + ) def _delete_startup_entry(self, key_name, service_name): registry_path = RUN_PATH_FORMAT % key_name @@ -398,17 +424,17 @@ def run_user_script(self, user_script): script_path = "$env:TMP\\coriolis_user_script.ps1" try: - utils.write_winrm_file( - self._conn, - script_path, - user_script) + utils.write_winrm_file(self._conn, script_path, user_script) except Exception as err: raise exception.CoriolisException( - "Failed to copy user script to target system.") from err - - cmd = ('$ErrorActionPreference = "Stop"; powershell.exe ' - '-NonInteractive -ExecutionPolicy RemoteSigned ' - '-File "%(script)s" "%(os_root_dir)s"') % { + "Failed to copy user script to target system." + ) from err + + cmd = ( + '$ErrorActionPreference = "Stop"; powershell.exe ' + '-NonInteractive -ExecutionPolicy RemoteSigned ' + '-File "%(script)s" "%(os_root_dir)s"' + ) % { "script": script_path, "os_root_dir": self._os_root_dir, } @@ -416,64 +442,66 @@ def run_user_script(self, user_script): out = self._conn.exec_ps_command(cmd) LOG.debug("User script output: %s" % out) except Exception as err: - raise exception.CoriolisException( - "Failed to run user script.") from err + raise exception.CoriolisException("Failed to run user script.") from err def _disable_cloudbase_init(self): key_name = str(uuid.uuid4()) self._load_registry_hive( "HKLM\\%s" % key_name, - "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir, + ) try: if self._check_cloudbase_init_exists(key_name): - self._event_manager.progress_update( - "Disabling cloudbase-init") + self._event_manager.progress_update("Disabling cloudbase-init") self._set_service_start_mode( - key_name, CLOUDBASEINIT_SERVICE_NAME, - SERVICE_START_DISABLED) + key_name, CLOUDBASEINIT_SERVICE_NAME, SERVICE_START_DISABLED + ) finally: self._unload_registry_hive("HKLM\\%s" % key_name) def _check_cloudbase_init_exists(self, key_name): - reg_service_path = (SERVICE_PATH_FORMAT % - (key_name, CLOUDBASEINIT_SERVICE_NAME)) - return self._conn.exec_ps_command( - "Test-Path %s" % reg_service_path) == "True" + reg_service_path = SERVICE_PATH_FORMAT % (key_name, CLOUDBASEINIT_SERVICE_NAME) + return self._conn.exec_ps_command("Test-Path %s" % reg_service_path) == "True" - def _setup_existing_cbslinit_service(self, key_name, image_path, - service_account="LocalSystem"): - reg_service_path = (SERVICE_PATH_FORMAT % - (key_name, CLOUDBASEINIT_SERVICE_NAME)) + def _setup_existing_cbslinit_service( + self, key_name, image_path, service_account="LocalSystem" + ): + reg_service_path = SERVICE_PATH_FORMAT % (key_name, CLOUDBASEINIT_SERVICE_NAME) self._conn.exec_ps_command( "Set-ItemProperty -Path '%s' -Name 'ImagePath' -Value '%s' " - "-Force" % (reg_service_path, image_path)) + "-Force" % (reg_service_path, image_path) + ) self._conn.exec_ps_command( "Set-ItemProperty -Path '%s' -Name 'ObjectName' " - "-Value '%s' -Force" % (reg_service_path, service_account)) + "-Value '%s' -Force" % (reg_service_path, service_account) + ) - self._set_service_start_mode(key_name, CLOUDBASEINIT_SERVICE_NAME, - SERVICE_START_AUTO) + self._set_service_start_mode( + key_name, CLOUDBASEINIT_SERVICE_NAME, SERVICE_START_AUTO + ) def _get_cbslinit_base_dir(self): return "%sCloudbase-Init" % self._os_root_dir def _get_cbslinit_scripts_dir(self, base_dir): - return ("%s\\LocalScripts" % base_dir) + return "%s\\LocalScripts" % base_dir def _write_local_script(self, base_dir, script_path, priority=50): scripts_dir = self._get_cbslinit_scripts_dir(base_dir) - script = "%s\\%d-%s" % ( - scripts_dir, priority, - os.path.basename(script_path)) + script = "%s\\%d-%s" % (scripts_dir, priority, os.path.basename(script_path)) with open(script_path, 'r') as fd: contents = fd.read() - utils.write_winrm_file( - self._conn, script, contents) - - def _write_cloudbase_init_conf(self, cloudbaseinit_base_dir, - local_base_dir, com_port="COM1", - metadata_services=None, plugins=None): + utils.write_winrm_file(self._conn, script, contents) + + def _write_cloudbase_init_conf( + self, + cloudbaseinit_base_dir, + local_base_dir, + com_port="COM1", + metadata_services=None, + plugins=None, + ): if metadata_services is None: metadata_services = CLOUDBASE_INIT_DEFAULT_METADATA_SVCS @@ -481,16 +509,16 @@ def _write_cloudbase_init_conf(self, cloudbaseinit_base_dir, plugins = CLOUDBASE_INIT_DEFAULT_PLUGINS elif type(plugins) is not list: raise exception.CoriolisException( - "Invalid plugins parameter. Must be list.") + "Invalid plugins parameter. Must be list." + ) LOG.info("Writing Cloudbase-Init configuration files") conf_dir = "%s\\conf" % cloudbaseinit_base_dir - scripts_dir = self._get_cbslinit_scripts_dir( - cloudbaseinit_base_dir) - self._conn.exec_ps_command("mkdir '%s' -Force" % conf_dir, - ignore_stdout=True) - self._conn.exec_ps_command("mkdir '%s' -Force" % scripts_dir, - ignore_stdout=True) + scripts_dir = self._get_cbslinit_scripts_dir(cloudbaseinit_base_dir) + self._conn.exec_ps_command("mkdir '%s' -Force" % conf_dir, ignore_stdout=True) + self._conn.exec_ps_command( + "mkdir '%s' -Force" % scripts_dir, ignore_stdout=True + ) conf_file_path = "%s\\cloudbase-init.conf" % conf_dir @@ -512,66 +540,74 @@ def _write_cloudbase_init_conf(self, cloudbaseinit_base_dir, "debug = true\r\n" "san_policy = OnlineAll\r\n" "metadata_services = %(metadata_services)s\r\n" - "logging_serial_port_settings = %(com_port)s,9600,N,8\r\n" % - {"bin_path": "%s\\Bin" % local_base_dir, - "log_path": "%s\\Log" % local_base_dir, - "scripts_path": "%s\\LocalScripts" % local_base_dir, - "com_port": com_port, - "metadata_services": ",".join(metadata_services), - "plugins": ",".join(plugins)}) - - utils.write_winrm_file( - self._conn, - conf_file_path, - conf_content) + "logging_serial_port_settings = %(com_port)s,9600,N,8\r\n" + % { + "bin_path": "%s\\Bin" % local_base_dir, + "log_path": "%s\\Log" % local_base_dir, + "scripts_path": "%s\\LocalScripts" % local_base_dir, + "com_port": com_port, + "metadata_services": ",".join(metadata_services), + "plugins": ",".join(plugins), + } + ) + + utils.write_winrm_file(self._conn, conf_file_path, conf_content) disks_script = os.path.join( - utils.get_resources_bin_dir(), - "bring-disks-online.ps1") + utils.get_resources_bin_dir(), "bring-disks-online.ps1" + ) - self._write_local_script( - cloudbaseinit_base_dir, disks_script, - priority=99) + self._write_local_script(cloudbaseinit_base_dir, disks_script, priority=99) - def _install_cloudbase_init(self, download_url, - metadata_services=None, enabled_plugins=None, - com_port="COM1"): + def _install_cloudbase_init( + self, + download_url, + metadata_services=None, + enabled_plugins=None, + com_port="COM1", + ): self._event_manager.progress_update("Adding cloudbase-init") cloudbaseinit_base_dir = self._get_cbslinit_base_dir() key_name = str(uuid.uuid4()) self._load_registry_hive( "HKLM\\%s" % key_name, - "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir, + ) try: cloudbaseinit_zip_path = "c:\\cloudbaseinit.zip" self._event_manager.progress_update("Downloading cloudbase-init") - utils.retry_on_error(sleep_seconds=5)( - self._conn.download_file)( - download_url, - cloudbaseinit_zip_path) + utils.retry_on_error(sleep_seconds=5)(self._conn.download_file)( + download_url, cloudbaseinit_zip_path + ) self._event_manager.progress_update("Installing cloudbase-init") - self._expand_archive(cloudbaseinit_zip_path, - cloudbaseinit_base_dir, overwrite=False) + self._expand_archive( + cloudbaseinit_zip_path, cloudbaseinit_base_dir, overwrite=False + ) log_dir = "%s\\Log" % cloudbaseinit_base_dir - self._conn.exec_ps_command("mkdir '%s' -Force" % log_dir, - ignore_stdout=True) + self._conn.exec_ps_command( + "mkdir '%s' -Force" % log_dir, ignore_stdout=True + ) local_base_dir = "C%s" % cloudbaseinit_base_dir[1:] self._write_cloudbase_init_conf( - cloudbaseinit_base_dir, local_base_dir, + cloudbaseinit_base_dir, + local_base_dir, metadata_services=metadata_services, - plugins=enabled_plugins, com_port=com_port) + plugins=enabled_plugins, + com_port=com_port, + ) image_path = ( '"%(path)s\\Bin\\OpenStackService.exe" cloudbase-init ' '"%(path)s\\Python\\Python.exe" ' '"%(path)s\\Python\\Scripts\\cloudbase-init.exe" ' - '--config-file "%(path)s\\conf\\cloudbase-init.conf"' % { - 'path': local_base_dir}) + '--config-file "%(path)s\\conf\\cloudbase-init.conf"' + % {'path': local_base_dir} + ) self._event_manager.progress_update("Enabling cloudbase-init") @@ -583,7 +619,8 @@ def _install_cloudbase_init(self, download_url, service_name=CLOUDBASEINIT_SERVICE_NAME, image_path=image_path, display_name="Cloud Initialization Service", - description="Service wrapper for cloudbase-init") + description="Service wrapper for cloudbase-init", + ) finally: self._unload_registry_hive("HKLM\\%s" % key_name) @@ -594,45 +631,54 @@ def _compile_static_ip_conf_from_registry(self, key_name): interfaces_reg_path = INTERFACES_PATH_FORMAT % key_name interfaces = self._conn.exec_ps_command( "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " - "'[^\\\\]+$').Matches).Value" % interfaces_reg_path) + "'[^\\\\]+$').Matches).Value" % interfaces_reg_path + ) for interface in interfaces.splitlines(): reg_path = '%s\\%s' % (interfaces_reg_path, interface) enable_dhcp = self._conn.exec_ps_command( - "(Get-ItemProperty -Path '%s').EnableDHCP" % reg_path) + "(Get-ItemProperty -Path '%s').EnableDHCP" % reg_path + ) if enable_dhcp == '0': default_gateway = self._conn.exec_ps_command( - "(Get-ItemProperty -Path '%s').DefaultGateway" % reg_path) + "(Get-ItemProperty -Path '%s').DefaultGateway" % reg_path + ) name_server = self._conn.exec_ps_command( - "(Get-ItemProperty -Path '%s').NameServer" % reg_path) + "(Get-ItemProperty -Path '%s').NameServer" % reg_path + ) ip_addresses = self._conn.exec_ps_command( - "(Get-ItemProperty -Path '%s').IPAddress" % reg_path) + "(Get-ItemProperty -Path '%s').IPAddress" % reg_path + ) subnet_masks = self._conn.exec_ps_command( - "(Get-ItemProperty -Path '%s').SubnetMask" % reg_path) + "(Get-ItemProperty -Path '%s').SubnetMask" % reg_path + ) if not (ip_addresses and subnet_masks): LOG.warning( "No IP Address or Subnet Mask found for interface: " - "'%s'" % interface) + "'%s'" % interface + ) continue prefix_lengths = [] for submask in subnet_masks.splitlines(): - prefix_lengths.append( - ipaddress.IPv4Network((0, submask)).prefixlen) + prefix_lengths.append(ipaddress.IPv4Network((0, submask)).prefixlen) ip_info = { "ip_addresses": ip_addresses.splitlines(), "prefix_lengths": prefix_lengths, "default_gateway": default_gateway, - "dns_addresses": name_server} + "dns_addresses": name_server, + } LOG.debug( "Found static IP configuration for interface '%s': " - "%s" % (interface, ip_info)) + "%s" % (interface, ip_info) + ) ips_info.append(ip_info) else: LOG.debug( "Could not find a static IP configuration for interface: " - "'%s'" % interface) + "'%s'" % interface + ) continue return ips_info @@ -651,21 +697,23 @@ def _get_static_nics_info(self, nics_info, ips_info): if not nic_ips: LOG.warning( f"Skipping NIC ('{nic.get('mac_address')}'). It has no " - f"detected IP addresses") + f"detected IP addresses" + ) continue diff = set(nic_ips) - set(reg_ip_addresses) if diff: LOG.warning( f"The IP addresses {list(diff)} found on the source " "VM's NIC were not found in the registry. These IPs will " - "be skipped in the static IP configuration process") - ip_matches = list( - set(reg_ip_addresses).intersection(set(nic_ips))) + "be skipped in the static IP configuration process" + ) + ip_matches = list(set(reg_ip_addresses).intersection(set(nic_ips))) if not ip_matches: LOG.warning( f"Couldn't find any static IP configuration that " f"matches the addresses {list(nic_ips)} of the source " - f"NIC ({nic.get('mac_address')}). Skipping") + f"NIC ({nic.get('mac_address')}). Skipping" + ) continue static_nic['ip_addresses'] = ip_matches static_nics_info.append(static_nic) @@ -679,7 +727,8 @@ def _write_static_ip_script(self, base_dir, nics_info, ips_info): ips_info_dump = json.dumps(ips_info) contents = STATIC_IP_SCRIPT_TEMPLATE % { 'nics_info': base64.b64encode(nics_info_dump.encode()).decode(), - 'ips_info': base64.b64encode(ips_info_dump.encode()).decode()} + 'ips_info': base64.b64encode(ips_info_dump.encode()).decode(), + } utils.write_winrm_file(self._conn, script_path, contents.encode()) def set_net_config(self, nics_info, dhcp): @@ -689,7 +738,8 @@ def set_net_config(self, nics_info, dhcp): key_name = str(uuid.uuid4()) self._load_registry_hive( "HKLM\\%s" % key_name, - "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self._os_root_dir, + ) try: cbslinit_base_dir = self._get_cbslinit_base_dir() ips_info = self._compile_static_ip_conf_from_registry(key_name) @@ -698,11 +748,13 @@ def set_net_config(self, nics_info, dhcp): LOG.debug(f"Detected static NICS info: {static_nics_info}") if static_nics_info: self._write_static_ip_script( - cbslinit_base_dir, static_nics_info, ips_info) + cbslinit_base_dir, static_nics_info, ips_info + ) else: LOG.warning( "No static IP configuration found on the source VM. " - "Static IP configuration will be skipped.") + "Static IP configuration will be skipped." + ) finally: self._unload_registry_hive("HKLM\\%s" % key_name) diff --git a/coriolis/policies/base.py b/coriolis/policies/base.py index a19874e9..b8e3d4a5 100644 --- a/coriolis/policies/base.py +++ b/coriolis/policies/base.py @@ -3,7 +3,6 @@ from oslo_policy import policy - # NOTE: the policy prefix is convened to be the 'service_type' of # the Coriolis endpoint in the Keystone Service Catalog. CORIOLIS_POLICIES_PREFIX = "migration" @@ -11,13 +10,13 @@ # NOTE: the below are default rules which can be fallen back to: POLICY_DEFAULT_RULES = [ policy.RuleDefault( - "admin", - "is_admin:True", - "Default admin rule which is omnipotent."), + "admin", "is_admin:True", "Default admin rule which is omnipotent." + ), policy.RuleDefault( "admin_or_owner", "is_admin:True or project_id:%(project_id)s", - "Default rule for most API accesses."), + "Default rule for most API accesses.", + ), ] diff --git a/coriolis/policies/deployments.py b/coriolis/policies/deployments.py index a24c9d41..63ad0a87 100644 --- a/coriolis/policies/deployments.py +++ b/coriolis/policies/deployments.py @@ -5,14 +5,12 @@ from coriolis.policies import base - DEPLOYMENTS_POLICY_PREFIX = "%s:deployments" % base.CORIOLIS_POLICIES_PREFIX DEPLOYMENTS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_deployments_policy_label(rule_label): - return "%s:%s" % ( - DEPLOYMENTS_POLICY_PREFIX, rule_label) + return "%s:%s" % (DEPLOYMENTS_POLICY_PREFIX, rule_label) DEPLOYMENTS_POLICY_DEFAULT_RULES = [ @@ -20,34 +18,19 @@ def get_deployments_policy_label(rule_label): get_deployments_policy_label('create'), DEPLOYMENTS_POLICY_DEFAULT_RULE, "Create a deployment", - [ - { - "path": "/deployments", - "method": "POST" - } - ] + [{"path": "/deployments", "method": "POST"}], ), policy.DocumentedRuleDefault( get_deployments_policy_label('list'), DEPLOYMENTS_POLICY_DEFAULT_RULE, "List deployments", - [ - { - "path": "/deployments", - "method": "GET" - } - ] + [{"path": "/deployments", "method": "GET"}], ), policy.DocumentedRuleDefault( get_deployments_policy_label('show'), DEPLOYMENTS_POLICY_DEFAULT_RULE, "Show details for a deployment", - [ - { - "path": "/deployment/{deployment_id}", - "method": "GET" - } - ] + [{"path": "/deployment/{deployment_id}", "method": "GET"}], ), # TODO(aznashwan): deployment actions should ideally be # declared in a separate module @@ -55,24 +38,14 @@ def get_deployments_policy_label(rule_label): get_deployments_policy_label('cancel'), DEPLOYMENTS_POLICY_DEFAULT_RULE, "Cancel a running Deployment", - [ - { - "path": "/deployments/{deployment_id}/actions/", - "method": "POST" - } - ] + [{"path": "/deployments/{deployment_id}/actions/", "method": "POST"}], ), policy.DocumentedRuleDefault( get_deployments_policy_label('delete'), DEPLOYMENTS_POLICY_DEFAULT_RULE, "Delete Deployment", - [ - { - "path": "/deployment/{deployment_id}", - "method": "DELETE" - } - ] - ) + [{"path": "/deployment/{deployment_id}", "method": "DELETE"}], + ), ] diff --git a/coriolis/policies/diagnostics.py b/coriolis/policies/diagnostics.py index d2d4bef2..b5dae247 100644 --- a/coriolis/policies/diagnostics.py +++ b/coriolis/policies/diagnostics.py @@ -5,15 +5,12 @@ from coriolis.policies import base - -DIAGNOSTICS_POLICY_PREFIX = "%s:diagnostics" % ( - base.CORIOLIS_POLICIES_PREFIX) +DIAGNOSTICS_POLICY_PREFIX = "%s:diagnostics" % (base.CORIOLIS_POLICIES_PREFIX) DIAGNOSTICS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_diagnostics_policy_label(rule_label): - return "%s:%s" % ( - DIAGNOSTICS_POLICY_PREFIX, rule_label) + return "%s:%s" % (DIAGNOSTICS_POLICY_PREFIX, rule_label) DIAGNOSTICS_POLICY_DEFAULT_RULES = [ @@ -21,12 +18,7 @@ def get_diagnostics_policy_label(rule_label): get_diagnostics_policy_label('get'), DIAGNOSTICS_POLICY_DEFAULT_RULE, "Get diagnostics information from coriolis components", - [ - { - "path": "/diagnostics", - "method": "GET" - } - ] + [{"path": "/diagnostics", "method": "GET"}], ) ] diff --git a/coriolis/policies/endpoints.py b/coriolis/policies/endpoints.py index 17dddf41..fd54493c 100644 --- a/coriolis/policies/endpoints.py +++ b/coriolis/policies/endpoints.py @@ -6,14 +6,12 @@ from coriolis.policies import base - ENDPOINTS_POLICY_PREFIX = "%s:endpoints" % base.CORIOLIS_POLICIES_PREFIX ENDPOINTS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_endpoints_policy_label(rule_label): - return "%s:%s" % ( - ENDPOINTS_POLICY_PREFIX, rule_label) + return "%s:%s" % (ENDPOINTS_POLICY_PREFIX, rule_label) ENDPOINTS_POLICY_DEFAULT_RULES = [ @@ -21,79 +19,44 @@ def get_endpoints_policy_label(rule_label): get_endpoints_policy_label('create'), ENDPOINTS_POLICY_DEFAULT_RULE, "Create an endpoint", - [ - { - "path": "/endpoints", - "method": "POST" - } - ] + [{"path": "/endpoints", "method": "POST"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list'), ENDPOINTS_POLICY_DEFAULT_RULE, "List endpoints", - [ - { - "path": "/endpoints", - "method": "GET" - } - ] + [{"path": "/endpoints", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('show'), ENDPOINTS_POLICY_DEFAULT_RULE, "Show details for endpoint", - [ - { - "path": "/endpoints/{endpoint_id}", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}", "method": "GET"}], ), # TODO(aznashwan): double-check this: policy.DocumentedRuleDefault( get_endpoints_policy_label('update'), ENDPOINTS_POLICY_DEFAULT_RULE, "Update details for endpoint", - [ - { - "path": "/endpoints/{endpoint_id}", - "method": "PUT" - } - ] + [{"path": "/endpoints/{endpoint_id}", "method": "PUT"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('delete'), ENDPOINTS_POLICY_DEFAULT_RULE, "Delete endpoint", - [ - { - "path": "/endpoints/{endpoint_id}", - "method": "DELETE" - } - ] + [{"path": "/endpoints/{endpoint_id}", "method": "DELETE"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('validate_connection'), ENDPOINTS_POLICY_DEFAULT_RULE, "Validate endpoint connection info", - [ - { - "path": "/endpoints/{endpoint_id}/actions", - "method": "POST" - } - ] + [{"path": "/endpoints/{endpoint_id}/actions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_instances'), ENDPOINTS_POLICY_DEFAULT_RULE, "List instances available for migration/replication", - [ - { - "path": "/endpoints/{endpoint_id}/instances", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/instances", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('get_instance'), @@ -102,53 +65,33 @@ def get_endpoints_policy_label(rule_label): [ { "path": "/endpoints/{endpoint_id}/instances/{instance_name}", - "method": "GET" + "method": "GET", } - ] + ], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_networks'), ENDPOINTS_POLICY_DEFAULT_RULE, "List networks available on the given endpoint", - [ - { - "path": "/endpoints/{endpoint_id}/networks", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/networks", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_storage'), ENDPOINTS_POLICY_DEFAULT_RULE, "List storage types available on the given endpoint", - [ - { - "path": "/endpoints/{endpoint_id}/storage", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/storage", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_destination_options'), ENDPOINTS_POLICY_DEFAULT_RULE, "List available destination options for endpoint", - [ - { - "path": "/endpoints/{endpoint_id}/destination-options", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/destination-options", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_source_options'), ENDPOINTS_POLICY_DEFAULT_RULE, "List available source options for endpoint", - [ - { - "path": "/endpoints/{endpoint_id}/source-options", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/source-options", "method": "GET"}], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_source_minion_pool_options'), @@ -157,9 +100,9 @@ def get_endpoints_policy_label(rule_label): [ { "path": "/endpoints/{endpoint_id}/source-minion-pool-options", - "method": "GET" + "method": "GET", } - ] + ], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('list_destination_minion_pool_options'), @@ -167,22 +110,16 @@ def get_endpoints_policy_label(rule_label): "List available destination pool options for endpoint", [ { - "path": ( - "/endpoints/{endpoint_id}/destination-minion-pool-options"), # noqa - "method": "GET" + "path": ("/endpoints/{endpoint_id}/destination-minion-pool-options"), # noqa + "method": "GET", } - ] + ], ), policy.DocumentedRuleDefault( get_endpoints_policy_label('export_inventory'), ENDPOINTS_POLICY_DEFAULT_RULE, "Export VM inventory as CSV for a supported endpoint", - [ - { - "path": "/endpoints/{endpoint_id}/inventory", - "method": "GET" - } - ] + [{"path": "/endpoints/{endpoint_id}/inventory", "method": "GET"}], ), ] diff --git a/coriolis/policies/general.py b/coriolis/policies/general.py index 103827ed..066835d2 100644 --- a/coriolis/policies/general.py +++ b/coriolis/policies/general.py @@ -5,14 +5,12 @@ from coriolis.policies import base - PROVIDERS_POLICY_PREFIX = "%s:providers" % base.CORIOLIS_POLICIES_PREFIX PROVIDERS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_providers_policy_label(rule_label): - return "%s:%s" % ( - PROVIDERS_POLICY_PREFIX, rule_label) + return "%s:%s" % (PROVIDERS_POLICY_PREFIX, rule_label) PROVIDERS_POLICY_DEFAULT_RULES = [ @@ -20,12 +18,7 @@ def get_providers_policy_label(rule_label): get_providers_policy_label('list'), PROVIDERS_POLICY_DEFAULT_RULE, "List available provider plugins and their capabilities", - [ - { - "path": "/providers", - "method": "GET" - } - ] + [{"path": "/providers", "method": "GET"}], ) ] diff --git a/coriolis/policies/minion_pools.py b/coriolis/policies/minion_pools.py index e93259dd..749ef7ea 100644 --- a/coriolis/policies/minion_pools.py +++ b/coriolis/policies/minion_pools.py @@ -6,14 +6,12 @@ from coriolis.policies import base - MINION_POOLS_POLICY_PREFIX = "%s:minion_pools" % base.CORIOLIS_POLICIES_PREFIX MINION_POOLS_DEFAULT_RULE = "rule:admin_or_owner" def get_minion_pools_policy_label(rule_label): - return "%s:%s" % ( - MINION_POOLS_POLICY_PREFIX, rule_label) + return "%s:%s" % (MINION_POOLS_POLICY_PREFIX, rule_label) MINION_POOLS_DEFAULT_RULES = [ @@ -21,90 +19,50 @@ def get_minion_pools_policy_label(rule_label): get_minion_pools_policy_label('create'), MINION_POOLS_DEFAULT_RULE, "Create a minion pool", - [ - { - "path": "/minion_pools", - "method": "POST" - } - ] + [{"path": "/minion_pools", "method": "POST"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('list'), MINION_POOLS_DEFAULT_RULE, "List minion_pools", - [ - { - "path": "/minion_pools", - "method": "GET" - } - ] + [{"path": "/minion_pools", "method": "GET"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('show'), MINION_POOLS_DEFAULT_RULE, "Show details for minion pool", - [ - { - "path": "/minion_pools/{minion_pool_id}", - "method": "GET" - } - ] + [{"path": "/minion_pools/{minion_pool_id}", "method": "GET"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('update'), MINION_POOLS_DEFAULT_RULE, "Update details for minion pool", - [ - { - "path": "/minion_pools/{minion_pool_id}", - "method": "PUT" - } - ] + [{"path": "/minion_pools/{minion_pool_id}", "method": "PUT"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('delete'), MINION_POOLS_DEFAULT_RULE, "Delete minion pool", - [ - { - "path": "/minion_pools/{minion_pool_id}", - "method": "DELETE" - } - ] + [{"path": "/minion_pools/{minion_pool_id}", "method": "DELETE"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('allocate'), MINION_POOLS_DEFAULT_RULE, "Allocate Minion Pool", - [ - { - "path": "/minion_pools/{minion_pool_id}/actions", - "method": "POST" - } - ] + [{"path": "/minion_pools/{minion_pool_id}/actions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('refresh'), MINION_POOLS_DEFAULT_RULE, "Refresh Minion Pool", - [ - { - "path": "/minion_pools/{minion_pool_id}/actions", - "method": "POST" - } - ] + [{"path": "/minion_pools/{minion_pool_id}/actions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_minion_pools_policy_label('deallocate'), MINION_POOLS_DEFAULT_RULE, "Deallocate Minion Pool", - [ - { - "path": "/minion_pools/{minion_pool_id}/actions", - "method": "POST" - } - ] - ) + [{"path": "/minion_pools/{minion_pool_id}/actions", "method": "POST"}], + ), ] diff --git a/coriolis/policies/regions.py b/coriolis/policies/regions.py index cc40e908..df502629 100644 --- a/coriolis/policies/regions.py +++ b/coriolis/policies/regions.py @@ -6,14 +6,12 @@ from coriolis.policies import base - REGIONS_POLICY_PREFIX = "%s:regions" % base.CORIOLIS_POLICIES_PREFIX REGIONS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_regions_policy_label(rule_label): - return "%s:%s" % ( - REGIONS_POLICY_PREFIX, rule_label) + return "%s:%s" % (REGIONS_POLICY_PREFIX, rule_label) REGIONS_POLICY_DEFAULT_RULES = [ @@ -21,57 +19,32 @@ def get_regions_policy_label(rule_label): get_regions_policy_label('create'), REGIONS_POLICY_DEFAULT_RULE, "Create a region", - [ - { - "path": "/regions", - "method": "POST" - } - ] + [{"path": "/regions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_regions_policy_label('list'), REGIONS_POLICY_DEFAULT_RULE, "List regions", - [ - { - "path": "/regions", - "method": "GET" - } - ] + [{"path": "/regions", "method": "GET"}], ), policy.DocumentedRuleDefault( get_regions_policy_label('show'), REGIONS_POLICY_DEFAULT_RULE, "Show details for region", - [ - { - "path": "/regions/{region_id}", - "method": "GET" - } - ] + [{"path": "/regions/{region_id}", "method": "GET"}], ), policy.DocumentedRuleDefault( get_regions_policy_label('update'), REGIONS_POLICY_DEFAULT_RULE, "Update details for region", - [ - { - "path": "/regions/{region_id}", - "method": "PUT" - } - ] + [{"path": "/regions/{region_id}", "method": "PUT"}], ), policy.DocumentedRuleDefault( get_regions_policy_label('delete'), REGIONS_POLICY_DEFAULT_RULE, "Delete region", - [ - { - "path": "/regions/{region_id}", - "method": "DELETE" - } - ] - ) + [{"path": "/regions/{region_id}", "method": "DELETE"}], + ), ] diff --git a/coriolis/policies/services.py b/coriolis/policies/services.py index a2582b55..114cd0e1 100644 --- a/coriolis/policies/services.py +++ b/coriolis/policies/services.py @@ -6,14 +6,12 @@ from coriolis.policies import base - SERVICES_POLICY_PREFIX = "%s:services" % base.CORIOLIS_POLICIES_PREFIX SERVICES_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_services_policy_label(rule_label): - return "%s:%s" % ( - SERVICES_POLICY_PREFIX, rule_label) + return "%s:%s" % (SERVICES_POLICY_PREFIX, rule_label) SERVICES_POLICY_DEFAULT_RULES = [ @@ -21,57 +19,32 @@ def get_services_policy_label(rule_label): get_services_policy_label('create'), SERVICES_POLICY_DEFAULT_RULE, "Create a service", - [ - { - "path": "/services", - "method": "POST" - } - ] + [{"path": "/services", "method": "POST"}], ), policy.DocumentedRuleDefault( get_services_policy_label('list'), SERVICES_POLICY_DEFAULT_RULE, "List services", - [ - { - "path": "/services", - "method": "GET" - } - ] + [{"path": "/services", "method": "GET"}], ), policy.DocumentedRuleDefault( get_services_policy_label('show'), SERVICES_POLICY_DEFAULT_RULE, "Show details for service", - [ - { - "path": "/services/{service_id}", - "method": "GET" - } - ] + [{"path": "/services/{service_id}", "method": "GET"}], ), policy.DocumentedRuleDefault( get_services_policy_label('update'), SERVICES_POLICY_DEFAULT_RULE, "Update details for service", - [ - { - "path": "/services/{service_id}", - "method": "PUT" - } - ] + [{"path": "/services/{service_id}", "method": "PUT"}], ), policy.DocumentedRuleDefault( get_services_policy_label('delete'), SERVICES_POLICY_DEFAULT_RULE, "Delete service", - [ - { - "path": "/services/{service_id}", - "method": "DELETE" - } - ] - ) + [{"path": "/services/{service_id}", "method": "DELETE"}], + ), ] diff --git a/coriolis/policies/transfer_schedules.py b/coriolis/policies/transfer_schedules.py index 40951872..524c9af8 100644 --- a/coriolis/policies/transfer_schedules.py +++ b/coriolis/policies/transfer_schedules.py @@ -5,15 +5,14 @@ from coriolis.policies import base - TRANSFER_SCHEDULES_POLICY_PREFIX = "%s:transfer_schedules" % ( - base.CORIOLIS_POLICIES_PREFIX) + base.CORIOLIS_POLICIES_PREFIX +) TRANSFER_SCHEDULES_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_transfer_schedules_policy_label(rule_label): - return "%s:%s" % ( - TRANSFER_SCHEDULES_POLICY_PREFIX, rule_label) + return "%s:%s" % (TRANSFER_SCHEDULES_POLICY_PREFIX, rule_label) TRANSFER_SCHEDULES_POLICY_DEFAULT_RULES = [ @@ -21,34 +20,19 @@ def get_transfer_schedules_policy_label(rule_label): get_transfer_schedules_policy_label('create'), TRANSFER_SCHEDULES_POLICY_DEFAULT_RULE, "Create a new execution schedule for a given Transfer", - [ - { - "path": "/transfers/{transfer_id}/schedules", - "method": "POST" - } - ] + [{"path": "/transfers/{transfer_id}/schedules", "method": "POST"}], ), policy.DocumentedRuleDefault( get_transfer_schedules_policy_label('list'), TRANSFER_SCHEDULES_POLICY_DEFAULT_RULE, "List execution schedules for a given Transfer", - [ - { - "path": "/transfers/{transfer_id}/schedules", - "method": "GET" - } - ] + [{"path": "/transfers/{transfer_id}/schedules", "method": "GET"}], ), policy.DocumentedRuleDefault( get_transfer_schedules_policy_label('show'), TRANSFER_SCHEDULES_POLICY_DEFAULT_RULE, "Show details for an execution schedule for a given Transfer", - [ - { - "path": "/transfers/{transfer_id}/schedules/{schedule_id}", - "method": "GET" - } - ] + [{"path": "/transfers/{transfer_id}/schedules/{schedule_id}", "method": "GET"}], ), policy.DocumentedRuleDefault( get_transfer_schedules_policy_label('update'), @@ -56,11 +40,10 @@ def get_transfer_schedules_policy_label(rule_label): "Update an existing execution schedule for a given Transfer", [ { - "path": ( - "/transfers/{transfer_id}/schedules/{schedule_id}"), - "method": "PUT" + "path": ("/transfers/{transfer_id}/schedules/{schedule_id}"), + "method": "PUT", } - ] + ], ), policy.DocumentedRuleDefault( get_transfer_schedules_policy_label('delete'), @@ -69,10 +52,10 @@ def get_transfer_schedules_policy_label(rule_label): [ { "path": "/transfers/{transfer_id}/schedules/{schedule_id}", - "method": "DELETE" + "method": "DELETE", } - ] - ) + ], + ), ] diff --git a/coriolis/policies/transfer_tasks_executions.py b/coriolis/policies/transfer_tasks_executions.py index b653149b..589b42cf 100644 --- a/coriolis/policies/transfer_tasks_executions.py +++ b/coriolis/policies/transfer_tasks_executions.py @@ -5,15 +5,14 @@ from coriolis.policies import base - TRANSFER_EXECUTIONS_POLICY_PREFIX = "%s:transfer_executions" % ( - base.CORIOLIS_POLICIES_PREFIX) + base.CORIOLIS_POLICIES_PREFIX +) TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_transfer_executions_policy_label(rule_label): - return "%s:%s" % ( - TRANSFER_EXECUTIONS_POLICY_PREFIX, rule_label) + return "%s:%s" % (TRANSFER_EXECUTIONS_POLICY_PREFIX, rule_label) TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULES = [ @@ -21,23 +20,13 @@ def get_transfer_executions_policy_label(rule_label): get_transfer_executions_policy_label('create'), TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULE, "Create a new execution for a given Transfer", - [ - { - "path": "/transfers/{transfer_id}/executions", - "method": "POST" - } - ] + [{"path": "/transfers/{transfer_id}/executions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_transfer_executions_policy_label('list'), TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULE, "List Executions for a given Transfer", - [ - { - "path": "/transfers/{transfer_id}/executions", - "method": "GET" - } - ] + [{"path": "/transfers/{transfer_id}/executions", "method": "GET"}], ), policy.DocumentedRuleDefault( get_transfer_executions_policy_label('show'), @@ -46,9 +35,9 @@ def get_transfer_executions_policy_label(rule_label): [ { "path": "/transfers/{transfer_id}/executions/{execution_id}", - "method": "GET" + "method": "GET", } - ] + ], ), # TODO(aznashwan): transfer actions should ideally be # declared in a separate module @@ -58,12 +47,10 @@ def get_transfer_executions_policy_label(rule_label): "Cancel a Transfer execution", [ { - "path": ( - "/transfers/{transfer_id}/executions/" - "{execution_id}/actions"), - "method": "POST" + "path": ("/transfers/{transfer_id}/executions/{execution_id}/actions"), + "method": "POST", } - ] + ], ), policy.DocumentedRuleDefault( get_transfer_executions_policy_label('delete'), @@ -72,10 +59,10 @@ def get_transfer_executions_policy_label(rule_label): [ { "path": "/transfers/{transfer_id}/executions/{execution_id}", - "method": "DELETE" + "method": "DELETE", } - ] - ) + ], + ), ] diff --git a/coriolis/policies/transfers.py b/coriolis/policies/transfers.py index 85978d5a..25398e45 100644 --- a/coriolis/policies/transfers.py +++ b/coriolis/policies/transfers.py @@ -5,14 +5,12 @@ from coriolis.policies import base - TRANSFERS_POLICY_PREFIX = "%s:transfers" % base.CORIOLIS_POLICIES_PREFIX TRANSFERS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" def get_transfers_policy_label(rule_label): - return "%s:%s" % ( - TRANSFERS_POLICY_PREFIX, rule_label) + return "%s:%s" % (TRANSFERS_POLICY_PREFIX, rule_label) TRANSFERS_POLICY_DEFAULT_RULES = [ @@ -20,34 +18,19 @@ def get_transfers_policy_label(rule_label): get_transfers_policy_label('create'), TRANSFERS_POLICY_DEFAULT_RULE, "Create a Transfer", - [ - { - "path": "/transfers", - "method": "POST" - } - ] + [{"path": "/transfers", "method": "POST"}], ), policy.DocumentedRuleDefault( get_transfers_policy_label('list'), TRANSFERS_POLICY_DEFAULT_RULE, "List Transfers", - [ - { - "path": "/transfers", - "method": "GET" - } - ] + [{"path": "/transfers", "method": "GET"}], ), policy.DocumentedRuleDefault( get_transfers_policy_label('show'), TRANSFERS_POLICY_DEFAULT_RULE, "Show details for Transfer", - [ - { - "path": "/transfers/{transfer_id}", - "method": "GET" - } - ] + [{"path": "/transfers/{transfer_id}", "method": "GET"}], ), # TODO(aznashwan): transfer actions should ideally be # declared in a separate module @@ -55,36 +38,20 @@ def get_transfers_policy_label(rule_label): get_transfers_policy_label('delete_disks'), TRANSFERS_POLICY_DEFAULT_RULE, "Delete Transfer Disks", - [ - { - "path": "/transfers/{transfer_id}/actions", - "method": "POST" - } - ] + [{"path": "/transfers/{transfer_id}/actions", "method": "POST"}], ), policy.DocumentedRuleDefault( get_transfers_policy_label('delete'), TRANSFERS_POLICY_DEFAULT_RULE, "Delete Transfer", - [ - { - "path": "/transfers/{transfer_id}", - "method": "DELETE" - } - ] + [{"path": "/transfers/{transfer_id}", "method": "DELETE"}], ), policy.DocumentedRuleDefault( get_transfers_policy_label('update'), TRANSFERS_POLICY_DEFAULT_RULE, "Update Transfer", - [ - { - "path": "/transfers/{transfer_id}", - "method": "POST" - } - ] - ) - + [{"path": "/transfers/{transfer_id}", "method": "POST"}], + ), ] diff --git a/coriolis/policy.py b/coriolis/policy.py index 297a4c92..50ca72f6 100644 --- a/coriolis/policy.py +++ b/coriolis/policy.py @@ -7,20 +7,20 @@ from oslo_log import log as logging from oslo_policy import policy -from coriolis import exception -from coriolis.policies import base -from coriolis.policies import deployments -from coriolis.policies import diagnostics -from coriolis.policies import endpoints -from coriolis.policies import general -from coriolis.policies import minion_pools -from coriolis.policies import regions -from coriolis.policies import services -from coriolis.policies import transfer_schedules -from coriolis.policies import transfer_tasks_executions -from coriolis.policies import transfers -from coriolis import utils - +from coriolis import exception, utils +from coriolis.policies import ( + base, + deployments, + diagnostics, + endpoints, + general, + minion_pools, + regions, + services, + transfer_schedules, + transfer_tasks_executions, + transfers, +) LOG = logging.getLogger(__name__) @@ -28,9 +28,18 @@ _ENFORCER = None DEFAULT_POLICIES_MODULES = [ - base, deployments, endpoints, general, transfers, - transfer_schedules, transfer_tasks_executions, diagnostics, regions, - services, minion_pools] + base, + deployments, + endpoints, + general, + transfers, + transfer_schedules, + transfer_tasks_executions, + diagnostics, + regions, + services, + minion_pools, +] def reset(): @@ -51,8 +60,9 @@ def init(): def register_rules(enforcer): - enforcer.register_defaults(itertools.chain(*[ - m.list_rules() for m in DEFAULT_POLICIES_MODULES])) + enforcer.register_defaults( + itertools.chain(*[m.list_rules() for m in DEFAULT_POLICIES_MODULES]) + ) def get_enforcer(): @@ -60,9 +70,8 @@ def get_enforcer(): return _ENFORCER -def check_policy_for_context( - context, action, target, exc=None, do_raise=True): - """ Checks the validity of the given action of the given target based on +def check_policy_for_context(context, action, target, exc=None, do_raise=True): + """Checks the validity of the given action of the given target based on set policies. On success, returns a value where bool(val) == True. On failure and if `do_raise` is False, returns False. @@ -75,15 +84,19 @@ def check_policy_for_context( exc = exception.PolicyNotAuthorized try: result = _ENFORCER.authorize( - action, target, credentials, - do_raise=do_raise, exc=exc, action=action) + action, target, credentials, do_raise=do_raise, exc=exc, action=action + ) except Exception as ex: LOG.debug( "Policy check for '%(action)s' with target '%(target)s' failed " - "with credentials: %(credentials)s.\nException: '%(trace)s'", { - 'action': action, 'target': target, - 'credentials': credentials, 'trace': - utils.get_exception_details()}) + "with credentials: %(credentials)s.\nException: '%(trace)s'", + { + 'action': action, + 'target': target, + 'credentials': credentials, + 'trace': utils.get_exception_details(), + }, + ) raise exc(str(ex)) diff --git a/coriolis/providers/api.py b/coriolis/providers/api.py index 5da606bc..0d2e23ea 100644 --- a/coriolis/providers/api.py +++ b/coriolis/providers/api.py @@ -13,4 +13,5 @@ def get_available_providers(self, ctxt): def get_provider_schemas(self, ctxt, platform_name, provider_type): return self._rpc_client.get_provider_schemas( - ctxt, platform_name, int(provider_type)) + ctxt, platform_name, int(provider_type) + ) diff --git a/coriolis/providers/backup_writers.py b/coriolis/providers/backup_writers.py index 811ca3d8..c7b468ad 100644 --- a/coriolis/providers/backup_writers.py +++ b/coriolis/providers/backup_writers.py @@ -15,27 +15,28 @@ import time import uuid +import paramiko from oslo_config import cfg from oslo_log import log as logging -import paramiko from six import with_metaclass -from coriolis import constants -from coriolis import data_transfer -from coriolis import exception +from coriolis import constants, data_transfer, exception, utils from coriolis.providers import provider_utils -from coriolis import utils CONF = cfg.CONF opts = [ - cfg.BoolOpt('compress_transfers', - default=True, - help='Use compression if possible during disk transfers'), - cfg.IntOpt('disk_checksum_timeout', - default=3600, - help='Maximum number of seconds to wait for a disk checksum ' - 'job to complete on the backup writer. Larger disks may ' - 'require a higher value.'), + cfg.BoolOpt( + 'compress_transfers', + default=True, + help='Use compression if possible during disk transfers', + ), + cfg.IntOpt( + 'disk_checksum_timeout', + default=3600, + help='Maximum number of seconds to wait for a disk checksum ' + 'job to complete on the backup writer. Larger disks may ' + 'require a higher value.', + ), ] CONF.register_opts(opts) _CORIOLIS_HTTP_WRITER_CMD = "coriolis-writer" @@ -45,11 +46,7 @@ BACKUP_WRITER_HTTP = "http_backup_writer" BACKUP_WRITER_FILE = "file_backup_writer" -BACKUP_WRITERS = [ - BACKUP_WRITER_SSH, - BACKUP_WRITER_HTTP, - BACKUP_WRITER_FILE -] +BACKUP_WRITERS = [BACKUP_WRITER_SSH, BACKUP_WRITER_HTTP, BACKUP_WRITER_FILE] _WRITER_ERR_MAP = { -1: "ERR_MORE_MSG", @@ -86,8 +83,9 @@ def _disable_lvm2_lvmetad(ssh): if utils.test_ssh_path(ssh, cfg): utils.exec_ssh_cmd( ssh, - 'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" ' - '%s' % cfg, get_pty=True) + 'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" %s' % cfg, + get_pty=True, + ) # NOTE: lvm2-lvmetad is the name of the lvmetad service # on both debian and RHEL based systems. It needs to be stopped # before we begin disk replication. We disable it in the config @@ -95,12 +93,14 @@ def _disable_lvm2_lvmetad(ssh): # a dependency. As the service may not actually exist, even though # the config is present, we ignore errors when stopping it. utils.ignore_exceptions(utils.exec_ssh_cmd)( - ssh, "sudo service lvm2-lvmetad stop", get_pty=True) + ssh, "sudo service lvm2-lvmetad stop", get_pty=True + ) # disable volume groups. Any volume groups that have volumes in use # will remain online. However, volume groups belonging to disks # that have been synced at least once, will be deactivated. utils.ignore_exceptions(utils.exec_ssh_cmd)( - ssh, "sudo vgchange -an", get_pty=True) + ssh, "sudo vgchange -an", get_pty=True + ) def _disable_lvm_metad_udev_rule(ssh): @@ -112,7 +112,8 @@ def _disable_lvm_metad_udev_rule(ssh): """ rule_paths = [ "/lib/udev/rules.d/69-lvm-metad.rules", - "/lib/udev/rules.d/69-dm-lvm.rules"] + "/lib/udev/rules.d/69-dm-lvm.rules", + ] for path in rule_paths: if utils.test_ssh_path(ssh, path): utils.exec_ssh_cmd(ssh, "sudo rm %s" % path, get_pty=True) @@ -125,19 +126,17 @@ def _check_deserialize_key(key): res = key elif type(key) is str: LOG.trace("Deserializing PEM-encoded private key.") - res = utils.deserialize_key( - key, CONF.serialization.temp_keypair_password) + res = utils.deserialize_key(key, CONF.serialization.temp_keypair_password) else: raise exception.CoriolisException( "Private key must be either a PEM-encoded string or " - "a paramiko.RSAKey instance. Got type '%s'." % ( - type(key))) + "a paramiko.RSAKey instance. Got type '%s'." % (type(key)) + ) return res class BackupWritersFactory(object): - def __init__(self, writer_connection_info, volumes_info): self._validate_info(writer_connection_info) self._type = writer_connection_info["backend"] @@ -147,31 +146,31 @@ def __init__(self, writer_connection_info, volumes_info): def get_writer(self): if self._type == BACKUP_WRITER_SSH: return SSHBackupWriter.from_connection_info( - self._conn_info, self._volumes_info) + self._conn_info, self._volumes_info + ) elif self._type == BACKUP_WRITER_HTTP: return HTTPBackupWriter.from_connection_info( - self._conn_info, self._volumes_info) + self._conn_info, self._volumes_info + ) elif self._type == BACKUP_WRITER_FILE: return FileBackupWriter.from_connection_info( - self._conn_info, self._volumes_info) - raise exception.CoriolisException( - "Invalid backup writer type: %s" % self._type) + self._conn_info, self._volumes_info + ) + raise exception.CoriolisException("Invalid backup writer type: %s" % self._type) def _validate_info(self, info): if type(info) is not dict: - raise exception.CoriolisException( - "Invalid backup writer connection info.") + raise exception.CoriolisException("Invalid backup writer connection info.") wrt_type = info.get("backend", None) if wrt_type is None: - raise exception.CoriolisException( - "Missing backend name in connection info") + raise exception.CoriolisException("Missing backend name in connection info") if wrt_type not in BACKUP_WRITERS: raise exception.CoriolisException( - "Invalid backup writer type: %s" % wrt_type) + "Invalid backup writer type: %s" % wrt_type + ) wrt_conn_info = info.get("connection_details") if wrt_conn_info is None: - raise exception.CoriolisException( - "Missing credentials in connection info") + raise exception.CoriolisException("Missing credentials in connection info") class BaseBackupWriterImpl(with_metaclass(abc.ABCMeta)): @@ -276,8 +275,7 @@ def from_connection_info(cls, info, volumes_info): class SSHBackupWriterImpl(BaseBackupWriterImpl): - def __init__(self, path, disk_id, compress_transfer=False, - encoder_count=3): + def __init__(self, path, disk_id, compress_transfer=False, encoder_count=3): self._msg_id = None self._stdin = None self._stdout = None @@ -310,21 +308,24 @@ def _exec_helper_cmd(self): self._msg_id = 0 self._offset = 0 self._stdin, self._stdout, self._stderr = self._ssh.exec_command( - "chmod +x write_data && sudo ./write_data") + "chmod +x write_data && sudo ./write_data" + ) def _encode_data(self, content, offset, msg_id): msg = data_transfer.encode_data( - msg_id, self._path, - offset, content, - compress=self._compress_transfer) + msg_id, self._path, offset, content, compress=self._compress_transfer + ) LOG.debug( "Guest path: %(path)s, offset: %(offset)d, content len: " "%(content_len)d, msg len: %(msg_len)d", - {"path": self._path, - "offset": offset, - "content_len": len(content), - "msg_len": len(msg)}) + { + "path": self._path, + "offset": offset, + "content_len": len(content), + "msg_len": len(msg), + }, + ) return msg def _encode_eod(self): @@ -339,8 +340,9 @@ def _send_msg(self, data): ret_val = self._stdout.channel.recv_exit_status() if int(ret_val) > 0: raise exception.CoriolisException( - "write_data exited with error code %r (%s)" % ( - ret_val, _WRITER_ERR_MAP.get(int(ret_val)))) + "write_data exited with error code %r (%s)" + % (ret_val, _WRITER_ERR_MAP.get(int(ret_val))) + ) self._stdin.write(data) self._stdin.flush() @@ -350,8 +352,7 @@ def _open(self): self._exec_helper_cmd() self._sender_thread = utils.start_thread(self._sender) for _ in range(self._encoder_cnt): - self._encoder_threads.append( - utils.start_thread(self._encoder)) + self._encoder_threads.append(utils.start_thread(self._encoder)) def seek(self, pos): self._offset = pos @@ -388,9 +389,8 @@ def _encoder(self): continue try: data = self._encode_data( - payload["data"], - payload["offset"], - payload["msg_id"]) + payload["data"], payload["offset"], payload["msg_id"] + ) self._sender_q.put(data) except BaseException as err: LOG.error("Backup encoder failed.") @@ -402,13 +402,12 @@ def _encoder(self): def write(self, data): if self._closing: - raise exception.CoriolisException( - "Attempted to write to a closed writer.") + raise exception.CoriolisException("Attempted to write to a closed writer.") if self._exception: raise exception.CoriolisException( - "Failed to write data. See log " - "for details.") from self._exception + "Failed to write data. See log for details." + ) from self._exception payload = { "offset": self._offset, @@ -422,16 +421,20 @@ def write(self, data): def _wait_for_queues(self): LOG.info("Waiting for unfinished transfers to complete") timeout = datetime.datetime.now() + datetime.timedelta(seconds=600) - while (self._enc_q.unfinished_tasks or - self._sender_q.unfinished_tasks) and not self._exception: - LOG.info("Waiting for unfinished transfers to complete, " - f"encoder tasks: {self._enc_q.unfinished_tasks}, " - f"sender tasks: {self._sender_q.unfinished_tasks}.") + while ( + self._enc_q.unfinished_tasks or self._sender_q.unfinished_tasks + ) and not self._exception: + LOG.info( + "Waiting for unfinished transfers to complete, " + f"encoder tasks: {self._enc_q.unfinished_tasks}, " + f"sender tasks: {self._sender_q.unfinished_tasks}." + ) time.sleep(0.5) now = datetime.datetime.now() if now >= timeout: raise exception.CoriolisException( - "Timed out waiting for data transfer to finish") + "Timed out waiting for data transfer to finish" + ) def close(self): self._closing = True @@ -441,8 +444,8 @@ def close(self): # We can raise here. Any SSH socket cleanup will happen # in _handle_exception() raise exception.CoriolisException( - "Exception occurred during data transfer. " - "Check logs for more details.") from self._exception + "Exception occurred during data transfer. Check logs for more details." + ) from self._exception if self._ssh: self._send_msg(self._encode_eod()) @@ -477,11 +480,12 @@ def _handle_exception(self, ex): # TODO(alexpilotti): map error codes to error messages raise exception.CoriolisException( "An exception occurred while writing data on target. " - "Exit code: %s" % ret_val) + "Exit code: %s" % ret_val + ) else: raise exception.CoriolisException( - "An exception occurred while writing data on target: %s" % - ex) + "An exception occurred while writing data on target: %s" % ex + ) class SSHBackupWriter(BaseBackupWriter): @@ -507,10 +511,10 @@ def from_connection_info(cls, info, volumes_info): if not all([ip, port, username]): raise exception.CoriolisException( "Connection info is invalid for SSHBackupWriter. " - "The following fields are required: %s" % ", ".join(required)) + "The following fields are required: %s" % ", ".join(required) + ) if pkey is None and password is None: - raise exception.CoriolisException( - "Either pkey or password are required") + raise exception.CoriolisException("Either pkey or password are required") if pkey: pkey = _check_deserialize_key(pkey) @@ -522,18 +526,14 @@ def _get_impl(self, path, disk_id): _disable_lvm_metad_udev_rule(ssh) _disable_lvm2_lvmetad(ssh) - matching_devs = [ - v for v in self._volumes_info if v["disk_id"] == disk_id] + matching_devs = [v for v in self._volumes_info if v["disk_id"] == disk_id] if not matching_devs: - base_msg = ( - "Could not locate disk with ID '%s' in volumes_info" % - disk_id) + base_msg = "Could not locate disk with ID '%s' in volumes_info" % disk_id LOG.error("%s: %s", base_msg, self._volumes_info) raise exception.CoriolisException(base_msg) elif len(matching_devs) > 1: - base_msg = ( - "Multiple disks with ID '%s' in volumes_info" % disk_id) + base_msg = "Multiple disks with ID '%s' in volumes_info" % disk_id LOG.error("%s: %s", base_msg, self._volumes_info) raise exception.CoriolisException(base_msg) @@ -548,8 +548,7 @@ def _get_impl(self, path, disk_id): def _copy_helper_cmd(self, ssh): with self._lock: sftp = ssh.open_sftp() - local_path = os.path.join( - utils.get_resources_bin_dir(), 'write_data') + local_path = os.path.join(utils.get_resources_bin_dir(), 'write_data') try: # Check if the remote file already exists sftp.stat('write_data') @@ -562,16 +561,21 @@ def _copy_helper_cmd(self, ssh): @utils.retry_on_error(sleep_seconds=30) def _connect_ssh(self): - LOG.info("Connecting to SSH host: %(ip)s:%(port)s" % - {"ip": self._ip, "port": self._port}) + LOG.info( + "Connecting to SSH host: %(ip)s:%(port)s" + % {"ip": self._ip, "port": self._port} + ) return utils.connect_ssh( - self._ip, self._port, self._username, - pkey=self._pkey, password=self._password) + self._ip, + self._port, + self._username, + pkey=self._pkey, + password=self._password, + ) class HTTPBackupWriterImpl(BaseBackupWriterImpl): - def __init__(self, path, disk_id, - compress_transfer=None, compressor_count=3): + def __init__(self, path, disk_id, compress_transfer=None, compressor_count=3): self._offset = None self._session = None self._ip = None @@ -607,17 +611,15 @@ def _set_info(self, info): self._key = info.get("client_key") self._ca = info.get("ca_crt") self._id = info.get("id") - if not all([self._ip, self._port, self._crt, - self._key, self._ca, self._id]): + if not all([self._ip, self._port, self._crt, self._key, self._ca, self._id]): raise exception.CoriolisException( - "Missing required info when creating HTTPBackupWriter") + "Missing required info when creating HTTPBackupWriter" + ) @property def _uri(self): b64_path = base64.b64encode(self._path.encode()).decode() - return "https://%s:%s/api/v2/device/%s" % ( - self._ip, self._port, b64_path - ) + return "https://%s:%s/api/v2/device/%s" % (self._ip, self._port, b64_path) @utils.retry_on_error() def _acquire(self): @@ -625,9 +627,9 @@ def _acquire(self): uri = "%s/acquire" % self._uri headers = {"X-Client-Token": self._id} resp = self._session.post( - uri, headers=headers, timeout=CONF.default_requests_timeout) - LOG.debug("Returned code: %d. Msg: %s" % ( - resp.status_code, resp.content)) + uri, headers=headers, timeout=CONF.default_requests_timeout + ) + LOG.debug("Returned code: %d. Msg: %s" % (resp.status_code, resp.content)) resp.raise_for_status() @utils.retry_on_error() @@ -636,18 +638,16 @@ def _release(self): uri = "%s/release" % self._uri headers = {"X-Client-Token": self._id} resp = self._session.post( - uri, headers=headers, timeout=CONF.default_requests_timeout) - LOG.debug("Returned code: %d. Msg: %s" % - (resp.status_code, resp.content)) + uri, headers=headers, timeout=CONF.default_requests_timeout + ) + LOG.debug("Returned code: %d. Msg: %s" % (resp.status_code, resp.content)) resp.raise_for_status() def _init_session(self): if self._session: self._session.close() sess = provider_utils.ProviderSession() - sess.cert = ( - self._crt, - self._key) + sess.cert = (self._crt, self._key) sess.verify = self._ca self._session = sess @@ -660,8 +660,7 @@ def _open(self): self._compressor_count = 1 self._compressor_threads = [] for _ in range(self._compressor_count): - self._compressor_threads.append( - utils.start_thread(self._compressor)) + self._compressor_threads.append(utils.start_thread(self._compressor)) def seek(self, pos): self._offset = pos @@ -693,7 +692,8 @@ def _compressor(self): if self._compress_transfer: try: chunk, compressed = data_transfer.compression_proxy( - chunk, constants.COMPRESSION_FORMAT_GZIP) + chunk, constants.COMPRESSION_FORMAT_GZIP + ) if compressed: send_payload["encoding"] = 'gzip' except BaseException as err: @@ -730,24 +730,28 @@ def send(): LOG.debug( "Guest path: %(path)s, offset: %(offset)d, content len: " "%(content_len)d", - {"path": self._path, - "offset": offset, - "content_len": len(chunk)}) + {"path": self._path, "offset": offset, "content_len": len(chunk)}, + ) resp = self._session.post( - self._uri, headers=headers, data=chunk, - timeout=CONF.default_requests_timeout) + self._uri, + headers=headers, + data=chunk, + timeout=CONF.default_requests_timeout, + ) LOG.debug( - "Response code: %r, content: %r" % - (resp.status_code, resp.content)) + "Response code: %r, content: %r" % (resp.status_code, resp.content) + ) try: resp.raise_for_status() self._write_error = False except Exception as err: LOG.warning( "Error writing chunk to disk %s at offset" - " %s: %s" % (self._path, payload["offset"], err)) + " %s: %s" % (self._path, payload["offset"], err) + ) self._write_error = True raise + try: send() except BaseException as err: @@ -766,9 +770,7 @@ def send(): @utils.retry_on_error() def write(self, data): if self._closing: - raise exception.CoriolisException( - "Attempted to write to a closed writer." - ) + raise exception.CoriolisException("Attempted to write to a closed writer.") if self._exception: raise exception.CoriolisException(self._exception) @@ -781,17 +783,21 @@ def write(self, data): def _wait_for_queues(self): timeout = datetime.datetime.now() + datetime.timedelta(seconds=600) - while (self._comp_q.unfinished_tasks or - self._sender_q.unfinished_tasks) and not self._exception: + while ( + self._comp_q.unfinished_tasks or self._sender_q.unfinished_tasks + ) and not self._exception: # No error recorded, and we have tasks in the queue - LOG.info("Waiting for unfinished transfers to complete, " - f"compressor tasks: {self._comp_q.unfinished_tasks}, " - f"sender tasks: {self._sender_q.unfinished_tasks}.") + LOG.info( + "Waiting for unfinished transfers to complete, " + f"compressor tasks: {self._comp_q.unfinished_tasks}, " + f"sender tasks: {self._sender_q.unfinished_tasks}." + ) time.sleep(0.5) now = datetime.datetime.now() if now >= timeout: raise exception.CoriolisException( - "Timed out waiting for data transfer to finish") + "Timed out waiting for data transfer to finish" + ) def _create_checksum_job(self, algorithm, start_offset=0, end_offset=0): """Creates a full-disk checksum job on the writer. @@ -814,8 +820,8 @@ def _create_checksum_job(self, algorithm, start_offset=0, end_offset=0): } resp = self._session.post( - uri, headers=headers, json=body, - timeout=CONF.default_requests_timeout) + uri, headers=headers, json=body, timeout=CONF.default_requests_timeout + ) resp.raise_for_status() return resp.json()["job_id"] @@ -825,8 +831,7 @@ def _get_checksum_job_status(self, job_id): self._ensure_session() uri = "%s/checksumJob/%s" % (self._uri, job_id) - resp = self._session.get( - uri, timeout=CONF.default_requests_timeout) + resp = self._session.get(uri, timeout=CONF.default_requests_timeout) resp.raise_for_status() return resp.json() @@ -836,8 +841,7 @@ def _delete_checksum_job(self, job_id): self._ensure_session() uri = "%s/checksumJob/%s" % (self._uri, job_id) - resp = self._session.delete( - uri, timeout=CONF.default_requests_timeout) + resp = self._session.delete(uri, timeout=CONF.default_requests_timeout) resp.raise_for_status() def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): @@ -855,8 +859,9 @@ def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): self._wait_for_queues() if self._exception: raise exception.CoriolisException( - "Cannot checksum disk '%s', pending write error: %s" % ( - self._path, self._exception)) + "Cannot checksum disk '%s', pending write error: %s" + % (self._path, self._exception) + ) timeout = CONF.disk_checksum_timeout deadline = time.monotonic() + timeout @@ -873,13 +878,15 @@ def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): if execution_status == _CHECKSUM_JOB_FAILED: raise exception.CoriolisException( - "Checksum job failed for disk '%s': %s" % ( - self._path, status.get("error_message", ""))) + "Checksum job failed for disk '%s': %s" + % (self._path, status.get("error_message", "")) + ) if time.monotonic() >= deadline: raise exception.CoriolisException( "Timed out waiting for checksum job for disk '%s' " - "after %d seconds." % (self._path, timeout)) + "after %d seconds." % (self._path, timeout) + ) time.sleep(_CHECKSUM_JOB_POLL_INTERVAL) finally: @@ -887,8 +894,8 @@ def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): self._delete_checksum_job(job_id) except Exception: LOG.warning( - "Failed to delete checksum job %s for disk %s", - job_id, self._path) + "Failed to delete checksum job %s for disk %s", job_id, self._path + ) def close(self): self._closing = True @@ -900,8 +907,9 @@ def close(self): try: self._release() except Exception as err: - LOG.error("Failed to release disk %s: %s. Ignoring." % ( - self._path, err)) + LOG.error( + "Failed to release disk %s: %s. Ignoring." % (self._path, err) + ) raise exception.CoriolisException(self._exception) self._release() @@ -920,11 +928,9 @@ def close(self): class HTTPBackupWriterBootstrapper(object): - def __init__(self, ssh_conn_info, writer_port): self._lock = threading.Lock() - self._writer_cmd = os.path.join( - "/usr/bin", _CORIOLIS_HTTP_WRITER_CMD) + self._writer_cmd = os.path.join("/usr/bin", _CORIOLIS_HTTP_WRITER_CMD) self._writer_port = writer_port self._ip = ssh_conn_info.get("ip") self._port = ssh_conn_info.get("port", 22) @@ -933,56 +939,68 @@ def __init__(self, ssh_conn_info, writer_port): self._pkey = ssh_conn_info.get("pkey") if not all([self._ip, self._port, self._username]): raise exception.CoriolisException( - "Invalid SSH connection info. IP, port and" - " username are mandatory") + "Invalid SSH connection info. IP, port and username are mandatory" + ) if self._password is None and self._pkey is None: - raise exception.CoriolisException( - "Either password or pkey are required") + raise exception.CoriolisException("Either password or pkey are required") if self._pkey: self._pkey = _check_deserialize_key(self._pkey) self._ssh = self._connect_ssh() @utils.retry_on_error(sleep_seconds=30) def _connect_ssh(self): - LOG.info("Connecting to SSH host: %(ip)s:%(port)s" % - {"ip": self._ip, "port": self._port}) + LOG.info( + "Connecting to SSH host: %(ip)s:%(port)s" + % {"ip": self._ip, "port": self._port} + ) return utils.connect_ssh( - self._ip, self._port, self._username, - pkey=self._pkey, password=self._password) + self._ip, + self._port, + self._username, + pkey=self._pkey, + password=self._password, + ) def _inject_dport_allow_rule(self, ssh): cmd = ( "sudo nft insert rule ip filter INPUT tcp dport %(port)s counter " "accept || " - "sudo iptables -I INPUT -p tcp --dport %(port)s -j ACCEPT" % { - "port": self._writer_port}) + "sudo iptables -I INPUT -p tcp --dport %(port)s -j ACCEPT" + % {"port": self._writer_port} + ) try: utils.exec_ssh_cmd(ssh, cmd, get_pty=True) except exception.CoriolisException: LOG.warn( "Could not inject TCP FW rule. Error was: %s", - utils.get_exception_details()) + utils.get_exception_details(), + ) def _add_firewalld_port(self, ssh): cmd = "sudo firewall-cmd --add-port=%s/tcp" % self._writer_port try: utils.exec_ssh_cmd(ssh, cmd, get_pty=True) except exception.CoriolisException: - LOG.warn("Could not add TCP port to firewalld. Error was: %s", - utils.get_exception_details()) + LOG.warn( + "Could not add TCP port to firewalld. Error was: %s", + utils.get_exception_details(), + ) def _change_binary_se_context(self, ssh): cmd = "sudo chcon -t bin_t %s" % self._writer_cmd try: utils.exec_ssh_cmd(ssh, cmd, get_pty=True) except exception.CoriolisException: - LOG.warn("Could not change SELinux context of writer binary. " - "Error was:%s", utils.get_exception_details()) + LOG.warn( + "Could not change SELinux context of writer binary. Error was:%s", + utils.get_exception_details(), + ) @utils.retry_on_error() def _copy_writer(self, ssh): local_path = os.path.join( - utils.get_resources_bin_dir(), _CORIOLIS_HTTP_WRITER_CMD) + utils.get_resources_bin_dir(), _CORIOLIS_HTTP_WRITER_CMD + ) remote_tmp_path = os.path.join("/tmp", _CORIOLIS_HTTP_WRITER_CMD) with self._lock: sftp = ssh.open_sftp() @@ -995,25 +1013,19 @@ def _copy_writer(self, ssh): sftp.put(local_path, remote_tmp_path) utils.exec_ssh_cmd( ssh, - "sudo mv %s %s" % ( - remote_tmp_path, self._writer_cmd), - get_pty=True + "sudo mv %s %s" % (remote_tmp_path, self._writer_cmd), + get_pty=True, ) utils.exec_ssh_cmd( - ssh, - "sudo chmod +x %s" % self._writer_cmd, - get_pty=True + ssh, "sudo chmod +x %s" % self._writer_cmd, get_pty=True ) finally: sftp.close() def _fetch_remote_file(self, ssh, remote_file, local_file): with open(local_file, 'wb') as fd: - utils.exec_ssh_cmd( - ssh, - "sudo chmod +r %s" % remote_file, get_pty=True) - data = utils.retry_on_error()( - utils.read_ssh_file)(ssh, remote_file) + utils.exec_ssh_cmd(ssh, "sudo chmod +r %s" % remote_file, get_pty=True) + data = utils.retry_on_error()(utils.read_ssh_file)(ssh, remote_file) fd.write(data) def _setup_certificates(self, ssh): @@ -1033,49 +1045,57 @@ def _setup_certificates(self, ssh): remote_srv_key = os.path.join(remote_base_dir, srv_key_name) exist = [] - for i in (remote_ca_crt, remote_client_crt, remote_client_key, - remote_srv_crt, remote_srv_key): + for i in ( + remote_ca_crt, + remote_client_crt, + remote_client_key, + remote_srv_crt, + remote_srv_key, + ): exist.append(utils.test_ssh_path(ssh, i)) if not all(exist): - utils.exec_ssh_cmd( - ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo %(writer_cmd)s generate-certificates -output-dir " - "%(cert_dir)s -certificate-hosts %(extra_hosts)s" % { + "%(cert_dir)s -certificate-hosts %(extra_hosts)s" + % { "writer_cmd": self._writer_cmd, "cert_dir": remote_base_dir, "extra_hosts": self._ip, }, - get_pty=True) + get_pty=True, + ) return { "srv_crt": remote_srv_crt, "srv_key": remote_srv_key, "ca_crt": remote_ca_crt, "client_crt": remote_client_crt, - "client_key": remote_client_key + "client_key": remote_client_key, } def _read_remote_file_sudo(self, remote_path): contents = utils.exec_ssh_cmd( - self._ssh, 'sudo cat "%s"' % remote_path, get_pty=True) + self._ssh, 'sudo cat "%s"' % remote_path, get_pty=True + ) return contents def _init_writer(self, ssh, cert_paths): - cmdline = ("%(cmd)s run -ca-cert %(ca_cert)s -key " - "%(srv_key)s -cert %(srv_cert)s -listen-port " - "%(listen_port)s") % { - "cmd": self._writer_cmd, - "ca_cert": cert_paths["ca_crt"], - "srv_key": cert_paths["srv_key"], - "srv_cert": cert_paths["srv_crt"], - "listen_port": self._writer_port, + cmdline = ( + "%(cmd)s run -ca-cert %(ca_cert)s -key " + "%(srv_key)s -cert %(srv_cert)s -listen-port " + "%(listen_port)s" + ) % { + "cmd": self._writer_cmd, + "ca_cert": cert_paths["ca_crt"], + "srv_key": cert_paths["srv_key"], + "srv_cert": cert_paths["srv_crt"], + "listen_port": self._writer_port, } self._change_binary_se_context(ssh) - utils.create_service( - ssh, cmdline, _CORIOLIS_HTTP_WRITER_CMD, start=True) + utils.create_service(ssh, cmdline, _CORIOLIS_HTTP_WRITER_CMD, start=True) self._inject_dport_allow_rule(ssh) self._add_firewalld_port(ssh) @@ -1083,25 +1103,21 @@ def setup_writer(self): _disable_lvm_metad_udev_rule(self._ssh) _disable_lvm2_lvmetad(self._ssh) self._copy_writer(self._ssh) - paths = utils.retry_on_error()( - self._setup_certificates)(self._ssh) - utils.retry_on_error()( - self._init_writer)(self._ssh, paths) + paths = utils.retry_on_error()(self._setup_certificates)(self._ssh) + utils.retry_on_error()(self._init_writer)(self._ssh, paths) return { "ip": self._ip, "port": self._writer_port, "certificates": { "client_crt": self._read_remote_file_sudo(paths["client_crt"]), "client_key": self._read_remote_file_sudo(paths["client_key"]), - "ca_crt": self._read_remote_file_sudo(paths["ca_crt"]) - } + "ca_crt": self._read_remote_file_sudo(paths["ca_crt"]), + }, } class HTTPBackupWriter(BaseBackupWriter): - - def __init__(self, ip, port, volumes_info, certificates, - compressor_count=3): + def __init__(self, ip, port, volumes_info, certificates, compressor_count=3): self._ip = ip self._port = port self._volumes_info = volumes_info @@ -1112,8 +1128,7 @@ def __init__(self, ip, port, volumes_info, certificates, self._certificates = certificates self._crt_dir = tempfile.mkdtemp() if not self._certificates: - raise exception.CoriolisException( - "certificates is mandatory") + raise exception.CoriolisException("certificates is mandatory") self._cert_paths = None @classmethod @@ -1144,16 +1159,18 @@ def from_connection_info(cls, conn_info, volumes_info): required = ["ip", "port", "certificates"] if not all([ip, port, certs]): raise exception.CoriolisException( - "Missing required connection info: %s" % ", ".join(required)) + "Missing required connection info: %s" % ", ".join(required) + ) required_cert_options = ["client_crt", "client_key", "ca_crt"] missing_cert_options = [ - opt for opt in required_cert_options - if opt not in certs] + opt for opt in required_cert_options if opt not in certs + ] if missing_cert_options: raise exception.CoriolisException( "Missing the following HTTPBackupWriter fields from the " - "'certificates' options: %s" % missing_cert_options) + "'certificates' options: %s" % missing_cert_options + ) return cls(ip, port, volumes_info, certs) @@ -1166,15 +1183,14 @@ def __del__(self): def _wait_for_conn(self): LOG.debug( - "waiting for coriolis-writer connectivity %s:%s" % ( - self._ip, self._writer_port)) - utils.wait_for_port_connectivity( - self._ip, self._writer_port) + "waiting for coriolis-writer connectivity %s:%s" + % (self._ip, self._writer_port) + ) + utils.wait_for_port_connectivity(self._ip, self._writer_port) def _write_cert_files(self): if not self._certificates: - raise exception.CoriolisException( - "certificates not set") + raise exception.CoriolisException("certificates not set") if self._cert_paths: return self._cert_paths @@ -1198,18 +1214,23 @@ def _get_impl(self, path, disk_id): cert_paths = self._write_cert_files() self._wait_for_conn() - path = [v for v in self._volumes_info - if v["disk_id"] == disk_id][0]["volume_dev"] + path = [v for v in self._volumes_info if v["disk_id"] == disk_id][0][ + "volume_dev" + ] impl = HTTPBackupWriterImpl( - path, disk_id, + path, + disk_id, compressor_count=self._compressor_count, - compress_transfer=CONF.compress_transfers) - impl._set_info({ - "ip": self._ip, - "port": self._writer_port, - "client_crt": cert_paths["client_crt"], - "client_key": cert_paths["client_key"], - "ca_crt": cert_paths["ca_crt"], - "id": self._id, - }) + compress_transfer=CONF.compress_transfers, + ) + impl._set_info( + { + "ip": self._ip, + "port": self._writer_port, + "client_crt": cert_paths["client_crt"], + "client_key": cert_paths["client_key"], + "ca_crt": cert_paths["ca_crt"], + "id": self._id, + } + ) return impl diff --git a/coriolis/providers/base.py b/coriolis/providers/base.py index 59f34815..0c00f653 100644 --- a/coriolis/providers/base.py +++ b/coriolis/providers/base.py @@ -7,8 +7,7 @@ from oslo_log import log as logging from six import with_metaclass -from coriolis import context -from coriolis import exception +from coriolis import context, exception from coriolis.osmorphing import base as base_osmorphing from coriolis.osmorphing.osdetect import base as base_osdetect @@ -16,7 +15,6 @@ class BaseProvider(object, with_metaclass(abc.ABCMeta)): - @property def platform(self) -> str: """Platform type.""" @@ -24,7 +22,6 @@ def platform(self) -> str: class BaseEndpointProvider(BaseProvider): - @abc.abstractmethod def validate_connection( self, @@ -139,8 +136,7 @@ def get_shared_library_directories( return [] -class BaseEndpointDestinationOptionsProvider( - object, with_metaclass(abc.ABCMeta)): +class BaseEndpointDestinationOptionsProvider(object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_target_environment_options( self, @@ -206,8 +202,7 @@ def get_target_environment_options( pass -class BaseEndpointSourceOptionsProvider( - object, with_metaclass(abc.ABCMeta)): +class BaseEndpointSourceOptionsProvider(object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_source_environment_options( self, @@ -274,7 +269,6 @@ def get_source_environment_options( class BaseInstanceProvider(BaseProvider): - @abc.abstractmethod def get_os_morphing_tools( self, @@ -316,7 +310,6 @@ def get_custom_os_detect_tools( class BaseImportInstanceProvider(BaseInstanceProvider): - @abc.abstractmethod def get_target_environment_schema(self) -> str: """Retrieve the provider specific target environment schema. @@ -342,10 +335,11 @@ def _get_destination_instance_name( :param instance_name: fallback instance name """ dest_instance_name = export_info.get("name", instance_name) - LOG.debug('Destination instance name for "%(instance_name)s": ' - '"%(dest_instance_name)s"', - {"instance_name": instance_name, - "dest_instance_name": dest_instance_name}) + LOG.debug( + 'Destination instance name for "%(instance_name)s": ' + '"%(dest_instance_name)s"', + {"instance_name": instance_name, "dest_instance_name": dest_instance_name}, + ) return dest_instance_name @abc.abstractmethod @@ -399,8 +393,7 @@ def delete_os_morphing_resources( pass -class BaseReplicaExportValidationProvider( - object, with_metaclass(abc.ABCMeta)): +class BaseReplicaExportValidationProvider(object, with_metaclass(abc.ABCMeta)): """Validate replica export parameters.""" @abc.abstractmethod @@ -424,8 +417,7 @@ def validate_replica_export_input( return {} -class BaseReplicaImportValidationProvider( - object, with_metaclass(abc.ABCMeta)): +class BaseReplicaImportValidationProvider(object, with_metaclass(abc.ABCMeta)): """Validate replica import parameters.""" @abc.abstractmethod @@ -479,7 +471,6 @@ def validate_replica_deployment_input( class BaseReplicaImportProvider(BaseImportInstanceProvider): - @abc.abstractmethod def deploy_replica_instance( self, @@ -809,7 +800,6 @@ def restore_replica_disk_snapshots( class BaseExportInstanceProvider(BaseInstanceProvider): - @abc.abstractmethod def get_source_environment_schema(self) -> str: """Retrieve the provider specific source environment schema. @@ -824,7 +814,6 @@ def get_source_environment_schema(self) -> str: class BaseReplicaExportProvider(BaseExportInstanceProvider): - @abc.abstractmethod def get_replica_instance_info( self, @@ -1002,18 +991,23 @@ def get_optimal_flavor( pass -def get_os_morphing_tools_helper(conn, os_morphing_tools_clss, - hypervisor_type, os_type, os_root_dir, - os_root_dev, event_manager): +def get_os_morphing_tools_helper( + conn, + os_morphing_tools_clss, + hypervisor_type, + os_type, + os_root_dir, + os_root_dev, + event_manager, +): if os_type and os_type not in os_morphing_tools_clss: - raise exception.OSMorphingToolsNotFound( - "Unsupported OS type: %s" % os_type) + raise exception.OSMorphingToolsNotFound("Unsupported OS type: %s" % os_type) for cls in os_morphing_tools_clss.get( - os_type, itertools.chain(*os_morphing_tools_clss.values())): + os_type, itertools.chain(*os_morphing_tools_clss.values()) + ): LOG.debug("Checking using OSMorphing class: %s", cls) - tools = cls( - conn, os_root_dir, os_root_dev, hypervisor_type, event_manager) + tools = cls(conn, os_root_dir, os_root_dev, hypervisor_type, event_manager) LOG.debug("Testing OS morphing tools: %s", cls.__name__) os_info = tools.check_os() if os_info: @@ -1046,9 +1040,10 @@ def get_storage( class BaseUpdateSourceReplicaProvider(object, with_metaclass(abc.ABCMeta)): - """ Class for replica export providers which offer the functionality + """Class for replica export providers which offer the functionality of updating the parameters for a replica. """ + @abc.abstractmethod def check_update_source_environment_params( self, @@ -1059,7 +1054,7 @@ def check_update_source_environment_params( old_params: dict, new_params: dict, ) -> list[dict]: - """ Checks that any existing replica resources for the VM given by its + """Checks that any existing replica resources for the VM given by its `export_info` which were replicated using the `old_params` is compatible with the `new_params`. The `old_params` and `new_params` refer to either the `source_environment` replica fields. @@ -1074,11 +1069,11 @@ def check_update_source_environment_params( """ -class BaseUpdateDestinationReplicaProvider( - object, with_metaclass(abc.ABCMeta)): - """ Class for replica import providers which offer the functionality +class BaseUpdateDestinationReplicaProvider(object, with_metaclass(abc.ABCMeta)): + """Class for replica import providers which offer the functionality of updating the parameters for a replica. """ + @abc.abstractmethod def check_update_destination_environment_params( self, @@ -1089,7 +1084,7 @@ def check_update_destination_environment_params( old_params: dict, new_params: dict, ) -> list[dict]: - """ Checks that any existing replica resources for the VM given by its + """Checks that any existing replica resources for the VM given by its `export_info` which were replicated using the `old_params` is compatible with the `new_params`. The `old_params` and `new_params` refer to the `destination_environment` replica fields. @@ -1104,8 +1099,7 @@ def check_update_destination_environment_params( """ -class _BaseMinionPoolProvider( - object, with_metaclass(abc.ABCMeta)): +class _BaseMinionPoolProvider(object, with_metaclass(abc.ABCMeta)): """Minion Pool management functionality. Coriolis users will choose whether minion pools should be used or not. @@ -1395,8 +1389,7 @@ def healthcheck_minion( pass -class BaseEndpointInventoryExportProvider( - object, with_metaclass(abc.ABCMeta)): +class BaseEndpointInventoryExportProvider(object, with_metaclass(abc.ABCMeta)): """Capability class for providers that support VM inventory CSV export. Providers that implement this class will be offered in the UI as @@ -1406,8 +1399,7 @@ class BaseEndpointInventoryExportProvider( """ @abc.abstractmethod - def export_instance_inventory( - self, ctxt, connection_info, source_environment): + def export_instance_inventory(self, ctxt, connection_info, source_environment): """Export the full VM inventory as a CSV-formatted string. :param ctxt: Coriolis request context @@ -1429,7 +1421,6 @@ class BaseSourceMinionPoolProvider(_BaseMinionPoolProvider): class BaseDestinationMinionPoolProvider(_BaseMinionPoolProvider): - @abc.abstractmethod def validate_osmorphing_minion_compatibility_for_transfer( self, @@ -1439,7 +1430,7 @@ def validate_osmorphing_minion_compatibility_for_transfer( environment_options: list[dict], minion_properties: dict, ): - """ Validates compatibility between the OSMorphing pool's options and + """Validates compatibility between the OSMorphing pool's options and the options selected for a given transfer. Should raise if any options of the minions in the pool might be deemed incompatible with the desired transfer options. @@ -1463,7 +1454,7 @@ def get_additional_os_morphing_info( target_environment: dict, instance_deployment_info: dict, ) -> dict: - """ This method should return any additional 'osmorphing_info' + """This method should return any additional 'osmorphing_info' as defined in coriolis.schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA :param ctxt: Coriolis request context diff --git a/coriolis/providers/factory.py b/coriolis/providers/factory.py index 5f2e4b7c..7420bc8d 100644 --- a/coriolis/providers/factory.py +++ b/coriolis/providers/factory.py @@ -3,15 +3,11 @@ from oslo_config import cfg -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, utils from coriolis.providers import base -from coriolis import utils serialization_opts = [ - cfg.ListOpt('providers', - default=[], - help='List of provider class paths'), + cfg.ListOpt('providers', default=[], help='List of provider class paths'), ] CONF = cfg.CONF @@ -21,33 +17,35 @@ constants.PROVIDER_TYPE_TRANSFER_EXPORT: base.BaseReplicaExportProvider, constants.PROVIDER_TYPE_TRANSFER_IMPORT: base.BaseReplicaImportProvider, constants.PROVIDER_TYPE_ENDPOINT: base.BaseEndpointProvider, - constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS: - base.BaseEndpointDestinationOptionsProvider, - constants.PROVIDER_TYPE_ENDPOINT_INSTANCES: - base.BaseEndpointInstancesProvider, - constants.PROVIDER_TYPE_ENDPOINT_NETWORKS: - base.BaseEndpointNetworksProvider, - constants.PROVIDER_TYPE_ENDPOINT_STORAGE: - base.BaseEndpointStorageProvider, + constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS: base.BaseEndpointDestinationOptionsProvider, + constants.PROVIDER_TYPE_ENDPOINT_INSTANCES: base.BaseEndpointInstancesProvider, + constants.PROVIDER_TYPE_ENDPOINT_NETWORKS: base.BaseEndpointNetworksProvider, + constants.PROVIDER_TYPE_ENDPOINT_STORAGE: base.BaseEndpointStorageProvider, constants.PROVIDER_TYPE_OS_MORPHING: base.BaseImportInstanceProvider, constants.PROVIDER_TYPE_INSTANCE_FLAVOR: base.BaseInstanceFlavorProvider, constants.PROVIDER_TYPE_SETUP_LIBS: base.BaseProviderSetupExtraLibsMixin, constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT: ( - base.BaseReplicaExportValidationProvider), + base.BaseReplicaExportValidationProvider + ), constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT: ( - base.BaseReplicaImportValidationProvider), + base.BaseReplicaImportValidationProvider + ), constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE: ( - base.BaseUpdateSourceReplicaProvider), + base.BaseUpdateSourceReplicaProvider + ), constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE: ( - base.BaseUpdateDestinationReplicaProvider), + base.BaseUpdateDestinationReplicaProvider + ), constants.PROVIDER_TYPE_SOURCE_ENDPOINT_OPTIONS: ( - base.BaseEndpointSourceOptionsProvider), - constants.PROVIDER_TYPE_SOURCE_MINION_POOL: ( - base.BaseSourceMinionPoolProvider), + base.BaseEndpointSourceOptionsProvider + ), + constants.PROVIDER_TYPE_SOURCE_MINION_POOL: (base.BaseSourceMinionPoolProvider), constants.PROVIDER_TYPE_DESTINATION_MINION_POOL: ( - base.BaseDestinationMinionPoolProvider), + base.BaseDestinationMinionPoolProvider + ), constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT: ( - base.BaseEndpointInventoryExportProvider), + base.BaseEndpointInventoryExportProvider + ), } @@ -59,8 +57,7 @@ def get_available_providers(): provider_data = providers.get(cls.platform, {}) provider_types = provider_data.get("types", []) - if (provider_class in cls.__mro__ and - provider_type not in provider_types): + if provider_class in cls.__mro__ and provider_type not in provider_types: provider_types.append(provider_type) provider_data["types"] = sorted(provider_types) @@ -68,8 +65,7 @@ def get_available_providers(): return providers -def get_provider( - platform_name, provider_type, event_handler, raise_if_not_found=True): +def get_provider(platform_name, provider_type, event_handler, raise_if_not_found=True): for provider in CONF.providers: cls = utils.load_class(provider) parent = PROVIDER_TYPE_MAP.get(provider_type) @@ -80,7 +76,8 @@ def get_provider( if raise_if_not_found: raise exception.NotFound( - "Provider not found for: %(platform_name)s, %(provider_type)s" % - {"platform_name": platform_name, "provider_type": provider_type}) + "Provider not found for: %(platform_name)s, %(provider_type)s" + % {"platform_name": platform_name, "provider_type": provider_type} + ) return None diff --git a/coriolis/providers/provider_utils.py b/coriolis/providers/provider_utils.py index e384683b..335081f9 100644 --- a/coriolis/providers/provider_utils.py +++ b/coriolis/providers/provider_utils.py @@ -1,7 +1,7 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. -from oslo_log import log as logging import requests +from oslo_log import log as logging from coriolis import exception @@ -9,10 +9,14 @@ def get_storage_mapping_for_disk( - storage_mappings, disk_info, storage_backends, - config_default=None, error_on_missing_mapping=True, - error_on_backend_not_found=True): - """ Returns the storage backend identifier from the given list of + storage_mappings, + disk_info, + storage_backends, + config_default=None, + error_on_missing_mapping=True, + error_on_backend_not_found=True, +): + """Returns the storage backend identifier from the given list of `storage_backends` to map for the disk given by its `disk_info`. Order of mapping resolution is: @@ -39,16 +43,24 @@ def get_storage_mapping_for_disk( """ disk_mappings = { mapping['disk_id']: mapping['destination'] - for mapping in storage_mappings.get("disk_mappings", [])} + for mapping in storage_mappings.get("disk_mappings", []) + } backend_mappings = { mapping['source']: mapping['destination'] - for mapping in storage_mappings.get('backend_mappings', [])} + for mapping in storage_mappings.get('backend_mappings', []) + } LOG.debug( "Resolving disk storage backend mapping for disk '%s' from available " "backends: %s (disk_mappings=%s, backend_mappings=%s, default=%s, " - "config_default=%s)", disk_info, storage_backends, disk_mappings, - backend_mappings, storage_mappings.get('default'), config_default) + "config_default=%s)", + disk_info, + storage_backends, + disk_mappings, + backend_mappings, + storage_mappings.get('default'), + config_default, + ) mapped_backend = None @@ -61,56 +73,58 @@ def get_storage_mapping_for_disk( mapped_backend = disk_mappings[disk_id] LOG.debug( "Found mapping for disk ID '%s' in the 'disk_mappings': %s", - disk_id, mapped_backend) + disk_id, + mapped_backend, + ) # 2) check for backend mapping if available: if not mapped_backend: if 'storage_backend_identifier' in disk_info: if disk_info['storage_backend_identifier'] in backend_mappings: mapped_backend = backend_mappings[ - disk_info['storage_backend_identifier']] + disk_info['storage_backend_identifier'] + ] LOG.debug( - "Found mapping for disk ID '%s' in the " - "'backend_mappings': %s", disk_id, mapped_backend) + "Found mapping for disk ID '%s' in the 'backend_mappings': %s", + disk_id, + mapped_backend, + ) else: LOG.debug( "'storage_backend_identifier' for disk '%s' is not mapped " "in the 'backend_mappings' from the 'storage_mappings'.", - disk_info) + disk_info, + ) else: - LOG.debug( - "No 'storage_backend_identifier' set for disk '%s'", - disk_info) + LOG.debug("No 'storage_backend_identifier' set for disk '%s'", disk_info) # 3) use provided default: if not mapped_backend: mapped_backend = storage_mappings.get('default', config_default) if mapped_backend is None: - LOG.warn( - "Could not find mapped storage backend for disk '%s'", disk_info) + LOG.warn("Could not find mapped storage backend for disk '%s'", disk_info) if error_on_missing_mapping: raise exception.DiskStorageMappingNotFound(id=disk_id) if mapped_backend: - if mapped_backend not in [ - backend['name'] for backend in storage_backends]: + if mapped_backend not in [backend['name'] for backend in storage_backends]: LOG.warn( "Mapped storage backend for disk '%s' ('%s') does not exist!", - disk_info, mapped_backend) + disk_info, + mapped_backend, + ) if error_on_backend_not_found: - raise exception.StorageBackendNotFound( - storage_name=mapped_backend) + raise exception.StorageBackendNotFound(storage_name=mapped_backend) - LOG.info( - "Mapped storage backend for disk '%s' is: %s", - disk_info, mapped_backend) + LOG.info("Mapped storage backend for disk '%s' is: %s", disk_info, mapped_backend) return mapped_backend -def check_changed_storage_mappings(volumes_info, old_storage_mappings, - new_storage_mappings): +def check_changed_storage_mappings( + volumes_info, old_storage_mappings, new_storage_mappings +): if not volumes_info: return @@ -119,24 +133,30 @@ def check_changed_storage_mappings(volumes_info, old_storage_mappings, new_backend_mappings = new_storage_mappings.get('backend_mappings', []) new_disk_mappings = new_storage_mappings.get('disk_mappings', []) - old_backend_mappings_set = set([ - tuple(mapping.values()) for mapping in old_backend_mappings]) - old_disk_mappings_set = set([ - tuple(mapping.values()) for mapping in old_disk_mappings]) - new_backend_mappings_set = set([ - tuple(mapping.values()) for mapping in new_backend_mappings]) - new_disk_mappings_set = set([ - tuple(mapping.values()) for mapping in new_disk_mappings]) - - if (not old_backend_mappings_set.issubset(new_backend_mappings_set) or - not old_disk_mappings_set.issubset(new_disk_mappings_set)): - raise exception.CoriolisException("Modifying storage mappings is " - "not supported.") + old_backend_mappings_set = set( + [tuple(mapping.values()) for mapping in old_backend_mappings] + ) + old_disk_mappings_set = set( + [tuple(mapping.values()) for mapping in old_disk_mappings] + ) + new_backend_mappings_set = set( + [tuple(mapping.values()) for mapping in new_backend_mappings] + ) + new_disk_mappings_set = set( + [tuple(mapping.values()) for mapping in new_disk_mappings] + ) + + if not old_backend_mappings_set.issubset( + new_backend_mappings_set + ) or not old_disk_mappings_set.issubset(new_disk_mappings_set): + raise exception.CoriolisException( + "Modifying storage mappings is not supported." + ) class ProviderSession(requests.Session): - def merge_environment_settings( - self, url, proxies, stream, verify, *args, **kwargs): + def merge_environment_settings(self, url, proxies, stream, verify, *args, **kwargs): verify = self.verify return super(ProviderSession, self).merge_environment_settings( - url, proxies, stream, verify, *args, **kwargs) + url, proxies, stream, verify, *args, **kwargs + ) diff --git a/coriolis/providers/replicator.py b/coriolis/providers/replicator.py index b9d1c12a..3fe29b47 100644 --- a/coriolis/providers/replicator.py +++ b/coriolis/providers/replicator.py @@ -7,15 +7,14 @@ import tempfile import time +import paramiko from oslo_config import cfg from oslo_log import log as logging from oslo_utils import units -import paramiko from sshtunnel import SSHTunnelForwarder -from coriolis import exception +from coriolis import exception, utils from coriolis.providers import provider_utils -from coriolis import utils LOG = logging.getLogger(__name__) @@ -32,12 +31,14 @@ DEFAULT_REPLICATOR_PORT = 4433 replicator_opts = [ - cfg.IntOpt('port', - default=DEFAULT_REPLICATOR_PORT, - help='The replicator TCP port.'), - cfg.IntOpt('default_requests_timeout', - default=60, - help='Number of seconds for HTTP request timeouts.'), + cfg.IntOpt( + 'port', default=DEFAULT_REPLICATOR_PORT, help='The replicator TCP port.' + ), + cfg.IntOpt( + 'default_requests_timeout', + default=60, + help='Number of seconds for HTTP request timeouts.', + ), ] CONF = cfg.CONF @@ -45,10 +46,16 @@ class Client(object): - - def __init__(self, ip, port, credentials, ssh_conn_info, - event_handler, use_compression=False, - use_tunnel=False): + def __init__( + self, + ip, + port, + credentials, + ssh_conn_info, + event_handler, + use_compression=False, + use_tunnel=False, + ): self._ip = ip self._use_tunnel = use_tunnel self._port = port @@ -95,31 +102,33 @@ def _test_connection(self): self._setup_tunnel_connection() else: self._event_manager.progress_update( - "Testing direct connection to replicator (%s:%s)" % ( - self._ip, self._port)) + "Testing direct connection to replicator (%s:%s)" + % (self._ip, self._port) + ) try: - utils.wait_for_port_connectivity( - self._ip, self._port, max_wait=30) + utils.wait_for_port_connectivity(self._ip, self._port, max_wait=30) return except BaseException as err: - LOG.debug("failed to connect to %s:%s Error: %s " - "Trying tunneled connection" % ( - self._ip, self._port, err)) + LOG.debug( + "failed to connect to %s:%s Error: %s " + "Trying tunneled connection" % (self._ip, self._port, err) + ) self._event_manager.progress_update( - "Direct connection to replicator failed. " - "Setting up tunnel.") + "Direct connection to replicator failed. Setting up tunnel." + ) self._setup_tunnel_connection() self._event_manager.progress_update( - "Testing connection to replicator (%s:%s)" % ( - self.repl_host, self.repl_port)) + "Testing connection to replicator (%s:%s)" + % (self.repl_host, self.repl_port) + ) try: utils.wait_for_port_connectivity( - self.repl_host, self.repl_port, max_wait=30) + self.repl_host, self.repl_port, max_wait=30 + ) except BaseException as err: self._tunnel.stop() - LOG.warning( - "failed to connect to replicator: %s" % err) + LOG.warning("failed to connect to replicator: %s" % err) raise def _get_ssh_tunnel(self): @@ -135,8 +144,7 @@ def _get_ssh_tunnel(self): pkey = self._ssh_conn_info.get("pkey") password = self._ssh_conn_info.get("password") if any([pkey, password]) is False: - raise exception.CoriolisException( - "Either password or pkey is required") + raise exception.CoriolisException("Either password or pkey is required") server = SSHTunnelForwarder( (remote_host, remote_port), @@ -156,9 +164,7 @@ def raw_disk_uri(self, disk_name): def _get_session(self): sess = provider_utils.ProviderSession() - sess.cert = ( - self._creds["client_cert"], - self._creds["client_key"]) + sess.cert = (self._creds["client_cert"], self._creds["client_key"]) sess.verify = self._creds["ca_cert"] return sess @@ -171,8 +177,8 @@ def get_status(self, device=None, brief=True): "brief": brief, } status = self._cli.get( - uri, params=params, - timeout=CONF.replicator.default_requests_timeout) + uri, params=params, timeout=CONF.replicator.default_requests_timeout + ) status.raise_for_status() return status.json() @@ -183,24 +189,22 @@ def get_chunks(self, device, skip_zeros=False): "skipZeros": skip_zeros, } chunks = self._cli.get( - uri, params=params, - timeout=CONF.replicator.default_requests_timeout) + uri, params=params, timeout=CONF.replicator.default_requests_timeout + ) chunks.raise_for_status() return chunks.json() @utils.retry_on_error() def get_changes(self, device): uri = "%s/api/v1/dev/%s/chunks/changes/" % (self._base_uri, device) - chunks = self._cli.get( - uri, timeout=CONF.replicator.default_requests_timeout) + chunks = self._cli.get(uri, timeout=CONF.replicator.default_requests_timeout) chunks.raise_for_status() return chunks.json() @utils.retry_on_error() def get_disk_size(self, disk): diskUri = self.raw_disk_uri(disk) - info = self._cli.head( - diskUri, timeout=CONF.replicator.default_requests_timeout) + info = self._cli.head(diskUri, timeout=CONF.replicator.default_requests_timeout) info.raise_for_status() return int(info.headers["Content-Length"]) @@ -214,8 +218,7 @@ def get_disk_checksum(self, device): """ uri = "%s/api/v1/dev/%s/checksum" % (self._base_uri, device) - resp = self._cli.get( - uri, timeout=CONF.replicator.default_requests_timeout) + resp = self._cli.get(uri, timeout=CONF.replicator.default_requests_timeout) resp.raise_for_status() return resp.json() @@ -234,18 +237,26 @@ def download_chunk(self, disk, chunk): headers["Accept-encoding"] = "identity" data = self._cli.get( - diskUri, headers=headers, - timeout=CONF.replicator.default_requests_timeout) + diskUri, headers=headers, timeout=CONF.replicator.default_requests_timeout + ) data.raise_for_status() return data.content class Replicator(object): - - def __init__(self, conn_info, event_manager, volumes_info, replica_state, - use_compression=None, ignore_mounted=True, - hash_method=HASH_METHOD_SHA256, watch_devices=True, - chunk_size=10485760, use_tunnel=False): + def __init__( + self, + conn_info, + event_manager, + volumes_info, + replica_state, + use_compression=None, + ignore_mounted=True, + hash_method=HASH_METHOD_SHA256, + watch_devices=True, + chunk_size=10485760, + use_tunnel=False, + ): self._event_manager = event_manager self._repl_state = replica_state self._conn_info = conn_info @@ -268,28 +279,27 @@ def __init__(self, conn_info, event_manager, volumes_info, replica_state, def __del__(self): if self._cert_dir is not None: - utils.ignore_exceptions( - shutil.rmtree)(self._cert_dir) + utils.ignore_exceptions(shutil.rmtree)(self._cert_dir) def _init_replicator_client(self, credentials): """ Helper function to create an instance of the replicator client. """ - ssh_conn_info = self._parse_source_ssh_conn_info( - self._conn_info) - args = self._parse_replicator_conn_info( - self._conn_info) + ssh_conn_info = self._parse_source_ssh_conn_info(self._conn_info) + args = self._parse_replicator_conn_info(self._conn_info) self._cli = Client( - args["ip"], args["port"], - credentials, ssh_conn_info, + args["ip"], + args["port"], + credentials, + ssh_conn_info, self._event_manager, use_compression=self._use_compression, - use_tunnel=self._use_tunnel) + use_tunnel=self._use_tunnel, + ) def _setup_ssh(self): - args = self._parse_source_ssh_conn_info( - self._conn_info) + args = self._parse_source_ssh_conn_info(self._conn_info) ssh = self._get_ssh_client(args) return ssh @@ -303,20 +313,22 @@ def _reconnect_ssh(self): def init_replicator(self): try: self._credentials = utils.retry_on_error(sleep_seconds=5)( - self._setup_replicator)(self._ssh) + self._setup_replicator + )(self._ssh) except Exception: LOG.warn("Failed to setup replicator, trying to reconnect ssh") self._reconnect_ssh() self._credentials = utils.retry_on_error(sleep_seconds=5)( - self._setup_replicator)(self._ssh) - utils.retry_on_error()( - self._init_replicator_client)(self._credentials) + self._setup_replicator + )(self._ssh) + utils.retry_on_error()(self._init_replicator_client)(self._credentials) LOG.debug( "Disk status after Replicator initialization: %s", - self._cli.get_status(device=None, brief=True)) + self._cli.get_status(device=None, brief=True), + ) def get_current_disks_status(self): - """ Returns a list of the current status of the attached data disks. + """Returns a list of the current status of the attached data disks. The root disk of the worker VM is NOT returned. The result is a list with elements of the following format: [{ @@ -347,9 +359,14 @@ def get_current_disks_status(self): return self._cli.get_status() def attach_new_disk( - self, disk_id, attachf, previous_disks_status=None, - retry_period=30, retry_count=10): - """ Returns the volumes_info for the disk attached by running + self, + disk_id, + attachf, + previous_disks_status=None, + retry_period=30, + retry_count=10, + ): + """Returns the volumes_info for the disk attached by running `attachf`. This is achieved by comparing the disk statuses before and after the execution of the attachment operation. @@ -365,27 +382,26 @@ def attach_new_disk( return: dict(): returns the volumes_info associated to the new disk. """ # check if volume with given ID is present in self._volumes_info: - matching_vols = [ - vol for vol in self._volumes_info - if vol['disk_id'] == disk_id] + matching_vols = [vol for vol in self._volumes_info if vol['disk_id'] == disk_id] if not matching_vols: raise exception.CoriolisException( "No information regarding volume with ID '%s'. " - "Cannot track its attachment." % disk_id) + "Cannot track its attachment." % disk_id + ) if len(matching_vols) > 1: raise exception.CoriolisException( - "Multiple volumes info with ID '%s' found: %s" % ( - disk_id, matching_vols)) + "Multiple volumes info with ID '%s' found: %s" + % (disk_id, matching_vols) + ) vol_info = matching_vols[0] # get/refresh current device paths: if not previous_disks_status: previous_disks_status = self._cli.get_status() LOG.debug( - "Disks status pre-attachment of %s: %s", - disk_id, previous_disks_status) - previous_device_paths = [ - dev['device-path'] for dev in previous_disks_status] + "Disks status pre-attachment of %s: %s", disk_id, previous_disks_status + ) + previous_device_paths = [dev['device-path'] for dev in previous_disks_status] # run attachment function and get new device paths: attachf() @@ -396,25 +412,32 @@ def attach_new_disk( for i in range(retry_count): try: new_disks_status = self._cli.get_status() - new_device_paths = [dev['device-path'] - for dev in new_disks_status] + new_device_paths = [dev['device-path'] for dev in new_disks_status] LOG.debug( "Polled devices while waiting for disk '%s' to attach " - "(try %d/%d): %s", disk_id, i + 1, retry_count, - new_device_paths) + "(try %d/%d): %s", + disk_id, + i + 1, + retry_count, + new_device_paths, + ) # check for missing/multiple new device paths: - missing_device_paths = ( - set(previous_device_paths) - set(new_device_paths)) + missing_device_paths = set(previous_device_paths) - set( + new_device_paths + ) if missing_device_paths: LOG.warn( "The following devices from the previous disk state " - "qeury are no longer detected: %s", [ - dev for dev in previous_disks_status - if dev['device-path'] in missing_device_paths]) - - new_device_paths = set( - new_device_paths) - set(previous_device_paths) + "qeury are no longer detected: %s", + [ + dev + for dev in previous_disks_status + if dev['device-path'] in missing_device_paths + ], + ) + + new_device_paths = set(new_device_paths) - set(previous_device_paths) except Exception: LOG.debug("Failed to get new device status") if new_device_paths: @@ -422,33 +445,45 @@ def attach_new_disk( else: LOG.debug( "Sleeping %d seconds for disk '%s' to get attached.", - retry_period, disk_id) + retry_period, + disk_id, + ) time.sleep(retry_period) if not new_device_paths: raise exception.CoriolisException( "No new device paths have appeared after volume attachment of " - "disk with ID: %s" % disk_id) + "disk with ID: %s" % disk_id + ) if len(new_device_paths) > 1: raise exception.CoriolisException( "Multiple device paths have appeared after attachment of disk " - "with ID %s: %s" % ( + "with ID %s: %s" + % ( disk_id, - [dev for dev in new_disks_status - if dev['device-path'] in new_device_paths])) + [ + dev + for dev in new_disks_status + if dev['device-path'] in new_device_paths + ], + ) + ) # record the new 'disk_path' for the volume: vol_info['disk_path'] = new_device_paths.pop() LOG.debug( "New device following attachment of disk '%s': %s", - disk_id, vol_info['disk_path']) + disk_id, + vol_info['disk_path'], + ) return vol_info def wait_for_chunks(self): if self._cli is None: raise exception.CoriolisException( - "replicator not initialized. Run init_replicator()") + "replicator not initialized. Run init_replicator()" + ) perc_steps = {} while True: @@ -461,11 +496,12 @@ def wait_for_chunks(self): if perc_step is None: perc_step = self._event_manager.add_percentage_step( "Performing chunking for disk %s (total size %.2f MB)" - % (devName, dev_size), 100) + % (devName, dev_size), + 100, + ) perc_steps[devName] = perc_step perc_done = vol["checksum-status"]["percentage"] - self._event_manager.set_percentage_step( - perc_step, perc_done) + self._event_manager.set_percentage_step(perc_step, perc_done) done.append(int(perc_done) == 100) if all(done): break @@ -497,9 +533,13 @@ def _get_ssh_client(self, args): gets a paramiko SSH client """ return utils.connect_ssh( - args["hostname"], args["port"], args["username"], - pkey=args.get("pkey"), password=args.get("password"), - banner_timeout=args.get("banner_timeout")) + args["hostname"], + args["port"], + args["username"], + pkey=args.get("pkey"), + password=args.get("password"), + banner_timeout=args.get("banner_timeout"), + ) def _parse_source_ssh_conn_info(self, conn_info): # if we get valid SSH connection info we can @@ -510,22 +550,23 @@ def _parse_source_ssh_conn_info(self, conn_info): port = conn_info.get('port', 22) password = conn_info.get('password', None) pkey = conn_info.get('pkey', None) - missing = [ - field for field in required - if field not in conn_info] + missing = [field for field in required if field not in conn_info] if missing: raise exception.CoriolisException( "Missing some required fields from source replication " - "worker VM connection info: %s" % missing) + "worker VM connection info: %s" % missing + ) if any([password, pkey]) is False: raise exception.CoriolisException( "Either 'password' or 'pkey' for source worker VM is required " - "to initialize the Coriolis replicator.") + "to initialize the Coriolis replicator." + ) if pkey: if type(pkey) is str: pkey = utils.deserialize_key( - pkey, CONF.serialization.temp_keypair_password) + pkey, CONF.serialization.temp_keypair_password + ) args = { "hostname": conn_info["ip"], @@ -566,38 +607,37 @@ def _copy_file(self, ssh, localPath, remotePath): sftp = paramiko.SFTPClient.from_transport(ssh.get_transport()) sftp.put(localPath, tmp) - utils.exec_ssh_cmd( - ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=True) sftp.close() @utils.retry_on_error() def _copy_replicator_cmd(self, ssh): - local_path = os.path.join( - utils.get_resources_bin_dir(), 'replicator') + local_path = os.path.join(utils.get_resources_bin_dir(), 'replicator') self._copy_file(ssh, local_path, REPLICATOR_PATH) - utils.exec_ssh_cmd( - ssh, "sudo chmod +x %s" % REPLICATOR_PATH, get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo chmod +x %s" % REPLICATOR_PATH, get_pty=True) def _setup_replicator_group(self, ssh, group_name=REPLICATOR_GROUP_NAME): - """ Sets up a group with the given name and adds the + """Sets up a group with the given name and adds the user we're connected as to it. Returns True if the group already existed, else False. """ group_exists = utils.exec_ssh_cmd( ssh, - "getent group %(group)s > /dev/null && echo 1 || echo 0" % { - "group": REPLICATOR_GROUP_NAME}) + "getent group %(group)s > /dev/null && echo 1 || echo 0" + % {"group": REPLICATOR_GROUP_NAME}, + ) if int(group_exists) == 0: - utils.exec_ssh_cmd( - ssh, "sudo groupadd %s" % group_name, get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo groupadd %s" % group_name, get_pty=True) # NOTE: this is required in order for the user we connected # as to be able to read the certs: # NOTE2: the group change will only take effect after we reconnect: utils.exec_ssh_cmd( - ssh, "sudo usermod -aG %s %s" % ( - REPLICATOR_GROUP_NAME, self._conn_info['username']), - get_pty=True) + ssh, + "sudo usermod -aG %s %s" + % (REPLICATOR_GROUP_NAME, self._conn_info['username']), + get_pty=True, + ) return int(group_exists) == 1 @@ -605,47 +645,52 @@ def _setup_replicator_user(self, ssh): # check for and create replicator user: user_exists = utils.exec_ssh_cmd( ssh, - "getent passwd %(user)s > /dev/null && echo 1 || echo 0" % { - "user": REPLICATOR_USERNAME}) + "getent passwd %(user)s > /dev/null && echo 1 || echo 0" + % {"user": REPLICATOR_USERNAME}, + ) if int(user_exists) == 0: utils.exec_ssh_cmd( - ssh, "sudo useradd -m -s /bin/bash -g %s %s" % ( - REPLICATOR_GROUP_NAME, REPLICATOR_USERNAME), - get_pty=True) + ssh, + "sudo useradd -m -s /bin/bash -g %s %s" + % (REPLICATOR_GROUP_NAME, REPLICATOR_USERNAME), + get_pty=True, + ) utils.exec_ssh_cmd( - ssh, "sudo usermod -aG disk %s" % REPLICATOR_USERNAME, - get_pty=True) + ssh, "sudo usermod -aG disk %s" % REPLICATOR_USERNAME, get_pty=True + ) @utils.retry_on_error() def _exec_replicator(self, ssh, port, certs, state_file): - cmdline = ("%(replicator_path)s run -hash-method=%(hash_method)s " - "-ignore-mounted-disks=%(ignore_mounted)s " - "-listen-port=%(listen_port)s " - "-chunk-size=%(chunk_size)s " - "-watch-devices=%(watch_devs)s " - "-state-file=%(state_file)s " - "-ca-cert=%(ca_cert)s -cert=%(srv_cert)s " - "-key=%(srv_key)s" % { - "replicator_path": REPLICATOR_PATH, - "hash_method": self._hash_method, - "ignore_mounted": json.dumps(self._ignore_mounted), - "watch_devs": json.dumps(self._watch_devices), - "listen_port": str(port), - "state_file": state_file, - "chunk_size": self._chunk_size, - "ca_cert": certs["ca_crt"], - "srv_cert": certs["srv_crt"], - "srv_key": certs["srv_key"], - }) + cmdline = ( + "%(replicator_path)s run -hash-method=%(hash_method)s " + "-ignore-mounted-disks=%(ignore_mounted)s " + "-listen-port=%(listen_port)s " + "-chunk-size=%(chunk_size)s " + "-watch-devices=%(watch_devs)s " + "-state-file=%(state_file)s " + "-ca-cert=%(ca_cert)s -cert=%(srv_cert)s " + "-key=%(srv_key)s" + % { + "replicator_path": REPLICATOR_PATH, + "hash_method": self._hash_method, + "ignore_mounted": json.dumps(self._ignore_mounted), + "watch_devs": json.dumps(self._watch_devices), + "listen_port": str(port), + "state_file": state_file, + "chunk_size": self._chunk_size, + "ca_cert": certs["ca_crt"], + "srv_cert": certs["srv_crt"], + "srv_key": certs["srv_key"], + } + ) utils.create_service( - ssh, cmdline, REPLICATOR_SVC_NAME, - run_as=REPLICATOR_USERNAME) + ssh, cmdline, REPLICATOR_SVC_NAME, run_as=REPLICATOR_USERNAME + ) def _fetch_remote_file(self, ssh, remote_file, local_file): # TODO(gsamfira): make this re-usable with open(local_file, 'wb') as fd: - data = utils.retry_on_error()( - utils.read_ssh_file)(ssh, remote_file) + data = utils.retry_on_error()(utils.read_ssh_file)(ssh, remote_file) fd.write(data) @utils.retry_on_error(sleep_seconds=5) @@ -677,33 +722,47 @@ def _setup_certificates(self, ssh, args): client_key = os.path.join(self._cert_dir, client_key_name) exist = [] - for i in (remote_ca_crt, remote_client_crt, remote_client_key, - remote_srv_crt, remote_srv_key): + for i in ( + remote_ca_crt, + remote_client_crt, + remote_client_key, + remote_srv_crt, + remote_srv_key, + ): exist.append(utils.test_ssh_path(ssh, i)) force_fetch = False if not all(exist): - utils.exec_ssh_cmd( - ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo %(replicator_cmd)s gen-certs -output-dir " - "%(cert_dir)s -certificate-hosts 127.0.0.1,%(extra_hosts)s" % { + "%(cert_dir)s -certificate-hosts 127.0.0.1,%(extra_hosts)s" + % { "replicator_cmd": REPLICATOR_PATH, "cert_dir": remote_base_dir, "extra_hosts": ip, }, - get_pty=True) + get_pty=True, + ) utils.exec_ssh_cmd( - ssh, "sudo chown -R %(user)s:%(group)s %(cert_dir)s" % { + ssh, + "sudo chown -R %(user)s:%(group)s %(cert_dir)s" + % { "cert_dir": remote_base_dir, "user": REPLICATOR_USERNAME, - "group": REPLICATOR_GROUP_NAME - }, get_pty=True) + "group": REPLICATOR_GROUP_NAME, + }, + get_pty=True, + ) utils.exec_ssh_cmd( - ssh, "sudo chmod -R g+r %(cert_dir)s" % { + ssh, + "sudo chmod -R g+r %(cert_dir)s" + % { "cert_dir": remote_base_dir, - }, get_pty=True) + }, + get_pty=True, + ) force_fetch = True exists = [] @@ -735,8 +794,10 @@ def _change_binary_se_context(self, ssh): try: utils.exec_ssh_cmd(ssh, cmd, get_pty=True) except exception.CoriolisException: - LOG.warn("Could not change SELinux context of replicator binary. " - "Error was:%s", utils.get_exception_details()) + LOG.warn( + "Could not change SELinux context of replicator binary. Error was:%s", + utils.get_exception_details(), + ) @utils.retry_on_error() def _setup_replicator(self, ssh): @@ -744,23 +805,22 @@ def _setup_replicator(self, ssh): # start service state_file = self._get_replicator_state_file() self._copy_file(ssh, state_file, REPLICATOR_STATE) - utils.exec_ssh_cmd( - ssh, "sudo chmod 755 %s" % REPLICATOR_STATE, get_pty=True) + utils.exec_ssh_cmd(ssh, "sudo chmod 755 %s" % REPLICATOR_STATE, get_pty=True) os.remove(state_file) args = self._parse_replicator_conn_info(self._conn_info) self._copy_replicator_cmd(ssh) self._change_binary_se_context(ssh) group_existed = self._setup_replicator_group( - ssh, group_name=REPLICATOR_GROUP_NAME) + ssh, group_name=REPLICATOR_GROUP_NAME + ) if not group_existed: # NOTE: we must reconnect so that our user being added to the new # Replicator group can take effect: ssh = self._reconnect_ssh() self._setup_replicator_user(ssh) certs = self._setup_certificates(ssh, args) - self._exec_replicator( - ssh, args["port"], certs["remote"], REPLICATOR_STATE) + self._exec_replicator(ssh, args["port"], certs["remote"], REPLICATOR_STATE) self.start() return certs["local"] @@ -785,35 +845,42 @@ def _verify_disk_checksum(self, dev_name, destination): if the checksums do not match. """ self._event_manager.progress_update( - "Verifying disk integrity for /dev/%s" % dev_name) + "Verifying disk integrity for /dev/%s" % dev_name + ) source = self._cli.get_disk_checksum(dev_name) end_offset = self._cli.get_disk_size(dev_name) writer = destination.get_disk_checksum( - source["algorithm"], end_offset=end_offset) + source["algorithm"], end_offset=end_offset + ) if writer is None: self._event_manager.progress_update( "Disk integrity check skipped for /dev/%s " - "(writer does not support checksums)" % dev_name) + "(writer does not support checksums)" % dev_name + ) return if source["algorithm"] != writer["algorithm"]: raise exception.ChecksumAlgorithmMismatch( disk=dev_name, source_alg=source["algorithm"], - dest_alg=writer["algorithm"]) + dest_alg=writer["algorithm"], + ) if source["checksum"] != writer["checksum"]: raise exception.ChecksumMismatch( disk=dev_name, source_checksum=source["checksum"], - dest_checksum=writer["checksum"]) + dest_checksum=writer["checksum"], + ) self._event_manager.progress_update( - "Disk integrity verified for /dev/%s (checksum: %s)" % ( - dev_name, source["checksum"])) + "Disk integrity verified for /dev/%s (checksum: %s)" + % (dev_name, source["checksum"]) + ) def replicate_disks( - self, source_volumes_info, backup_writer, verify_checksum=False): + self, source_volumes_info, backup_writer, verify_checksum=False + ): """ Fetch the block diff and send it to the backup_writer. If the target_is_zeroed parameter is set to True, on initial @@ -846,7 +913,8 @@ def replicate_disks( if dst_vol_idx is None: raise exception.CoriolisException( "failed to find a coresponding volume in volumes_info" - " for %s" % volume["disk_id"]) + " for %s" % volume["disk_id"] + ) dst_vol = self._volumes_info[dst_vol_idx] @@ -858,16 +926,16 @@ def replicate_disks( if isInitial and dst_vol.get("zeroed", False) is True: # This is an initial sync of the disk, and we can # skip zero chunks - chunks = self._cli.get_chunks( - devName, skip_zeros=True) + chunks = self._cli.get_chunks(devName, skip_zeros=True) else: # subsequent sync. Get changes. chunks = self._cli.get_changes(devName) if not chunks: self._event_manager.progress_update( - "No new chunks to replicate for disk \"%s\" (%s)" % ( - volume['disk_id'], devName)) + "No new chunks to replicate for disk \"%s\" (%s)" + % (volume['disk_id'], devName) + ) self._repl_state = curr_state return self._repl_state @@ -875,10 +943,9 @@ def replicate_disks( msg = ( "Replicating changed data for disk \"%s\" (device \"%s\", " - "written chunks: %.2f MB)") % ( - volume["disk_id"], devName, size) - perc_step = self._event_manager.add_percentage_step( - msg, len(chunks)) + "written chunks: %.2f MB)" + ) % (volume["disk_id"], devName, size) + perc_step = self._event_manager.add_percentage_step(msg, len(chunks)) total = 0 with backup_writer.open("", volume['disk_id']) as destination: @@ -888,8 +955,7 @@ def replicate_disks( data = self._cli.download_chunk(devName, chunk) destination.write(data) total += 1 - self._event_manager.set_percentage_step( - perc_step, total) + self._event_manager.set_percentage_step(perc_step, total) if verify_checksum: self._verify_disk_checksum(devName, destination) @@ -900,29 +966,27 @@ def replicate_disks( return self._repl_state def _download_full_disk(self, disk, path): - self._event_manager.progress_update( - "Downloading %s as thick file" % disk) + self._event_manager.progress_update("Downloading %s as thick file" % disk) diskUri = self._cli.raw_disk_uri(disk) size = self._cli.get_disk_size(disk) perc_step = self._event_manager.add_percentage_step( - "Downloading full disk /dev/%s" % disk, size) + "Downloading full disk /dev/%s" % disk, size + ) total = 0 - with self._cli._cli.get(diskUri, stream=True, - timeout=(CONF.replicator. - default_requests_timeout)) as dw: + with self._cli._cli.get( + diskUri, stream=True, timeout=(CONF.replicator.default_requests_timeout) + ) as dw: with open(path, 'wb') as dsk: for chunk in dw.iter_content(chunk_size=self._chunk_size): if chunk: writen = dsk.write(chunk) total += writen - self._event_manager.set_percentage_step( - perc_step, total) + self._event_manager.set_percentage_step(perc_step, total) def _download_sparse_disk(self, disk, path, chunks): - self._event_manager.progress_update( - "Downloading %s as sparse file" % disk) + self._event_manager.progress_update("Downloading %s as sparse file" % disk) size = self._cli.get_disk_size(disk) size_from_chunks = self._get_size_from_chunks(chunks) total = 0 @@ -930,8 +994,9 @@ def _download_sparse_disk(self, disk, path, chunks): # create sparse file fp.truncate(size) perc_step = self._event_manager.add_percentage_step( - "Downloading spart disk /dev/%s (%s MB)" % ( - disk, size_from_chunks), len(chunks)) + "Downloading spart disk /dev/%s (%s MB)" % (disk, size_from_chunks), + len(chunks), + ) for chunk in chunks: offset = int(chunk["offset"]) # seek to offset @@ -941,8 +1006,7 @@ def _download_sparse_disk(self, disk, path, chunks): fp.write(data) total += 1 - self._event_manager.set_percentage_step( - perc_step, total) + self._event_manager.set_percentage_step(perc_step, total) def download_disk(self, disk, path): """ @@ -955,8 +1019,7 @@ def download_disk(self, disk, path): if diskName.startswith("/dev"): # just get the name diskName = diskName[5:] - chunks = self._cli.get_chunks( - device=diskName, skip_zeros=True) + chunks = self._cli.get_chunks(device=diskName, skip_zeros=True) if len(chunks) == 0: self._download_full_disk(diskName, path) else: diff --git a/coriolis/qemu.py b/coriolis/qemu.py index 01fe5647..202abb27 100644 --- a/coriolis/qemu.py +++ b/coriolis/qemu.py @@ -22,24 +22,27 @@ class QObject(ctypes.Structure): - _fields_ = [("type", ctypes.c_void_p), - ("refcnt", ctypes.c_size_t)] + _fields_ = [("type", ctypes.c_void_p), ("refcnt", ctypes.c_size_t)] class QString(ctypes.Structure): - _fields_ = [("base", QObject), - ("string", ctypes.c_char_p), - ("length", ctypes.c_size_t), - ("capacity", ctypes.c_size_t)] + _fields_ = [ + ("base", QObject), + ("string", ctypes.c_char_p), + ("length", ctypes.c_size_t), + ("capacity", ctypes.c_size_t), + ] class Error(ctypes.Structure): - _fields_ = [("msg", ctypes.c_char_p), - ("err_class", ctypes.c_int), - ("src", ctypes.c_char_p), - ("func", ctypes.c_char_p), - ("line", ctypes.c_int), - ("hint", ctypes.c_void_p)] + _fields_ = [ + ("msg", ctypes.c_char_p), + ("err_class", ctypes.c_int), + ("src", ctypes.c_char_p), + ("func", ctypes.c_char_p), + ("line", ctypes.c_int), + ("hint", ctypes.c_void_p), + ] _libqemu.qemu_vfree.argtypes = [ctypes.c_void_p] @@ -54,8 +57,7 @@ class Error(ctypes.Structure): _libqemu.qemu_init_exec_dir.restype = None qemu_init_exec_dir = _libqemu.qemu_init_exec_dir -_libqemu.qemu_init_main_loop.argtypes = [ - ctypes.POINTER(ctypes.POINTER(Error))] +_libqemu.qemu_init_main_loop.argtypes = [ctypes.POINTER(ctypes.POINTER(Error))] _libqemu.qemu_init_main_loop.res_type = ctypes.c_int qemu_init_main_loop = _libqemu.qemu_init_main_loop @@ -80,7 +82,10 @@ class Error(ctypes.Structure): qdict_new = _libqemu.qdict_new _libqemu.qdict_put_obj.argtypes = [ - ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(QObject)] + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.POINTER(QObject), +] _libqemu.qdict_put_obj.restype = None qdict_put_obj = _libqemu.qdict_put_obj @@ -89,8 +94,12 @@ class Error(ctypes.Structure): bdrv_init = _libqemu.bdrv_init _libqemu.blk_new_open.argtypes = [ - ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_int, - ctypes.POINTER(ctypes.POINTER(Error))] + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(Error)), +] _libqemu.blk_new_open.restype = ctypes.c_void_p blk_new_open = _libqemu.blk_new_open @@ -107,7 +116,11 @@ class Error(ctypes.Structure): blk_nb_sectors = _libqemu.blk_nb_sectors _libqemu.blk_pread.argtypes = [ - ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, ctypes.c_int] + ctypes.c_void_p, + ctypes.c_int64, + ctypes.c_void_p, + ctypes.c_int, +] _libqemu.blk_pread.res_type = ctypes.c_int blk_pread = _libqemu.blk_pread @@ -116,7 +129,12 @@ class Error(ctypes.Structure): blk_unref = _libqemu.blk_unref _libqemu.bdrv_get_block_status_above.argtypes = [ - ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int, - ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_void_p)] + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_int64, + ctypes.c_int, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_void_p), +] _libqemu.bdrv_get_block_status_above.restype = ctypes.c_int64 bdrv_get_block_status_above = _libqemu.bdrv_get_block_status_above diff --git a/coriolis/qemu_reader.py b/coriolis/qemu_reader.py index 71425cfd..d0568a06 100644 --- a/coriolis/qemu_reader.py +++ b/coriolis/qemu_reader.py @@ -4,8 +4,7 @@ import contextlib import ctypes -from coriolis import exception -from coriolis import qemu +from coriolis import exception, qemu class QEMUDiskImageReaderImpl(object): @@ -37,7 +36,8 @@ def _open(self): options = qemu.qdict_new() blk = qemu.blk_new_open( - self._path.encode(), None, options, 0, ctypes.byref(error)) + self._path.encode(), None, options, 0, ctypes.byref(error) + ) if not blk: raise exception.QEMUException(error.contents.msg.decode()) @@ -52,9 +52,10 @@ def disk_size(self): def _get_sectors(self, offset, size): start_sector = offset >> qemu.BDRV_SECTOR_BITS - return (start_sector, - min(self._total_sectors - start_sector, - size >> qemu.BDRV_SECTOR_BITS)) + return ( + start_sector, + min(self._total_sectors - start_sector, size >> qemu.BDRV_SECTOR_BITS), + ) def get_block_status(self, offset, size): start_sector, num_sectors = self._get_sectors(offset, size) @@ -64,11 +65,15 @@ def get_block_status(self, offset, size): while True: pnum = ctypes.c_int(0) status = qemu.bdrv_get_block_status_above( - self._bs, None, start_sector + sectors, num_sectors - sectors, - ctypes.byref(pnum), ctypes.byref(self._block_driver_state)) + self._bs, + None, + start_sector + sectors, + num_sectors - sectors, + ctypes.byref(pnum), + ctypes.byref(self._block_driver_state), + ) if status < 0 or pnum.value == 0: - raise exception.QEMUException( - 'bdrv_get_block_status_above failed') + raise exception.QEMUException('bdrv_get_block_status_above failed') allocated = (status & qemu.BDRV_BLOCK_ALLOCATED) > 0 zero_block = (status & qemu.BDRV_BLOCK_ZERO) > 0 @@ -94,8 +99,7 @@ def read(self, offset, size): self._buf_size = size read_size = num_sectors << qemu.BDRV_SECTOR_BITS - ret = qemu.blk_pread( - self._blk, offset, self._buf, read_size) + ret = qemu.blk_pread(self._blk, offset, self._buf, read_size) if ret < 0: raise exception.QEMUException("blk_pread failed") diff --git a/coriolis/regions/api.py b/coriolis/regions/api.py index 7a37b688..b6e1fda8 100644 --- a/coriolis/regions/api.py +++ b/coriolis/regions/api.py @@ -10,11 +10,13 @@ def __init__(self): def create(self, ctxt, region_name, description, enabled=True): return self._rpc_client.create_region( - ctxt, region_name, description=description, enabled=enabled) + ctxt, region_name, description=description, enabled=enabled + ) def update(self, ctxt, region_id, updated_values): return self._rpc_client.update_region( - ctxt, region_id, updated_values=updated_values) + ctxt, region_id, updated_values=updated_values + ) def delete(self, ctxt, region_id): self._rpc_client.delete_region(ctxt, region_id) diff --git a/coriolis/rpc.py b/coriolis/rpc.py index 9530fe46..f3a4d3dc 100644 --- a/coriolis/rpc.py +++ b/coriolis/rpc.py @@ -1,22 +1,24 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -import coriolis.exception import oslo_messaging as messaging - from oslo_config import cfg from oslo_log import log as logging +import coriolis.exception from coriolis import context - rpc_opts = [ - cfg.StrOpt('messaging_transport_url', - default="rabbit://guest:guest@127.0.0.1:5672/", - help='Messaging transport url'), - cfg.IntOpt('default_messaging_timeout', - default=60, - help='Number of seconds for messaging timeouts.') + cfg.StrOpt( + 'messaging_transport_url', + default="rabbit://guest:guest@127.0.0.1:5672/", + help='Messaging transport url', + ), + cfg.IntOpt( + 'default_messaging_timeout', + default=60, + help='Number of seconds for messaging timeouts.', + ), ] CONF = cfg.CONF @@ -24,13 +26,11 @@ LOG = logging.getLogger(__name__) -ALLOWED_EXMODS = [ - coriolis.exception.__name__] +ALLOWED_EXMODS = [coriolis.exception.__name__] _TRANSPORT = None class RequestContextSerializer(messaging.Serializer): - def __init__(self, base): self._base = base @@ -53,15 +53,15 @@ def deserialize_context(self, ctxt): def _get_transport(): return messaging.get_transport( - cfg.CONF, CONF.messaging_transport_url, - allowed_remote_exmods=ALLOWED_EXMODS) + cfg.CONF, CONF.messaging_transport_url, allowed_remote_exmods=ALLOWED_EXMODS + ) def get_server(target, endpoints, serializer=None): serializer = RequestContextSerializer(serializer) - return messaging.get_rpc_server(_get_transport(), target, endpoints, - executor='threading', - serializer=serializer) + return messaging.get_rpc_server( + _get_transport(), target, endpoints, executor='threading', serializer=serializer + ) def init(): @@ -72,7 +72,7 @@ def init(): class BaseRPCClient(object): - """ Wrapper for 'oslo_messaging.RPCClient' which automatically + """Wrapper for 'oslo_messaging.RPCClient' which automatically instantiates and cleans up transports for each call. """ @@ -85,8 +85,7 @@ def __init__(self, target, timeout=None, serializer=None): self._transport_conn = None def __repr__(self): - return "" % ( - self._target, self._timeout) + return "" % (self._target, self._timeout) @property def _transport(self): @@ -100,9 +99,11 @@ def _transport(self): def _rpc_client(self): return messaging.RPCClient( - self._transport, self._target, + self._transport, + self._target, serializer=self._serializer, - timeout=self._timeout) + timeout=self._timeout, + ) def _call(self, ctxt, method, **kwargs): client = self._rpc_client() diff --git a/coriolis/scheduler/filters/base.py b/coriolis/scheduler/filters/base.py index 89091716..2cbfe5cf 100644 --- a/coriolis/scheduler/filters/base.py +++ b/coriolis/scheduler/filters/base.py @@ -7,16 +7,13 @@ class BaseServiceFilter(with_metaclass(abc.ABCMeta)): - def is_service_acceptable(self, service): return self.rate_service(service) > 0 def filter_services(self, services): - return [ - service for service in services - if self.is_service_acceptable(service)] + return [service for service in services if self.is_service_acceptable(service)] @abc.abstractmethod def rate_service(self, service): - """ Returns a rating out of 100 for the service. """ + """Returns a rating out of 100 for the service.""" pass diff --git a/coriolis/scheduler/filters/trivial_filters.py b/coriolis/scheduler/filters/trivial_filters.py index 2680ac3a..4acdcc1e 100644 --- a/coriolis/scheduler/filters/trivial_filters.py +++ b/coriolis/scheduler/filters/trivial_filters.py @@ -5,29 +5,29 @@ from coriolis.scheduler.filters import base - LOG = logging.getLogger(__name__) class RegionsFilter(base.BaseServiceFilter): - def __init__(self, regions, any_region=False): self._regions = regions self._any_region = any_region def __repr__(self): return "<%s(regions=%s, any_region=%s)>" % ( - self.__class__.__name__, self._regions, self._any_region) + self.__class__.__name__, + self._regions, + self._any_region, + ) def rate_service(self, service): if not self._regions: LOG.debug( - "No regions specified for this filter (%s). " - "Presuming service is valid.") + "No regions specified for this filter (%s). Presuming service is valid." + ) return 100 - service_regions = [ - region.id for region in service.mapped_regions] + service_regions = [region.id for region in service.mapped_regions] found = [] missing = [] for region in self._regions: @@ -38,26 +38,29 @@ def rate_service(self, service): if not found: LOG.debug( - "None of the requested regions are available on service (%s): " - "%s", service.id, self._regions) + "None of the requested regions are available on service (%s): %s", + service.id, + self._regions, + ) return 0 if not self._any_region and missing: LOG.debug( "The following required regions are missing from service " - "with ID '%s': %s", service.id, missing) + "with ID '%s': %s", + service.id, + missing, + ) return 0 return 100 class TopicFilter(base.BaseServiceFilter): - def __init__(self, topic): self._topic = topic def __repr__(self): - return "<%s(topic=%s)>" % ( - self.__class__.__name__, self._topic) + return "<%s(topic=%s)>" % (self.__class__.__name__, self._topic) def rate_service(self, service): if service.topic == self._topic: @@ -66,13 +69,11 @@ def rate_service(self, service): class EnabledFilter(base.BaseServiceFilter): - def __init__(self, enabled=True): self._enabled = enabled def __repr__(self): - return "<%s(enabled=%s)>" % ( - self.__class__.__name__, self._enabled) + return "<%s(enabled=%s)>" % (self.__class__.__name__, self._enabled) def rate_service(self, service): if service.enabled == self._enabled: @@ -81,9 +82,8 @@ def rate_service(self, service): class ProviderTypesFilter(base.BaseServiceFilter): - def __init__(self, provider_requirements): - """ Filters based on requested provider capabilities. + """Filters based on requested provider capabilities. :param provider_requirements: dict of the form { "": [constants.PROVIDER_TYPE_*, ...]} """ @@ -91,26 +91,35 @@ def __init__(self, provider_requirements): def __repr__(self): return "<%s(provider_requirements=%s)>" % ( - self.__class__.__name__, self._provider_requirements) + self.__class__.__name__, + self._provider_requirements, + ) def rate_service(self, service): for platform_type in self._provider_requirements: if platform_type not in service.providers: LOG.debug( "Service with ID '%s' does not have a provider for " - "platform type '%s'", service.id, platform_type) + "platform type '%s'", + service.id, + platform_type, + ) return 0 - available_types = service.providers[ - platform_type].get('types', []) + available_types = service.providers[platform_type].get('types', []) missing_types = [ - typ for typ in self._provider_requirements[platform_type] - if typ not in available_types] + typ + for typ in self._provider_requirements[platform_type] + if typ not in available_types + ] if missing_types: LOG.debug( "Service with ID '%s' is missing the following required " "provider types for platform '%s': %s", - service.id, platform_type, missing_types) + service.id, + platform_type, + missing_types, + ) return 0 return 100 diff --git a/coriolis/scheduler/rpc/client.py b/coriolis/scheduler/rpc/client.py index 93729165..e66b3b42 100644 --- a/coriolis/scheduler/rpc/client.py +++ b/coriolis/scheduler/rpc/client.py @@ -1,27 +1,24 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -import oslo_messaging as messaging import random import time -from coriolis import constants -from coriolis import exception -from coriolis import rpc -from coriolis.tasks import factory as tasks_factory -from coriolis import utils - +import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log as logging +from coriolis import constants, exception, rpc, utils +from coriolis.tasks import factory as tasks_factory VERSION = "1.0" LOG = logging.getLogger(__name__) scheduler_opts = [ - cfg.IntOpt("scheduler_rpc_timeout", - help="Number of seconds until RPC calls to the " - "scheduler timeout.") + cfg.IntOpt( + "scheduler_rpc_timeout", + help="Number of seconds until RPC calls to the scheduler timeout.", + ) ] CONF = cfg.CONF @@ -33,21 +30,23 @@ def __init__(self, timeout=None): target = messaging.Target(topic='coriolis_scheduler', version=VERSION) if timeout is None: timeout = CONF.scheduler.scheduler_rpc_timeout - super(SchedulerClient, self).__init__( - target, timeout=timeout) + super(SchedulerClient, self).__init__(target, timeout=timeout) def get_diagnostics(self, ctxt): return self._call(ctxt, 'get_diagnostics') def get_workers_for_specs( - self, ctxt, provider_requirements=None, - region_sets=None, enabled=None): + self, ctxt, provider_requirements=None, region_sets=None, enabled=None + ): return self._call( - ctxt, 'get_workers_for_specs', region_sets=region_sets, - enabled=enabled, provider_requirements=provider_requirements) - - def get_any_worker_service( - self, ctxt, random_choice=False, raise_if_none=True): + ctxt, + 'get_workers_for_specs', + region_sets=region_sets, + enabled=enabled, + provider_requirements=provider_requirements, + ) + + def get_any_worker_service(self, ctxt, random_choice=False, raise_if_none=True): services = self.get_workers_for_specs(ctxt) if not services: if raise_if_none: @@ -57,25 +56,38 @@ def get_any_worker_service( if random_choice: service = random.choice(services) LOG.debug( - "Selected service with ID '%s' for any-worker request.", - service['id']) + "Selected service with ID '%s' for any-worker request.", service['id'] + ) return service def get_worker_service_for_specs( - self, ctxt, provider_requirements=None, region_sets=None, - enabled=True, random_choice=False, raise_on_no_matches=True): + self, + ctxt, + provider_requirements=None, + region_sets=None, + enabled=True, + random_choice=False, + raise_on_no_matches=True, + ): """Utility method which ensures at least one service matching the provided requirements exists and is usable. """ - requirements_str = ( - "enabled=%s; region_sets=%s; provider_requirements=%s" % ( - enabled, region_sets, provider_requirements)) + requirements_str = "enabled=%s; region_sets=%s; provider_requirements=%s" % ( + enabled, + region_sets, + provider_requirements, + ) LOG.info( "Requesting Worker Service from scheduler with the following " - "specifications: %s", requirements_str) + "specifications: %s", + requirements_str, + ) services = self.get_workers_for_specs( - ctxt, provider_requirements=provider_requirements, - region_sets=region_sets, enabled=enabled) + ctxt, + provider_requirements=provider_requirements, + region_sets=region_sets, + enabled=enabled, + ) if not services: if raise_on_no_matches: raise exception.NoSuitableWorkerServiceError() @@ -83,7 +95,9 @@ def get_worker_service_for_specs( LOG.debug( "Was offered Worker Services with the following IDs for " "requirements '%s': %s", - requirements_str, [s["id"] for s in services]) + requirements_str, + [s["id"] for s in services], + ) selected_service = services[0] if random_choice: @@ -91,13 +105,22 @@ def get_worker_service_for_specs( LOG.info( "Was offered Worker Service with ID '%s' for requirements: %s", - selected_service['id'], requirements_str) + selected_service['id'], + requirements_str, + ) return selected_service def get_worker_service_for_task( - self, ctxt, task, origin_endpoint, destination_endpoint, - retry_count=5, retry_period=2, random_choice=True): - """ Gets a worker service for the task with the given properties + self, + ctxt, + task, + origin_endpoint, + destination_endpoint, + retry_count=5, + retry_period=2, + random_choice=True, + ): + """Gets a worker service for the task with the given properties and source/target endpoints. :param task: Dict of the form: { @@ -111,64 +134,84 @@ def get_worker_service_for_task( LOG.debug( "Compiling required Worker Service specs for task with " "ID '%s' (type '%s') from endpoints '%s' to '%s'", - task['id'], task['task_type'], origin_endpoint['id'], - destination_endpoint['id']) - task_cls = tasks_factory.get_task_runner_class( - task['task_type']) + task['id'], + task['task_type'], + origin_endpoint['id'], + destination_endpoint['id'], + ) + task_cls = tasks_factory.get_task_runner_class(task['task_type']) # determine required Coriolis regions based on the endpoints: required_region_sets = [] origin_endpoint_region_ids = [ - r['id'] for r in origin_endpoint['mapped_regions']] + r['id'] for r in origin_endpoint['mapped_regions'] + ] destination_endpoint_region_ids = [ - r['id'] for r in destination_endpoint['mapped_regions']] + r['id'] for r in destination_endpoint['mapped_regions'] + ] required_platform = task_cls.get_required_platform() if required_platform in ( - constants.TASK_PLATFORM_SOURCE, - constants.TASK_PLATFORM_BILATERAL): + constants.TASK_PLATFORM_SOURCE, + constants.TASK_PLATFORM_BILATERAL, + ): required_region_sets.append(origin_endpoint_region_ids) if required_platform in ( - constants.TASK_PLATFORM_DESTINATION, - constants.TASK_PLATFORM_BILATERAL): + constants.TASK_PLATFORM_DESTINATION, + constants.TASK_PLATFORM_BILATERAL, + ): required_region_sets.append(destination_endpoint_region_ids) # determine provider requirements: provider_requirements = {} required_provider_types = task_cls.get_required_provider_types() if constants.PROVIDER_PLATFORM_SOURCE in required_provider_types: - provider_requirements[origin_endpoint['type']] = ( - required_provider_types[ - constants.PROVIDER_PLATFORM_SOURCE]) + provider_requirements[origin_endpoint['type']] = required_provider_types[ + constants.PROVIDER_PLATFORM_SOURCE + ] if constants.PROVIDER_PLATFORM_DESTINATION in required_provider_types: provider_requirements[destination_endpoint['type']] = ( - required_provider_types[ - constants.PROVIDER_PLATFORM_DESTINATION]) + required_provider_types[constants.PROVIDER_PLATFORM_DESTINATION] + ) worker_service = None for i in range(retry_count): try: LOG.debug( "Requesting Worker Service for task with ID '%s' (type " - "'%s') from endpoints '%s' to '%s'", task['id'], - task['task_type'], origin_endpoint['id'], - destination_endpoint['id']) + "'%s') from endpoints '%s' to '%s'", + task['id'], + task['task_type'], + origin_endpoint['id'], + destination_endpoint['id'], + ) worker_service = self.get_worker_service_for_specs( - ctxt, provider_requirements=provider_requirements, - region_sets=required_region_sets, enabled=True, - random_choice=random_choice) + ctxt, + provider_requirements=provider_requirements, + region_sets=required_region_sets, + enabled=True, + random_choice=random_choice, + ) LOG.debug( "Scheduler has granted Worker Service '%s' for task with " "ID '%s' (type '%s') from endpoints '%s' to '%s'", - worker_service['id'], task['id'], task['task_type'], - origin_endpoint['id'], destination_endpoint['id']) + worker_service['id'], + task['id'], + task['task_type'], + origin_endpoint['id'], + destination_endpoint['id'], + ) return worker_service except Exception: LOG.warn( "Failed to schedule task with ID '%s' (attempt %d/%d). " "Waiting %d seconds and then retrying. Error was: %s", - task['id'], i + 1, retry_count, retry_period, - utils.get_exception_details()) + task['id'], + i + 1, + retry_count, + retry_period, + utils.get_exception_details(), + ) time.sleep(retry_period) message = ( @@ -176,8 +219,9 @@ def get_worker_service_for_task( " there are no Coriolis Worker services able to perform the task " "on the platforms and in the Coriolis Regions required by the " "selected source/destination Coriolis Endpoints. Please review" - " the Conductor and Scheduler logs for more exact details." % ( - task['id'], retry_count)) + " the Conductor and Scheduler logs for more exact details." + % (task['id'], retry_count) + ) # db_api.set_task_status( # ctxt, task.id, constants.TASK_STATUS_FAILED_TO_SCHEDULE, # exception_details=message) diff --git a/coriolis/scheduler/rpc/server.py b/coriolis/scheduler/rpc/server.py index bec093e6..27b5f68b 100644 --- a/coriolis/scheduler/rpc/server.py +++ b/coriolis/scheduler/rpc/server.py @@ -4,13 +4,10 @@ from oslo_config import cfg from oslo_log import log as logging +from coriolis import constants, exception, utils from coriolis.conductor.rpc import client as rpc_conductor_client -from coriolis import constants from coriolis.db import api as db_api -from coriolis import exception from coriolis.scheduler.filters import trivial_filters -from coriolis import utils - VERSION = "1.0" @@ -33,21 +30,21 @@ def get_diagnostics(self, ctxt): def _get_all_worker_services(self, ctxt): services = db_api.get_services(ctxt) services = trivial_filters.TopicFilter( - constants.WORKER_MAIN_MESSAGING_TOPIC).filter_services( - services) + constants.WORKER_MAIN_MESSAGING_TOPIC + ).filter_services(services) if not services: raise exception.NoWorkerServiceError() return services def _get_weighted_filtered_services( - self, services, filters, minimum_per_filter_rating=1): - """ Returns list of services and their scores for the given filters. + self, services, filters, minimum_per_filter_rating=1 + ): + """Returns list of services and their scores for the given filters. Services which are rejected by any filter will be excluded. """ if not filters: - LOG.warn( - "No filters provided. Presuming all services acceptable.") + LOG.warn("No filters provided. Presuming all services acceptable.") return [(service, 100) for service in services] scores = [] @@ -55,7 +52,9 @@ def _get_weighted_filtered_services( service_ids = [service.id for service in services] LOG.debug( "Running following filters on worker services '%s': %s", - service_ids, filters) + service_ids, + filters, + ) for service in services: total_score = 0 @@ -69,8 +68,8 @@ def _get_weighted_filtered_services( total_score = total_score + rating if not acceptable: LOG.debug( - "Service with ID '%s' was rejected by filter %r", - service.id, flt) + "Service with ID '%s' was rejected by filter %r", service.id, flt + ) continue scores.append((service, total_score)) @@ -79,21 +78,22 @@ def _get_weighted_filtered_services( message = ( "None of the inspected Coriolis Worker services (IDs %s) " "matched the requested filtering criteria (minimum score %d) " - "for the following required filters: %s" % ( - [s.id for s in services], - minimum_per_filter_rating, filters)) + "for the following required filters: %s" + % ([s.id for s in services], minimum_per_filter_rating, filters) + ) raise exception.NoSuitableWorkerServiceError(message) LOG.debug( - "Determined following scores for services based on filters '%s': " - "%s", filters, scores) + "Determined following scores for services based on filters '%s': %s", + filters, + scores, + ) - return sorted( - scores, key=lambda s: s[1], reverse=True) + return sorted(scores, key=lambda s: s[1], reverse=True) def _filter_regions( - self, ctxt, region_ids, enabled=True, check_all_exist=True, - regions_cache=None): + self, ctxt, region_ids, enabled=True, check_all_exist=True, regions_cache=None + ): found_regions = [] filtered_regions = [] regions = regions_cache @@ -107,20 +107,25 @@ def _filter_regions( filtered_regions.append(region) if check_all_exist: - missing_regions = set(region_ids).difference( - set(found_regions)) + missing_regions = set(region_ids).difference(set(found_regions)) if missing_regions: raise exception.RegionNotFound( "Failed to schedule job on regions %s as one or more " - "of the proposed regions (%s) do not exist." % ( - region_ids, missing_regions)) + "of the proposed regions (%s) do not exist." + % (region_ids, missing_regions) + ) return filtered_regions def get_workers_for_specs( - self, ctxt, provider_requirements=None, - region_sets=None, enabled=None, filter_disabled_regions=True): - """ Returns a list of enabled Worker Services with the specified + self, + ctxt, + provider_requirements=None, + region_sets=None, + enabled=None, + filter_disabled_regions=True, + ): + """Returns a list of enabled Worker Services with the specified parameters. :param provider_requirements: dict of the form { "": [constants.PROVIDER_TYPE_*, ...]} @@ -132,9 +137,13 @@ def get_workers_for_specs( worker_services = self._get_all_worker_services(ctxt) LOG.debug( - "Searching for Worker Services with specs: %s" % { + "Searching for Worker Services with specs: %s" + % { "provider_requirements": provider_requirements, - "region_sets": region_sets, "enabled": enabled}) + "region_sets": region_sets, + "enabled": enabled, + } + ) if enabled is not None: filters.append(trivial_filters.EnabledFilter(enabled=enabled)) @@ -143,25 +152,35 @@ def get_workers_for_specs( if not region_set: continue filtered_regions = self._filter_regions( - ctxt, region_set, enabled=filter_disabled_regions, - check_all_exist=True) + ctxt, + region_set, + enabled=filter_disabled_regions, + check_all_exist=True, + ) if not filtered_regions: raise exception.NoSuitableRegionError( "None of the selected Regions (%s) are enabled or " - "otherwise usable." % region_set) + "otherwise usable." % region_set + ) filters.append( - trivial_filters.RegionsFilter( - region_set, any_region=True)) + trivial_filters.RegionsFilter(region_set, any_region=True) + ) if provider_requirements: - filters.append( - trivial_filters.ProviderTypesFilter(provider_requirements)) + filters.append(trivial_filters.ProviderTypesFilter(provider_requirements)) filtered_services = self._get_weighted_filtered_services( - worker_services, filters) + worker_services, filters + ) LOG.info( - "Found Worker Services %s for specs: %s" % ( - filtered_services, { + "Found Worker Services %s for specs: %s" + % ( + filtered_services, + { "provider_requirements": provider_requirements, - "region_sets": region_sets, "enabled": enabled})) + "region_sets": region_sets, + "enabled": enabled, + }, + ) + ) return [s[0] for s in filtered_services] diff --git a/coriolis/scheduler/scheduler_utils.py b/coriolis/scheduler/scheduler_utils.py index defd6dca..a2493c13 100644 --- a/coriolis/scheduler/scheduler_utils.py +++ b/coriolis/scheduler/scheduler_utils.py @@ -5,24 +5,20 @@ from oslo_log import log as logging -from coriolis import constants +from coriolis import constants, exception from coriolis.db import api as db_api -from coriolis import exception from coriolis.scheduler.rpc import client as rpc_scheduler_client from coriolis.transfer_cron.rpc import client as rpc_cron_client from coriolis.worker.rpc import client as rpc_worker_client - VERSION = "1.0" LOG = logging.getLogger(__name__) RPC_TOPIC_TO_CLIENT_CLASS_MAP = { constants.WORKER_MAIN_MESSAGING_TOPIC: rpc_worker_client.WorkerClient, - constants.SCHEDULER_MAIN_MESSAGING_TOPIC: ( - rpc_scheduler_client.SchedulerClient), - constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC: ( - rpc_cron_client.TransferCronClient) + constants.SCHEDULER_MAIN_MESSAGING_TOPIC: (rpc_scheduler_client.SchedulerClient), + constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC: (rpc_cron_client.TransferCronClient), } @@ -30,23 +26,22 @@ def get_rpc_client_for_service(service, *client_args, **client_kwargs): rpc_client_class = RPC_TOPIC_TO_CLIENT_CLASS_MAP.get(service.topic) if not rpc_client_class: raise exception.NotFound( - "No RPC client class for service with topic '%s'." % ( - service.topic)) + "No RPC client class for service with topic '%s'." % (service.topic) + ) topic = service.topic if service.topic == constants.WORKER_MAIN_MESSAGING_TOPIC: # NOTE: coriolis.service.MessagingService-type services (such # as the worker), always have a dedicated per-host queue # which can be used to target the service: - topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ({ - "main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC, - "host": service.host}) + topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ( + {"main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC, "host": service.host} + ) return rpc_client_class(*client_args, topic=topic, **client_kwargs) -def get_any_worker_service( - scheduler_client, ctxt, random_choice=False, raw_dict=False): +def get_any_worker_service(scheduler_client, ctxt, random_choice=False, raw_dict=False): services = scheduler_client.get_workers_for_specs(ctxt) if not services: raise exception.NoWorkerServiceError() @@ -60,8 +55,9 @@ def get_any_worker_service( def get_worker_rpc_for_host(host, *client_args, **client_kwargs): rpc_client_class = RPC_TOPIC_TO_CLIENT_CLASS_MAP[ - constants.WORKER_MAIN_MESSAGING_TOPIC] - topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ({ - "main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC, - "host": host}) + constants.WORKER_MAIN_MESSAGING_TOPIC + ] + topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ( + {"main_topic": constants.WORKER_MAIN_MESSAGING_TOPIC, "host": host} + ) return rpc_client_class(*client_args, topic=topic, **client_kwargs) diff --git a/coriolis/schemas.py b/coriolis/schemas.py index 3a84be73..3163b86c 100644 --- a/coriolis/schemas.py +++ b/coriolis/schemas.py @@ -9,9 +9,7 @@ import jsonschema from oslo_log import log as logging -from coriolis import exception -from coriolis import utils - +from coriolis import exception, utils LOG = logging.getLogger(__name__) @@ -23,9 +21,11 @@ PROVIDER_TARGET_ENVIRONMENT_SCHEMA_NAME = "target_environment_schema.json" PROVIDER_SOURCE_ENVIRONMENT_SCHEMA_NAME = "source_environment_schema.json" PROVIDER_SOURCE_MINION_POOL_ENVIRONMENT_SCHEMA_NAME = ( - "source_minion_pool_environment_schema.json") + "source_minion_pool_environment_schema.json" +) PROVIDER_DESTINATION_MINION_POOL_ENVIRONMENT_SCHEMA_NAME = ( - "destination_minion_pool_environment_schema.json") + "destination_minion_pool_environment_schema.json" +) _CORIOLIS_VM_EXPORT_INFO_SCHEMA_NAME = "vm_export_info_schema.json" _CORIOLIS_VM_INSTANCE_INFO_SCHEMA_NAME = "vm_instance_info_schema.json" @@ -38,18 +38,19 @@ _CORIOLIS_STORAGE_MAPPINGS_SCHEMA_NAME = "storage_mappings_schema.json" _CORIOLIS_VM_STORAGE_SCHEMA_NAME = "vm_storage_schema.json" _CORIOLIS_VOLUMES_INFO_SCHEMA_NAME = "volumes_info_schema.json" -_CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA_NAME = ( - "disk_sync_resources_info_schema.json") +_CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA_NAME = "disk_sync_resources_info_schema.json" _CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA_NAME = ( - "disk_sync_resources_conn_info_schema.json") + "disk_sync_resources_conn_info_schema.json" +) _CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA_NAME = ( - "replication_worker_conn_info_schema.json") + "replication_worker_conn_info_schema.json" +) _CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA_NAME = ( - "detected_os_morphing_info_schema.json") + "detected_os_morphing_info_schema.json" +) -def get_schema(package_name, schema_name, - schemas_directory=DEFAULT_SCHEMAS_DIRECTORY): +def get_schema(package_name, schema_name, schemas_directory=DEFAULT_SCHEMAS_DIRECTORY): """Loads the schema using jinja2 template loading. Loads the schema with the given 'schema_name' using jinja2 template @@ -69,16 +70,21 @@ def get_schema(package_name, schema_name, LOG.trace( "Failed to load module '%s' using Jinja2. Attempting to load " "parent '%s'. Error was: %s", - package_name, parent_package, str(ex)) - package_loader = jinja2.PackageLoader( - parent_package, schemas_directory) + package_name, + parent_package, + str(ex), + ) + package_loader = jinja2.PackageLoader(parent_package, schemas_directory) template_env = jinja2.Environment(loader=package_loader) schema = json.loads(template_env.get_template(schema_name).render()) - LOG.debug("Succesfully loaded and parsed schema '%s' from '%s'.", - schema_name, package_name) + LOG.debug( + "Succesfully loaded and parsed schema '%s' from '%s'.", + schema_name, + package_name, + ) return schema @@ -94,11 +100,12 @@ def validate_value(val, schema, format_checker=None, raise_on_error=True): except jsonschema.exceptions.ValidationError as ex: if raise_on_error: raise exception.SchemaValidationException( - "Schema validation failed: %s" % str(ex)) + "Schema validation failed: %s" % str(ex) + ) else: LOG.warn( - "Schema validation failed, ignoring: %s", - utils.get_exception_details()) + "Schema validation failed, ignoring: %s", utils.get_exception_details() + ) return False return True @@ -115,46 +122,51 @@ def validate_string(json_string, schema): # Global schemas CORIOLIS_VM_EXPORT_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_VM_EXPORT_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_VM_EXPORT_INFO_SCHEMA_NAME +) CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA = get_schema( - __name__, _CORIOLIS_OS_MORPHING_RES_SCHEMA_NAME) + __name__, _CORIOLIS_OS_MORPHING_RES_SCHEMA_NAME +) CORIOLIS_VM_INSTANCE_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_VM_INSTANCE_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_VM_INSTANCE_INFO_SCHEMA_NAME +) -CORIOLIS_VM_NETWORK_SCHEMA = get_schema( - __name__, _CORIOLIS_VM_NETWORK_SCHEMA_NAME) +CORIOLIS_VM_NETWORK_SCHEMA = get_schema(__name__, _CORIOLIS_VM_NETWORK_SCHEMA_NAME) -SCHEDULE_API_BODY_SCHEMA = get_schema( - __name__, _SCHEDULE_API_BODY_SCHEMA_NAME) +SCHEDULE_API_BODY_SCHEMA = get_schema(__name__, _SCHEDULE_API_BODY_SCHEMA_NAME) CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA = get_schema( - __name__, _CORIOLIS_DESTINATION_OPTIONS_SCHEMA_NAME) + __name__, _CORIOLIS_DESTINATION_OPTIONS_SCHEMA_NAME +) CORIOLIS_SOURCE_ENVIRONMENT_OPTIONS_SCHEMA = get_schema( - __name__, _CORIOLIS_SOURCE_OPTIONS_SCHEMA_NAME) + __name__, _CORIOLIS_SOURCE_OPTIONS_SCHEMA_NAME +) -CORIOLIS_NETWORK_MAP_SCHEMA = get_schema( - __name__, _CORIOLIS_NETWORK_MAP_SCHEMA_NAME) +CORIOLIS_NETWORK_MAP_SCHEMA = get_schema(__name__, _CORIOLIS_NETWORK_MAP_SCHEMA_NAME) CORIOLIS_STORAGE_MAPPINGS_SCHEMA = get_schema( - __name__, _CORIOLIS_STORAGE_MAPPINGS_SCHEMA_NAME) + __name__, _CORIOLIS_STORAGE_MAPPINGS_SCHEMA_NAME +) -CORIOLIS_VM_STORAGE_SCHEMA = get_schema( - __name__, _CORIOLIS_VM_STORAGE_SCHEMA_NAME) +CORIOLIS_VM_STORAGE_SCHEMA = get_schema(__name__, _CORIOLIS_VM_STORAGE_SCHEMA_NAME) -CORIOLIS_VOLUMES_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_VOLUMES_INFO_SCHEMA_NAME) +CORIOLIS_VOLUMES_INFO_SCHEMA = get_schema(__name__, _CORIOLIS_VOLUMES_INFO_SCHEMA_NAME) CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA_NAME +) CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA_NAME +) CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA_NAME +) CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA = get_schema( - __name__, _CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA_NAME) + __name__, _CORIOLIS_DETECTED_OS_MORPHING_INFO_SCHEMA_NAME +) diff --git a/coriolis/schemas_exceptions.py b/coriolis/schemas_exceptions.py index bff242f4..5ab5570f 100644 --- a/coriolis/schemas_exceptions.py +++ b/coriolis/schemas_exceptions.py @@ -11,22 +11,25 @@ class CoriolisSchemaException(exception.CoriolisException): """Base class for all coriolis schema handling exceptions.""" + message = "Exception occured during schema validation: %(msg)s." class CoriolisSchemaValidationError( - CoriolisSchemaException, jsonschema.ValidationError): + CoriolisSchemaException, jsonschema.ValidationError +): """Raised when a schema validation has failed.""" + message = "Failed to validate JSON schema: %(msg)s." -class CoriolisSchemaParsingError( - CoriolisSchemaException, ValueError): +class CoriolisSchemaParsingError(CoriolisSchemaException, ValueError): """Raised when decoding a JSON schema or when validating a JSON value.""" + message = "Failed to parse JSON for schema validation: %(msg)s." -class CoriolisSchemaLoadingException( - CoriolisSchemaException, jinja2.TemplateNotFound): +class CoriolisSchemaLoadingException(CoriolisSchemaException, jinja2.TemplateNotFound): """Raised when schema files are not found.""" + message = "Failed to load schema: %(msg)s." diff --git a/coriolis/secrets.py b/coriolis/secrets.py index cec04879..ffba81ca 100644 --- a/coriolis/secrets.py +++ b/coriolis/secrets.py @@ -4,13 +4,11 @@ import copy import json -from barbicanclient import client as barbican_client import keystoneauth1 +from barbicanclient import client as barbican_client from oslo_log import log as logging -from coriolis import keystone -from coriolis import utils - +from coriolis import keystone, utils LOG = logging.getLogger(__name__) @@ -32,7 +30,9 @@ def get_secret(ctxt, secret_ref): except keystoneauth1.exceptions.http.Unauthorized: LOG.debug( "Error occured while fetching secret with trust ID, retrying " - "without. Error was: %s", utils.get_exception_details()) + "without. Error was: %s", + utils.get_exception_details(), + ) ctxt = copy.deepcopy(ctxt) ctxt.trust_id = None payload = _get_barbican_secret_payload(ctxt, secret_ref) diff --git a/coriolis/service.py b/coriolis/service.py index 4d21f447..9c54d393 100644 --- a/coriolis/service.py +++ b/coriolis/service.py @@ -7,31 +7,37 @@ import sys import threading +import oslo_messaging as messaging from cheroot import wsgi from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging -import oslo_messaging as messaging from oslo_service import service from oslo_service import wsgi as oslo_wsgi -from coriolis import rpc -from coriolis import utils - +from coriolis import rpc, utils service_opts = [ - cfg.StrOpt('api_migration_listen', - default="0.0.0.0", - help='IP address on which the Migration API listens'), - cfg.PortOpt('api_migration_listen_port', - default=7667, - help='Port on which the Migration API listens'), - cfg.IntOpt('api_migration_workers', - help='Number of workers for the Migration API service. ' - 'The default is equal to the number of CPUs available.'), - cfg.IntOpt('messaging_workers', - help='Number of workers for the messaging service. ' - 'The default is equal to the number of CPUs available.'), + cfg.StrOpt( + 'api_migration_listen', + default="0.0.0.0", + help='IP address on which the Migration API listens', + ), + cfg.PortOpt( + 'api_migration_listen_port', + default=7667, + help='Port on which the Migration API listens', + ), + cfg.IntOpt( + 'api_migration_workers', + help='Number of workers for the Migration API service. ' + 'The default is equal to the number of CPUs available.', + ), + cfg.IntOpt( + 'messaging_workers', + help='Number of workers for the messaging service. ' + 'The default is equal to the number of CPUs available.', + ), ] CONF = cfg.CONF @@ -40,7 +46,7 @@ def get_worker_count_from_args(argv): - """ Parses the args for '--worker-process-count' and returns a tuple + """Parses the args for '--worker-process-count' and returns a tuple containing the count (defaults to logical CPU count if --worker-process-count is not present), as well as the unprocessed args. """ @@ -51,19 +57,23 @@ def _check_positive_worker_count(worker_count): if count <= 0: raise argparse.ArgumentTypeError( "Worker process count must be a strictly positive integer, " - "got: %s" % worker_count) + "got: %s" % worker_count + ) return count + parser.add_argument( - '--worker-process-count', metavar='N', + '--worker-process-count', + metavar='N', type=_check_positive_worker_count, help="Number of worker processes for this service. Defaults to the " - "number of logical CPU cores on the system.") + "number of logical CPU cores on the system.", + ) args, unknown_args = parser.parse_known_args(args=argv) return args.worker_process_count, unknown_args def check_locks_dir_empty(): - """ Checks whether the locks dir is empty and warns otherwise. + """Checks whether the locks dir is empty and warns otherwise. NOTE: external oslo_concurrency locks work based on listing open file descriptors so this check is not necessarily conclusive, though all freshly @@ -80,27 +90,26 @@ def check_locks_dir_empty(): return if not os.path.exists(locks_dir): - LOG.warn( - "Configured 'lock_path' directory '%s' does NOT exist!", - locks_dir) + LOG.warn("Configured 'lock_path' directory '%s' does NOT exist!", locks_dir) return if not os.path.isdir(locks_dir): - LOG.warn( - "Configured 'lock_path' directory '%s' is NOT a directory!", - locks_dir) + LOG.warn("Configured 'lock_path' directory '%s' is NOT a directory!", locks_dir) return locks_dir_contents = os.listdir(locks_dir) if locks_dir_contents: LOG.warn( "Configured 'lock_path' directory '%s' is NOT empty: %s", - locks_dir, locks_dir_contents) + locks_dir, + locks_dir_contents, + ) return LOG.info( "Successfully checked 'lock_path' directory '%s' exists and is empty.", - locks_dir) + locks_dir, + ) def get_application(): @@ -126,16 +135,16 @@ def __init__(self, name, worker_count=None, init_rpc=True): self._workers = int(worker_count) else: self._workers = ( - CONF.api_migration_workers or processutils.get_worker_count()) + CONF.api_migration_workers or processutils.get_worker_count() + ) self._loader = oslo_wsgi.Loader(CONF) self._app = self._loader.load_app(name) bind_addr = (self._host, self._port) self._server = wsgi.Server( - bind_addr=bind_addr, - wsgi_app=self._app, - server_name=name) + bind_addr=bind_addr, wsgi_app=self._app, server_name=name + ) def get_workers_count(self): return self._workers @@ -143,10 +152,7 @@ def get_workers_count(self): def start(self): self._server.prepare() - self._thread = threading.Thread( - target=self._server.serve, - daemon=True - ) + self._thread = threading.Thread(target=self._server.serve, daemon=True) self._thread.start() def stop(self): @@ -164,13 +170,12 @@ def reset(self): class MessagingService(service.ServiceBase): - def __init__(self, topic, endpoints, version, - worker_count=None, init_rpc=True): + def __init__(self, topic, endpoints, version, worker_count=None, init_rpc=True): if init_rpc: rpc.init() - target = messaging.Target(topic=topic, - server=utils.get_hostname(), - version=version) + target = messaging.Target( + topic=topic, server=utils.get_hostname(), version=version + ) self._server = rpc.get_server(target, endpoints) # NOTE: oslo_service fork()'s, which won't work on Windows... diff --git a/coriolis/services/api.py b/coriolis/services/api.py index 958989d6..6e326088 100644 --- a/coriolis/services/api.py +++ b/coriolis/services/api.py @@ -8,15 +8,13 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create( - self, ctxt, host, binary, topic, mapped_regions, - enabled): + def create(self, ctxt, host, binary, topic, mapped_regions, enabled): return self._rpc_client.register_service( - ctxt, host, binary, topic, enabled, mapped_regions) + ctxt, host, binary, topic, enabled, mapped_regions + ) def update(self, ctxt, service_id, updated_values): - return self._rpc_client.update_service( - ctxt, service_id, updated_values) + return self._rpc_client.update_service(ctxt, service_id, updated_values) def delete(self, ctxt, region_id): self._rpc_client.delete_service(ctxt, region_id) diff --git a/coriolis/taskflow/base.py b/coriolis/taskflow/base.py index 167adc56..4297b143 100644 --- a/coriolis/taskflow/base.py +++ b/coriolis/taskflow/base.py @@ -6,33 +6,31 @@ from taskflow import task as taskflow_tasks from taskflow.types import failure -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, utils from coriolis.scheduler.rpc import client as rpc_scheduler_client from coriolis.tasks import factory as tasks_factory -from coriolis import utils from coriolis.worker.rpc import client as rpc_worker_client - -TASK_RETURN_VALUE_FORMAT = "%s-result" % ( - constants.TASK_LOCK_NAME_FORMAT) +TASK_RETURN_VALUE_FORMAT = "%s-result" % (constants.TASK_LOCK_NAME_FORMAT) LOG = logging.getLogger(__name__) taskflow_opts = [ cfg.IntOpt( - "worker_task_execution_timeout", default=3600, + "worker_task_execution_timeout", + default=3600, help="Number of seconds until Coriolis tasks which are executed" - "remotely on a Worker Service through taskflow timeout.")] + "remotely on a Worker Service through taskflow timeout.", + ) +] CONF = cfg.CONF CONF.register_opts(taskflow_opts, 'taskflow') class BaseCoriolisTaskflowTask(taskflow_tasks.Task): - """ Base class for all TaskFlow tasks within Coriolis. """ + """Base class for all TaskFlow tasks within Coriolis.""" - def _get_error_str_for_flow_failures( - self, flow_failures, full_tracebacks=True): + def _get_error_str_for_flow_failures(self, flow_failures, full_tracebacks=True): if not flow_failures: return "No flow failures provided." @@ -40,7 +38,7 @@ def _get_error_str_for_flow_failures( return "No flow failures present." res = "" - for (task_id, task_failure) in flow_failures.items(): + for task_id, task_failure in flow_failures.items(): label = "Error message" failure_str = task_failure.exception_str if full_tracebacks: @@ -48,9 +46,7 @@ def _get_error_str_for_flow_failures( failure_str = task_failure.traceback_str else: failure_str = task_failure.exception_str - if isinstance( - task_failure.exception, - exception.TaskProcessException): + if isinstance(task_failure.exception, exception.TaskProcessException): # NOTE: TaskProcessException contains a full trace # from the worker service so we must split it: exception_lines = task_failure.exception_str.split('\n') @@ -59,9 +55,7 @@ def _get_error_str_for_flow_failures( failure_str = exception_lines[-2].strip() else: failure_str = exception_lines[-1].strip() - res = ( - "%s %s for task '%s': %s\n" % ( - res, label, task_id, failure_str)) + res = "%s %s for task '%s': %s\n" % (res, label, task_id, failure_str) if res: # remove extra newline: res = res[:-1] @@ -74,20 +68,28 @@ def revert(self, *args, **kwargs): # it means that this is the task which error'd out: LOG.error( "Taskflow task '%s' is reverting after errorring out with the " - "following trace: %s", self.name, result.traceback_str) + "following trace: %s", + self.name, + result.traceback_str, + ) else: # else the failures were from other tasks: flow_failures = kwargs.get('flow_failures', {}) LOG.error( "Taskflow task '%s' is reverting after the failure of one " - "or more other tasks (%s) Tracebacks were:\n%s" % ( - self.name, list(flow_failures.keys()), + "or more other tasks (%s) Tracebacks were:\n%s" + % ( + self.name, + list(flow_failures.keys()), self._get_error_str_for_flow_failures( - flow_failures, full_tracebacks=True))) + flow_failures, full_tracebacks=True + ), + ) + ) class BaseRunWorkerTask(BaseCoriolisTaskflowTask): - """ Base taskflow.Task implementation for tasks which can be run + """Base taskflow.Task implementation for tasks which can be run on the worker service. This class can be seen as an "adapter" between the current coriolis.tasks.TaskRunner classes and taskflow ones. @@ -104,9 +106,16 @@ class BaseRunWorkerTask(BaseCoriolisTaskflowTask): """ def __init__( - self, task_name, task_id, task_instance, main_task_runner_type, - cleanup_task_runner_type=None, depends_on=None, - raise_on_cleanup_failure=False, **kwargs): + self, + task_name, + task_id, + task_instance, + main_task_runner_type, + cleanup_task_runner_type=None, + depends_on=None, + raise_on_cleanup_failure=False, + **kwargs, + ): self._task_id = task_id self._task_name = task_name self._task_instance = task_instance @@ -120,8 +129,7 @@ def __init__( @property def _scheduler_client(self): if not getattr(self, '_scheduler_client_instance', None): - self._scheduler_client_instance = ( - rpc_scheduler_client.SchedulerClient()) + self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance def _set_provides_for_dependencies(self, kwargs): @@ -132,9 +140,7 @@ def _set_provides_for_dependencies(self, kwargs): kwargs['provides'] = [dep] def _set_requires_for_dependencies(self, kwargs, depends_on): - dep_requirements = [ - TASK_RETURN_VALUE_FORMAT % dep_id - for dep_id in depends_on] + dep_requirements = [TASK_RETURN_VALUE_FORMAT % dep_id for dep_id in depends_on] if kwargs.get('requires') is not None: kwargs['requires'].extend(dep_requirements) elif dep_requirements: @@ -144,17 +150,19 @@ def _set_requires_for_dependencies(self, kwargs, depends_on): def _set_requires_for_task_info_fields(self, kwargs): new_requires = kwargs.get('requires', []) main_task_runner = tasks_factory.get_task_runner_class( - self._main_task_runner_type) + self._main_task_runner_type + ) main_task_deps = main_task_runner.get_required_task_info_properties() new_requires.extend(main_task_deps) if self._cleanup_task_runner_type: cleanup_task_runner = tasks_factory.get_task_runner_class( - self._cleanup_task_runner_type) + self._cleanup_task_runner_type + ) cleanup_task_deps = list( - set( - cleanup_task_runner.get_required_task_info_properties( - )).difference( - main_task_runner.get_returned_task_info_properties())) + set(cleanup_task_runner.get_required_task_info_properties()).difference( + main_task_runner.get_returned_task_info_properties() + ) + ) new_requires.extend(cleanup_task_deps) kwargs['requires'] = new_requires @@ -163,97 +171,141 @@ def _set_requires_for_task_info_fields(self, kwargs): def _set_provides_for_task_info_fields(self, kwargs): new_provides = kwargs.get('provides', []) main_task_runner = tasks_factory.get_task_runner_class( - self._main_task_runner_type) + self._main_task_runner_type + ) main_task_res = main_task_runner.get_returned_task_info_properties() new_provides.extend(main_task_res) if self._cleanup_task_runner_type: cleanup_task_runner = tasks_factory.get_task_runner_class( - self._cleanup_task_runner_type) + self._cleanup_task_runner_type + ) cleanup_task_res = list( - set( - cleanup_task_runner.get_returned_task_info_properties( - )).difference( - main_task_runner.get_returned_task_info_properties())) + set(cleanup_task_runner.get_returned_task_info_properties()).difference( + main_task_runner.get_returned_task_info_properties() + ) + ) new_provides.extend(cleanup_task_res) kwargs['provides'] = new_provides return kwargs def _get_worker_service_rpc_for_task( - self, ctxt, task_id, task_type, origin, destination, - retry_count=5, retry_period=2, - rpc_timeout=CONF.taskflow.worker_task_execution_timeout): - task_info = { - "id": task_id, - "task_type": task_type} + self, + ctxt, + task_id, + task_type, + origin, + destination, + retry_count=5, + retry_period=2, + rpc_timeout=CONF.taskflow.worker_task_execution_timeout, + ): + task_info = {"id": task_id, "task_type": task_type} worker_service = self._scheduler_client.get_worker_service_for_task( - ctxt, task_info, origin, destination, retry_count=retry_count, - retry_period=retry_period, random_choice=True) + ctxt, + task_info, + origin, + destination, + retry_count=retry_count, + retry_period=retry_period, + random_choice=True, + ) LOG.debug( "[Task '%s'] Was offered the following worker service for " "executing Taskflow worker task '%s': %s", - self._task_name, task_id, worker_service['id']) + self._task_name, + task_id, + worker_service['id'], + ) return rpc_worker_client.WorkerClient.from_service_definition( - worker_service, timeout=rpc_timeout) + worker_service, timeout=rpc_timeout + ) - def _execute_task( - self, ctxt, task_id, task_type, origin, destination, task_info): + def _execute_task(self, ctxt, task_id, task_type, origin, destination, task_info): worker_rpc = self._get_worker_service_rpc_for_task( - ctxt, self._task_id, task_type, origin, destination) + ctxt, self._task_id, task_type, origin, destination + ) try: LOG.debug( "[Task '%s'] Starting to run task '%s' (type '%s') " - "on worker service." % ( - self._task_id, self._task_name, task_type)) + "on worker service." % (self._task_id, self._task_name, task_type) + ) res = worker_rpc.run_task( - ctxt, self._task_id, task_type, origin, destination, - self._task_instance, task_info) + ctxt, + self._task_id, + task_type, + origin, + destination, + self._task_instance, + task_info, + ) LOG.debug( "[Task '%s'] Taskflow worker task '%s' (type %s) has " - "successfully run and returned the following info: %s" % ( - self._task_name, task_id, task_type, res)) + "successfully run and returned the following info: %s" + % (self._task_name, task_id, task_type, res) + ) return res except Exception: LOG.debug( "[Task %s] Exception occurred while executing task '%s' " - "(type '%s') on the worker service: %s", self._task_name, - task_id, task_type, utils.get_exception_details()) + "(type '%s') on the worker service: %s", + self._task_name, + task_id, + task_type, + utils.get_exception_details(), + ) raise def execute(self, context, origin, destination, task_info): res = self._execute_task( - context, self._task_id, self._main_task_runner_type, origin, - destination, task_info) + context, + self._task_id, + self._main_task_runner_type, + origin, + destination, + task_info, + ) return res def revert(self, context, origin, destination, task_info, **kwargs): super(BaseRunWorkerTask, self).revert( - context, origin, destination, task_info, **kwargs) + context, origin, destination, task_info, **kwargs + ) if not self._cleanup_task_runner_type: LOG.debug( "Task '%s' (main type '%s') had no cleanup task runner " "associated with it. Skipping any reversion logic", - self._task_name, self._main_task_runner_type) + self._task_name, + self._main_task_runner_type, + ) return try: res = self._execute_task( - context, self._task_id, self._cleanup_task_runner_type, - origin, destination, task_info) + context, + self._task_id, + self._cleanup_task_runner_type, + origin, + destination, + task_info, + ) except Exception: LOG.warn( "Task cleanup for '%s' (main task type '%s', cleanup task type" "'%s') has failed with the following trace: %s", - self._task_name, self._main_task_runner_type, - self._cleanup_task_runner_type, utils.get_exception_details()) + self._task_name, + self._main_task_runner_type, + self._cleanup_task_runner_type, + utils.get_exception_details(), + ) if self._raise_on_cleanup_failure: raise return LOG.debug( "Reversion of taskflow task '%s' (ID '%s') was successfully " - "executed using task runner '%s' with the following result: %s" % - (self._task_name, self._task_id, self._cleanup_task_runner_type, - res)) + "executed using task runner '%s' with the following result: %s" + % (self._task_name, self._task_id, self._cleanup_task_runner_type, res) + ) diff --git a/coriolis/taskflow/runner.py b/coriolis/taskflow/runner.py index 7e709cf9..0edab412 100644 --- a/coriolis/taskflow/runner.py +++ b/coriolis/taskflow/runner.py @@ -3,8 +3,8 @@ import multiprocessing import sys - from logging import handlers + from oslo_config import cfg from oslo_log import log as logging from six.moves import queue @@ -13,7 +13,6 @@ from coriolis import utils - LOG = logging.getLogger(__name__) TASKFLOW_EXECUTION_ORDER_PARALLEL = 'parallel' @@ -25,12 +24,13 @@ class TaskFlowRunner(object): - def __init__( - self, service_name, - execution_order=TASKFLOW_EXECUTION_ORDER_PARALLEL, - executor=TASKFLOW_EXECUTOR_THREADED, - max_workers=1): + self, + service_name, + execution_order=TASKFLOW_EXECUTION_ORDER_PARALLEL, + executor=TASKFLOW_EXECUTOR_THREADED, + max_workers=1, + ): self._service_name = service_name self._execution_order = execution_order @@ -41,24 +41,34 @@ def _log_flow_transition(self, state, details): LOG.debug( "[TaskFlowRunner(%s)] Flow '%s' (internal UUID '%s') transitioned" " from '%s' state to '%s'", - self._service_name, details['flow_name'], details['flow_uuid'], - details['old_state'], state) + self._service_name, + details['flow_name'], + details['flow_uuid'], + details['old_state'], + state, + ) def _log_task_transition(self, state, details): LOG.debug( "[TaskFlowRunner(%s)] Task '%s' (internal UUID '%s') transitioned" " from '%s' state to '%s'", - self._service_name, details['task_name'], details['task_uuid'], - details['old_state'], state) + self._service_name, + details['task_name'], + details['task_uuid'], + details['old_state'], + state, + ) def _setup_engine_for_flow(self, flow, store=None): engine = engines.load( - flow, store, executor=self._executor, - engine=self._execution_order, max_workers=self._max_workers) - engine.notifier.register( - notifier.Notifier.ANY, self._log_flow_transition) - engine.atom_notifier.register( - notifier.Notifier.ANY, self._log_task_transition) + flow, + store, + executor=self._executor, + engine=self._execution_order, + max_workers=self._max_workers, + ) + engine.notifier.register(notifier.Notifier.ANY, self._log_flow_transition) + engine.atom_notifier.register(notifier.Notifier.ANY, self._log_task_transition) return engine def _run_flow(self, flow, store=None): @@ -77,12 +87,16 @@ def _run_flow(self, flow, store=None): except Exception: LOG.warn( "Fatal error occurred while attempting to run flow '%s'. " - "Full trace was: %s", flow.name, - utils.get_exception_details()) + "Full trace was: %s", + flow.name, + utils.get_exception_details(), + ) raise LOG.info( "Successfully ran flow with name '%s'. Statistics were: %s", - flow.name, engine.statistics) + flow.name, + engine.statistics, + ) def run_flow(self, flow, store=None): self._run_flow(flow, store=store) @@ -118,13 +132,15 @@ def _spawn_process_flow(self, flow, store=None): mp_ctx = multiprocessing.get_context('spawn') mp_log_q = mp_ctx.Queue() process = mp_ctx.Process( - target=self._run_flow_in_process, - args=(flow, mp_log_q, store)) + target=self._run_flow_in_process, args=(flow, mp_log_q, store) + ) LOG.debug("Starting new background process for flow '%s'", flow.name) process.start() LOG.debug( - "Sucessfully started background process for flow '%s' with " - "PID: '%d'", flow.name, process.pid) + "Sucessfully started background process for flow '%s' with PID: '%d'", + flow.name, + process.pid, + ) # TODO(lpetrut): one logger thread per subprocess may be excessive when # having a large number of concurrent jobs. It may be worth having a @@ -135,12 +151,11 @@ def _spawn_process_flow(self, flow, store=None): # queues, we'd probably need pipes instead. There's also the option of # using select/poll/epoll directly. utils.start_thread( - target=self._handle_mp_log_events, - args=(process, mp_log_q), - daemon=True) + target=self._handle_mp_log_events, args=(process, mp_log_q), daemon=True + ) def run_flow_in_background(self, flow, store=None): - """ Starts the given flow in the background in a separate process. + """Starts the given flow in the background in a separate process. Does NOT return/store any result. All tasks must be "self-sufficient" and record their own results in diff --git a/coriolis/taskflow/utils.py b/coriolis/taskflow/utils.py index fdd5bf4b..9c47e7f6 100644 --- a/coriolis/taskflow/utils.py +++ b/coriolis/taskflow/utils.py @@ -3,12 +3,11 @@ from oslo_log import log as logging - LOG = logging.getLogger(__name__) class DummyDecider(object): - """ A callable to decide execution in a pre-defined manner. """ + """A callable to decide execution in a pre-defined manner.""" def __init__(self, allow=True): self._allow = allow @@ -16,5 +15,7 @@ def __init__(self, allow=True): def __call__(self, history): LOG.debug( "Dummy decider returning '%s'. Provided task history was: %s", - self._allow, history) + self._allow, + history, + ) return self._allow diff --git a/coriolis/tasks/base.py b/coriolis/tasks/base.py index aaa71f53..f5c11349 100644 --- a/coriolis/tasks/base.py +++ b/coriolis/tasks/base.py @@ -3,20 +3,20 @@ import abc +import paramiko from oslo_config import cfg from oslo_log import log as logging -import paramiko from six import with_metaclass -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, utils from coriolis.providers import factory as providers_factory -from coriolis import utils serialization_opts = [ - cfg.StrOpt('temp_keypair_password', - default=None, - help='Password to be used when serializing temporary keys'), + cfg.StrOpt( + 'temp_keypair_password', + default=None, + help='Password to be used when serializing temporary keys', + ), ] CONF = cfg.CONF @@ -25,132 +25,144 @@ class TaskRunner(with_metaclass(abc.ABCMeta)): - - def get_shared_libs_for_providers( - self, ctxt, origin, destination, event_handler): - """ Returns a list of directories containing libraries needed - for both the source and destination providers. """ + def get_shared_libs_for_providers(self, ctxt, origin, destination, event_handler): + """Returns a list of directories containing libraries needed + for both the source and destination providers.""" required_libs = [] platform = self.get_required_platform() if platform in [ - constants.TASK_PLATFORM_SOURCE, - constants.TASK_PLATFORM_BILATERAL]: + constants.TASK_PLATFORM_SOURCE, + constants.TASK_PLATFORM_BILATERAL, + ]: required_libs.extend( - get_shared_lib_dirs_for_provider(ctxt, origin, event_handler)) + get_shared_lib_dirs_for_provider(ctxt, origin, event_handler) + ) if platform in [ - constants.TASK_PLATFORM_DESTINATION, - constants.TASK_PLATFORM_BILATERAL]: + constants.TASK_PLATFORM_DESTINATION, + constants.TASK_PLATFORM_BILATERAL, + ]: required_libs.extend( - get_shared_lib_dirs_for_provider( - ctxt, destination, event_handler)) + get_shared_lib_dirs_for_provider(ctxt, destination, event_handler) + ) return required_libs @classmethod @abc.abstractmethod def get_required_task_info_properties(cls): - """ Returns a list of the string fields which are required - to be present during the tasks' run method. """ + """Returns a list of the string fields which are required + to be present during the tasks' run method.""" raise NotImplementedError( "No required task info properties specified for task class of " - "type '%s'." % cls) + "type '%s'." % cls + ) @classmethod @abc.abstractmethod def get_returned_task_info_properties(cls): - """ Returns a list of the string fields which are returned by the + """Returns a list of the string fields which are returned by the tasks' run method to be added to the task info. """ raise NotImplementedError( "No returned task info properties specified for task class of " - "type '%s'." % cls) + "type '%s'." % cls + ) @classmethod @abc.abstractmethod def get_required_provider_types(cls): - """ Returns a dict with 'source/destination' as keys containing a list + """Returns a dict with 'source/destination' as keys containing a list of all the provider types (constants.PROVIDER_TYPE_*) required for the task. """ raise NotImplementedError( - "No required provider types specified for task class of " - "type '%s'." % cls) + "No required provider types specified for task class of type '%s'." % cls + ) @classmethod @abc.abstractmethod def get_required_platform(cls): - """ Returns whether the task operates on the source platform, the + """Returns whether the task operates on the source platform, the destination, or both. (constants.TASK_PLATFORM_*) """ raise NotImplementedError( - "No required platform specified for task class of " - "type '%s'." % cls) + "No required platform specified for task class of type '%s'." % cls + ) @abc.abstractmethod - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): - """ The actual logic run by the task. + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + """The actual logic run by the task. Should return a dict with all the fields declared by 'self.get_returned_task_info_properties'. Must be implemented in all child classes. """ raise NotImplementedError( - "No base run method implemented for task class of type '%s'." % ( - self.__class__)) + "No base run method implemented for task class of type '%s'." + % (self.__class__) + ) - def run(self, ctxt, instance, origin, destination, task_info, - event_handler): - """ Runs the task with the given params and returns + def run(self, ctxt, instance, origin, destination, task_info, event_handler): + """Runs the task with the given params and returns a dict with the results. NOTE: This should NOT modify the existing task_info in any way. """ missing_info_props = [ - prop for prop in self.get_required_task_info_properties() - if prop not in task_info] + prop + for prop in self.get_required_task_info_properties() + if prop not in task_info + ] if missing_info_props: raise exception.CoriolisException( "Task type '%s' asked to run on task info with " - "missing properties: %s" % ( - self.__class__, missing_info_props)) + "missing properties: %s" % (self.__class__, missing_info_props) + ) result = self._run( - ctxt, instance, origin, destination, task_info, event_handler) + ctxt, instance, origin, destination, task_info, event_handler + ) if type(result) is not dict: raise exception.CoriolisException( "Task type '%s' returned result of type %s " - "instead of a dict: %s" % ( - self.__class__, type(result), result)) + "instead of a dict: %s" % (self.__class__, type(result), result) + ) missing_returns = [ - prop for prop in self.get_returned_task_info_properties() - if prop not in result.keys()] + prop + for prop in self.get_returned_task_info_properties() + if prop not in result.keys() + ] if missing_returns: raise exception.CoriolisException( "Task type '%s' failed to return the following " "declared return values in its result: %s. " - "Result was: %s" % ( - self.__class__, missing_returns, - utils.sanitize_task_info(result))) + "Result was: %s" + % (self.__class__, missing_returns, utils.sanitize_task_info(result)) + ) undeclared_returns = [ - prop for prop in result.keys() - if prop not in self.get_returned_task_info_properties()] + prop + for prop in result.keys() + if prop not in self.get_returned_task_info_properties() + ] if undeclared_returns: raise exception.CoriolisException( "Task type '%s' returned the following undeclared " - "keys in its result: %s" % ( - self.__class__, undeclared_returns)) + "keys in its result: %s" % (self.__class__, undeclared_returns) + ) return result def get_shared_lib_dirs_for_provider(ctxt, endpoint, event_handler): provider = providers_factory.get_provider( - endpoint['type'], constants.PROVIDER_TYPE_SETUP_LIBS, - event_handler, raise_if_not_found=False) + endpoint['type'], + constants.PROVIDER_TYPE_SETUP_LIBS, + event_handler, + raise_if_not_found=False, + ) if provider: conn_info = get_connection_info(ctxt, endpoint) return provider.get_shared_library_directories(ctxt, conn_info) @@ -162,27 +174,23 @@ def get_connection_info(ctxt, data): return utils.get_secret_connection_info(ctxt, connection_info) -def marshal_migr_conn_info( - migr_connection_info, private_key_field_name="pkey"): - if migr_connection_info and ( - private_key_field_name in migr_connection_info): +def marshal_migr_conn_info(migr_connection_info, private_key_field_name="pkey"): + if migr_connection_info and (private_key_field_name in migr_connection_info): migr_connection_info = migr_connection_info.copy() pkey = migr_connection_info[private_key_field_name] if isinstance(pkey, str) is False: - migr_connection_info[private_key_field_name] = ( - utils.serialize_key( - pkey, CONF.serialization.temp_keypair_password)) + migr_connection_info[private_key_field_name] = utils.serialize_key( + pkey, CONF.serialization.temp_keypair_password + ) return migr_connection_info -def unmarshal_migr_conn_info( - migr_connection_info, private_key_field_name="pkey"): - if migr_connection_info and ( - private_key_field_name in migr_connection_info): +def unmarshal_migr_conn_info(migr_connection_info, private_key_field_name="pkey"): + if migr_connection_info and (private_key_field_name in migr_connection_info): migr_connection_info = migr_connection_info.copy() pkey_str = migr_connection_info[private_key_field_name] if isinstance(pkey_str, paramiko.rsakey.RSAKey) is False: - migr_connection_info[private_key_field_name] = ( - utils.deserialize_key( - pkey_str, CONF.serialization.temp_keypair_password)) + migr_connection_info[private_key_field_name] = utils.deserialize_key( + pkey_str, CONF.serialization.temp_keypair_password + ) return migr_connection_info diff --git a/coriolis/tasks/factory.py b/coriolis/tasks/factory.py index e638ea45..601b6c10 100644 --- a/coriolis/tasks/factory.py +++ b/coriolis/tasks/factory.py @@ -1,128 +1,74 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis import constants -from coriolis import exception -from coriolis.tasks import migration_tasks -from coriolis.tasks import minion_pool_tasks -from coriolis.tasks import osmorphing_tasks -from coriolis.tasks import replica_tasks +from coriolis import constants, exception +from coriolis.tasks import ( + migration_tasks, + minion_pool_tasks, + osmorphing_tasks, + replica_tasks, +) _TASKS_MAP = { - constants.TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT: - migration_tasks.FinalizeInstanceDeploymentTask, - constants.TASK_TYPE_CLEANUP_FAILED_INSTANCE_DEPLOYMENT: - migration_tasks.CleanupFailedInstanceDeploymentTask, - constants.TASK_TYPE_GET_OPTIMAL_FLAVOR: - migration_tasks.GetOptimalFlavorTask, - constants.TASK_TYPE_DEPLOY_OS_MORPHING_RESOURCES: - osmorphing_tasks.DeployOSMorphingResourcesTask, - constants.TASK_TYPE_OS_MORPHING: - osmorphing_tasks.OSMorphingTask, - constants.TASK_TYPE_DELETE_OS_MORPHING_RESOURCES: - osmorphing_tasks.DeleteOSMorphingResourcesTask, - constants.TASK_TYPE_GET_INSTANCE_INFO: - replica_tasks.GetInstanceInfoTask, - constants.TASK_TYPE_REPLICATE_DISKS: - replica_tasks.ReplicateDisksTask, - constants.TASK_TYPE_SHUTDOWN_INSTANCE: - replica_tasks.ShutdownInstanceTask, - constants.TASK_TYPE_DEPLOY_TRANSFER_DISKS: - replica_tasks.DeployReplicaDisksTask, - constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS: - replica_tasks.DeleteReplicaSourceDiskSnapshotsTask, - constants.TASK_TYPE_DELETE_TRANSFER_DISKS: - replica_tasks.DeleteReplicaDisksTask, - constants.TASK_TYPE_DEPLOY_TRANSFER_TARGET_RESOURCES: - replica_tasks.DeployReplicaTargetResourcesTask, - constants.TASK_TYPE_DELETE_TRANSFER_TARGET_RESOURCES: - replica_tasks.DeleteReplicaTargetResourcesTask, - constants.TASK_TYPE_DEPLOY_TRANSFER_SOURCE_RESOURCES: - replica_tasks.DeployReplicaSourceResourcesTask, - constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_RESOURCES: - replica_tasks.DeleteReplicaSourceResourcesTask, - constants.TASK_TYPE_DEPLOY_INSTANCE_RESOURCES: - replica_tasks.DeployReplicaInstanceResourcesTask, - constants.TASK_TYPE_CREATE_TRANSFER_DISK_SNAPSHOTS: - replica_tasks.CreateReplicaDiskSnapshotsTask, - constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS: - replica_tasks.DeleteReplicaTargetDiskSnapshotsTask, - constants.TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS: - replica_tasks.RestoreReplicaDiskSnapshotsTask, - constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS: - replica_tasks.ValidateReplicaExecutionSourceInputsTask, - constants.TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS: - replica_tasks.ValidateReplicaExecutionDestinationInputsTask, - constants.TASK_TYPE_VALIDATE_DEPLOYMENT_INPUTS: - replica_tasks.ValidateReplicaDeploymentParametersTask, - constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER: - replica_tasks.UpdateSourceReplicaTask, - constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER: - replica_tasks.UpdateDestinationReplicaTask, - constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS: - minion_pool_tasks.ValidateSourceMinionPoolOptionsTask, - constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS: - minion_pool_tasks.ValidateDestinationMinionPoolOptionsTask, - constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE: - minion_pool_tasks.CreateSourceMinionMachineTask, - constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE: - minion_pool_tasks.CreateDestinationMinionMachineTask, - constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE: - minion_pool_tasks.DeleteSourceMinionMachineTask, - constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE: - minion_pool_tasks.DeleteDestinationMinionMachineTask, - constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES: - minion_pool_tasks.SetUpSourcePoolSupportingResourcesTask, - constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES: - minion_pool_tasks.SetUpDestinationPoolSupportingResources, - constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES: - minion_pool_tasks.TearDownSourcePoolSupportingResourcesTask, - constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES: - minion_pool_tasks.TearDownDestinationPoolSupportingResources, - constants.TASK_TYPE_ATTACH_VOLUMES_TO_SOURCE_MINION: - minion_pool_tasks.AttachVolumesToSourceMinionTask, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION: - minion_pool_tasks.DetachVolumesFromSourceMinionTask, - constants.TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION: - minion_pool_tasks.AttachVolumesToDestinationMinionTask, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_DESTINATION_MINION: - minion_pool_tasks.DetachVolumesFromDestinationMinionTask, - constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION: - minion_pool_tasks.AttachVolumesToOSMorphingMinionTask, - constants.TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION: - minion_pool_tasks.DetachVolumesFromOSMorphingMinionTask, - constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY: - minion_pool_tasks.ValidateSourceMinionCompatibilityTask, - constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY: - minion_pool_tasks.ValidateDestinationMinionCompatibilityTask, - constants.TASK_TYPE_VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY: - minion_pool_tasks.ValidateOSMorphingMinionCompatibilityTask, - constants.TASK_TYPE_RELEASE_SOURCE_MINION: - minion_pool_tasks.ReleaseSourceMinionTask, - constants.TASK_TYPE_RELEASE_DESTINATION_MINION: - minion_pool_tasks.ReleaseDestinationMinionTask, - constants.TASK_TYPE_RELEASE_OSMORPHING_MINION: - minion_pool_tasks.ReleaseOSMorphingMinionTask, - constants.TASK_TYPE_COLLECT_OSMORPHING_INFO: - minion_pool_tasks.CollectOSMorphingInfoTask, - constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION: - minion_pool_tasks.HealthcheckSourceMinionMachineTask, - constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION: - minion_pool_tasks.HealthcheckDestinationMinionTask, - constants.TASK_TYPE_POWER_ON_SOURCE_MINION: - minion_pool_tasks.PowerOnSourceMinionTask, - constants.TASK_TYPE_POWER_OFF_SOURCE_MINION: - minion_pool_tasks.PowerOffSourceMinionTask, - constants.TASK_TYPE_POWER_ON_DESTINATION_MINION: - minion_pool_tasks.PowerOnDestinationMinionTask, - constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION: - minion_pool_tasks.PowerOffDestinationMinionTask + constants.TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT: migration_tasks.FinalizeInstanceDeploymentTask, + constants.TASK_TYPE_CLEANUP_FAILED_INSTANCE_DEPLOYMENT: migration_tasks.CleanupFailedInstanceDeploymentTask, + constants.TASK_TYPE_GET_OPTIMAL_FLAVOR: migration_tasks.GetOptimalFlavorTask, + constants.TASK_TYPE_DEPLOY_OS_MORPHING_RESOURCES: osmorphing_tasks.DeployOSMorphingResourcesTask, + constants.TASK_TYPE_OS_MORPHING: osmorphing_tasks.OSMorphingTask, + constants.TASK_TYPE_DELETE_OS_MORPHING_RESOURCES: osmorphing_tasks.DeleteOSMorphingResourcesTask, + constants.TASK_TYPE_GET_INSTANCE_INFO: replica_tasks.GetInstanceInfoTask, + constants.TASK_TYPE_REPLICATE_DISKS: replica_tasks.ReplicateDisksTask, + constants.TASK_TYPE_SHUTDOWN_INSTANCE: replica_tasks.ShutdownInstanceTask, + constants.TASK_TYPE_DEPLOY_TRANSFER_DISKS: replica_tasks.DeployReplicaDisksTask, + constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS: replica_tasks.DeleteReplicaSourceDiskSnapshotsTask, + constants.TASK_TYPE_DELETE_TRANSFER_DISKS: replica_tasks.DeleteReplicaDisksTask, + constants.TASK_TYPE_DEPLOY_TRANSFER_TARGET_RESOURCES: replica_tasks.DeployReplicaTargetResourcesTask, + constants.TASK_TYPE_DELETE_TRANSFER_TARGET_RESOURCES: replica_tasks.DeleteReplicaTargetResourcesTask, + constants.TASK_TYPE_DEPLOY_TRANSFER_SOURCE_RESOURCES: replica_tasks.DeployReplicaSourceResourcesTask, + constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_RESOURCES: replica_tasks.DeleteReplicaSourceResourcesTask, + constants.TASK_TYPE_DEPLOY_INSTANCE_RESOURCES: replica_tasks.DeployReplicaInstanceResourcesTask, + constants.TASK_TYPE_CREATE_TRANSFER_DISK_SNAPSHOTS: replica_tasks.CreateReplicaDiskSnapshotsTask, + constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS: replica_tasks.DeleteReplicaTargetDiskSnapshotsTask, + constants.TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS: replica_tasks.RestoreReplicaDiskSnapshotsTask, + constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS: replica_tasks.ValidateReplicaExecutionSourceInputsTask, + constants.TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS: replica_tasks.ValidateReplicaExecutionDestinationInputsTask, + constants.TASK_TYPE_VALIDATE_DEPLOYMENT_INPUTS: replica_tasks.ValidateReplicaDeploymentParametersTask, + constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER: replica_tasks.UpdateSourceReplicaTask, + constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER: replica_tasks.UpdateDestinationReplicaTask, + constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS: minion_pool_tasks.ValidateSourceMinionPoolOptionsTask, + constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS: minion_pool_tasks.ValidateDestinationMinionPoolOptionsTask, + constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE: minion_pool_tasks.CreateSourceMinionMachineTask, + constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE: minion_pool_tasks.CreateDestinationMinionMachineTask, + constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE: minion_pool_tasks.DeleteSourceMinionMachineTask, + constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE: minion_pool_tasks.DeleteDestinationMinionMachineTask, + constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES: minion_pool_tasks.SetUpSourcePoolSupportingResourcesTask, + constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES: minion_pool_tasks.SetUpDestinationPoolSupportingResources, + constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES: minion_pool_tasks.TearDownSourcePoolSupportingResourcesTask, + constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES: minion_pool_tasks.TearDownDestinationPoolSupportingResources, + constants.TASK_TYPE_ATTACH_VOLUMES_TO_SOURCE_MINION: minion_pool_tasks.AttachVolumesToSourceMinionTask, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_SOURCE_MINION: minion_pool_tasks.DetachVolumesFromSourceMinionTask, + constants.TASK_TYPE_ATTACH_VOLUMES_TO_DESTINATION_MINION: minion_pool_tasks.AttachVolumesToDestinationMinionTask, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_DESTINATION_MINION: minion_pool_tasks.DetachVolumesFromDestinationMinionTask, + constants.TASK_TYPE_ATTACH_VOLUMES_TO_OSMORPHING_MINION: minion_pool_tasks.AttachVolumesToOSMorphingMinionTask, + constants.TASK_TYPE_DETACH_VOLUMES_FROM_OSMORPHING_MINION: minion_pool_tasks.DetachVolumesFromOSMorphingMinionTask, + constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_COMPATIBILITY: minion_pool_tasks.ValidateSourceMinionCompatibilityTask, + constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_COMPATIBILITY: minion_pool_tasks.ValidateDestinationMinionCompatibilityTask, + constants.TASK_TYPE_VALIDATE_OSMORPHING_MINION_POOL_COMPATIBILITY: minion_pool_tasks.ValidateOSMorphingMinionCompatibilityTask, + constants.TASK_TYPE_RELEASE_SOURCE_MINION: minion_pool_tasks.ReleaseSourceMinionTask, + constants.TASK_TYPE_RELEASE_DESTINATION_MINION: minion_pool_tasks.ReleaseDestinationMinionTask, + constants.TASK_TYPE_RELEASE_OSMORPHING_MINION: minion_pool_tasks.ReleaseOSMorphingMinionTask, + constants.TASK_TYPE_COLLECT_OSMORPHING_INFO: minion_pool_tasks.CollectOSMorphingInfoTask, + constants.TASK_TYPE_HEALTHCHECK_SOURCE_MINION: minion_pool_tasks.HealthcheckSourceMinionMachineTask, + constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION: minion_pool_tasks.HealthcheckDestinationMinionTask, + constants.TASK_TYPE_POWER_ON_SOURCE_MINION: minion_pool_tasks.PowerOnSourceMinionTask, + constants.TASK_TYPE_POWER_OFF_SOURCE_MINION: minion_pool_tasks.PowerOffSourceMinionTask, + constants.TASK_TYPE_POWER_ON_DESTINATION_MINION: minion_pool_tasks.PowerOnDestinationMinionTask, + constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION: minion_pool_tasks.PowerOffDestinationMinionTask, } def get_task_runner_class(task_type): cls = _TASKS_MAP.get(task_type) if not cls: - raise exception.NotFound( - "TaskRunner not found for task type: %s" % task_type) + raise exception.NotFound("TaskRunner not found for task type: %s" % task_type) return cls diff --git a/coriolis/tasks/migration_tasks.py b/coriolis/tasks/migration_tasks.py index d71add64..89a7b976 100644 --- a/coriolis/tasks/migration_tasks.py +++ b/coriolis/tasks/migration_tasks.py @@ -3,17 +3,14 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import events +from coriolis import constants, events from coriolis.providers import factory as providers_factory -from coriolis.tasks import base -from coriolis.tasks import replica_tasks +from coriolis.tasks import base, replica_tasks LOG = logging.getLogger(__name__) class GetOptimalFlavorTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -30,21 +27,22 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_INSTANCE_FLAVOR] + constants.PROVIDER_TYPE_INSTANCE_FLAVOR + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_INSTANCE_FLAVOR, - event_handler) + destination["type"], constants.PROVIDER_TYPE_INSTANCE_FLAVOR, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) target_environment = task_info["target_environment"] export_info = task_info["export_info"] flavor = provider.get_optimal_flavor( - ctxt, connection_info, target_environment, export_info) + ctxt, connection_info, target_environment, export_info + ) instance_deployment_info = task_info.get("instance_deployment_info") if instance_deployment_info is None: @@ -52,24 +50,25 @@ def _run(self, ctxt, instance, origin, destination, task_info, instance_deployment_info["selected_flavor"] = flavor events.EventManager(event_handler).progress_update( - "Selected flavor: %s" % flavor) + "Selected flavor: %s" % flavor + ) - return { - "instance_deployment_info": instance_deployment_info} + return {"instance_deployment_info": instance_deployment_info} class DeployMigrationSourceResourcesTask( - replica_tasks.DeployReplicaSourceResourcesTask): + replica_tasks.DeployReplicaSourceResourcesTask +): pass class DeployMigrationTargetResourcesTask( - replica_tasks.DeployReplicaTargetResourcesTask): + replica_tasks.DeployReplicaTargetResourcesTask +): pass -class CreateInstanceDisksTask( - replica_tasks.DeployReplicaDisksTask): +class CreateInstanceDisksTask(replica_tasks.DeployReplicaDisksTask): pass @@ -78,45 +77,56 @@ class CleanupInstanceTargetStorageTask(replica_tasks.DeleteReplicaDisksTask): class CleanupInstanceSourceStorageTask( - replica_tasks.DeleteReplicaSourceDiskSnapshotsTask): + replica_tasks.DeleteReplicaSourceDiskSnapshotsTask +): pass class FinalizeInstanceDeploymentTask( - replica_tasks.FinalizeReplicaInstanceDeploymentTask): + replica_tasks.FinalizeReplicaInstanceDeploymentTask +): pass class CleanupFailedInstanceDeploymentTask( - replica_tasks.CleanupFailedReplicaInstanceDeploymentTask): + replica_tasks.CleanupFailedReplicaInstanceDeploymentTask +): pass class ValidateMigrationSourceInputsTask( - replica_tasks.ValidateReplicaExecutionSourceInputsTask): + replica_tasks.ValidateReplicaExecutionSourceInputsTask +): pass class ValidateMigrationDestinationInputsTask( - replica_tasks.ValidateReplicaExecutionDestinationInputsTask): + replica_tasks.ValidateReplicaExecutionDestinationInputsTask +): def _validate_provider_replica_import_input( - self, provider, ctxt, conn_info, target_environment, export_info): + self, provider, ctxt, conn_info, target_environment, export_info + ): provider.validate_replica_import_input( - ctxt, conn_info, target_environment, export_info, + ctxt, + conn_info, + target_environment, + export_info, check_os_morphing_resources=True, - check_final_vm_params=True) + check_final_vm_params=True, + ) class DeleteMigrationSourceResourcesTask( - replica_tasks.DeleteReplicaSourceResourcesTask): + replica_tasks.DeleteReplicaSourceResourcesTask +): pass class DeleteMigrationTargetResourcesTask( - replica_tasks.DeleteReplicaTargetResourcesTask): + replica_tasks.DeleteReplicaTargetResourcesTask +): pass -class DeployInstanceResourcesTask( - replica_tasks.DeployReplicaInstanceResourcesTask): +class DeployInstanceResourcesTask(replica_tasks.DeployReplicaInstanceResourcesTask): pass diff --git a/coriolis/tasks/minion_pool_tasks.py b/coriolis/tasks/minion_pool_tasks.py index 75e24571..e75b0140 100644 --- a/coriolis/tasks/minion_pool_tasks.py +++ b/coriolis/tasks/minion_pool_tasks.py @@ -3,9 +3,7 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import events -from coriolis import exception +from coriolis import constants, events, exception from coriolis.providers import factory as providers_factory from coriolis.tasks import base @@ -14,18 +12,21 @@ SOURCE_MINION_TASK_INFO_FIELD_MAPPINGS = { "origin_minion_provider_properties": "source_resources", - "origin_minion_connection_info": "source_resources_connection_info"} + "origin_minion_connection_info": "source_resources_connection_info", +} TARGET_MINION_TASK_INFO_FIELD_MAPPINGS = { "destination_minion_provider_properties": "target_resources", "destination_minion_backup_writer_connection_info": ( - "target_resources_connection_info")} + "target_resources_connection_info" + ), +} OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS = { "osmorphing_minion_provider_properties": "os_morphing_resources", - "osmorphing_minion_connection_info": "osmorphing_connection_info"} + "osmorphing_minion_connection_info": "osmorphing_connection_info", +} -def _get_required_minion_pool_provider_types_for_platform( - platform_type): +def _get_required_minion_pool_provider_types_for_platform(platform_type): provider_type = None if platform_type == constants.PROVIDER_PLATFORM_SOURCE: provider_type = constants.PROVIDER_TYPE_SOURCE_MINION_POOL @@ -34,9 +35,9 @@ def _get_required_minion_pool_provider_types_for_platform( else: raise NotImplementedError( "Cannot determine required minion pool provider type for " - "platform of type '%s'" % platform_type) - return { - platform_type: [provider_type]} + "platform of type '%s'" % platform_type + ) + return {platform_type: [provider_type]} def _get_platform_to_target(required_platform, origin, destination): @@ -47,51 +48,55 @@ def _get_platform_to_target(required_platform, origin, destination): platform_to_target = destination else: raise NotImplementedError( - "Unknown minion pool platform '%s'" % ( - required_platform)) + "Unknown minion pool platform '%s'" % (required_platform) + ) return platform_to_target -def _check_missing_minion_properties(provider_type, minion_properties, - properties_to_check=None): +def _check_missing_minion_properties( + provider_type, minion_properties, properties_to_check=None +): if properties_to_check is None: properties_to_check = [] - missing = [ - key for key in properties_to_check - if key not in minion_properties] + missing = [key for key in properties_to_check if key not in minion_properties] if missing: LOG.warn( "Provider of type '%s' failed to return the following minion " "property keys: %s. Allowing run to completion for later " - "cleanup.", provider_type, missing) + "cleanup.", + provider_type, + missing, + ) def _get_minion_conn_info(minion_properties): minion_connection_info = {} if 'connection_info' in minion_properties: minion_connection_info = base.marshal_migr_conn_info( - minion_properties['connection_info']) + minion_properties['connection_info'] + ) return minion_connection_info def _get_minion_backup_writer_conn_info(minion_properties): minion_backup_writer_conn = {} if 'backup_writer_connection_info' in minion_properties: - minion_backup_writer_conn = minion_properties[ - 'backup_writer_connection_info'] + minion_backup_writer_conn = minion_properties['backup_writer_connection_info'] if 'connection_details' in minion_backup_writer_conn: minion_backup_writer_conn['connection_details'] = ( base.marshal_migr_conn_info( - minion_backup_writer_conn['connection_details'])) + minion_backup_writer_conn['connection_details'] + ) + ) return minion_backup_writer_conn class _BaseValidateMinionPoolOptionsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): raise NotImplementedError( - "No Minion pool options validation platform specified.") + "No Minion pool options validation platform specified." + ) @classmethod def get_required_task_info_properties(cls): @@ -104,115 +109,147 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) - - def _run(self, ctxt, minion_pool_machine_id, origin, destination, - task_info, event_handler): + cls.get_required_platform() + ) + + def _run( + self, + ctxt, + minion_pool_machine_id, + origin, + destination, + task_info, + event_handler, + ): # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) environment_options = task_info['pool_environment_options'] provider.validate_minion_pool_environment_options( - ctxt, connection_info, environment_options) + ctxt, connection_info, environment_options + ) return {} class ValidateSourceMinionPoolOptionsTask(_BaseValidateMinionPoolOptionsTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE -class ValidateDestinationMinionPoolOptionsTask( - _BaseValidateMinionPoolOptionsTask): - +class ValidateDestinationMinionPoolOptionsTask(_BaseValidateMinionPoolOptionsTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION class _BaseCreateMinionMachineTask(base.TaskRunner): - @classmethod def get_required_platform(cls): raise NotImplementedError( - "No minion pool creation required platform specified.") + "No minion pool creation required platform specified." + ) @classmethod def get_required_task_info_properties(cls): return [ - "pool_environment_options", "pool_shared_resources", - "pool_identifier", "pool_os_type"] + "pool_environment_options", + "pool_shared_resources", + "pool_identifier", + "pool_os_type", + ] @classmethod def get_returned_task_info_properties(cls): return [ - "minion_provider_properties", "minion_connection_info", - "minion_backup_writer_connection_info"] + "minion_provider_properties", + "minion_connection_info", + "minion_backup_writer_connection_info", + ] @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) - - def _run(self, ctxt, minion_pool_machine_id, origin, destination, - task_info, event_handler): + cls.get_required_platform() + ) + + def _run( + self, + ctxt, + minion_pool_machine_id, + origin, + destination, + task_info, + event_handler, + ): # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) pool_identifier = task_info['pool_identifier'] environment_options = task_info['pool_environment_options'] pool_shared_resources = task_info['pool_shared_resources'] pool_os_type = task_info["pool_os_type"] minion_properties = provider.create_minion( - ctxt, connection_info, environment_options, pool_identifier, - pool_os_type, pool_shared_resources, minion_pool_machine_id) + ctxt, + connection_info, + environment_options, + pool_identifier, + pool_os_type, + pool_shared_resources, + minion_pool_machine_id, + ) _check_missing_minion_properties( - provider_type, minion_properties, + provider_type, + minion_properties, properties_to_check=[ - "connection_info", "minion_provider_properties", - "backup_writer_connection_info"]) + "connection_info", + "minion_provider_properties", + "backup_writer_connection_info", + ], + ) return { "minion_connection_info": _get_minion_conn_info(minion_properties), "minion_backup_writer_connection_info": ( - _get_minion_backup_writer_conn_info(minion_properties)), + _get_minion_backup_writer_conn_info(minion_properties) + ), "minion_provider_properties": minion_properties.get( - "minion_provider_properties")} + "minion_provider_properties" + ), + } class CreateSourceMinionMachineTask(_BaseCreateMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE class CreateDestinationMinionMachineTask(_BaseCreateMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION class _BaseDeleteMinionMachineTask(base.TaskRunner): - @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion deletion required platform specified.") + raise NotImplementedError("No minion deletion required platform specified.") @classmethod def get_required_task_info_properties(cls): @@ -225,45 +262,52 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) - - def _run(self, ctxt, minion_pool_machine_id, origin, destination, - task_info, event_handler): + cls.get_required_platform() + ) + + def _run( + self, + ctxt, + minion_pool_machine_id, + origin, + destination, + task_info, + event_handler, + ): # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) minion_provider_properties = task_info['minion_provider_properties'] - provider.delete_minion( - ctxt, connection_info, minion_provider_properties) + provider.delete_minion(ctxt, connection_info, minion_provider_properties) return {} class DeleteSourceMinionMachineTask(_BaseDeleteMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE class DeleteDestinationMinionMachineTask(_BaseDeleteMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION class _BaseSetUpPoolSupportingResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): raise NotImplementedError( - "No pool shared resource setup required platform specified.") + "No pool shared resource setup required platform specified." + ) @classmethod def get_required_task_info_properties(cls): @@ -276,48 +320,55 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) - - def _run(self, ctxt, minion_pool_machine_id, origin, destination, - task_info, event_handler): + cls.get_required_platform() + ) + + def _run( + self, + ctxt, + minion_pool_machine_id, + origin, + destination, + task_info, + event_handler, + ): # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) pool_identifier = task_info['pool_identifier'] environment_options = task_info['pool_environment_options'] pool_shared_resources = provider.set_up_pool_shared_resources( - ctxt, connection_info, environment_options, pool_identifier) + ctxt, connection_info, environment_options, pool_identifier + ) return {"pool_shared_resources": pool_shared_resources} -class SetUpSourcePoolSupportingResourcesTask( - _BaseSetUpPoolSupportingResourcesTask): - +class SetUpSourcePoolSupportingResourcesTask(_BaseSetUpPoolSupportingResourcesTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE -class SetUpDestinationPoolSupportingResources( - _BaseSetUpPoolSupportingResourcesTask): - +class SetUpDestinationPoolSupportingResources(_BaseSetUpPoolSupportingResourcesTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION class _BaseTearDownPoolSupportingResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): raise NotImplementedError( - "No pool tear down shared resoures required platform specified.") + "No pool tear down shared resoures required platform specified." + ) @classmethod def get_required_task_info_properties(cls): @@ -330,53 +381,62 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) - - def _run(self, ctxt, minion_pool_machine_id, origin, destination, - task_info, event_handler): + cls.get_required_platform() + ) + + def _run( + self, + ctxt, + minion_pool_machine_id, + origin, + destination, + task_info, + event_handler, + ): # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) environment_options = task_info['pool_environment_options'] pool_shared_resources = task_info['pool_shared_resources'] provider.tear_down_pool_shared_resources( - ctxt, connection_info, environment_options, - pool_shared_resources) + ctxt, connection_info, environment_options, pool_shared_resources + ) return {"pool_shared_resources": None} class TearDownSourcePoolSupportingResourcesTask( - _BaseTearDownPoolSupportingResourcesTask): - + _BaseTearDownPoolSupportingResourcesTask +): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE class TearDownDestinationPoolSupportingResources( - _BaseTearDownPoolSupportingResourcesTask): - + _BaseTearDownPoolSupportingResourcesTask +): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION class _BaseVolumesMinionMachineAttachmentTask(base.TaskRunner): - """ The purposes of the volume attachment tasks are to: + """The purposes of the volume attachment tasks are to: 1) attach the volumes of the minions 2) return any updated properties for the minions if needed """ @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion disk attachment platform specified") + raise NotImplementedError("No minion disk attachment platform specified") @classmethod def get_required_task_info_properties(cls): @@ -392,87 +452,101 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) @classmethod def _get_volumes_info_from_task_info(cls, task_info): - raise NotImplementedError( - "No minion volumes info retrieval logic implemented.") + raise NotImplementedError("No minion volumes info retrieval logic implemented.") @classmethod def _get_minion_properties_task_info_field(cls): raise NotImplementedError( - "No minion disk attachment task info field specified.") + "No minion disk attachment task info field specified." + ) @classmethod def _get_minion_connection_info_task_info_field(cls): raise NotImplementedError( - "No minion disk attachment task info field specified.") + "No minion disk attachment task info field specified." + ) @classmethod def _get_provider_disk_operation(cls, provider): raise NotImplementedError( - "No minion disk attachment provider operation specified.") + "No minion disk attachment provider operation specified." + ) @classmethod def _get_minion_task_info_field_mappings(cls): - raise NotImplementedError( - "No minion task info field mappings provided.") + raise NotImplementedError("No minion task info field mappings provided.") - def _check_missing_disk_op_result_keys(self, platform_to_target, - disk_op_res): + def _check_missing_disk_op_result_keys(self, platform_to_target, disk_op_res): missing_result_props = [ - prop for prop in ["volumes_info", "minion_properties"] - if prop not in disk_op_res] + prop + for prop in ["volumes_info", "minion_properties"] + if prop not in disk_op_res + ] if missing_result_props: raise exception.CoriolisException( "The following properties were missing from minion disk " - "operation '%s' from platform '%s': %s" % ( + "operation '%s' from platform '%s': %s" + % ( self._get_provider_disk_operation.__name__, - platform_to_target, missing_result_props)) + platform_to_target, + missing_result_props, + ) + ) - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): platform_to_target = _get_platform_to_target( - self.get_required_platform(), origin, destination) + self.get_required_platform(), origin, destination + ) connection_info = base.get_connection_info(ctxt, platform_to_target) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - platform_to_target["type"], provider_type, event_handler) + platform_to_target["type"], provider_type, event_handler + ) volumes_info = self._get_volumes_info_from_task_info(task_info) - minion_properties = task_info[ - self._get_minion_properties_task_info_field()] + minion_properties = task_info[self._get_minion_properties_task_info_field()] minion_connection_info = task_info[ - self._get_minion_connection_info_task_info_field()] + self._get_minion_connection_info_task_info_field() + ] res = self._get_provider_disk_operation(provider)( - ctxt, connection_info, minion_properties, - minion_connection_info, volumes_info) + ctxt, + connection_info, + minion_properties, + minion_connection_info, + volumes_info, + ) self._check_missing_disk_op_result_keys(platform_to_target, res) field_name_map = self._get_minion_task_info_field_mappings() result = { "volumes_info": res['volumes_info'], - self._get_minion_properties_task_info_field(): res[ - "minion_properties"], - field_name_map[ - self._get_minion_properties_task_info_field()]: res[ - "minion_properties"]} - - result.update({ - field_name_map[field]: task_info[field] - for field in field_name_map - if field_name_map[field] not in result}) + self._get_minion_properties_task_info_field(): res["minion_properties"], + field_name_map[self._get_minion_properties_task_info_field()]: res[ + "minion_properties" + ], + } + + result.update( + { + field_name_map[field]: task_info[field] + for field in field_name_map + if field_name_map[field] not in result + } + ) return result -class _BaseAttachVolumesToTransferMinionTask( - _BaseVolumesMinionMachineAttachmentTask): - +class _BaseAttachVolumesToTransferMinionTask(_BaseVolumesMinionMachineAttachmentTask): @classmethod def _get_volumes_info_from_task_info(cls, task_info): return task_info["volumes_info"] @@ -480,22 +554,21 @@ def _get_volumes_info_from_task_info(cls, task_info): @classmethod def get_required_task_info_properties(cls): fields = super( - _BaseAttachVolumesToTransferMinionTask, - cls).get_required_task_info_properties() + _BaseAttachVolumesToTransferMinionTask, cls + ).get_required_task_info_properties() fields.append("volumes_info") return fields @classmethod def get_returned_task_info_properties(cls): fields = super( - _BaseAttachVolumesToTransferMinionTask, - cls).get_returned_task_info_properties() + _BaseAttachVolumesToTransferMinionTask, cls + ).get_returned_task_info_properties() fields.append("volumes_info") return fields class AttachVolumesToSourceMinionTask(_BaseAttachVolumesToTransferMinionTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @@ -522,15 +595,12 @@ def _get_provider_disk_operation(cls, provider): class DetachVolumesFromSourceMinionTask(AttachVolumesToSourceMinionTask): - @classmethod def _get_provider_disk_operation(cls, provider): return provider.detach_volumes_from_minion -class AttachVolumesToDestinationMinionTask( - _BaseAttachVolumesToTransferMinionTask): - +class AttachVolumesToDestinationMinionTask(_BaseAttachVolumesToTransferMinionTask): @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -552,35 +622,30 @@ def _get_minion_task_info_field_mappings(cls): return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS -class DetachVolumesFromDestinationMinionTask( - AttachVolumesToDestinationMinionTask): - +class DetachVolumesFromDestinationMinionTask(AttachVolumesToDestinationMinionTask): @classmethod def _get_provider_disk_operation(cls, provider): return provider.detach_volumes_from_minion -class AttachVolumesToOSMorphingMinionTask( - _BaseVolumesMinionMachineAttachmentTask): - +class AttachVolumesToOSMorphingMinionTask(_BaseVolumesMinionMachineAttachmentTask): @classmethod def _get_volumes_info_from_task_info(cls, task_info): - return task_info[ - "instance_deployment_info"]["volumes_info"] + return task_info["instance_deployment_info"]["volumes_info"] @classmethod def get_required_task_info_properties(cls): fields = super( - AttachVolumesToOSMorphingMinionTask, - cls).get_required_task_info_properties() + AttachVolumesToOSMorphingMinionTask, cls + ).get_required_task_info_properties() fields.append("instance_deployment_info") return fields @classmethod def get_returned_task_info_properties(cls): fields = super( - AttachVolumesToOSMorphingMinionTask, - cls).get_returned_task_info_properties() + AttachVolumesToOSMorphingMinionTask, cls + ).get_returned_task_info_properties() fields.append("instance_deployment_info") return fields @@ -608,11 +673,10 @@ def _get_minion_task_info_field_mappings(cls): def _clear_mapped_minion_task_info_field(cls): return False - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): - res = super( - AttachVolumesToOSMorphingMinionTask, self)._run( - ctxt, instance, origin, destination, task_info, event_handler) + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + res = super(AttachVolumesToOSMorphingMinionTask, self)._run( + ctxt, instance, origin, destination, task_info, event_handler + ) instance_deployment_info = task_info['instance_deployment_info'] if 'volumes_info' in res: @@ -623,9 +687,7 @@ def _run(self, ctxt, instance, origin, destination, return res -class DetachVolumesFromOSMorphingMinionTask( - AttachVolumesToOSMorphingMinionTask): - +class DetachVolumesFromOSMorphingMinionTask(AttachVolumesToOSMorphingMinionTask): @classmethod def _get_provider_disk_operation(cls, provider): return provider.detach_volumes_from_minion @@ -636,7 +698,7 @@ def _clear_mapped_minion_task_info_field(cls): class _BaseValidateMinionCompatibilityTask(base.TaskRunner): - """ The purposes of the minion validation tasks are to: + """The purposes of the minion validation tasks are to: 1) run the afferent validation method on the provider 2) "translate" the fields related to the minion into fields which are to be consumed by the other tasks @@ -645,33 +707,37 @@ class _BaseValidateMinionCompatibilityTask(base.TaskRunner): @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion validation platform specified") + raise NotImplementedError("No minion validation platform specified") @classmethod def get_required_task_info_properties(cls): - base_props = set([ - "export_info", - cls._get_transfer_properties_task_info_field(), - cls._get_minion_properties_task_info_field()]) - base_props = base_props.union(set( - cls._get_minion_task_info_field_mappings().keys())) + base_props = set( + [ + "export_info", + cls._get_transfer_properties_task_info_field(), + cls._get_minion_properties_task_info_field(), + ] + ) + base_props = base_props.union( + set(cls._get_minion_task_info_field_mappings().keys()) + ) return list(base_props) @classmethod def get_returned_task_info_properties(cls): - return list( - cls._get_minion_task_info_field_mappings().values()) + return list(cls._get_minion_task_info_field_mappings().values()) @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) @classmethod def _get_provider_pool_validation_operation(cls, provider): raise NotImplementedError( - "No minion pool provider validation method was specified.") + "No minion pool provider validation method was specified." + ) @classmethod def _get_transfer_properties_task_info_field(cls): @@ -681,49 +747,43 @@ def _get_transfer_properties_task_info_field(cls): elif platform == constants.PROVIDER_PLATFORM_DESTINATION: return "target_environment" raise exception.CoriolisException( - "Unknown minion pool validation operation platform '%s'" % ( - platform)) + "Unknown minion pool validation operation platform '%s'" % (platform) + ) @classmethod def _get_minion_properties_task_info_field(cls): - raise NotImplementedError( - "No minion validation task info field specified.") + raise NotImplementedError("No minion validation task info field specified.") @classmethod def _get_minion_task_info_field_mappings(cls): - raise NotImplementedError( - "No minion task info field mappings provided.") + raise NotImplementedError("No minion task info field mappings provided.") - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): platform_to_target = _get_platform_to_target( - self.get_required_platform(), origin, destination) + self.get_required_platform(), origin, destination + ) connection_info = base.get_connection_info(ctxt, platform_to_target) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - platform_to_target["type"], provider_type, event_handler) + platform_to_target["type"], provider_type, event_handler + ) export_info = task_info["export_info"] - minion_properties = task_info[ - self._get_minion_properties_task_info_field()] - transfer_properties = task_info[ - self._get_transfer_properties_task_info_field()] + minion_properties = task_info[self._get_minion_properties_task_info_field()] + transfer_properties = task_info[self._get_transfer_properties_task_info_field()] validation_op = self._get_provider_pool_validation_operation(provider) validation_op( - ctxt, connection_info, export_info, transfer_properties, - minion_properties) + ctxt, connection_info, export_info, transfer_properties, minion_properties + ) field_mappings = self._get_minion_task_info_field_mappings() - return { - field_mappings[field]: task_info[field] - for field in field_mappings} + return {field_mappings[field]: task_info[field] for field in field_mappings} -class ValidateSourceMinionCompatibilityTask( - _BaseValidateMinionCompatibilityTask): - +class ValidateSourceMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE @@ -741,9 +801,7 @@ def _get_minion_task_info_field_mappings(cls): return SOURCE_MINION_TASK_INFO_FIELD_MAPPINGS -class ValidateDestinationMinionCompatibilityTask( - _BaseValidateMinionCompatibilityTask): - +class ValidateDestinationMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION @@ -761,9 +819,7 @@ def _get_minion_task_info_field_mappings(cls): return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS -class ValidateOSMorphingMinionCompatibilityTask( - _BaseValidateMinionCompatibilityTask): - +class ValidateOSMorphingMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION @@ -782,21 +838,18 @@ def _get_minion_task_info_field_mappings(cls): class _BaseReleaseMinionTask(base.TaskRunner): - """ The purpose of releasal tasks is to clear (set to None) all of the + """The purpose of releasal tasks is to clear (set to None) all of the fields afferent to the minion for the respective task type. """ @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion releasing platform specified") + raise NotImplementedError("No minion releasing platform specified") @classmethod def get_required_task_info_properties(cls): prop_mappings = cls._get_minion_task_info_field_mappings() - return list( - set(prop_mappings.keys()).union( - prop_mappings.values())) + return list(set(prop_mappings.keys()).union(prop_mappings.values())) @classmethod def get_returned_task_info_properties(cls): @@ -805,24 +858,20 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) @classmethod def _get_minion_task_info_field_mappings(cls): - raise NotImplementedError( - "No minion task info field mappings provided.") + raise NotImplementedError("No minion task info field mappings provided.") - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) event_manager.progress_update("Releasing minion machine") - return { - field: None - for field in self.get_returned_task_info_properties()} + return {field: None for field in self.get_returned_task_info_properties()} class ReleaseSourceMinionTask(_BaseReleaseMinionTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_SOURCE @@ -833,7 +882,6 @@ def _get_minion_task_info_field_mappings(cls): class ReleaseDestinationMinionTask(_BaseReleaseMinionTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION @@ -844,7 +892,6 @@ def _get_minion_task_info_field_mappings(cls): class ReleaseOSMorphingMinionTask(_BaseReleaseMinionTask): - @classmethod def get_required_platform(cls): return constants.PROVIDER_PLATFORM_DESTINATION @@ -855,7 +902,6 @@ def _get_minion_task_info_field_mappings(cls): class CollectOSMorphingInfoTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -863,7 +909,8 @@ def get_required_platform(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) @classmethod def get_required_task_info_properties(cls): @@ -873,36 +920,36 @@ def get_required_task_info_properties(cls): def get_returned_task_info_properties(cls): return ["osmorphing_info"] - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - destination["type"], provider_type, event_handler) + destination["type"], provider_type, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) target_environment = task_info["target_environment"] instance_deployment_info = task_info["instance_deployment_info"] result = provider.get_additional_os_morphing_info( - ctxt, connection_info, target_environment, - instance_deployment_info) + ctxt, connection_info, target_environment, instance_deployment_info + ) if not isinstance(result, dict) or 'osmorphing_info' not in result: raise exception.CoriolisException( "'get_additional_os_morphing_info' method for provider of type" - " '%s' failed to return OSMorphing info.") + " '%s' failed to return OSMorphing info." + ) - return { - "osmorphing_info": result["osmorphing_info"]} + return {"osmorphing_info": result["osmorphing_info"]} class _BaseHealthcheckMinionMachineTask(base.TaskRunner): - """ Calls into the provider to healthcheck the minion machine. """ + """Calls into the provider to healthcheck the minion machine.""" @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion healthcheck platform specified") + raise NotImplementedError("No minion healthcheck platform specified") @classmethod def get_required_task_info_properties(cls): @@ -915,54 +962,56 @@ def get_returned_task_info_properties(cls): @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): platform_to_target = _get_platform_to_target( - self.get_required_platform(), origin, destination) + self.get_required_platform(), origin, destination + ) connection_info = base.get_connection_info(ctxt, platform_to_target) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - platform_to_target["type"], provider_type, event_handler) + platform_to_target["type"], provider_type, event_handler + ) minion_properties = task_info['minion_provider_properties'] minion_connection_info = base.unmarshal_migr_conn_info( - task_info['minion_connection_info']) + task_info['minion_connection_info'] + ) provider.healthcheck_minion( - ctxt, connection_info, minion_properties, minion_connection_info) + ctxt, connection_info, minion_properties, minion_connection_info + ) return {} class HealthcheckSourceMinionMachineTask(_BaseHealthcheckMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE class HealthcheckDestinationMinionTask(_BaseHealthcheckMinionMachineTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION class _BasePowerCycleMinionTask(base.TaskRunner): - @classmethod def get_required_platform(cls): - raise NotImplementedError( - "No minion power cycle platform specified") + raise NotImplementedError("No minion power cycle platform specified") @classmethod def get_required_provider_types(cls): return _get_required_minion_pool_provider_types_for_platform( - cls.get_required_platform()) + cls.get_required_platform() + ) @classmethod def get_required_task_info_properties(cls): @@ -974,19 +1023,20 @@ def get_returned_task_info_properties(cls): @classmethod def _get_minion_power_cycle_op(cls, provider): - raise NotImplementedError( - "No minion power cycle operation implemented.") + raise NotImplementedError("No minion power cycle operation implemented.") - def _run(self, ctxt, instance, origin, destination, - task_info, event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): platform_to_target = _get_platform_to_target( - self.get_required_platform(), origin, destination) + self.get_required_platform(), origin, destination + ) connection_info = base.get_connection_info(ctxt, platform_to_target) provider_type = self.get_required_provider_types()[ - self.get_required_platform()][0] + self.get_required_platform() + ][0] provider = providers_factory.get_provider( - platform_to_target["type"], provider_type, event_handler) + platform_to_target["type"], provider_type, event_handler + ) power_cycle_op = self._get_minion_power_cycle_op(provider) minion_properties = task_info['minion_provider_properties'] power_cycle_op(ctxt, connection_info, minion_properties) @@ -995,42 +1045,36 @@ def _run(self, ctxt, instance, origin, destination, class _BasePowerOnMinionTask(_BasePowerCycleMinionTask): - @classmethod def _get_minion_power_cycle_op(cls, provider): return provider.start_minion class PowerOnSourceMinionTask(_BasePowerOnMinionTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE class PowerOnDestinationMinionTask(_BasePowerOnMinionTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION class _BasePowerOffMinionTask(_BasePowerCycleMinionTask): - @classmethod def _get_minion_power_cycle_op(cls, provider): return provider.shutdown_minion class PowerOffSourceMinionTask(_BasePowerOffMinionTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE class PowerOffDestinationMinionTask(_BasePowerOffMinionTask): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION diff --git a/coriolis/tasks/osmorphing_tasks.py b/coriolis/tasks/osmorphing_tasks.py index 3005a525..9aac488e 100644 --- a/coriolis/tasks/osmorphing_tasks.py +++ b/coriolis/tasks/osmorphing_tasks.py @@ -3,28 +3,22 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, schemas from coriolis.osmorphing import manager as osmorphing_manager from coriolis.providers import factory as providers_factory -from coriolis import schemas from coriolis.tasks import base - LOG = logging.getLogger(__name__) class OSMorphingTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod def get_required_task_info_properties(cls): - return [ - "osmorphing_info", "osmorphing_connection_info", - "user_scripts"] + return ["osmorphing_info", "osmorphing_connection_info", "user_scripts"] @classmethod def get_returned_task_info_properties(cls): @@ -34,24 +28,26 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT], + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ], constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT], + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ], } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): origin_provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) destination_provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) osmorphing_connection_info = base.unmarshal_migr_conn_info( - task_info['osmorphing_connection_info']) + task_info['osmorphing_connection_info'] + ) osmorphing_info = task_info.get('osmorphing_info', {}) user_scripts = task_info.get("user_scripts") @@ -61,8 +57,7 @@ def _run(self, ctxt, instance, origin, destination, task_info, if not instance_script: os_type = osmorphing_info.get("os_type") if os_type: - instance_script = user_scripts.get( - "global", {}).get(os_type) + instance_script = user_scripts.get("global", {}).get(os_type) osmorphing_manager.morph_image( origin_provider, @@ -70,13 +65,13 @@ def _run(self, ctxt, instance, origin, destination, task_info, osmorphing_connection_info, osmorphing_info, instance_script, - event_handler) + event_handler, + ) return {} class DeployOSMorphingResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -88,67 +83,73 @@ def get_required_task_info_properties(cls): @classmethod def get_returned_task_info_properties(cls): return [ - "os_morphing_resources", "osmorphing_info", - "osmorphing_connection_info"] + "os_morphing_resources", + "osmorphing_info", + "osmorphing_connection_info", + ] @classmethod def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_OS_MORPHING] + constants.PROVIDER_TYPE_OS_MORPHING + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, - event_handler) + destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) target_environment = task_info["target_environment"] instance_deployment_info = task_info["instance_deployment_info"] import_info = provider.deploy_os_morphing_resources( - ctxt, connection_info, target_environment, - instance_deployment_info) + ctxt, connection_info, target_environment, instance_deployment_info + ) schemas.validate_value( - import_info, schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA, + import_info, + schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA, # NOTE: we avoid raising so that the cleanup task # can [try] to deal with the temporary resources. - raise_on_error=False) + raise_on_error=False, + ) os_morphing_resources = import_info.get('os_morphing_resources') if not os_morphing_resources: raise exception.InvalidTaskResult( "Target provider for '%s' did NOT return any " - "'os_morphing_resources'." % ( - destination["type"])) + "'os_morphing_resources'." % (destination["type"]) + ) - osmorphing_connection_info = import_info.get( - 'osmorphing_connection_info') + osmorphing_connection_info = import_info.get('osmorphing_connection_info') if not osmorphing_connection_info: raise exception.InvalidTaskResult( "Target provider '%s' did NOT return any " - "'osmorphing_connection_info'." % ( - destination["type"])) + "'osmorphing_connection_info'." % (destination["type"]) + ) osmorphing_connection_info = base.marshal_migr_conn_info( - osmorphing_connection_info) + osmorphing_connection_info + ) os_morphing_info = import_info.get("osmorphing_info", {}) if not os_morphing_info: LOG.warn( "Target provider for '%s' did NOT return any " "'osmorphing_info'. Defaulting to %s", - destination["type"], os_morphing_info) + destination["type"], + os_morphing_info, + ) return { "os_morphing_resources": os_morphing_resources, "osmorphing_connection_info": osmorphing_connection_info, - "osmorphing_info": os_morphing_info} + "osmorphing_info": os_morphing_info, + } class DeleteOSMorphingResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -165,21 +166,20 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_OS_MORPHING] + constants.PROVIDER_TYPE_OS_MORPHING + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, - event_handler) + destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) os_morphing_resources = task_info.get("os_morphing_resources") target_environment = task_info["target_environment"] provider.delete_os_morphing_resources( - ctxt, connection_info, target_environment, os_morphing_resources) + ctxt, connection_info, target_environment, os_morphing_resources + ) - return { - "os_morphing_resources": None, - "osmorphing_connection_info": None} + return {"os_morphing_resources": None, "osmorphing_connection_info": None} diff --git a/coriolis/tasks/replica_tasks.py b/coriolis/tasks/replica_tasks.py index 0407e8bc..79e1b9a3 100644 --- a/coriolis/tasks/replica_tasks.py +++ b/coriolis/tasks/replica_tasks.py @@ -3,14 +3,10 @@ from oslo_log import log as logging -from coriolis import constants -from coriolis import events -from coriolis import exception +from coriolis import constants, events, exception, schemas, utils from coriolis.providers import backup_writers from coriolis.providers import factory as providers_factory -from coriolis import schemas from coriolis.tasks import base -from coriolis import utils LOG = logging.getLogger(__name__) @@ -19,57 +15,70 @@ def _get_volumes_info(task_info): volumes_info = task_info.get("volumes_info", []) if not volumes_info: raise exception.InvalidActionTasksExecutionState( - "No volumes information present") + "No volumes information present" + ) return volumes_info def _check_ensure_volumes_info_ordering(export_info, volumes_info): - """ Returns a new list of volumes_info, ensuring that the order of + """Returns a new list of volumes_info, ensuring that the order of the disks in 'volumes_info' is consistent with the order that the disks appear in 'export_info[devices][disks]' """ instance = export_info.get( - 'instance_name', - export_info.get('name', export_info['id'])) + 'instance_name', export_info.get('name', export_info['id']) + ) - vol_info_cpy = utils.sanitize_task_info( - {"volumes_info": volumes_info}).get("volumes_info", []) + vol_info_cpy = utils.sanitize_task_info({"volumes_info": volumes_info}).get( + "volumes_info", [] + ) ordered_volumes_info = [] for disk in export_info['devices']['disks']: disk_id = disk['id'] - matching_volumes = [ - vol for vol in volumes_info if vol['disk_id'] == disk_id] + matching_volumes = [vol for vol in volumes_info if vol['disk_id'] == disk_id] if not matching_volumes: LOG.error( - "Could not find source disk '%s' (ID '%s') in Replica " - "volumes info: %s", disk, disk_id, vol_info_cpy) + "Could not find source disk '%s' (ID '%s') in Replica volumes info: %s", + disk, + disk_id, + vol_info_cpy, + ) raise exception.InvalidActionTasksExecutionState( "Source disk with ID '%s' not recognized. If this disk is " "newly added to the instance, please make sure to Execute the " "Replica before Updating. Check logs for more " - "information." % disk_id) + "information." % disk_id + ) elif len(matching_volumes) > 1: LOG.error( - "Multiple disks with ID '%s' found in Replica volumes " - "info: %s", disk_id, vol_info_cpy) + "Multiple disks with ID '%s' found in Replica volumes info: %s", + disk_id, + vol_info_cpy, + ) raise exception.InvalidActionTasksExecutionState( "Multiple disks with ID '%s' found in Replica volumes info. " "Please check that the instance doesn't have the same volume " "attached twice, or whether there's a UUID collision between " - "its disks. Check the logs for more information." % disk_id) + "its disks. Check the logs for more information." % disk_id + ) ordered_volumes_info.append(matching_volumes[0]) ordered_vol_info_cpy = utils.sanitize_task_info( - {"volumes_info": ordered_volumes_info}).get("volumes_info", []) + {"volumes_info": ordered_volumes_info} + ).get("volumes_info", []) LOG.debug( - "volumes_info returned by provider for instance " - "'%s': %s", instance, vol_info_cpy) + "volumes_info returned by provider for instance '%s': %s", + instance, + vol_info_cpy, + ) LOG.debug( - "volumes_info for instance '%s' after " - "reordering: %s", instance, ordered_vol_info_cpy) + "volumes_info for instance '%s' after reordering: %s", + instance, + ordered_vol_info_cpy, + ) return ordered_volumes_info @@ -105,7 +114,7 @@ def _update_export_info(old_export_info, result_export_info): class GetInstanceInfoTask(base.TaskRunner): - """ Task which gathers the export info for a VM. """ + """Task which gathers the export info for a VM.""" @classmethod def get_required_platform(cls): @@ -123,32 +132,31 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) source_environment = task_info['source_environment'] old_export_info = task_info.get('export_info', {}) export_info = provider.get_replica_instance_info( - ctxt, connection_info, source_environment, instance) + ctxt, connection_info, source_environment, instance + ) # Validate the output - schemas.validate_value( - export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + schemas.validate_value(export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) _update_export_info(old_export_info, export_info) - return { - 'export_info': export_info} + return {'export_info': export_info} class ShutdownInstanceTask(base.TaskRunner): - """ Task which shuts down a VM. """ + """Task which shuts down a VM.""" @classmethod def get_required_platform(cls): @@ -166,24 +174,22 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) source_environment = task_info['source_environment'] - provider.shutdown_instance(ctxt, connection_info, source_environment, - instance) + provider.shutdown_instance(ctxt, connection_info, source_environment, instance) return {} class ReplicateDisksTask(base.TaskRunner): - @classmethod def get_required_platform(cls): # NOTE: considering Replication reads from one end (be it PMR minion @@ -194,10 +200,13 @@ def get_required_platform(cls): @classmethod def get_required_task_info_properties(cls): return [ - "export_info", "volumes_info", "source_environment", + "export_info", + "volumes_info", + "source_environment", "source_resources", "source_resources_connection_info", - "target_resources_connection_info"] + "target_resources_connection_info", + ] @classmethod def get_returned_task_info_properties(cls): @@ -207,67 +216,71 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) export_info = task_info["export_info"] volumes_info = _get_volumes_info(task_info) schemas.validate_value( {"volumes_info": volumes_info}, - schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA) + schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA, + ) migr_source_conn_info = task_info["source_resources_connection_info"] if migr_source_conn_info: schemas.validate_value( migr_source_conn_info, - schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA) - migr_source_conn_info = base.unmarshal_migr_conn_info( - migr_source_conn_info) + schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA, + ) + migr_source_conn_info = base.unmarshal_migr_conn_info(migr_source_conn_info) migr_target_conn_info = task_info["target_resources_connection_info"] if migr_target_conn_info: schemas.validate_value( migr_target_conn_info, - schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA) - migr_target_conn_info['connection_details'] = ( - base.unmarshal_migr_conn_info( - migr_target_conn_info['connection_details'])) + schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA, + ) + migr_target_conn_info['connection_details'] = base.unmarshal_migr_conn_info( + migr_target_conn_info['connection_details'] + ) incremental = task_info.get("incremental", True) source_environment = task_info['source_environment'] source_resources = task_info.get('source_resources', {}) volumes_info = provider.replicate_disks( - ctxt, connection_info, source_environment, instance, - source_resources, migr_source_conn_info, migr_target_conn_info, - volumes_info, incremental) - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + ctxt, + connection_info, + source_environment, + instance, + source_resources, + migr_source_conn_info, + migr_target_conn_info, + volumes_info, + incremental, + ) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) - volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + volumes_info = _check_ensure_volumes_info_ordering(export_info, volumes_info) - return { - 'volumes_info': volumes_info} + return {'volumes_info': volumes_info} class DeployReplicaDisksTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod def get_required_task_info_properties(cls): - return [ - "export_info", "volumes_info", "target_environment"] + return ["export_info", "volumes_info", "target_environment"] @classmethod def get_returned_task_info_properties(cls): @@ -277,43 +290,43 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): target_environment = task_info['target_environment'] export_info = task_info["export_info"] provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) volumes_info = task_info.get("volumes_info", []) volumes_info = provider.deploy_replica_disks( - ctxt, connection_info, target_environment, instance, export_info, - volumes_info) - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + ctxt, + connection_info, + target_environment, + instance, + export_info, + volumes_info, + ) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) - volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + volumes_info = _check_ensure_volumes_info_ordering(export_info, volumes_info) - return { - 'volumes_info': volumes_info} + return {'volumes_info': volumes_info} class DeleteReplicaSourceDiskSnapshotsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @classmethod def get_required_task_info_properties(cls): - return [ - "volumes_info", "source_environment"] + return ["volumes_info", "source_environment"] @classmethod def get_returned_task_info_properties(cls): @@ -323,43 +336,41 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) if not task_info.get("volumes_info"): - LOG.debug( - "No volumes_info present. Skipping source snapshot deletion.") + LOG.debug("No volumes_info present. Skipping source snapshot deletion.") event_manager.progress_update( - "No previous volumes information present, nothing to delete") + "No previous volumes information present, nothing to delete" + ) return {'volumes_info': []} provider = providers_factory.get_provider( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) source_environment = task_info['source_environment'] volumes_info = _get_volumes_info(task_info) volumes_info = provider.delete_replica_source_snapshots( - ctxt, connection_info, source_environment, volumes_info) + ctxt, connection_info, source_environment, volumes_info + ) - return { - 'volumes_info': volumes_info} + return {'volumes_info': volumes_info} class DeleteReplicaDisksTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod def get_required_task_info_properties(cls): - return [ - "volumes_info", "target_environment"] + return ["volumes_info", "target_environment"] @classmethod def get_returned_task_info_properties(cls): @@ -369,42 +380,41 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) if not task_info.get("volumes_info"): - LOG.debug( - "No volumes_info present. Skipping disk deletion.") + LOG.debug("No volumes_info present. Skipping disk deletion.") event_manager.progress_update( - "No previous volumes information present, nothing to delete") + "No previous volumes information present, nothing to delete" + ) return {'volumes_info': []} provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) volumes_info = _get_volumes_info(task_info) target_environment = task_info['target_environment'] volumes_info = provider.delete_replica_disks( - ctxt, connection_info, target_environment, volumes_info) + ctxt, connection_info, target_environment, volumes_info + ) if volumes_info: LOG.warn( "'volumes_info' should have been void after disk " - "deletion task but it is: %s" % ( - utils.sanitize_task_info({ - 'volumes_info': volumes_info}))) + "deletion task but it is: %s" + % (utils.sanitize_task_info({'volumes_info': volumes_info})) + ) - return { - 'volumes_info': []} + return {'volumes_info': []} class DeployReplicaSourceResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @@ -421,61 +431,67 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) source_environment = task_info.get('source_environment', {}) export_info = task_info['export_info'] replica_resources_info = provider.deploy_replica_source_resources( - ctxt, connection_info, export_info, source_environment) + ctxt, connection_info, export_info, source_environment + ) - migr_connection_info = replica_resources_info.get( - "connection_info", {}) + migr_connection_info = replica_resources_info.get("connection_info", {}) if 'connection_info' not in replica_resources_info: LOG.warn( "Replica source provider for '%s' did NOT return any " "'connection_info'. Defaulting to '%s'", - origin["type"], migr_connection_info) + origin["type"], + migr_connection_info, + ) else: migr_connection_info = replica_resources_info['connection_info'] if migr_connection_info: - migr_connection_info = base.marshal_migr_conn_info( - migr_connection_info) + migr_connection_info = base.marshal_migr_conn_info(migr_connection_info) schemas.validate_value( migr_connection_info, schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA, # NOTE: we avoid raising so that the cleanup task # can [try] to deal with the temporary resources. - raise_on_error=False) + raise_on_error=False, + ) else: LOG.warn( "Replica source provider for '%s' returned empty " "'connection_info' in source resources deployment: %s", - origin["type"], migr_connection_info) + origin["type"], + migr_connection_info, + ) migr_resources = {} if 'migr_resources' not in replica_resources_info: LOG.warn( "Replica source provider for '%s' did NOT return any " "'migr_resources'. Defaulting to %s", - origin["type"], migr_resources) + origin["type"], + migr_resources, + ) else: migr_resources = replica_resources_info['migr_resources'] return { "source_resources": migr_resources, - "source_resources_connection_info": migr_connection_info} + "source_resources_connection_info": migr_connection_info, + } class DeleteReplicaSourceResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @@ -492,14 +508,14 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - event_handler) + origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, origin) migr_resources = task_info["source_resources"] @@ -509,15 +525,13 @@ def _run(self, ctxt, instance, origin, destination, task_info, if migr_resources: provider.delete_replica_source_resources( - ctxt, connection_info, source_environment, migr_resources) + ctxt, connection_info, source_environment, migr_resources + ) - return { - "source_resources": None, - "source_resources_connection_info": None} + return {"source_resources": None, "source_resources_connection_info": None} class DeployReplicaTargetResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -528,68 +542,72 @@ def get_required_task_info_properties(cls): @classmethod def get_returned_task_info_properties(cls): - return [ - "volumes_info", "target_resources", - "target_resources_connection_info"] + return ["volumes_info", "target_resources", "target_resources_connection_info"] @classmethod def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } def _validate_connection_info(self, migr_connection_info): try: - backup_writers.BackupWritersFactory( - migr_connection_info, None).get_writer() + backup_writers.BackupWritersFactory(migr_connection_info, None).get_writer() except Exception as err: LOG.warn( "Seemingly invalid backup writer conn info. Replica will " - "likely fail during disk Replication. Error is: %s" % ( - str(err))) + "likely fail during disk Replication. Error is: %s" % (str(err)) + ) if migr_connection_info: if 'connection_details' in migr_connection_info: migr_connection_info['connection_details'] = ( base.marshal_migr_conn_info( - migr_connection_info['connection_details'])) + migr_connection_info['connection_details'] + ) + ) schemas.validate_value( migr_connection_info, schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA, # NOTE: we avoid raising so that the cleanup task # can [try] to deal with the temporary resources. - raise_on_error=False) + raise_on_error=False, + ) - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): target_environment = task_info["target_environment"] export_info = task_info['export_info'] provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) volumes_info = _get_volumes_info(task_info) replica_resources_info = provider.deploy_replica_target_resources( - ctxt, connection_info, target_environment, volumes_info) + ctxt, connection_info, target_environment, volumes_info + ) schemas.validate_value( replica_resources_info, schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA, # NOTE: we avoid raising so that the cleanup task # can [try] to deal with the temporary resources. - raise_on_error=False) + raise_on_error=False, + ) if "volumes_info" in replica_resources_info: volumes_info = replica_resources_info["volumes_info"] volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + export_info, volumes_info + ) else: LOG.warn( "Replica target provider for '%s' did not return any " - "'volumes_info'. Using the previous value of it.") + "'volumes_info'. Using the previous value of it." + ) migr_connection_info = {} if 'connection_info' in replica_resources_info: @@ -599,25 +617,29 @@ def _run(self, ctxt, instance, origin, destination, task_info, LOG.warn( "Replica target provider for '%s' did NOT return any " "'connection_info'. Defaulting to %s", - destination["type"], migr_connection_info) + destination["type"], + migr_connection_info, + ) target_resources = {} if 'migr_resources' not in replica_resources_info: LOG.warn( "Replica target provider for '%s' did NOT return any " "'migr_resources'. Defaulting to %s", - destination["type"], target_resources) + destination["type"], + target_resources, + ) else: target_resources = replica_resources_info["migr_resources"] return { "volumes_info": volumes_info, "target_resources": target_resources, - "target_resources_connection_info": migr_connection_info} + "target_resources_connection_info": migr_connection_info, + } class DeleteReplicaTargetResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -628,21 +650,20 @@ def get_required_task_info_properties(cls): @classmethod def get_returned_task_info_properties(cls): - return [ - "target_resources", "target_resources_connection_info"] + return ["target_resources", "target_resources_connection_info"] @classmethod def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) migr_resources = task_info.get("target_resources") @@ -650,23 +671,20 @@ def _run(self, ctxt, instance, origin, destination, task_info, if migr_resources: provider.delete_replica_target_resources( - ctxt, connection_info, target_environment, migr_resources) + ctxt, connection_info, target_environment, migr_resources + ) - return { - "target_resources": None, - "target_resources_connection_info": None} + return {"target_resources": None, "target_resources_connection_info": None} class DeployReplicaInstanceResourcesTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod def get_required_task_info_properties(cls): - return [ - "export_info", "target_environment", "clone_disks", "volumes_info"] + return ["export_info", "target_environment", "clone_disks", "volumes_info"] @classmethod def get_returned_task_info_properties(cls): @@ -676,17 +694,17 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): target_environment = task_info["target_environment"] export_info = task_info["export_info"] provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) volumes_info = _get_volumes_info(task_info) @@ -694,16 +712,19 @@ def _run(self, ctxt, instance, origin, destination, task_info, LOG.debug("Clone disks: %s", clone_disks) import_info = provider.deploy_replica_instance( - ctxt, connection_info, target_environment, instance, - export_info, volumes_info, clone_disks) + ctxt, + connection_info, + target_environment, + instance, + export_info, + volumes_info, + clone_disks, + ) - return { - "instance_deployment_info": import_info[ - 'instance_deployment_info']} + return {"instance_deployment_info": import_info['instance_deployment_info']} class FinalizeReplicaInstanceDeploymentTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -720,32 +741,32 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) target_environment = task_info["target_environment"] instance_deployment_info = task_info["instance_deployment_info"] result = provider.finalize_replica_instance_deployment( - ctxt, connection_info, target_environment, - instance_deployment_info) + ctxt, connection_info, target_environment, instance_deployment_info + ) if result is None: LOG.warn( "'None' was returned as result for Finalize Replica Instance " - "deployment task '%s'.", task_info) + "deployment task '%s'.", + task_info, + ) - return { - "transfer_result": result} + return {"transfer_result": result} class CleanupFailedReplicaInstanceDeploymentTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -762,28 +783,26 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) target_environment = task_info["target_environment"] instance_deployment_info = task_info["instance_deployment_info"] provider.cleanup_failed_replica_instance_deployment( - ctxt, connection_info, target_environment, - instance_deployment_info) + ctxt, connection_info, target_environment, instance_deployment_info + ) - return { - "instance_deployment_info": None} + return {"instance_deployment_info": None} class CreateReplicaDiskSnapshotsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -800,14 +819,14 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) export_info = task_info['export_info'] target_environment = task_info["target_environment"] @@ -815,19 +834,16 @@ def _run(self, ctxt, instance, origin, destination, task_info, volumes_info = _get_volumes_info(task_info) volumes_info = provider.create_replica_disk_snapshots( - ctxt, connection_info, target_environment, volumes_info) - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + ctxt, connection_info, target_environment, volumes_info + ) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) - volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + volumes_info = _check_ensure_volumes_info_ordering(export_info, volumes_info) - return { - "volumes_info": volumes_info} + return {"volumes_info": volumes_info} class DeleteReplicaTargetDiskSnapshotsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -844,34 +860,31 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): export_info = task_info['export_info'] provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) volumes_info = _get_volumes_info(task_info) target_environment = task_info["target_environment"] volumes_info = provider.delete_replica_target_disk_snapshots( - ctxt, connection_info, target_environment, volumes_info) - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + ctxt, connection_info, target_environment, volumes_info + ) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) - volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + volumes_info = _check_ensure_volumes_info_ordering(export_info, volumes_info) - return { - "volumes_info": volumes_info} + return {"volumes_info": volumes_info} class RestoreReplicaDiskSnapshotsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -888,14 +901,14 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): provider = providers_factory.get_provider( - destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - event_handler) + destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler + ) connection_info = base.get_connection_info(ctxt, destination) export_info = task_info['export_info'] target_environment = task_info["target_environment"] @@ -903,19 +916,16 @@ def _run(self, ctxt, instance, origin, destination, task_info, volumes_info = _get_volumes_info(task_info) volumes_info = provider.restore_replica_disk_snapshots( - ctxt, connection_info, target_environment, volumes_info) - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + ctxt, connection_info, target_environment, volumes_info + ) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) - volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + volumes_info = _check_ensure_volumes_info_ordering(export_info, volumes_info) - return { - "volumes_info": volumes_info} + return {"volumes_info": volumes_info} class ValidateReplicaExecutionSourceInputsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @@ -932,31 +942,37 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT] + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) origin_type = origin["type"] source_provider = providers_factory.get_provider( - origin_type, constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, - event_handler, raise_if_not_found=False) + origin_type, + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, + event_handler, + raise_if_not_found=False, + ) origin_connection_info = base.get_connection_info(ctxt, origin) if not source_provider: event_manager.progress_update( "Replica Export Provider for platform '%s' does not support " - "Replica input validation" % origin_type) + "Replica input validation" % origin_type + ) else: source_provider.validate_replica_export_input( - ctxt, origin_connection_info, instance, - source_environment=task_info["source_environment"]) + ctxt, + origin_connection_info, + instance, + source_environment=task_info["source_environment"], + ) return {} class ValidateReplicaExecutionDestinationInputsTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -973,31 +989,38 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT + ] } def _validate_provider_replica_import_input( - self, provider, ctxt, conn_info, target_environment, export_info): + self, provider, ctxt, conn_info, target_environment, export_info + ): provider.validate_replica_import_input( - ctxt, conn_info, target_environment, export_info, + ctxt, + conn_info, + target_environment, + export_info, check_os_morphing_resources=False, - check_final_vm_params=False) + check_final_vm_params=False, + ) - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) destination_type = destination["type"] - destination_connection_info = base.get_connection_info( - ctxt, destination) + destination_connection_info = base.get_connection_info(ctxt, destination) destination_provider = providers_factory.get_provider( destination_type, - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, event_handler, - raise_if_not_found=False) + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, + event_handler, + raise_if_not_found=False, + ) if not destination_provider: event_manager.progress_update( "Replica Import Provider for platform '%s' does not support " - "Replica input validation" % destination_type) + "Replica input validation" % destination_type + ) return {} export_info = task_info.get("export_info") @@ -1005,18 +1028,22 @@ def _run(self, ctxt, instance, origin, destination, task_info, raise exception.InvalidActionTasksExecutionState( "Instance export info is not set. Cannot perform " "Replica Import validation for destination platform " - "'%s'" % destination_type) + "'%s'" % destination_type + ) target_environment = task_info["target_environment"] self._validate_provider_replica_import_input( - destination_provider, ctxt, destination_connection_info, - target_environment, export_info) + destination_provider, + ctxt, + destination_connection_info, + target_environment, + export_info, + ) return {} class ValidateReplicaDeploymentParametersTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -1033,43 +1060,43 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT] + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) - destination_connection_info = base.get_connection_info( - ctxt, destination) + destination_connection_info = base.get_connection_info(ctxt, destination) destination_type = destination["type"] export_info = task_info["export_info"] # validate Export info: - schemas.validate_value( - export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + schemas.validate_value(export_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) # validate destination params: destination_provider = providers_factory.get_provider( destination_type, - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, event_handler, - raise_if_not_found=False) + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, + event_handler, + raise_if_not_found=False, + ) if not destination_provider: event_manager.progress_update( "Replica Deployment Provider for platform '%s' does not " - "support Replica Deployment input validation" % ( - destination_type)) + "support Replica Deployment input validation" % (destination_type) + ) return {} # NOTE: the target environment JSON schema should have been validated # upon accepting the Replica API creation request. target_environment = task_info['target_environment'] destination_provider.validate_replica_deployment_input( - ctxt, destination_connection_info, target_environment, export_info) + ctxt, destination_connection_info, target_environment, export_info + ) return {} class UpdateSourceReplicaTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_SOURCE @@ -1086,11 +1113,11 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_SOURCE: [ - constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE] + constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) volumes_info = task_info.get("volumes_info", []) @@ -1100,44 +1127,46 @@ def _run(self, ctxt, instance, origin, destination, task_info, # the previous value of it: old_source_env = origin.get('source_environment') if not new_source_env: - event_manager.progress_update( - "No new source environment options provided") - return { - 'volumes_info': volumes_info, - 'source_environment': old_source_env} + event_manager.progress_update("No new source environment options provided") + return {'volumes_info': volumes_info, 'source_environment': old_source_env} source_provider = providers_factory.get_provider( - origin["type"], constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, - event_handler, raise_if_not_found=False) + origin["type"], + constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, + event_handler, + raise_if_not_found=False, + ) if not source_provider: raise exception.InvalidActionTasksExecutionState( "Replica source provider plugin for '%s' does not support" - " updating Replicas" % origin["type"]) + " updating Replicas" % origin["type"] + ) origin_connection_info = base.get_connection_info(ctxt, origin) LOG.info("Checking source provider environment params") - volumes_info = ( - source_provider.check_update_source_environment_params( - ctxt, origin_connection_info, instance, volumes_info, - old_source_env, new_source_env)) + volumes_info = source_provider.check_update_source_environment_params( + ctxt, + origin_connection_info, + instance, + volumes_info, + old_source_env, + new_source_env, + ) if volumes_info: - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) else: LOG.warn( "Source update method for '%s' source provider did NOT " "return any volumes info. Defaulting to old value.", - origin["type"]) + origin["type"], + ) volumes_info = task_info.get("volumes_info", []) - return { - "volumes_info": volumes_info, - "source_environment": new_source_env} + return {"volumes_info": volumes_info, "source_environment": new_source_env} class UpdateDestinationReplicaTask(base.TaskRunner): - @classmethod def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @@ -1154,11 +1183,11 @@ def get_returned_task_info_properties(cls): def get_required_provider_types(cls): return { constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE] + constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE + ] } - def _run(self, ctxt, instance, origin, destination, task_info, - event_handler): + def _run(self, ctxt, instance, origin, destination, task_info, event_handler): event_manager = events.EventManager(event_handler) volumes_info = task_info.get("volumes_info", []) @@ -1169,42 +1198,49 @@ def _run(self, ctxt, instance, origin, destination, task_info, old_destination_env = destination.get('target_environment', {}) if not new_destination_env: event_manager.progress_update( - "No new destination environment options provided") + "No new destination environment options provided" + ) return { "target_environment": old_destination_env, - "volumes_info": volumes_info} + "volumes_info": volumes_info, + } destination_provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE, - event_handler, raise_if_not_found=False) + event_handler, + raise_if_not_found=False, + ) if not destination_provider: raise exception.InvalidActionTasksExecutionState( "Replica destination provider plugin for '%s' does not " - "support updating Replicas" % destination["type"]) + "support updating Replicas" % destination["type"] + ) - destination_connection_info = base.get_connection_info( - ctxt, destination) + destination_connection_info = base.get_connection_info(ctxt, destination) export_info = task_info.get("export_info", {}) LOG.info("Checking destination provider environment params") - volumes_info = ( - destination_provider.check_update_destination_environment_params( - ctxt, destination_connection_info, export_info, volumes_info, - old_destination_env, new_destination_env)) + volumes_info = destination_provider.check_update_destination_environment_params( + ctxt, + destination_connection_info, + export_info, + volumes_info, + old_destination_env, + new_destination_env, + ) if volumes_info: - schemas.validate_value( - volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + schemas.validate_value(volumes_info, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) volumes_info = _check_ensure_volumes_info_ordering( - export_info, volumes_info) + export_info, volumes_info + ) else: LOG.warn( "Destination update method for '%s' dest provider did NOT " "return any volumes info. Defaulting to old value.", - destination["type"]) + destination["type"], + ) volumes_info = task_info.get("volumes_info", []) - return { - "volumes_info": volumes_info, - "target_environment": new_destination_env} + return {"volumes_info": volumes_info, "target_environment": new_destination_env} diff --git a/coriolis/tests/api/middleware/test_auth.py b/coriolis/tests/api/middleware/test_auth.py index 0a656348..8c988c24 100644 --- a/coriolis/tests/api/middleware/test_auth.py +++ b/coriolis/tests/api/middleware/test_auth.py @@ -3,12 +3,12 @@ from unittest import mock -from oslo_middleware import request_id import webob +from oslo_middleware import request_id -from coriolis.api.middleware import auth -from coriolis.api import wsgi from coriolis import context +from coriolis.api import wsgi +from coriolis.api.middleware import auth from coriolis.tests import test_base @@ -22,12 +22,7 @@ def setUp(self): @mock.patch.object(context, "RequestContext") @mock.patch.object(auth.CoriolisKeystoneContext, "_get_project_id") @mock.patch.object(auth.CoriolisKeystoneContext, "_get_user") - def test__call__( - self, - mock_get_user, - mock__get_project_id, - mock_request_context - ): + def test__call__(self, mock_get_user, mock__get_project_id, mock_request_context): req_mock = mock.Mock() expected_roles = ['1', '2', '3'] @@ -40,9 +35,7 @@ def test__call__( expected_req_id = 'mock_req_id' req_mock.remote_addr = expected_remote_address - req_mock.environ = { - request_id.ENV_REQUEST_ID: expected_req_id - } + req_mock.environ = {request_id.ENV_REQUEST_ID: expected_req_id} req_mock.headers = { 'X_ROLE': '1,2,3', @@ -50,16 +43,12 @@ def test__call__( 'X-Project-Domain-Name': expected_project_domain_name, 'X-User-Domain-Name': expected_user_domain_name, 'X_AUTH_TOKEN': expected_auth_token, - 'X_SERVICE_CATALOG': - str(expected_service_catalog).replace("'", '"'), + 'X_SERVICE_CATALOG': str(expected_service_catalog).replace("'", '"'), } result = self.auth(req_mock) - self.assertEqual( - self.auth.application, - result - ) + self.assertEqual(self.auth.application, result) mock_get_user.assert_called_once_with(req_mock) mock__get_project_id.assert_called_once_with(req_mock) @@ -73,22 +62,18 @@ def test__call__( auth_token=expected_auth_token, remote_address=expected_remote_address, service_catalog=expected_service_catalog, - request_id=expected_req_id + request_id=expected_req_id, ) self.assertEqual( - req_mock.environ['coriolis.context'], - mock_request_context.return_value + req_mock.environ['coriolis.context'], mock_request_context.return_value ) @mock.patch.object(context, "RequestContext") @mock.patch.object(auth.CoriolisKeystoneContext, "_get_project_id") @mock.patch.object(auth.CoriolisKeystoneContext, "_get_user") def test__call__req_id_is_bytes( - self, - mock_get_user, - mock__get_project_id, - mock_request_context + self, mock_get_user, mock__get_project_id, mock_request_context ): req_mock = mock.Mock() @@ -102,9 +87,7 @@ def test__call__req_id_is_bytes( expected_req_id = 'mock_req_id' req_mock.remote_addr = expected_remote_address - req_mock.environ = { - request_id.ENV_REQUEST_ID: expected_req_id.encode() - } + req_mock.environ = {request_id.ENV_REQUEST_ID: expected_req_id.encode()} req_mock.headers = { 'X_ROLE': '1,2,3', @@ -112,16 +95,12 @@ def test__call__req_id_is_bytes( 'X-Project-Domain-Name': expected_project_domain_name, 'X-User-Domain-Name': expected_user_domain_name, 'X_AUTH_TOKEN': expected_auth_token, - 'X_SERVICE_CATALOG': - str(expected_service_catalog).replace("'", '"'), + 'X_SERVICE_CATALOG': str(expected_service_catalog).replace("'", '"'), } result = self.auth(req_mock) - self.assertEqual( - self.auth.application, - result - ) + self.assertEqual(self.auth.application, result) mock_get_user.assert_called_once_with(req_mock) mock__get_project_id.assert_called_once_with(req_mock) @@ -135,12 +114,11 @@ def test__call__req_id_is_bytes( auth_token=expected_auth_token, remote_address=expected_remote_address, service_catalog=expected_service_catalog, - request_id=expected_req_id + request_id=expected_req_id, ) self.assertEqual( - req_mock.environ['coriolis.context'], - mock_request_context.return_value + req_mock.environ['coriolis.context'], mock_request_context.return_value ) @mock.patch.object(auth.CoriolisKeystoneContext, "_get_project_id") @@ -174,11 +152,7 @@ def test__call__no_headers( req_mock.headers = {} - self.assertRaises( - webob.exc.HTTPUnauthorized, - self.auth, - req_mock - ) + self.assertRaises(webob.exc.HTTPUnauthorized, self.auth, req_mock) def test_get_project_id_tenant_id(self): req_mock = mock.Mock() @@ -191,10 +165,7 @@ def test_get_project_id_tenant_id(self): result = self.auth._get_project_id(req_mock) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_project_id_tenant(self): req_mock = mock.Mock() @@ -207,21 +178,14 @@ def test_get_project_id_tenant(self): result = self.auth._get_project_id(req_mock) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_project_id_no_tenant(self): req_mock = mock.Mock() req_mock.headers = {} - self.assertRaises( - webob.exc.HTTPBadRequest, - self.auth._get_project_id, - req_mock - ) + self.assertRaises(webob.exc.HTTPBadRequest, self.auth._get_project_id, req_mock) def test_get_user(self): req_mock = mock.Mock() @@ -237,10 +201,7 @@ def test_get_user(self): result = self.auth._get_user(req_mock) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_user_has_user(self): req_mock = mock.Mock() @@ -253,10 +214,7 @@ def test_get_user_has_user(self): result = self.auth._get_user(req_mock) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_user_has_user_id(self): req_mock = mock.Mock() @@ -269,18 +227,11 @@ def test_get_user_has_user_id(self): result = self.auth._get_user(req_mock) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_user_none(self): req_mock = mock.Mock() req_mock.headers = {} - self.assertRaises( - webob.exc.HTTPUnauthorized, - self.auth._get_user, - req_mock - ) + self.assertRaises(webob.exc.HTTPUnauthorized, self.auth._get_user, req_mock) diff --git a/coriolis/tests/api/test_common.py b/coriolis/tests/api/test_common.py index 1d4a46b7..078c6056 100644 --- a/coriolis/tests/api/test_common.py +++ b/coriolis/tests/api/test_common.py @@ -26,8 +26,8 @@ def test_get_paging_params_unspecified(self): def test_get_sort_params(self): req = webob.Request.blank( - '/some-path?' - 'sort_key=key0&sort_dir=dir0&sort_key=key1&sort_dir=dir1') + '/some-path?sort_key=key0&sort_dir=dir0&sort_key=key1&sort_dir=dir1' + ) sort_keys, sort_dirs = common.get_sort_params(req) diff --git a/coriolis/tests/api/test_wsgi.py b/coriolis/tests/api/test_wsgi.py index e6b6e5ec..4ce58ea6 100644 --- a/coriolis/tests/api/test_wsgi.py +++ b/coriolis/tests/api/test_wsgi.py @@ -3,8 +3,8 @@ import webob.exc -from coriolis.api import wsgi from coriolis import exception +from coriolis.api import wsgi from coriolis.tests import test_base diff --git a/coriolis/tests/api/v1/test_diagnostics.py b/coriolis/tests/api/v1/test_diagnostics.py index da03eeb2..e35b0444 100644 --- a/coriolis/tests/api/v1/test_diagnostics.py +++ b/coriolis/tests/api/v1/test_diagnostics.py @@ -18,11 +18,7 @@ def setUp(self): @mock.patch.object(api.API, 'get') @mock.patch.object(diagnostic_view, 'collection') - def test_index( - self, - mock_collection, - mock_get - ): + def test_index(self, mock_collection, mock_get): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} diff --git a/coriolis/tests/api/v1/test_endpoint_actions.py b/coriolis/tests/api/v1/test_endpoint_actions.py index b23049d7..20302205 100644 --- a/coriolis/tests/api/v1/test_endpoint_actions.py +++ b/coriolis/tests/api/v1/test_endpoint_actions.py @@ -5,11 +5,10 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import endpoint_actions from coriolis.endpoints import api -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class EndpointActionsControllerTestCase(test_base.CoriolisBaseTestCase): @@ -20,10 +19,7 @@ def setUp(self): self.endpoint_api = endpoint_actions.EndpointActionsController() @mock.patch.object(api.API, 'validate_connection') - def test_validate_connection( - self, - mock_validate_connection - ): + def test_validate_connection(self, mock_validate_connection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -34,29 +30,22 @@ def test_validate_connection( mock_validate_connection.return_value = (is_valid, message) expected_result = { - "validate-connection": - {"valid": is_valid, "message": message} + "validate-connection": {"valid": is_valid, "message": message} } - result = testutils.get_wrapped_function( - self.endpoint_api._validate_connection)( - mock_req, - id, - body # type: ignore + result = testutils.get_wrapped_function(self.endpoint_api._validate_connection)( + mock_req, + id, + body, # type: ignore ) mock_context.can.assert_called_once_with( - 'migration:endpoints:validate_connection') - mock_validate_connection.assert_called_once_with(mock_context, id) - self.assertEqual( - expected_result, - result + 'migration:endpoints:validate_connection' ) + mock_validate_connection.assert_called_once_with(mock_context, id) + self.assertEqual(expected_result, result) @mock.patch.object(api.API, 'validate_connection') - def test_validate_connection_except_not_found( - self, - mock_validate_connection - ): + def test_validate_connection_except_not_found(self, mock_validate_connection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -66,18 +55,16 @@ def test_validate_connection_except_not_found( self.assertRaises( exc.HTTPNotFound, - testutils.get_wrapped_function( - self.endpoint_api._validate_connection), + testutils.get_wrapped_function(self.endpoint_api._validate_connection), mock_req, id, - body + body, ) mock_validate_connection.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'validate_connection') def test_validate_connection_except_invalid_parameter_value( - self, - mock_validate_connection + self, mock_validate_connection ): mock_req = mock.Mock() mock_context = mock.Mock() @@ -90,10 +77,9 @@ def test_validate_connection_except_invalid_parameter_value( self.assertRaises( exc.HTTPNotFound, - testutils.get_wrapped_function( - self.endpoint_api._validate_connection), + testutils.get_wrapped_function(self.endpoint_api._validate_connection), mock_req, id, - body + body, ) mock_validate_connection.assert_called_once_with(mock_context, id) diff --git a/coriolis/tests/api/v1/test_endpoint_destination_minion_pool_options.py b/coriolis/tests/api/v1/test_endpoint_destination_minion_pool_options.py index 598cc032..50b8f495 100644 --- a/coriolis/tests/api/v1/test_endpoint_destination_minion_pool_options.py +++ b/coriolis/tests/api/v1/test_endpoint_destination_minion_pool_options.py @@ -3,12 +3,11 @@ from unittest import mock -from coriolis.api.v1 import endpoint_destination_minion_pool_options \ - as endpoint +from coriolis import utils +from coriolis.api.v1 import endpoint_destination_minion_pool_options as endpoint from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.tests import test_base -from coriolis import utils class EndpointDestinationMinionPoolOptionsControllerTestCase( @@ -19,18 +18,14 @@ class EndpointDestinationMinionPoolOptionsControllerTestCase( """ def setUp(self): - super( - EndpointDestinationMinionPoolOptionsControllerTestCase, - self - ).setUp() - self.minion_api = \ - endpoint.EndpointDestinationMinionPoolOptionsController() + super(EndpointDestinationMinionPoolOptionsControllerTestCase, self).setUp() + self.minion_api = endpoint.EndpointDestinationMinionPoolOptionsController() @mock.patch.object(utils, 'decode_base64_param') - @mock.patch.object(endpoint_options_view, - 'destination_minion_pool_options_collection') - @mock.patch.object(api.API, - 'get_endpoint_destination_minion_pool_options') + @mock.patch.object( + endpoint_options_view, 'destination_minion_pool_options_collection' + ) + @mock.patch.object(api.API, 'get_endpoint_destination_minion_pool_options') def test_index( self, mock_get_endpoint_destination_minion_pool_options, @@ -43,39 +38,35 @@ def test_index( mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env options = mock.sentinel.options - mock_req.GET = { - 'env': env, - 'options': options - } + mock_req.GET = {'env': env, 'options': options} mock_decode_base64_param.side_effect = [env, options] expected_calls = [ mock.call.mock_get_diagnostics_policy_label(env, is_json=True), - mock.call.mock_get_diagnostics_policy_label(options, is_json=True)] + mock.call.mock_get_diagnostics_policy_label(options, is_json=True), + ] result = self.minion_api.index(mock_req, endpoint_id) mock_context.can.assert_called_once_with( - 'migration:endpoints:list_destination_minion_pool_options') + 'migration:endpoints:list_destination_minion_pool_options' + ) mock_decode_base64_param.assert_has_calls(expected_calls) - (mock_get_endpoint_destination_minion_pool_options. - assert_called_once_with)( - mock_context, endpoint_id, - env=env, - option_names=options) - (mock_destination_minion_pool_options_collection. - assert_called_once_with)( - mock_get_endpoint_destination_minion_pool_options.return_value) + (mock_get_endpoint_destination_minion_pool_options.assert_called_once_with)( + mock_context, endpoint_id, env=env, option_names=options + ) + (mock_destination_minion_pool_options_collection.assert_called_once_with)( + mock_get_endpoint_destination_minion_pool_options.return_value + ) self.assertEqual( - mock_destination_minion_pool_options_collection.return_value, - result + mock_destination_minion_pool_options_collection.return_value, result ) @mock.patch.object(utils, 'decode_base64_param') - @mock.patch.object(endpoint_options_view, - 'destination_minion_pool_options_collection') - @mock.patch.object(api.API, - 'get_endpoint_destination_minion_pool_options') + @mock.patch.object( + endpoint_options_view, 'destination_minion_pool_options_collection' + ) + @mock.patch.object(api.API, 'get_endpoint_destination_minion_pool_options') def test_index_no_env_and_options( self, mock_get_endpoint_destination_minion_pool_options, @@ -91,15 +82,12 @@ def test_index_no_env_and_options( result = self.minion_api.index(mock_req, endpoint_id) mock_decode_base64_param.assert_not_called() - (mock_get_endpoint_destination_minion_pool_options. - assert_called_once_with)( - mock_context, endpoint_id, - env={}, - option_names={}) - (mock_destination_minion_pool_options_collection. - assert_called_once_with)( - mock_get_endpoint_destination_minion_pool_options.return_value) + (mock_get_endpoint_destination_minion_pool_options.assert_called_once_with)( + mock_context, endpoint_id, env={}, option_names={} + ) + (mock_destination_minion_pool_options_collection.assert_called_once_with)( + mock_get_endpoint_destination_minion_pool_options.return_value + ) self.assertEqual( - mock_destination_minion_pool_options_collection.return_value, - result + mock_destination_minion_pool_options_collection.return_value, result ) diff --git a/coriolis/tests/api/v1/test_endpoint_destination_options.py b/coriolis/tests/api/v1/test_endpoint_destination_options.py index ec6b3163..3dfcf920 100644 --- a/coriolis/tests/api/v1/test_endpoint_destination_options.py +++ b/coriolis/tests/api/v1/test_endpoint_destination_options.py @@ -3,16 +3,14 @@ from unittest import mock +from coriolis import utils from coriolis.api.v1 import endpoint_destination_options as endpoint from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.tests import test_base -from coriolis import utils -class EndpointDestinationOptionsControllerTestCase( - test_base.CoriolisBaseTestCase -): +class EndpointDestinationOptionsControllerTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Endpoint Destination Options v1 API""" def setUp(self): @@ -34,37 +32,31 @@ def test_index( mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env options = mock.sentinel.options - mock_req.GET = { - 'env': env, - 'options': options - } + mock_req.GET = {'env': env, 'options': options} mock_decode_base64_param.side_effect = [env, options] expected_calls = [ mock.call.mock_decode_base64_param(env, is_json=True), - mock.call.mock_decode_base64_param(options, is_json=True)] + mock.call.mock_decode_base64_param(options, is_json=True), + ] result = self.endpoint_api.index(mock_req, endpoint_id) mock_context.can.assert_called_once_with( - 'migration:endpoints:list_destination_options') + 'migration:endpoints:list_destination_options' + ) mock_decode_base64_param.assert_has_calls(expected_calls) mock_get_endpoint_destination_options.assert_called_once_with( - mock_context, endpoint_id, - env=env, - option_names=options) + mock_context, endpoint_id, env=env, option_names=options + ) mock_destination_options_collection.assert_called_once_with( - mock_get_endpoint_destination_options.return_value) - self.assertEqual( - mock_destination_options_collection.return_value, - result + mock_get_endpoint_destination_options.return_value ) + self.assertEqual(mock_destination_options_collection.return_value, result) @mock.patch.object(utils, 'decode_base64_param') - @mock.patch.object(endpoint_options_view, - 'destination_options_collection') - @mock.patch.object(api.API, - 'get_endpoint_destination_options') + @mock.patch.object(endpoint_options_view, 'destination_options_collection') + @mock.patch.object(api.API, 'get_endpoint_destination_options') def test_index_no_env_and_options( self, mock_get_endpoint_destination_options, @@ -81,12 +73,9 @@ def test_index_no_env_and_options( mock_decode_base64_param.assert_not_called() mock_get_endpoint_destination_options.assert_called_once_with( - mock_context, endpoint_id, - env={}, - option_names={}) + mock_context, endpoint_id, env={}, option_names={} + ) mock_destination_options_collection.assert_called_once_with( - mock_get_endpoint_destination_options.return_value) - self.assertEqual( - mock_destination_options_collection.return_value, - result + mock_get_endpoint_destination_options.return_value ) + self.assertEqual(mock_destination_options_collection.return_value, result) diff --git a/coriolis/tests/api/v1/test_endpoint_instances.py b/coriolis/tests/api/v1/test_endpoint_instances.py index e374b9a2..027adb16 100644 --- a/coriolis/tests/api/v1/test_endpoint_instances.py +++ b/coriolis/tests/api/v1/test_endpoint_instances.py @@ -3,12 +3,12 @@ from unittest import mock +from coriolis import utils from coriolis.api import common from coriolis.api.v1 import endpoint_instances as endpoint from coriolis.api.v1.views import endpoint_resources_view from coriolis.endpoint_resources import api from coriolis.tests import test_base -from coriolis import utils class EndpointInstanceControllerTestCase(test_base.CoriolisBaseTestCase): @@ -35,30 +35,29 @@ def test_index( mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env instance_name_pattern = mock.sentinel.instance_name_pattern - mock_req.GET = { - 'env': env, - 'name': instance_name_pattern - } + mock_req.GET = {'env': env, 'name': instance_name_pattern} marker = 'mock_marker' limit = 'mock_limit' mock_get_paging_params.return_value = (marker, limit) result = self.endpoint_api.index(mock_req, endpoint_id) - mock_context.can.assert_called_once_with( - 'migration:endpoints:list_instances') + mock_context.can.assert_called_once_with('migration:endpoints:list_instances') mock_get_paging_params.assert_called_once_with(mock_req) mock_decode_base64_param.assert_called_once_with(env, is_json=True) mock_get_endpoint_instances.assert_called_once_with( - mock_context, endpoint_id, + mock_context, + endpoint_id, mock_decode_base64_param.return_value, - marker, limit, instance_name_pattern, refresh=False) + marker, + limit, + instance_name_pattern, + refresh=False, + ) mock_instances_collection.assert_called_once_with( - mock_get_endpoint_instances.return_value) - self.assertEqual( - mock_instances_collection.return_value, - result + mock_get_endpoint_instances.return_value ) + self.assertEqual(mock_instances_collection.return_value, result) @mock.patch.object(common, 'get_paging_params') @mock.patch.object(utils, 'decode_base64_param') @@ -83,14 +82,12 @@ def test_index_no_env_and_options( mock_get_paging_params.assert_called_once_with(mock_req) mock_decode_base64_param.assert_not_called() mock_get_endpoint_instances.assert_called_once_with( - mock_context, endpoint_id, - {}, None, None, None, refresh=False) + mock_context, endpoint_id, {}, None, None, None, refresh=False + ) mock_instances_collection.assert_called_once_with( - mock_get_endpoint_instances.return_value) - self.assertEqual( - mock_instances_collection.return_value, - result + mock_get_endpoint_instances.return_value ) + self.assertEqual(mock_instances_collection.return_value, result) @mock.patch.object(utils, 'decode_base64_param') @mock.patch.object(endpoint_resources_view, 'instance_single') @@ -114,23 +111,20 @@ def test_show( expected_calls = [ mock.call.mock_decode_base64_param(id), - mock.call.mock_decode_base64_param(env, is_json=True)] + mock.call.mock_decode_base64_param(env, is_json=True), + ] result = self.endpoint_api.show(mock_req, endpoint_id, id) - mock_context.can.assert_called_once_with( - 'migration:endpoints:get_instance') + mock_context.can.assert_called_once_with('migration:endpoints:get_instance') mock_decode_base64_param.assert_has_calls(expected_calls) mock_get_endpoint_instance.assert_called_once_with( - mock_context, endpoint_id, - env, - id) + mock_context, endpoint_id, env, id + ) mock_instance_single.assert_called_once_with( - mock_get_endpoint_instance.return_value) - self.assertEqual( - mock_instance_single.return_value, - result + mock_get_endpoint_instance.return_value ) + self.assertEqual(mock_instance_single.return_value, result) @mock.patch.object(utils, 'decode_base64_param') @mock.patch.object(endpoint_resources_view, 'instance_single') @@ -152,11 +146,9 @@ def test_show_no_env( mock_decode_base64_param.assert_called_once_with(id) mock_get_endpoint_instance.assert_called_once_with( - mock_context, endpoint_id, {}, - mock_decode_base64_param.return_value) + mock_context, endpoint_id, {}, mock_decode_base64_param.return_value + ) mock_instance_single.assert_called_once_with( - mock_get_endpoint_instance.return_value) - self.assertEqual( - mock_instance_single.return_value, - result + mock_get_endpoint_instance.return_value ) + self.assertEqual(mock_instance_single.return_value, result) diff --git a/coriolis/tests/api/v1/test_endpoint_inventory.py b/coriolis/tests/api/v1/test_endpoint_inventory.py index e187c0ec..0b2ef547 100644 --- a/coriolis/tests/api/v1/test_endpoint_inventory.py +++ b/coriolis/tests/api/v1/test_endpoint_inventory.py @@ -3,11 +3,11 @@ from unittest import mock -from coriolis.api.v1 import endpoint_inventory as endpoint +from coriolis import utils from coriolis.api import wsgi as api_wsgi +from coriolis.api.v1 import endpoint_inventory as endpoint from coriolis.endpoint_resources import api from coriolis.tests import test_base -from coriolis import utils class EndpointInventoryControllerTestCase(test_base.CoriolisBaseTestCase): @@ -34,12 +34,11 @@ def test_index( response = self.endpoint_api.index(mock_req, endpoint_id) - mock_context.can.assert_called_once_with( - 'migration:endpoints:export_inventory') + mock_context.can.assert_called_once_with('migration:endpoints:export_inventory') mock_decode_base64_param.assert_called_once_with(env, is_json=True) mock_get_endpoint_inventory_csv.assert_called_once_with( - mock_context, endpoint_id, - mock_decode_base64_param.return_value) + mock_context, endpoint_id, mock_decode_base64_param.return_value + ) self.assertIsInstance(response, api_wsgi.ResponseObject) self.assertEqual(response.code, 200) self.assertEqual(response.obj, 'vm_id,vm_name\n') @@ -62,6 +61,7 @@ def test_index_no_env( mock_decode_base64_param.assert_not_called() mock_get_endpoint_inventory_csv.assert_called_once_with( - mock_context, endpoint_id, {}) + mock_context, endpoint_id, {} + ) self.assertIsInstance(response, api_wsgi.ResponseObject) self.assertEqual(response.code, 200) diff --git a/coriolis/tests/api/v1/test_endpoint_networks.py b/coriolis/tests/api/v1/test_endpoint_networks.py index e1acf9ba..b6fadda3 100644 --- a/coriolis/tests/api/v1/test_endpoint_networks.py +++ b/coriolis/tests/api/v1/test_endpoint_networks.py @@ -3,11 +3,11 @@ from unittest import mock +from coriolis import utils from coriolis.api.v1 import endpoint_networks as endpoint from coriolis.api.v1.views import endpoint_resources_view from coriolis.endpoint_resources import api from coriolis.tests import test_base -from coriolis import utils class EndpointNetworkControllerTestCase(test_base.CoriolisBaseTestCase): @@ -31,24 +31,19 @@ def test_index( endpoint_id = mock.sentinel.endpoint_id mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env - mock_req.GET = { - 'env': env - } + mock_req.GET = {'env': env} result = self.endpoint_api.index(mock_req, endpoint_id) - mock_context.can.assert_called_once_with( - 'migration:endpoints:list_networks') + mock_context.can.assert_called_once_with('migration:endpoints:list_networks') mock_decode_base64_param.assert_called_once_with(env, is_json=True) mock_get_endpoint_networks.assert_called_once_with( - mock_context, endpoint_id, - mock_decode_base64_param.return_value) + mock_context, endpoint_id, mock_decode_base64_param.return_value + ) mock_networks_collection.assert_called_once_with( - mock_get_endpoint_networks.return_value) - self.assertEqual( - mock_networks_collection.return_value, - result + mock_get_endpoint_networks.return_value ) + self.assertEqual(mock_networks_collection.return_value, result) @mock.patch.object(utils, 'decode_base64_param') @mock.patch.object(endpoint_resources_view, 'networks_collection') @@ -69,10 +64,9 @@ def test_index_no_env( mock_decode_base64_param.assert_not_called() mock_get_endpoint_networks.assert_called_once_with( - mock_context, endpoint_id, {}) + mock_context, endpoint_id, {} + ) mock_networks_collection.assert_called_once_with( - mock_get_endpoint_networks.return_value) - self.assertEqual( - mock_networks_collection.return_value, - result + mock_get_endpoint_networks.return_value ) + self.assertEqual(mock_networks_collection.return_value, result) diff --git a/coriolis/tests/api/v1/test_endpoint_source_minion_pool_options.py b/coriolis/tests/api/v1/test_endpoint_source_minion_pool_options.py index 7d43c482..9e54b285 100644 --- a/coriolis/tests/api/v1/test_endpoint_source_minion_pool_options.py +++ b/coriolis/tests/api/v1/test_endpoint_source_minion_pool_options.py @@ -3,28 +3,24 @@ from unittest import mock +from coriolis import utils from coriolis.api.v1 import endpoint_source_minion_pool_options as endpoint from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.tests import test_base -from coriolis import utils -class EndpointSourceMinionPoolOptionsControllerTestCase( - test_base.CoriolisBaseTestCase -): +class EndpointSourceMinionPoolOptionsControllerTestCase(test_base.CoriolisBaseTestCase): """ Test suite for the Coriolis Endpoint Source Minion Pool Options v1 API """ def setUp(self): super(EndpointSourceMinionPoolOptionsControllerTestCase, self).setUp() - self.endpoint_api = \ - endpoint.EndpointSourceMinionPoolOptionsController() + self.endpoint_api = endpoint.EndpointSourceMinionPoolOptionsController() @mock.patch.object(utils, 'decode_base64_param') - @mock.patch.object(endpoint_options_view, - 'source_minion_pool_options_collection') + @mock.patch.object(endpoint_options_view, 'source_minion_pool_options_collection') @mock.patch.object(api.API, 'get_endpoint_source_minion_pool_options') def test_index( self, @@ -38,37 +34,32 @@ def test_index( mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env options = mock.sentinel.options - mock_req.GET = { - 'env': env, - 'options': options - } + mock_req.GET = {'env': env, 'options': options} mock_decode_base64_param.side_effect = [env, options] expected_calls = [ mock.call.mock_decode_base64_param(env, is_json=True), - mock.call.mock_decode_base64_param(options, is_json=True)] + mock.call.mock_decode_base64_param(options, is_json=True), + ] result = self.endpoint_api.index(mock_req, endpoint_id) mock_context.can.assert_called_once_with( - 'migration:endpoints:list_source_minion_pool_options') + 'migration:endpoints:list_source_minion_pool_options' + ) mock_decode_base64_param.assert_has_calls(expected_calls) - (mock_get_endpoint_source_minion_pool_options. - assert_called_once_with)( - mock_context, endpoint_id, - env=env, - option_names=options) - (mock_source_minion_pool_options_collection. - assert_called_once_with)( - mock_get_endpoint_source_minion_pool_options.return_value) + (mock_get_endpoint_source_minion_pool_options.assert_called_once_with)( + mock_context, endpoint_id, env=env, option_names=options + ) + (mock_source_minion_pool_options_collection.assert_called_once_with)( + mock_get_endpoint_source_minion_pool_options.return_value + ) self.assertEqual( - mock_source_minion_pool_options_collection.return_value, - result + mock_source_minion_pool_options_collection.return_value, result ) @mock.patch.object(utils, 'decode_base64_param') - @mock.patch.object(endpoint_options_view, - 'source_minion_pool_options_collection') + @mock.patch.object(endpoint_options_view, 'source_minion_pool_options_collection') @mock.patch.object(api.API, 'get_endpoint_source_minion_pool_options') def test_index_no_env( self, @@ -86,12 +77,11 @@ def test_index_no_env( mock_decode_base64_param.assert_not_called() mock_get_endpoint_source_minion_pool_options.assert_called_once_with( - mock_context, endpoint_id, - env={}, - option_names={}) + mock_context, endpoint_id, env={}, option_names={} + ) mock_source_minion_pool_options_collection.assert_called_once_with( - mock_get_endpoint_source_minion_pool_options.return_value) + mock_get_endpoint_source_minion_pool_options.return_value + ) self.assertEqual( - mock_source_minion_pool_options_collection.return_value, - result + mock_source_minion_pool_options_collection.return_value, result ) diff --git a/coriolis/tests/api/v1/test_endpoint_source_options.py b/coriolis/tests/api/v1/test_endpoint_source_options.py index d8f4ce5f..7b52065d 100644 --- a/coriolis/tests/api/v1/test_endpoint_source_options.py +++ b/coriolis/tests/api/v1/test_endpoint_source_options.py @@ -3,11 +3,11 @@ from unittest import mock +from coriolis import utils from coriolis.api.v1 import endpoint_source_options as endpoint from coriolis.api.v1.views import endpoint_options_view from coriolis.endpoint_options import api from coriolis.tests import test_base -from coriolis import utils class EndpointSourceOptionsControllerTestCase(test_base.CoriolisBaseTestCase): @@ -32,33 +32,27 @@ def test_index( mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env options = mock.sentinel.options - mock_req.GET = { - 'env': env, - 'options': options - } + mock_req.GET = {'env': env, 'options': options} mock_decode_base64_param.side_effect = [env, options] expected_calls = [ - mock.call.mock_decode_base64_param( - env, is_json=True), - mock.call.mock_decode_base64_param( - options, is_json=True)] + mock.call.mock_decode_base64_param(env, is_json=True), + mock.call.mock_decode_base64_param(options, is_json=True), + ] result = self.endpoint_api.index(mock_req, endpoint_id) mock_context.can.assert_called_once_with( - 'migration:endpoints:list_source_options') + 'migration:endpoints:list_source_options' + ) mock_decode_base64_param.assert_has_calls(expected_calls) mock_get_endpoint_source_options.assert_called_once_with( - mock_context, endpoint_id, - env=env, - option_names=options) + mock_context, endpoint_id, env=env, option_names=options + ) mock_source_options_collection.assert_called_once_with( - mock_get_endpoint_source_options.return_value) - self.assertEqual( - mock_source_options_collection.return_value, - result + mock_get_endpoint_source_options.return_value ) + self.assertEqual(mock_source_options_collection.return_value, result) @mock.patch.object(utils, 'decode_base64_param') @mock.patch.object(endpoint_options_view, 'source_options_collection') @@ -79,12 +73,9 @@ def test_index_no_env( mock_decode_base64_param.assert_not_called() mock_get_endpoint_source_options.assert_called_once_with( - mock_context, endpoint_id, - env={}, - option_names={}) + mock_context, endpoint_id, env={}, option_names={} + ) mock_source_options_collection.assert_called_once_with( - mock_get_endpoint_source_options.return_value) - self.assertEqual( - mock_source_options_collection.return_value, - result + mock_get_endpoint_source_options.return_value ) + self.assertEqual(mock_source_options_collection.return_value, result) diff --git a/coriolis/tests/api/v1/test_endpoint_storage.py b/coriolis/tests/api/v1/test_endpoint_storage.py index af9dff75..e36fc31b 100644 --- a/coriolis/tests/api/v1/test_endpoint_storage.py +++ b/coriolis/tests/api/v1/test_endpoint_storage.py @@ -3,11 +3,11 @@ from unittest import mock +from coriolis import utils from coriolis.api.v1 import endpoint_storage as endpoint from coriolis.api.v1.views import endpoint_resources_view from coriolis.endpoint_resources import api from coriolis.tests import test_base -from coriolis import utils class EndpointStorageControllerTestCase(test_base.CoriolisBaseTestCase): @@ -31,24 +31,19 @@ def test_index( endpoint_id = mock.sentinel.endpoint_id mock_req.environ = {'coriolis.context': mock_context} env = mock.sentinel.env - mock_req.GET = { - 'env': env - } + mock_req.GET = {'env': env} result = self.endpoint_api.index(mock_req, endpoint_id) - mock_context.can.assert_called_once_with( - 'migration:endpoints:list_storage') + mock_context.can.assert_called_once_with('migration:endpoints:list_storage') mock_decode_base64_param.assert_called_once_with(env, is_json=True) mock_storage_collection.assert_called_once_with( - mock_context, endpoint_id, - mock_decode_base64_param.return_value) + mock_context, endpoint_id, mock_decode_base64_param.return_value + ) mock_get_endpoint_storage.assert_called_once_with( - mock_storage_collection.return_value) - self.assertEqual( - mock_get_endpoint_storage.return_value, - result + mock_storage_collection.return_value ) + self.assertEqual(mock_get_endpoint_storage.return_value, result) @mock.patch.object(utils, 'decode_base64_param') @mock.patch.object(endpoint_resources_view, 'storage_collection') @@ -68,11 +63,8 @@ def test_index_no_env( result = self.endpoint_api.index(mock_req, endpoint_id) mock_decode_base64_param.assert_not_called() - mock_storage_collection.assert_called_once_with( - mock_context, endpoint_id, {}) + mock_storage_collection.assert_called_once_with(mock_context, endpoint_id, {}) mock_get_endpoint_storage.assert_called_once_with( - mock_storage_collection.return_value) - self.assertEqual( - mock_get_endpoint_storage.return_value, - result + mock_storage_collection.return_value ) + self.assertEqual(mock_get_endpoint_storage.return_value, result) diff --git a/coriolis/tests/api/v1/test_endpoints.py b/coriolis/tests/api/v1/test_endpoints.py index 17f415cc..be8a7d6b 100644 --- a/coriolis/tests/api/v1/test_endpoints.py +++ b/coriolis/tests/api/v1/test_endpoints.py @@ -5,12 +5,11 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import endpoints from coriolis.api.v1.views import endpoint_view from coriolis.endpoints import api -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class EndpointControllerTestCase(test_base.CoriolisBaseTestCase): @@ -36,13 +35,8 @@ def test_show( mock_context.can.assert_called_once_with('migration:endpoints:show') mock_get_endpoint.assert_called_once_with(mock_context, id) - mock_single.assert_called_once_with( - mock_get_endpoint.return_value - ) - self.assertEqual( - mock_single.return_value, - result - ) + mock_single.assert_called_once_with(mock_get_endpoint.return_value) + self.assertEqual(mock_single.return_value, result) @mock.patch.object(api.API, 'get_endpoint') def test_show_no_endpoint( @@ -55,12 +49,7 @@ def test_show_no_endpoint( id = mock.sentinel.id mock_get_endpoint.return_value = None - self.assertRaises( - exc.HTTPNotFound, - self.endpoint_api.show, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.endpoint_api.show, mock_req, id) mock_context.can.assert_called_once_with('migration:endpoints:show') mock_get_endpoint.assert_called_once_with(mock_context, id) @@ -79,61 +68,59 @@ def test_index( mock_context.can.assert_called_once_with('migration:endpoints:list') mock_get_endpoints.assert_called_once_with(mock_context) - mock_collection.assert_called_once_with( - mock_get_endpoints.return_value - ) - self.assertEqual( - mock_collection.return_value, - result - ) + mock_collection.assert_called_once_with(mock_get_endpoints.return_value) + self.assertEqual(mock_collection.return_value, result) def test__validate_create_body(self): mock_body = { - 'endpoint': - { - 'name': 'mock_name', - 'type': 'mock_type', - 'connection_info': 'mock_connection_info', - } + 'endpoint': { + 'name': 'mock_name', + 'type': 'mock_type', + 'connection_info': 'mock_connection_info', + } } endpoint = testutils.get_wrapped_function( - self.endpoint_api._validate_create_body)( - self.endpoint_api, - body=mock_body, # type: ignore + self.endpoint_api._validate_create_body + )( + self.endpoint_api, + body=mock_body, # type: ignore ) self.assertEqual( - ('mock_name', 'mock_type', None, 'mock_connection_info', []), - endpoint + ('mock_name', 'mock_type', None, 'mock_connection_info', []), endpoint ) def test__validate_create_body_all_keys(self): mock_body = { - 'endpoint': - { - 'name': 'mock_name', - 'type': 'mock_type', - 'description': 'mock_description', - 'connection_info': 'mock_connection_info', - 'mapped_regions': ['mapped_region_1', 'mapped_region_1'], - } + 'endpoint': { + 'name': 'mock_name', + 'type': 'mock_type', + 'description': 'mock_description', + 'connection_info': 'mock_connection_info', + 'mapped_regions': ['mapped_region_1', 'mapped_region_1'], + } } endpoint = testutils.get_wrapped_function( - self.endpoint_api._validate_create_body)( - self.endpoint_api, - body=mock_body, # type: ignore + self.endpoint_api._validate_create_body + )( + self.endpoint_api, + body=mock_body, # type: ignore ) self.assertEqual( - ('mock_name', 'mock_type', 'mock_description', - 'mock_connection_info', ['mapped_region_1', 'mapped_region_1']), - endpoint + ( + 'mock_name', + 'mock_type', + 'mock_description', + 'mock_connection_info', + ['mapped_region_1', 'mapped_region_1'], + ), + endpoint, ) def test__validate_create_body_no_endpoint(self): mock_body = {} self.assertRaises( KeyError, - testutils.get_wrapped_function( - self.endpoint_api._validate_create_body), + testutils.get_wrapped_function(self.endpoint_api._validate_create_body), self.endpoint_api, body=mock_body, # type: ignore ) @@ -141,12 +128,7 @@ def test__validate_create_body_no_endpoint(self): @mock.patch.object(endpoint_view, 'single') @mock.patch.object(api.API, 'create') @mock.patch.object(endpoints.EndpointController, '_validate_create_body') - def test_create( - self, - mock__validate_create_body, - mock_create, - mock_single - ): + def test_create(self, mock__validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -156,7 +138,7 @@ def test_create( mock.sentinel.endpoint_type, mock.sentinel.description, mock.sentinel.connection_info, - mock.sentinel.mapped_regions + mock.sentinel.mapped_regions, ) result = self.endpoint_api.create(mock_req, body) @@ -168,53 +150,42 @@ def test_create( mock.sentinel.endpoint_type, mock.sentinel.description, mock.sentinel.connection_info, - mock.sentinel.mapped_regions - ) - self.assertEqual( - mock_single.return_value, - result + mock.sentinel.mapped_regions, ) + self.assertEqual(mock_single.return_value, result) def test__validate_update_body(self): mock_body = { - 'endpoint': - { - 'name': 'mock_name', - 'description': 'mock_description', - 'connection_info': 'mock_connection_info', - 'mapped_regions': ['mapped_region_1', 'mapped_region_1'], - } + 'endpoint': { + 'name': 'mock_name', + 'description': 'mock_description', + 'connection_info': 'mock_connection_info', + 'mapped_regions': ['mapped_region_1', 'mapped_region_1'], + } } endpoint = testutils.get_wrapped_function( - self.endpoint_api._validate_update_body)( - self.endpoint_api, - body=mock_body, # type: ignore - ) - self.assertEqual( - mock_body['endpoint'], - endpoint + self.endpoint_api._validate_update_body + )( + self.endpoint_api, + body=mock_body, # type: ignore ) + self.assertEqual(mock_body['endpoint'], endpoint) def test__validate_update_body_no_keys(self): - mock_body = { - 'endpoint': {} - } + mock_body = {'endpoint': {}} endpoint = testutils.get_wrapped_function( - self.endpoint_api._validate_update_body)( - self.endpoint_api, - body=mock_body, # type: ignore - ) - self.assertEqual( - {}, - endpoint + self.endpoint_api._validate_update_body + )( + self.endpoint_api, + body=mock_body, # type: ignore ) + self.assertEqual({}, endpoint) def test__validate_update_body_no_endpoint(self): mock_body = {} self.assertRaises( KeyError, - testutils.get_wrapped_function( - self.endpoint_api._validate_update_body), + testutils.get_wrapped_function(self.endpoint_api._validate_update_body), self.endpoint_api, body=mock_body, # type: ignore ) @@ -222,12 +193,7 @@ def test__validate_update_body_no_endpoint(self): @mock.patch.object(endpoint_view, 'single') @mock.patch.object(api.API, 'update') @mock.patch.object(endpoints.EndpointController, '_validate_update_body') - def test_update( - self, - mock__validate_update_body, - mock_update, - mock_single - ): + def test_update(self, mock__validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -239,51 +205,31 @@ def test_update( mock_context.can.assert_called_once_with('migration:endpoints:update') mock__validate_update_body.assert_called_once_with(body) mock_update.assert_called_once_with( - mock_context, id, - mock__validate_update_body.return_value - ) - self.assertEqual( - mock_single.return_value, - result + mock_context, id, mock__validate_update_body.return_value ) + self.assertEqual(mock_single.return_value, result) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id - self.assertRaises( - exc.HTTPNoContent, - self.endpoint_api.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNoContent, self.endpoint_api.delete, mock_req, id) mock_context.can.assert_called_once_with('migration:endpoints:delete') mock_delete.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_delete.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.endpoint_api.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.endpoint_api.delete, mock_req, id) mock_context.can.assert_called_once_with('migration:endpoints:delete') mock_delete.assert_called_once_with(mock_context, id) diff --git a/coriolis/tests/api/v1/test_minion_pool_actions.py b/coriolis/tests/api/v1/test_minion_pool_actions.py index 7e1f86c6..431884b4 100644 --- a/coriolis/tests/api/v1/test_minion_pool_actions.py +++ b/coriolis/tests/api/v1/test_minion_pool_actions.py @@ -6,12 +6,11 @@ import ddt from webob import exc +from coriolis import exception from coriolis.api.v1 import minion_pool_actions as minion from coriolis.api.v1.views import minion_pool_view -from coriolis import exception from coriolis.minion_pools import api -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils @ddt.ddt @@ -24,34 +23,22 @@ def setUp(self): @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'allocate_minion_pool') - def test__allocate_pool( - self, - mock_allocate_minion_pool, - mock_single - ): + def test__allocate_pool(self, mock_allocate_minion_pool, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_body = {} - result = testutils.get_wrapped_function( - self.minion._allocate_pool)( - mock_req, - id, - mock_body + result = testutils.get_wrapped_function(self.minion._allocate_pool)( + mock_req, id, mock_body ) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:minion_pools:allocate") + mock_context.can.assert_called_once_with("migration:minion_pools:allocate") mock_allocate_minion_pool.assert_called_once_with(mock_context, id) - mock_single.assert_called_once_with( - mock_allocate_minion_pool.return_value) + mock_single.assert_called_once_with(mock_allocate_minion_pool.return_value) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'allocate_minion_pool') @@ -75,44 +62,31 @@ def test__allocate_pool_raises( testutils.get_wrapped_function(self.minion._allocate_pool), mock_req, id, - mock_body + mock_body, ) - mock_context.can.assert_called_once_with( - "migration:minion_pools:allocate") + mock_context.can.assert_called_once_with("migration:minion_pools:allocate") mock_allocate_minion_pool.assert_called_once_with(mock_context, id) mock_single.assert_not_called() @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'refresh_minion_pool') - def test__refresh_pool( - self, - mock_refresh_minion_pool, - mock_single - ): + def test__refresh_pool(self, mock_refresh_minion_pool, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_body = {} - result = testutils.get_wrapped_function( - self.minion._refresh_pool)( - mock_req, - id, - mock_body + result = testutils.get_wrapped_function(self.minion._refresh_pool)( + mock_req, id, mock_body ) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:minion_pools:refresh") + mock_context.can.assert_called_once_with("migration:minion_pools:refresh") mock_refresh_minion_pool.assert_called_once_with(mock_context, id) - mock_single.assert_called_once_with( - mock_refresh_minion_pool.return_value) + mock_single.assert_called_once_with(mock_refresh_minion_pool.return_value) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'refresh_minion_pool') @@ -136,45 +110,33 @@ def test__refresh_pool_raises( testutils.get_wrapped_function(self.minion._refresh_pool), mock_req, id, - mock_body + mock_body, ) - mock_context.can.assert_called_once_with( - "migration:minion_pools:refresh") + mock_context.can.assert_called_once_with("migration:minion_pools:refresh") mock_refresh_minion_pool.assert_called_once_with(mock_context, id) mock_single.assert_not_called() @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'deallocate_minion_pool') - def test__deallocate_pool( - self, - mock_deallocate_minion_pool, - mock_single - ): + def test__deallocate_pool(self, mock_deallocate_minion_pool, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_body = {'deallocate': {}} - result = testutils.get_wrapped_function( - self.minion._deallocate_pool)( - mock_req, - id, - mock_body + result = testutils.get_wrapped_function(self.minion._deallocate_pool)( + mock_req, id, mock_body ) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:minion_pools:deallocate") + mock_context.can.assert_called_once_with("migration:minion_pools:deallocate") mock_deallocate_minion_pool.assert_called_once_with( - mock_context, id, force=False) - mock_single.assert_called_once_with( - mock_deallocate_minion_pool.return_value) + mock_context, id, force=False + ) + mock_single.assert_called_once_with(mock_deallocate_minion_pool.return_value) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'deallocate_minion_pool') @@ -198,11 +160,11 @@ def test__deallocate_pool_raises( testutils.get_wrapped_function(self.minion._deallocate_pool), mock_req, id, - mock_body + mock_body, ) - mock_context.can.assert_called_once_with( - "migration:minion_pools:deallocate") + mock_context.can.assert_called_once_with("migration:minion_pools:deallocate") mock_deallocate_minion_pool.assert_called_once_with( - mock_context, id, force=False) + mock_context, id, force=False + ) mock_single.assert_not_called() diff --git a/coriolis/tests/api/v1/test_minion_pools.py b/coriolis/tests/api/v1/test_minion_pools.py index 96d1b545..2da04ea5 100644 --- a/coriolis/tests/api/v1/test_minion_pools.py +++ b/coriolis/tests/api/v1/test_minion_pools.py @@ -6,14 +6,12 @@ import ddt from webob import exc +from coriolis import constants, exception from coriolis.api.v1 import minion_pools from coriolis.api.v1.views import minion_pool_view -from coriolis import constants from coriolis.endpoints import api as endpoints_api -from coriolis import exception from coriolis.minion_pools import api -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils @ddt.ddt @@ -26,11 +24,7 @@ def setUp(self): @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'get_minion_pool') - def test_show( - self, - mock_get_minion_pool, - mock_single - ): + def test_show(self, mock_get_minion_pool, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -38,35 +32,22 @@ def test_show( result = self.minion_pools.show(mock_req, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:minion_pools:show") mock_get_minion_pool.assert_called_once_with(mock_context, id) - mock_single.assert_called_once_with( - mock_get_minion_pool.return_value) + mock_single.assert_called_once_with(mock_get_minion_pool.return_value) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'get_minion_pool') - def test_show_not_found( - self, - mock_get_minion_pool, - mock_single - ): + def test_show_not_found(self, mock_get_minion_pool, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_get_minion_pool.return_value = None - self.assertRaises( - exc.HTTPNotFound, - self.minion_pools.show, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.minion_pools.show, mock_req, id) mock_context.can.assert_called_once_with("migration:minion_pools:show") mock_get_minion_pool.assert_called_once_with(mock_context, id) @@ -74,26 +55,18 @@ def test_show_not_found( @mock.patch.object(minion_pool_view, 'collection') @mock.patch.object(api.API, 'get_minion_pools') - def test_index( - self, - mock_get_minion_pools, - mock_collection - ): + def test_index(self, mock_get_minion_pools, mock_collection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} result = self.minion_pools.index(mock_req) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) mock_context.can.assert_called_once_with("migration:minion_pools:list") mock_get_minion_pools.assert_called_once_with(mock_context) - mock_collection.assert_called_once_with( - mock_get_minion_pools.return_value) + mock_collection.assert_called_once_with(mock_get_minion_pools.return_value) @ddt.file_data('data/minion_pools_retention_strategy.yml') def test__check_pool_retention_strategy( @@ -104,25 +77,19 @@ def test__check_pool_retention_strategy( if exception_raised: self.assertRaisesRegex( Exception, - "Invalid minion pool retention strategy '%s'" - % pool_retention_strategy, + "Invalid minion pool retention strategy '%s'" % pool_retention_strategy, self.minion_pools._check_pool_retention_strategy, - pool_retention_strategy + pool_retention_strategy, ) else: strategy = getattr(constants, pool_retention_strategy) self.assertEqual( - None, - self.minion_pools._check_pool_retention_strategy(strategy) + None, self.minion_pools._check_pool_retention_strategy(strategy) ) @ddt.file_data('data/minion_pools_numeric_values.yml') - def test__check_pool_numeric_values( - self, - config, - exception_raised - ): + def test__check_pool_numeric_values(self, config, exception_raised): minimum_minions = config.get("minimum_minions", None) maximum_minions = config.get("maximum_minions", None) minion_max_idle_time = config.get("minion_max_idle_time", None) @@ -133,18 +100,20 @@ def test__check_pool_numeric_values( self.minion_pools._check_pool_numeric_values, minimum_minions, maximum_minions, - minion_max_idle_time + minion_max_idle_time, ) else: self.assertEqual( self.minion_pools._check_pool_numeric_values( - minimum_minions, maximum_minions, minion_max_idle_time), - None) + minimum_minions, maximum_minions, minion_max_idle_time + ), + None, + ) - @mock.patch.object(minion_pools.MinionPoolController, - '_check_pool_retention_strategy') - @mock.patch.object(minion_pools.MinionPoolController, - '_check_pool_numeric_values') + @mock.patch.object( + minion_pools.MinionPoolController, '_check_pool_retention_strategy' + ) + @mock.patch.object(minion_pools.MinionPoolController, '_check_pool_numeric_values') @ddt.file_data('data/minion_pools_validate_create_body.yml') def test__validate_create_body( self, @@ -152,25 +121,25 @@ def test__validate_create_body( mock__check_pool_retention_strategy, config, exception_raised, - expected_result + expected_result, ): expected_validation_api_method = config.get( - "expected_validation_api_method", None) + "expected_validation_api_method", None + ) ctxt = {} body = config["body"] minion_pool = body["minion_pool"] - minion_pool["os_type"] = getattr( - constants, minion_pool["os_type"], None) - minion_pool["platform"] = getattr( - constants, minion_pool["platform"], None) + minion_pool["os_type"] = getattr(constants, minion_pool["os_type"], None) + minion_pool["platform"] = getattr(constants, minion_pool["platform"], None) if 'minion_retention_strategy' in minion_pool: - minion_pool["minion_retention_strategy"] = \ - getattr( - constants, minion_pool["minion_retention_strategy"], - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE) + minion_pool["minion_retention_strategy"] = getattr( + constants, + minion_pool["minion_retention_strategy"], + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE, + ) minion_retention_strategy = minion_pool.get( 'minion_retention_strategy', - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE, ) minimum_minions = minion_pool.get('minimum_minions', 1) maximum_minions = minion_pool.get('maximum_minions', 1) @@ -180,50 +149,39 @@ def test__validate_create_body( self.assertRaisesRegex( Exception, exception_raised, - testutils.get_wrapped_function( - self.minion_pools._validate_create_body), + testutils.get_wrapped_function(self.minion_pools._validate_create_body), self.minion_pools, ctxt, - body + body, ) else: with mock.patch.object( - endpoints_api.API, - expected_validation_api_method) as mock_validation_api_method: - + endpoints_api.API, expected_validation_api_method + ) as mock_validation_api_method: result = testutils.get_wrapped_function( - self.minion_pools._validate_create_body)( - self.minion_pools, - ctxt, - body, + self.minion_pools._validate_create_body + )( + self.minion_pools, + ctxt, + body, ) - self.assertEqual( - tuple(expected_result), - result - ) + self.assertEqual(tuple(expected_result), result) mock_validation_api_method.assert_called_once_with( - ctxt, - minion_pool["endpoint_id"], - minion_pool["environment_options"] + ctxt, minion_pool["endpoint_id"], minion_pool["environment_options"] ) mock__check_pool_numeric_values.assert_called_once_with( - minimum_minions, maximum_minions, minion_max_idle_time) + minimum_minions, maximum_minions, minion_max_idle_time + ) mock__check_pool_retention_strategy.assert_called_once_with( minion_retention_strategy ) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'create') - @mock.patch.object(minion_pools.MinionPoolController, - '_validate_create_body') - def test_create( - self, - mock_validate_create_body, - mock_create, - mock_single - ): + @mock.patch.object(minion_pools.MinionPoolController, '_validate_create_body') + def test_create(self, mock_validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -232,31 +190,20 @@ def test_create( result = self.minion_pools.create(mock_req, mock_body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:minion_pools:create") - mock_validate_create_body.assert_called_once_with( - mock_context, mock_body) + mock_context.can.assert_called_once_with("migration:minion_pools:create") + mock_validate_create_body.assert_called_once_with(mock_context, mock_body) mock_create.assert_called_once() mock_single.assert_called_once_with(mock_create.return_value) @ddt.file_data('data/minion_pools_validate_enviroment_options.yml') def test__validate_updated_environment_options( - self, - platform, - expected_api_method, - exception_raised + self, platform, expected_api_method, exception_raised ): platform = getattr(constants, platform, None) endpoint_id = mock.sentinel.endpoint_id - minion_pool = { - 'platform': platform, - 'endpoint_id': endpoint_id - } + minion_pool = {'platform': platform, 'endpoint_id': endpoint_id} enviroment_options = mock.sentinel.enviroment_options context = mock.sentinel.context @@ -267,27 +214,28 @@ def test__validate_updated_environment_options( self.minion_pools._validate_updated_environment_options, context, minion_pool, - enviroment_options + enviroment_options, ) else: - with mock.patch.object(endpoints_api.API, - expected_api_method) as mock_api_method: + with mock.patch.object( + endpoints_api.API, expected_api_method + ) as mock_api_method: self.minion_pools._validate_updated_environment_options( - context, - minion_pool, - enviroment_options + context, minion_pool, enviroment_options ) mock_api_method.assert_called_once_with( - context, endpoint_id, enviroment_options) + context, endpoint_id, enviroment_options + ) - @mock.patch.object(minion_pools.MinionPoolController, - '_validate_updated_environment_options') + @mock.patch.object( + minion_pools.MinionPoolController, '_validate_updated_environment_options' + ) @mock.patch.object(api.API, 'get_minion_pool') - @mock.patch.object(minion_pools.MinionPoolController, - '_check_pool_retention_strategy') - @mock.patch.object(minion_pools.MinionPoolController, - '_check_pool_numeric_values') + @mock.patch.object( + minion_pools.MinionPoolController, '_check_pool_retention_strategy' + ) + @mock.patch.object(minion_pools.MinionPoolController, '_check_pool_numeric_values') @ddt.file_data('data/minion_pools_validate_update_body.yml') def test__validate_update_body( self, @@ -297,77 +245,60 @@ def test__validate_update_body( mock_validate_updated_environment_options, config, exception_raised, - expected_result + expected_result, ): body = config["body"] minion_pool = body["minion_pool"] environment_options = minion_pool.get('environment_options', {}) - minion_retention_strategy = minion_pool.get( - 'minion_retention_strategy', "") + minion_retention_strategy = minion_pool.get('minion_retention_strategy', "") validate_get_minion_pool = config.get("validate_get_minion_pool", None) minimum_minions = minion_pool.get('minimum_minions', 1) maximum_minions = minion_pool.get('maximum_minions', 1) minion_max_idle_time = minion_pool.get('minion_max_idle_time', 1) mock_context = mock.Mock() id = mock.sentinel.id - mock_get_minion_pool.return_value = { - 'minimum_minions': 1, - 'maximum_minions': 1 - } + mock_get_minion_pool.return_value = {'minimum_minions': 1, 'maximum_minions': 1} if exception_raised: self.assertRaisesRegex( Exception, exception_raised, - testutils.get_wrapped_function( - self.minion_pools._validate_update_body), + testutils.get_wrapped_function(self.minion_pools._validate_update_body), self.minion_pools, id, mock_context, - body + body, ) else: result = testutils.get_wrapped_function( - self.minion_pools._validate_update_body)( - self.minion_pools, - id, - mock_context, - body, + self.minion_pools._validate_update_body + )( + self.minion_pools, + id, + mock_context, + body, ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) if minion_retention_strategy: mock_check_pool_retention_strategy.assert_called_once_with( minion_retention_strategy ) if validate_get_minion_pool: - mock_get_minion_pool.assert_called_once_with( - mock_context, id - ) + mock_get_minion_pool.assert_called_once_with(mock_context, id) mock_check_pool_numeric_values.assert_called_once_with( minimum_minions, maximum_minions, minion_max_idle_time ) if environment_options: - (mock_validate_updated_environment_options. - assert_called_once_with)( - mock_context, mock_get_minion_pool.return_value, - environment_options + (mock_validate_updated_environment_options.assert_called_once_with)( + mock_context, mock_get_minion_pool.return_value, environment_options ) @mock.patch.object(minion_pool_view, 'single') @mock.patch.object(api.API, 'update') - @mock.patch.object(minion_pools.MinionPoolController, - '_validate_update_body') - def test_update( - self, - mock_validate_update_body, - mock_update, - mock_single - ): + @mock.patch.object(minion_pools.MinionPoolController, '_validate_update_body') + def test_update(self, mock_validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -376,55 +307,34 @@ def test_update( result = self.minion_pools.update(mock_req, id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:minion_pools:update") - mock_validate_update_body.assert_called_once_with( - id, mock_context, body) + mock_context.can.assert_called_once_with("migration:minion_pools:update") + mock_validate_update_body.assert_called_once_with(id, mock_context, body) mock_update.assert_called_once_with( - mock_context, id, mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id - self.assertRaises( - exc.HTTPNoContent, - self.minion_pools.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNoContent, self.minion_pools.delete, mock_req, id) mock_delete.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_delete.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.minion_pools.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.minion_pools.delete, mock_req, id) - mock_context.can.assert_called_once_with( - "migration:minion_pools:delete") + mock_context.can.assert_called_once_with("migration:minion_pools:delete") mock_delete.assert_called_once_with(mock_context, id) diff --git a/coriolis/tests/api/v1/test_provider_schemas.py b/coriolis/tests/api/v1/test_provider_schemas.py index 4389c084..857f7e10 100644 --- a/coriolis/tests/api/v1/test_provider_schemas.py +++ b/coriolis/tests/api/v1/test_provider_schemas.py @@ -16,26 +16,20 @@ def setUp(self): self.provider_schemas = provider_schemas.ProviderSchemasController() @mock.patch.object(api.API, 'get_provider_schemas') - def test_index( - self, - get_provider_schemas - ): + def test_index(self, get_provider_schemas): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} mock_platform_name = "mock_platform_name" mock_provider_type = "mock_provider_type" - expected_result = { - 'schemas': get_provider_schemas.return_value - } + expected_result = {'schemas': get_provider_schemas.return_value} result = self.provider_schemas.index( - mock_req, mock_platform_name, mock_provider_type) - - self.assertEqual( - expected_result, - result + mock_req, mock_platform_name, mock_provider_type ) + self.assertEqual(expected_result, result) + get_provider_schemas.assert_called_once_with( - mock_context, mock_platform_name, mock_provider_type) + mock_context, mock_platform_name, mock_provider_type + ) diff --git a/coriolis/tests/api/v1/test_providers.py b/coriolis/tests/api/v1/test_providers.py index 207d4239..293b56ba 100644 --- a/coriolis/tests/api/v1/test_providers.py +++ b/coriolis/tests/api/v1/test_providers.py @@ -16,23 +16,15 @@ def setUp(self): self.providers = providers.ProviderController() @mock.patch.object(api.API, 'get_available_providers') - def test_index( - self, - mock_get_available_providers - ): + def test_index(self, mock_get_available_providers): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} - expected_result = { - 'providers': mock_get_available_providers.return_value - } + expected_result = {'providers': mock_get_available_providers.return_value} result = self.providers.index(mock_req) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_context.can.assert_called_once_with("migration:providers:list") mock_get_available_providers.assert_called_once_with(mock_context) diff --git a/coriolis/tests/api/v1/test_regions.py b/coriolis/tests/api/v1/test_regions.py index 89dacdbf..84120629 100644 --- a/coriolis/tests/api/v1/test_regions.py +++ b/coriolis/tests/api/v1/test_regions.py @@ -5,12 +5,11 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import regions from coriolis.api.v1.views import region_view -from coriolis import exception from coriolis.regions import api -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class RegionControllerTestCase(test_base.CoriolisBaseTestCase): @@ -22,11 +21,7 @@ def setUp(self): @mock.patch.object(region_view, 'single') @mock.patch.object(api.API, 'get_region') - def test_show( - self, - mock_get_region, - mock_single - ): + def test_show(self, mock_get_region, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -34,53 +29,35 @@ def test_show( result = self.regions.show(mock_req, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:regions:show") mock_get_region.assert_called_once_with(mock_context, id) mock_single.assert_called_once_with(mock_get_region.return_value) @mock.patch.object(api.API, 'get_region') - def test_show_not_found( - self, - mock_get_region - ): + def test_show_not_found(self, mock_get_region): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_get_region.return_value = None - self.assertRaises( - exc.HTTPNotFound, - self.regions.show, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.regions.show, mock_req, id) mock_context.can.assert_called_once_with("migration:regions:show") mock_get_region.assert_called_once_with(mock_context, id) @mock.patch.object(region_view, 'collection') @mock.patch.object(api.API, 'get_regions') - def test_index( - self, - mock_get_regions, - mock_collection - ): + def test_index(self, mock_get_regions, mock_collection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} result = self.regions.index(mock_req) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) mock_context.can.assert_called_once_with("migration:regions:list") mock_get_regions.assert_called_once_with(mock_context) @@ -93,30 +70,19 @@ def test_validate_create_body( description = mock.sentinel.description enabled = False body = { - "region": { - "name": name, - "description": description, - "enabled": enabled - } + "region": {"name": name, "description": description, "enabled": enabled} } - result = testutils.get_wrapped_function( - self.regions._validate_create_body)(self.regions, body=body) - - self.assertEqual( - (name, description, enabled), - result + result = testutils.get_wrapped_function(self.regions._validate_create_body)( + self.regions, body=body ) + self.assertEqual((name, description, enabled), result) + @mock.patch.object(region_view, 'single') @mock.patch.object(api.API, 'create') @mock.patch.object(regions.RegionController, '_validate_create_body') - def test_create( - self, - mock_validate_create_body, - mock_create, - mock_single - ): + def test_create(self, mock_validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -124,25 +90,18 @@ def test_create( description = mock.sentinel.description enabled = True mock_validate_create_body.return_value = (name, description, enabled) - region = { - "name": name, - "description": description, - "enabled": enabled - } + region = {"name": name, "description": description, "enabled": enabled} body = {"region": region} result = self.regions.create(mock_req, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:regions:create") mock_validate_create_body.assert_called_once_with(body) mock_create.assert_called_once_with( - mock_context, region_name=name, description=description, - enabled=enabled) + mock_context, region_name=name, description=description, enabled=enabled + ) mock_single.assert_called_once_with(mock_create.return_value) def test_validate_update_body( @@ -155,7 +114,7 @@ def test_validate_update_body( "name": name, "description": description, "enabled": enabled, - "mock_key": "mock_value" + "mock_key": "mock_value", } expected_result = { "name": name, @@ -164,23 +123,16 @@ def test_validate_update_body( } body = {"region": region} - result = testutils.get_wrapped_function( - self.regions._validate_update_body)(self.regions, body=body) - - self.assertEqual( - expected_result, - result + result = testutils.get_wrapped_function(self.regions._validate_update_body)( + self.regions, body=body ) + self.assertEqual(expected_result, result) + @mock.patch.object(region_view, 'single') @mock.patch.object(api.API, 'update') @mock.patch.object(regions.RegionController, '_validate_update_body') - def test_update( - self, - mock_validate_update_body, - mock_update, - mock_single - ): + def test_update(self, mock_validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -189,55 +141,36 @@ def test_update( result = self.regions.update(mock_req, id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:regions:update") mock_validate_update_body.assert_called_once_with(body) mock_update.assert_called_once_with( - mock_context, id, - mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) mock_single.assert_called_once_with(mock_update.return_value) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id - self.assertRaises( - exc.HTTPNoContent, - self.regions.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNoContent, self.regions.delete, mock_req, id) mock_context.can.assert_called_once_with("migration:regions:delete") mock_delete.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_delete.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.regions.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.regions.delete, mock_req, id) mock_context.can.assert_called_once_with("migration:regions:delete") mock_delete.assert_called_once_with(mock_context, id) diff --git a/coriolis/tests/api/v1/test_router.py b/coriolis/tests/api/v1/test_router.py index 3dc14549..4a8a9128 100644 --- a/coriolis/tests/api/v1/test_router.py +++ b/coriolis/tests/api/v1/test_router.py @@ -3,30 +3,32 @@ from unittest import mock -from coriolis.api.v1 import deployment_actions -from coriolis.api.v1 import deployments -from coriolis.api.v1 import diagnostics -from coriolis.api.v1 import endpoint_actions -from coriolis.api.v1 import endpoint_destination_minion_pool_options -from coriolis.api.v1 import endpoint_destination_options -from coriolis.api.v1 import endpoint_instances -from coriolis.api.v1 import endpoint_networks -from coriolis.api.v1 import endpoint_source_minion_pool_options -from coriolis.api.v1 import endpoint_source_options -from coriolis.api.v1 import endpoint_storage -from coriolis.api.v1 import endpoints -from coriolis.api.v1 import minion_pool_actions -from coriolis.api.v1 import minion_pools -from coriolis.api.v1 import provider_schemas -from coriolis.api.v1 import providers -from coriolis.api.v1 import regions -from coriolis.api.v1 import router -from coriolis.api.v1 import services -from coriolis.api.v1 import transfer_actions -from coriolis.api.v1 import transfer_schedules -from coriolis.api.v1 import transfer_tasks_execution_actions -from coriolis.api.v1 import transfer_tasks_executions -from coriolis.api.v1 import transfers +from coriolis.api.v1 import ( + deployment_actions, + deployments, + diagnostics, + endpoint_actions, + endpoint_destination_minion_pool_options, + endpoint_destination_options, + endpoint_instances, + endpoint_networks, + endpoint_source_minion_pool_options, + endpoint_source_options, + endpoint_storage, + endpoints, + minion_pool_actions, + minion_pools, + provider_schemas, + providers, + regions, + router, + services, + transfer_actions, + transfer_schedules, + transfer_tasks_execution_actions, + transfer_tasks_executions, + transfers, +) from coriolis.tests import test_base @@ -52,8 +54,7 @@ def setUp(self): @mock.patch.object(endpoint_networks, 'create_resource') @mock.patch.object(endpoint_instances, 'create_resource') @mock.patch.object(endpoint_actions, 'create_resource') - @mock.patch.object(endpoint_destination_minion_pool_options, - 'create_resource') + @mock.patch.object(endpoint_destination_minion_pool_options, 'create_resource') @mock.patch.object(endpoint_source_minion_pool_options, 'create_resource') @mock.patch.object(minion_pool_actions, 'create_resource') @mock.patch.object(minion_pools, 'create_resource') @@ -85,75 +86,76 @@ def test_setup_routes( mock_transfer_schedules_create_resource, mock_diagnostics_create_resource, mock_deployment_actions_create_resource, - mock_deployments_create_resource + mock_deployments_create_resource, ): ext_mgr = mock.sentinel.ext_mgr mapper = mock.Mock() resource_calls = [ mock.call( - 'provider', 'providers', - controller=mock_providers_create_resource.return_value + 'provider', + 'providers', + controller=mock_providers_create_resource.return_value, ), mock.call( - 'region', 'regions', + 'region', + 'regions', controller=mock_regions_create_resource.return_value, - collection={'detail': 'GET'} + collection={'detail': 'GET'}, ), mock.call( - 'endpoint', 'endpoints', + 'endpoint', + 'endpoints', controller=mock_endpoints_create_resource.return_value, collection={'detail': 'GET'}, - member={'action': 'POST'} + member={'action': 'POST'}, ), mock.call( - 'service', 'services', + 'service', + 'services', controller=mock_services_create_resource.return_value, - collection={'detail': 'GET'} + collection={'detail': 'GET'}, ), mock.call( - 'minion_pool', 'minion_pools', + 'minion_pool', + 'minion_pools', controller=mock_minion_pools_create_resource.return_value, - collection={'detail': 'GET'} + collection={'detail': 'GET'}, ), mock.call( 'minion_pool_options', 'endpoints/{endpoint_id}/source-minion-pool-options', - controller= - mock_endpoint_source_minion_pool_options_create_resource. - return_value, + controller=mock_endpoint_source_minion_pool_options_create_resource.return_value, ), mock.call( 'minion_pool_options', 'endpoints/{endpoint_id}/destination-minion-pool-options', - controller= - mock_endpoint_destination_minion_pool_options_create_resource. - return_value, + controller=mock_endpoint_destination_minion_pool_options_create_resource.return_value, ), mock.call( - 'instance', 'endpoints/{endpoint_id}/instances', - controller= - mock_endpoint_instances_create_resource.return_value, + 'instance', + 'endpoints/{endpoint_id}/instances', + controller=mock_endpoint_instances_create_resource.return_value, ), mock.call( - 'network', 'endpoints/{endpoint_id}/networks', + 'network', + 'endpoints/{endpoint_id}/networks', controller=mock_endpoint_networks_create_resource.return_value, ), mock.call( - 'storage', 'endpoints/{endpoint_id}/storage', + 'storage', + 'endpoints/{endpoint_id}/storage', controller=mock_endpoint_storage_create_resource.return_value, ), mock.call( 'destination_options', 'endpoints/{endpoint_id}/destination-options', - controller= - mock_endpoint_destination_options_create_resource.return_value, + controller=mock_endpoint_destination_options_create_resource.return_value, ), mock.call( 'source_options', 'endpoints/{endpoint_id}/source-options', - controller= - mock_endpoint_source_options_create_resource.return_value, + controller=mock_endpoint_source_options_create_resource.return_value, ), mock.call( 'provider_schemas', @@ -161,36 +163,37 @@ def test_setup_routes( controller=mock_provider_schemas_create_resource.return_value, ), mock.call( - 'transfer', 'transfers', + 'transfer', + 'transfers', controller=mock_transfers_create_resource.return_value, collection={'detail': 'GET'}, - member={'action': 'POST'} + member={'action': 'POST'}, ), mock.call( 'execution', 'transfers/{transfer_id}/executions', - controller= - mock_transfer_tasks_executions_create_resource.return_value, + controller=mock_transfer_tasks_executions_create_resource.return_value, collection={'detail': 'GET'}, - member={'action': 'POST'} + member={'action': 'POST'}, ), mock.call( 'transfer_schedule', 'transfers/{transfer_id}/schedules', - controller= - mock_transfer_schedules_create_resource.return_value, + controller=mock_transfer_schedules_create_resource.return_value, collection={'index': 'GET'}, - member={'action': 'POST'} + member={'action': 'POST'}, ), mock.call( - 'diagnostics', 'diagnostics', + 'diagnostics', + 'diagnostics', controller=mock_diagnostics_create_resource.return_value, ), mock.call( - 'deployment', 'deployments', + 'deployment', + 'deployments', controller=mock_deployments_create_resource.return_value, collection={'detail': 'GET'}, - member={'action': 'POST'} + member={'action': 'POST'}, ), ] @@ -198,42 +201,37 @@ def test_setup_routes( mock.call( 'minion_pool_actions', '/{project_id}/minion_pools/{id}/actions', - controller= - mock_minion_pool_actions_create_resource.return_value, + controller=mock_minion_pool_actions_create_resource.return_value, action='action', - conditions={'method': 'POST'} + conditions={'method': 'POST'}, ), mock.call( 'endpoint_actions', '/{project_id}/endpoints/{id}/actions', - controller= - mock_endpoint_actions_create_resource.return_value, + controller=mock_endpoint_actions_create_resource.return_value, action='action', - conditions={'method': 'POST'} + conditions={'method': 'POST'}, ), mock.call( 'transfer_actions', '/{project_id}/transfers/{id}/actions', controller=mock_transfer_actions_create_resource.return_value, action='action', - conditions={'method': 'POST'} + conditions={'method': 'POST'}, ), mock.call( 'transfer_tasks_execution_actions', - '/{project_id}/transfers/{transfer_id}/' - 'executions/{id}/actions', - controller= - mock_transfer_tasks_execution_actions_create_resource. - return_value, + '/{project_id}/transfers/{transfer_id}/executions/{id}/actions', + controller=mock_transfer_tasks_execution_actions_create_resource.return_value, action='action', - conditions={'method': 'POST'} + conditions={'method': 'POST'}, ), mock.call( - 'deployment_actions', '/{project_id}/deployments/{id}/actions', - controller=( - mock_deployment_actions_create_resource.return_value), + 'deployment_actions', + '/{project_id}/deployments/{id}/actions', + controller=(mock_deployment_actions_create_resource.return_value), action='action', - conditions={"method": "POST"} + conditions={"method": "POST"}, ), ] diff --git a/coriolis/tests/api/v1/test_services.py b/coriolis/tests/api/v1/test_services.py index 0dceaba7..54fe0f78 100644 --- a/coriolis/tests/api/v1/test_services.py +++ b/coriolis/tests/api/v1/test_services.py @@ -5,12 +5,11 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import services from coriolis.api.v1.views import service_view -from coriolis import exception from coriolis.services import api -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class ServiceControllerTestCase(test_base.CoriolisBaseTestCase): @@ -34,14 +33,10 @@ def test_show( result = self.services.show(mock_req, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:services:show") - mock_get_service.assert_called_once_with( - mock_context, id) + mock_get_service.assert_called_once_with(mock_context, id) mock_single.assert_called_once_with(mock_get_service.return_value) @mock.patch.object(service_view, 'single') @@ -57,35 +52,22 @@ def test_show_no_service( id = mock.sentinel.id mock_get_service.return_value = None - self.assertRaises( - exc.HTTPNotFound, - self.services.show, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.services.show, mock_req, id) mock_context.can.assert_called_once_with("migration:services:show") - mock_get_service.assert_called_once_with( - mock_context, id) + mock_get_service.assert_called_once_with(mock_context, id) mock_single.assert_not_called() @mock.patch.object(service_view, 'collection') @mock.patch.object(api.API, 'get_services') - def test_index( - self, - mock_get_services, - mock_collection - ): + def test_index(self, mock_get_services, mock_collection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} result = self.services.index(mock_req) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) mock_context.can.assert_called_once_with("migration:services:list") mock_get_services.assert_called_once_with(mock_context) @@ -105,27 +87,20 @@ def test_validate_create_body( "binary": binary, "topic": topic, "mapped_regions": mapped_regions, - "enabled": enabled + "enabled": enabled, } } - result = testutils.get_wrapped_function( - self.services._validate_create_body)(self.services, body=body) - - self.assertEqual( - (host, binary, topic, mapped_regions, enabled), - result + result = testutils.get_wrapped_function(self.services._validate_create_body)( + self.services, body=body ) + self.assertEqual((host, binary, topic, mapped_regions, enabled), result) + @mock.patch.object(service_view, 'single') @mock.patch.object(api.API, 'create') @mock.patch.object(services.ServiceController, '_validate_create_body') - def test_create( - self, - mock_validate_create_body, - mock_create, - mock_single - ): + def test_create(self, mock_validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -135,33 +110,38 @@ def test_create( mapped_regions = ["region_1", "region_2"] enabled = True mock_validate_create_body.return_value = ( - host, binary, topic, mapped_regions, enabled) + host, + binary, + topic, + mapped_regions, + enabled, + ) service = { "host": host, "binary": binary, "topic": topic, "mapped_regions": mapped_regions, - "enabled": enabled + "enabled": enabled, } body = {"service": service} result = self.services.create(mock_req, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:services:create") mock_validate_create_body.assert_called_once_with(body) mock_create.assert_called_once_with( - mock_context, host=host, binary=binary, topic=topic, - mapped_regions=mapped_regions, enabled=enabled) + mock_context, + host=host, + binary=binary, + topic=topic, + mapped_regions=mapped_regions, + enabled=enabled, + ) mock_single.assert_called_once_with(mock_create.return_value) - def test_validate_update_body( - self - ): + def test_validate_update_body(self): host = mock.sentinel.host binary = mock.sentinel.binary topic = mock.sentinel.topic @@ -172,41 +152,30 @@ def test_validate_update_body( "binary": binary, "topic": topic, "mapped_regions": mapped_regions, - "enabled": enabled + "enabled": enabled, } body = {"service": service} - result = testutils.get_wrapped_function( - self.services._validate_update_body)(self.services, body=body) - - self.assertEqual( - {'enabled': enabled, 'mapped_regions': mapped_regions}, - result + result = testutils.get_wrapped_function(self.services._validate_update_body)( + self.services, body=body ) - def test_validate_update_body_no_keys( - self - ): + self.assertEqual({'enabled': enabled, 'mapped_regions': mapped_regions}, result) + + def test_validate_update_body_no_keys(self): service = {} body = {"service": service} - result = testutils.get_wrapped_function( - self.services._validate_update_body)(self.services, body=body) - - self.assertEqual( - {}, - result + result = testutils.get_wrapped_function(self.services._validate_update_body)( + self.services, body=body ) + self.assertEqual({}, result) + @mock.patch.object(service_view, 'single') @mock.patch.object(api.API, 'update') @mock.patch.object(services.ServiceController, '_validate_update_body') - def test_update( - self, - mock_validate_update_body, - mock_update, - mock_single - ): + def test_update(self, mock_validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -215,55 +184,36 @@ def test_update( result = self.services.update(mock_req, id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:services:update") mock_validate_update_body.assert_called_once_with(body) mock_update.assert_called_once_with( - mock_context, id, - mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) mock_single.assert_called_once_with(mock_update.return_value) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id - self.assertRaises( - exc.HTTPNoContent, - self.services.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNoContent, self.services.delete, mock_req, id) mock_context.can.assert_called_once_with("migration:services:delete") mock_delete.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_delete.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.services.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.services.delete, mock_req, id) mock_context.can.assert_called_once_with("migration:services:delete") mock_delete.assert_called_once_with(mock_context, id) diff --git a/coriolis/tests/api/v1/test_transfer_actions.py b/coriolis/tests/api/v1/test_transfer_actions.py index bbfc4aaa..90ce44be 100644 --- a/coriolis/tests/api/v1/test_transfer_actions.py +++ b/coriolis/tests/api/v1/test_transfer_actions.py @@ -5,11 +5,10 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import transfer_actions from coriolis.api.v1.views import transfer_tasks_execution_view -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils from coriolis.transfers import api @@ -22,41 +21,26 @@ def setUp(self): @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'delete_disks') - def test_delete_disks( - self, - mock_delete_disks, - mock_single - ): + def test_delete_disks(self, mock_delete_disks, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id body = mock.sentinel.body - result = testutils.get_wrapped_function( - self.transfer_actions._delete_disks)( - mock_req, - id, - body + result = testutils.get_wrapped_function(self.transfer_actions._delete_disks)( + mock_req, id, body ) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfers:delete_disks") + mock_context.can.assert_called_once_with("migration:transfers:delete_disks") mock_delete_disks.assert_called_once_with(mock_context, id) mock_single.assert_called_once_with(mock_delete_disks.return_value) @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'delete_disks') - def test_delete_disks_not_found( - self, - mock_delete_disks, - mock_single - ): + def test_delete_disks_not_found(self, mock_delete_disks, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -66,25 +50,19 @@ def test_delete_disks_not_found( self.assertRaises( exc.HTTPNotFound, - testutils.get_wrapped_function( - self.transfer_actions._delete_disks), + testutils.get_wrapped_function(self.transfer_actions._delete_disks), req=mock_req, id=id, - body=body + body=body, ) - mock_context.can.assert_called_once_with( - "migration:transfers:delete_disks") + mock_context.can.assert_called_once_with("migration:transfers:delete_disks") mock_delete_disks.assert_called_once_with(mock_context, id) mock_single.assert_not_called() @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'delete_disks') - def test_delete_disks_invalid_parameter_value( - self, - mock_delete_disks, - mock_single - ): + def test_delete_disks_invalid_parameter_value(self, mock_delete_disks, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -94,14 +72,12 @@ def test_delete_disks_invalid_parameter_value( self.assertRaises( exc.HTTPNotFound, - testutils.get_wrapped_function( - self.transfer_actions._delete_disks), + testutils.get_wrapped_function(self.transfer_actions._delete_disks), req=mock_req, id=id, - body=body + body=body, ) - mock_context.can.assert_called_once_with( - "migration:transfers:delete_disks") + mock_context.can.assert_called_once_with("migration:transfers:delete_disks") mock_delete_disks.assert_called_once_with(mock_context, id) mock_single.assert_not_called() diff --git a/coriolis/tests/api/v1/test_transfer_schedules.py b/coriolis/tests/api/v1/test_transfer_schedules.py index 771ea400..7a7ce410 100644 --- a/coriolis/tests/api/v1/test_transfer_schedules.py +++ b/coriolis/tests/api/v1/test_transfer_schedules.py @@ -1,16 +1,15 @@ # Copyright 2023 Cloudbase Solutions Srl # All Rights Reserved. +import datetime from unittest import mock -import datetime import jsonschema from webob import exc +from coriolis import exception, schemas from coriolis.api.v1 import transfer_schedules from coriolis.api.v1.views import transfer_schedule_view -from coriolis import exception -from coriolis import schemas from coriolis.tests import test_base from coriolis.transfer_cron import api @@ -20,16 +19,11 @@ class TransferScheduleControllerTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(TransferScheduleControllerTestCase, self).setUp() - self.transfer_schedules = ( - transfer_schedules.TransferScheduleController()) + self.transfer_schedules = transfer_schedules.TransferScheduleController() @mock.patch.object(transfer_schedule_view, 'single') @mock.patch.object(api.API, 'get_schedule') - def test_show( - self, - mock_get_schedule, - mock_single - ): + def test_show(self, mock_get_schedule, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -38,24 +32,15 @@ def test_show( result = self.transfer_schedules.show(mock_req, transfer_id, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:show") - mock_get_schedule.assert_called_once_with( - mock_context, transfer_id, id) + mock_context.can.assert_called_once_with("migration:transfer_schedules:show") + mock_get_schedule.assert_called_once_with(mock_context, transfer_id, id) mock_single.assert_called_once_with(mock_get_schedule.return_value) @mock.patch.object(transfer_schedule_view, 'single') @mock.patch.object(api.API, 'get_schedule') - def test_show_not_found( - self, - mock_get_schedule, - mock_single - ): + def test_show_not_found(self, mock_get_schedule, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -64,26 +49,16 @@ def test_show_not_found( mock_get_schedule.return_value = None self.assertRaises( - exc.HTTPNotFound, - self.transfer_schedules.show, - mock_req, - transfer_id, - id + exc.HTTPNotFound, self.transfer_schedules.show, mock_req, transfer_id, id ) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:show") - mock_get_schedule.assert_called_once_with( - mock_context, transfer_id, id) + mock_context.can.assert_called_once_with("migration:transfer_schedules:show") + mock_get_schedule.assert_called_once_with(mock_context, transfer_id, id) mock_single.assert_not_called() @mock.patch.object(transfer_schedule_view, 'collection') @mock.patch.object(api.API, 'get_schedules') - def test_index( - self, - mock_get_schedules, - mock_collection - ): + def test_index(self, mock_get_schedules, mock_collection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -91,89 +66,66 @@ def test_index( mock_req.GET = {"show_expired": "False"} result = self.transfer_schedules.index(mock_req, transfer_id) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:list") + mock_context.can.assert_called_once_with("migration:transfer_schedules:list") mock_get_schedules.assert_called_once_with( - mock_context, - transfer_id, - expired=False + mock_context, transfer_id, expired=False ) - mock_collection.assert_called_once_with( - mock_get_schedules.return_value) + mock_collection.assert_called_once_with(mock_get_schedules.return_value) @mock.patch('coriolis.schemas.SCHEDULE_API_BODY_SCHEMA') @mock.patch.object(schemas, 'validate_value') def test_validate_schedule( - self, - mock_validate_value, - mock_schedule_api_body_schema + self, mock_validate_value, mock_schedule_api_body_schema ): schedule = mock.sentinel.schedule result = self.transfer_schedules._validate_schedule(schedule) - self.assertEqual( - schedule, - result - ) + self.assertEqual(schedule, result) mock_validate_value.assert_called_once_with( - schedule, mock_schedule_api_body_schema['properties']['schedule']) + schedule, mock_schedule_api_body_schema['properties']['schedule'] + ) - def test_validate_expiration_date_is_none( - self - ): + def test_validate_expiration_date_is_none(self): expiration_date = None - result = self.transfer_schedules._validate_expiration_date( - expiration_date) + result = self.transfer_schedules._validate_expiration_date(expiration_date) - self.assertEqual( - None, - result - ) + self.assertEqual(None, result) - def test_validate_expiration_date_past( - self - ): + def test_validate_expiration_date_past(self): expiration_date = '1970-1-1' self.assertRaises( exception.InvalidInput, self.transfer_schedules._validate_expiration_date, - expiration_date + expiration_date, ) - def test_validate_expiration_date( - self - ): + def test_validate_expiration_date(self): expiration_date = '9999-12-31' - result = self.transfer_schedules._validate_expiration_date( - expiration_date) + result = self.transfer_schedules._validate_expiration_date(expiration_date) - self.assertEqual( - datetime.datetime(9999, 12, 31), - result - ) + self.assertEqual(datetime.datetime(9999, 12, 31), result) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_expiration_date') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_expiration_date' + ) @mock.patch.object(schemas, 'validate_value') @mock.patch.object(jsonschema, 'FormatChecker') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_schedule') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_schedule' + ) def test_validate_create_body( self, mock_validate_schedule, mock_format_checker, mock_validate_value, - mock_validate_expiration_date + mock_validate_expiration_date, ): schedule = mock.sentinel.schedule date = mock.sentinel.date @@ -189,35 +141,35 @@ def test_validate_create_body( False, mock_validate_expiration_date.return_value, True, - False + False, ) result = self.transfer_schedules._validate_create_body(mock_body) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_validate_schedule.assert_called_once_with(schedule) mock_validate_value.assert_called_once_with( - mock_body, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=mock_format_checker.return_value + mock_body, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=mock_format_checker.return_value, ) mock_validate_expiration_date.assert_called_once_with(date) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_expiration_date') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_expiration_date' + ) @mock.patch.object(schemas, 'validate_value') @mock.patch.object(jsonschema, 'FormatChecker') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_schedule') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_schedule' + ) def test_validate_create_body_no_expiration_date( self, mock_validate_schedule, mock_format_checker, mock_validate_value, - mock_validate_expiration_date + mock_validate_expiration_date, ): schedule = mock.sentinel.schedule mock_body = { @@ -231,20 +183,18 @@ def test_validate_create_body_no_expiration_date( False, None, True, - False + False, ) result = self.transfer_schedules._validate_create_body(mock_body) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_validate_schedule.assert_called_once_with(schedule) mock_validate_value.assert_called_once_with( - mock_body, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=mock_format_checker.return_value + mock_body, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=mock_format_checker.return_value, ) mock_validate_expiration_date.assert_not_called() @@ -257,21 +207,23 @@ def test_validate_create_body_no_schedule( self.assertRaises( exception.InvalidInput, self.transfer_schedules._validate_create_body, - mock_body + mock_body, ) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_expiration_date') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_expiration_date' + ) @mock.patch.object(schemas, 'validate_value') @mock.patch.object(jsonschema, 'FormatChecker') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_schedule') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_schedule' + ) def test_validate_update_body( self, mock_validate_schedule, mock_format_checker, mock_validate_value, - mock_validate_expiration_date + mock_validate_expiration_date, ): schedule = mock.sentinel.schedule date = mock.sentinel.date @@ -290,62 +242,54 @@ def test_validate_update_body( "auto_deploy": True, } - result = self.transfer_schedules._validate_update_body( - mock_update_body) + result = self.transfer_schedules._validate_update_body(mock_update_body) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_validate_schedule.assert_called_once_with(schedule) mock_validate_value.assert_called_once_with( - expected_result, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=mock_format_checker.return_value + expected_result, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=mock_format_checker.return_value, ) mock_validate_expiration_date.assert_called_once_with(date) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_expiration_date') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_expiration_date' + ) @mock.patch.object(schemas, 'validate_value') @mock.patch.object(jsonschema, 'FormatChecker') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_schedule') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_schedule' + ) def test_validate_update_body_none( self, mock_validate_schedule, mock_format_checker, mock_validate_value, - mock_validate_expiration_date + mock_validate_expiration_date, ): mock_update_body = {} expected_result = {} - result = self.transfer_schedules._validate_update_body( - mock_update_body) + result = self.transfer_schedules._validate_update_body(mock_update_body) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_validate_schedule.assert_not_called() mock_validate_value.assert_called_once_with( - expected_result, schemas.SCHEDULE_API_BODY_SCHEMA, - format_checker=mock_format_checker.return_value + expected_result, + schemas.SCHEDULE_API_BODY_SCHEMA, + format_checker=mock_format_checker.return_value, ) mock_validate_expiration_date.assert_not_called() @mock.patch.object(transfer_schedule_view, 'single') @mock.patch.object(api.API, 'create') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_create_body') - def test_create( - self, - mock_validate_create_body, - mock_create, - mock_single - ): + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_create_body' + ) + def test_create(self, mock_validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -354,24 +298,27 @@ def test_create( schedule = mock.sentinel.schedule exp_date = mock.sentinel.exp_date mock_validate_create_body.return_value = ( - schedule, False, exp_date, True, False) + schedule, + False, + exp_date, + True, + False, + ) result = self.transfer_schedules.create(mock_req, transfer_id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:create") + mock_context.can.assert_called_once_with("migration:transfer_schedules:create") mock_validate_create_body.assert_called_once_with(body) mock_create.assert_called_once_with( - mock_context, transfer_id, schedule, False, exp_date, True, False) + mock_context, transfer_id, schedule, False, exp_date, True, False + ) mock_single.assert_called_once_with(mock_create.return_value) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_create_body') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_create_body' + ) def test_create_except( self, mock_validate_create_body, @@ -388,23 +335,18 @@ def test_create_except( self.transfer_schedules.create, mock_req, transfer_id, - body + body, ) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:create") + mock_context.can.assert_called_once_with("migration:transfer_schedules:create") mock_validate_create_body.assert_called_once_with(body) @mock.patch.object(transfer_schedule_view, 'single') @mock.patch.object(api.API, 'update') - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_update_body') - def test_update( - self, - mock_validate_update_body, - mock_update, - mock_single - ): + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_update_body' + ) + def test_update(self, mock_validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -412,24 +354,20 @@ def test_update( id = mock.sentinel.id body = mock.sentinel.body - result = self.transfer_schedules.update( - mock_req, transfer_id, id, body) + result = self.transfer_schedules.update(mock_req, transfer_id, id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:update") + mock_context.can.assert_called_once_with("migration:transfer_schedules:update") mock_validate_update_body.assert_called_once_with(body) mock_update.assert_called_once_with( - mock_context, transfer_id, id, - mock_validate_update_body.return_value) + mock_context, transfer_id, id, mock_validate_update_body.return_value + ) mock_single.assert_called_once_with(mock_update.return_value) - @mock.patch.object(transfer_schedules.TransferScheduleController, - '_validate_update_body') + @mock.patch.object( + transfer_schedules.TransferScheduleController, '_validate_update_body' + ) def test_update_except( self, mock_validate_update_body, @@ -448,18 +386,14 @@ def test_update_except( mock_req, transfer_id, id, - body + body, ) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:update") + mock_context.can.assert_called_once_with("migration:transfer_schedules:update") mock_validate_update_body.assert_called_once_with(body) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -467,13 +401,8 @@ def test_delete( id = mock.sentinel.id self.assertRaises( - exc.HTTPNoContent, - self.transfer_schedules.delete, - mock_req, - transfer_id, - id + exc.HTTPNoContent, self.transfer_schedules.delete, mock_req, transfer_id, id ) - mock_context.can.assert_called_once_with( - "migration:transfer_schedules:delete") + mock_context.can.assert_called_once_with("migration:transfer_schedules:delete") mock_delete.assert_called_once_with(mock_context, transfer_id, id) diff --git a/coriolis/tests/api/v1/test_transfer_tasks_execution_actions.py b/coriolis/tests/api/v1/test_transfer_tasks_execution_actions.py index bddfb363..71438f45 100644 --- a/coriolis/tests/api/v1/test_transfer_tasks_execution_actions.py +++ b/coriolis/tests/api/v1/test_transfer_tasks_execution_actions.py @@ -6,33 +6,24 @@ import ddt from webob import exc -from coriolis.api.v1 import transfer_tasks_execution_actions as transfer_api from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.api.v1 import transfer_tasks_execution_actions as transfer_api +from coriolis.tests import test_base, testutils from coriolis.transfer_tasks_executions import api @ddt.ddt -class TransferTasksExecutionActionsControllerTestCase( - test_base.CoriolisBaseTestCase -): +class TransferTasksExecutionActionsControllerTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Transfer Tasks Execution Actions v1 API""" def setUp(self): super(TransferTasksExecutionActionsControllerTestCase, self).setUp() - self.transfer_api = ( - transfer_api.TransferTasksExecutionActionsController()) + self.transfer_api = transfer_api.TransferTasksExecutionActionsController() @mock.patch.object(api.API, 'cancel') @ddt.file_data('data/transfer_task_execution_actions_cancel.yml') def test_cancel( - self, - mock_cancel, - config, - expected_force, - exception_raised, - expected_result + self, mock_cancel, config, expected_force, exception_raised, expected_result ): mock_req = mock.Mock() mock_context = mock.Mock() @@ -41,8 +32,7 @@ def test_cancel( transfer_id = mock.sentinel.transfer_id body = config["body"] if exception_raised: - mock_cancel.side_effect = getattr(exception, exception_raised)( - "err") + mock_cancel.side_effect = getattr(exception, exception_raised)("err") self.assertRaises( getattr(exc, expected_result), @@ -50,10 +40,10 @@ def test_cancel( mock_req, transfer_id, id, - body + body, ) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:cancel") + mock_context.can.assert_called_once_with("migration:transfer_executions:cancel") mock_cancel.assert_called_once_with( - mock_context, transfer_id, id, expected_force) + mock_context, transfer_id, id, expected_force + ) diff --git a/coriolis/tests/api/v1/test_transfer_tasks_executions.py b/coriolis/tests/api/v1/test_transfer_tasks_executions.py index 0bf6fabb..c2b33bbd 100644 --- a/coriolis/tests/api/v1/test_transfer_tasks_executions.py +++ b/coriolis/tests/api/v1/test_transfer_tasks_executions.py @@ -5,9 +5,9 @@ from webob import exc +from coriolis import exception from coriolis.api.v1 import transfer_tasks_executions as transfer_api from coriolis.api.v1.views import transfer_tasks_execution_view -from coriolis import exception from coriolis.tests import test_base from coriolis.transfer_tasks_executions import api @@ -21,11 +21,7 @@ def setUp(self): @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'get_execution') - def test_show( - self, - mock_get_execution, - mock_single - ): + def test_show(self, mock_get_execution, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -34,24 +30,15 @@ def test_show( result = self.transfer_api.show(mock_req, transfer_id, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:show") - mock_get_execution.assert_called_once_with( - mock_context, transfer_id, id) + mock_context.can.assert_called_once_with("migration:transfer_executions:show") + mock_get_execution.assert_called_once_with(mock_context, transfer_id, id) mock_single.assert_called_once_with(mock_get_execution.return_value) @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'get_execution') - def test_show_not_found( - self, - mock_get_execution, - mock_single - ): + def test_show_not_found(self, mock_get_execution, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -60,17 +47,11 @@ def test_show_not_found( mock_get_execution.return_value = None self.assertRaises( - exc.HTTPNotFound, - self.transfer_api.show, - mock_req, - transfer_id, - id + exc.HTTPNotFound, self.transfer_api.show, mock_req, transfer_id, id ) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:show") - mock_get_execution.assert_called_once_with( - mock_context, transfer_id, id) + mock_context.can.assert_called_once_with("migration:transfer_executions:show") + mock_get_execution.assert_called_once_with(mock_context, transfer_id, id) mock_single.assert_not_called() @mock.patch("coriolis.api.common.get_paging_params") @@ -90,7 +71,8 @@ def test_index( transfer_id = mock.sentinel.transfer_id mock_get_sort_params.return_value = ( mock.sentinel.sort_keys, - mock.sentinel.sort_dirs) + mock.sentinel.sort_dirs, + ) mock_get_paging_params.return_value = ( mock.sentinel.marker, mock.sentinel.limit, @@ -98,30 +80,23 @@ def test_index( result = self.transfer_api.index(mock_req, transfer_id) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:list") + mock_context.can.assert_called_once_with("migration:transfer_executions:list") mock_get_executions.assert_called_once_with( - mock_context, transfer_id, include_tasks=False, + mock_context, + transfer_id, + include_tasks=False, marker=mock.sentinel.marker, limit=mock.sentinel.limit, sort_keys=mock.sentinel.sort_keys, sort_dirs=mock.sentinel.sort_dirs, ) - mock_collection.assert_called_once_with( - mock_get_executions.return_value) + mock_collection.assert_called_once_with(mock_get_executions.return_value) @mock.patch.object(transfer_tasks_execution_view, 'collection') @mock.patch.object(api.API, 'get_executions') - def test_detail( - self, - mock_get_executions, - mock_collection - ): + def test_detail(self, mock_get_executions, mock_collection): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -129,25 +104,17 @@ def test_detail( result = self.transfer_api.detail(mock_req, transfer_id) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:show") + mock_context.can.assert_called_once_with("migration:transfer_executions:show") mock_get_executions.assert_called_once_with( - mock_context, transfer_id, include_tasks=True) - mock_collection.assert_called_once_with( - mock_get_executions.return_value) + mock_context, transfer_id, include_tasks=True + ) + mock_collection.assert_called_once_with(mock_get_executions.return_value) @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'create') - def test_create( - self, - mock_create, - mock_single - ): + def test_create(self, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -157,24 +124,15 @@ def test_create( result = self.transfer_api.create(mock_req, transfer_id, mock_body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:create") - mock_create.assert_called_once_with( - mock_context, transfer_id, True, True) + mock_context.can.assert_called_once_with("migration:transfer_executions:create") + mock_create.assert_called_once_with(mock_context, transfer_id, True, True) mock_single.assert_called_once_with(mock_create.return_value) @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'create') - def test_create_no_executions( - self, - mock_create, - mock_single - ): + def test_create_no_executions(self, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -183,22 +141,14 @@ def test_create_no_executions( result = self.transfer_api.create(mock_req, transfer_id, mock_body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:create") - mock_create.assert_called_once_with( - mock_context, transfer_id, False, False) + mock_context.can.assert_called_once_with("migration:transfer_executions:create") + mock_create.assert_called_once_with(mock_context, transfer_id, False, False) mock_single.assert_called_once_with(mock_create.return_value) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -206,22 +156,14 @@ def test_delete( id = mock.sentinel.id self.assertRaises( - exc.HTTPNoContent, - self.transfer_api.delete, - mock_req, - transfer_id, - id + exc.HTTPNoContent, self.transfer_api.delete, mock_req, transfer_id, id ) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:delete") + mock_context.can.assert_called_once_with("migration:transfer_executions:delete") mock_delete.assert_called_once_with(mock_context, transfer_id, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -230,13 +172,8 @@ def test_delete_not_found( mock_delete.side_effect = exception.NotFound() self.assertRaises( - exc.HTTPNotFound, - self.transfer_api.delete, - mock_req, - transfer_id, - id + exc.HTTPNotFound, self.transfer_api.delete, mock_req, transfer_id, id ) - mock_context.can.assert_called_once_with( - "migration:transfer_executions:delete") + mock_context.can.assert_called_once_with("migration:transfer_executions:delete") mock_delete.assert_called_once_with(mock_context, transfer_id, id) diff --git a/coriolis/tests/api/v1/test_transfers.py b/coriolis/tests/api/v1/test_transfers.py index 25fdfea9..aab4155f 100644 --- a/coriolis/tests/api/v1/test_transfers.py +++ b/coriolis/tests/api/v1/test_transfers.py @@ -6,14 +6,12 @@ import ddt from webob import exc +from coriolis import exception from coriolis.api.v1 import transfers from coriolis.api.v1 import utils as api_utils -from coriolis.api.v1.views import transfer_tasks_execution_view -from coriolis.api.v1.views import transfer_view +from coriolis.api.v1.views import transfer_tasks_execution_view, transfer_view from coriolis.endpoints import api as endpoints_api -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils from coriolis.transfers import api @@ -28,12 +26,7 @@ def setUp(self): @mock.patch.object(transfer_view, 'single') @mock.patch.object(api.API, 'get_transfer') @mock.patch.object(api_utils, 'get_bool_url_arg') - def test_show( - self, - mock_get_bool_url_arg, - mock_get_transfer, - mock_single - ): + def test_show(self, mock_get_bool_url_arg, mock_get_transfer, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -42,17 +35,14 @@ def test_show( result = self.transfers.show(mock_req, id) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) mock_context.can.assert_called_once_with("migration:transfers:show") mock_get_bool_url_arg.assert_called_once_with( - mock_req, 'include_task_info', default=False) + mock_req, 'include_task_info', default=False + ) mock_get_transfer.assert_called_once_with( - mock_context, id, - include_task_info=mock_get_bool_url_arg.return_value + mock_context, id, include_task_info=mock_get_bool_url_arg.return_value ) mock_single.assert_called_once_with(mock_get_transfer.return_value) @@ -71,17 +61,12 @@ def test_show_no_transfer( id = mock.sentinel.id mock_get_transfer.return_value = None - self.assertRaises( - exc.HTTPNotFound, - self.transfers.show, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.transfers.show, mock_req, id) mock_context.can.assert_called_once_with("migration:transfers:show") mock_get_transfer.assert_called_once_with( - mock_context, id, - include_task_info=mock_get_bool_url_arg.return_value) + mock_context, id, include_task_info=mock_get_bool_url_arg.return_value + ) mock_single.assert_not_called() @mock.patch("coriolis.api.common.get_paging_params") @@ -102,7 +87,8 @@ def test_list( mock_req.environ = {'coriolis.context': mock_context} mock_get_sort_params.return_value = ( mock.sentinel.sort_keys, - mock.sentinel.sort_dirs) + mock.sentinel.sort_dirs, + ) mock_get_paging_params.return_value = ( mock.sentinel.marker, mock.sentinel.limit, @@ -112,14 +98,11 @@ def test_list( result = self.transfers._list(mock_req) - self.assertEqual( - mock_collection.return_value, - result - ) + self.assertEqual(mock_collection.return_value, result) expected_calls = [ mock.call(mock_req, 'show_deleted', default=False), - mock.call(mock_req, 'include_task_info', default=False) + mock.call(mock_req, 'include_task_info', default=False), ] mock_get_bool_url_arg.assert_has_calls(expected_calls) @@ -133,8 +116,7 @@ def test_list( sort_keys=mock.sentinel.sort_keys, sort_dirs=mock.sentinel.sort_dirs, ) - mock_collection.assert_called_once_with( - mock_get_transfers.return_value) + mock_collection.assert_called_once_with(mock_get_transfers.return_value) @mock.patch.object(api_utils, 'validate_instances_list_for_transfer') @mock.patch.object(endpoints_api.API, 'validate_source_environment') @@ -153,7 +135,7 @@ def test_validate_create_body( mock_validate_instances_list_for_transfer, config, exception_raised, - expected_result + expected_result, ): ctxt = {} body = config["body"] @@ -172,48 +154,40 @@ def test_validate_create_body( self.assertRaisesRegex( Exception, exception_raised, - testutils.get_wrapped_function( - self.transfers._validate_create_body), + testutils.get_wrapped_function(self.transfers._validate_create_body), self.transfers, ctxt, - body + body, ) mock_validate_network_map.assert_not_called() else: result = testutils.get_wrapped_function( - self.transfers._validate_create_body)( - self.transfers, - ctxt, - body, + self.transfers._validate_create_body + )( + self.transfers, + ctxt, + body, ) - self.assertEqual( - tuple(expected_result), - result - ) + self.assertEqual(tuple(expected_result), result) mock_validate_network_map.assert_called_once_with(network_map) mock_validate_target_environment.assert_called_once_with( - ctxt, destination_endpoint_id, destination_environment) + ctxt, destination_endpoint_id, destination_environment + ) mock_validate_user_scripts.assert_called_once_with(user_scripts) - mock_validate_storage_mappings.assert_called_once_with( - storage_mappings) + mock_validate_storage_mappings.assert_called_once_with(storage_mappings) mock_validate_source_environment.assert_called_once_with( - ctxt, origin_endpoint_id, source_environment) - mock_validate_instances_list_for_transfer.assert_called_once_with( - instances) + ctxt, origin_endpoint_id, source_environment + ) + mock_validate_instances_list_for_transfer.assert_called_once_with(instances) @mock.patch.object(transfer_view, 'single') @mock.patch.object(api.API, 'create') @mock.patch.object(transfers.TransferController, '_validate_create_body') - def test_create( - self, - mock_validate_create_body, - mock_create, - mock_single - ): + def test_create(self, mock_validate_create_body, mock_create, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -222,129 +196,102 @@ def test_create( result = self.transfers.create(mock_req, mock_body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfers:create") - mock_validate_create_body.assert_called_once_with( - mock_context, mock_body) + mock_context.can.assert_called_once_with("migration:transfers:create") + mock_validate_create_body.assert_called_once_with(mock_context, mock_body) mock_create.assert_called_once() mock_single.assert_called_once_with(mock_create.return_value) @mock.patch.object(api.API, 'delete') - def test_delete( - self, - mock_delete - ): + def test_delete(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id - self.assertRaises( - exc.HTTPNoContent, - self.transfers.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNoContent, self.transfers.delete, mock_req, id) mock_delete.assert_called_once_with(mock_context, id) @mock.patch.object(api.API, 'delete') - def test_delete_not_found( - self, - mock_delete - ): + def test_delete_not_found(self, mock_delete): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} id = mock.sentinel.id mock_delete.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.transfers.delete, - mock_req, - id - ) + self.assertRaises(exc.HTTPNotFound, self.transfers.delete, mock_req, id) mock_context.can.assert_called_once_with("migration:transfers:delete") mock_delete.assert_called_once_with(mock_context, id) @ddt.file_data('data/transfers_update_storage_mappings.yml') - def test_update_storage_mappings( - self, - config, - expected_result, - logs_expected - ): + def test_update_storage_mappings(self, config, expected_result, logs_expected): original_storage_mappings = config['original_storage_mappings'] new_storage_mappings = config['new_storage_mappings'] if logs_expected: with self.assertLogs('coriolis.api.v1.transfers', level='INFO'): result = self.transfers._update_storage_mappings( - original_storage_mappings, new_storage_mappings) + original_storage_mappings, new_storage_mappings + ) else: result = self.transfers._update_storage_mappings( - original_storage_mappings, new_storage_mappings) + original_storage_mappings, new_storage_mappings + ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_get_updated_user_scripts( self, ): original_user_scripts = { - 'global': {"mock_global_scripts_1": "mock_value", - "mock_global_scripts_2": "mock_value"}, - 'instances': {"mock_instance_scripts": "mock_value"} + 'global': { + "mock_global_scripts_1": "mock_value", + "mock_global_scripts_2": "mock_value", + }, + 'instances': {"mock_instance_scripts": "mock_value"}, } new_user_scripts = { 'global': {"mock_global_scripts_1": "mock_new_value"}, - 'instances': {"mock_instance_scripts": "mock_new_value"} + 'instances': {"mock_instance_scripts": "mock_new_value"}, } expected_result = { - 'global': {"mock_global_scripts_1": "mock_new_value", - "mock_global_scripts_2": "mock_value"}, - 'instances': {"mock_instance_scripts": "mock_new_value"} + 'global': { + "mock_global_scripts_1": "mock_new_value", + "mock_global_scripts_2": "mock_value", + }, + 'instances': {"mock_instance_scripts": "mock_new_value"}, } result = self.transfers._get_updated_user_scripts( - original_user_scripts, new_user_scripts) - - self.assertEqual( - expected_result, - result + original_user_scripts, new_user_scripts ) + self.assertEqual(expected_result, result) + def test_get_updated_user_scripts_new_user_scripts_empty( self, ): original_user_scripts = { - 'global': {"mock_global_scripts_1": "mock_value", - "mock_global_scripts_2": "mock_value"}, - 'instances': {"mock_instance_scripts": "mock_value"} + 'global': { + "mock_global_scripts_1": "mock_value", + "mock_global_scripts_2": "mock_value", + }, + 'instances': {"mock_instance_scripts": "mock_value"}, } new_user_scripts = {} result = self.transfers._get_updated_user_scripts( - original_user_scripts, new_user_scripts) - - self.assertEqual( - original_user_scripts, - result + original_user_scripts, new_user_scripts ) - @mock.patch.object(transfers.TransferController, - '_get_updated_user_scripts') + self.assertEqual(original_user_scripts, result) + + @mock.patch.object(transfers.TransferController, '_get_updated_user_scripts') @mock.patch.object(api_utils, 'validate_user_scripts') - @mock.patch.object(transfers.TransferController, - '_update_storage_mappings') + @mock.patch.object(transfers.TransferController, '_update_storage_mappings') @ddt.file_data('data/transfers_get_merged_transfer_values.yml') def test_get_merged_transfer_values( self, @@ -352,7 +299,7 @@ def test_get_merged_transfer_values( mock_validate_user_scripts, mock_get_updated_user_scripts, config, - expected_result + expected_result, ): transfer = config['transfer'] updated_values = config['updated_values'] @@ -360,38 +307,33 @@ def test_get_merged_transfer_values( transfer_user_scripts = transfer.get('user_scripts', {}) updated_user_scripts = updated_values.get('user_scripts', {}) new_storage_mappings = updated_values.get('storage_mappings', {}) - expected_result['storage_mappings'] = \ + expected_result['storage_mappings'] = mock_update_storage_mappings.return_value + expected_result['destination_environment']['storage_mappings'] = ( mock_update_storage_mappings.return_value - expected_result['destination_environment'][ - 'storage_mappings'] = mock_update_storage_mappings.return_value - expected_result['user_scripts'] = \ - mock_get_updated_user_scripts.return_value - mock_validate_user_scripts.side_effect = ["mock_scripts", - "mock_new_scripts"] - - result = self.transfers._get_merged_transfer_values( - transfer, updated_values) - - self.assertEqual( - expected_result, - result ) + expected_result['user_scripts'] = mock_get_updated_user_scripts.return_value + mock_validate_user_scripts.side_effect = ["mock_scripts", "mock_new_scripts"] + + result = self.transfers._get_merged_transfer_values(transfer, updated_values) + + self.assertEqual(expected_result, result) mock_update_storage_mappings.assert_called_once_with( - original_storage_mapping, new_storage_mappings) + original_storage_mapping, new_storage_mappings + ) mock_validate_user_scripts.assert_has_calls( - [mock.call(transfer_user_scripts), - mock.call(updated_user_scripts)]) + [mock.call(transfer_user_scripts), mock.call(updated_user_scripts)] + ) mock_get_updated_user_scripts.assert_called_once_with( - "mock_scripts", "mock_new_scripts") + "mock_scripts", "mock_new_scripts" + ) @mock.patch.object(api_utils, 'validate_user_scripts') @mock.patch.object(api_utils, 'validate_storage_mappings') @mock.patch.object(api_utils, 'validate_network_map') @mock.patch.object(endpoints_api.API, 'validate_target_environment') @mock.patch.object(endpoints_api.API, 'validate_source_environment') - @mock.patch.object(transfers.TransferController, - '_get_merged_transfer_values') + @mock.patch.object(transfers.TransferController, '_get_merged_transfer_values') @mock.patch.object(api.API, 'get_transfer') @ddt.file_data('data/transfers_validate_update_body.yml') def test_validate_update_body( @@ -404,7 +346,7 @@ def test_validate_update_body( mock_validate_storage_mappings, mock_validate_user_scripts, config, - expected_result + expected_result, ): body = config['body'] transfer = config['transfer'] @@ -414,34 +356,29 @@ def test_validate_update_body( mock_get_transfer.return_value = transfer mock_get_merged_transfer_values.return_value = transfer_body - result = testutils.get_wrapped_function( - self.transfers._validate_update_body)( - self.transfers, - id, - context, - body + result = testutils.get_wrapped_function(self.transfers._validate_update_body)( + self.transfers, id, context, body ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_get_transfer.assert_called_once_with(context, id) - mock_get_merged_transfer_values.assert_called_once_with( - transfer, transfer_body) + mock_get_merged_transfer_values.assert_called_once_with(transfer, transfer_body) mock_validate_source_environment.assert_called_once_with( - context, transfer['origin_endpoint_id'], - transfer_body['source_environment']) + context, transfer['origin_endpoint_id'], transfer_body['source_environment'] + ) mock_validate_target_environment.assert_called_once_with( - context, transfer['destination_endpoint_id'], - transfer_body['destination_environment']) - mock_validate_network_map.assert_called_once_with( - transfer_body['network_map']) + context, + transfer['destination_endpoint_id'], + transfer_body['destination_environment'], + ) + mock_validate_network_map.assert_called_once_with(transfer_body['network_map']) mock_validate_storage_mappings.assert_called_once_with( - transfer_body['storage_mappings']) + transfer_body['storage_mappings'] + ) mock_validate_user_scripts.assert_called_once_with( - transfer_body['user_scripts']) + transfer_body['user_scripts'] + ) @mock.patch.object(api.API, 'get_transfer') @ddt.file_data('data/transfers_validate_update_body_raises.yml') @@ -455,12 +392,11 @@ def test_validate_update_body_raises( self.assertRaises( exc.HTTPBadRequest, - testutils.get_wrapped_function( - self.transfers._validate_update_body), + testutils.get_wrapped_function(self.transfers._validate_update_body), self.transfers, id, context, - body + body, ) mock_get_transfer.assert_called_once_with(context, id) @@ -468,12 +404,7 @@ def test_validate_update_body_raises( @mock.patch.object(transfer_tasks_execution_view, 'single') @mock.patch.object(api.API, 'update') @mock.patch.object(transfers.TransferController, '_validate_update_body') - def test_update( - self, - mock_validate_update_body, - mock_update, - mock_single - ): + def test_update(self, mock_validate_update_body, mock_update, mock_single): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -482,27 +413,18 @@ def test_update( result = self.transfers.update(mock_req, id, body) - self.assertEqual( - mock_single.return_value, - result - ) + self.assertEqual(mock_single.return_value, result) - mock_context.can.assert_called_once_with( - "migration:transfers:update") - mock_validate_update_body.assert_called_once_with( - id, mock_context, body) + mock_context.can.assert_called_once_with("migration:transfers:update") + mock_validate_update_body.assert_called_once_with(id, mock_context, body) mock_update.assert_called_once_with( - mock_context, id, - mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) mock_single.assert_called_once_with(mock_update.return_value) @mock.patch.object(api.API, 'update') @mock.patch.object(transfers.TransferController, '_validate_update_body') - def test_update_not_found( - self, - mock_validate_update_body, - mock_update - ): + def test_update_not_found(self, mock_validate_update_body, mock_update): mock_req = mock.Mock() mock_context = mock.Mock() mock_req.environ = {'coriolis.context': mock_context} @@ -510,28 +432,18 @@ def test_update_not_found( body = mock.sentinel.body mock_update.side_effect = exception.NotFound() - self.assertRaises( - exc.HTTPNotFound, - self.transfers.update, - mock_req, - id, - body - ) + self.assertRaises(exc.HTTPNotFound, self.transfers.update, mock_req, id, body) - mock_context.can.assert_called_once_with( - "migration:transfers:update") - mock_validate_update_body.assert_called_once_with( - id, mock_context, body) + mock_context.can.assert_called_once_with("migration:transfers:update") + mock_validate_update_body.assert_called_once_with(id, mock_context, body) mock_update.assert_called_once_with( - mock_context, id, - mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) @mock.patch.object(api.API, 'update') @mock.patch.object(transfers.TransferController, '_validate_update_body') def test_update_not_invalid_parameter_value( - self, - mock_validate_update_body, - mock_update + self, mock_validate_update_body, mock_update ): mock_req = mock.Mock() mock_context = mock.Mock() @@ -540,18 +452,10 @@ def test_update_not_invalid_parameter_value( body = mock.sentinel.body mock_update.side_effect = exception.InvalidParameterValue('err') - self.assertRaises( - exc.HTTPNotFound, - self.transfers.update, - mock_req, - id, - body - ) + self.assertRaises(exc.HTTPNotFound, self.transfers.update, mock_req, id, body) - mock_context.can.assert_called_once_with( - "migration:transfers:update") - mock_validate_update_body.assert_called_once_with( - id, mock_context, body) + mock_context.can.assert_called_once_with("migration:transfers:update") + mock_validate_update_body.assert_called_once_with(id, mock_context, body) mock_update.assert_called_once_with( - mock_context, id, - mock_validate_update_body.return_value) + mock_context, id, mock_validate_update_body.return_value + ) diff --git a/coriolis/tests/api/v1/test_utils.py b/coriolis/tests/api/v1/test_utils.py index 72fd9960..d38d6a6c 100644 --- a/coriolis/tests/api/v1/test_utils.py +++ b/coriolis/tests/api/v1/test_utils.py @@ -6,9 +6,8 @@ import ddt from webob import exc +from coriolis import exception, schemas from coriolis.api.v1 import utils -from coriolis import exception -from coriolis import schemas from coriolis.tests import test_base @@ -16,39 +15,27 @@ class UtilsTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Utils v1 API""" - def test_get_bool_url_arg_( - self - ): + def test_get_bool_url_arg_(self): mock_req = mock.Mock() mock_req.GET.get.return_value = "true" arg_name = "show_deleted" result = utils.get_bool_url_arg(mock_req, arg_name, default=False) - self.assertEqual( - True, - result - ) + self.assertEqual(True, result) mock_req.GET.get.assert_called_once_with(arg_name, False) - def test_get_bool_url_arg_false( - self - ): + def test_get_bool_url_arg_false(self): mock_req = mock.Mock() mock_req.GET.get.return_value = False arg_name = "show_deleted" result = utils.get_bool_url_arg(mock_req, arg_name, False) - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) mock_req.GET.get.assert_called_once_with(arg_name, False) - def test_get_bool_url_arg_invalid_json( - self - ): + def test_get_bool_url_arg_invalid_json(self): mock_req = mock.MagicMock() mock_req.GET.get.return_value = "}invalid{" arg_name = "include_task_info" @@ -56,69 +43,56 @@ def test_get_bool_url_arg_invalid_json( with self.assertLogs('coriolis.api.v1.utils', level='WARN'): result = utils.get_bool_url_arg(mock_req, arg_name, default=False) - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) mock_req.GET.get.assert_called_once_with(arg_name, False) @mock.patch.object(schemas, 'validate_value') - def test_validate_network_map( - self, - mock_validate_value - ): + def test_validate_network_map(self, mock_validate_value): network_map = "mock_network_map" utils.validate_network_map(network_map) mock_validate_value.assert_called_once_with( - network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA) + network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA + ) @mock.patch.object(schemas, 'validate_value') def test_validate_network_map_schema_validation_exception( - self, - mock_validate_value + self, mock_validate_value ): network_map = "mock_network_map" mock_validate_value.side_effect = exception.SchemaValidationException() - self.assertRaises( - exc.HTTPBadRequest, - utils.validate_network_map, - network_map - ) + self.assertRaises(exc.HTTPBadRequest, utils.validate_network_map, network_map) mock_validate_value.assert_called_once_with( - network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA) + network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA + ) @mock.patch.object(schemas, 'validate_value') - def test_validate_storage_mappings( - self, - mock_validate_value - ): + def test_validate_storage_mappings(self, mock_validate_value): storage_mappings = "mock_storage_mappings" utils.validate_storage_mappings(storage_mappings) mock_validate_value.assert_called_once_with( - storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA) + storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA + ) @mock.patch.object(schemas, 'validate_value') def test_validate_storage_mappings_schema_validation_exception( - self, - mock_validate_value + self, mock_validate_value ): storage_mappings = "mock_storage_mappings" mock_validate_value.side_effect = exception.SchemaValidationException() self.assertRaises( - exc.HTTPBadRequest, - utils.validate_storage_mappings, - storage_mappings + exc.HTTPBadRequest, utils.validate_storage_mappings, storage_mappings ) mock_validate_value.assert_called_once_with( - storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA) + storage_mappings, schemas.CORIOLIS_STORAGE_MAPPINGS_SCHEMA + ) @mock.patch.object(utils, '_build_keyerror_message') def test_format_keyerror_message(self, mock_build_keyerror_message): @@ -128,32 +102,20 @@ def mock_fun(body): result = mock_fun({'endpoint': "expected_result"}) - self.assertEqual( - "expected_result", - result - ) + self.assertEqual("expected_result", result) with self.assertLogs('coriolis.api.v1.utils', level='ERROR'): - self.assertRaises( - exception.InvalidInput, - mock_fun, - '' - ) + self.assertRaises(exception.InvalidInput, mock_fun, '') mock_build_keyerror_message.assert_not_called() with self.assertLogs('coriolis.api.v1.utils', level='ERROR'): - self.assertRaises( - exception.InvalidInput, - mock_fun, - {} - ) + self.assertRaises(exception.InvalidInput, mock_fun, {}) mock_build_keyerror_message.assert_called_once_with( 'endpoint', 'create', 'endpoint' ) @mock.patch.object(utils, '_build_keyerror_message') - def test_format_keyerror_message_empty_keyerror( - self, mock_build_keyerror_message): + def test_format_keyerror_message_empty_keyerror(self, mock_build_keyerror_message): @utils.format_keyerror_message(resource='endpoint', method='create') def mock_fun(body): @@ -175,7 +137,7 @@ def test_build_keyerror_message_resource_equals_key( self.assertEqual( 'The mock_resource creation body needs to be encased inside the' ' "mock_resource" key', - result + result, ) def test_build_keyerror_message_resource_not_equals_key( @@ -188,9 +150,8 @@ def test_build_keyerror_message_resource_not_equals_key( result = utils._build_keyerror_message(resource, method, key) self.assertEqual( - 'The mock_resource update body lacks a required attribute: ' - '"mock_key"', - result + 'The mock_resource update body lacks a required attribute: "mock_key"', + result, ) def test_validate_user_scripts( @@ -199,17 +160,14 @@ def test_validate_user_scripts( user_scripts = { 'global': { 'linux': 'mock_scripts_linux', - 'windows': 'mock_scripts_windows' + 'windows': 'mock_scripts_windows', }, - 'instances': {} + 'instances': {}, } result = utils.validate_user_scripts(user_scripts) - self.assertEqual( - user_scripts, - result - ) + self.assertEqual(user_scripts, result) def test_validate_user_scripts_none( self, @@ -218,39 +176,25 @@ def test_validate_user_scripts_none( result = utils.validate_user_scripts(user_scripts) - self.assertEqual( - {}, - result - ) + self.assertEqual({}, result) @ddt.file_data('data/utils_validate_user_scripts_raises.yml') - def test_validate_user_scripts_invalid_input( - self, - user_scripts - ): + def test_validate_user_scripts_invalid_input(self, user_scripts): self.assertRaises( - exception.InvalidInput, - utils.validate_user_scripts, - user_scripts + exception.InvalidInput, utils.validate_user_scripts, user_scripts ) @ddt.file_data('data/utils_validate_instances_list_for_transfer.yml') def test_validate_instances_list_for_transfer( - self, - instances, - expected_result, - exception_raised + self, instances, expected_result, exception_raised ): if exception_raised: self.assertRaises( exception.InvalidInput, utils.validate_instances_list_for_transfer, - instances + instances, ) else: result = utils.validate_instances_list_for_transfer(instances) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) diff --git a/coriolis/tests/api/v1/views/test_diagnostic_view.py b/coriolis/tests/api/v1/views/test_diagnostic_view.py index f454057b..54310c8d 100644 --- a/coriolis/tests/api/v1/views/test_diagnostic_view.py +++ b/coriolis/tests/api/v1/views/test_diagnostic_view.py @@ -6,7 +6,7 @@ class DiagnosticViewTestCase(test_base.CoriolisBaseTestCase): - "Test suite for the Coriolis api v1 views.""" + "Test suite for the Coriolis api v1 views." def test_single(self): mock_single = "mock_single" @@ -14,10 +14,7 @@ def test_single(self): result = diagnostic_view.single(mock_single) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_single_none(self): mock_single = None @@ -25,10 +22,7 @@ def test_single_none(self): result = diagnostic_view.single(mock_single) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_collection(self): mock_collection = {"mock_key_1": "value_1", "mock_key_2": "value_2"} @@ -36,10 +30,7 @@ def test_collection(self): result = diagnostic_view.collection(mock_collection) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_collection_none(self): mock_collection = {} @@ -47,7 +38,4 @@ def test_collection_none(self): result = diagnostic_view.collection(mock_collection) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) diff --git a/coriolis/tests/api/v1/views/test_endpoint_options_view.py b/coriolis/tests/api/v1/views/test_endpoint_options_view.py index 2b4f2062..78c16ae4 100644 --- a/coriolis/tests/api/v1/views/test_endpoint_options_view.py +++ b/coriolis/tests/api/v1/views/test_endpoint_options_view.py @@ -6,23 +6,20 @@ class EndpointOptionsViewTestCase(test_base.CoriolisApiViewsTestCase): - def test_destination_minion_pool_options_collection(self): - fun = getattr(endpoint_options_view, - 'destination_minion_pool_options_collection') + fun = getattr( + endpoint_options_view, 'destination_minion_pool_options_collection' + ) self._collection_view_test(fun, "destination_minion_pool_options") def test_destination_options_collection(self): - fun = getattr(endpoint_options_view, - 'destination_options_collection') + fun = getattr(endpoint_options_view, 'destination_options_collection') self._collection_view_test(fun, "destination_options") def test_source_minion_pool_options_collection(self): - fun = getattr(endpoint_options_view, - 'source_minion_pool_options_collection') + fun = getattr(endpoint_options_view, 'source_minion_pool_options_collection') self._collection_view_test(fun, "source_minion_pool_options") def test_source_options_collection(self): - fun = getattr(endpoint_options_view, - 'source_options_collection') + fun = getattr(endpoint_options_view, 'source_options_collection') self._collection_view_test(fun, "source_options") diff --git a/coriolis/tests/api/v1/views/test_endpoint_resources_view.py b/coriolis/tests/api/v1/views/test_endpoint_resources_view.py index 81f7b203..98611f30 100644 --- a/coriolis/tests/api/v1/views/test_endpoint_resources_view.py +++ b/coriolis/tests/api/v1/views/test_endpoint_resources_view.py @@ -6,28 +6,22 @@ class EndpointResourcesViewTestCase(test_base.CoriolisApiViewsTestCase): - def test_instance_single(self): - fun = getattr(endpoint_resources_view, - 'instance_single') + fun = getattr(endpoint_resources_view, 'instance_single') self._single_view_test(fun, "instance") def test_instances_collection(self): - fun = getattr(endpoint_resources_view, - 'instances_collection') + fun = getattr(endpoint_resources_view, 'instances_collection') self._collection_view_test(fun, "instances") def test_network_single(self): - fun = getattr(endpoint_resources_view, - 'network_single') + fun = getattr(endpoint_resources_view, 'network_single') self._single_view_test(fun, "network") def test_networks_collection(self): - fun = getattr(endpoint_resources_view, - 'networks_collection') + fun = getattr(endpoint_resources_view, 'networks_collection') self._collection_view_test(fun, "networks") def test_storage_collection(self): - fun = getattr(endpoint_resources_view, - 'storage_collection') + fun = getattr(endpoint_resources_view, 'storage_collection') self._single_view_test(fun, "storage") diff --git a/coriolis/tests/api/v1/views/test_endpoint_view.py b/coriolis/tests/api/v1/views/test_endpoint_view.py index ad23e1c8..7a2b18a9 100644 --- a/coriolis/tests/api/v1/views/test_endpoint_view.py +++ b/coriolis/tests/api/v1/views/test_endpoint_view.py @@ -17,66 +17,52 @@ def setUp(self): @mock.patch.object(view_utils, 'format_opt') def test_format_endpoint(self, mock_format_opt): - mock_format_opt.return_value = { - "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - "mock_key": "mock_value" - } - - expected_result = { - 'mapped_regions': ['mock_id1', 'mock_id2'], - 'mock_key': 'mock_value' - } - - endpoint = mock.sentinel.endpoint - keys = mock.sentinel.keys - result = endpoint_view._format_endpoint(endpoint, keys) - - mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = { + "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + "mock_key": "mock_value", + } + + expected_result = { + 'mapped_regions': ['mock_id1', 'mock_id2'], + 'mock_key': 'mock_value', + } + + endpoint = mock.sentinel.endpoint + keys = mock.sentinel.keys + result = endpoint_view._format_endpoint(endpoint, keys) + + mock_format_opt.assert_called_once_with(endpoint, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_endpoint_no_keys(self, mock_format_opt): - mock_format_opt.return_value = { - "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - } + mock_format_opt.return_value = { + "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + } - expected_result = { - 'mapped_regions': ['mock_id1', 'mock_id2'], - } + expected_result = { + 'mapped_regions': ['mock_id1', 'mock_id2'], + } - endpoint = mock.sentinel.endpoint - keys = mock.sentinel.keys - result = endpoint_view._format_endpoint(endpoint, keys) + endpoint = mock.sentinel.endpoint + keys = mock.sentinel.keys + result = endpoint_view._format_endpoint(endpoint, keys) - mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.assert_called_once_with(endpoint, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_endpoint_no_mapped_regions(self, mock_format_opt): - mock_format_opt.return_value = { - "mock_key": "mock_value" - } - - expected_result = { - 'mapped_regions': [], - 'mock_key': 'mock_value' - } - - endpoint = mock.sentinel.endpoint - keys = mock.sentinel.keys - result = endpoint_view._format_endpoint(endpoint, keys) - - mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = {"mock_key": "mock_value"} + + expected_result = {'mapped_regions': [], 'mock_key': 'mock_value'} + + endpoint = mock.sentinel.endpoint + keys = mock.sentinel.keys + result = endpoint_view._format_endpoint(endpoint, keys) + + mock_format_opt.assert_called_once_with(endpoint, keys) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(endpoint_view, 'single') diff --git a/coriolis/tests/api/v1/views/test_minion_pool_view.py b/coriolis/tests/api/v1/views/test_minion_pool_view.py index 5d6ab6ce..20365ca1 100644 --- a/coriolis/tests/api/v1/views/test_minion_pool_view.py +++ b/coriolis/tests/api/v1/views/test_minion_pool_view.py @@ -17,36 +17,40 @@ def test_format_minion_pool( mock_format_opt, ): mock_minion_pool_dict = { - 'minion_machines': [{ - 'connection_info': { - 'pkey': 'mock_key', - 'password': 'mock_key', - 'certificates': {'key': 'mock_key'} - }, - 'backup_writer_connection_info': { - 'connection_details': { + 'minion_machines': [ + { + 'connection_info': { 'pkey': 'mock_key', 'password': 'mock_key', - 'certificates': {'key': 'mock_key'} + 'certificates': {'key': 'mock_key'}, + }, + 'backup_writer_connection_info': { + 'connection_details': { + 'pkey': 'mock_key', + 'password': 'mock_key', + 'certificates': {'key': 'mock_key'}, + }, }, } - }], + ], } expected_result = { - 'minion_machines': [{ - 'connection_info': { - 'pkey': '***', - 'password': '***', - 'certificates': {'key': '***'} - }, - 'backup_writer_connection_info': { - 'connection_details': { + 'minion_machines': [ + { + 'connection_info': { 'pkey': '***', 'password': '***', - 'certificates': {'key': '***'} + 'certificates': {'key': '***'}, + }, + 'backup_writer_connection_info': { + 'connection_details': { + 'pkey': '***', + 'password': '***', + 'certificates': {'key': '***'}, + }, }, } - }], + ], } mock_format_opt.return_value = mock_minion_pool_dict @@ -55,10 +59,7 @@ def test_format_minion_pool( result = view._format_minion_pool(endpoint, keys) mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_minion_pool_connection_none( @@ -66,20 +67,24 @@ def test_format_minion_pool_connection_none( mock_format_opt, ): mock_minion_pool_dict = { - 'minion_machines': [{ - 'connection_info': None, - 'backup_writer_connection_info': { - 'connection_details': None, + 'minion_machines': [ + { + 'connection_info': None, + 'backup_writer_connection_info': { + 'connection_details': None, + }, } - }], + ], } expected_result = { - 'minion_machines': [{ - 'connection_info': None, - 'backup_writer_connection_info': { - 'connection_details': None, + 'minion_machines': [ + { + 'connection_info': None, + 'backup_writer_connection_info': { + 'connection_details': None, + }, } - }], + ], } mock_format_opt.return_value = mock_minion_pool_dict @@ -88,10 +93,7 @@ def test_format_minion_pool_connection_none( result = view._format_minion_pool(endpoint, keys) mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_minion_pool_no_minion_machines( @@ -106,10 +108,7 @@ def test_format_minion_pool_no_minion_machines( result = view._format_minion_pool(endpoint, keys) mock_format_opt.assert_called_once_with(endpoint, keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(view, 'single') diff --git a/coriolis/tests/api/v1/views/test_region_view.py b/coriolis/tests/api/v1/views/test_region_view.py index 674fbcdd..ed0cfed0 100644 --- a/coriolis/tests/api/v1/views/test_region_view.py +++ b/coriolis/tests/api/v1/views/test_region_view.py @@ -17,75 +17,60 @@ def setUp(self): @mock.patch.object(view_utils, 'format_opt') def test_format_region(self, mock_format_opt): - mock_format_opt.return_value = { - "mapped_endpoints": [{'id': 'endpoint_1'}, - {'id': 'endpoint_2'}], - "mapped_services": [{'id': 'service_1'}, - {'id': 'service_2'}], - "mock_key": "mock_value" - } - - expected_result = { - 'mapped_endpoints': ['endpoint_1', 'endpoint_2'], - "mapped_services": ['service_1', 'service_2'], - 'mock_key': 'mock_value' - } - - region = mock.sentinel.region - keys = mock.sentinel.keys - result = region_view._format_region(region, keys) - - mock_format_opt.assert_called_once_with(region, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = { + "mapped_endpoints": [{'id': 'endpoint_1'}, {'id': 'endpoint_2'}], + "mapped_services": [{'id': 'service_1'}, {'id': 'service_2'}], + "mock_key": "mock_value", + } + + expected_result = { + 'mapped_endpoints': ['endpoint_1', 'endpoint_2'], + "mapped_services": ['service_1', 'service_2'], + 'mock_key': 'mock_value', + } + + region = mock.sentinel.region + keys = mock.sentinel.keys + result = region_view._format_region(region, keys) + + mock_format_opt.assert_called_once_with(region, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_region_no_keys(self, mock_format_opt): - mock_format_opt.return_value = { - 'mapped_endpoints': [{'id': 'endpoint_1'}, - {'id': 'endpoint_2'}], - 'mapped_services': [{'id': 'service_1'}, - {'id': 'service_2'}], - } - - expected_result = { - 'mapped_endpoints': ['endpoint_1', 'endpoint_2'], - 'mapped_services': ['service_1', 'service_2'], - } - - region = mock.sentinel.region - keys = mock.sentinel.keys - result = region_view._format_region(region, keys) - - mock_format_opt.assert_called_once_with(region, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = { + 'mapped_endpoints': [{'id': 'endpoint_1'}, {'id': 'endpoint_2'}], + 'mapped_services': [{'id': 'service_1'}, {'id': 'service_2'}], + } + + expected_result = { + 'mapped_endpoints': ['endpoint_1', 'endpoint_2'], + 'mapped_services': ['service_1', 'service_2'], + } + + region = mock.sentinel.region + keys = mock.sentinel.keys + result = region_view._format_region(region, keys) + + mock_format_opt.assert_called_once_with(region, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_region_no_mapped_regions(self, mock_format_opt): - mock_format_opt.return_value = { - "mock_key": "mock_value" - } - - expected_result = { - 'mapped_endpoints': [], - 'mapped_services': [], - 'mock_key': 'mock_value' - } - - region = mock.sentinel.region - keys = mock.sentinel.keys - result = region_view._format_region(region, keys) - - mock_format_opt.assert_called_once_with(region, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = {"mock_key": "mock_value"} + + expected_result = { + 'mapped_endpoints': [], + 'mapped_services': [], + 'mock_key': 'mock_value', + } + + region = mock.sentinel.region + keys = mock.sentinel.keys + result = region_view._format_region(region, keys) + + mock_format_opt.assert_called_once_with(region, keys) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(region_view, 'single') diff --git a/coriolis/tests/api/v1/views/test_service_view.py b/coriolis/tests/api/v1/views/test_service_view.py index 1cd79061..b2a2c707 100644 --- a/coriolis/tests/api/v1/views/test_service_view.py +++ b/coriolis/tests/api/v1/views/test_service_view.py @@ -17,66 +17,52 @@ def setUp(self): @mock.patch.object(view_utils, 'format_opt') def test_format_service(self, mock_format_opt): - mock_format_opt.return_value = { - "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - "mock_key": "mock_value" - } - - expected_result = { - "mapped_regions": ['mock_id1', 'mock_id2'], - 'mock_key': 'mock_value' - } - - service = mock.sentinel.service - keys = mock.sentinel.keys - result = service_view._format_service(service, keys) - - mock_format_opt.assert_called_once_with(service, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = { + "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + "mock_key": "mock_value", + } + + expected_result = { + "mapped_regions": ['mock_id1', 'mock_id2'], + 'mock_key': 'mock_value', + } + + service = mock.sentinel.service + keys = mock.sentinel.keys + result = service_view._format_service(service, keys) + + mock_format_opt.assert_called_once_with(service, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_service_no_keys(self, mock_format_opt): - mock_format_opt.return_value = { - "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - } + mock_format_opt.return_value = { + "mapped_regions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + } - expected_result = { - "mapped_regions": ['mock_id1', 'mock_id2'], - } + expected_result = { + "mapped_regions": ['mock_id1', 'mock_id2'], + } - service = mock.sentinel.service - keys = mock.sentinel.keys - result = service_view._format_service(service, keys) + service = mock.sentinel.service + keys = mock.sentinel.keys + result = service_view._format_service(service, keys) - mock_format_opt.assert_called_once_with(service, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.assert_called_once_with(service, keys) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_service_no_mapped_regions(self, mock_format_opt): - mock_format_opt.return_value = { - "mock_key": "mock_value" - } - - expected_result = { - 'mapped_regions': [], - 'mock_key': 'mock_value' - } - - service = mock.sentinel.service - keys = mock.sentinel.keys - result = service_view._format_service(service, keys) - - mock_format_opt.assert_called_once_with(service, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = {"mock_key": "mock_value"} + + expected_result = {'mapped_regions': [], 'mock_key': 'mock_value'} + + service = mock.sentinel.service + keys = mock.sentinel.keys + result = service_view._format_service(service, keys) + + mock_format_opt.assert_called_once_with(service, keys) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(service_view, 'single') diff --git a/coriolis/tests/api/v1/views/test_transfer_task_execution_view.py b/coriolis/tests/api/v1/views/test_transfer_task_execution_view.py index 4cf58ff5..297cb667 100644 --- a/coriolis/tests/api/v1/views/test_transfer_task_execution_view.py +++ b/coriolis/tests/api/v1/views/test_transfer_task_execution_view.py @@ -3,9 +3,9 @@ from unittest import mock +from coriolis import constants from coriolis.api.v1.views import transfer_tasks_execution_view as view from coriolis.api.v1.views import utils as view_utils -from coriolis import constants from coriolis.tests import test_base @@ -14,16 +14,9 @@ class TransferTaskExecutionViewTestCase(test_base.CoriolisApiViewsTestCase): @mock.patch.object(view, '_sort_tasks') @mock.patch.object(view_utils, 'format_opt') - def test_format_transfer_tasks_execution( - self, - mock_format_opt, - mock_sort_tasks - ): + def test_format_transfer_tasks_execution(self, mock_format_opt, mock_sort_tasks): mock_tasks = ['mock_task1', 'mock_task2'] - mock_execution = { - 'tasks': mock_tasks, - 'mock_key': 'mock_value' - } + mock_execution = {'tasks': mock_tasks, 'mock_key': 'mock_value'} mock_sort_tasks.return_value = mock_execution keys = mock.sentinel.keys @@ -31,31 +24,21 @@ def test_format_transfer_tasks_execution( mock_sort_tasks.assert_called_once_with(mock_tasks) mock_format_opt.assert_called_once_with(mock_execution["tasks"], keys) - self.assertEqual( - mock_format_opt.return_value, - result - ) + self.assertEqual(mock_format_opt.return_value, result) @mock.patch.object(view, '_sort_tasks') @mock.patch.object(view_utils, 'format_opt') def test_format_transfer_tasks_execution_no_tasks( - self, - mock_format_opt, - mock_sort_tasks + self, mock_format_opt, mock_sort_tasks ): - mock_execution = { - 'mock_key': 'mock_value' - } + mock_execution = {'mock_key': 'mock_value'} keys = mock.sentinel.keys result = view.format_transfer_tasks_execution(mock_execution, keys) mock_sort_tasks.assert_not_called() mock_format_opt.assert_called_once_with(mock_execution, keys) - self.assertEqual( - mock_format_opt.return_value, - result - ) + self.assertEqual(mock_format_opt.return_value, result) def test_sort_tasks(self): mock_tasks = [ @@ -72,10 +55,7 @@ def test_sort_tasks(self): result = view._sort_tasks(mock_tasks) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_sort_tasks_no_filter(self): mock_tasks = [ @@ -93,20 +73,14 @@ def test_sort_tasks_no_filter(self): result = view._sort_tasks(mock_tasks, False) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_sort_tasks_no_tasks(self): expected_result = [] result = view._sort_tasks(expected_result) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(view, 'single') diff --git a/coriolis/tests/api/v1/views/test_transfer_view.py b/coriolis/tests/api/v1/views/test_transfer_view.py index cbcdc957..82d0117b 100644 --- a/coriolis/tests/api/v1/views/test_transfer_view.py +++ b/coriolis/tests/api/v1/views/test_transfer_view.py @@ -18,90 +18,74 @@ def setUp(self): @mock.patch.object(view, 'format_transfer_tasks_execution') @mock.patch.object(view_utils, 'format_opt') - def test_format_transfer(self, mock_format_opt, - mock_format_transfer_tasks_execution): - mock_format_opt.return_value = { - "executions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - "mock_key": "mock_value" - } - - expected_calls = [ - mock.call.mock_format_transfer_tasks_execution( - {'id': 'mock_id1'}), - mock.call.mock_format_transfer_tasks_execution( - {'id': 'mock_id2'})] - expected_result = { - "executions": - [mock_format_transfer_tasks_execution.return_value, - mock_format_transfer_tasks_execution.return_value], - 'mock_key': 'mock_value' - } - - transfer = mock.sentinel.transfer - keys = mock.sentinel.keys - result = transfer_view._format_transfer(transfer, keys) - - mock_format_opt.assert_called_once_with(transfer, keys) - mock_format_transfer_tasks_execution.assert_has_calls( - expected_calls - ) - self.assertEqual( - expected_result, - result - ) + def test_format_transfer( + self, mock_format_opt, mock_format_transfer_tasks_execution + ): + mock_format_opt.return_value = { + "executions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + "mock_key": "mock_value", + } + + expected_calls = [ + mock.call.mock_format_transfer_tasks_execution({'id': 'mock_id1'}), + mock.call.mock_format_transfer_tasks_execution({'id': 'mock_id2'}), + ] + expected_result = { + "executions": [ + mock_format_transfer_tasks_execution.return_value, + mock_format_transfer_tasks_execution.return_value, + ], + 'mock_key': 'mock_value', + } + + transfer = mock.sentinel.transfer + keys = mock.sentinel.keys + result = transfer_view._format_transfer(transfer, keys) + + mock_format_opt.assert_called_once_with(transfer, keys) + mock_format_transfer_tasks_execution.assert_has_calls(expected_calls) + self.assertEqual(expected_result, result) @mock.patch.object(view, 'format_transfer_tasks_execution') @mock.patch.object(view_utils, 'format_opt') - def test_format_transfer_no_keys(self, mock_format_opt, - mock_format_transfer_tasks_execution): - mock_format_opt.return_value = { - "executions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], - } - - expected_calls = [ - mock.call.mock_format_transfer_tasks_execution( - {'id': 'mock_id1'}), - mock.call.mock_format_transfer_tasks_execution( - {'id': 'mock_id2'})] - expected_result = { - "executions": - [mock_format_transfer_tasks_execution.return_value, - mock_format_transfer_tasks_execution.return_value], - } - - transfer = mock.sentinel.transfer - keys = mock.sentinel.keys - result = transfer_view._format_transfer(transfer, keys) - - mock_format_opt.assert_called_once_with(transfer, keys) - mock_format_transfer_tasks_execution.assert_has_calls( - expected_calls - ) - self.assertEqual( - expected_result, - result - ) + def test_format_transfer_no_keys( + self, mock_format_opt, mock_format_transfer_tasks_execution + ): + mock_format_opt.return_value = { + "executions": [{'id': 'mock_id1'}, {'id': 'mock_id2'}], + } + + expected_calls = [ + mock.call.mock_format_transfer_tasks_execution({'id': 'mock_id1'}), + mock.call.mock_format_transfer_tasks_execution({'id': 'mock_id2'}), + ] + expected_result = { + "executions": [ + mock_format_transfer_tasks_execution.return_value, + mock_format_transfer_tasks_execution.return_value, + ], + } + + transfer = mock.sentinel.transfer + keys = mock.sentinel.keys + result = transfer_view._format_transfer(transfer, keys) + + mock_format_opt.assert_called_once_with(transfer, keys) + mock_format_transfer_tasks_execution.assert_has_calls(expected_calls) + self.assertEqual(expected_result, result) @mock.patch.object(view_utils, 'format_opt') def test_format_transfer_no_executions(self, mock_format_opt): - mock_format_opt.return_value = { - "mock_key": "mock_value" - } - - expected_result = { - 'executions': [], - 'mock_key': 'mock_value' - } - - transfer = mock.sentinel.transfer - keys = mock.sentinel.keys - result = transfer_view._format_transfer(transfer, keys) - - mock_format_opt.assert_called_once_with(transfer, keys) - self.assertEqual( - expected_result, - result - ) + mock_format_opt.return_value = {"mock_key": "mock_value"} + + expected_result = {'executions': [], 'mock_key': 'mock_value'} + + transfer = mock.sentinel.transfer + keys = mock.sentinel.keys + result = transfer_view._format_transfer(transfer, keys) + + mock_format_opt.assert_called_once_with(transfer, keys) + self.assertEqual(expected_result, result) def test_single(self): fun = getattr(transfer_view, 'single') diff --git a/coriolis/tests/api/v1/views/test_utils.py b/coriolis/tests/api/v1/views/test_utils.py index 794ce92c..82376589 100644 --- a/coriolis/tests/api/v1/views/test_utils.py +++ b/coriolis/tests/api/v1/views/test_utils.py @@ -16,10 +16,7 @@ def test_format_opt(self): result = view_utils.format_opt(mock_option, mock_keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_format_opt_key_not_in_options(self): mock_option = {"mock_key_1": "value_1", "mock_key_2": "value_2"} @@ -29,10 +26,7 @@ def test_format_opt_key_not_in_options(self): result = view_utils.format_opt(mock_option, mock_keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) def test_format_opt_keys_none(self): mock_option = {"mock_key_1": "value_1", "mock_key_2": "value_2"} @@ -42,7 +36,4 @@ def test_format_opt_keys_none(self): result = view_utils.format_opt(mock_option, mock_keys) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) diff --git a/coriolis/tests/cmd/test_api.py b/coriolis/tests/cmd/test_api.py index 4ec8d0f9..c1ecda7e 100644 --- a/coriolis/tests/cmd/test_api.py +++ b/coriolis/tests/cmd/test_api.py @@ -4,10 +4,9 @@ import sys from unittest import mock +from coriolis import service, utils from coriolis.cmd import api -from coriolis import service from coriolis.tests import test_base -from coriolis import utils class ApiTestCase(test_base.CoriolisBaseTestCase): @@ -19,8 +18,7 @@ class ApiTestCase(test_base.CoriolisBaseTestCase): @mock.patch('coriolis.cmd.api.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -39,17 +37,19 @@ def test_main( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_WSGIService.assert_called_once_with( - 'coriolis-api', worker_count=worker_count) + 'coriolis-api', worker_count=worker_count + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_WSGIService.return_value, - workers=mock_WSGIService.return_value. - get_workers_count.return_value) + mock_conf, + mock_WSGIService.return_value, + workers=mock_WSGIService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) @mock.patch.object(service, 'service') @mock.patch.object(service, 'WSGIService') @@ -57,8 +57,7 @@ def test_main( @mock.patch('coriolis.cmd.api.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main_no_worker_count( self, mock_gmr_setup, @@ -67,7 +66,7 @@ def test_main_no_worker_count( mock_conf, mock_setup_logging, mock_WSGIService, - mock_service + mock_service, ): worker_count = None args = ['mock_arg_1', 'mock_arg_2'] @@ -77,14 +76,16 @@ def test_main_no_worker_count( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_WSGIService.assert_called_once_with( - 'coriolis-api', worker_count=mock_conf.api.worker_count) + 'coriolis-api', worker_count=mock_conf.api.worker_count + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_WSGIService.return_value, - workers=mock_WSGIService.return_value. - get_workers_count.return_value) + mock_conf, + mock_WSGIService.return_value, + workers=mock_WSGIService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/cmd/test_conductor.py b/coriolis/tests/cmd/test_conductor.py index e6a072a6..f1e67d3e 100644 --- a/coriolis/tests/cmd/test_conductor.py +++ b/coriolis/tests/cmd/test_conductor.py @@ -4,12 +4,10 @@ import sys from unittest import mock +from coriolis import constants, service, utils from coriolis.cmd import conductor from coriolis.conductor.rpc import server as rpc_server -from coriolis import constants -from coriolis import service from coriolis.tests import test_base -from coriolis import utils class ConductorTestCase(test_base.CoriolisBaseTestCase): @@ -23,8 +21,7 @@ class ConductorTestCase(test_base.CoriolisBaseTestCase): @mock.patch('coriolis.cmd.conductor.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -35,7 +32,7 @@ def test_main( mock_check_locks_dir_empty, mock_MessagingService, mock_ConductorServerEndpoint, - mock_service + mock_service, ): worker_count = mock.sentinel.worker_count args = ['mock_arg_1', 'mock_arg_2'] @@ -45,20 +42,23 @@ def test_main( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_check_locks_dir_empty.assert_called_once() mock_MessagingService.assert_called_once_with( constants.CONDUCTOR_MAIN_MESSAGING_TOPIC, [mock_ConductorServerEndpoint.return_value], - rpc_server.VERSION, worker_count=worker_count) + rpc_server.VERSION, + worker_count=worker_count, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) @mock.patch.object(service, 'service') @mock.patch.object(rpc_server, 'ConductorServerEndpoint') @@ -68,8 +68,7 @@ def test_main( @mock.patch('coriolis.cmd.conductor.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main_no_worker_count( self, mock_gmr_setup, @@ -80,7 +79,7 @@ def test_main_no_worker_count( mock_check_locks_dir_empty, mock_MessagingService, mock_ConductorServerEndpoint, - mock_service + mock_service, ): worker_count = None args = ['mock_arg_1', 'mock_arg_2'] @@ -90,17 +89,20 @@ def test_main_no_worker_count( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_check_locks_dir_empty.assert_called_once() mock_MessagingService.assert_called_once_with( constants.CONDUCTOR_MAIN_MESSAGING_TOPIC, [mock_ConductorServerEndpoint.return_value], - rpc_server.VERSION, worker_count=mock_conf.conductor.worker_count) + rpc_server.VERSION, + worker_count=mock_conf.conductor.worker_count, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/cmd/test_db_sync.py b/coriolis/tests/cmd/test_db_sync.py index 51e6c7c0..ae843652 100644 --- a/coriolis/tests/cmd/test_db_sync.py +++ b/coriolis/tests/cmd/test_db_sync.py @@ -4,10 +4,10 @@ import sys from unittest import mock +from coriolis import utils from coriolis.cmd import db_sync from coriolis.db import api as db_api from coriolis.tests import test_base -from coriolis import utils class DBSyncTestCase(test_base.CoriolisBaseTestCase): @@ -19,17 +19,13 @@ class DBSyncTestCase(test_base.CoriolisBaseTestCase): @mock.patch('coriolis.cmd.db_sync.CONF') @mock.patch.object(sys, 'argv') def test_main( - self, - mock_argv, - mock_conf, - mock_setup_logging, - mock_get_engine, - mock_db_sync + self, mock_argv, mock_conf, mock_setup_logging, mock_get_engine, mock_db_sync ): db_sync.main() mock_conf.assert_called_once_with( - mock_argv[1:], project='coriolis', version="1.0.0") + mock_argv[1:], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_get_engine.assert_called_once() mock_db_sync.assert_called_once_with(mock_get_engine.return_value) diff --git a/coriolis/tests/cmd/test_minion_manager.py b/coriolis/tests/cmd/test_minion_manager.py index 6ed31dca..602ad71b 100644 --- a/coriolis/tests/cmd/test_minion_manager.py +++ b/coriolis/tests/cmd/test_minion_manager.py @@ -4,12 +4,10 @@ import sys from unittest import mock +from coriolis import constants, service, utils from coriolis.cmd import minion_manager -from coriolis import constants from coriolis.minion_manager.rpc import server as rpc_server -from coriolis import service from coriolis.tests import test_base -from coriolis import utils class MinionManagerTestCase(test_base.CoriolisBaseTestCase): @@ -21,8 +19,7 @@ class MinionManagerTestCase(test_base.CoriolisBaseTestCase): @mock.patch.object(utils, 'setup_logging') @mock.patch('coriolis.cmd.minion_manager.CONF') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -31,23 +28,25 @@ def test_main( mock_setup_logging, mock_MinionManagerServerEndpoint, mock_MessagingService, - mock_service + mock_service, ): minion_manager.main() mock_conf.assert_called_once_with( - mock_argv[1:], project='coriolis', version="1.0.0") + mock_argv[1:], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_MinionManagerServerEndpoint.assert_called_once() mock_MessagingService.assert_called_once_with( constants.MINION_MANAGER_MAIN_MESSAGING_TOPIC, [mock_MinionManagerServerEndpoint.return_value], rpc_server.VERSION, - worker_count=mock_conf.minion_manager.worker_count) + worker_count=mock_conf.minion_manager.worker_count, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/cmd/test_replica_cron.py b/coriolis/tests/cmd/test_replica_cron.py index df264228..e2c4e63d 100644 --- a/coriolis/tests/cmd/test_replica_cron.py +++ b/coriolis/tests/cmd/test_replica_cron.py @@ -4,12 +4,10 @@ import sys from unittest import mock +from coriolis import constants, service, utils from coriolis.cmd import transfer_cron -from coriolis import constants -from coriolis import service from coriolis.tests import test_base from coriolis.transfer_cron.rpc import server as rpc_server -from coriolis import utils class TransferCronTestCase(test_base.CoriolisBaseTestCase): @@ -21,8 +19,7 @@ class TransferCronTestCase(test_base.CoriolisBaseTestCase): @mock.patch.object(utils, 'setup_logging') @mock.patch('coriolis.cmd.transfer_cron.CONF') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -31,23 +28,25 @@ def test_main( mock_setup_logging, mock_TransferCronServerEndpoint, mock_MessagingService, - mock_service + mock_service, ): transfer_cron.main() mock_conf.assert_called_once_with( - mock_argv[1:], project='coriolis', version="1.0.0") + mock_argv[1:], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_TransferCronServerEndpoint.assert_called_once() mock_MessagingService.assert_called_once_with( constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC, [mock_TransferCronServerEndpoint.return_value], rpc_server.VERSION, - worker_count=1) + worker_count=1, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/cmd/test_scheduler.py b/coriolis/tests/cmd/test_scheduler.py index 2fc39d86..77dbc797 100644 --- a/coriolis/tests/cmd/test_scheduler.py +++ b/coriolis/tests/cmd/test_scheduler.py @@ -4,12 +4,10 @@ import sys from unittest import mock +from coriolis import constants, service, utils from coriolis.cmd import scheduler -from coriolis import constants from coriolis.scheduler.rpc import server as rpc_server -from coriolis import service from coriolis.tests import test_base -from coriolis import utils class SchedulerTestCase(test_base.CoriolisBaseTestCase): @@ -21,8 +19,7 @@ class SchedulerTestCase(test_base.CoriolisBaseTestCase): @mock.patch.object(utils, 'setup_logging') @mock.patch('coriolis.cmd.scheduler.CONF') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -31,23 +28,25 @@ def test_main( mock_setup_logging, mock_SchedulerServerEndpoint, mock_MessagingService, - mock_service + mock_service, ): scheduler.main() mock_conf.assert_called_once_with( - mock_argv[1:], project='coriolis', version="1.0.0") + mock_argv[1:], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_SchedulerServerEndpoint.assert_called_once() mock_MessagingService.assert_called_once_with( constants.SCHEDULER_MAIN_MESSAGING_TOPIC, [mock_SchedulerServerEndpoint.return_value], rpc_server.VERSION, - worker_count=mock_conf.scheduler.worker_count) + worker_count=mock_conf.scheduler.worker_count, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/cmd/test_worker.py b/coriolis/tests/cmd/test_worker.py index 0bb0d54f..8e4efcf6 100644 --- a/coriolis/tests/cmd/test_worker.py +++ b/coriolis/tests/cmd/test_worker.py @@ -4,11 +4,9 @@ import sys from unittest import mock +from coriolis import constants, service, utils from coriolis.cmd import worker -from coriolis import constants -from coriolis import service from coriolis.tests import test_base -from coriolis import utils from coriolis.worker.rpc import server as rpc_server @@ -22,8 +20,7 @@ class WorkerTestCase(test_base.CoriolisBaseTestCase): @mock.patch('coriolis.cmd.worker.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main( self, mock_gmr_setup, @@ -33,7 +30,7 @@ def test_main( mock_setup_logging, mock_MessagingService, mock_WorkerServerEndpoint, - mock_service + mock_service, ): worker_count = mock.sentinel.worker_count args = ['mock_arg_1', 'mock_arg_2'] @@ -43,19 +40,23 @@ def test_main( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_MessagingService.assert_called_once_with( constants.WORKER_MAIN_MESSAGING_TOPIC, [mock_WorkerServerEndpoint.return_value], - rpc_server.VERSION, worker_count=worker_count, init_rpc=False) + rpc_server.VERSION, + worker_count=worker_count, + init_rpc=False, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) @mock.patch.object(service, 'service') @mock.patch.object(rpc_server, 'WorkerServerEndpoint') @@ -64,8 +65,7 @@ def test_main( @mock.patch('coriolis.cmd.worker.CONF') @mock.patch.object(service, 'get_worker_count_from_args') @mock.patch.object(sys, 'argv') - @mock.patch( - 'oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') + @mock.patch('oslo_reports.guru_meditation_report.TextGuruMeditation.setup_autorun') def test_main_no_worker_count( self, mock_gmr_setup, @@ -75,7 +75,7 @@ def test_main_no_worker_count( mock_setup_logging, mock_MessagingService, mock_WorkerServerEndpoint, - mock_service + mock_service, ): worker_count = None args = ['mock_arg_1', 'mock_arg_2'] @@ -85,17 +85,20 @@ def test_main_no_worker_count( mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( - ['mock_arg_2'], project='coriolis', version="1.0.0") + ['mock_arg_2'], project='coriolis', version="1.0.0" + ) mock_setup_logging.assert_called_once() mock_MessagingService.assert_called_once_with( constants.WORKER_MAIN_MESSAGING_TOPIC, [mock_WorkerServerEndpoint.return_value], - rpc_server.VERSION, worker_count=mock_conf.worker.worker_count, - init_rpc=False) + rpc_server.VERSION, + worker_count=mock_conf.worker.worker_count, + init_rpc=False, + ) mock_service.launch.assert_called_once_with( - mock_conf, mock_MessagingService.return_value, - workers=mock_MessagingService.return_value. - get_workers_count.return_value) + mock_conf, + mock_MessagingService.return_value, + workers=mock_MessagingService.return_value.get_workers_count.return_value, + ) mock_service.launch.return_value.wait.assert_called_once() - mock_gmr_setup.assert_called_once_with( - version="1.0.0", conf=mock_conf) + mock_gmr_setup.assert_called_once_with(version="1.0.0", conf=mock_conf) diff --git a/coriolis/tests/conductor/rpc/test_client.py b/coriolis/tests/conductor/rpc/test_client.py index ddd3686c..d18f2f12 100644 --- a/coriolis/tests/conductor/rpc/test_client.py +++ b/coriolis/tests/conductor/rpc/test_client.py @@ -3,8 +3,8 @@ from unittest import mock -from coriolis.conductor.rpc import client from coriolis import constants +from coriolis.conductor.rpc import client from coriolis.tests import test_base INSTANCE_ARGS = { @@ -23,23 +23,20 @@ "instances": ['mock_instance_1', 'mock_instance_2'], "notes": "mock_notes", "network_map": {'mock_network_key': 'mock_network_value'}, - "storage_mappings": { - 'mock_destination_key': 'mock_destination_value' - }, + "storage_mappings": {'mock_destination_key': 'mock_destination_value'}, "source_environment": {'mock_source_key': 'mock_source_value'}, "user_scripts": {'mock_scripts_key': 'mock_scripts_value'}, } class ConductorClientTestCase(test_base.CoriolisRPCClientTestCase): - def setUp(self): super(ConductorClientTestCase, self).setUp() self.client = client.ConductorClient() self._mock_pagination_args = dict( - marker="mock_marker", limit=5, - sort_keys=["mock_column"], sort_dirs=["desc"]) + marker="mock_marker", limit=5, sort_keys=["mock_column"], sort_dirs=["desc"] + ) def test_create_endpoint(self): args = { @@ -47,7 +44,7 @@ def test_create_endpoint(self): "endpoint_type": "mock_endpoint_type", "description": "mock_description", "connection_info": "mock_connection_info", - "mapped_regions": "mock_mapped_regions" + "mapped_regions": "mock_mapped_regions", } self._test(self.client.create_endpoint, args) @@ -81,7 +78,7 @@ def test_get_endpoint_instances(self): "marker": None, "limit": None, "instance_name_pattern": None, - "refresh": False + "refresh": False, } self._test(self.client.get_endpoint_instances, args) @@ -89,7 +86,7 @@ def test_get_endpoint_instance(self): args = { "endpoint_id": "mock_endpoint_id", "source_environment": "mock_source_environment", - "instance_name": "mock_instance_name" + "instance_name": "mock_instance_name", } self._test(self.client.get_endpoint_instance, args) @@ -97,7 +94,7 @@ def test_get_endpoint_source_options(self): args = { "endpoint_id": "mock_endpoint_id", "env": "mock_env", - "option_names": "mock_option_names" + "option_names": "mock_option_names", } self._test(self.client.get_endpoint_source_options, args) @@ -105,42 +102,28 @@ def test_get_endpoint_destination_options(self): args = { "endpoint_id": "mock_endpoint_id", "env": "mock_env", - "option_names": "mock_option_names" + "option_names": "mock_option_names", } self._test(self.client.get_endpoint_destination_options, args) def test_get_endpoint_networks(self): - args = { - "endpoint_id": "mock_endpoint_id", - "env": "mock_env" - } + args = {"endpoint_id": "mock_endpoint_id", "env": "mock_env"} self._test(self.client.get_endpoint_networks, args) def test_get_endpoint_storage(self): - args = { - "endpoint_id": "mock_endpoint_id", - "env": "mock_env" - } + args = {"endpoint_id": "mock_endpoint_id", "env": "mock_env"} self._test(self.client.get_endpoint_storage, args) def test_validate_endpoint_connection(self): - args = { - "endpoint_id": "mock_endpoint_id" - } + args = {"endpoint_id": "mock_endpoint_id"} self._test(self.client.validate_endpoint_connection, args) def test_validate_endpoint_target_environment(self): - args = { - "endpoint_id": "mock_endpoint_id", - "target_env": "mock_target_env" - } + args = {"endpoint_id": "mock_endpoint_id", "target_env": "mock_target_env"} self._test(self.client.validate_endpoint_target_environment, args) def test_validate_endpoint_source_environment(self): - args = { - "endpoint_id": "mock_endpoint_id", - "source_env": "mock_source_env" - } + args = {"endpoint_id": "mock_endpoint_id", "source_env": "mock_source_env"} self._test(self.client.validate_endpoint_source_environment, args) def test_get_available_providers(self): @@ -150,7 +133,7 @@ def test_get_available_providers(self): def test_get_provider_schemas(self): args = { "platform_name": "mock_platform_name", - "provider_type": "mock_provider_type" + "provider_type": "mock_provider_type", } self._test(self.client.get_provider_schemas, args) @@ -174,22 +157,19 @@ def test_get_transfer_tasks_execution(self): args = { "transfer_id": "mock_transfer_id", "execution_id": "mock_execution_id", - "include_task_info": False + "include_task_info": False, } self._test(self.client.get_transfer_tasks_execution, args) def test_delete_transfer_tasks_execution(self): - args = { - "transfer_id": "mock_transfer_id", - "execution_id": "mock_execution_id" - } + args = {"transfer_id": "mock_transfer_id", "execution_id": "mock_execution_id"} self._test(self.client.delete_transfer_tasks_execution, args) def test_cancel_transfer_tasks_execution(self): args = { "transfer_id": "mock_transfer_id", "execution_id": "mock_execution_id", - "force": "mock_force" + "force": "mock_force", } self._test(self.client.cancel_transfer_tasks_execution, args) @@ -228,9 +208,7 @@ def test_delete_transfer(self): self._test(self.client.delete_transfer, args) def test_delete_transfer_disks(self): - args = { - "transfer_id": "mock_transfer_id" - } + args = {"transfer_id": "mock_transfer_id"} self._test(self.client.delete_transfer_disks, args) def test_deploy_transfer_instances(self): @@ -247,37 +225,28 @@ def test_deploy_transfer_instances(self): self._test(self.client.deploy_transfer_instances, args) def test_set_task_host(self): - args = { - "task_id": "mock_task_id", - "host": "mock_host" - } + args = {"task_id": "mock_task_id", "host": "mock_host"} self._test(self.client.set_task_host, args) def test_set_task_process(self): - args = { - "task_id": "mock_task_id", - "process_id": "mock_process_id" - } + args = {"task_id": "mock_task_id", "process_id": "mock_process_id"} self._test(self.client.set_task_process, args) def test_task_completed(self): - args = { - "task_id": "mock_task_id", - "task_result": "mock_task_result" - } + args = {"task_id": "mock_task_id", "task_result": "mock_task_result"} self._test(self.client.task_completed, args) def test_confirm_task_cancellation(self): args = { "task_id": "mock_task_id", - "cancellation_details": "mock_cancellation_details" + "cancellation_details": "mock_cancellation_details", } self._test(self.client.confirm_task_cancellation, args) def test_set_task_error(self): args = { "task_id": "mock_task_id", - "exception_details": "mock_exception_details" + "exception_details": "mock_exception_details", } self._test(self.client.set_task_error, args) @@ -285,7 +254,7 @@ def test_add_task_event(self): args = { "task_id": "mock_task_id", "level": "mock_level", - "message": "mock_message" + "message": "mock_message", } self._test(self.client.add_task_event, args, rpc_op='_cast') @@ -295,10 +264,9 @@ def test_update_task_progress_update(self): "progress_update_index": "mock_progress_update_index", "new_current_step": "mock_new_current_step", "new_total_steps": None, - "new_message": None + "new_message": None, } - self._test( - self.client.update_task_progress_update, args, rpc_op='_cast') + self._test(self.client.update_task_progress_update, args, rpc_op='_cast') def test_update_task_progress_update_sync_final(self): args = { @@ -310,10 +278,8 @@ def test_update_task_progress_update_sync_final(self): } with mock.patch.object(self.client, '_call') as op_mock: ctxt = mock.sentinel.ctxt - self.client.update_task_progress_update( - ctxt, sync=True, **args) - op_mock.assert_called_once_with( - ctxt, 'update_task_progress_update', **args) + self.client.update_task_progress_update(ctxt, sync=True, **args) + op_mock.assert_called_once_with(ctxt, 'update_task_progress_update', **args) def test_create_transfer_schedule(self): args = { @@ -330,36 +296,30 @@ def test_update_transfer_schedule(self): args = { "transfer_id": "mock_transfer_id", "schedule_id": "mock_schedule_id", - "updated_values": "mock_updated_values" + "updated_values": "mock_updated_values", } self._test(self.client.update_transfer_schedule, args) def test_delete_transfer_schedule(self): - args = { - "transfer_id": "mock_transfer_id", - "schedule_id": "mock_schedule_id" - } + args = {"transfer_id": "mock_transfer_id", "schedule_id": "mock_schedule_id"} self._test(self.client.delete_transfer_schedule, args) def test_get_transfer_schedules(self): - args = { - "transfer_id": None, - "expired": True - } + args = {"transfer_id": None, "expired": True} self._test(self.client.get_transfer_schedules, args) def test_get_transfer_schedule(self): args = { "transfer_id": "mock_transfer_id", "schedule_id": "mock_schedule_id", - "expired": True + "expired": True, } self._test(self.client.get_transfer_schedule, args) def test_update_transfer(self): args = { "transfer_id": "mock_transfer_id", - "updated_properties": "mock_updated_properties" + "updated_properties": "mock_updated_properties", } self._test(self.client.update_transfer, args) @@ -373,7 +333,7 @@ def test_create_region(self): args = { "region_name": "mock_region_name", "description": "mock_description", - "enabled": True + "enabled": True, } self._test(self.client.create_region, args) @@ -381,22 +341,15 @@ def test_get_regions(self): self._test(self.client.get_regions, args={}) def test_get_region(self): - args = { - "region_id": "mock_region_id" - } + args = {"region_id": "mock_region_id"} self._test(self.client.get_region, args) def test_update_region(self): - args = { - "region_id": "mock_region_id", - "updated_values": "mock_updated_values" - } + args = {"region_id": "mock_region_id", "updated_values": "mock_updated_values"} self._test(self.client.update_region, args) def test_delete_region(self): - args = { - "region_id": "mock_region_id" - } + args = {"region_id": "mock_region_id"} self._test(self.client.delete_region, args) def test_register_service(self): @@ -407,113 +360,95 @@ def test_register_service(self): "enabled": "mock_enabled", "mapped_regions": "mock_mapped_regions", "providers": None, - "specs": None + "specs": None, } self._test(self.client.register_service, args) def test_check_service_registered(self): - args = { - "host": "mock_host", - "binary": "mock_binary", - "topic": "mock_topic" - } + args = {"host": "mock_host", "binary": "mock_binary", "topic": "mock_topic"} self._test(self.client.check_service_registered, args) def test_refresh_service_status(self): - args = { - "service_id": "mock_service_id" - } + args = {"service_id": "mock_service_id"} self._test(self.client.refresh_service_status, args) def test_get_services(self): self._test(self.client.get_services, args={}) def test_get_service(self): - args = { - "service_id": "mock_service_id" - } + args = {"service_id": "mock_service_id"} self._test(self.client.get_service, args) def test_update_service(self): args = { "service_id": "mock_service_id", - "updated_values": "mock_updated_values" + "updated_values": "mock_updated_values", } self._test(self.client.update_service, args) def test_delete_service(self): - args = { - "service_id": "mock_service_id" - } + args = {"service_id": "mock_service_id"} self._test(self.client.delete_service, args) def test_confirm_transfer_minions_allocation(self): args = { "transfer_id": "mock_transfer_id", - "minion_machine_allocations": "mock_minion_machine_allocations" + "minion_machine_allocations": "mock_minion_machine_allocations", } self._test(self.client.confirm_transfer_minions_allocation, args) def test_report_transfer_minions_allocation_error(self): args = { "transfer_id": "mock_transfer_id", - "minion_allocation_error_details": - "mock_minion_allocation_error_details" + "minion_allocation_error_details": "mock_minion_allocation_error_details", } self._test(self.client.report_transfer_minions_allocation_error, args) def test_confirm_deployment_minions_allocation(self): args = { "deployment_id": "mock_deployment_id", - "minion_machine_allocations": "mock_minion_machine_allocations" + "minion_machine_allocations": "mock_minion_machine_allocations", } self._test(self.client.confirm_deployment_minions_allocation, args) def test_report_deployment_minions_allocation_error(self): args = { "deployment_id": "mock_deployment_id", - "minion_allocation_error_details": - "mock_minion_allocation_error_details" + "minion_allocation_error_details": "mock_minion_allocation_error_details", } - self._test( - self.client.report_deployment_minions_allocation_error, args) + self._test(self.client.report_deployment_minions_allocation_error, args) def test_add_task_progress_update(self): args = { "task_id": "mock_task_id", "message": "mock_message", "initial_step": "mock_message", - "total_steps": "mock_message" + "total_steps": "mock_message", } ctxt = mock.sentinel.ctxt with mock.patch.object(self.client, '_cast') as op_mock: - self.client.add_task_progress_update( - ctxt, return_event=False, **args) - op_mock.assert_called_once_with( - ctxt, 'add_task_progress_update', **args) + self.client.add_task_progress_update(ctxt, return_event=False, **args) + op_mock.assert_called_once_with(ctxt, 'add_task_progress_update', **args) with mock.patch.object(self.client, '_call') as op_mock: - self.client.add_task_progress_update( - ctxt, return_event=True, **args) - op_mock.assert_called_once_with( - ctxt, 'add_task_progress_update', **args) + self.client.add_task_progress_update(ctxt, return_event=True, **args) + op_mock.assert_called_once_with(ctxt, 'add_task_progress_update', **args) class ConductorTaskRpcEventHandlerTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(ConductorTaskRpcEventHandlerTestCase, self).setUp() self.ctxt = mock.Mock() self.task_id = mock.sentinel.task_id - self.client = client.ConductorTaskRpcEventHandler( - self.ctxt, self.task_id) + self.client = client.ConductorTaskRpcEventHandler(self.ctxt, self.task_id) def test_get_progress_update_identifier(self): self.assertEqual( mock.sentinel.index, client.ConductorTaskRpcEventHandler.get_progress_update_identifier( - {'index': mock.sentinel.index}) + {'index': mock.sentinel.index} + ), ) @mock.patch.object(client.ConductorClient, 'add_task_progress_update') @@ -523,19 +458,17 @@ def test_add_progress_update(self, mock_add_task_progress_update): total_steps = mock.sentinel.total_steps return_event = False result = self.client.add_progress_update( - message, initial_step, total_steps, return_event) - - self.assertEqual( - mock_add_task_progress_update.return_value, - result + message, initial_step, total_steps, return_event ) + + self.assertEqual(mock_add_task_progress_update.return_value, result) mock_add_task_progress_update.assert_called_once_with( self.ctxt, self.task_id, message, initial_step=initial_step, total_steps=total_steps, - return_event=return_event + return_event=return_event, ) @mock.patch.object(client.ConductorClient, 'update_task_progress_update') @@ -546,7 +479,8 @@ def test_update_progress_update(self, mock_update_task_progress_update): new_message = None self.client.update_progress_update( - update_identifier, new_current_step, new_total_steps, new_message) + update_identifier, new_current_step, new_total_steps, new_message + ) mock_update_task_progress_update.assert_called_once_with( self.ctxt, @@ -566,8 +500,5 @@ def test_add_event(self, mock_add_task_event): self.client.add_event(message, level) mock_add_task_event.assert_called_once_with( - self.ctxt, - self.task_id, - level, - message + self.ctxt, self.task_id, level, message ) diff --git a/coriolis/tests/conductor/rpc/test_server.py b/coriolis/tests/conductor/rpc/test_server.py index 096cd33a..54b6f90e 100644 --- a/coriolis/tests/conductor/rpc/test_server.py +++ b/coriolis/tests/conductor/rpc/test_server.py @@ -3,24 +3,18 @@ import copy import logging -from unittest import mock import uuid +from unittest import mock import ddt from oslo_concurrency import lockutils from oslo_config import cfg +from coriolis import constants, context, exception, keystone, schemas, utils from coriolis.conductor.rpc import server -from coriolis import constants -from coriolis import context from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models -from coriolis import exception -from coriolis import keystone -from coriolis import schemas -from coriolis.tests import test_base -from coriolis.tests import testutils -from coriolis import utils +from coriolis.tests import test_base, testutils from coriolis.worker.rpc import client as rpc_worker_client @@ -38,20 +32,17 @@ def setUp(self): self._licensing_client = mock.Mock() self.server._licensing_client = self._licensing_client self._mock_pagination_args = dict( - marker="mock_marker", limit=5, - sort_keys=["mock_column"], sort_dirs=["desc"]) + marker="mock_marker", limit=5, sort_keys=["mock_column"], sort_dirs=["desc"] + ) - @mock.patch.object( - rpc_worker_client.WorkerClient, "from_service_definition" - ) + @mock.patch.object(rpc_worker_client.WorkerClient, "from_service_definition") @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") def test_get_all_diagnostics( - self, - mock_scheduler_client, - mock_from_service_definition + self, mock_scheduler_client, mock_from_service_definition ): mock_scheduler_client.get_workers_for_specs.side_effect = ( - CoriolisTestException()) + CoriolisTestException() + ) self.assertRaises( CoriolisTestException, lambda: self.server.get_all_diagnostics(mock.sentinel.context), @@ -59,44 +50,38 @@ def test_get_all_diagnostics( mock_scheduler_client.get_workers_for_specs.side_effect = None diagnostics = self.server.get_all_diagnostics(mock.sentinel.context) - assert ( - mock_scheduler_client.get_diagnostics.return_value in diagnostics - ) + assert mock_scheduler_client.get_diagnostics.return_value in diagnostics mock_scheduler_client.get_workers_for_specs.return_value = { mock.sentinel.diagnostics } diagnostics = self.server.get_all_diagnostics(mock.sentinel.context) - assert ( - mock_scheduler_client.get_diagnostics.return_value in diagnostics - ) - mock_scheduler_client.get_workers_for_specs.return_value = [{ - 'host': mock.sentinel.host - }] + assert mock_scheduler_client.get_diagnostics.return_value in diagnostics + mock_scheduler_client.get_workers_for_specs.return_value = [ + {'host': mock.sentinel.host} + ] mock_from_service_definition.side_effect = ( - mock.sentinel.rpc_worker, CoriolisTestException()) + mock.sentinel.rpc_worker, + CoriolisTestException(), + ) self.assertRaises( CoriolisTestException, lambda: self.server.get_all_diagnostics(mock.sentinel.context), ) - @mock.patch.object( - rpc_worker_client.WorkerClient, "from_service_definition" - ) + @mock.patch.object(rpc_worker_client.WorkerClient, "from_service_definition") @mock.patch.object(db_api, "get_service") @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") def test_get_worker_service_rpc_for_specs( - self, - mock_scheduler_client, - mock_get_service, - mock_from_service_definition, + self, + mock_scheduler_client, + mock_get_service, + mock_from_service_definition, ): # returns dictionary with id and rpc mock_scheduler_client.get_worker_service_for_specs.return_value = { "id": mock.sentinel.worker_id } - result = self.server._get_worker_service_rpc_for_specs( - mock.sentinel.context - ) + result = self.server._get_worker_service_rpc_for_specs(mock.sentinel.context) worker_service = mock_scheduler_client.get_worker_service_for_specs worker_service.assert_called_once_with( @@ -123,16 +108,13 @@ def test_check_delete_reservation_for_transfer(self): def test_check_delete_reservation_for_transfer_no_licensing_client(self): transfer_action = mock.Mock() self.server._licensing_client = None - with self.assertLogs( - 'coriolis.conductor.rpc.server', level=logging.WARNING): + with self.assertLogs('coriolis.conductor.rpc.server', level=logging.WARNING): self.server._check_delete_reservation_for_transfer(transfer_action) def test_check_delete_reservation_for_transfer_delete_fails(self): transfer_action = mock.Mock() - self._licensing_client.delete_reservation.side_effect = \ - CoriolisTestException() - with self.assertLogs( - 'coriolis.conductor.rpc.server', level=logging.WARNING): + self._licensing_client.delete_reservation.side_effect = CoriolisTestException() + with self.assertLogs('coriolis.conductor.rpc.server', level=logging.WARNING): self.server._check_delete_reservation_for_transfer(transfer_action) self._licensing_client.delete_reservation.assert_called_once_with( transfer_action.reservation_id @@ -145,13 +127,13 @@ def test_check_delete_reservation_for_transfer_delete_fails(self): @mock.patch.object(models, "Endpoint") @mock.patch.object(uuid, "uuid4") def test_create_endpoint( - self, - mock_uuid4, - mock_endpoint_model, - mock_add_endpoint, - mock_update_endpoint, - mock_delete_endpoint, - mock_get_endpoint, + self, + mock_uuid4, + mock_endpoint_model, + mock_add_endpoint, + mock_update_endpoint, + mock_delete_endpoint, + mock_get_endpoint, ): endpoint = self.server.create_endpoint( mock.sentinel.context, @@ -160,9 +142,7 @@ def test_create_endpoint( mock.sentinel.description, mock.sentinel.connection_info, ) - self.assertEqual( - mock_endpoint_model.return_value.name, mock.sentinel.name - ) + self.assertEqual(mock_endpoint_model.return_value.name, mock.sentinel.name) self.assertEqual( mock_endpoint_model.return_value.type, mock.sentinel.endpoint_type ) @@ -246,8 +226,9 @@ def test_get_endpoints(self, mock_get_endpoints): def test_get_endpoint(self, mock_get_endpoint): def call_get_endpoint(): return testutils.get_wrapped_function(self.server.get_endpoint)( - self, mock.sentinel.context, - mock.sentinel.endpoint_id # type: ignore + self, + mock.sentinel.context, + mock.sentinel.endpoint_id, # type: ignore ) endpoint = call_get_endpoint() @@ -264,8 +245,9 @@ def call_get_endpoint(): def test_get_endpoint_not_found(self, mock_get_endpoint): def call_get_endpoint(): return testutils.get_wrapped_function(self.server.get_endpoint)( - self, mock.sentinel.context, - mock.sentinel.endpoint_id # type: ignore + self, + mock.sentinel.context, + mock.sentinel.endpoint_id, # type: ignore ) mock_get_endpoint.return_value = None @@ -274,12 +256,13 @@ def call_get_endpoint(): @mock.patch.object(db_api, "delete_endpoint") @mock.patch.object(db_api, "get_endpoint_transfers_count") def test_delete_endpoint( - self, mock_get_endpoint_transfers_count, mock_delete_endpoint + self, mock_get_endpoint_transfers_count, mock_delete_endpoint ): def call_delete_endpoint(): return testutils.get_wrapped_function(self.server.delete_endpoint)( - self, mock.sentinel.context, - mock.sentinel.endpoint_id # type: ignore + self, + mock.sentinel.context, + mock.sentinel.endpoint_id, # type: ignore ) mock_get_endpoint_transfers_count.return_value = 0 @@ -297,7 +280,7 @@ def call_delete_endpoint(): ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_instances( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): instances = self.server.get_endpoint_instances( mock.sentinel.context, @@ -333,7 +316,7 @@ def test_get_endpoint_instances( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_instance( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): instance = self.server.get_endpoint_instance( mock.sentinel.context, @@ -358,16 +341,14 @@ def test_get_endpoint_instance( ) rpc_return_value = mock_get_worker_service_rpc_for_specs.return_value - self.assertEqual( - instance, rpc_return_value.get_endpoint_instance.return_value - ) + self.assertEqual(instance, rpc_return_value.get_endpoint_instance.return_value) @mock.patch.object( server.ConductorServerEndpoint, "_get_worker_service_rpc_for_specs" ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_source_options( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): options = self.server.get_endpoint_source_options( mock.sentinel.context, @@ -409,7 +390,7 @@ def test_get_endpoint_source_options( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_destination_options( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): options = self.server.get_endpoint_destination_options( mock.sentinel.context, @@ -434,14 +415,13 @@ def test_get_endpoint_destination_options( ) rpc_return_value = mock_get_worker_service_rpc_for_specs.return_value - rpc_return_value.get_endpoint_destination_options\ - .assert_called_once_with( - mock.sentinel.context, - mock_get_endpoint.return_value.type, - mock_get_endpoint.return_value.connection_info, - mock.sentinel.environment, - mock.sentinel.option_names, - ) + rpc_return_value.get_endpoint_destination_options.assert_called_once_with( + mock.sentinel.context, + mock_get_endpoint.return_value.type, + mock_get_endpoint.return_value.connection_info, + mock.sentinel.environment, + mock.sentinel.option_names, + ) self.assertEqual( options, @@ -453,7 +433,7 @@ def test_get_endpoint_destination_options( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_networks( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): networks = self.server.get_endpoint_networks( mock.sentinel.context, @@ -484,16 +464,14 @@ def test_get_endpoint_networks( mock.sentinel.environment, ) - self.assertEqual( - networks, rpc_return_value.get_endpoint_networks.return_value - ) + self.assertEqual(networks, rpc_return_value.get_endpoint_networks.return_value) @mock.patch.object( server.ConductorServerEndpoint, "_get_worker_service_rpc_for_specs" ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_storage_pools( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): storage_pools = self.server.get_endpoint_storage( mock.sentinel.context, @@ -533,7 +511,7 @@ def test_get_endpoint_storage_pools( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_endpoint_inventory_csv( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): result = self.server.get_endpoint_inventory_csv( mock.sentinel.context, @@ -565,8 +543,7 @@ def test_get_endpoint_inventory_csv( ) self.assertEqual( - result, - rpc_return_value.get_endpoint_inventory_csv.return_value + result, rpc_return_value.get_endpoint_inventory_csv.return_value ) @mock.patch.object( @@ -574,7 +551,7 @@ def test_get_endpoint_inventory_csv( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_validate_endpoint_connection( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): self.server.validate_endpoint_connection( mock.sentinel.context, mock.sentinel.endpoint_id @@ -589,9 +566,7 @@ def test_validate_endpoint_connection( enabled=True, region_sets=[[]], provider_requirements={ - mock_get_endpoint.return_value.type: [ - constants.PROVIDER_TYPE_ENDPOINT - ] + mock_get_endpoint.return_value.type: [constants.PROVIDER_TYPE_ENDPOINT] }, ) @@ -607,7 +582,7 @@ def test_validate_endpoint_connection( ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_validate_endpoint_target_environment( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): self.server.validate_endpoint_target_environment( mock.sentinel.context, @@ -624,25 +599,22 @@ def test_validate_endpoint_target_environment( enabled=True, region_sets=[[]], provider_requirements={ - mock_get_endpoint.return_value.type: [ - constants.PROVIDER_TYPE_ENDPOINT - ] + mock_get_endpoint.return_value.type: [constants.PROVIDER_TYPE_ENDPOINT] }, ) - mock_get_worker_service_rpc_for_specs.return_value\ - .validate_endpoint_target_environment.assert_called_once_with( - mock.sentinel.context, - mock_get_endpoint.return_value.type, - mock.sentinel.target_env, - ) + mock_get_worker_service_rpc_for_specs.return_value.validate_endpoint_target_environment.assert_called_once_with( + mock.sentinel.context, + mock_get_endpoint.return_value.type, + mock.sentinel.target_env, + ) @mock.patch.object( server.ConductorServerEndpoint, "_get_worker_service_rpc_for_specs" ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_validate_endpoint_source_environment( - self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs + self, mock_get_endpoint, mock_get_worker_service_rpc_for_specs ): self.server.validate_endpoint_source_environment( mock.sentinel.context, @@ -659,72 +631,57 @@ def test_validate_endpoint_source_environment( enabled=True, region_sets=[[]], provider_requirements={ - mock_get_endpoint.return_value.type: [ - constants.PROVIDER_TYPE_ENDPOINT - ] + mock_get_endpoint.return_value.type: [constants.PROVIDER_TYPE_ENDPOINT] }, ) - mock_get_worker_service_rpc_for_specs.return_value\ - .validate_endpoint_source_environment.assert_called_once_with( - mock.sentinel.context, - mock_get_endpoint.return_value.type, - mock.sentinel.source_env, - ) + mock_get_worker_service_rpc_for_specs.return_value.validate_endpoint_source_environment.assert_called_once_with( + mock.sentinel.context, + mock_get_endpoint.return_value.type, + mock.sentinel.source_env, + ) - @mock.patch.object( - rpc_worker_client.WorkerClient, "from_service_definition" - ) + @mock.patch.object(rpc_worker_client.WorkerClient, "from_service_definition") @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") def test_get_available_providers( - self, mock_scheduler_client, mock_service_definition + self, mock_scheduler_client, mock_service_definition ): providers = self.server.get_available_providers(mock.sentinel.context) mock_service_definition.assert_called_once_with( - mock_scheduler_client.get_any_worker_service( - mock.sentinel.context)) - mock_service_definition.return_value\ - .get_available_providers.assert_called_once_with( - mock.sentinel.context - ) + mock_scheduler_client.get_any_worker_service(mock.sentinel.context) + ) + mock_service_definition.return_value.get_available_providers.assert_called_once_with( + mock.sentinel.context + ) self.assertEqual( providers, - mock_service_definition - .return_value.get_available_providers.return_value, + mock_service_definition.return_value.get_available_providers.return_value, ) @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") - @mock.patch.object( - rpc_worker_client.WorkerClient, "from_service_definition" - ) - def test_get_provider_schemas( - self, mock_service_definition, mock_scheduler_client - ): + @mock.patch.object(rpc_worker_client.WorkerClient, "from_service_definition") + def test_get_provider_schemas(self, mock_service_definition, mock_scheduler_client): provider_schemas = self.server.get_provider_schemas( mock.sentinel.context, mock.sentinel.platform_name, mock.sentinel.provider_type, ) mock_service_definition.assert_called_once_with( - mock_scheduler_client.get_any_worker_service( - mock.sentinel.context)) - mock_service_definition.return_value\ - .get_provider_schemas.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.platform_name, - mock.sentinel.provider_type, - ) + mock_scheduler_client.get_any_worker_service(mock.sentinel.context) + ) + mock_service_definition.return_value.get_provider_schemas.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.platform_name, + mock.sentinel.provider_type, + ) self.assertEqual( provider_schemas, - mock_service_definition.return_value - .get_provider_schemas.return_value, + mock_service_definition.return_value.get_provider_schemas.return_value, ) @mock.patch.object(models, "Task") @mock.patch.object(uuid, "uuid4", return_value="task_id") - def test_create_task( - self, mock_uuid4, mock_task_model - ): + def test_create_task(self, mock_uuid4, mock_task_model): task1 = mock.sentinel.task1 task1.id = mock.sentinel.task1_id task2 = mock.sentinel.task2 @@ -794,21 +751,16 @@ def test_get_task_origin(self, mock_get_endpoint): expected_result = { "connection_info": mock.sentinel.connection_info, "type": mock.sentinel.type, - "source_environment": mock.sentinel.source_environment + "source_environment": mock.sentinel.source_environment, } - result = self.server._get_task_origin( - mock.sentinel.context, - action - ) + result = self.server._get_task_origin(mock.sentinel.context, action) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_get_endpoint.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin_endpoint_id) + mock.sentinel.context, mock.sentinel.origin_endpoint_id + ) @mock.patch.object(server.ConductorServerEndpoint, "get_endpoint") def test_get_task_destination(self, mock_get_endpoint): @@ -823,32 +775,25 @@ def test_get_task_destination(self, mock_get_endpoint): expected_result = { "connection_info": mock.sentinel.connection_info, "type": mock.sentinel.type, - "target_environment": mock.sentinel.destination_environment + "target_environment": mock.sentinel.destination_environment, } - result = self.server._get_task_destination( - mock.sentinel.context, - action - ) + result = self.server._get_task_destination(mock.sentinel.context, action) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) mock_get_endpoint.assert_called_once_with( - mock.sentinel.context, mock.sentinel.destination_endpoint_id) + mock.sentinel.context, mock.sentinel.destination_endpoint_id + ) - @mock.patch.object( - rpc_worker_client.WorkerClient, "from_service_definition" - ) + @mock.patch.object(rpc_worker_client.WorkerClient, "from_service_definition") @mock.patch.object(db_api, "set_task_status") @mock.patch.object(server.ConductorServerEndpoint, "_scheduler_client") def test_get_worker_service_rpc_for_task( - self, - mock_scheduler_client, - mock_set_task_status, - mock_service_definition, + self, + mock_scheduler_client, + mock_set_task_status, + mock_service_definition, ): task_mock = mock.Mock() service = self.server._get_worker_service_rpc_for_task( @@ -857,16 +802,15 @@ def test_get_worker_service_rpc_for_task( mock.sentinel.origin_endpoint, mock.sentinel.destination_endpoint, ) - mock_scheduler_client.get_worker_service_for_task\ - .assert_called_once_with( - mock.sentinel.context, - {"id": task_mock.id, "task_type": task_mock.task_type}, - mock.sentinel.origin_endpoint, - mock.sentinel.destination_endpoint, - retry_count=5, - retry_period=2, - random_choice=True, - ) + mock_scheduler_client.get_worker_service_for_task.assert_called_once_with( + mock.sentinel.context, + {"id": task_mock.id, "task_type": task_mock.task_type}, + mock.sentinel.origin_endpoint, + mock.sentinel.destination_endpoint, + retry_count=5, + retry_period=2, + random_choice=True, + ) mock_service_definition.assert_called_once_with( mock_scheduler_client.get_worker_service_for_task.return_value ) @@ -892,10 +836,10 @@ def test_get_worker_service_rpc_for_task( exception_details="test", ) - @mock.patch.object(server.ConductorServerEndpoint, - "_set_tasks_execution_status") - @mock.patch.object(server.ConductorServerEndpoint, - "_get_worker_service_rpc_for_task") + @mock.patch.object(server.ConductorServerEndpoint, "_set_tasks_execution_status") + @mock.patch.object( + server.ConductorServerEndpoint, "_get_worker_service_rpc_for_task" + ) @mock.patch.object(db_api, "set_task_status") @mock.patch.object(db_api, "get_endpoint") @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") @@ -919,7 +863,8 @@ def test_begin_tasks( mock_context.trust_id = None mock_get_endpoint.side_effect = [ mock.sentinel.origin_endpoint_id, - mock.sentinel.destination_endpoint_id] + mock.sentinel.destination_endpoint_id, + ] task = mock.Mock( id=mock.sentinel.task_id, @@ -929,10 +874,7 @@ def test_begin_tasks( depends_on=None, on_error=None, ) - execution = mock.Mock( - id=mock.sentinel.execution_id, - tasks=[task] - ) + execution = mock.Mock(id=mock.sentinel.execution_id, tasks=[task]) self.server._begin_tasks( mock_context, @@ -945,40 +887,43 @@ def test_begin_tasks( mock_create_trust.assert_called_once_with(mock_context) mock_get_task_origin.assert_called_once_with(mock_context, mock_action) - mock_get_task_destination.assert_called_once_with( - mock_context, mock_action) - mock_get_endpoint.assert_has_calls([ - mock.call(mock_context, mock_action.origin_endpoint_id), - mock.call(mock_context, mock_action.destination_endpoint_id) - ]) + mock_get_task_destination.assert_called_once_with(mock_context, mock_action) + mock_get_endpoint.assert_has_calls( + [ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id), + ] + ) mock_set_task_status.assert_called_once_with( - mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING) + mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING + ) mock_get_worker_service_rpc_for_task.assert_called_once_with( mock_context, task, mock.sentinel.origin_endpoint_id, mock.sentinel.destination_endpoint_id, retry_count=mock.sentinel.scheduling_retry_count, - retry_period=mock.sentinel.scheduling_retry_period + retry_period=mock.sentinel.scheduling_retry_period, ) - (mock_get_worker_service_rpc_for_task.return_value.begin_task. - assert_called_once_with)( + ( + mock_get_worker_service_rpc_for_task.return_value.begin_task.assert_called_once_with + )( mock_context, task_id=mock.sentinel.task_id, task_type=task.task_type, origin=mock_get_task_origin.return_value, destination=mock_get_task_destination.return_value, instance=task.instance, - task_info=expected_task_instance + task_info=expected_task_instance, ) mock_set_tasks_execution_status.assert_called_once_with( mock_context, execution, constants.TASK_STATUS_RUNNING ) - @mock.patch.object(server.ConductorServerEndpoint, - "_cancel_tasks_execution") - @mock.patch.object(server.ConductorServerEndpoint, - "_get_worker_service_rpc_for_task") + @mock.patch.object(server.ConductorServerEndpoint, "_cancel_tasks_execution") + @mock.patch.object( + server.ConductorServerEndpoint, "_get_worker_service_rpc_for_task" + ) @mock.patch.object(db_api, "set_task_status") @mock.patch.object(db_api, "get_endpoint") @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") @@ -992,7 +937,7 @@ def test_begin_tasks_begin_task_raises( mock_get_endpoint, mock_set_task_status, mock_get_worker_service_rpc_for_task, - mock_cancel_tasks_execution + mock_cancel_tasks_execution, ): mock_context = mock.Mock() task_info = {'instance_1': "mock_instance_1"} @@ -1002,7 +947,8 @@ def test_begin_tasks_begin_task_raises( mock_context.trust_id = None mock_get_endpoint.side_effect = [ mock.sentinel.origin_endpoint_id, - mock.sentinel.destination_endpoint_id] + mock.sentinel.destination_endpoint_id, + ] task = mock.Mock( id=mock.sentinel.task_id, @@ -1012,12 +958,10 @@ def test_begin_tasks_begin_task_raises( depends_on=None, on_error=None, ) - execution = mock.Mock( - id=mock.sentinel.execution_id, - tasks=[task] - ) - (mock_get_worker_service_rpc_for_task.return_value.begin_task - .side_effect) = CoriolisTestException() + execution = mock.Mock(id=mock.sentinel.execution_id, tasks=[task]) + ( + mock_get_worker_service_rpc_for_task.return_value.begin_task.side_effect + ) = CoriolisTestException() self.assertRaises( CoriolisTestException, @@ -1032,39 +976,43 @@ def test_begin_tasks_begin_task_raises( mock_create_trust.assert_called_once_with(mock_context) mock_get_task_origin.assert_called_once_with(mock_context, mock_action) - mock_get_task_destination.assert_called_once_with( - mock_context, mock_action) - mock_get_endpoint.assert_has_calls([ - mock.call(mock_context, mock_action.origin_endpoint_id), - mock.call(mock_context, mock_action.destination_endpoint_id) - ]) + mock_get_task_destination.assert_called_once_with(mock_context, mock_action) + mock_get_endpoint.assert_has_calls( + [ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id), + ] + ) mock_set_task_status.assert_called_once_with( - mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING) + mock_context, mock.sentinel.task_id, constants.TASK_STATUS_PENDING + ) mock_get_worker_service_rpc_for_task.assert_called_once_with( mock_context, task, mock.sentinel.origin_endpoint_id, mock.sentinel.destination_endpoint_id, retry_count=mock.sentinel.scheduling_retry_count, - retry_period=mock.sentinel.scheduling_retry_period + retry_period=mock.sentinel.scheduling_retry_period, ) - (mock_get_worker_service_rpc_for_task.return_value.begin_task. - assert_called_once_with)( + ( + mock_get_worker_service_rpc_for_task.return_value.begin_task.assert_called_once_with + )( mock_context, task_id=mock.sentinel.task_id, task_type=task.task_type, origin=mock_get_task_origin.return_value, destination=mock_get_task_destination.return_value, instance=task.instance, - task_info=expected_task_instance + task_info=expected_task_instance, ) mock_cancel_tasks_execution.assert_called_once_with( - mock_context, execution, requery=True) + mock_context, execution, requery=True + ) - @mock.patch.object(server.ConductorServerEndpoint, - "_cancel_tasks_execution") - @mock.patch.object(server.ConductorServerEndpoint, - "_get_worker_service_rpc_for_task") + @mock.patch.object(server.ConductorServerEndpoint, "_cancel_tasks_execution") + @mock.patch.object( + server.ConductorServerEndpoint, "_get_worker_service_rpc_for_task" + ) @mock.patch.object(db_api, "set_task_status") @mock.patch.object(db_api, "get_endpoint") @mock.patch.object(server.ConductorServerEndpoint, "_get_task_destination") @@ -1078,7 +1026,7 @@ def test_begin_tasks_no_newly_started_tasks( mock_get_endpoint, mock_set_task_status, mock_get_worker_service_rpc_for_task, - mock_cancel_tasks_execution + mock_cancel_tasks_execution, ): mock_context = mock.Mock() task_info = mock.Mock() @@ -1088,7 +1036,8 @@ def test_begin_tasks_no_newly_started_tasks( mock_context.trust_id = None mock_get_endpoint.side_effect = [ mock.sentinel.origin_endpoint_id, - mock.sentinel.destination_endpoint_id] + mock.sentinel.destination_endpoint_id, + ] task = mock.Mock( id=mock.sentinel.task_id, @@ -1097,10 +1046,7 @@ def test_begin_tasks_no_newly_started_tasks( depends_on=None, on_error=None, ) - execution = mock.Mock( - id=mock.sentinel.execution_id, - tasks=[task] - ) + execution = mock.Mock(id=mock.sentinel.execution_id, tasks=[task]) self.assertRaises( exception.InvalidActionTasksExecutionState, @@ -1115,16 +1061,18 @@ def test_begin_tasks_no_newly_started_tasks( mock_create_trust.assert_called_once_with(mock_context) mock_get_task_origin.assert_called_once_with(mock_context, mock_action) - mock_get_task_destination.assert_called_once_with( - mock_context, mock_action) - mock_get_endpoint.assert_has_calls([ - mock.call(mock_context, mock_action.origin_endpoint_id), - mock.call(mock_context, mock_action.destination_endpoint_id) - ]) + mock_get_task_destination.assert_called_once_with(mock_context, mock_action) + mock_get_endpoint.assert_has_calls( + [ + mock.call(mock_context, mock_action.origin_endpoint_id), + mock.call(mock_context, mock_action.destination_endpoint_id), + ] + ) mock_set_task_status.assert_not_called() mock_get_worker_service_rpc_for_task.assert_not_called() - (mock_get_worker_service_rpc_for_task.return_value.begin_task. - assert_not_called)() + ( + mock_get_worker_service_rpc_for_task.return_value.begin_task.assert_not_called + )() mock_cancel_tasks_execution.assert_not_called() @mock.patch.object(server.ConductorServerEndpoint, "_create_task") @@ -1133,8 +1081,7 @@ def test_begin_tasks_no_newly_started_tasks( ) @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer") def test_delete_transfer_disks_invalid_state( - self, mock_get_transfer, - mock_check_transfer_running, mock_create_task + self, mock_get_transfer, mock_check_transfer_running, mock_create_task ): mock_transfer = mock_get_transfer.return_value mock_transfer.instances = [mock.sentinel.instance] @@ -1164,13 +1111,11 @@ def test_delete_transfer_disks_invalid_state( @ddt.file_data("data/execution_tasks_config.yml") @ddt.unpack def test_check_execution_tasks_sanity( - self, tasks_config, init_task_info, expected_result + self, tasks_config, init_task_info, expected_result ): def convert_to_task(task_config): instance_task = mock.Mock() - instance_task.instance = task_config.get( - "instance", mock.sentinel.instance - ) + instance_task.instance = task_config.get("instance", mock.sentinel.instance) instance_task.id = task_config.get("id", str(uuid.uuid4())) instance_task.status = task_config.get( "status", constants.TASK_STATUS_SCHEDULED @@ -1199,9 +1144,7 @@ def convert_to_task(task_config): } if not expected_result: - self.server._check_execution_tasks_sanity( - execution, initial_task_info - ) + self.server._check_execution_tasks_sanity(execution, initial_task_info) else: exception_mappings = { "INVALID_STATE": exception.InvalidTaskState, @@ -1214,89 +1157,60 @@ def convert_to_task(task_config): exception_mappings[expected_result["type"]], expected_result.get("message", ""), ): - self.server._check_execution_tasks_sanity( - execution, initial_task_info - ) + self.server._check_execution_tasks_sanity(execution, initial_task_info) @mock.patch.object(copy, "deepcopy") - @mock.patch.object( - server.ConductorServerEndpoint, - "_get_transfer_tasks_execution" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_begin_tasks" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_set_tasks_execution_status" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_minion_manager_client" - ) + @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, "_begin_tasks") + @mock.patch.object(server.ConductorServerEndpoint, "_set_tasks_execution_status") + @mock.patch.object(server.ConductorServerEndpoint, "_minion_manager_client") @mock.patch.object(db_api, "add_transfer_tasks_execution") @mock.patch.object(db_api, "update_transfer_action_info_for_instance") - @mock.patch.object( - server.ConductorServerEndpoint, - "_check_execution_tasks_sanity" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_create_task" - ) + @mock.patch.object(server.ConductorServerEndpoint, "_check_execution_tasks_sanity") + @mock.patch.object(server.ConductorServerEndpoint, "_create_task") @mock.patch.object(uuid, "uuid4") @mock.patch.object(models, "TasksExecution") + @mock.patch.object(server.ConductorServerEndpoint, "_check_minion_pools_for_action") @mock.patch.object( - server.ConductorServerEndpoint, - "_check_minion_pools_for_action" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_check_transfer_running_executions" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_check_reservation_for_transfer" + server.ConductorServerEndpoint, "_check_transfer_running_executions" ) @mock.patch.object( - server.ConductorServerEndpoint, - "_get_transfer" + server.ConductorServerEndpoint, "_check_reservation_for_transfer" ) + @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer") @ddt.file_data("data/execute_transfer_tasks_config.yml") @ddt.unpack def test_execute_transfer_tasks( - self, - mock_get_transfer, - mock_check_reservation, - mock_check_transfer_running_executions, - mock_check_minion_pools_for_action, - mock_tasks_execution, - mock_uuid4, - mock_create_task, - mock_check_execution_tasks_sanity, - mock_update_transfer_action_info_for_instance, - mock_add_transfer_tasks_execution, - mock_minion_manager_client, - mock_set_tasks_execution_status, - mock_begin_tasks, - mock_get_transfer_tasks_execution, - mock_deepcopy, - config, - expected_tasks, + self, + mock_get_transfer, + mock_check_reservation, + mock_check_transfer_running_executions, + mock_check_minion_pools_for_action, + mock_tasks_execution, + mock_uuid4, + mock_create_task, + mock_check_execution_tasks_sanity, + mock_update_transfer_action_info_for_instance, + mock_add_transfer_tasks_execution, + mock_minion_manager_client, + mock_set_tasks_execution_status, + mock_begin_tasks, + mock_get_transfer_tasks_execution, + mock_deepcopy, + config, + expected_tasks, ): has_origin_minion_pool = config.get("origin_minion_pool", False) has_target_minion_pool = config.get("target_minion_pool", False) shutdown_instances = config.get("shutdown_instances", False) def call_execute_transfer_tasks(): - return testutils\ - .get_wrapped_function(self.server.execute_transfer_tasks)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - shutdown_instances, # type: ignore - ) + return testutils.get_wrapped_function(self.server.execute_transfer_tasks)( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + shutdown_instances, # type: ignore + ) instances = [mock.sentinel.instance1, mock.sentinel.instance2] mock_transfer = mock.Mock( @@ -1304,19 +1218,21 @@ def call_execute_transfer_tasks(): network_map=mock.sentinel.network_map, info={mock.sentinel.instance1: {'volume_info': None}}, origin_minion_pool_id=mock.sentinel.origin_minion_pool_id - if has_origin_minion_pool else None, + if has_origin_minion_pool + else None, destination_minion_pool_id=mock.sentinel.destination_minion_pool_id - if has_target_minion_pool else None, + if has_target_minion_pool + else None, ) mock_get_transfer.return_value = mock_transfer def create_task_side_effect( - instance, - task_type, - execution, - depends_on=None, - on_error=False, - on_error_only=False + instance, + task_type, + execution, + depends_on=None, + on_error=False, + on_error_only=False, ): return mock.Mock( id=task_type, @@ -1338,79 +1254,90 @@ def create_task_side_effect( ) mock_check_reservation.assert_called_once_with(mock_transfer) mock_check_transfer_running_executions.assert_called_once_with( - mock.sentinel.context, mock_transfer) + mock.sentinel.context, mock_transfer + ) mock_check_minion_pools_for_action.assert_called_once_with( - mock.sentinel.context, mock_transfer) + mock.sentinel.context, mock_transfer + ) - mock_deepcopy.assert_called_once_with( - mock_transfer.destination_environment) + mock_deepcopy.assert_called_once_with(mock_transfer.destination_environment) for instance in instances: assert instance in mock_transfer.info self.assertEqual( mock_transfer.info[instance]['source_environment'], - mock_transfer.source_environment) + mock_transfer.source_environment, + ) self.assertEqual( mock_transfer.info[instance]['target_environment'], - mock_deepcopy.return_value) + mock_deepcopy.return_value, + ) # generic tasks - mock_create_task.assert_has_calls([ - mock.call( - instance, - constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS, - mock_tasks_execution.return_value), - mock.call( - instance, - constants.TASK_TYPE_GET_INSTANCE_INFO, - mock_tasks_execution.return_value), - mock.call( - instance, - constants.TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS, - mock_tasks_execution.return_value, - depends_on=[constants.TASK_TYPE_GET_INSTANCE_INFO]), - ]) + mock_create_task.assert_has_calls( + [ + mock.call( + instance, + constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS, + mock_tasks_execution.return_value, + ), + mock.call( + instance, + constants.TASK_TYPE_GET_INSTANCE_INFO, + mock_tasks_execution.return_value, + ), + mock.call( + instance, + constants.TASK_TYPE_VALIDATE_TRANSFER_DESTINATION_INPUTS, + mock_tasks_execution.return_value, + depends_on=[constants.TASK_TYPE_GET_INSTANCE_INFO], + ), + ] + ) # tasks defined in the yaml config for task in expected_tasks: kwargs = {} if 'on_error' in task: kwargs = {'on_error': task['on_error']} - mock_create_task.assert_has_calls([ + mock_create_task.assert_has_calls( + [ + mock.call( + instance, + task['type'], + mock_tasks_execution.return_value, + depends_on=task['depends_on'], + **kwargs, + ) + ] + ) + + mock_update_transfer_action_info_for_instance.assert_has_calls( + [ mock.call( + mock.sentinel.context, + mock_transfer.id, instance, - task['type'], - mock_tasks_execution.return_value, - depends_on=task['depends_on'], - **kwargs, + mock_transfer.info[instance], ) - ]) - - mock_update_transfer_action_info_for_instance.assert_has_calls([ - mock.call( - mock.sentinel.context, - mock_transfer.id, - instance, - mock_transfer.info[instance], - ) - ]) + ] + ) mock_check_execution_tasks_sanity.assert_called_once_with( - mock_tasks_execution.return_value, - mock_transfer.info) + mock_tasks_execution.return_value, mock_transfer.info + ) mock_add_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock_tasks_execution.return_value) + mock.sentinel.context, mock_tasks_execution.return_value + ) if any([has_origin_minion_pool, has_target_minion_pool]): - mock_minion_manager_client\ - .allocate_minion_machines_for_transfer.assert_called_once_with( - mock.sentinel.context, - mock_transfer, - ) + mock_minion_manager_client.allocate_minion_machines_for_transfer.assert_called_once_with( + mock.sentinel.context, + mock_transfer, + ) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, mock_tasks_execution.return_value, @@ -1421,31 +1348,31 @@ def create_task_side_effect( mock.sentinel.context, mock_transfer, mock_tasks_execution.return_value, - delete_trust_id=True + delete_trust_id=True, ) mock_get_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock_tasks_execution.return_value.id, - to_dict=True) + to_dict=True, + ) self.assertEqual( mock_tasks_execution.return_value.status, - constants.EXECUTION_STATUS_UNEXECUTED) + constants.EXECUTION_STATUS_UNEXECUTED, + ) self.assertEqual( mock_tasks_execution.return_value.type, - constants.EXECUTION_TYPE_TRANSFER_EXECUTION) - self.assertEqual( - result, mock_get_transfer_tasks_execution.return_value) + constants.EXECUTION_TYPE_TRANSFER_EXECUTION, + ) + self.assertEqual(result, mock_get_transfer_tasks_execution.return_value) @mock.patch.object(db_api, "get_transfer_tasks_executions") - def test_get_transfer_tasks_executions( - self, - mock_get_transfer_tasks_executions - ): + def test_get_transfer_tasks_executions(self, mock_get_transfer_tasks_executions): result = testutils.get_wrapped_function( - self.server.get_transfer_tasks_executions)( + self.server.get_transfer_tasks_executions + )( self.server, mock.sentinel.context, mock.sentinel.transfer_id, @@ -1454,10 +1381,7 @@ def test_get_transfer_tasks_executions( **self._mock_pagination_args, ) - self.assertEqual( - mock_get_transfer_tasks_executions.return_value, - result - ) + self.assertEqual(mock_get_transfer_tasks_executions.return_value, result) mock_get_transfer_tasks_executions.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, @@ -1468,172 +1392,147 @@ def test_get_transfer_tasks_executions( ) @mock.patch.object(db_api, "get_transfer_tasks_execution") - def test_get_transfer_tasks_execution( - self, - mock_get_transfer_tasks_execution - ): + def test_get_transfer_tasks_execution(self, mock_get_transfer_tasks_execution): result = testutils.get_wrapped_function( - self.server.get_transfer_tasks_execution)( + self.server.get_transfer_tasks_execution + )( self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, - include_task_info=False + include_task_info=False, ) - self.assertEqual( - mock_get_transfer_tasks_execution.return_value, - result - ) + self.assertEqual(mock_get_transfer_tasks_execution.return_value, result) mock_get_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, include_task_info=False, - to_dict=True + to_dict=True, ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_transfer_tasks_execution' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer_tasks_execution') @mock.patch.object(db_api, 'delete_transfer_tasks_execution') def test_delete_transfer_tasks_execution( - self, - mock_delete_transfer_tasks_execution, - mock_get_transfer_tasks_execution + self, mock_delete_transfer_tasks_execution, mock_get_transfer_tasks_execution ): def call_delete_transfer_tasks_execution(): return testutils.get_wrapped_function( - self.server.delete_transfer_tasks_execution)( + self.server.delete_transfer_tasks_execution + )( self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, # type: ignore ) + call_delete_transfer_tasks_execution() mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id + ) mock_delete_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.execution_id + ) # raises exception if status is active mock_get_transfer_tasks_execution.return_value.status = ( - constants.EXECUTION_STATUS_RUNNING) + constants.EXECUTION_STATUS_RUNNING + ) self.assertRaises( exception.InvalidActionTasksExecutionState, - call_delete_transfer_tasks_execution) + call_delete_transfer_tasks_execution, + ) - @mock.patch.object(server.ConductorServerEndpoint, - '_get_transfer_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') def test_cancel_transfer_tasks_execution( - self, - mock_cancel_transfer_tasks_execution, - mock_get_transfer_tasks_execution + self, mock_cancel_transfer_tasks_execution, mock_get_transfer_tasks_execution ): - mock_get_transfer_tasks_execution.return_value.status = constants\ - .EXECUTION_STATUS_RUNNING - testutils.get_wrapped_function( - self.server.cancel_transfer_tasks_execution)( + mock_get_transfer_tasks_execution.return_value.status = ( + constants.EXECUTION_STATUS_RUNNING + ) + testutils.get_wrapped_function(self.server.cancel_transfer_tasks_execution)( self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, - False + False, ) mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id + ) mock_cancel_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_get_transfer_tasks_execution.return_value, - force=False) + force=False, + ) mock_get_transfer_tasks_execution.reset_mock() mock_cancel_transfer_tasks_execution.reset_mock() - mock_get_transfer_tasks_execution.return_value.status = constants\ - .EXECUTION_STATUS_CANCELLING - testutils.get_wrapped_function( - self.server.cancel_transfer_tasks_execution)( + mock_get_transfer_tasks_execution.return_value.status = ( + constants.EXECUTION_STATUS_CANCELLING + ) + testutils.get_wrapped_function(self.server.cancel_transfer_tasks_execution)( self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, - True + True, ) mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id + ) mock_cancel_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_get_transfer_tasks_execution.return_value, - force=True) + force=True, + ) - @mock.patch.object(server.ConductorServerEndpoint, - '_get_transfer_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') def test_cancel_transfer_tasks_execution_status_not_active( - self, - mock_cancel_transfer_tasks_execution, - mock_get_transfer_tasks_execution + self, mock_cancel_transfer_tasks_execution, mock_get_transfer_tasks_execution ): self.assertRaises( exception.InvalidTransferState, - testutils.get_wrapped_function( - self.server.cancel_transfer_tasks_execution), + testutils.get_wrapped_function(self.server.cancel_transfer_tasks_execution), self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, - False + False, ) mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id + ) mock_cancel_transfer_tasks_execution.assert_not_called() - @mock.patch.object(server.ConductorServerEndpoint, - '_get_transfer_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') def test_cancel_transfer_tasks_execution_status_cancelling_no_force( - self, - mock_cancel_transfer_tasks_execution, - mock_get_transfer_tasks_execution + self, mock_cancel_transfer_tasks_execution, mock_get_transfer_tasks_execution ): - mock_get_transfer_tasks_execution.return_value.status = constants\ - .EXECUTION_STATUS_CANCELLING + mock_get_transfer_tasks_execution.return_value.status = ( + constants.EXECUTION_STATUS_CANCELLING + ) self.assertRaises( exception.InvalidTransferState, - testutils.get_wrapped_function( - self.server.cancel_transfer_tasks_execution), + testutils.get_wrapped_function(self.server.cancel_transfer_tasks_execution), self.server, mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, - False + False, ) mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.execution_id) + mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id + ) mock_cancel_transfer_tasks_execution.assert_not_called() @mock.patch.object(db_api, 'get_transfer_tasks_execution') - def test__get_transfer_tasks_execution( - self, - mock_get_transfer_tasks_execution - ): + def test__get_transfer_tasks_execution(self, mock_get_transfer_tasks_execution): result = self.server._get_transfer_tasks_execution( mock.sentinel.context, mock.sentinel.transfer_id, @@ -1641,21 +1540,18 @@ def test__get_transfer_tasks_execution( include_task_info=False, to_dict=False, ) - self.assertEqual( - mock_get_transfer_tasks_execution.return_value, - result - ) + self.assertEqual(mock_get_transfer_tasks_execution.return_value, result) mock_get_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.execution_id, include_task_info=False, - to_dict=False) + to_dict=False, + ) @mock.patch.object(db_api, 'get_transfer_tasks_execution') def test__get_transfer_tasks_execution_no_execution( - self, - mock_get_transfer_tasks_execution + self, mock_get_transfer_tasks_execution ): mock_get_transfer_tasks_execution.return_value = None self.assertRaises( @@ -1673,7 +1569,8 @@ def test__get_transfer_tasks_execution_no_execution( mock.sentinel.transfer_id, mock.sentinel.execution_id, include_task_info=False, - to_dict=False) + to_dict=False, + ) @mock.patch.object(db_api, 'get_transfers') def test_get_transfers(self, mock_get_transfers): @@ -1684,10 +1581,7 @@ def test_get_transfers(self, mock_get_transfers): **self._mock_pagination_args, ) - self.assertEqual( - mock_get_transfers.return_value, - result - ) + self.assertEqual(mock_get_transfers.return_value, result) mock_get_transfers.assert_called_once_with( mock.sentinel.context, include_tasks_executions=False, @@ -1702,25 +1596,24 @@ def test_get_transfer(self, mock_get_transfer): self.server, mock.sentinel.context, mock.sentinel.transfer_id, - include_task_info=False + include_task_info=False, ) - self.assertEqual( - mock_get_transfer.return_value, - result - ) + self.assertEqual(mock_get_transfer.return_value, result) mock_get_transfer.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=False, - to_dict=True + to_dict=True, ) @mock.patch.object(db_api, 'delete_transfer') - @mock.patch.object(server.ConductorServerEndpoint, - '_check_delete_reservation_for_transfer') - @mock.patch.object(server.ConductorServerEndpoint, - '_check_transfer_running_executions') + @mock.patch.object( + server.ConductorServerEndpoint, '_check_delete_reservation_for_transfer' + ) + @mock.patch.object( + server.ConductorServerEndpoint, '_check_transfer_running_executions' + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_delete_transfer( self, @@ -1730,85 +1623,68 @@ def test_delete_transfer( mock_delete_transfer, ): testutils.get_wrapped_function(self.server.delete_transfer)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id + self.server, mock.sentinel.context, mock.sentinel.transfer_id ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, mock.sentinel.transfer_id) + mock.sentinel.context, mock.sentinel.transfer_id + ) mock_check_transfer_running_executions.assert_called_once_with( - mock.sentinel.context, mock_get_transfer.return_value) + mock.sentinel.context, mock_get_transfer.return_value + ) mock_check_delete_reservation_for_transfer.assert_called_once_with( - mock_get_transfer.return_value) + mock_get_transfer.return_value + ) mock_delete_transfer.assert_called_once_with( - mock.sentinel.context, mock.sentinel.transfer_id) + mock.sentinel.context, mock.sentinel.transfer_id + ) - @mock.patch.object( - server.ConductorServerEndpoint, - 'get_transfer_tasks_execution' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_begin_tasks' - ) + @mock.patch.object(server.ConductorServerEndpoint, 'get_transfer_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') @mock.patch.object(db_api, "add_transfer_tasks_execution") @mock.patch.object(db_api, "update_transfer_action_info_for_instance") - @mock.patch.object( - server.ConductorServerEndpoint, - '_check_execution_tasks_sanity' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_check_execution_tasks_sanity') @mock.patch.object(copy, "deepcopy") - @mock.patch.object( - server.ConductorServerEndpoint, - '_create_task' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_create_task') @mock.patch.object(uuid, "uuid4") @mock.patch.object(models, "TasksExecution") @mock.patch.object( - server.ConductorServerEndpoint, - '_check_transfer_running_executions' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_transfer' + server.ConductorServerEndpoint, '_check_transfer_running_executions' ) + @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_delete_transfer_disks( - self, - mock_get_transfer, - mock_check_transfer_running_executions, - mock_tasks_execution, - mock_uuid4, - mock_create_task, - mock_deepcopy, - mock_check_execution_tasks_sanity, - mock_update_transfer_action_info_for_instance, - mock_add_transfer_tasks_execution, - mock_begin_tasks, - mock_get_transfer_tasks_execution, + self, + mock_get_transfer, + mock_check_transfer_running_executions, + mock_tasks_execution, + mock_uuid4, + mock_create_task, + mock_deepcopy, + mock_check_execution_tasks_sanity, + mock_update_transfer_action_info_for_instance, + mock_add_transfer_tasks_execution, + mock_begin_tasks, + mock_get_transfer_tasks_execution, ): def call_delete_transfer_disks(): - return testutils.get_wrapped_function( - self.server.delete_transfer_disks)( + return testutils.get_wrapped_function(self.server.delete_transfer_disks)( self.server, mock.sentinel.context, mock.sentinel.transfer_id, # type: ignore ) + instances = [mock.Mock(), mock.Mock()] mock_transfer = mock.Mock( instances=instances, id=mock.sentinel.transfer_id, network_map=mock.sentinel.network_map, - info={ - instance: instance - for instance in instances - } + info={instance: instance for instance in instances}, ) def create_task_side_effect( - instance, - task_type, - execution, - depends_on=None, + instance, + task_type, + execution, + depends_on=None, ): return mock.Mock( id=task_type, @@ -1824,100 +1700,91 @@ def create_task_side_effect( result = call_delete_transfer_disks() mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_check_transfer_running_executions.assert_called_once_with( - mock.sentinel.context, - mock_transfer + mock.sentinel.context, mock_transfer ) self.assertEqual( mock_tasks_execution.return_value.status, - constants.EXECUTION_STATUS_UNEXECUTED + constants.EXECUTION_STATUS_UNEXECUTED, ) self.assertEqual( mock_tasks_execution.return_value.type, - constants.EXECUTION_TYPE_TRANSFER_DISKS_DELETE + constants.EXECUTION_TYPE_TRANSFER_DISKS_DELETE, ) for instance in instances: assert instance in mock_transfer.info - mock_create_task.assert_has_calls([ - mock.call( - instance, - constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS, - mock_tasks_execution.return_value, - ), - mock.call( - instance, - constants.TASK_TYPE_DELETE_TRANSFER_DISKS, - mock_tasks_execution.return_value, - depends_on=[ - constants - .TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS - ], - ), - ]) + mock_create_task.assert_has_calls( + [ + mock.call( + instance, + constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS, + mock_tasks_execution.return_value, + ), + mock.call( + instance, + constants.TASK_TYPE_DELETE_TRANSFER_DISKS, + mock_tasks_execution.return_value, + depends_on=[ + constants.TASK_TYPE_DELETE_TRANSFER_SOURCE_DISK_SNAPSHOTS + ], + ), + ] + ) - mock_update_transfer_action_info_for_instance\ - .assert_has_calls([mock.call( - mock.sentinel.context, - mock_transfer.id, - instance, - mock_transfer.info[instance], - )]) + mock_update_transfer_action_info_for_instance.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + mock_transfer.id, + instance, + mock_transfer.info[instance], + ) + ] + ) - mock_deepcopy.assert_called_once_with( - mock_transfer.destination_environment) + mock_deepcopy.assert_called_once_with(mock_transfer.destination_environment) mock_check_execution_tasks_sanity.assert_called_once_with( mock_tasks_execution.return_value, mock_transfer.info, ) mock_add_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock_tasks_execution.return_value + mock.sentinel.context, mock_tasks_execution.return_value ) mock_begin_tasks.assert_called_once_with( - mock.sentinel.context, - mock_transfer, - mock_tasks_execution.return_value + mock.sentinel.context, mock_transfer, mock_tasks_execution.return_value ) mock_get_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_transfer.id, - mock_tasks_execution.return_value.id + mock_tasks_execution.return_value.id, ) - self.assertEqual( - result, mock_get_transfer_tasks_execution.return_value) + self.assertEqual(result, mock_get_transfer_tasks_execution.return_value) # raises exception if instances have no volumes info instances[0].get.return_value = None instances[1].get.return_value = None - self.assertRaises( - exception.InvalidTransferState, - call_delete_transfer_disks - ) + self.assertRaises(exception.InvalidTransferState, call_delete_transfer_disks) # raises exception if instance not in transfer.info instances[0].get.return_value = mock.sentinel.volume_info instances[1].get.return_value = mock.sentinel.volume_info mock_transfer.info = {} - self.assertRaises( - exception.InvalidTransferState, - call_delete_transfer_disks - ) + self.assertRaises(exception.InvalidTransferState, call_delete_transfer_disks) def test_check_endpoints(self): origin_endpoint = mock.Mock() destination_endpoint = mock.Mock() self.server._check_endpoints( - mock.sentinel.context, origin_endpoint, destination_endpoint) + mock.sentinel.context, origin_endpoint, destination_endpoint + ) def test_check_endpoints_same_destination_id(self): origin_endpoint = mock.Mock() @@ -1929,30 +1796,30 @@ def test_check_endpoints_same_destination_id(self): self.server._check_endpoints, mock.sentinel.context, origin_endpoint, - destination_endpoint + destination_endpoint, ) def test_check_endpoints_same_destination_connection_info(self): origin_endpoint = mock.Mock() destination_endpoint = mock.Mock() - origin_endpoint.connection_info = \ - mock.sentinel.origin_endpoint_connection_info - destination_endpoint.connection_info = \ + origin_endpoint.connection_info = mock.sentinel.origin_endpoint_connection_info + destination_endpoint.connection_info = ( mock.sentinel.origin_endpoint_connection_info + ) self.assertRaises( exception.SameDestination, self.server._check_endpoints, mock.sentinel.context, origin_endpoint, - destination_endpoint + destination_endpoint, ) @mock.patch.object(server.ConductorServerEndpoint, 'get_transfer') @mock.patch.object(db_api, 'add_transfer') - @mock.patch.object(server.ConductorServerEndpoint, - '_create_reservation_for_transfer') - @mock.patch.object(server.ConductorServerEndpoint, - '_check_minion_pools_for_action') + @mock.patch.object( + server.ConductorServerEndpoint, '_create_reservation_for_transfer' + ) + @mock.patch.object(server.ConductorServerEndpoint, '_check_minion_pools_for_action') @mock.patch.object(models, 'Transfer') @mock.patch.object(server.ConductorServerEndpoint, '_check_endpoints') @mock.patch.object(server.ConductorServerEndpoint, 'get_endpoint') @@ -1964,10 +1831,12 @@ def test_create_instances_transfer( mock_check_minion_pools_for_action, mock_create_reservation_for_transfer, mock_add_transfer, - mock_get_transfer + mock_get_transfer, ): - mock_get_endpoint.side_effect = mock.sentinel.origin_endpoint_id, \ - mock.sentinel.destination_endpoint_id + mock_get_endpoint.side_effect = ( + mock.sentinel.origin_endpoint_id, + mock.sentinel.destination_endpoint_id, + ) mock_transfer.return_value = mock.Mock() result = self.server.create_instances_transfer( mock.sentinel.context, @@ -1983,20 +1852,19 @@ def test_create_instances_transfer( mock.sentinel.network_map, mock.sentinel.storage_mappings, notes=None, - user_scripts=None + user_scripts=None, ) - self.assertEqual( - mock_get_transfer.return_value, - result + self.assertEqual(mock_get_transfer.return_value, result) + mock_get_endpoint.assert_has_calls( + [ + mock.call(mock.sentinel.context, mock.sentinel.origin_endpoint_id), + mock.call(mock.sentinel.context, mock.sentinel.destination_endpoint_id), + ] ) - mock_get_endpoint.assert_has_calls([ - mock.call(mock.sentinel.context, mock.sentinel.origin_endpoint_id), - mock.call(mock.sentinel.context, - mock.sentinel.destination_endpoint_id)]) mock_check_endpoints.assert_called_once_with( mock.sentinel.context, mock.sentinel.origin_endpoint_id, - mock.sentinel.destination_endpoint_id + mock.sentinel.destination_endpoint_id, ) self.assertEqual( ( @@ -2005,36 +1873,42 @@ def test_create_instances_transfer( mock_transfer.return_value.destination_endpoint_id, mock_transfer.return_value.origin_minion_pool_id, mock_transfer.return_value.destination_minion_pool_id, - (mock_transfer.return_value. - instance_osmorphing_minion_pool_mappings), + (mock_transfer.return_value.instance_osmorphing_minion_pool_mappings), mock_transfer.return_value.source_environment, mock_transfer.return_value.destination_environment, mock_transfer.return_value.info, mock_transfer.return_value.notes, - mock_transfer.return_value.user_scripts), + mock_transfer.return_value.user_scripts, + ), ( mock.sentinel.origin_endpoint_id, mock.sentinel.destination_endpoint_id, mock.sentinel.destination_endpoint_id, mock.sentinel.origin_minion_pool_id, mock.sentinel.destination_minion_pool_id, - (mock.sentinel. - instance_osmorphing_minion_pool_mappings), + (mock.sentinel.instance_osmorphing_minion_pool_mappings), mock.sentinel.source_environment, mock.sentinel.destination_environment, - {mock.sentinel.instance_1: {'volumes_info': []}, - mock.sentinel.instance_2: {'volumes_info': []}}, + { + mock.sentinel.instance_1: {'volumes_info': []}, + mock.sentinel.instance_2: {'volumes_info': []}, + }, None, - {}) + {}, + ), ) mock_check_minion_pools_for_action.assert_called_once_with( - mock.sentinel.context, mock_transfer.return_value) + mock.sentinel.context, mock_transfer.return_value + ) mock_create_reservation_for_transfer.assert_called_once_with( - mock_transfer.return_value) + mock_transfer.return_value + ) mock_add_transfer.assert_called_once_with( - mock.sentinel.context, mock_transfer.return_value) + mock.sentinel.context, mock_transfer.return_value + ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, mock_transfer.return_value.id) + mock.sentinel.context, mock_transfer.return_value.id + ) @mock.patch.object(db_api, 'get_transfer') def test__get_transfer(self, mock_get_transfer): @@ -2042,17 +1916,14 @@ def test__get_transfer(self, mock_get_transfer): mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=False, - to_dict=False - ) - self.assertEqual( - mock_get_transfer.return_value, - result + to_dict=False, ) + self.assertEqual(mock_get_transfer.return_value, result) mock_get_transfer.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=False, - to_dict=False + to_dict=False, ) @mock.patch.object(db_api, 'get_transfer') @@ -2064,26 +1935,18 @@ def test__get_transfer_not_found(self, mock_get_transfer): mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=False, - to_dict=False + to_dict=False, ) mock_get_transfer.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=False, - to_dict=False + to_dict=False, ) @ddt.data( - ( - None, - [], - {} - ), - ( - {'instances': {}}, - [], - {'instances': {}} - ), + (None, [], {}), + ({'instances': {}}, [], {'instances': {}}), ( { 'instances': { @@ -2091,67 +1954,44 @@ def test__get_transfer_not_found(self, mock_get_transfer): } }, ["mock_instance_2"], - { - 'instances': {} - } + {'instances': {}}, ), ( { 'instances': { "mock_instance_1": "mock_value_1\r\n", - "mock_instance_2": "mock_value_2\r\n" + "mock_instance_2": "mock_value_2\r\n", } }, ["mock_instance_2"], - { - 'instances': { - "mock_instance_2": "mock_value_2\n" - } - } + {'instances': {"mock_instance_2": "mock_value_2\n"}}, ), ( { - 'global': { - "linux": "mock_value_1\r\n" - }, + 'global': {"linux": "mock_value_1\r\n"}, 'instances': { "mock_instance_1": "mock_value_1\n\r", - "mock_instance_2": "mock_value_2\n\r" - } + "mock_instance_2": "mock_value_2\n\r", + }, }, ["mock_instance_2"], { - 'instances': { - "mock_instance_2": "mock_value_2\n" - }, - 'global': { - "linux": "mock_value_1\n" - } - } + 'instances': {"mock_instance_2": "mock_value_2\n"}, + 'global': {"linux": "mock_value_1\n"}, + }, ), ( - { - 'global': { - "linux": "mock_value_1\n\r" - } - }, + {'global': {"linux": "mock_value_1\n\r"}}, [], - { - 'global': { - "linux": "mock_value_1\n" - } - } - ) + {'global': {"linux": "mock_value_1\n"}}, + ), ) @ddt.unpack - def test_normalize_user_scripts( - self, user_scripts, instances, expected_result): + def test_normalize_user_scripts(self, user_scripts, instances, expected_result): try: - with self.assertLogs( - 'coriolis.conductor.rpc.server', level='WARN'): - result = self.server._normalize_user_scripts( - user_scripts, instances) + with self.assertLogs('coriolis.conductor.rpc.server', level='WARN'): + result = self.server._normalize_user_scripts(user_scripts, instances) except AssertionError: pass @@ -2163,32 +2003,24 @@ def test_get_deployment(self, mock_get_deployment): self.server, mock.sentinel.context, mock.sentinel.deployment_id, - include_task_info=False - ) - self.assertEqual( - mock_get_deployment.return_value, - result + include_task_info=False, ) + self.assertEqual(mock_get_deployment.return_value, result) mock_get_deployment.assert_called_once_with( mock.sentinel.context, mock.sentinel.deployment_id, include_task_info=False, - to_dict=True + to_dict=True, ) @mock.patch.object(db_api, 'get_transfer_deployments') - def test_check_running_transfer_deployments( - self, - mock_get_transfer_deployments - ): + def test_check_running_transfer_deployments(self, mock_get_transfer_deployments): deployment_1 = mock.Mock() deployment_2 = mock.Mock() deployment_1.executions = [mock.Mock()] - deployment_1.executions[0].status = \ - constants.EXECUTION_STATUS_COMPLETED + deployment_1.executions[0].status = constants.EXECUTION_STATUS_COMPLETED deployment_2.executions = [mock.Mock()] - deployment_2.executions[0].status = \ - constants.EXECUTION_STATUS_ERROR + deployment_2.executions[0].status = constants.EXECUTION_STATUS_ERROR deployments = [deployment_1, deployment_2] mock_get_transfer_deployments.return_value = deployments self.server._check_running_transfer_deployments( @@ -2202,16 +2034,14 @@ def test_check_running_transfer_deployments( @mock.patch.object(db_api, 'get_transfer_deployments') def test_check_running_transfer_deployments_invalid_transfer_state( - self, - mock_get_transfer_deployments + self, mock_get_transfer_deployments ): deployment_1 = mock.Mock() deployment_2 = mock.Mock() deployment_1.executions = [mock.Mock()] deployment_1.executions[0].status = constants.EXECUTION_STATUS_RUNNING deployment_2.executions = [mock.Mock()] - deployment_2.executions[0].status = \ - constants.EXECUTION_STATUS_COMPLETED + deployment_2.executions[0].status = constants.EXECUTION_STATUS_COMPLETED deployments = [deployment_1, deployment_2] mock_get_transfer_deployments.return_value = deployments self.assertRaises( @@ -2244,28 +2074,22 @@ def test_check_running_executions_invalid_state(self): self.assertRaises( exception.InvalidActionTasksExecutionState, self.server._check_running_executions, - action + action, ) - @mock.patch.object(server.ConductorServerEndpoint, - '_check_running_transfer_deployments') - @mock.patch.object(server.ConductorServerEndpoint, - '_check_running_executions') + @mock.patch.object( + server.ConductorServerEndpoint, '_check_running_transfer_deployments' + ) + @mock.patch.object(server.ConductorServerEndpoint, '_check_running_executions') def test_check_transfer_running_executions( - self, - mock_check_running_executions, - mock_check_running_transfer_deployments + self, mock_check_running_executions, mock_check_running_transfer_deployments ): transfer = mock.Mock() - self.server._check_transfer_running_executions( - mock.sentinel.context, - transfer - ) + self.server._check_transfer_running_executions(mock.sentinel.context, transfer) mock_check_running_executions.assert_called_once_with(transfer) mock_check_running_transfer_deployments.assert_called_once_with( - mock.sentinel.context, - transfer.id + mock.sentinel.context, transfer.id ) def test_check_valid_transfer_tasks_execution(self): @@ -2279,12 +2103,8 @@ def test_check_valid_transfer_tasks_execution(self): type=constants.EXECUTION_TYPE_TRANSFER_EXECUTION, status=constants.EXECUTION_STATUS_COMPLETED, ) - mock_transfer = mock.Mock( - executions=[execution1, execution2] - ) - self.server._check_valid_transfer_tasks_execution( - mock_transfer - ) + mock_transfer = mock.Mock(executions=[execution1, execution2]) + self.server._check_valid_transfer_tasks_execution(mock_transfer) # raises exception if all executions are incomplete execution1.status = constants.EXECUTION_STATUS_UNEXECUTED @@ -2293,49 +2113,38 @@ def test_check_valid_transfer_tasks_execution(self): self.assertRaises( exception.InvalidTransferState, self.server._check_valid_transfer_tasks_execution, - mock_transfer + mock_transfer, ) # doesn't raise exception if all executions are incomplete # and is forced - self.server._check_valid_transfer_tasks_execution( - mock_transfer, - True - ) + self.server._check_valid_transfer_tasks_execution(mock_transfer, True) # doesn't raise exception if only one execution is completed execution1.status = constants.EXECUTION_STATUS_COMPLETED execution2.status = constants.EXECUTION_STATUS_UNEXECUTED - self.server._check_valid_transfer_tasks_execution( - mock_transfer - ) + self.server._check_valid_transfer_tasks_execution(mock_transfer) mock_transfer.executions = [] self.assertRaises( exception.InvalidTransferState, self.server._check_valid_transfer_tasks_execution, - mock_transfer + mock_transfer, ) - @mock.patch.object(server.ConductorServerEndpoint, - "get_available_providers") + @mock.patch.object(server.ConductorServerEndpoint, "get_available_providers") def test_get_provider_types(self, mock_get_available_providers): endpoint = mock.Mock() - result = self.server._get_provider_types( - mock.sentinel.context, - endpoint - ) + result = self.server._get_provider_types(mock.sentinel.context, endpoint) self.assertEqual( - (mock_get_available_providers.return_value. - get(endpoint.type)["types"]), - result + (mock_get_available_providers.return_value.get(endpoint.type)["types"]), + result, ) - mock_get_available_providers.assert_called_once_with( - mock.sentinel.context) + mock_get_available_providers.assert_called_once_with(mock.sentinel.context) mock_get_available_providers.reset_mock() endpoint.type = "mock_type" @@ -2344,31 +2153,35 @@ def test_get_provider_types(self, mock_get_available_providers): exception.NotFound, self.server._get_provider_types, mock.sentinel.context, - endpoint + endpoint, ) - mock_get_available_providers.assert_called_once_with( - mock.sentinel.context) + mock_get_available_providers.assert_called_once_with(mock.sentinel.context) - @mock.patch.object(server.ConductorServerEndpoint, - '_normalize_user_scripts') - @mock.patch.object(server.ConductorServerEndpoint, - '_validate_deployment_inputs') + @mock.patch.object(server.ConductorServerEndpoint, '_normalize_user_scripts') + @mock.patch.object(server.ConductorServerEndpoint, '_validate_deployment_inputs') @mock.patch.object(models, 'Deployment') @mock.patch.object(db_api, 'add_deployment') @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') @mock.patch.object(server.ConductorServerEndpoint, '_execute_deployment') @mock.patch.object(server.ConductorServerEndpoint, 'get_deployment') def test_deploy_transfer_instances( - self, mock_get_deployment, mock_execute_deployment, - mock_get_transfer, mock_add_deployment, mock_deployment_model, - mock_validate_deployment_inputs, mock_normalize_user_scripts): + self, + mock_get_deployment, + mock_execute_deployment, + mock_get_transfer, + mock_add_deployment, + mock_deployment_model, + mock_validate_deployment_inputs, + mock_normalize_user_scripts, + ): mock_normalize_user_scripts.return_value = {'instances': {}} transfer_mock = mock.MagicMock() transfer_mock.instance_osmorphing_minion_pool_mappings = { - mock.sentinel.instance1: mock.sentinel.pool1} + mock.sentinel.instance1: mock.sentinel.pool1 + } transfer_mock.user_scripts = {} transfer_mock.clone_disks = False transfer_mock.skip_os_morphing = True @@ -2378,18 +2191,18 @@ def test_deploy_transfer_instances( deployment = mock_deployment_model.return_value force = False - result = testutils.get_wrapped_function( - self.server.deploy_transfer_instances)( - self.server, - mock.sentinel.ctxt, - mock.sentinel.transfer_id, - force=force, - clone_disks=True, - instance_osmorphing_minion_pool_mappings=osm_pool_mappings, - skip_os_morphing=False, - user_scripts=mock.sentinel.user_scripts, - wait_for_execution=None, - trust_id=None) + result = testutils.get_wrapped_function(self.server.deploy_transfer_instances)( + self.server, + mock.sentinel.ctxt, + mock.sentinel.transfer_id, + force=force, + clone_disks=True, + instance_osmorphing_minion_pool_mappings=osm_pool_mappings, + skip_os_morphing=False, + user_scripts=mock.sentinel.user_scripts, + wait_for_execution=None, + trust_id=None, + ) self.assertEqual(mock_get_deployment.return_value, result) self.assertEqual({'instances': {}}, deployment.user_scripts) @@ -2398,23 +2211,27 @@ def test_deploy_transfer_instances( self.assertEqual( mock.sentinel.pool2, deployment.instance_osmorphing_minion_pool_mappings[ - mock.sentinel.instance1]) + mock.sentinel.instance1 + ], + ) self.assertTrue( - transfer_mock.destination_environment is not - deployment.destination_environment) + transfer_mock.destination_environment + is not deployment.destination_environment + ) mock_normalize_user_scripts.assert_called_once_with( - mock.sentinel.user_scripts, transfer_mock.instances) + mock.sentinel.user_scripts, transfer_mock.instances + ) mock_get_transfer.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.transfer_id, - include_task_info=True) - mock_add_deployment.assert_called_once_with( - mock.sentinel.ctxt, deployment) + mock.sentinel.ctxt, mock.sentinel.transfer_id, include_task_info=True + ) + mock_add_deployment.assert_called_once_with(mock.sentinel.ctxt, deployment) mock_execute_deployment.assert_called_once_with( - mock.sentinel.ctxt, deployment, False) - mock_get_deployment.assert_called_once_with( - mock.sentinel.ctxt, deployment.id) + mock.sentinel.ctxt, deployment, False + ) + mock_get_deployment.assert_called_once_with(mock.sentinel.ctxt, deployment.id) mock_validate_deployment_inputs.assert_called_once_with( - mock.sentinel.ctxt, deployment, transfer_mock, force) + mock.sentinel.ctxt, deployment, transfer_mock, force + ) @mock.patch.object(models, 'Deployment') @mock.patch.object(db_api, 'add_deployment') @@ -2422,11 +2239,17 @@ def test_deploy_transfer_instances( @mock.patch.object(server.ConductorServerEndpoint, '_execute_deployment') @mock.patch.object(server.ConductorServerEndpoint, 'get_deployment') def test_deploy_transfer_instances_wait_for_execution( - self, mock_get_deployment, mock_execute_deployment, - mock_get_transfer, mock_add_deployment, mock_deployment_model): + self, + mock_get_deployment, + mock_execute_deployment, + mock_get_transfer, + mock_add_deployment, + mock_deployment_model, + ): transfer_mock = mock.MagicMock() transfer_mock.instance_osmorphing_minion_pool_mappings = { - mock.sentinel.instance1: mock.sentinel.pool1} + mock.sentinel.instance1: mock.sentinel.pool1 + } transfer_mock.user_scripts = {} transfer_mock.clone_disks = False transfer_mock.skip_os_morphing = True @@ -2434,144 +2257,110 @@ def test_deploy_transfer_instances_wait_for_execution( mock_get_transfer.return_value = transfer_mock deployment = mock_deployment_model.return_value - result = testutils.get_wrapped_function( - self.server.deploy_transfer_instances)( - self.server, - mock.sentinel.ctxt, - mock.sentinel.transfer_id, - force=False, - clone_disks=None, - instance_osmorphing_minion_pool_mappings=None, - skip_os_morphing=None, - user_scripts=None, - wait_for_execution=mock.sentinel.wait_for_execution, - trust_id=mock.sentinel.trust_id) + result = testutils.get_wrapped_function(self.server.deploy_transfer_instances)( + self.server, + mock.sentinel.ctxt, + mock.sentinel.transfer_id, + force=False, + clone_disks=None, + instance_osmorphing_minion_pool_mappings=None, + skip_os_morphing=None, + user_scripts=None, + wait_for_execution=mock.sentinel.wait_for_execution, + trust_id=mock.sentinel.trust_id, + ) self.assertEqual(mock_get_deployment.return_value, result) self.assertEqual(transfer_mock.user_scripts, deployment.user_scripts) self.assertEqual(transfer_mock.clone_disks, deployment.clone_disks) - self.assertEqual( - transfer_mock.skip_os_morphing, deployment.skip_os_morphing) + self.assertEqual(transfer_mock.skip_os_morphing, deployment.skip_os_morphing) self.assertEqual( transfer_mock.instance_osmorphing_minion_pool_mappings, - deployment.instance_osmorphing_minion_pool_mappings) + deployment.instance_osmorphing_minion_pool_mappings, + ) self.assertTrue( - transfer_mock.destination_environment is not - deployment.destination_environment) + transfer_mock.destination_environment + is not deployment.destination_environment + ) mock_get_transfer.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.transfer_id, - include_task_info=True) - mock_add_deployment.assert_called_once_with( - mock.sentinel.ctxt, deployment) + mock.sentinel.ctxt, mock.sentinel.transfer_id, include_task_info=True + ) + mock_add_deployment.assert_called_once_with(mock.sentinel.ctxt, deployment) mock_execute_deployment.assert_not_called() - mock_get_deployment.assert_called_once_with( - mock.sentinel.ctxt, deployment.id) + mock_get_deployment.assert_called_once_with(mock.sentinel.ctxt, deployment.id) - def test_get_instance_scripts( - self - ): + def test_get_instance_scripts(self): user_scripts = { "global": 'mock_user_scripts', "instances": { mock.sentinel.instance_1: mock.sentinel.scripts_1, mock.sentinel.instance_2: mock.sentinel.scripts_2, - } + }, } expected_result = { 'global': 'mock_user_scripts', - 'instances': {mock.sentinel.instance_1: mock.sentinel.scripts_1} + 'instances': {mock.sentinel.instance_1: mock.sentinel.scripts_1}, } result = self.server._get_instance_scripts( user_scripts, mock.sentinel.instance_1 ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) - def test_get_instance_scripts_no_instance_script( - self - ): + def test_get_instance_scripts_no_instance_script(self): user_scripts = { "global": 'mock_user_scripts', "instances": { mock.sentinel.instance_1: None, mock.sentinel.instance_2: mock.sentinel.scripts_2, - } - } - expected_result = { - 'global': 'mock_user_scripts', - 'instances': {} + }, } + expected_result = {'global': 'mock_user_scripts', 'instances': {}} result = self.server._get_instance_scripts( user_scripts, mock.sentinel.instance_1 ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) - def test_get_instance_scripts_no_user_scripts( - self - ): + def test_get_instance_scripts_no_user_scripts(self): user_scripts = None - expected_result = { - 'global': {}, - 'instances': {} - } + expected_result = {'global': {}, 'instances': {}} result = self.server._get_instance_scripts( user_scripts, mock.sentinel.instance_1 ) - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) - @mock.patch.object(server.ConductorServerEndpoint, - "_minion_manager_client") - def test_deallocate_minion_machines_for_action( - self, - mock_minion_manager_client - ): + @mock.patch.object(server.ConductorServerEndpoint, "_minion_manager_client") + def test_deallocate_minion_machines_for_action(self, mock_minion_manager_client): action = mock.Mock() result = self.server._deallocate_minion_machines_for_action( - mock.sentinel.context, - action + mock.sentinel.context, action ) self.assertEqual( - (mock_minion_manager_client.deallocate_minion_machines_for_action. - return_value), - result + ( + mock_minion_manager_client.deallocate_minion_machines_for_action.return_value + ), + result, + ) + ( + mock_minion_manager_client.deallocate_minion_machines_for_action.assert_called_once_with( + mock.sentinel.context, action.base_id + ) ) - (mock_minion_manager_client.deallocate_minion_machines_for_action. - assert_called_once_with( - mock.sentinel.context, - action.base_id - )) - @mock.patch.object(server.ConductorServerEndpoint, - "_minion_manager_client") - def test_check_minion_pools_for_action( - self, - mock_minion_manager_client - ): + @mock.patch.object(server.ConductorServerEndpoint, "_minion_manager_client") + def test_check_minion_pools_for_action(self, mock_minion_manager_client): action = mock.Mock() - self.server._check_minion_pools_for_action( - mock.sentinel.context, - action - ) + self.server._check_minion_pools_for_action(mock.sentinel.context, action) - (mock_minion_manager_client. - validate_minion_pool_selections_for_action.assert_called_once_with( - mock.sentinel.context, - action - )) + ( + mock_minion_manager_client.validate_minion_pool_selections_for_action.assert_called_once_with( + mock.sentinel.context, action + ) + ) @mock.patch.object(db_api, "update_transfer_action_info_for_instance") def test_update_task_info_for_minion_allocations( - self, - mock_update_transfer_action_info_for_instance + self, mock_update_transfer_action_info_for_instance ): action = mock.Mock() action.id = mock.sentinel.action_id @@ -2579,95 +2368,70 @@ def test_update_task_info_for_minion_allocations( action.info = { mock.sentinel.instance1: {}, mock.sentinel.instance2: { - 'origin_minion_machine_id': - mock.sentinel.origin_minion_id, - 'origin_minion_provider_properties': - mock.sentinel.origin_minion_provider_properties, - 'origin_minion_connection_info': - mock.sentinel.origin_minion_connection_info, - 'destination_minion_machine_id': - mock.sentinel.destination_minion_id, - 'destination_minion_provider_properties': - mock.sentinel.destination_minion_provider_properties, - 'destination_minion_connection_info': - mock.sentinel.destination_minion_connection_info, - 'destination_minion_backup_writer_connection_info': - mock.sentinel.destination_minion_backup_writer_connection_info, - 'osmorphing_minion_machine_id': - mock.sentinel.osmorphing_minion_id, - 'osmorphing_minion_provider_properties': - mock.sentinel.osmorphing_minion_provider_properties, - 'osmorphing_minion_connection_info': - mock.sentinel.osmorphing_minion_connection_info, + 'origin_minion_machine_id': mock.sentinel.origin_minion_id, + 'origin_minion_provider_properties': mock.sentinel.origin_minion_provider_properties, + 'origin_minion_connection_info': mock.sentinel.origin_minion_connection_info, + 'destination_minion_machine_id': mock.sentinel.destination_minion_id, + 'destination_minion_provider_properties': mock.sentinel.destination_minion_provider_properties, + 'destination_minion_connection_info': mock.sentinel.destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': mock.sentinel.destination_minion_backup_writer_connection_info, + 'osmorphing_minion_machine_id': mock.sentinel.osmorphing_minion_id, + 'osmorphing_minion_provider_properties': mock.sentinel.osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': mock.sentinel.osmorphing_minion_connection_info, }, - mock.sentinel.instance3: {} + mock.sentinel.instance3: {}, } minion_machine_allocations = { mock.sentinel.instance1: { 'origin_minion': { 'id': mock.sentinel.updated_origin_minion_id, - "provider_properties": - mock.sentinel.updated_origin_minion_provider_properties, - "connection_info": - mock.sentinel.updated_origin_minion_connection_info, + "provider_properties": mock.sentinel.updated_origin_minion_provider_properties, + "connection_info": mock.sentinel.updated_origin_minion_connection_info, }, 'destination_minion': { 'id': mock.sentinel.updated_destination_minion_id, - "provider_properties": - (mock.sentinel. - updated_destination_minion_provider_properties), - "connection_info": - mock.sentinel.updated_destination_minion_connection_info, - "backup_writer_connection_info": - (mock.sentinel. - updated_destination_minion_backup_writer_connection_info) + "provider_properties": ( + mock.sentinel.updated_destination_minion_provider_properties + ), + "connection_info": mock.sentinel.updated_destination_minion_connection_info, + "backup_writer_connection_info": ( + mock.sentinel.updated_destination_minion_backup_writer_connection_info + ), }, 'osmorphing_minion': { 'id': mock.sentinel.updated_osmorphing_minion_id, - "provider_properties": - (mock.sentinel. - updated_osmorphing_minion_provider_properties), - "connection_info": - mock.sentinel.updated_osmorphing_minion_connection_info, - } + "provider_properties": ( + mock.sentinel.updated_osmorphing_minion_provider_properties + ), + "connection_info": mock.sentinel.updated_osmorphing_minion_connection_info, + }, }, } expected_action_info = { mock.sentinel.instance1: { - 'origin_minion_machine_id': - mock.sentinel.updated_origin_minion_id, - 'origin_minion_provider_properties': - mock.sentinel.updated_origin_minion_provider_properties, - 'origin_minion_connection_info': - mock.sentinel.updated_origin_minion_connection_info, - 'destination_minion_machine_id': - mock.sentinel.updated_destination_minion_id, - 'destination_minion_provider_properties': - mock.sentinel.updated_destination_minion_provider_properties, - 'destination_minion_connection_info': - mock.sentinel.updated_destination_minion_connection_info, - 'destination_minion_backup_writer_connection_info': - (mock.sentinel. - updated_destination_minion_backup_writer_connection_info), - 'osmorphing_minion_machine_id': - mock.sentinel.updated_osmorphing_minion_id, - 'osmorphing_minion_provider_properties': - mock.sentinel.updated_osmorphing_minion_provider_properties, - 'osmorphing_minion_connection_info': - mock.sentinel.updated_osmorphing_minion_connection_info, + 'origin_minion_machine_id': mock.sentinel.updated_origin_minion_id, + 'origin_minion_provider_properties': mock.sentinel.updated_origin_minion_provider_properties, + 'origin_minion_connection_info': mock.sentinel.updated_origin_minion_connection_info, + 'destination_minion_machine_id': mock.sentinel.updated_destination_minion_id, + 'destination_minion_provider_properties': mock.sentinel.updated_destination_minion_provider_properties, + 'destination_minion_connection_info': mock.sentinel.updated_destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': ( + mock.sentinel.updated_destination_minion_backup_writer_connection_info + ), + 'osmorphing_minion_machine_id': mock.sentinel.updated_osmorphing_minion_id, + 'osmorphing_minion_provider_properties': mock.sentinel.updated_osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': mock.sentinel.updated_osmorphing_minion_connection_info, }, } self.server._update_task_info_for_minion_allocations( - mock.sentinel.context, - action, - minion_machine_allocations + mock.sentinel.context, action, minion_machine_allocations ) mock_update_transfer_action_info_for_instance.assert_called_once_with( mock.sentinel.context, mock.sentinel.action_id, mock.sentinel.instance1, - expected_action_info[mock.sentinel.instance1] + expected_action_info[mock.sentinel.instance1], ) minion_machine_allocations = { mock.sentinel.instance2: {}, @@ -2675,46 +2439,31 @@ def test_update_task_info_for_minion_allocations( } expected_action_info = { mock.sentinel.instance2: { - 'origin_minion_machine_id': - mock.sentinel.origin_minion_id, - 'origin_minion_provider_properties': - mock.sentinel.origin_minion_provider_properties, - 'origin_minion_connection_info': - mock.sentinel.origin_minion_connection_info, - 'destination_minion_machine_id': - mock.sentinel.destination_minion_id, - 'destination_minion_provider_properties': - mock.sentinel.destination_minion_provider_properties, - 'destination_minion_connection_info': - mock.sentinel.destination_minion_connection_info, - 'destination_minion_backup_writer_connection_info': - mock.sentinel.destination_minion_backup_writer_connection_info, - 'osmorphing_minion_machine_id': - mock.sentinel.osmorphing_minion_id, - 'osmorphing_minion_provider_properties': - mock.sentinel.osmorphing_minion_provider_properties, - 'osmorphing_minion_connection_info': - mock.sentinel.osmorphing_minion_connection_info, + 'origin_minion_machine_id': mock.sentinel.origin_minion_id, + 'origin_minion_provider_properties': mock.sentinel.origin_minion_provider_properties, + 'origin_minion_connection_info': mock.sentinel.origin_minion_connection_info, + 'destination_minion_machine_id': mock.sentinel.destination_minion_id, + 'destination_minion_provider_properties': mock.sentinel.destination_minion_provider_properties, + 'destination_minion_connection_info': mock.sentinel.destination_minion_connection_info, + 'destination_minion_backup_writer_connection_info': mock.sentinel.destination_minion_backup_writer_connection_info, + 'osmorphing_minion_machine_id': mock.sentinel.osmorphing_minion_id, + 'osmorphing_minion_provider_properties': mock.sentinel.osmorphing_minion_provider_properties, + 'osmorphing_minion_connection_info': mock.sentinel.osmorphing_minion_connection_info, }, } mock_update_transfer_action_info_for_instance.reset_mock() self.server._update_task_info_for_minion_allocations( - mock.sentinel.context, - action, - minion_machine_allocations + mock.sentinel.context, action, minion_machine_allocations ) mock_update_transfer_action_info_for_instance.assert_called_once_with( mock.sentinel.context, mock.sentinel.action_id, mock.sentinel.instance2, - expected_action_info[mock.sentinel.instance2] + expected_action_info[mock.sentinel.instance2], ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') - def test_get_last_execution_for_transfer( - self, - mock_get_transfer - ): + def test_get_last_execution_for_transfer(self, mock_get_transfer): transfer = mock.Mock() transfer.id = mock.sentinel.id execution1 = mock.Mock(id=mock.sentinel.execution_id1, number=1) @@ -2723,14 +2472,9 @@ def test_get_last_execution_for_transfer( transfer.executions = [execution1, execution2, execution3] mock_get_transfer.return_value = transfer result = self.server._get_last_execution_for_transfer( - mock.sentinel.context, - transfer, - requery=False - ) - self.assertEqual( - execution2, - result + mock.sentinel.context, transfer, requery=False ) + self.assertEqual(execution2, result) mock_get_transfer.assert_not_called() transfer.executions = None self.assertRaises( @@ -2738,16 +2482,14 @@ def test_get_last_execution_for_transfer( self.server._get_last_execution_for_transfer, mock.sentinel.context, transfer, - requery=True + requery=True, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, mock.sentinel.id) + mock.sentinel.context, mock.sentinel.id + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_deployment') - def test_get_execution_for_deployment( - self, - mock_get_deployment - ): + def test_get_execution_for_deployment(self, mock_get_deployment): deployment = mock.Mock() deployment.id = mock.sentinel.id execution1 = mock.Mock(id=mock.sentinel.execution_id1) @@ -2755,14 +2497,9 @@ def test_get_execution_for_deployment( deployment.executions = [execution1] mock_get_deployment.return_value = deployment result = self.server._get_execution_for_deployment( - mock.sentinel.context, - deployment, - requery=False - ) - self.assertEqual( - execution1, - result + mock.sentinel.context, deployment, requery=False ) + self.assertEqual(execution1, result) mock_get_deployment.assert_not_called() deployment.executions = [execution1, execution2] self.assertRaises( @@ -2770,25 +2507,28 @@ def test_get_execution_for_deployment( self.server._get_execution_for_deployment, mock.sentinel.context, deployment, - requery=True + requery=True, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, mock.sentinel.id) + mock.sentinel.context, mock.sentinel.id + ) deployment.executions = [] self.assertRaises( exception.InvalidDeploymentState, self.server._get_execution_for_deployment, mock.sentinel.context, deployment, - requery=False + requery=False, ) @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') @mock.patch.object(db_api, 'get_transfer_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_update_task_info_for_minion_allocations') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_last_execution_for_transfer') + @mock.patch.object( + server.ConductorServerEndpoint, '_update_task_info_for_minion_allocations' + ) + @mock.patch.object( + server.ConductorServerEndpoint, '_get_last_execution_for_transfer' + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_confirm_transfer_minions_allocation( self, @@ -2796,51 +2536,49 @@ def test_confirm_transfer_minions_allocation( mock_get_last_execution_for_transfer, mock_update_task_info_for_minion_allocations, mock_get_transfer_tasks_execution, - mock_begin_tasks + mock_begin_tasks, ): - mock_get_transfer.return_value.last_execution_status = \ + mock_get_transfer.return_value.last_execution_status = ( constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS - - testutils.get_wrapped_function( - self.server.confirm_transfer_minions_allocation)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.minion_machine_allocations ) - mock_get_transfer.assert_called_once_with( + testutils.get_wrapped_function(self.server.confirm_transfer_minions_allocation)( + self.server, mock.sentinel.context, mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.minion_machine_allocations, + ) + + mock_get_transfer.assert_called_once_with( + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_get_last_execution_for_transfer.assert_called_once_with( - mock.sentinel.context, - mock_get_transfer.return_value, - requery=False + mock.sentinel.context, mock_get_transfer.return_value, requery=False ) mock_update_task_info_for_minion_allocations.assert_called_once_with( mock.sentinel.context, mock_get_transfer.return_value, - mock.sentinel.minion_machine_allocations + mock.sentinel.minion_machine_allocations, ) mock_get_transfer_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_get_transfer.return_value.id, - mock_get_last_execution_for_transfer.return_value.id + mock_get_last_execution_for_transfer.return_value.id, ) mock_begin_tasks.assert_called_once_with( mock.sentinel.context, mock_get_transfer.return_value, - mock_get_transfer_tasks_execution.return_value + mock_get_transfer_tasks_execution.return_value, ) @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') @mock.patch.object(db_api, 'get_transfer_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_update_task_info_for_minion_allocations') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_last_execution_for_transfer') + @mock.patch.object( + server.ConductorServerEndpoint, '_update_task_info_for_minion_allocations' + ) + @mock.patch.object( + server.ConductorServerEndpoint, '_get_last_execution_for_transfer' + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_confirm_transfer_minions_allocation_unexpected_status( self, @@ -2848,267 +2586,257 @@ def test_confirm_transfer_minions_allocation_unexpected_status( mock_get_last_execution_for_transfer, mock_update_task_info_for_minion_allocations, mock_get_transfer_tasks_execution, - mock_begin_tasks + mock_begin_tasks, ): - mock_get_transfer.return_value.last_execution_status = \ + mock_get_transfer.return_value.last_execution_status = ( constants.EXECUTION_STATUS_CANCELED + ) self.assertRaises( exception.InvalidTransferState, testutils.get_wrapped_function( - self.server.confirm_transfer_minions_allocation), + self.server.confirm_transfer_minions_allocation + ), self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.minion_machine_allocations + mock.sentinel.minion_machine_allocations, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_get_last_execution_for_transfer.assert_not_called() mock_update_task_info_for_minion_allocations.assert_not_called() mock_get_transfer_tasks_execution.assert_not_called() mock_begin_tasks.assert_not_called() - @mock.patch.object(server.ConductorServerEndpoint, - '_set_tasks_execution_status') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_last_execution_for_transfer') + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') + @mock.patch.object( + server.ConductorServerEndpoint, '_get_last_execution_for_transfer' + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_report_transfer_minions_allocation_error( self, mock_get_transfer, mock_get_last_execution_for_transfer, mock_cancel_tasks_execution, - mock_set_tasks_execution_status + mock_set_tasks_execution_status, ): - mock_get_transfer.return_value.last_execution_status = \ + mock_get_transfer.return_value.last_execution_status = ( constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ) testutils.get_wrapped_function( - self.server.report_transfer_minions_allocation_error)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.minion_allocation_error_details + self.server.report_transfer_minions_allocation_error + )( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + mock.sentinel.minion_allocation_error_details, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_get_last_execution_for_transfer.assert_called_once_with( - mock.sentinel.context, - mock_get_transfer.return_value, - requery=False + mock.sentinel.context, mock_get_transfer.return_value, requery=False ) mock_cancel_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_get_last_execution_for_transfer.return_value, - requery=True + requery=True, ) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, mock_get_last_execution_for_transfer.return_value, - constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS, ) - @mock.patch.object(server.ConductorServerEndpoint, - '_set_tasks_execution_status') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_last_execution_for_transfer') + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') + @mock.patch.object( + server.ConductorServerEndpoint, '_get_last_execution_for_transfer' + ) @mock.patch.object(server.ConductorServerEndpoint, '_get_transfer') def test_report_transfer_minions_allocation_error_unexpected_status( self, mock_get_transfer, mock_get_last_execution_for_transfer, mock_cancel_tasks_execution, - mock_set_tasks_execution_status + mock_set_tasks_execution_status, ): - mock_get_transfer.return_value.last_execution_status = \ + mock_get_transfer.return_value.last_execution_status = ( constants.EXECUTION_STATUS_CANCELED + ) self.assertRaises( exception.InvalidTransferState, testutils.get_wrapped_function( - self.server.report_transfer_minions_allocation_error), + self.server.report_transfer_minions_allocation_error + ), self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.minion_allocation_error_details + mock.sentinel.minion_allocation_error_details, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_get_last_execution_for_transfer.assert_not_called() mock_cancel_tasks_execution.assert_not_called() mock_set_tasks_execution_status.assert_not_called() @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') - @mock.patch.object(server.ConductorServerEndpoint, - '_update_task_info_for_minion_allocations') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_execution_for_deployment') + @mock.patch.object( + server.ConductorServerEndpoint, '_update_task_info_for_minion_allocations' + ) + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_for_deployment') @mock.patch.object(server.ConductorServerEndpoint, '_get_deployment') def test_confirm_deployment_minions_allocation( self, mock_get_deployment, mock_get_execution_for_deployment, mock_update_task_info_for_minion_allocations, - mock_begin_tasks + mock_begin_tasks, ): - mock_get_deployment.return_value.last_execution_status = \ + mock_get_deployment.return_value.last_execution_status = ( constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ) testutils.get_wrapped_function( - self.server.confirm_deployment_minions_allocation)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.minion_machine_allocations + self.server.confirm_deployment_minions_allocation + )( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + mock.sentinel.minion_machine_allocations, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_get_execution_for_deployment.assert_called_once_with( - mock.sentinel.context, - mock_get_deployment.return_value, - requery=False + mock.sentinel.context, mock_get_deployment.return_value, requery=False ) mock_update_task_info_for_minion_allocations.assert_called_once_with( mock.sentinel.context, mock_get_deployment.return_value, - mock.sentinel.minion_machine_allocations + mock.sentinel.minion_machine_allocations, ) mock_begin_tasks.assert_called_once_with( mock.sentinel.context, mock_get_deployment.return_value, - mock_get_execution_for_deployment.return_value + mock_get_execution_for_deployment.return_value, ) @mock.patch.object(server.ConductorServerEndpoint, '_begin_tasks') - @mock.patch.object(server.ConductorServerEndpoint, - '_update_task_info_for_minion_allocations') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_execution_for_deployment') + @mock.patch.object( + server.ConductorServerEndpoint, '_update_task_info_for_minion_allocations' + ) + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_for_deployment') @mock.patch.object(server.ConductorServerEndpoint, '_get_deployment') def test_confirm_deployment_minions_allocation_unexpected_status( self, mock_get_deployment, mock_get_execution_for_deployment, mock_update_task_info_for_minion_allocations, - mock_begin_tasks + mock_begin_tasks, ): - mock_get_deployment.return_value.last_execution_status = \ + mock_get_deployment.return_value.last_execution_status = ( constants.EXECUTION_STATUS_CANCELED + ) self.assertRaises( exception.InvalidDeploymentState, testutils.get_wrapped_function( - self.server.confirm_deployment_minions_allocation), + self.server.confirm_deployment_minions_allocation + ), self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.minion_machine_allocations + mock.sentinel.minion_machine_allocations, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_get_execution_for_deployment.assert_not_called() mock_update_task_info_for_minion_allocations.assert_not_called() mock_begin_tasks.assert_not_called() - @mock.patch.object(server.ConductorServerEndpoint, - '_set_tasks_execution_status') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_execution_for_deployment') + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_for_deployment') @mock.patch.object(server.ConductorServerEndpoint, '_get_deployment') def test_report_deployment_minions_allocation_error( self, mock_get_deployment, mock_get_execution_for_deployment, mock_cancel_tasks_execution, - mock_set_tasks_execution_status + mock_set_tasks_execution_status, ): - mock_get_deployment.return_value.last_execution_status = \ + mock_get_deployment.return_value.last_execution_status = ( constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS + ) testutils.get_wrapped_function( - self.server.report_deployment_minions_allocation_error)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.minion_allocation_error_details + self.server.report_deployment_minions_allocation_error + )( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + mock.sentinel.minion_allocation_error_details, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_get_execution_for_deployment.assert_called_once_with( - mock.sentinel.context, - mock_get_deployment.return_value, - requery=False + mock.sentinel.context, mock_get_deployment.return_value, requery=False ) mock_cancel_tasks_execution.assert_called_once_with( mock.sentinel.context, mock_get_execution_for_deployment.return_value, - requery=True + requery=True, ) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, mock_get_execution_for_deployment.return_value, - constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS + constants.EXECUTION_STATUS_ERROR_ALLOCATING_MINIONS, ) - @mock.patch.object(server.ConductorServerEndpoint, - '_set_tasks_execution_status') - @mock.patch.object(server.ConductorServerEndpoint, - '_cancel_tasks_execution') - @mock.patch.object(server.ConductorServerEndpoint, - '_get_execution_for_deployment') + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_for_deployment') @mock.patch.object(server.ConductorServerEndpoint, '_get_deployment') def test_report_deployment_minions_allocation_error_unexpected_status( self, mock_get_deployment, mock_get_execution_for_deployment, mock_cancel_tasks_execution, - mock_set_tasks_execution_status + mock_set_tasks_execution_status, ): - mock_get_deployment.return_value.last_execution_status = \ + mock_get_deployment.return_value.last_execution_status = ( constants.EXECUTION_STATUS_CANCELED + ) self.assertRaises( exception.InvalidDeploymentState, testutils.get_wrapped_function( - self.server.report_deployment_minions_allocation_error), + self.server.report_deployment_minions_allocation_error + ), self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.minion_allocation_error_details + mock.sentinel.minion_allocation_error_details, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_get_execution_for_deployment.assert_not_called() mock_cancel_tasks_execution.assert_not_called() @@ -3129,12 +2857,12 @@ def test_report_deployment_minions_allocation_error_unexpected_status( 'WorkerClient', ) def test_cancel_tasks_execution_no_config( - self, - mock_worker_client, - mock_set_task_status, - mock_advance_execution_state, - mock_set_tasks_execution_status, - mock_get_tasks_execution + self, + mock_worker_client, + mock_set_task_status, + mock_advance_execution_state, + mock_set_tasks_execution_status, + mock_get_tasks_execution, ): execution = mock.Mock( id=mock.sentinel.execution_id, @@ -3142,8 +2870,8 @@ def test_cancel_tasks_execution_no_config( ) def call_cancel_tasks_execution( - requery=False, - force=False, + requery=False, + force=False, ): self.server._cancel_tasks_execution( mock.sentinel.context, @@ -3199,19 +2927,17 @@ def call_cancel_tasks_execution( ) ] call_cancel_tasks_execution() - mock_worker_client.return_value.cancel_task\ - .assert_called_once_with( - mock.sentinel.context, - execution.tasks[0].id, - execution.tasks[0].process_id, - False - ) + mock_worker_client.return_value.cancel_task.assert_called_once_with( + mock.sentinel.context, + execution.tasks[0].id, + execution.tasks[0].process_id, + False, + ) # if worker_rpc.cancel_task fails # _set_task_status should be called with FAILED_TO_CANCEL mock_set_task_status.reset_mock() - mock_worker_client.return_value.cancel_task\ - .side_effect = Exception() + mock_worker_client.return_value.cancel_task.side_effect = Exception() call_cancel_tasks_execution() mock_set_task_status.assert_any_call( mock.sentinel.context, @@ -3221,26 +2947,20 @@ def call_cancel_tasks_execution( ) @mock.patch.object(db_api, 'get_deployment') - def test__get_deployment( - self, - mock_get_deployment - ): + def test__get_deployment(self, mock_get_deployment): result = self.server._get_deployment( mock.sentinel.context, mock.sentinel.deployment_id, include_task_info=False, - to_dict=False - ) - self.assertEqual( - mock_get_deployment.return_value, - result + to_dict=False, ) + self.assertEqual(mock_get_deployment.return_value, result) mock_get_deployment.assert_called_once_with( mock.sentinel.context, mock.sentinel.deployment_id, include_task_info=False, - to_dict=False + to_dict=False, ) mock_get_deployment.reset_mock() mock_get_deployment.return_value = None @@ -3251,14 +2971,14 @@ def test__get_deployment( mock.sentinel.context, mock.sentinel.deployment_id, include_task_info=False, - to_dict=False + to_dict=False, ) mock_get_deployment.assert_called_once_with( mock.sentinel.context, mock.sentinel.deployment_id, include_task_info=False, - to_dict=False + to_dict=False, ) @mock.patch.object(db_api, 'get_tasks_execution') @@ -3277,23 +2997,21 @@ def test__get_deployment( ) @ddt.file_data("data/cancel_tasks_execution_config.yml") @ddt.unpack - def test_cancel_tasks_execution( - self, - mock_worker_client, - mock_set_task_status, - mock_advance_execution_state, - mock_set_tasks_execution_status, - mock_get_tasks_execution, - config, - expected_status, + def test_cancel_tasks_execution( + self, + mock_worker_client, + mock_set_task_status, + mock_advance_execution_state, + mock_set_tasks_execution_status, + mock_get_tasks_execution, + config, + expected_status, ): force = config.get('force', False) tasks = config.get('tasks', []) on_error = config.get('on_error') hides_exception_details = config.get('hides_exception_details', False) - depends_on = config.get('depends_on') and [ - mock.sentinel.depends_on - ] + depends_on = config.get('depends_on') and [mock.sentinel.depends_on] execution = mock.Mock( id=mock.sentinel.execution_id, status=constants.EXECUTION_STATUS_RUNNING, @@ -3304,8 +3022,9 @@ def test_cancel_tasks_execution( status=t, depends_on=depends_on, on_error=on_error, - ) for i, t in enumerate(tasks) - ] + ) + for i, t in enumerate(tasks) + ], ) self.server._cancel_tasks_execution( @@ -3323,17 +3042,20 @@ def test_cancel_tasks_execution( kwargs = {'exception_details': mock.ANY} if hides_exception_details: kwargs = {} - mock_set_task_status.assert_has_calls([ - mock.call( - mock.sentinel.context, - execution_task.id, - expected_status, - **kwargs - ) - ]) + mock_set_task_status.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + execution_task.id, + expected_status, + **kwargs, + ) + ] + ) - @mock.patch.object(server.ConductorServerEndpoint, - '_deallocate_minion_machines_for_action') + @mock.patch.object( + server.ConductorServerEndpoint, '_deallocate_minion_machines_for_action' + ) @mock.patch.object(db_api, 'get_action') @mock.patch.object(keystone, 'delete_trust') @mock.patch.object(db_api, 'set_execution_status') @@ -3342,31 +3064,24 @@ def test_set_tasks_execution_status( mock_set_execution_status, mock_delete_trust, mock_get_action, - mock_deallocate_minion_machines_for_action + mock_deallocate_minion_machines_for_action, ): context = mock.Mock() execution = mock.Mock() def call_set_tasks_execution_status(new_execution_status): self.server._set_tasks_execution_status( - context, - execution, - new_execution_status + context, execution, new_execution_status ) context.delete_trust_id = mock.sentinel.delete_trust_id - execution.status = (constants. - EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS) + execution.status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS call_set_tasks_execution_status(constants.EXECUTION_STATUS_COMPLETED) mock_set_execution_status.assert_called_once_with( - context, - execution.id, - constants.EXECUTION_STATUS_COMPLETED - ) - mock_delete_trust.assert_called_once_with( - context + context, execution.id, constants.EXECUTION_STATUS_COMPLETED ) + mock_delete_trust.assert_called_once_with(context) mock_get_action.assert_not_called() mock_set_execution_status.reset_mock() @@ -3376,23 +3091,19 @@ def call_set_tasks_execution_status(new_execution_status): call_set_tasks_execution_status(constants.EXECUTION_STATUS_CANCELED) mock_set_execution_status.assert_called_once_with( - context, - execution.id, - constants.EXECUTION_STATUS_CANCELED + context, execution.id, constants.EXECUTION_STATUS_CANCELED ) mock_delete_trust.assert_not_called() - mock_get_action.assert_called_once_with( - context, execution.action_id) + mock_get_action.assert_called_once_with(context, execution.action_id) mock_deallocate_minion_machines_for_action.assert_called_once_with( - context, mock_get_action.return_value) + context, mock_get_action.return_value + ) mock_set_execution_status.reset_mock() mock_get_action.reset_mock() call_set_tasks_execution_status(constants.EXECUTION_STATUS_RUNNING) mock_set_execution_status.assert_called_once_with( - context, - execution.id, - constants.EXECUTION_STATUS_RUNNING + context, execution.id, constants.EXECUTION_STATUS_RUNNING ) mock_delete_trust.assert_not_called() mock_get_action.assert_not_called() @@ -3401,15 +3112,13 @@ def call_set_tasks_execution_status(new_execution_status): @mock.patch.object(db_api, 'set_task_host_properties') @mock.patch.object(db_api, 'get_task') def test_set_task_host( - self, - mock_get_task, - mock_set_task_host_properties, - mock_set_task_status, + self, + mock_get_task, + mock_set_task_host_properties, + mock_set_task_status, ): def call_set_task_host(): - testutils.get_wrapped_function( - self.server.set_task_host - )( + testutils.get_wrapped_function(self.server.set_task_host)( self.server, mock.sentinel.context, mock.sentinel.task_id, @@ -3428,18 +3137,14 @@ def call_set_task_host(): ) mock_get_task.return_value.status = constants.TASK_STATUS_CANCELLING - self.assertRaises( - exception.TaskIsCancelling, - call_set_task_host - ) + self.assertRaises(exception.TaskIsCancelling, call_set_task_host) - mock_get_task.return_value.status = constants\ - .TASK_STATUS_CANCELLING_AFTER_COMPLETION + mock_get_task.return_value.status = ( + constants.TASK_STATUS_CANCELLING_AFTER_COMPLETION + ) call_set_task_host() mock_set_task_host_properties.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.task_id, - host=mock.sentinel.host + mock.sentinel.context, mock.sentinel.task_id, host=mock.sentinel.host ) mock_set_task_status.assert_called_once_with( mock.sentinel.context, @@ -3462,15 +3167,13 @@ def call_set_task_host(): @mock.patch.object(db_api, 'set_task_host_properties') @mock.patch.object(db_api, 'get_task') def test_set_task_process( - self, - mock_get_task, - mock_set_task_host_properties, - mock_set_task_status, + self, + mock_get_task, + mock_set_task_host_properties, + mock_set_task_status, ): def call_set_task_host(): - testutils.get_wrapped_function( - self.server.set_task_process - )( + testutils.get_wrapped_function(self.server.set_task_process)( self.server, mock.sentinel.context, mock.sentinel.task_id, @@ -3478,9 +3181,7 @@ def call_set_task_host(): ) # task status is not in accepted state - with self.assertRaisesRegex( - exception.InvalidTaskState, - "expected statuses"): + with self.assertRaisesRegex(exception.InvalidTaskState, "expected statuses"): call_set_task_host() mock_get_task.assert_called_once_with( @@ -3494,7 +3195,7 @@ def call_set_task_host(): mock_set_task_host_properties.assert_called_once_with( mock.sentinel.context, mock.sentinel.task_id, - process_id=mock.sentinel.process_id + process_id=mock.sentinel.process_id, ) mock_set_task_status.assert_called_once_with( mock.sentinel.context, @@ -3506,30 +3207,25 @@ def call_set_task_host(): mock_get_task.return_value = mock.Mock( host=None, ) - with self.assertRaisesRegex( - exception.InvalidTaskState, - "has no host"): + with self.assertRaisesRegex(exception.InvalidTaskState, "has no host"): call_set_task_host() - @mock.patch.object( - server.ConductorServerEndpoint, - '_set_tasks_execution_status' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') @mock.patch.object(db_api, 'set_task_status') @mock.patch.object(db_api, 'get_tasks_execution') def test_check_clean_execution_deadlock( - self, - mock_get_tasks_execution, - mock_set_task_status, - mock_set_tasks_execution_status, + self, + mock_get_tasks_execution, + mock_set_task_status, + mock_set_tasks_execution_status, ): execution = mock.Mock( id=mock.sentinel.execution_id, ) def call_check_clean_execution_deadlock( - task_statuses=None, - requery=False, + task_statuses=None, + requery=False, ): return self.server._check_clean_execution_deadlock( mock.sentinel.context, @@ -3545,10 +3241,7 @@ def call_check_clean_execution_deadlock( mock.sentinel.execution_id, ) # RUNNING is default state - self.assertEqual( - determined_state, - constants.EXECUTION_STATUS_RUNNING - ) + self.assertEqual(determined_state, constants.EXECUTION_STATUS_RUNNING) # is deadlocked with 2 tasks that should be stranded task_statuses = { @@ -3558,29 +3251,28 @@ def call_check_clean_execution_deadlock( determined_state = call_check_clean_execution_deadlock( task_statuses=task_statuses, ) - mock_set_task_status.assert_has_calls([ - mock.call( - mock.sentinel.context, - mock.sentinel.task_1, - constants.TASK_STATUS_CANCELED_FROM_DEADLOCK, - exception_details=mock.ANY, - ), - mock.call( - mock.sentinel.context, - mock.sentinel.task_2, - constants.TASK_STATUS_CANCELED_FROM_DEADLOCK, - exception_details=mock.ANY, - ), - ]) + mock_set_task_status.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + mock.sentinel.task_1, + constants.TASK_STATUS_CANCELED_FROM_DEADLOCK, + exception_details=mock.ANY, + ), + mock.call( + mock.sentinel.context, + mock.sentinel.task_2, + constants.TASK_STATUS_CANCELED_FROM_DEADLOCK, + exception_details=mock.ANY, + ), + ] + ) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, execution, constants.EXECUTION_STATUS_DEADLOCKED, ) - self.assertEqual( - determined_state, - constants.EXECUTION_STATUS_DEADLOCKED - ) + self.assertEqual(determined_state, constants.EXECUTION_STATUS_DEADLOCKED) # has a pending task, not deadlocked task_statuses = { @@ -3594,10 +3286,7 @@ def call_check_clean_execution_deadlock( ) mock_set_task_status.assert_not_called() mock_set_tasks_execution_status.assert_not_called() - self.assertEqual( - determined_state, - constants.EXECUTION_STATUS_RUNNING - ) + self.assertEqual(determined_state, constants.EXECUTION_STATUS_RUNNING) # deadlocked with 2 tasks but one is not stranded task_statuses = { @@ -3618,22 +3307,19 @@ def call_check_clean_execution_deadlock( execution, constants.EXECUTION_STATUS_DEADLOCKED, ) - self.assertEqual( - determined_state, - constants.EXECUTION_STATUS_DEADLOCKED - ) + self.assertEqual(determined_state, constants.EXECUTION_STATUS_DEADLOCKED) @mock.patch.object(db_api, 'get_tasks_execution') def test_get_execution_status_no_config( - self, - mock_get_tasks_execution, + self, + mock_get_tasks_execution, ): execution = mock.Mock( id=mock.sentinel.execution_id, ) def call_get_execution_status( - requery=False, + requery=False, ): return self.server._get_execution_status( mock.sentinel.context, @@ -3648,18 +3334,11 @@ def call_get_execution_status( mock.sentinel.execution_id, ) # default state is COMPLETED - self.assertEqual( - status, - constants.EXECUTION_STATUS_COMPLETED - ) + self.assertEqual(status, constants.EXECUTION_STATUS_COMPLETED) @ddt.file_data("data/get_execution_status_config.yml") @ddt.unpack - def test_get_execution_status( - self, - config, - expected_status - ): + def test_get_execution_status(self, config, expected_status): tasks = config.get('tasks', []) execution = mock.Mock( id=mock.sentinel.execution_id, @@ -3667,7 +3346,8 @@ def test_get_execution_status( mock.Mock( id=f'task_{status}', status=status, - ) for status in tasks + ) + for status in tasks ], ) status = self.server._get_execution_status( @@ -3677,51 +3357,34 @@ def test_get_execution_status( ) self.assertEqual(status, expected_status) + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') @mock.patch.object( - server.ConductorServerEndpoint, - '_set_tasks_execution_status' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_execution_status' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_cancel_tasks_execution' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_worker_service_rpc_for_task' + server.ConductorServerEndpoint, '_get_worker_service_rpc_for_task' ) @mock.patch.object(db_api, 'set_task_status') @mock.patch.object(db_api, 'get_endpoint') @mock.patch.object(db_api, 'get_action') + @mock.patch.object(server.ConductorServerEndpoint, '_get_task_destination') + @mock.patch.object(server.ConductorServerEndpoint, '_get_task_origin') @mock.patch.object( - server.ConductorServerEndpoint, - '_get_task_destination' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_task_origin' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_check_clean_execution_deadlock' + server.ConductorServerEndpoint, '_check_clean_execution_deadlock' ) @mock.patch.object(db_api, 'get_tasks_execution') def test_advance_execution_state_no_config( - self, - mock_get_tasks_execution, - mock_check_clean_execution_deadlock, - mock_get_task_origin, - mock_get_task_destination, - mock_get_action, - mock_get_endpoint, - mock_set_task_status, - mock_get_worker_service_rpc_for_task, - mock_cancel_tasks_execution, - mock_get_execution_status, - mock_set_tasks_execution_status, + self, + mock_get_tasks_execution, + mock_check_clean_execution_deadlock, + mock_get_task_origin, + mock_get_task_destination, + mock_get_action, + mock_get_endpoint, + mock_set_task_status, + mock_get_worker_service_rpc_for_task, + mock_cancel_tasks_execution, + mock_get_execution_status, + mock_set_tasks_execution_status, ): # no active status and requery started_tasks = self.server._advance_execution_state( @@ -3748,7 +3411,7 @@ def test_advance_execution_state_no_config( ) def call_advance_execution_state( - requery=False, + requery=False, ): return self.server._advance_execution_state( mock.sentinel.context, @@ -3759,8 +3422,7 @@ def call_advance_execution_state( # call with no tasks with self.assertRaisesRegex( - exception.InvalidActionTasksExecutionState, - "no tasks" + exception.InvalidActionTasksExecutionState, "no tasks" ): call_advance_execution_state() @@ -3775,28 +3437,28 @@ def call_advance_execution_state( started_tasks = call_advance_execution_state() mock_get_task_origin.assert_called_once_with( - mock.sentinel.context, - execution.action + mock.sentinel.context, execution.action ) mock_get_task_destination.assert_called_once_with( - mock.sentinel.context, - execution.action + mock.sentinel.context, execution.action ) mock_get_action.assert_called_once_with( mock.sentinel.context, execution.action_id, include_task_info=True, ) - mock_get_endpoint.assert_has_calls([ - mock.call( - mock.sentinel.context, - execution.action.origin_endpoint_id, - ), - mock.call( - mock.sentinel.context, - execution.action.destination_endpoint_id, - ), - ]) + mock_get_endpoint.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + execution.action.origin_endpoint_id, + ), + mock.call( + mock.sentinel.context, + execution.action.destination_endpoint_id, + ), + ] + ) mock_check_clean_execution_deadlock.assert_called_with( mock.sentinel.context, execution, @@ -3815,8 +3477,9 @@ def call_advance_execution_state( self.assertEqual(started_tasks, []) # execution is deadlocked - mock_check_clean_execution_deadlock\ - .return_value = constants.EXECUTION_STATUS_DEADLOCKED + mock_check_clean_execution_deadlock.return_value = ( + constants.EXECUTION_STATUS_DEADLOCKED + ) mock_get_execution_status.reset_mock() started_tasks = call_advance_execution_state() mock_get_execution_status.assert_not_called() @@ -3824,8 +3487,7 @@ def call_advance_execution_state( # last execution status is the execution status mock_check_clean_execution_deadlock.reset_mock(return_value=True) - mock_get_execution_status\ - .return_value = constants.EXECUTION_STATUS_RUNNING + mock_get_execution_status.return_value = constants.EXECUTION_STATUS_RUNNING mock_set_tasks_execution_status.reset_mock() call_advance_execution_state() mock_set_tasks_execution_status.assert_not_called() @@ -3844,9 +3506,7 @@ def call_advance_execution_state( 'test': 'info', }, } - mock_get_action.return_value = mock.Mock( - info=task_info - ) + mock_get_action.return_value = mock.Mock(info=task_info) started_tasks = call_advance_execution_state() mock_get_worker_service_rpc_for_task.assert_called_once_with( mock.sentinel.context, @@ -3854,21 +3514,19 @@ def call_advance_execution_state( mock.ANY, mock.ANY, ) - mock_get_worker_service_rpc_for_task.return_value\ - .begin_task.assert_called_once_with( - mock.sentinel.context, - task_id=mock.sentinel.task_1, - task_type=mock.sentinel.task_type, - origin=mock_get_task_origin.return_value, - destination=mock_get_task_destination.return_value, - instance=mock.sentinel.instance, - task_info=task_info[mock.sentinel.instance], - ) + mock_get_worker_service_rpc_for_task.return_value.begin_task.assert_called_once_with( + mock.sentinel.context, + task_id=mock.sentinel.task_1, + task_type=mock.sentinel.task_type, + origin=mock_get_task_origin.return_value, + destination=mock_get_task_destination.return_value, + instance=mock.sentinel.instance, + task_info=task_info[mock.sentinel.instance], + ) self.assertEqual(started_tasks, [task.id]) # handles worker service rpc error - mock_get_worker_service_rpc_for_task.side_effect = ( - CoriolisTestException()) + mock_get_worker_service_rpc_for_task.side_effect = CoriolisTestException() self.assertRaises( CoriolisTestException, call_advance_execution_state, @@ -3879,54 +3537,38 @@ def call_advance_execution_state( requery=True, ) + @mock.patch.object(server.ConductorServerEndpoint, '_set_tasks_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_get_execution_status') + @mock.patch.object(server.ConductorServerEndpoint, '_cancel_tasks_execution') @mock.patch.object( - server.ConductorServerEndpoint, - '_set_tasks_execution_status' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_execution_status' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_cancel_tasks_execution' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_worker_service_rpc_for_task' + server.ConductorServerEndpoint, '_get_worker_service_rpc_for_task' ) @mock.patch.object(db_api, 'set_task_status') @mock.patch.object(db_api, 'get_endpoint') @mock.patch.object(db_api, 'get_action') + @mock.patch.object(server.ConductorServerEndpoint, '_get_task_destination') + @mock.patch.object(server.ConductorServerEndpoint, '_get_task_origin') @mock.patch.object( - server.ConductorServerEndpoint, - '_get_task_destination' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_get_task_origin' - ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_check_clean_execution_deadlock' + server.ConductorServerEndpoint, '_check_clean_execution_deadlock' ) @mock.patch.object(db_api, 'get_tasks_execution') @ddt.file_data("data/advance_execution_state.yml") @ddt.unpack def test_advance_execution_state_scheduled_tasks( - self, - mock_get_tasks_execution, - mock_check_clean_execution_deadlock, - mock_get_task_origin, - mock_get_task_destination, - mock_get_action, - mock_get_endpoint, - mock_set_task_status, - mock_get_worker_service_rpc_for_task, - mock_cancel_tasks_execution, - mock_get_execution_status, - mock_set_tasks_execution_status, - config): + self, + mock_get_tasks_execution, + mock_check_clean_execution_deadlock, + mock_get_task_origin, + mock_get_task_destination, + mock_get_action, + mock_get_endpoint, + mock_set_task_status, + mock_get_worker_service_rpc_for_task, + mock_cancel_tasks_execution, + mock_get_execution_status, + mock_set_tasks_execution_status, + config, + ): tasks = config.get('tasks', []) execution = mock.Mock( status=constants.EXECUTION_STATUS_RUNNING, @@ -3937,7 +3579,8 @@ def test_advance_execution_state_scheduled_tasks( status=t['status'], on_error=t.get('on_error', None), depends_on=t.get('depends_on', []), - ) for i, t in enumerate(tasks) + ) + for i, t in enumerate(tasks) ], ) @@ -3954,50 +3597,50 @@ def test_advance_execution_state_scheduled_tasks( kwargs = {'exception_details': mock.ANY} if task['expected_status'] == constants.TASK_STATUS_PENDING: kwargs = {} - mock_set_task_status.assert_has_calls([ - mock.call( - mock.sentinel.context, - task['id'], - task['expected_status'], - **kwargs - ) - ]) + mock_set_task_status.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + task['id'], + task['expected_status'], + **kwargs, + ) + ] + ) self.assertEqual( started_tasks, - [t['id'] for t in tasks - if 'expected_status' in t and t['expected_status'] == - constants.TASK_STATUS_PENDING] + [ + t['id'] + for t in tasks + if 'expected_status' in t + and t['expected_status'] == constants.TASK_STATUS_PENDING + ], ) @mock.patch.object(db_api, 'update_transfer_action_info_for_instance') def test_update_transfer_volumes_info( - self, - mock_update_transfer_action_info_for_instance + self, mock_update_transfer_action_info_for_instance ): self.server._update_transfer_volumes_info( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.instance, - mock.sentinel.updated_task_info + mock.sentinel.updated_task_info, ) mock_update_transfer_action_info_for_instance.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.instance, - mock.sentinel.updated_task_info + mock.sentinel.updated_task_info, ) - @mock.patch.object(server.ConductorServerEndpoint, - '_update_transfer_volumes_info') + @mock.patch.object(server.ConductorServerEndpoint, '_update_transfer_volumes_info') @mock.patch.object(lockutils, 'lock') @mock.patch.object(db_api, 'get_deployment') def test_update_volumes_info_for_deployment_parent_transfer( - self, - mock_get_deployment, - mock_lock, - mock_update_transfer_volumes_info + self, mock_get_deployment, mock_lock, mock_update_transfer_volumes_info ): deployment = mock.Mock() mock_get_deployment.return_value = deployment @@ -4006,50 +3649,43 @@ def test_update_volumes_info_for_deployment_parent_transfer( mock.sentinel.context, mock.sentinel.deployment_id, mock.sentinel.instance, - mock.sentinel.updated_task_info + mock.sentinel.updated_task_info, ) mock_get_deployment.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.deployment_id + mock.sentinel.context, mock.sentinel.deployment_id ) mock_lock.assert_called_once_with( - constants.TRANSFER_LOCK_NAME_FORMAT % - mock_get_deployment.return_value.transfer_id, - external=True + constants.TRANSFER_LOCK_NAME_FORMAT + % mock_get_deployment.return_value.transfer_id, + external=True, ) mock_update_transfer_volumes_info.assert_called_once_with( mock.sentinel.context, mock_get_deployment.return_value.transfer_id, mock.sentinel.instance, - mock.sentinel.updated_task_info + mock.sentinel.updated_task_info, ) - @mock.patch.object( - server.ConductorServerEndpoint, - '_minion_manager_client' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_minion_manager_client') @mock.patch.object(db_api, 'update_minion_machine') @mock.patch.object(db_api, 'update_transfer') - @mock.patch.object( - server.ConductorServerEndpoint, - '_update_transfer_volumes_info' - ) + @mock.patch.object(server.ConductorServerEndpoint, '_update_transfer_volumes_info') @mock.patch.object(db_api, 'set_transfer_action_result') @mock.patch.object(schemas, 'validate_value') @mock.patch.object( server.ConductorServerEndpoint, - '_update_volumes_info_for_deployment_parent_transfer' + '_update_volumes_info_for_deployment_parent_transfer', ) def test_handle_post_task_actions( - self, - mock_update_volumes_info_for_deployment_parent_transfer, - mock_validate_value, - mock_set_transfer_action_result, - mock_update_transfer_volumes_info, - mock_update_transfer, - mock_update_minion_machine, - mock_minion_manager_client, + self, + mock_update_volumes_info_for_deployment_parent_transfer, + mock_validate_value, + mock_set_transfer_action_result, + mock_update_transfer_volumes_info, + mock_update_transfer, + mock_update_minion_machine, + mock_minion_manager_client, ): task = mock.Mock( task_type=constants.TASK_TYPE_RESTORE_TRANSFER_DISK_SNAPSHOTS, @@ -4069,11 +3705,11 @@ def call_handle_post_task_actions(): execution, task_info, ) + call_handle_post_task_actions() # no volumes_info - mock_update_volumes_info_for_deployment_parent_transfer\ - .assert_not_called() + mock_update_volumes_info_for_deployment_parent_transfer.assert_not_called() # has volumes_info task_info["volumes_info"] = [ @@ -4082,25 +3718,22 @@ def call_handle_post_task_actions(): } ] call_handle_post_task_actions() - mock_update_volumes_info_for_deployment_parent_transfer\ - .assert_called_once_with( - mock.sentinel.context, - mock.sentinel.action_id, - mock.sentinel.instance, - {"volumes_info": task_info["volumes_info"]}, - ) + mock_update_volumes_info_for_deployment_parent_transfer.assert_called_once_with( + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.instance, + {"volumes_info": task_info["volumes_info"]}, + ) - task.task_type = constants\ - .TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS + task.task_type = constants.TASK_TYPE_DELETE_TRANSFER_TARGET_DISK_SNAPSHOTS call_handle_post_task_actions() # no clone_disks, reset volumes_info - mock_update_volumes_info_for_deployment_parent_transfer\ - .assert_called_with( - mock.sentinel.context, - mock.sentinel.action_id, - mock.sentinel.instance, - {"volumes_info": []}, - ) + mock_update_volumes_info_for_deployment_parent_transfer.assert_called_with( + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.instance, + {"volumes_info": []}, + ) # has clone_disks task_info['clone_disks'] = [ @@ -4108,11 +3741,9 @@ def call_handle_post_task_actions(): 'id': 'clone_disk_id', } ] - mock_update_volumes_info_for_deployment_parent_transfer\ - .reset_mock() + mock_update_volumes_info_for_deployment_parent_transfer.reset_mock() call_handle_post_task_actions() - mock_update_volumes_info_for_deployment_parent_transfer\ - .assert_not_called() + mock_update_volumes_info_for_deployment_parent_transfer.assert_not_called() types = [ constants.TASK_TYPE_FINALIZE_INSTANCE_DEPLOYMENT, @@ -4134,8 +3765,7 @@ def call_handle_post_task_actions(): ] call_handle_post_task_actions() mock_validate_value.assert_called_once_with( - task_info['transfer_result'], - schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA + task_info['transfer_result'], schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA ) mock_set_transfer_action_result.assert_called_once_with( mock.sentinel.context, @@ -4146,9 +3776,7 @@ def call_handle_post_task_actions(): # handles schema validation error mock_set_transfer_action_result.reset_mock() - mock_validate_value.side_effect = ( - exception.SchemaValidationException() - ) + mock_validate_value.side_effect = exception.SchemaValidationException() call_handle_post_task_actions() mock_set_transfer_action_result.assert_not_called() mock_validate_value.side_effect = None @@ -4190,9 +3818,7 @@ def call_handle_post_task_actions(): ] call_handle_post_task_actions() mock_update_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.action_id, - task_info + mock.sentinel.context, mock.sentinel.action_id, task_info ) mock_update_transfer.reset_mock() @@ -4212,8 +3838,7 @@ def call_handle_post_task_actions(): mock_update_minion_machine.assert_called_once_with( mock.sentinel.context, task_info['origin_minion_machine_id'], - {'provider_properties': [ - {'minion_provider_properties': 'value'}]}, + {'provider_properties': [{'minion_provider_properties': 'value'}]}, ) mock_update_minion_machine.reset_mock() @@ -4233,8 +3858,7 @@ def call_handle_post_task_actions(): mock_update_minion_machine.assert_called_once_with( mock.sentinel.context, task_info['destination_minion_machine_id'], - {'provider_properties': [ - {'minion_provider_properties': 'value'}]}, + {'provider_properties': [{'minion_provider_properties': 'value'}]}, ) mock_update_minion_machine.reset_mock() @@ -4254,19 +3878,17 @@ def call_handle_post_task_actions(): mock_update_minion_machine.assert_called_once_with( mock.sentinel.context, task_info['osmorphing_minion_machine_id'], - {'provider_properties': [ - {'minion_provider_properties': 'value'}]}, + {'provider_properties': [{'minion_provider_properties': 'value'}]}, ) mock_update_minion_machine.reset_mock() # TASK_TYPE_RELEASE_SOURCE_MINION task.task_type = constants.TASK_TYPE_RELEASE_SOURCE_MINION call_handle_post_task_actions() - mock_minion_manager_client.deallocate_minion_machine\ - .assert_called_once_with( - mock.sentinel.context, - task_info['origin_minion_machine_id'], - ) + mock_minion_manager_client.deallocate_minion_machine.assert_called_once_with( + mock.sentinel.context, + task_info['origin_minion_machine_id'], + ) mock_minion_manager_client.deallocate_minion_machine.reset_mock() # TASK_TYPE_RELEASE_DESTINATION_MINION @@ -4275,29 +3897,26 @@ def call_handle_post_task_actions(): # destination minion machine is the same as osmorphing minion machine task_info['destination_minion_machine_id'] = ['other id'] call_handle_post_task_actions() - mock_minion_manager_client.deallocate_minion_machine\ - .assert_called_once_with( - mock.sentinel.context, - task_info['destination_minion_machine_id'], - ) + mock_minion_manager_client.deallocate_minion_machine.assert_called_once_with( + mock.sentinel.context, + task_info['destination_minion_machine_id'], + ) mock_minion_manager_client.deallocate_minion_machine.reset_mock() # destination minion machine is different # from osmorphing minion machine task_info['destination_minion_machine_id'] = ['minion_machine_id'] call_handle_post_task_actions() - mock_minion_manager_client.deallocate_minion_machine\ - .assert_not_called() + mock_minion_manager_client.deallocate_minion_machine.assert_not_called() mock_minion_manager_client.deallocate_minion_machine.reset_mock() # TASK_TYPE_RELEASE_OSMORPHING_MINION task.task_type = constants.TASK_TYPE_RELEASE_OSMORPHING_MINION call_handle_post_task_actions() - mock_minion_manager_client.deallocate_minion_machine\ - .assert_called_once_with( - mock.sentinel.context, - task_info['osmorphing_minion_machine_id'], - ) + mock_minion_manager_client.deallocate_minion_machine.assert_called_once_with( + mock.sentinel.context, + task_info['osmorphing_minion_machine_id'], + ) mock_minion_manager_client.deallocate_minion_machine.reset_mock() # for any other type of task nothing is called @@ -4305,8 +3924,7 @@ def call_handle_post_task_actions(): call_handle_post_task_actions() mock_update_transfer.assert_not_called() mock_update_minion_machine.assert_not_called() - mock_minion_manager_client.deallocate_minion_machine\ - .assert_not_called() + mock_minion_manager_client.deallocate_minion_machine.assert_not_called() @mock.patch.object(utils, "sanitize_task_info") @mock.patch.object(db_api, "get_task") @@ -4318,16 +3936,16 @@ def call_handle_post_task_actions(): @ddt.file_data("data/task_completed_config.yml") @ddt.unpack def test_task_completed( - self, - mock_lock, - mock_update_transfer_action_info, - mock_get_action, - mock_get_tasks_execution, - mock_set_task_status, - mock_get_task, - mock_sanitize_task_info, - config, - expected_status, + self, + mock_lock, + mock_update_transfer_action_info, + mock_get_action, + mock_get_tasks_execution, + mock_set_task_status, + mock_get_task, + mock_sanitize_task_info, + config, + expected_status, ): task_status = config['task_status'] has_exception_details = config.get('has_exception_details', True) @@ -4342,16 +3960,13 @@ def test_task_completed( action_id=mock.sentinel.action_id, tasks=[ mock.Mock( - id=mock.sentinel.task_id, - status=constants.TASK_STATUS_COMPLETED + id=mock.sentinel.task_id, status=constants.TASK_STATUS_COMPLETED ) - ] + ], ) self.server.task_completed( - mock.sentinel.context, - mock.sentinel.task_id, - mock.sentinel.task_result + mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.task_result ) if expected_status is None: mock_set_task_status.assert_not_called() @@ -4387,19 +4002,12 @@ def test_task_completed( mock_update_transfer_action_info.reset_mock() # no task result - self.server.task_completed( - mock.sentinel.context, - mock.sentinel.task_id, - None - ) + self.server.task_completed(mock.sentinel.context, mock.sentinel.task_id, None) mock_update_transfer_action_info.assert_not_called() self.assertEqual(2, mock_get_action.call_count) @mock.patch.object(db_api, 'set_task_status') - def test_cancel_execution_for_osmorphing_debugging( - self, - mock_set_task_status - ): + def test_cancel_execution_for_osmorphing_debugging(self, mock_set_task_status): subtask = mock.Mock() execution = mock.Mock() execution.tasks = [subtask] @@ -4407,8 +4015,7 @@ def test_cancel_execution_for_osmorphing_debugging( subtask.task_type = constants.TASK_TYPE_OS_MORPHING subtask.status = constants.TASK_STATUS_PENDING self.server._cancel_execution_for_osmorphing_debugging( - mock.sentinel.context, - execution + mock.sentinel.context, execution ) mock_set_task_status.assert_not_called() @@ -4417,31 +4024,28 @@ def test_cancel_execution_for_osmorphing_debugging( exception.CoriolisException, self.server._cancel_execution_for_osmorphing_debugging, mock.sentinel.context, - execution + execution, ) subtask.status = constants.TASK_STATUS_COMPLETED self.server._cancel_execution_for_osmorphing_debugging( - mock.sentinel.context, - execution + mock.sentinel.context, execution ) mock_set_task_status.assert_not_called() subtask.status = constants.TASK_STATUS_SCHEDULED subtask.on_error = True self.server._cancel_execution_for_osmorphing_debugging( - mock.sentinel.context, - execution + mock.sentinel.context, execution ) mock_set_task_status.assert_called_once_with( mock.sentinel.context, subtask.id, constants.TASK_STATUS_CANCELED_FOR_DEBUGGING, - exception_details=mock.ANY + exception_details=mock.ANY, ) - @mock.patch.object(server.ConductorServerEndpoint, - "_advance_execution_state") + @mock.patch.object(server.ConductorServerEndpoint, "_advance_execution_state") @mock.patch.object(db_api, "get_tasks_execution") @mock.patch.object(db_api, "set_task_status") @mock.patch.object(db_api, "get_task") @@ -4455,7 +4059,7 @@ def test_confirm_task_cancellation( mock_advance_execution_state, task_status, expected_final_status, - expected_advance_execution_state_call + expected_advance_execution_state_call, ): task = mock.Mock() task.status = getattr(constants, task_status) @@ -4469,42 +4073,35 @@ def test_confirm_task_cancellation( self.server, mock.sentinel.context, mock.sentinel.task_id, - mock.sentinel.cancellation_details + mock.sentinel.cancellation_details, ) mock_set_task_status.assert_called_once_with( mock.sentinel.context, task.id, expected_final_status, - exception_details=mock.ANY + exception_details=mock.ANY, ) if expected_advance_execution_state_call: mock_get_tasks_execution.assert_called_once_with( - mock.sentinel.context, task.execution_id) + mock.sentinel.context, task.execution_id + ) mock_advance_execution_state.assert_called_once_with( mock.sentinel.context, mock_get_tasks_execution.return_value, - requery=False + requery=False, ) else: mock_get_tasks_execution.assert_not_called() mock_advance_execution_state.assert_not_called() @mock.patch.object( - server.ConductorServerEndpoint, - "_check_delete_reservation_for_transfer" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_cancel_tasks_execution" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_set_tasks_execution_status" + server.ConductorServerEndpoint, "_check_delete_reservation_for_transfer" ) + @mock.patch.object(server.ConductorServerEndpoint, "_cancel_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, "_set_tasks_execution_status") @mock.patch.object( - server.ConductorServerEndpoint, - "_cancel_execution_for_osmorphing_debugging" + server.ConductorServerEndpoint, "_cancel_execution_for_osmorphing_debugging" ) @mock.patch.object(lockutils, "lock") @mock.patch.object(db_api, "get_action") @@ -4514,18 +4111,18 @@ def test_confirm_task_cancellation( @ddt.file_data("data/set_task_error_config.yml") @ddt.unpack def test_set_task_error( - self, - mock_get_task, - mock_set_task_status, - mock_get_tasks_execution, - mock_get_action, - mock_lock, - mock_cancel_execution_for_osmorphing_debugging, - mock_set_tasks_execution_status, - mock_cancel_tasks_execution, - mock_check_delete_reservation_for_transfer, - config, - expected_status, + self, + mock_get_task, + mock_set_task_status, + mock_get_tasks_execution, + mock_get_action, + mock_lock, + mock_cancel_execution_for_osmorphing_debugging, + mock_set_tasks_execution_status, + mock_cancel_tasks_execution, + mock_check_delete_reservation_for_transfer, + config, + expected_status, ): task_status = config['task_status'] mock_get_tasks_execution.return_value = mock.Mock( @@ -4533,10 +4130,9 @@ def test_set_task_error( action_id=mock.sentinel.action_id, tasks=[ mock.Mock( - id=mock.sentinel.task_id, - status=constants.TASK_STATUS_COMPLETED + id=mock.sentinel.task_id, status=constants.TASK_STATUS_COMPLETED ) - ] + ], ) mock_get_task.return_value = mock.Mock( status=task_status, @@ -4559,11 +4155,7 @@ def test_set_task_error( @mock.patch.object(db_api, "add_task_event") @mock.patch.object(db_api, "get_task") - def test_add_task_event( - self, - mock_get_task, - mock_add_task_event - ): + def test_add_task_event(self, mock_get_task, mock_add_task_event): task = mock.Mock() mock_get_task.return_value = task task.status = constants.EXECUTION_STATUS_COMPLETED @@ -4575,12 +4167,11 @@ def test_add_task_event( mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.level, - mock.sentinel.message + mock.sentinel.message, ) mock_get_task.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.task_id + mock.sentinel.context, mock.sentinel.task_id ) mock_add_task_event.assert_not_called() @@ -4592,26 +4183,23 @@ def test_add_task_event( mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.level, - mock.sentinel.message + mock.sentinel.message, ) mock_get_task.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.task_id + mock.sentinel.context, mock.sentinel.task_id ) mock_add_task_event.assert_called_once_with( mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.level, - mock.sentinel.message + mock.sentinel.message, ) @mock.patch.object(db_api, "add_task_progress_update") @mock.patch.object(db_api, "get_task") def test_add_task_progress_update( - self, - mock_get_task, - mock_add_task_progress_update + self, mock_get_task, mock_add_task_progress_update ): task = mock.Mock() mock_get_task.return_value = task @@ -4619,58 +4207,47 @@ def test_add_task_progress_update( self.assertRaises( exception.InvalidTaskState, - testutils.get_wrapped_function( - self.server.add_task_progress_update), + testutils.get_wrapped_function(self.server.add_task_progress_update), self.server, mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.message, initial_step=mock.sentinel.initial_step, - total_steps=mock.sentinel.total_steps + total_steps=mock.sentinel.total_steps, ) mock_get_task.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.task_id + mock.sentinel.context, mock.sentinel.task_id ) mock_add_task_progress_update.assert_not_called() mock_get_task.reset_mock() task.status = constants.EXECUTION_STATUS_RUNNING - result = testutils.get_wrapped_function( - self.server.add_task_progress_update)( + result = testutils.get_wrapped_function(self.server.add_task_progress_update)( self.server, mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.message, initial_step=mock.sentinel.initial_step, - total_steps=mock.sentinel.total_steps + total_steps=mock.sentinel.total_steps, ) - self.assertEqual( - mock_add_task_progress_update.return_value, - result - ) + self.assertEqual(mock_add_task_progress_update.return_value, result) mock_get_task.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.task_id + mock.sentinel.context, mock.sentinel.task_id ) mock_add_task_progress_update.assert_called_once_with( mock.sentinel.context, mock.sentinel.task_id, mock.sentinel.message, initial_step=mock.sentinel.initial_step, - total_steps=mock.sentinel.total_steps + total_steps=mock.sentinel.total_steps, ) @mock.patch.object(db_api, "update_task_progress_update") - def test_update_task_progress_update( - self, - mock_update_task_progress_update - ): - testutils.get_wrapped_function( - self.server.update_task_progress_update)( + def test_update_task_progress_update(self, mock_update_task_progress_update): + testutils.get_wrapped_function(self.server.update_task_progress_update)( self.server, mock.sentinel.context, mock.sentinel.task_id, @@ -4690,26 +4267,20 @@ def test_update_task_progress_update( ) @mock.patch.object(db_api, "get_transfer_schedule") - def test__get_transfer_schedule( - self, - mock_get_transfer_schedule - ): + def test__get_transfer_schedule(self, mock_get_transfer_schedule): result = self.server._get_transfer_schedule( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, - expired=True + expired=True, ) - self.assertEqual( - mock_get_transfer_schedule.return_value, - result - ) + self.assertEqual(mock_get_transfer_schedule.return_value, result) mock_get_transfer_schedule.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, - expired=True + expired=True, ) mock_get_transfer_schedule.reset_mock() @@ -4721,14 +4292,14 @@ def test__get_transfer_schedule( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, - expired=False + expired=False, ) mock_get_transfer_schedule.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, - expired=False + expired=False, ) @mock.patch.object(server.ConductorServerEndpoint, "get_transfer_schedule") @@ -4742,7 +4313,7 @@ def test_create_transfer_schedule( mock_get_transfer, mock_transfer_schedule, mock_add_transfer_schedule, - mock_get_transfer_schedule + mock_get_transfer_schedule, ): context = mock.Mock() transfer_schedule = mock.Mock() @@ -4756,13 +4327,10 @@ def test_create_transfer_schedule( mock.sentinel.enabled, mock.sentinel.exp_date, mock.sentinel.shutdown_instance, - mock.sentinel.auto_deploy + mock.sentinel.auto_deploy, ) - self.assertEqual( - mock_get_transfer_schedule.return_value, - result - ) + self.assertEqual(mock_get_transfer_schedule.return_value, result) self.assertEqual( ( transfer_schedule.transfer, @@ -4772,7 +4340,7 @@ def test_create_transfer_schedule( transfer_schedule.enabled, transfer_schedule.shutdown_instance, transfer_schedule.auto_deploy, - transfer_schedule.trust_id + transfer_schedule.trust_id, ), ( mock_get_transfer.return_value, @@ -4782,8 +4350,8 @@ def test_create_transfer_schedule( mock.sentinel.enabled, mock.sentinel.shutdown_instance, mock.sentinel.auto_deploy, - mock.sentinel.trust_id - ) + mock.sentinel.trust_id, + ), ) mock_create_trust.assert_called_once_with(context) mock_get_transfer.assert_called_once_with( @@ -4792,44 +4360,33 @@ def test_create_transfer_schedule( ) mock_transfer_schedule.assert_called_once() mock_add_transfer_schedule.assert_called_once_with( - context, - transfer_schedule, - mock.ANY + context, transfer_schedule, mock.ANY ) mock_get_transfer_schedule.assert_called_once_with( - context, - mock.sentinel.transfer_id, - transfer_schedule.id + context, mock.sentinel.transfer_id, transfer_schedule.id ) - @mock.patch.object( - server.ConductorServerEndpoint, "_get_transfer_schedule") + @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer_schedule") @mock.patch.object(db_api, "update_transfer_schedule") def test_update_transfer_schedule( - self, - mock_update_transfer_schedule, - mock_get_transfer_schedule + self, mock_update_transfer_schedule, mock_get_transfer_schedule ): - result = testutils.get_wrapped_function( - self.server.update_transfer_schedule)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.schedule_id, - mock.sentinel.updated_values, + result = testutils.get_wrapped_function(self.server.update_transfer_schedule)( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + mock.sentinel.schedule_id, + mock.sentinel.updated_values, ) - self.assertEqual( - mock_get_transfer_schedule.return_value, - result - ) + self.assertEqual(mock_get_transfer_schedule.return_value, result) mock_update_transfer_schedule.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, mock.sentinel.updated_values, None, - mock.ANY + mock.ANY, ) mock_get_transfer_schedule.assert_called_once_with( mock.sentinel.context, @@ -4849,14 +4406,10 @@ def test_cleanup_schedule_resources( schedule = mock.Mock() schedule.trust_id = None - self.server._cleanup_schedule_resources( - mock.sentinel.context, - schedule - ) + self.server._cleanup_schedule_resources(mock.sentinel.context, schedule) mock_transfer_cron_client.unregister.assert_called_once_with( - mock.sentinel.context, - schedule + mock.sentinel.context, schedule ) mock_get_admin_context.assert_not_called() mock_delete_trust.assert_not_called() @@ -4864,26 +4417,18 @@ def test_cleanup_schedule_resources( mock_transfer_cron_client.reset_mock() schedule.trust_id = mock.sentinel.trust_id - self.server._cleanup_schedule_resources( - mock.sentinel.context, - schedule - ) + self.server._cleanup_schedule_resources(mock.sentinel.context, schedule) mock_transfer_cron_client.unregister.assert_called_once_with( - mock.sentinel.context, - schedule + mock.sentinel.context, schedule ) - mock_get_admin_context.assert_called_once_with( - trust_id=mock.sentinel.trust_id) - mock_delete_trust.assert_called_once_with( - mock_get_admin_context.return_value) + mock_get_admin_context.assert_called_once_with(trust_id=mock.sentinel.trust_id) + mock_delete_trust.assert_called_once_with(mock_get_admin_context.return_value) @mock.patch.object(db_api, "delete_transfer_schedule") @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer") def test_delete_transfer_schedule( - self, - mock_get_transfer, - mock_delete_transfer_schedule + self, mock_get_transfer, mock_delete_transfer_schedule ): transfer = mock.Mock() transfer.last_execution_status = constants.EXECUTION_STATUS_COMPLETED @@ -4893,19 +4438,18 @@ def test_delete_transfer_schedule( self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.schedule_id + mock.sentinel.schedule_id, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_delete_transfer_schedule.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, None, - mock.ANY + mock.ANY, ) mock_get_transfer.reset_mock() @@ -4914,78 +4458,62 @@ def test_delete_transfer_schedule( self.assertRaises( exception.InvalidTransferState, - testutils.get_wrapped_function( - self.server.delete_transfer_schedule), + testutils.get_wrapped_function(self.server.delete_transfer_schedule), self.server, mock.sentinel.context, mock.sentinel.transfer_id, - mock.sentinel.schedule_id + mock.sentinel.schedule_id, ) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id + mock.sentinel.context, mock.sentinel.transfer_id ) mock_delete_transfer_schedule.assert_not_called() @mock.patch.object(db_api, "get_transfer_schedules") def test_get_transfer_schedules(self, mock_get_transfer_schedules): - result = testutils.get_wrapped_function( - self.server.get_transfer_schedules)( - self.server, - mock.sentinel.context, - transfer_id=None, - expired=True + result = testutils.get_wrapped_function(self.server.get_transfer_schedules)( + self.server, mock.sentinel.context, transfer_id=None, expired=True ) - self.assertEqual( - mock_get_transfer_schedules.return_value, - result - ) + self.assertEqual(mock_get_transfer_schedules.return_value, result) mock_get_transfer_schedules.assert_called_once_with( - mock.sentinel.context, - transfer_id=None, - expired=True + mock.sentinel.context, transfer_id=None, expired=True ) @mock.patch.object(db_api, "get_transfer_schedule") def test_get_transfer_schedule(self, mock_get_transfer_schedule): - result = testutils.get_wrapped_function( - self.server.get_transfer_schedule)( - self.server, - mock.sentinel.context, - mock.sentinel.transfer_id, - mock.sentinel.schedule_id, - expired=True + result = testutils.get_wrapped_function(self.server.get_transfer_schedule)( + self.server, + mock.sentinel.context, + mock.sentinel.transfer_id, + mock.sentinel.schedule_id, + expired=True, ) - self.assertEqual( - mock_get_transfer_schedule.return_value, - result - ) + self.assertEqual(mock_get_transfer_schedule.return_value, result) mock_get_transfer_schedule.assert_called_once_with( mock.sentinel.context, mock.sentinel.transfer_id, mock.sentinel.schedule_id, - expired=True + expired=True, ) - @mock.patch.object(server.ConductorServerEndpoint, - "get_transfer_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, "get_transfer_tasks_execution") @mock.patch.object(server.ConductorServerEndpoint, "_begin_tasks") @mock.patch.object(db_api, "add_transfer_tasks_execution") @mock.patch.object(db_api, "update_transfer_action_info_for_instance") - @mock.patch.object(server.ConductorServerEndpoint, - "_check_execution_tasks_sanity") + @mock.patch.object(server.ConductorServerEndpoint, "_check_execution_tasks_sanity") @mock.patch.object(server.ConductorServerEndpoint, "_create_task") @mock.patch.object(utils, "sanitize_task_info") @mock.patch.object(models, "TasksExecution") - @mock.patch.object(server.ConductorServerEndpoint, - "_check_valid_transfer_tasks_execution") - @mock.patch.object(server.ConductorServerEndpoint, - "_check_transfer_running_executions") - @mock.patch.object(server.ConductorServerEndpoint, - "_check_minion_pools_for_action") + @mock.patch.object( + server.ConductorServerEndpoint, "_check_valid_transfer_tasks_execution" + ) + @mock.patch.object( + server.ConductorServerEndpoint, "_check_transfer_running_executions" + ) + @mock.patch.object(server.ConductorServerEndpoint, "_check_minion_pools_for_action") @mock.patch.object(models, "Transfer") @mock.patch.object(server.ConductorServerEndpoint, "_get_transfer") @ddt.file_data("data/update_transfer_config.yml") @@ -5007,7 +4535,7 @@ def test_update_transfer( mock_get_transfer_tasks_execution, config, has_updated_values, - has_transfer_instance + has_transfer_instance, ): transfer = mock.Mock() dummy = mock.Mock() @@ -5023,17 +4551,12 @@ def test_update_transfer( self.server, mock.sentinel.context, mock.sentinel.transfer_id, - updated_properties + updated_properties, ) - self.assertEqual( - mock_get_transfer_tasks_execution.return_value, - result - ) + self.assertEqual(mock_get_transfer_tasks_execution.return_value, result) mock_get_transfer.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - include_task_info=True + mock.sentinel.context, mock.sentinel.transfer_id, include_task_info=True ) mock_check_transfer_running_executions.assert_called_once_with( mock.sentinel.context, @@ -5043,32 +4566,22 @@ def test_update_transfer( mock_get_transfer.return_value, force=True, ) - self.assertEqual( - execution.action, - mock_get_transfer.return_value - ) + self.assertEqual(execution.action, mock_get_transfer.return_value) mock_check_execution_tasks_sanity.assert_called_once_with( - execution, - transfer.info + execution, transfer.info ) mock_add_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - execution + mock.sentinel.context, execution ) mock_begin_tasks.assert_called_once_with( - mock.sentinel.context, - transfer, - execution + mock.sentinel.context, transfer, execution ) mock_get_transfer_tasks_execution.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.transfer_id, - execution.id + mock.sentinel.context, mock.sentinel.transfer_id, execution.id ) if has_updated_values: mock_check_minion_pools_for_action.assert_called_once_with( - mock.sentinel.context, - dummy + mock.sentinel.context, dummy ) else: mock_check_minion_pools_for_action.assert_not_called() @@ -5077,103 +4590,87 @@ def test_update_transfer( create_task_calls = [] update_transfer_action_info_for_instance_calls = [] for instance in config['transfer'].get("info", {}): + expected_sanitize_task_info_calls.append(mock.call(mock.ANY)) expected_sanitize_task_info_calls.append( - mock.call(mock.ANY)) - expected_sanitize_task_info_calls.append( - mock.call(transfer.info[instance])) - create_task_calls.append(mock.call( - instance, - constants.TASK_TYPE_GET_INSTANCE_INFO, - execution)) - create_task_calls.append(mock.call( - instance, - constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, - execution)) - create_task_calls.append(mock.call( - instance, - constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER, - execution, - depends_on=mock.ANY)) + mock.call(transfer.info[instance]) + ) + create_task_calls.append( + mock.call( + instance, constants.TASK_TYPE_GET_INSTANCE_INFO, execution + ) + ) + create_task_calls.append( + mock.call( + instance, constants.TASK_TYPE_UPDATE_SOURCE_TRANSFER, execution + ) + ) + create_task_calls.append( + mock.call( + instance, + constants.TASK_TYPE_UPDATE_DESTINATION_TRANSFER, + execution, + depends_on=mock.ANY, + ) + ) update_transfer_action_info_for_instance_calls.append( mock.call( mock.sentinel.context, transfer.id, instance, - transfer.info[instance]) + transfer.info[instance], + ) ) - mock_sanitize_task_info.assert_has_calls( - expected_sanitize_task_info_calls) + mock_sanitize_task_info.assert_has_calls(expected_sanitize_task_info_calls) mock_create_task.assert_has_calls(create_task_calls) mock_update_transfer_action_info_for_instance.assert_has_calls( - update_transfer_action_info_for_instance_calls) + update_transfer_action_info_for_instance_calls + ) @mock.patch.object(server.ConductorServerEndpoint, "get_region") @mock.patch.object(db_api, "add_region") @mock.patch.object(models, "Region") - def test_create_region( - self, - mock_Region, - mock_add_region, - mock_get_region - ): + def test_create_region(self, mock_Region, mock_add_region, mock_get_region): result = self.server.create_region( mock.sentinel.context, mock.sentinel.region_name, description=mock.sentinel.description, - enabled=True + enabled=True, ) - self.assertEqual( - mock_get_region.return_value, - result - ) + self.assertEqual(mock_get_region.return_value, result) self.assertEqual( ( mock_Region.return_value.name, mock_Region.return_value.description, - mock_Region.return_value.enabled + mock_Region.return_value.enabled, ), - ( - mock.sentinel.region_name, - mock.sentinel.description, - True - ) + (mock.sentinel.region_name, mock.sentinel.description, True), ) mock_Region.assert_called_once() mock_add_region.assert_called_once_with( - mock.sentinel.context, - mock_Region.return_value + mock.sentinel.context, mock_Region.return_value ) mock_get_region.assert_called_once_with( - mock.sentinel.context, - mock_Region.return_value.id + mock.sentinel.context, mock_Region.return_value.id ) @mock.patch.object(db_api, "get_regions") def test_get_regions(self, mock_get_regions): result = self.server.get_regions(mock.sentinel.context) - self.assertEqual( - mock_get_regions.return_value, - result - ) + self.assertEqual(mock_get_regions.return_value, result) mock_get_regions.assert_called_once_with(mock.sentinel.context) @mock.patch.object(db_api, "get_region") def test_get_region(self, mock_get_region): - result = testutils.get_wrapped_function( - self.server.get_region)( - self.server, - mock.sentinel.context, - mock.sentinel.region_id + result = testutils.get_wrapped_function(self.server.get_region)( + self.server, mock.sentinel.context, mock.sentinel.region_id ) - self.assertEqual( - mock_get_region.return_value, - result - ) + self.assertEqual(mock_get_region.return_value, result) mock_get_region.assert_called_once_with( - mock.sentinel.context, mock.sentinel.region_id) + mock.sentinel.context, mock.sentinel.region_id + ) mock_get_region.reset_mock() mock_get_region.return_value = None @@ -5185,44 +4682,36 @@ def test_get_region(self, mock_get_region): mock.sentinel.region_id, ) mock_get_region.assert_called_once_with( - mock.sentinel.context, mock.sentinel.region_id) + mock.sentinel.context, mock.sentinel.region_id + ) @mock.patch.object(db_api, "get_region") @mock.patch.object(db_api, "update_region") - def test_update_region( - self, - mock_update_region, - mock_get_region - ): - result = testutils.get_wrapped_function( - self.server.update_region)( + def test_update_region(self, mock_update_region, mock_get_region): + result = testutils.get_wrapped_function(self.server.update_region)( self.server, mock.sentinel.context, mock.sentinel.region_id, - mock.sentinel.updated_values + mock.sentinel.updated_values, ) - self.assertEqual( - mock_get_region.return_value, - result - ) + self.assertEqual(mock_get_region.return_value, result) mock_update_region.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.region_id, - mock.sentinel.updated_values + mock.sentinel.context, mock.sentinel.region_id, mock.sentinel.updated_values ) mock_get_region.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.region_id + mock.sentinel.context, mock.sentinel.region_id ) @mock.patch.object(db_api, "delete_region") def test_delete_region(self, mock_delete_region): testutils.get_wrapped_function(self.server.delete_region)( - self.server, mock.sentinel.context, mock.sentinel.region_id) + self.server, mock.sentinel.context, mock.sentinel.region_id + ) mock_delete_region.assert_called_once_with( - mock.sentinel.context, mock.sentinel.region_id) + mock.sentinel.context, mock.sentinel.region_id + ) @mock.patch.object(rpc_worker_client.WorkerClient, "get_service_status") @mock.patch.object(db_api, "delete_service") @@ -5239,7 +4728,7 @@ def test_register_service( mock_get_service, mock_update_service, mock_delete_service, - mock_get_service_status + mock_get_service_status, ): self.assertRaises( exception.Conflict, @@ -5251,14 +4740,14 @@ def test_register_service( mock.sentinel.enabled, mapped_regions=mock.sentinel.mapped_regions, providers=mock.sentinel.providers, - specs=mock.sentinel.specs + specs=mock.sentinel.specs, ) mock_find_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - topic=mock.sentinel.topic + topic=mock.sentinel.topic, ) mock_find_service.reset_mock() @@ -5272,13 +4761,10 @@ def test_register_service( mock.sentinel.enabled, mapped_regions=mock.sentinel.mapped_regions, providers=mock.sentinel.providers, - specs=mock.sentinel.specs + specs=mock.sentinel.specs, ) - self.assertEqual( - mock_get_service.return_value, - result - ) + self.assertEqual(mock_get_service.return_value, result) self.assertEqual( ( mock_Service.return_value.host, @@ -5286,7 +4772,7 @@ def test_register_service( mock_Service.return_value.topic, mock_Service.return_value.enabled, mock_Service.return_value.providers, - mock_Service.return_value.specs + mock_Service.return_value.specs, ), ( mock.sentinel.host, @@ -5294,30 +4780,28 @@ def test_register_service( mock.sentinel.topic, mock.sentinel.enabled, mock.sentinel.providers, - mock.sentinel.specs - ) + mock.sentinel.specs, + ), ) mock_find_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - topic=mock.sentinel.topic + topic=mock.sentinel.topic, ) mock_Service.assert_called_once() mock_add_service.assert_called_once_with( - mock.sentinel.context, - mock_Service.return_value + mock.sentinel.context, mock_Service.return_value ) mock_get_service_status.assert_not_called() mock_update_service.assert_called_once_with( mock.sentinel.context, mock_Service.return_value.id, - {"mapped_regions": mock.sentinel.mapped_regions} + {"mapped_regions": mock.sentinel.mapped_regions}, ) mock_delete_service.assert_not_called() mock_get_service.assert_called_once_with( - mock.sentinel.context, - mock_Service.return_value.id + mock.sentinel.context, mock_Service.return_value.id ) mock_add_service.reset_mock() @@ -5334,32 +4818,26 @@ def test_register_service( mock.sentinel.enabled, mapped_regions=mock.sentinel.mapped_regions, providers=None, - specs=None + specs=None, ) self.assertEqual( - ( - mock_Service.return_value.providers, - mock_Service.return_value.specs - ), + (mock_Service.return_value.providers, mock_Service.return_value.specs), ( mock_get_service_status.return_value["providers"], - mock_get_service_status.return_value["specs"] - ) + mock_get_service_status.return_value["specs"], + ), ) mock_add_service.assert_called_once_with( - mock.sentinel.context, - mock_Service.return_value + mock.sentinel.context, mock_Service.return_value ) - mock_get_service_status.assert_called_once_with( - mock.sentinel.context) + mock_get_service_status.assert_called_once_with(mock.sentinel.context) mock_update_service.assert_called_once_with( mock.sentinel.context, mock_Service.return_value.id, - {"mapped_regions": mock.sentinel.mapped_regions} + {"mapped_regions": mock.sentinel.mapped_regions}, ) mock_delete_service.assert_called_once_with( - mock.sentinel.context, - mock_Service.return_value.id + mock.sentinel.context, mock_Service.return_value.id ) mock_get_service.assert_not_called() @@ -5369,18 +4847,15 @@ def test_check_service_registered(self, mock_find_service): mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - mock.sentinel.topic + mock.sentinel.topic, ) - self.assertEqual( - mock_find_service.return_value, - result - ) + self.assertEqual(mock_find_service.return_value, result) mock_find_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - topic=mock.sentinel.topic + topic=mock.sentinel.topic, ) @mock.patch.object(db_api, "find_service") @@ -5390,82 +4865,60 @@ def test_check_service_registered_no_service(self, mock_find_service): mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - mock.sentinel.topic + mock.sentinel.topic, ) - self.assertEqual( - None, - result - ) + self.assertEqual(None, result) mock_find_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.host, mock.sentinel.binary, - topic=mock.sentinel.topic + topic=mock.sentinel.topic, ) @mock.patch.object(db_api, "update_service") @mock.patch.object(rpc_worker_client.WorkerClient, "get_service_status") @mock.patch.object(db_api, "get_service", new_callable=mock.Mock) def test_refresh_service_status( - self, - mock_get_service, - mock_get_service_status, - mock_update_service + self, mock_get_service, mock_get_service_status, mock_update_service ): - result = testutils.get_wrapped_function( - self.server.refresh_service_status)( + result = testutils.get_wrapped_function(self.server.refresh_service_status)( self.server, mock.sentinel.context, mock.sentinel.service_id, ) - self.assertEqual( - mock_get_service.return_value, - result + self.assertEqual(mock_get_service.return_value, result) + mock_get_service.assert_has_calls( + [mock.call(mock.sentinel.context, mock.sentinel.service_id)] * 2 ) - mock_get_service.assert_has_calls([ - mock.call( - mock.sentinel.context, - mock.sentinel.service_id - ) - ] * 2) - mock_get_service_status.assert_called_once_with( - mock.sentinel.context) + mock_get_service_status.assert_called_once_with(mock.sentinel.context) mock_update_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.service_id, { "providers": mock_get_service_status.return_value["providers"], "specs": mock_get_service_status.return_value["specs"], - "status": constants.SERVICE_STATUS_UP - } + "status": constants.SERVICE_STATUS_UP, + }, ) @mock.patch.object(db_api, "get_services") def test_get_services(self, mock_get_services): result = self.server.get_services(mock.sentinel.context) - self.assertEqual( - mock_get_services.return_value, - result - ) + self.assertEqual(mock_get_services.return_value, result) mock_get_services.assert_called_once_with(mock.sentinel.context) @mock.patch.object(db_api, "get_service") def test_get_service(self, mock_get_service): - result = testutils.get_wrapped_function( - self.server.get_service)( - self.server, - mock.sentinel.context, - mock.sentinel.service_id + result = testutils.get_wrapped_function(self.server.get_service)( + self.server, mock.sentinel.context, mock.sentinel.service_id ) - self.assertEqual( - mock_get_service.return_value, - result - ) + self.assertEqual(mock_get_service.return_value, result) mock_get_service.assert_called_once_with( - mock.sentinel.context, mock.sentinel.service_id) + mock.sentinel.context, mock.sentinel.service_id + ) mock_get_service.reset_mock() mock_get_service.return_value = None @@ -5477,64 +4930,47 @@ def test_get_service(self, mock_get_service): mock.sentinel.service_id, ) mock_get_service.assert_called_once_with( - mock.sentinel.context, mock.sentinel.service_id) + mock.sentinel.context, mock.sentinel.service_id + ) @mock.patch.object(db_api, "delete_service") def test_delete_service(self, mock_delete_service): testutils.get_wrapped_function(self.server.delete_service)( - self.server, - mock.sentinel.context, - mock.sentinel.service_id + self.server, mock.sentinel.context, mock.sentinel.service_id ) mock_delete_service.assert_called_once_with( - mock.sentinel.context, mock.sentinel.service_id) + mock.sentinel.context, mock.sentinel.service_id + ) @mock.patch.object(db_api, "get_service") @mock.patch.object(db_api, "update_service") - def test_update_service( - self, - mock_update_service, - mock_get_service - ): - result = testutils.get_wrapped_function( - self.server.update_service)( + def test_update_service(self, mock_update_service, mock_get_service): + result = testutils.get_wrapped_function(self.server.update_service)( self.server, mock.sentinel.context, mock.sentinel.service_id, - mock.sentinel.updated_values + mock.sentinel.updated_values, ) - self.assertEqual( - mock_get_service.return_value, - result - ) + self.assertEqual(mock_get_service.return_value, result) mock_update_service.assert_called_once_with( mock.sentinel.context, mock.sentinel.service_id, - mock.sentinel.updated_values + mock.sentinel.updated_values, ) mock_get_service.assert_called_once_with( - mock.sentinel.context, - mock.sentinel.service_id + mock.sentinel.context, mock.sentinel.service_id ) @mock.patch.object(cfg.CONF, "conductor") @mock.patch.object( - server.ConductorServerEndpoint, - "_check_delete_reservation_for_transfer" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_cancel_tasks_execution" - ) - @mock.patch.object( - server.ConductorServerEndpoint, - "_set_tasks_execution_status" + server.ConductorServerEndpoint, "_check_delete_reservation_for_transfer" ) + @mock.patch.object(server.ConductorServerEndpoint, "_cancel_tasks_execution") + @mock.patch.object(server.ConductorServerEndpoint, "_set_tasks_execution_status") @mock.patch.object( - server.ConductorServerEndpoint, - "_cancel_execution_for_osmorphing_debugging" + server.ConductorServerEndpoint, "_cancel_execution_for_osmorphing_debugging" ) @mock.patch.object(lockutils, "lock") @mock.patch.object(db_api, "get_action") @@ -5542,17 +4978,17 @@ def test_update_service( @mock.patch.object(db_api, "set_task_status") @mock.patch.object(db_api, "get_task") def test_set_task_error_os_morphing( - self, - mock_get_task, - mock_set_task_status, - mock_get_tasks_execution, - mock_get_action, - mock_lock, - mock_cancel_execution_for_osmorphing_debugging, - mock_set_tasks_execution_status, - mock_cancel_tasks_execution, - mock_check_delete_reservation_for_transfer, - mock_conf_conductor, + self, + mock_get_task, + mock_set_task_status, + mock_get_tasks_execution, + mock_get_action, + mock_lock, + mock_cancel_execution_for_osmorphing_debugging, + mock_set_tasks_execution_status, + mock_cancel_tasks_execution, + mock_check_delete_reservation_for_transfer, + mock_conf_conductor, ): execution = mock.Mock( type=constants.EXECUTION_TYPE_TRANSFER_UPDATE, @@ -5563,7 +4999,7 @@ def test_set_task_error_os_morphing( status=constants.TASK_STATUS_COMPLETED, task_type=constants.TASK_TYPE_OS_MORPHING, ) - ] + ], ) # non running tasks of type OS Morphing with debugging enabled @@ -5579,12 +5015,14 @@ def test_set_task_error_os_morphing( mock.sentinel.exception_details, ) mock_cancel_execution_for_osmorphing_debugging.assert_called_once_with( - mock.sentinel.context, mock_get_tasks_execution.return_value, ) + mock.sentinel.context, + mock_get_tasks_execution.return_value, + ) self.assertEqual(2, mock_set_task_status.call_count) mock_set_tasks_execution_status.assert_called_once_with( mock.sentinel.context, mock_get_tasks_execution.return_value, - constants.EXECUTION_STATUS_CANCELED_FOR_DEBUGGING + constants.EXECUTION_STATUS_CANCELED_FOR_DEBUGGING, ) mock_cancel_tasks_execution.assert_not_called() diff --git a/coriolis/tests/conductor/rpc/test_utils.py b/coriolis/tests/conductor/rpc/test_utils.py index 1ce3664a..f11be9f9 100644 --- a/coriolis/tests/conductor/rpc/test_utils.py +++ b/coriolis/tests/conductor/rpc/test_utils.py @@ -2,12 +2,11 @@ # All Rights Reserved. import time - from unittest import mock +from coriolis import utils from coriolis.conductor.rpc import utils as rpc_utils from coriolis.tests import test_base -from coriolis import utils class CoriolisTestException(Exception): @@ -20,13 +19,10 @@ class ConductorUtilsTestCase(test_base.CoriolisBaseTestCase): @mock.patch.object(utils, 'get_exception_details') @mock.patch.object(time, 'sleep') def test_check_create_registration_for_service( - self, - mock_sleep, - mock_get_exception_details + self, mock_sleep, mock_get_exception_details ): conductor_rpc = mock.Mock() - conductor_rpc.check_service_registered.return_value = { - 'id': mock.sentinel.id} + conductor_rpc.check_service_registered.return_value = {'id': mock.sentinel.id} result = rpc_utils.check_create_registration_for_service( conductor_rpc, mock.sentinel.request_context, @@ -37,26 +33,23 @@ def test_check_create_registration_for_service( mapped_regions=None, providers=mock.sentinel.providers, specs=mock.sentinel.specs, - retry_period=30 + retry_period=30, ) - self.assertEqual( - conductor_rpc.update_service.return_value, - result - ) + self.assertEqual(conductor_rpc.update_service.return_value, result) conductor_rpc.check_service_registered.assert_called_once_with( mock.sentinel.request_context, mock.sentinel.host, mock.sentinel.binary, - mock.sentinel.topic + mock.sentinel.topic, ) conductor_rpc.update_service.assert_called_once_with( mock.sentinel.request_context, mock.sentinel.id, updated_values={ "providers": mock.sentinel.providers, - "specs": mock.sentinel.specs - } + "specs": mock.sentinel.specs, + }, ) conductor_rpc.register_service.assert_not_called() mock_sleep.assert_not_called() @@ -65,13 +58,13 @@ def test_check_create_registration_for_service( @mock.patch.object(utils, 'get_exception_details') @mock.patch.object(time, 'sleep') def test_check_create_registration_for_service_register_service( - self, - mock_sleep, - mock_get_exception_details + self, mock_sleep, mock_get_exception_details ): conductor_rpc = mock.Mock() - conductor_rpc.check_service_registered.side_effect = \ - [CoriolisTestException(), None] + conductor_rpc.check_service_registered.side_effect = [ + CoriolisTestException(), + None, + ] period = 30 with self.assertLogs('coriolis.conductor.rpc.utils', level='WARN'): result = rpc_utils.check_create_registration_for_service( @@ -84,20 +77,20 @@ def test_check_create_registration_for_service_register_service( mapped_regions=None, providers=mock.sentinel.providers, specs=mock.sentinel.specs, - retry_period=30 + retry_period=30, ) - self.assertEqual( - conductor_rpc.register_service.return_value, - result - ) + self.assertEqual(conductor_rpc.register_service.return_value, result) conductor_rpc.check_service_registered.assert_has_calls( - [mock.call( - mock.sentinel.request_context, - mock.sentinel.host, - mock.sentinel.binary, - mock.sentinel.topic - )] * 2 + [ + mock.call( + mock.sentinel.request_context, + mock.sentinel.host, + mock.sentinel.binary, + mock.sentinel.topic, + ) + ] + * 2 ) conductor_rpc.register_service.assert_called_once_with( mock.sentinel.request_context, @@ -107,7 +100,7 @@ def test_check_create_registration_for_service_register_service( False, mapped_regions=None, providers=mock.sentinel.providers, - specs=mock.sentinel.specs + specs=mock.sentinel.specs, ) conductor_rpc.update_service.assert_not_called() mock_sleep.assert_called_once_with(period) diff --git a/coriolis/tests/cron/test_cron.py b/coriolis/tests/cron/test_cron.py index a9889b35..927a8dd3 100644 --- a/coriolis/tests/cron/test_cron.py +++ b/coriolis/tests/cron/test_cron.py @@ -2,16 +2,15 @@ # All Rights Reserved. import datetime -import ddt - -import schedule import sys import time from unittest import mock +import ddt +import schedule + +from coriolis import exception, schemas from coriolis.cron import cron -from coriolis import exception -from coriolis import schemas from coriolis.tests import test_base @@ -30,10 +29,7 @@ def setUp(self, *_): mock_on_error = mock.Mock() mock_job_callable = mock.Mock() args = ['arg1', 'arg2'] - kw = { - "arg3": "mock_arg3", - "arg4": "mock_arg4" - } + kw = {"arg3": "mock_arg3", "arg4": "mock_arg4"} self.cron = cron.CronJob( mock.sentinel.name, mock.sentinel.description, @@ -44,7 +40,7 @@ def setUp(self, *_): mock_on_error, mock_job_callable, *args, - **kw + **kw, ) @mock.patch.object(schemas, 'validate_value') @@ -63,13 +59,13 @@ def test__init__(self, mock_validate_value): mock_on_success, mock_on_error, mock_job_callable, - args + args, ) self.assertIsInstance(self.cron, cron.CronJob) mock_validate_value.assert_called_once_with( mock.sentinel.schedule, - schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"] + schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"], ) @mock.patch.object(schemas, 'validate_value') @@ -91,12 +87,12 @@ def test__init__raises(self, mock_validate_value): mock_on_error, mock_job_callable, args, - mock.sentinel.kw + mock.sentinel.kw, ) mock_validate_value.assert_called_once_with( mock.sentinel.schedule, - schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"] + schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"], ) mock_job_callable = "invalid_job" @@ -114,7 +110,7 @@ def test__init__raises(self, mock_validate_value): mock_on_error, mock_job_callable, args, - mock.sentinel.kw + mock.sentinel.kw, ) mock_validate_value.assert_not_called() @@ -134,7 +130,7 @@ def test__init__raises(self, mock_validate_value): mock_on_error, mock_job_callable, args, - mock.sentinel.kw + mock.sentinel.kw, ) mock_on_success = mock.Mock() @@ -152,68 +148,41 @@ def test__init__raises(self, mock_validate_value): mock_on_error, mock_job_callable, args, - mock.sentinel.kw + mock.sentinel.kw, ) @ddt.data( - { - 'pairs': [("mock_field1", "mock_field2")], - 'expected_result': [False] - }, - { - 'pairs': [("mock_field1", "mock_field1")], - 'expected_result': [True] - }, - { - 'pairs': [("mock_field1", None)], - 'expected_result': [True] - }, - { - 'pairs': [(None, "mock_field2")], - 'expected_result': [False] - }, - { - 'pairs': [(None, None)], - 'expected_result': [True] - }, + {'pairs': [("mock_field1", "mock_field2")], 'expected_result': [False]}, + {'pairs': [("mock_field1", "mock_field1")], 'expected_result': [True]}, + {'pairs': [("mock_field1", None)], 'expected_result': [True]}, + {'pairs': [(None, "mock_field2")], 'expected_result': [False]}, + {'pairs': [(None, None)], 'expected_result': [True]}, ) def test_compare(self, data): pairs = data['pairs'] result = self.cron._compare(pairs) - self.assertEqual( - data['expected_result'], - result - ) + self.assertEqual(data['expected_result'], result) def test_is_expired(self): self.cron._expires = datetime.datetime.fromisoformat('2000-01-01') result = self.cron.is_expired() - self.assertEqual( - True, - result - ) + self.assertEqual(True, result) self.cron._expires = datetime.datetime.fromisoformat('2099-01-01') result = self.cron.is_expired() - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) self.cron._expires = None result = self.cron.is_expired() - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) @mock.patch.object(cron.CronJob, 'is_expired') @ddt.data( @@ -221,24 +190,12 @@ def test_is_expired(self): 'schedule': { 'year': 4, # year is not in SCHEDULE_FIELDS 'month': 1, - 'hour': 0 + 'hour': 0, }, - 'expected_result': True + 'expected_result': True, }, - { - 'schedule': { - 'month': 2, - 'hour': 0 - }, - 'expected_result': False - }, - { - 'schedule': { - 'month': 1, - 'hour': 1 - }, - 'expected_result': False - } + {'schedule': {'month': 2, 'hour': 0}, 'expected_result': False}, + {'schedule': {'month': 1, 'hour': 1}, 'expected_result': False}, ) def test_should_run(self, data, mock_is_expired): mock_is_expired.return_value = False @@ -248,10 +205,7 @@ def test_should_run(self, data, mock_is_expired): result = self.cron.should_run(dt) - self.assertEqual( - data['expected_result'], - result - ) + self.assertEqual(data['expected_result'], result) @mock.patch.object(cron.CronJob, '_compare') @mock.patch.object(cron.CronJob, 'is_expired') @@ -259,33 +213,20 @@ def test_should_run_false(self, mock_is_expired, mock_compare): mock_is_expired.return_value = True dt = datetime.datetime.fromisoformat('2099-01-01') self.cron._enabled = True - self.cron.schedule = { - 'month': 1, - 'hour': 0 - } + self.cron.schedule = {'month': 1, 'hour': 0} - self.assertRaises( - exception.CoriolisException, - self.cron.should_run, - None - ) + self.assertRaises(exception.CoriolisException, self.cron.should_run, None) result = self.cron.should_run(dt) - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) mock_is_expired.return_value = False self.cron._enabled = False result = self.cron.should_run(dt) - self.assertEqual( - False, - result - ) + self.assertEqual(False, result) mock_compare.assert_not_called() @@ -303,47 +244,45 @@ def test_start(self, mock_send_status): self.cron.start() - self.cron._func.assert_called_once_with( - *self.cron._args, **self.cron._kw) - self.cron._on_success.assert_called_once_with( - self.cron._func.return_value) + self.cron._func.assert_called_once_with(*self.cron._args, **self.cron._kw) + self.cron._on_success.assert_called_once_with(self.cron._func.return_value) mock_send_status.assert_called_once_with( None, - {"result": self.cron._func.return_value, - "description": self.cron._description, - "name": self.cron.name, - "error_info": None} + { + "result": self.cron._func.return_value, + "description": self.cron._description, + "name": self.cron.name, + "error_info": None, + }, ) @mock.patch.object(cron, 'LOG') @mock.patch.object(sys, 'exc_info') @mock.patch.object(cron.CronJob, '_send_status') - def test_start_on_error( - self, - mock_send_status, - mock_exc_info, - mock_LOG - ): + def test_start_on_error(self, mock_send_status, mock_exc_info, mock_LOG): mock_exc_info.return_value = "mock_exc_info" self.cron._func.side_effect = Exception('err_msg1') self.cron._on_error.side_effect = Exception('err_msg2') self.cron.start() - self.cron._func.assert_called_once_with( - *self.cron._args, **self.cron._kw) + self.cron._func.assert_called_once_with(*self.cron._args, **self.cron._kw) self.cron._on_error.assert_called_once_with("mock_exc_info") mock_send_status.assert_called_once_with( None, - {"result": None, - "description": self.cron._description, - "name": self.cron.name, - "error_info": "mock_exc_info"} + { + "result": None, + "description": self.cron._description, + "name": self.cron.name, + "error_info": "mock_exc_info", + }, + ) + mock_LOG.assert_has_calls( + [ + mock.call.exception(self.cron._func.side_effect), + mock.call.exception(self.cron._on_error.side_effect), + ] ) - mock_LOG.assert_has_calls([ - mock.call.exception(self.cron._func.side_effect), - mock.call.exception(self.cron._on_error.side_effect) - ]) @ddt.ddt @@ -365,43 +304,27 @@ def test_register(self, *_): datetime.datetime.fromisoformat('2099-01-01'), mock.Mock(), mock.Mock(), - mock.Mock() + mock.Mock(), ) self.cron.register(job) - self.assertEqual( - self.cron._jobs[mock.sentinel.name], - job - ) + self.assertEqual(self.cron._jobs[mock.sentinel.name], job) def test_register_no_job(self): - self.assertRaises( - ValueError, - self.cron.register, - None - ) + self.assertRaises(ValueError, self.cron.register, None) def test_unregister(self): - self.cron._jobs = { - 'job_1': 'mock_job', - 'job_2': 'mock_job' - } + self.cron._jobs = {'job_1': 'mock_job', 'job_2': 'mock_job'} self.cron.unregister('job_1') - self.assertEqual( - {'job_2': 'mock_job'}, - self.cron._jobs - ) + self.assertEqual({'job_2': 'mock_job'}, self.cron._jobs) def test_unregister_jobs_with_prefix(self): self.cron._jobs = { 'pre1_job_1': 'mock_job', 'pre1_job_2': 'mock_job', - 'pre2_job_3': 'mock_job' + 'pre2_job_3': 'mock_job', } self.cron.unregister_jobs_with_prefix('pre1') - self.assertEqual( - {'pre2_job_3': 'mock_job'}, - self.cron._jobs - ) + self.assertEqual({'pre2_job_3': 'mock_job'}, self.cron._jobs) @mock.patch('coriolis.utils.start_thread') def test_check_jobs(self, mock_spawn): @@ -415,42 +338,34 @@ def test_check_jobs(self, mock_spawn): self.cron._check_jobs() - mock_spawn.assert_called_once_with( - mock_job.start, - args=[self.cron._queue]) + mock_spawn.assert_called_once_with(mock_job.start, args=[self.cron._queue]) @mock.patch.object(time, 'sleep') @mock.patch.object(schedule, 'run_pending') def test_loop(self, mock_run_pending, mock_sleep): mock_sleep.side_effect = [None, CoriolisTestException()] - self.assertRaises( - CoriolisTestException, - self.cron._loop - ) + self.assertRaises(CoriolisTestException, self.cron._loop) mock_run_pending.assert_has_calls([mock.call(), mock.call()]) - mock_sleep.assert_has_calls([mock.call(.2), mock.call(.2)]) + mock_sleep.assert_has_calls([mock.call(0.2), mock.call(0.2)]) @mock.patch.object(cron, 'LOG') def test_result_loop(self, mock_LOG): job_info = { 'result': None, 'error_info': 'mock_err_info', - 'description': 'mock_description' + 'description': 'mock_description', } job_info2 = { 'result': 'mock_result', 'error_info': None, - 'description': 'mock_description' + 'description': 'mock_description', } self.cron._queue.put(job_info) self.cron._queue.put(job_info2) mock_LOG.info.side_effect = CoriolisTestException() - self.assertRaises( - CoriolisTestException, - self.cron._result_loop - ) + self.assertRaises(CoriolisTestException, self.cron._result_loop) mock_LOG.error.assert_called_once() mock_LOG.info.assert_called_once() @@ -466,40 +381,30 @@ def test_janitor(self, mock_sleep): } mock_sleep.side_effect = CoriolisTestException() - self.assertRaises( - CoriolisTestException, - self.cron._janitor - ) + self.assertRaises(CoriolisTestException, self.cron._janitor) - self.assertEqual( - {'job2': job2}, - self.cron._jobs - ) + self.assertEqual({'job2': job2}, self.cron._jobs) @mock.patch('coriolis.utils.start_thread') @mock.patch.object(schedule, 'every') def test_start(self, mock_every, mock_spawn): - mock_spawn.side_effect = [ - 'spawn_loop', 'spawn_janitor', 'spawn_result_loop'] + mock_spawn.side_effect = ['spawn_loop', 'spawn_janitor', 'spawn_result_loop'] self.cron.start() - mock_every.return_value.minute.do.assert_called_once_with( - self.cron._check_jobs) - mock_spawn.assert_has_calls([ - mock.call(self.cron._loop), - mock.call(self.cron._janitor), - mock.call(self.cron._result_loop), - ]) + mock_every.return_value.minute.do.assert_called_once_with(self.cron._check_jobs) + mock_spawn.assert_has_calls( + [ + mock.call(self.cron._loop), + mock.call(self.cron._janitor), + mock.call(self.cron._result_loop), + ] + ) self.assertEqual( - ['spawn_loop', 'spawn_janitor', 'spawn_result_loop'], - self.cron._threads + ['spawn_loop', 'spawn_janitor', 'spawn_result_loop'], self.cron._threads ) def test_stop(self): self.cron._should_stop = False self.cron.stop() - self.assertEqual( - True, - self.cron._should_stop - ) + self.assertEqual(True, self.cron._should_stop) diff --git a/coriolis/tests/db/sqlalchemy/test_api.py b/coriolis/tests/db/sqlalchemy/test_api.py index de79b5d8..c268f517 100644 --- a/coriolis/tests/db/sqlalchemy/test_api.py +++ b/coriolis/tests/db/sqlalchemy/test_api.py @@ -6,9 +6,8 @@ from oslo_config import cfg from oslo_db.sqlalchemy import session as db_session -from coriolis.db.sqlalchemy import api -from coriolis.db.sqlalchemy import migration from coriolis import exception +from coriolis.db.sqlalchemy import api, migration from coriolis.tests import test_base @@ -26,10 +25,7 @@ def test_get_facade_none(self, mock_EngineFacade): result = api.get_facade() - self.assertEqual( - mock_EngineFacade.return_value, - result - ) + self.assertEqual(mock_EngineFacade.return_value, result) mock_EngineFacade.assert_called_once_with(mock.sentinel.connection) @mock.patch.object(db_session, 'EngineFacade') @@ -38,98 +34,58 @@ def test_get_facade(self, mock_EngineFacade): result = api.get_facade() - self.assertEqual( - mock.sentinel.facade, - result - ) + self.assertEqual(mock.sentinel.facade, result) mock_EngineFacade.assert_not_called() @mock.patch.object(api, 'get_facade') def test_get_engine(self, mock_get_facade): result = api.get_engine() - self.assertEqual( - mock_get_facade.return_value.get_engine.return_value, - result - ) + self.assertEqual(mock_get_facade.return_value.get_engine.return_value, result) @mock.patch.object(api, 'get_facade') def test_get_session(self, mock_get_facade): result = api.get_session() - self.assertEqual( - mock_get_facade.return_value.get_session.return_value, - result - ) + self.assertEqual(mock_get_facade.return_value.get_session.return_value, result) def test_get_backend(self): result = api.get_backend() - self.assertEqual( - api, - result - ) + self.assertEqual(api, result) @mock.patch.object(migration, 'db_sync') @mock.patch.object(api, 'db_version') - def test_db_sync( - self, - mock_db_version, - mock_db_sync - ): + def test_db_sync(self, mock_db_version, mock_db_sync): result = api.db_sync(mock.sentinel.engine) - self.assertEqual( - mock_db_sync.return_value, - result - ) + self.assertEqual(mock_db_sync.return_value, result) mock_db_version.assert_not_called() - mock_db_sync.assert_called_once_with( - mock.sentinel.engine, version=None) + mock_db_sync.assert_called_once_with(mock.sentinel.engine, version=None) @mock.patch.object(migration, 'db_sync') @mock.patch.object(api, 'db_version') - def test_db_sync_version( - self, - mock_db_version, - mock_db_sync - ): + def test_db_sync_version(self, mock_db_version, mock_db_sync): mock_db_version.return_value = 1 result = api.db_sync(mock.sentinel.engine, version=1) - self.assertEqual( - mock_db_sync.return_value, - result - ) + self.assertEqual(mock_db_sync.return_value, result) mock_db_version.assert_called_once_with(mock.sentinel.engine) - mock_db_sync.assert_called_once_with( - mock.sentinel.engine, version=1) + mock_db_sync.assert_called_once_with(mock.sentinel.engine, version=1) @mock.patch.object(api, 'db_version') - def test_db_sync_version_raise( - self, - mock_db_version - ): + def test_db_sync_version_raise(self, mock_db_version): mock_db_version.return_value = 2 self.assertRaises( - exception.CoriolisException, - api.db_sync, - mock.sentinel.engine, - version=1 + exception.CoriolisException, api.db_sync, mock.sentinel.engine, version=1 ) mock_db_version.assert_called_once_with(mock.sentinel.engine) @mock.patch.object(migration, 'db_version') - def test_db_version( - self, - mock_db_version - ): + def test_db_version(self, mock_db_version): result = api.db_version(mock.sentinel.engine) - self.assertEqual( - mock_db_version.return_value, - result - ) + self.assertEqual(mock_db_version.return_value, result) mock_db_version.assert_called_once_with(mock.sentinel.engine) diff --git a/coriolis/tests/db/sqlalchemy/test_migration.py b/coriolis/tests/db/sqlalchemy/test_migration.py index 6f35ad59..9828baa2 100644 --- a/coriolis/tests/db/sqlalchemy/test_migration.py +++ b/coriolis/tests/db/sqlalchemy/test_migration.py @@ -20,15 +20,12 @@ def test_db_sync(self, mock_db_sync, mock_abspath): result = migration.db_sync(mock.sentinel.engine, mock.sentinel.version) - self.assertEqual( - mock_db_sync.return_value, - result - ) + self.assertEqual(mock_db_sync.return_value, result) mock_db_sync.assert_called_once_with( mock.sentinel.engine, "/abspath/migrate_repo", mock.sentinel.version, - init_version=0 + init_version=0, ) @mock.patch.object(os.path, 'abspath') @@ -40,9 +37,7 @@ def test_db_version(self, mock_db_version, mock_abspath): self.assertEqual(mock_db_version.return_value, result) mock_db_version.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - 0 + mock.sentinel.engine, "/abspath/migrate_repo", 0 ) @mock.patch.object(os.path, 'abspath') @@ -51,11 +46,10 @@ def test_db_version_control(self, mock_db_version_control, mock_abspath): mock_abspath.return_value = "/abspath" result = migration.db_version_control( - mock.sentinel.engine, mock.sentinel.version) + mock.sentinel.engine, mock.sentinel.version + ) self.assertEqual(mock_db_version_control.return_value, result) mock_db_version_control.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - mock.sentinel.version + mock.sentinel.engine, "/abspath/migrate_repo", mock.sentinel.version ) diff --git a/coriolis/tests/db/sqlalchemy/test_models.py b/coriolis/tests/db/sqlalchemy/test_models.py index 7b0c7610..c4b2000f 100644 --- a/coriolis/tests/db/sqlalchemy/test_models.py +++ b/coriolis/tests/db/sqlalchemy/test_models.py @@ -30,15 +30,12 @@ def test_to_dict(self): "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = task_event.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class MinionPoolEventTestCase(test_base.CoriolisBaseTestCase): @@ -64,15 +61,12 @@ def test_to_dict(self): "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = minion_pool_event.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class TaskProgressUpdateTestCase(test_base.CoriolisBaseTestCase): @@ -100,15 +94,12 @@ def test_to_dict(self): "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = task_progress_update.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class MinionPoolProgressUpdateTestCase(test_base.CoriolisBaseTestCase): @@ -139,15 +130,12 @@ def test_to_dict(self): "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = minion_pool_progress_update.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class TaskTestCase(test_base.CoriolisBaseTestCase): @@ -191,19 +179,17 @@ def test_to_dict(self): "events": [mock_event1.to_dict(), mock_event2.to_dict()], "progress_updates": [ mock_progress_update1.to_dict(), - mock_progress_update2.to_dict()], + mock_progress_update2.to_dict(), + ], "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = task.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class TasksExecutionTestCase(test_base.CoriolisBaseTestCase): @@ -233,15 +219,12 @@ def test_to_dict(self): "created_at": mock.sentinel.created_at, "updated_at": mock.sentinel.updated_at, "deleted_at": mock.sentinel.deleted_at, - "deleted": mock.sentinel.deleted + "deleted": mock.sentinel.deleted, } result = tasks_execution.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class BaseTransferActionTestCase(test_base.CoriolisBaseTestCase): @@ -254,16 +237,14 @@ def test_to_dict(self): transfer.base_id = mock.sentinel.base_id transfer.user_id = mock.sentinel.user_id transfer.project_id = mock.sentinel.project_id - transfer.destination_environment = \ - mock.sentinel.destination_environment + transfer.destination_environment = mock.sentinel.destination_environment transfer.type = mock.sentinel.type transfer.executions = [mock_execution1, mock_execution2] transfer.instances = mock.sentinel.instances transfer.reservation_id = mock.sentinel.reservation_id transfer.notes = mock.sentinel.notes transfer.origin_endpoint_id = mock.sentinel.origin_endpoint_id - transfer.destination_endpoint_id = \ - mock.sentinel.destination_endpoint_id + transfer.destination_endpoint_id = mock.sentinel.destination_endpoint_id transfer.transfer_result = mock.sentinel.transfer_result transfer.network_map = mock.sentinel.network_map transfer.storage_mappings = mock.sentinel.storage_mappings @@ -275,10 +256,10 @@ def test_to_dict(self): transfer.deleted = mock.sentinel.deleted transfer.origin_minion_pool_id = mock.sentinel.origin_minion_pool_id transfer.deleted = mock.sentinel.deleted - transfer.destination_minion_pool_id = \ - mock.sentinel.destination_minion_pool_id - transfer.instance_osmorphing_minion_pool_mappings = \ + transfer.destination_minion_pool_id = mock.sentinel.destination_minion_pool_id + transfer.instance_osmorphing_minion_pool_mappings = ( mock.sentinel.instance_osmorphing_minion_pool_mappings + ) transfer.user_scripts = mock.sentinel.user_scripts transfer.info = mock.sentinel.info transfer.clone_disks = True @@ -289,8 +270,7 @@ def test_to_dict(self): "project_id": mock.sentinel.project_id, "destination_environment": mock.sentinel.destination_environment, "type": mock.sentinel.type, - "executions": [ - mock_execution1.to_dict(), mock_execution2.to_dict()], + "executions": [mock_execution1.to_dict(), mock_execution2.to_dict()], "instances": mock.sentinel.instances, "reservation_id": mock.sentinel.reservation_id, "notes": mock.sentinel.notes, @@ -306,10 +286,8 @@ def test_to_dict(self): "deleted_at": mock.sentinel.deleted_at, "deleted": mock.sentinel.deleted, "origin_minion_pool_id": mock.sentinel.origin_minion_pool_id, - "destination_minion_pool_id": - mock.sentinel.destination_minion_pool_id, - "instance_osmorphing_minion_pool_mappings": - mock.sentinel.instance_osmorphing_minion_pool_mappings, + "destination_minion_pool_id": mock.sentinel.destination_minion_pool_id, + "instance_osmorphing_minion_pool_mappings": mock.sentinel.instance_osmorphing_minion_pool_mappings, "user_scripts": mock.sentinel.user_scripts, "info": mock.sentinel.info, "clone_disks": True, @@ -318,10 +296,7 @@ def test_to_dict(self): result = transfer.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class ReplicaTestCase(test_base.CoriolisBaseTestCase): @@ -333,10 +308,7 @@ def test_to_dict(self): result = transfer.to_dict() - self.assertEqual( - mock.sentinel.id, - result["id"] - ) + self.assertEqual(mock.sentinel.id, result["id"]) class DeploymentTestCase(test_base.CoriolisBaseTestCase): @@ -377,8 +349,9 @@ def test_to_dict(self): minion.connection_info = mock.sentinel.connection_info minion.allocated_action = mock.sentinel.allocated_action minion.last_used_at = mock.sentinel.last_used_at - minion.backup_writer_connection_info = \ + minion.backup_writer_connection_info = ( mock.sentinel.backup_writer_connection_info + ) minion.provider_properties = mock.sentinel.provider_properties expected_result = { "id": mock.sentinel.id, @@ -394,17 +367,13 @@ def test_to_dict(self): "connection_info": mock.sentinel.connection_info, "allocated_action": mock.sentinel.allocated_action, "last_used_at": mock.sentinel.last_used_at, - "backup_writer_connection_info": - mock.sentinel.backup_writer_connection_info, - "provider_properties": mock.sentinel.provider_properties + "backup_writer_connection_info": mock.sentinel.backup_writer_connection_info, + "provider_properties": mock.sentinel.provider_properties, } result = minion.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) class MinionPoolTestCase(test_base.CoriolisBaseTestCase): @@ -420,8 +389,7 @@ def test_to_dict(self): mock_progress_update2 = models.MinionPoolProgressUpdate() minion.minion_machines = [mock_minion_machine1, mock_minion_machine2] minion.events = [mock_event1, mock_event2] - minion.progress_updates = [ - mock_progress_update1, mock_progress_update2] + minion.progress_updates = [mock_progress_update1, mock_progress_update2] minion.id = mock.sentinel.id minion.name = mock.sentinel.name minion.notes = mock.sentinel.notes @@ -439,8 +407,7 @@ def test_to_dict(self): minion.minimum_minions = mock.sentinel.minimum_minions minion.maximum_minions = mock.sentinel.maximum_minions minion.minion_max_idle_time = mock.sentinel.minion_max_idle_time - minion.minion_retention_strategy = \ - mock.sentinel.minion_retention_strategy + minion.minion_retention_strategy = mock.sentinel.minion_retention_strategy expected_result = { "id": mock.sentinel.id, "name": mock.sentinel.name, @@ -459,22 +426,18 @@ def test_to_dict(self): "minimum_minions": mock.sentinel.minimum_minions, "maximum_minions": mock.sentinel.maximum_minions, "minion_max_idle_time": mock.sentinel.minion_max_idle_time, - "minion_retention_strategy": - mock.sentinel.minion_retention_strategy, + "minion_retention_strategy": mock.sentinel.minion_retention_strategy, "minion_machines": [ mock_minion_machine1.to_dict(), - mock_minion_machine2.to_dict()], - "events": [ - mock_event1.to_dict(), - mock_event2.to_dict()], + mock_minion_machine2.to_dict(), + ], + "events": [mock_event1.to_dict(), mock_event2.to_dict()], "progress_updates": [ mock_progress_update1.to_dict(), - mock_progress_update2.to_dict()], + mock_progress_update2.to_dict(), + ], } result = minion.to_dict() - self.assertEqual( - expected_result, - result - ) + self.assertEqual(expected_result, result) diff --git a/coriolis/tests/db/sqlalchemy/test_types.py b/coriolis/tests/db/sqlalchemy/test_types.py index dd510b49..9c7825b9 100644 --- a/coriolis/tests/db/sqlalchemy/test_types.py +++ b/coriolis/tests/db/sqlalchemy/test_types.py @@ -1,8 +1,8 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. -from unittest import mock import zlib +from unittest import mock from coriolis.db.sqlalchemy import types from coriolis.tests import test_base @@ -18,19 +18,13 @@ def test_load_dialect_impl(self): result = long_text.load_dialect_impl(mock_dialect) - self.assertEqual( - mock_dialect.type_descriptor.return_value, - result - ) + self.assertEqual(mock_dialect.type_descriptor.return_value, result) mock_dialect.name = 'sqlite' result = long_text.load_dialect_impl(mock_dialect) - self.assertEqual( - long_text.impl, - result - ) + self.assertEqual(long_text.impl, result) class DatabaseSqlalchemyBlobTestCase(test_base.CoriolisBaseTestCase): @@ -43,19 +37,13 @@ def test_load_dialect_impl(self): result = blob.load_dialect_impl(mock_dialect) - self.assertEqual( - mock_dialect.type_descriptor.return_value, - result - ) + self.assertEqual(mock_dialect.type_descriptor.return_value, result) mock_dialect.name = 'sqlite' result = blob.load_dialect_impl(mock_dialect) - self.assertEqual( - blob.impl, - result - ) + self.assertEqual(blob.impl, result) class DatabaseSqlalchemyJsonTestCase(test_base.CoriolisBaseTestCase): @@ -68,31 +56,22 @@ def setUp(self): def test_process_bind_param(self): mock_dialect = mock.Mock() - result = self.type.process_bind_param( - {"mock_key": "mock_value"}, mock_dialect) + result = self.type.process_bind_param({"mock_key": "mock_value"}, mock_dialect) - self.assertEqual( - '{"mock_key": "mock_value"}', - result - ) + self.assertEqual('{"mock_key": "mock_value"}', result) def test_process_result_value(self): mock_dialect = mock.Mock() result = self.type.process_result_value( - '{"mock_key": "mock_value"}', mock_dialect) - - self.assertEqual( - {"mock_key": "mock_value"}, - result + '{"mock_key": "mock_value"}', mock_dialect ) + self.assertEqual({"mock_key": "mock_value"}, result) + result = self.type.process_result_value(None, mock_dialect) - self.assertEqual( - None, - result - ) + self.assertEqual(None, result) class DatabaseSqlalchemyBsonTestCase(test_base.CoriolisBaseTestCase): @@ -105,41 +84,30 @@ def setUp(self): def test_process_bind_param(self): mock_dialect = mock.Mock() - result = self.type.process_bind_param( - {"mock_key": "mock_value"}, mock_dialect) + result = self.type.process_bind_param({"mock_key": "mock_value"}, mock_dialect) self.assertEqual( - '{"mock_key": "mock_value"}', - zlib.decompress(result).decode('utf-8') + '{"mock_key": "mock_value"}', zlib.decompress(result).decode('utf-8') ) def test_process_result_value(self): mock_dialect = mock.Mock() result = self.type.process_result_value( - zlib.compress('{"mock_key": "mock_value"}'.encode('utf-8')), - mock_dialect + zlib.compress('{"mock_key": "mock_value"}'.encode('utf-8')), mock_dialect ) - self.assertEqual( - {"mock_key": "mock_value"}, - result - ) + self.assertEqual({"mock_key": "mock_value"}, result) result = self.type.process_result_value( - '{"mock_key": "mock_value"}', mock_dialect) - - self.assertEqual( - {"mock_key": "mock_value"}, - result + '{"mock_key": "mock_value"}', mock_dialect ) + self.assertEqual({"mock_key": "mock_value"}, result) + result = self.type.process_result_value(None, mock_dialect) - self.assertEqual( - None, - result - ) + self.assertEqual(None, result) class DatabaseSqlalchemyListTestCase(test_base.CoriolisBaseTestCase): @@ -155,45 +123,30 @@ def test_load_dialect_impl(self): result = self.type.load_dialect_impl(mock_dialect) - self.assertEqual( - mock_dialect.type_descriptor.return_value, - result - ) + self.assertEqual(mock_dialect.type_descriptor.return_value, result) mock_dialect.name = 'sqlite' result = self.type.load_dialect_impl(mock_dialect) - self.assertEqual( - self.type.impl, - result - ) + self.assertEqual(self.type.impl, result) def test_process_bind_param(self): mock_dialect = mock.Mock() - result = self.type.process_bind_param( - {"mock_key": "mock_value"}, mock_dialect) + result = self.type.process_bind_param({"mock_key": "mock_value"}, mock_dialect) - self.assertEqual( - '{"mock_key": "mock_value"}', - result - ) + self.assertEqual('{"mock_key": "mock_value"}', result) def test_process_result_value(self): mock_dialect = mock.Mock() result = self.type.process_result_value( - '{"mock_key": "mock_value"}', mock_dialect) - - self.assertEqual( - {"mock_key": "mock_value"}, - result + '{"mock_key": "mock_value"}', mock_dialect ) + self.assertEqual({"mock_key": "mock_value"}, result) + result = self.type.process_result_value(None, mock_dialect) - self.assertEqual( - None, - result - ) + self.assertEqual(None, result) diff --git a/coriolis/tests/db/test_api.py b/coriolis/tests/db/test_api.py index 088010a8..9416cbf8 100644 --- a/coriolis/tests/db/test_api.py +++ b/coriolis/tests/db/test_api.py @@ -1,20 +1,18 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. import datetime -from unittest import mock import uuid +from unittest import mock import ddt -from oslo_utils import timeutils import sqlalchemy +from oslo_utils import timeutils -from coriolis import constants +from coriolis import constants, exception from coriolis.db import api from coriolis.db.sqlalchemy import api as sqlalchemy_api from coriolis.db.sqlalchemy import models -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils CONTEXT_MOCK = mock.MagicMock() DEFAULT_INSTANCE = "instance1" @@ -25,10 +23,14 @@ def get_valid_endpoint( - endpoint_id=None, user_id=DEFAULT_USER_ID, - project_id=DEFAULT_PROJECT_ID, connection_info=None, - endpoint_type="openstack", name="test_name", - description="Endpoint Description"): + endpoint_id=None, + user_id=DEFAULT_USER_ID, + project_id=DEFAULT_PROJECT_ID, + connection_info=None, + endpoint_type="openstack", + name="test_name", + description="Endpoint Description", +): if endpoint_id is None: endpoint_id = str(uuid.uuid4()) if connection_info is None: @@ -58,8 +60,7 @@ def create_valid_tasks_execution(): valid_task.execution = valid_tasks_execution valid_task.instance = DEFAULT_INSTANCE valid_task.status = constants.TASK_STATUS_RUNNING - valid_task.task_type = ( - constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS) + valid_task.task_type = constants.TASK_TYPE_VALIDATE_TRANSFER_SOURCE_INPUTS valid_task.index = 1 valid_task.on_error = False @@ -79,21 +80,19 @@ def create_valid_tasks_execution(): class BaseDBAPITestCase(test_base.CoriolisBaseTestCase): - - valid_data = { - "user_scope": {}, - "outer_scope": {} - } + valid_data = {"user_scope": {}, "outer_scope": {}} @classmethod def setup_scoped_data(cls, region_id, project_id="1"): data = dict() valid_endpoint_source = get_valid_endpoint( - endpoint_type='vmware', project_id=project_id) + endpoint_type='vmware', project_id=project_id + ) cls.session.add(valid_endpoint_source) data['source_endpoint'] = valid_endpoint_source valid_endpoint_destination = get_valid_endpoint( - endpoint_type='openstack', project_id=project_id) + endpoint_type='openstack', project_id=project_id + ) cls.session.add(valid_endpoint_destination) data['destination_endpoint'] = valid_endpoint_destination @@ -145,8 +144,7 @@ def setup_scoped_data(cls, region_id, project_id="1"): valid_deployment.instances = [DEFAULT_INSTANCE] valid_deployment.info = DEFAULT_TASK_INFO valid_deployment.origin_endpoint_id = valid_endpoint_source.id - valid_deployment.destination_endpoint_id = ( - valid_endpoint_destination.id) + valid_deployment.destination_endpoint_id = valid_endpoint_destination.id valid_deployment.transfer = valid_transfer deployment_execution = create_valid_tasks_execution() @@ -165,10 +163,10 @@ def setup_database_data(cls): cls.valid_region.enabled = True cls.session.add(cls.valid_region) - cls.valid_data['user_scope'] = cls.setup_scoped_data( - cls.valid_region.id) + cls.valid_data['user_scope'] = cls.setup_scoped_data(cls.valid_region.id) cls.valid_data['outer_scope'] = cls.setup_scoped_data( - cls.valid_region.id, project_id="2") + cls.valid_region.id, project_id="2" + ) cls.session.commit() @classmethod @@ -215,15 +213,17 @@ def test_get_session(self): def test_db_sync(self, mock_impl): self.assertEqual( api.db_sync(mock.sentinel.engine, version=mock.sentinel.version), - mock_impl.db_sync.return_value) + mock_impl.db_sync.return_value, + ) mock_impl.db_sync.assert_called_once_with( - mock.sentinel.engine, version=mock.sentinel.version) + mock.sentinel.engine, version=mock.sentinel.version + ) @mock.patch.object(api, 'IMPL') def test_db_version(self, mock_impl): self.assertEqual( - api.db_version(mock.sentinel.engine), - mock_impl.db_version.return_value) + api.db_version(mock.sentinel.engine), mock_impl.db_version.return_value + ) mock_impl.db_version.assert_called_once_with(mock.sentinel.engine) def test__session(self): @@ -231,27 +231,26 @@ def test__session(self): @mock.patch.object(api, 'get_session') def test__session_no_context(self, mock_get_session): - self.assertEqual( - api._session(None), - mock_get_session.return_value) + self.assertEqual(api._session(None), mock_get_session.return_value) @mock.patch.object(api, 'get_session') def test__session_sessionless_context(self, mock_get_session): context = mock.Mock(session=None) - self.assertEqual( - api._session(context), - mock_get_session.return_value) + self.assertEqual(api._session(context), mock_get_session.return_value) @ddt.data( {"kwargs": None, "expected_result": False}, {"kwargs": {}, "expected_result": False}, {"kwargs": {"user_id": None}, "expected_result": False}, - {"kwargs": {"user_id": "1", "project_id": None}, - "expected_result": False}, - {"kwargs": {"user_id": "1", "project_id": "1", "is_admin": True}, - "expected_result": False}, - {"kwargs": {"user_id": "1", "project_id": "1", "is_admin": False}, - "expected_result": True}, + {"kwargs": {"user_id": "1", "project_id": None}, "expected_result": False}, + { + "kwargs": {"user_id": "1", "project_id": "1", "is_admin": True}, + "expected_result": False, + }, + { + "kwargs": {"user_id": "1", "project_id": "1", "is_admin": False}, + "expected_result": True, + }, ) def test_is_user_context(self, data): kwargs = data.get('kwargs') @@ -259,35 +258,45 @@ def test_is_user_context(self, data): context = None else: context = mock.Mock(**data.get('kwargs', {})) - self.assertEqual( - api.is_user_context(context), data.get('expected_result')) + self.assertEqual(api.is_user_context(context), data.get('expected_result')) @mock.patch.object(api, '_session') def test__model_query(self, mock_session): self.assertEqual( api._model_query(mock.sentinel.context, mock.sentinel.model), - mock_session.return_value.query.return_value) - mock_session.assert_called_once_with( - mock.sentinel.context) - mock_session.return_value.query.assert_called_once_with( - mock.sentinel.model) + mock_session.return_value.query.return_value, + ) + mock_session.assert_called_once_with(mock.sentinel.context) + mock_session.return_value.query.assert_called_once_with(mock.sentinel.model) def test__update_sqlalchemy_object_fields_non_dict_values(self): self.assertRaises( - exception.InvalidInput, api._update_sqlalchemy_object_fields, - mock.ANY, mock.ANY, None) + exception.InvalidInput, + api._update_sqlalchemy_object_fields, + mock.ANY, + mock.ANY, + None, + ) def test__update_sqlalchemy_object_fields_conflict(self): updateable_fields = ["field1", "field2"] values_to_update = {"field1": "value1", "field3": "value3"} self.assertRaises( - exception.Conflict, api._update_sqlalchemy_object_fields, - mock.ANY, updateable_fields, values_to_update) + exception.Conflict, + api._update_sqlalchemy_object_fields, + mock.ANY, + updateable_fields, + values_to_update, + ) def test__update_sqlalchemy_object_fields_invalid_obj_field(self): self.assertRaises( - exception.InvalidInput, api._update_sqlalchemy_object_fields, - models.Endpoint, ["invalid_field"], {"invalid_field": "new_value"}) + exception.InvalidInput, + api._update_sqlalchemy_object_fields, + models.Endpoint, + ["invalid_field"], + {"invalid_field": "new_value"}, + ) def test__update_sqlalchemy_object_fields(self): obj = models.Endpoint() @@ -295,7 +304,8 @@ def test__update_sqlalchemy_object_fields(self): new_description = "updated test description" api._update_sqlalchemy_object_fields( - obj, ["description"], {"description": new_description}) + obj, ["description"], {"description": new_description} + ) self.assertEqual(obj.description, new_description) def test__soft_delete_aware_query_show_deleted_kwarg(self): @@ -304,11 +314,16 @@ def test__soft_delete_aware_query_show_deleted_kwarg(self): self.session.commit() testutils.get_wrapped_function(api.delete_endpoint)( - self.context, valid_endpoint.id) + self.context, valid_endpoint.id + ) self.context.show_deleted = False - result = api._soft_delete_aware_query( - self.context, models.Endpoint, show_deleted=True).filter( - models.Endpoint.id == valid_endpoint.id).first() + result = ( + api._soft_delete_aware_query( + self.context, models.Endpoint, show_deleted=True + ) + .filter(models.Endpoint.id == valid_endpoint.id) + .first() + ) self.assertEqual(result.id, valid_endpoint.id) self.assertIsNotNone(result.deleted_at) @@ -318,11 +333,14 @@ def test__soft_delete_aware_query_context_show_deleted(self): self.session.commit() testutils.get_wrapped_function(api.delete_endpoint)( - self.context, valid_endpoint.id) + self.context, valid_endpoint.id + ) self.context.show_deleted = True - result = api._soft_delete_aware_query( - self.context, models.Endpoint).filter( - models.Endpoint.id == valid_endpoint.id).first() + result = ( + api._soft_delete_aware_query(self.context, models.Endpoint) + .filter(models.Endpoint.id == valid_endpoint.id) + .first() + ) self.assertEqual(result.id, valid_endpoint.id) self.assertIsNotNone(result.deleted_at) @@ -348,8 +366,7 @@ def test_unmodified_input(self): sort_keys = ["created_at", "id"] sort_dirs = ["desc", "desc"] - ret_keys, ret_dirs = api.process_sort_params( - sort_keys, sort_dirs) + ret_keys, ret_dirs = api.process_sort_params(sort_keys, sort_dirs) self.assertEqual(sort_keys, ret_keys) self.assertEqual(sort_dirs, ret_dirs) @@ -359,8 +376,7 @@ def test_unmatched_input(self): sort_dirs = ["asc"] exp_dirs = ["asc", "asc"] - ret_keys, ret_dirs = api.process_sort_params( - sort_keys, sort_dirs) + ret_keys, ret_dirs = api.process_sort_params(sort_keys, sort_dirs) self.assertEqual(sort_keys, ret_keys) self.assertEqual(exp_dirs, ret_dirs) @@ -372,7 +388,8 @@ def test_default_keys_appended(self): exp_dirs = ["asc", "asc"] ret_keys, ret_dirs = api.process_sort_params( - sort_keys, sort_dirs, + sort_keys, + sort_dirs, default_keys=["id"], default_dir="asc", ) @@ -387,7 +404,8 @@ def test_default_keys_without_input(self): exp_dirs = ["asc"] ret_keys, ret_dirs = api.process_sort_params( - sort_keys, sort_dirs, + sort_keys, + sort_dirs, default_keys=["id"], default_dir="asc", ) @@ -397,16 +415,14 @@ def test_default_keys_without_input(self): class EndpointDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(EndpointDBAPITestCase, cls).setUpClass() - cls.valid_endpoint_source = cls.valid_data['user_scope'].get( - 'source_endpoint') + cls.valid_endpoint_source = cls.valid_data['user_scope'].get('source_endpoint') cls.valid_endpoint_region_mapping = cls.valid_data['user_scope'].get( - 'endpoint_mapping') - cls.outer_scope_endpoint = cls.valid_data['outer_scope'].get( - 'source_endpoint') + 'endpoint_mapping' + ) + cls.outer_scope_endpoint = cls.valid_data['outer_scope'].get('source_endpoint') def test_get_endpoints(self): result = api.get_endpoints(self.context) @@ -441,32 +457,49 @@ def test_add_endpoint(self): new_endpoint = get_valid_endpoint( endpoint_id=new_endpoint_id, connection_info={"conn_info": {"new": "info"}}, - endpoint_type="vmware", name="new_endpoint", - description="New Endpoint") + endpoint_type="vmware", + name="new_endpoint", + description="New Endpoint", + ) api.add_endpoint(self.context, new_endpoint) result = api.get_endpoint(self.context, new_endpoint_id) self.assertEqual(result, new_endpoint) def test_update_endpoint_not_found(self): self.assertRaises( - exception.NotFound, api.update_endpoint, - self.context, "invalid_id", mock.ANY) + exception.NotFound, + api.update_endpoint, + self.context, + "invalid_id", + mock.ANY, + ) def test_update_endpoint_invalid_values(self): self.assertRaises( - exception.InvalidInput, api.update_endpoint, - self.context, self.valid_endpoint_source.id, None) + exception.InvalidInput, + api.update_endpoint, + self.context, + self.valid_endpoint_source.id, + None, + ) def test_update_endpoint_invalid_column(self): self.assertRaises( - exception.Conflict, api.update_endpoint, - self.context, self.valid_endpoint_source.id, {"type": "openstack"}) + exception.Conflict, + api.update_endpoint, + self.context, + self.valid_endpoint_source.id, + {"type": "openstack"}, + ) def test_update_endpoint_region_not_found(self): self.assertRaises( - exception.NotFound, api.update_endpoint, self.context, + exception.NotFound, + api.update_endpoint, + self.context, self.valid_endpoint_source.id, - {"mapped_regions": ["invalid_region_id"]}) + {"mapped_regions": ["invalid_region_id"]}, + ) def test_update_endpoint(self): new_region_id = str(uuid.uuid4()) @@ -479,42 +512,51 @@ def test_update_endpoint(self): self.session.commit() api.update_endpoint( - self.context, self.valid_endpoint_source.id, - {"mapped_regions": [new_region_id], "name": new_endpoint_name}) + self.context, + self.valid_endpoint_source.id, + {"mapped_regions": [new_region_id], "name": new_endpoint_name}, + ) result = api.get_endpoint(self.context, self.valid_endpoint_source.id) old_endpoint_region_mapping = api.get_endpoint_region_mapping( - self.context, self.valid_endpoint_source.id, self.valid_region.id) + self.context, self.valid_endpoint_source.id, self.valid_region.id + ) new_endpoint_region_mapping = api.get_endpoint_region_mapping( - self.context, self.valid_endpoint_source.id, new_region_id)[0] + self.context, self.valid_endpoint_source.id, new_region_id + )[0] self.assertEqual(result.name, new_endpoint_name) self.assertEqual(old_endpoint_region_mapping, []) self.assertEqual(new_endpoint_region_mapping.region_id, new_region_id) self.assertEqual( - new_endpoint_region_mapping.endpoint_id, - self.valid_endpoint_source.id) + new_endpoint_region_mapping.endpoint_id, self.valid_endpoint_source.id + ) @mock.patch.object(api, 'delete_endpoint_region_mapping') @mock.patch.object(api, 'add_endpoint_region_mapping') @mock.patch.object(api, 'get_region') @mock.patch.object(api, '_update_sqlalchemy_object_fields') def test_update_endpoint_remapping_failure( - self, mock_update_obj, mock_get_region, mock_add_mapping, - mock_delete_mapping): + self, mock_update_obj, mock_get_region, mock_add_mapping, mock_delete_mapping + ): mock_add_mapping.side_effect = [Exception, None] self.assertRaises( - Exception, api.update_endpoint, - self.context, self.valid_endpoint_source.id, - {"mapped_regions": [mock.sentinel.region_id]}) - mock_get_region.assert_called_with( - self.context, mock.sentinel.region_id) + Exception, + api.update_endpoint, + self.context, + self.valid_endpoint_source.id, + {"mapped_regions": [mock.sentinel.region_id]}, + ) + mock_get_region.assert_called_with(self.context, mock.sentinel.region_id) mock_delete_mapping.side_effect = Exception mock_update_obj.side_effect = Exception self.assertRaises( - Exception, api.update_endpoint, self.context, + Exception, + api.update_endpoint, + self.context, self.valid_endpoint_source.id, - {"mapped_regions": [mock.sentinel.region_id]}) + {"mapped_regions": [mock.sentinel.region_id]}, + ) def test_delete_endpoint(self): new_endpoint = get_valid_endpoint() @@ -526,13 +568,15 @@ def test_delete_endpoint(self): api.delete_endpoint(self.context, new_endpoint_id) result = api.get_endpoint(self.context, new_endpoint_id) mappings = api.get_endpoint_region_mapping( - self.context, new_endpoint_id, self.valid_region.id) + self.context, new_endpoint_id, self.valid_region.id + ) self.assertIsNone(result) self.assertEqual(mappings, []) def test_delete_endpoint_not_found(self): self.assertRaises( - exception.NotFound, api.delete_endpoint, self.context, "no_id") + exception.NotFound, api.delete_endpoint, self.context, "no_id" + ) def test_delete_endpoint_admin_context(self): self.context.is_admin = True @@ -542,35 +586,34 @@ def test_delete_endpoint_admin_context(self): new_outer_scope_endpoint.project_id = "3" api.add_endpoint(self.context, new_outer_scope_endpoint) - api.delete_endpoint( - self.context, new_outer_scope_endpoint.id) + api.delete_endpoint(self.context, new_outer_scope_endpoint.id) result = api.get_endpoint(self.context, new_outer_scope_endpoint.id) self.assertIsNotNone(result.deleted_at) def test_delete_endpoint_out_of_user_scope(self): - new_outer_scope_endpoint = get_valid_endpoint( - user_id="3", project_id="3") + new_outer_scope_endpoint = get_valid_endpoint(user_id="3", project_id="3") self.session.add(new_outer_scope_endpoint) self.session.commit() self.assertRaises( - exception.NotFound, api.delete_endpoint, self.context, - new_outer_scope_endpoint.id) + exception.NotFound, + api.delete_endpoint, + self.context, + new_outer_scope_endpoint.id, + ) class TransferTasksExecutionDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(TransferTasksExecutionDBAPITestCase, cls).setUpClass() cls.valid_transfer = cls.valid_data['user_scope'].get('transfer') cls.valid_task = cls.valid_data['user_scope'].get('task') - cls.valid_tasks_execution = cls.valid_data['user_scope'].get( - 'tasks_execution') - cls.outer_scope_transfer = cls.valid_data['outer_scope'].get( - 'transfer') + cls.valid_tasks_execution = cls.valid_data['user_scope'].get('tasks_execution') + cls.outer_scope_transfer = cls.valid_data['outer_scope'].get('transfer') cls.outer_scope_tasks_execution = cls.valid_data['outer_scope'].get( - "tasks_execution") + "tasks_execution" + ) def setUp(self): super(TransferTasksExecutionDBAPITestCase, self).setUp() @@ -590,12 +633,14 @@ def _create_dummy_execution(action): def test_get_transfer_tasks_executions_include_info(self): result = api.get_transfer_tasks_executions( - self.context, self.valid_transfer.id, include_task_info=True) + self.context, self.valid_transfer.id, include_task_info=True + ) self.assertTrue(hasattr(result[0].action, 'info')) def test_get_transfer_tasks_executions_include_tasks(self): result = api.get_transfer_tasks_executions( - self.context, self.valid_transfer.id, include_tasks=True) + self.context, self.valid_transfer.id, include_tasks=True + ) tasks = [] for e in result: tasks.extend(e.tasks) @@ -604,55 +649,67 @@ def test_get_transfer_tasks_executions_include_tasks(self): def test_get_transfer_tasks_executions_to_dict(self): result = api.get_transfer_tasks_executions( - self.context, self.valid_transfer.id, to_dict=True) + self.context, self.valid_transfer.id, to_dict=True + ) execution_ids = [e['id'] for e in result] self.assertIn(self.valid_tasks_execution.id, execution_ids) def test_get_transfer_tasks_executions(self): - result = api.get_transfer_tasks_executions( - self.context, self.valid_transfer.id) + result = api.get_transfer_tasks_executions(self.context, self.valid_transfer.id) self.assertIn(self.valid_tasks_execution, result) def test_get_transfer_tasks_executions_admin(self): self.context.is_admin = True result = api.get_transfer_tasks_executions( - self.context, self.outer_scope_transfer.id) + self.context, self.outer_scope_transfer.id + ) self.assertIn(self.outer_scope_tasks_execution, result) def test_get_transfer_tasks_execution_out_of_user_scope(self): result = api.get_transfer_tasks_executions( - self.context, self.outer_scope_transfer.id) + self.context, self.outer_scope_transfer.id + ) self.assertEqual(result, []) def test_get_transfer_tasks_execution(self): result = api.get_transfer_tasks_execution( - self.context, self.valid_transfer.id, - self.valid_tasks_execution.id) + self.context, self.valid_transfer.id, self.valid_tasks_execution.id + ) self.assertEqual(result, self.valid_tasks_execution) def test_get_transfer_tasks_execution_admin(self): self.context.is_admin = True result = api.get_transfer_tasks_execution( - self.context, self.outer_scope_transfer.id, - self.outer_scope_tasks_execution.id) + self.context, + self.outer_scope_transfer.id, + self.outer_scope_tasks_execution.id, + ) self.assertEqual(result, self.outer_scope_tasks_execution) def test_get_transfer_tasks_execution_out_of_user_context(self): result = api.get_transfer_tasks_execution( - self.context, self.outer_scope_transfer.id, - self.outer_scope_tasks_execution.id) + self.context, + self.outer_scope_transfer.id, + self.outer_scope_tasks_execution.id, + ) self.assertIsNone(result) def test_get_transfer_tasks_execution_include_task_info(self): result = api.get_transfer_tasks_execution( - self.context, self.valid_transfer.id, - self.valid_tasks_execution.id, include_task_info=True) + self.context, + self.valid_transfer.id, + self.valid_tasks_execution.id, + include_task_info=True, + ) self.assertTrue(hasattr(result.action, 'info')) def test_get_transfer_tasks_execution_to_dict(self): result = api.get_transfer_tasks_execution( - self.context, self.valid_transfer.id, - self.valid_tasks_execution.id, to_dict=True) + self.context, + self.valid_transfer.id, + self.valid_tasks_execution.id, + to_dict=True, + ) self.assertEqual(result['id'], self.valid_tasks_execution.id) def test_add_transfer_tasks_execution(self): @@ -660,102 +717,121 @@ def test_add_transfer_tasks_execution(self): api.add_transfer_tasks_execution(self.context, new_tasks_execution) result = api.get_transfer_tasks_execution( - self.context, self.valid_transfer.id, new_tasks_execution.id) + self.context, self.valid_transfer.id, new_tasks_execution.id + ) self.assertEqual(new_tasks_execution, result) self.assertGreater(result.number, 0) def test_add_transfer_tasks_execution_admin(self): self.context.is_admin = True - new_tasks_execution = self._create_dummy_execution( - self.outer_scope_transfer) + new_tasks_execution = self._create_dummy_execution(self.outer_scope_transfer) api.add_transfer_tasks_execution(self.context, new_tasks_execution) result = api.get_transfer_tasks_execution( - self.context, self.outer_scope_transfer.id, new_tasks_execution.id) + self.context, self.outer_scope_transfer.id, new_tasks_execution.id + ) self.assertEqual(new_tasks_execution, result) def test_add_transfer_tasks_execution_out_of_user_context(self): - new_tasks_execution = self._create_dummy_execution( - self.outer_scope_transfer) + new_tasks_execution = self._create_dummy_execution(self.outer_scope_transfer) self.assertRaises( - exception.NotAuthorized, api.add_transfer_tasks_execution, - self.context, new_tasks_execution) + exception.NotAuthorized, + api.add_transfer_tasks_execution, + self.context, + new_tasks_execution, + ) def test_delete_transfer_tasks_execution(self): new_tasks_execution = self._create_dummy_execution(self.valid_transfer) api.add_transfer_tasks_execution(self.context, new_tasks_execution) - api.delete_transfer_tasks_execution( - self.context, new_tasks_execution.id) + api.delete_transfer_tasks_execution(self.context, new_tasks_execution.id) result = api.get_transfer_tasks_execution( - self.context, self.valid_transfer.id, new_tasks_execution.id) + self.context, self.valid_transfer.id, new_tasks_execution.id + ) self.assertIsNone(result) def test_delete_transfer_tasks_execution_admin(self): self.context.is_admin = True - new_tasks_execution = self._create_dummy_execution( - self.outer_scope_transfer) + new_tasks_execution = self._create_dummy_execution(self.outer_scope_transfer) api.add_transfer_tasks_execution(self.context, new_tasks_execution) - api.delete_transfer_tasks_execution( - self.context, new_tasks_execution.id) + api.delete_transfer_tasks_execution(self.context, new_tasks_execution.id) result = api.get_transfer_tasks_execution( - self.context, self.outer_scope_transfer.id, new_tasks_execution.id) + self.context, self.outer_scope_transfer.id, new_tasks_execution.id + ) self.assertIsNone(result) def test_delete_transfer_tasks_execution_out_of_user_scope(self): self.context.is_admin = True - new_tasks_execution = self._create_dummy_execution( - self.outer_scope_transfer) + new_tasks_execution = self._create_dummy_execution(self.outer_scope_transfer) api.add_transfer_tasks_execution(self.context, new_tasks_execution) self.context.is_admin = False self.assertRaises( - exception.NotAuthorized, api.delete_transfer_tasks_execution, - self.context, new_tasks_execution.id) + exception.NotAuthorized, + api.delete_transfer_tasks_execution, + self.context, + new_tasks_execution.id, + ) def test_delete_transfer_tasks_execution_not_found(self): self.context.is_admin = True self.assertRaises( - exception.NotFound, api.delete_transfer_tasks_execution, - self.context, "invalid_id") + exception.NotFound, + api.delete_transfer_tasks_execution, + self.context, + "invalid_id", + ) def test_set_execution_status_admin(self): self.context.is_admin = True new_status = constants.EXECUTION_STATUS_COMPLETED result = api.set_execution_status( - self.context, self.outer_scope_tasks_execution.id, new_status, - update_action_status=False) + self.context, + self.outer_scope_tasks_execution.id, + new_status, + update_action_status=False, + ) self.assertEqual(result.status, new_status) def test_set_execution_status_out_of_user_scope(self): self.assertRaises( - exception.NotFound, api.set_execution_status, self.context, - self.outer_scope_tasks_execution.id, mock.ANY, - update_action_status=False) + exception.NotFound, + api.set_execution_status, + self.context, + self.outer_scope_tasks_execution.id, + mock.ANY, + update_action_status=False, + ) def test_set_execution_status_not_found(self): self.assertRaises( - exception.NotFound, api.set_execution_status, self.context, - "invalid_id", mock.ANY, - update_action_status=False) + exception.NotFound, + api.set_execution_status, + self.context, + "invalid_id", + mock.ANY, + update_action_status=False, + ) def test_set_execution_status_update_action_status(self): new_status = constants.EXECUTION_STATUS_COMPLETED api.set_execution_status( - self.context, self.valid_tasks_execution.id, new_status) + self.context, self.valid_tasks_execution.id, new_status + ) self.assertEqual(self.valid_transfer.last_execution_status, new_status) class TransferSchedulesDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(TransferSchedulesDBAPITestCase, cls).setUpClass() cls.valid_transfer_schedule = cls.valid_data['user_scope'].get( - 'transfer_schedule') + 'transfer_schedule' + ) cls.valid_transfer = cls.valid_data['user_scope'].get('transfer') cls.outer_scope_transfer_schedule = cls.valid_data['outer_scope'].get( - 'transfer_schedule') - cls.outer_scope_transfer = cls.valid_data['outer_scope'].get( - 'transfer') + 'transfer_schedule' + ) + cls.outer_scope_transfer = cls.valid_data['outer_scope'].get('transfer') @staticmethod def _create_dummy_transfer_schedule(transfer, expiration_date): @@ -789,25 +865,27 @@ def test__get_transfer_schedules_filter_out_of_user_context(self): def test__get_transfer_schedules_filter_by_transfer(self): result = api._get_transfer_schedules_filter( - self.context, transfer_id=self.valid_transfer_schedule.transfer_id) + self.context, transfer_id=self.valid_transfer_schedule.transfer_id + ) self.assertEqual(result.first(), self.valid_transfer_schedule) def test__get_transfer_schedules_filter_by_schedule_id(self): result = api._get_transfer_schedules_filter( - self.context, schedule_id=self.valid_transfer_schedule.id).first() + self.context, schedule_id=self.valid_transfer_schedule.id + ).first() self.assertEqual(result, self.valid_transfer_schedule) def test__get_transfer_schedules_filter_by_not_expired(self): expiration_date = timeutils.utcnow() + datetime.timedelta(days=1) unexpired_transfer_schedule = self._create_dummy_transfer_schedule( - self.valid_transfer, expiration_date=expiration_date) + self.valid_transfer, expiration_date=expiration_date + ) self.session.add(unexpired_transfer_schedule) - expiration_null_transfer_schedule = ( - self._create_dummy_transfer_schedule( - self.valid_transfer, expiration_date=None)) + expiration_null_transfer_schedule = self._create_dummy_transfer_schedule( + self.valid_transfer, expiration_date=None + ) self.session.add(expiration_null_transfer_schedule) - result = api._get_transfer_schedules_filter( - self.context, expired=False).all() + result = api._get_transfer_schedules_filter(self.context, expired=False).all() self.assertIn(unexpired_transfer_schedule, result) self.assertIn(expiration_null_transfer_schedule, result) @@ -817,74 +895,91 @@ def test_get_transfer_schedules(self): def test_get_transfer_schedule(self): result = api.get_transfer_schedule( - self.context, self.valid_transfer.id, - self.valid_transfer_schedule.id) + self.context, self.valid_transfer.id, self.valid_transfer_schedule.id + ) self.assertEqual(result, self.valid_transfer_schedule) def test_update_transfer_schedule(self): pre_update_mock = mock.Mock() post_update_mock = mock.Mock() api.update_transfer_schedule( - self.context, self.valid_transfer.id, - self.valid_transfer_schedule.id, {"shutdown_instance": True}, + self.context, + self.valid_transfer.id, + self.valid_transfer_schedule.id, + {"shutdown_instance": True}, pre_update_callable=pre_update_mock, - post_update_callable=post_update_mock) + post_update_callable=post_update_mock, + ) result = api.get_transfer_schedule( - self.context, self.valid_transfer.id, - self.valid_transfer_schedule.id) + self.context, self.valid_transfer.id, self.valid_transfer_schedule.id + ) self.assertEqual(result.shutdown_instance, True) - pre_update_mock.assert_called_once_with( - schedule=self.valid_transfer_schedule) + pre_update_mock.assert_called_once_with(schedule=self.valid_transfer_schedule) post_update_mock.assert_called_once_with( - self.context, self.valid_transfer_schedule) + self.context, self.valid_transfer_schedule + ) def test_delete_transfer_schedule_not_found(self): - self.assertRaises(exception.NotFound, api.delete_transfer_schedule, - self.context, self.valid_transfer.id, "invalid") + self.assertRaises( + exception.NotFound, + api.delete_transfer_schedule, + self.context, + self.valid_transfer.id, + "invalid", + ) def test_delete_transfer_schedule_admin(self): self.context.is_admin = True outer_scope_schedule = self._create_dummy_transfer_schedule( - self.outer_scope_transfer, None) + self.outer_scope_transfer, None + ) self.session.add(outer_scope_schedule) api.delete_transfer_schedule( - self.context, self.outer_scope_transfer.id, - outer_scope_schedule.id) + self.context, self.outer_scope_transfer.id, outer_scope_schedule.id + ) result = api.get_transfer_schedule( - self.context, self.outer_scope_transfer.id, - outer_scope_schedule.id) + self.context, self.outer_scope_transfer.id, outer_scope_schedule.id + ) self.assertIsNone(result) def test_delete_transfer_schedule_out_of_user_context(self): outer_scope_schedule = self._create_dummy_transfer_schedule( - self.outer_scope_transfer, None) + self.outer_scope_transfer, None + ) self.session.add(outer_scope_schedule) self.assertRaises( - exception.NotAuthorized, api.delete_transfer_schedule, - self.context, self.outer_scope_transfer.id, - outer_scope_schedule.id) + exception.NotAuthorized, + api.delete_transfer_schedule, + self.context, + self.outer_scope_transfer.id, + outer_scope_schedule.id, + ) def test_delete_transfer_schedule(self): dummy_transfer_schedule = self._create_dummy_transfer_schedule( - self.valid_transfer, None) + self.valid_transfer, None + ) self.session.add(dummy_transfer_schedule) pre_delete_mock = mock.Mock() post_delete_mock = mock.Mock() api.delete_transfer_schedule( - self.context, self.valid_transfer.id, dummy_transfer_schedule.id, + self.context, + self.valid_transfer.id, + dummy_transfer_schedule.id, pre_delete_callable=pre_delete_mock, - post_delete_callable=post_delete_mock) + post_delete_callable=post_delete_mock, + ) result = api.get_transfer_schedule( - self.context, self.valid_transfer.id, dummy_transfer_schedule.id) + self.context, self.valid_transfer.id, dummy_transfer_schedule.id + ) self.assertIsNone(result) - pre_delete_mock.assert_called_once_with( - self.context, dummy_transfer_schedule) - post_delete_mock.assert_called_once_with( - self.context, dummy_transfer_schedule) + pre_delete_mock.assert_called_once_with(self.context, dummy_transfer_schedule) + post_delete_mock.assert_called_once_with(self.context, dummy_transfer_schedule) def test_delete_transfer_schedule_already_deleted(self): dummy_transfer_schedule = self._create_dummy_transfer_schedule( - self.valid_transfer, None) + self.valid_transfer, None + ) self.session.add(dummy_transfer_schedule) def pre_delete(context, schedule): @@ -893,45 +988,55 @@ def pre_delete(context, schedule): context.session.commit() self.assertRaises( - exception.NotFound, api.delete_transfer_schedule, - self.context, self.valid_transfer.id, dummy_transfer_schedule.id, - pre_delete_callable=pre_delete) + exception.NotFound, + api.delete_transfer_schedule, + self.context, + self.valid_transfer.id, + dummy_transfer_schedule.id, + pre_delete_callable=pre_delete, + ) def test_add_transfer_schedule(self): - new_schedule = self._create_dummy_transfer_schedule( - self.valid_transfer, None) + new_schedule = self._create_dummy_transfer_schedule(self.valid_transfer, None) post_add_mock = mock.Mock() api.add_transfer_schedule( - self.context, new_schedule, post_create_callable=post_add_mock) + self.context, new_schedule, post_create_callable=post_add_mock + ) result = api.get_transfer_schedule( - self.context, self.valid_transfer.id, new_schedule.id) + self.context, self.valid_transfer.id, new_schedule.id + ) self.assertEqual(result, new_schedule) post_add_mock.assert_called_once_with(self.context, new_schedule) def test_add_transfer_schedule_out_of_user_context(self): new_schedule = self._create_dummy_transfer_schedule( - self.outer_scope_transfer, None) + self.outer_scope_transfer, None + ) self.assertRaises( - exception.NotAuthorized, api.add_transfer_schedule, - self.context, new_schedule) + exception.NotAuthorized, + api.add_transfer_schedule, + self.context, + new_schedule, + ) class TransfersDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(TransfersDBAPITestCase, cls).setUpClass() cls.valid_transfer = cls.valid_data['user_scope'].get('transfer') cls.valid_transfer_execution = cls.valid_data['user_scope'].get( - 'tasks_execution') - cls.outer_scope_transfer = cls.valid_data['outer_scope'].get( - 'transfer') + 'tasks_execution' + ) + cls.outer_scope_transfer = cls.valid_data['outer_scope'].get('transfer') @staticmethod - def _create_dummy_transfer(scenario=constants.TRANSFER_SCENARIO_REPLICA, - origin_endpoint_id=str(uuid.uuid4()), - destination_endpoint_id=str(uuid.uuid4()), - project_id=DEFAULT_PROJECT_ID): + def _create_dummy_transfer( + scenario=constants.TRANSFER_SCENARIO_REPLICA, + origin_endpoint_id=str(uuid.uuid4()), + destination_endpoint_id=str(uuid.uuid4()), + project_id=DEFAULT_PROJECT_ID, + ): transfer = models.Transfer() transfer.id = str(uuid.uuid4()) transfer.user_id = project_id @@ -988,13 +1093,16 @@ def test_get_transfer_admin(self): def test_get_transfer_include_task_info(self): result = api.get_transfer( - self.context, self.valid_transfer.id, include_task_info=True) + self.context, self.valid_transfer.id, include_task_info=True + ) self.assertEqual(result.info, DEFAULT_TASK_INFO) def test_get_transfer_by_scenario(self): result = api.get_transfer( - self.context, self.valid_transfer.id, - transfer_scenario=constants.TRANSFER_SCENARIO_REPLICA) + self.context, + self.valid_transfer.id, + transfer_scenario=constants.TRANSFER_SCENARIO_REPLICA, + ) self.assertEqual(result, self.valid_transfer) def test_get_transfer_out_of_user_scope(self): @@ -1002,8 +1110,7 @@ def test_get_transfer_out_of_user_scope(self): self.assertIsNone(result) def test_get_transfer_to_dict(self): - result = api.get_transfer( - self.context, self.valid_transfer.id, to_dict=True) + result = api.get_transfer(self.context, self.valid_transfer.id, to_dict=True) self.assertEqual(result['id'], self.valid_transfer.id) result = api.get_transfer(self.context, "invalid", to_dict=True) @@ -1014,21 +1121,24 @@ def test_get_endpoint_transfers_count(self): dest_endpoint_id = str(uuid.uuid4()) dummy_transfer_replica = self._create_dummy_transfer( origin_endpoint_id=origin_endpoint_id, - destination_endpoint_id=dest_endpoint_id) + destination_endpoint_id=dest_endpoint_id, + ) dummy_transfer_migration = self._create_dummy_transfer( scenario=constants.TRANSFER_SCENARIO_LIVE_MIGRATION, origin_endpoint_id=origin_endpoint_id, - destination_endpoint_id=dest_endpoint_id) + destination_endpoint_id=dest_endpoint_id, + ) self.session.add(dummy_transfer_replica) self.session.add(dummy_transfer_migration) - result = api.get_endpoint_transfers_count( - self.context, origin_endpoint_id) + result = api.get_endpoint_transfers_count(self.context, origin_endpoint_id) self.assertEqual(result, 2) result = api.get_endpoint_transfers_count( - self.context, origin_endpoint_id, - transfer_scenario=constants.TRANSFER_SCENARIO_REPLICA) + self.context, + origin_endpoint_id, + transfer_scenario=constants.TRANSFER_SCENARIO_REPLICA, + ) self.assertEqual(result, 1) def test_add_transfer(self): @@ -1039,25 +1149,27 @@ def test_add_transfer(self): class DeploymentsDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(DeploymentsDBAPITestCase, cls).setUpClass() cls.user_deployment = cls.valid_data['user_scope'].get('deployment') - cls.outer_scope_deployment = cls.valid_data['outer_scope'].get( - 'deployment') + cls.outer_scope_deployment = cls.valid_data['outer_scope'].get('deployment') cls.user_deployment_execution = cls.valid_data['user_scope'].get( - 'deployment_execution') - cls.outer_scope_deployment_execution = cls.valid_data[ - 'outer_scope'].get('deployment_execution') + 'deployment_execution' + ) + cls.outer_scope_deployment_execution = cls.valid_data['outer_scope'].get( + 'deployment_execution' + ) cls.user_deployment_task = cls.user_deployment_execution.tasks[0] cls.user_transfer = cls.valid_data['user_scope'].get('transfer') @staticmethod - def _create_dummy_deployment(transfer_id, - origin_endpoint_id=str(uuid.uuid4()), - destination_endpoint_id=str(uuid.uuid4()), - project_id=DEFAULT_PROJECT_ID): + def _create_dummy_deployment( + transfer_id, + origin_endpoint_id=str(uuid.uuid4()), + destination_endpoint_id=str(uuid.uuid4()), + project_id=DEFAULT_PROJECT_ID, + ): deployment = models.Deployment() deployment.id = str(uuid.uuid4()) deployment.user_id = project_id @@ -1076,25 +1188,30 @@ def _create_dummy_deployment(transfer_id, def test_get_transfers_deployments_admin(self): self.context.is_admin = True result = api.get_transfer_deployments( - self.context, self.user_deployment.transfer_id) + self.context, self.user_deployment.transfer_id + ) self.assertIn(self.user_deployment, result) def test_get_transfer_deployments_out_of_user_context(self): result = api.get_transfer_deployments( - self.context, self.outer_scope_deployment.transfer_id) + self.context, self.outer_scope_deployment.transfer_id + ) self.assertNotIn(self.outer_scope_deployment, result) def test_get_transfer_deployments(self): result = api.get_transfer_deployments( - self.context, self.user_deployment.transfer_id) + self.context, self.user_deployment.transfer_id + ) self.assertIn(self.user_deployment, result) def test_get_deployments_admin(self): self.context.is_admin = True result = api.get_deployments(self.context) self.assertIn(self.outer_scope_deployment, result) - self.assertIn(self.outer_scope_deployment_execution, - self.outer_scope_deployment.executions) + self.assertIn( + self.outer_scope_deployment_execution, + self.outer_scope_deployment.executions, + ) def test_get_deployments_include_tasks(self): result = api.get_deployments(self.context, include_tasks=True) @@ -1125,18 +1242,17 @@ def test_get_deployments_to_dict(self): def test_get_deployment_admin(self): self.context.is_admin = True - result = api.get_deployment( - self.context, self.outer_scope_deployment.id) + result = api.get_deployment(self.context, self.outer_scope_deployment.id) self.assertEqual(self.outer_scope_deployment, result) def test_get_deployment_include_task_info(self): - result = api.get_deployment(self.context, self.user_deployment.id, - include_task_info=True) + result = api.get_deployment( + self.context, self.user_deployment.id, include_task_info=True + ) self.assertEqual(result.info, self.user_deployment.info) def test_get_deployment_out_of_user_context(self): - result = api.get_deployment( - self.context, self.outer_scope_deployment.id) + result = api.get_deployment(self.context, self.outer_scope_deployment.id) self.assertIsNone(result) def test_get_deployment(self): @@ -1144,8 +1260,7 @@ def test_get_deployment(self): self.assertEqual(result, self.user_deployment) def test_get_deployment_to_dict(self): - result = api.get_deployment(self.context, self.user_deployment.id, - to_dict=True) + result = api.get_deployment(self.context, self.user_deployment.id, to_dict=True) self.assertEqual(result['id'], self.user_deployment.id) def test_add_deployment(self): @@ -1156,13 +1271,11 @@ def test_add_deployment(self): class BaseTransferActionDBAPITestCase(BaseDBAPITestCase): - @classmethod def setUpClass(cls): super(BaseTransferActionDBAPITestCase, cls).setUpClass() cls.user_transfer = cls.valid_data['user_scope'].get('transfer') - cls.outer_scope_transfer = cls.valid_data['outer_scope'].get( - 'transfer') + cls.outer_scope_transfer = cls.valid_data['outer_scope'].get('transfer') def test_get_action_admin(self): self.context.is_admin = True @@ -1171,10 +1284,14 @@ def test_get_action_admin(self): def test_get_action_not_found(self): self.assertRaises( - exception.NotFound, api.get_action, self.context, - self.outer_scope_transfer.id) + exception.NotFound, + api.get_action, + self.context, + self.outer_scope_transfer.id, + ) def test_get_action_include_task_info(self): result = api.get_action( - self.context, self.user_transfer.id, include_task_info=True) + self.context, self.user_transfer.id, include_task_info=True + ) self.assertEqual(result.info, self.user_transfer.info) diff --git a/coriolis/tests/diagnostics/test_api.py b/coriolis/tests/diagnostics/test_api.py index 40e010de..ac296c0a 100644 --- a/coriolis/tests/diagnostics/test_api.py +++ b/coriolis/tests/diagnostics/test_api.py @@ -3,9 +3,9 @@ from unittest import mock +from coriolis import utils from coriolis.diagnostics import api from coriolis.tests import test_base -from coriolis import utils class DiagnosticsAPITestCase(test_base.CoriolisBaseTestCase): @@ -17,24 +17,17 @@ def setUp(self): self.diagnostics_api._conductor_cli = mock.Mock() @mock.patch.object(utils, "get_diagnostics_info") - def test_get( - self, - mock_get_diagnostics_info - ): - (self.diagnostics_api._conductor_cli.get_all_diagnostics. - return_value) = [mock.sentinel.all_diag] - mock_get_diagnostics_info.return_value = mock.sentinel.diag_info - expected_result = [ - mock.sentinel.all_diag, - mock.sentinel.diag_info + def test_get(self, mock_get_diagnostics_info): + (self.diagnostics_api._conductor_cli.get_all_diagnostics.return_value) = [ + mock.sentinel.all_diag ] + mock_get_diagnostics_info.return_value = mock.sentinel.diag_info + expected_result = [mock.sentinel.all_diag, mock.sentinel.diag_info] result = self.diagnostics_api.get(mock.sentinel.context) - self.assertEqual( - expected_result, - result - ) - (self.diagnostics_api._conductor_cli.get_all_diagnostics. - assert_called_once_with)(mock.sentinel.context) + self.assertEqual(expected_result, result) + ( + self.diagnostics_api._conductor_cli.get_all_diagnostics.assert_called_once_with + )(mock.sentinel.context) mock_get_diagnostics_info.assert_called_once() diff --git a/coriolis/tests/endpoint_options/test_api.py b/coriolis/tests/endpoint_options/test_api.py index 42b268a1..016e5a0e 100644 --- a/coriolis/tests/endpoint_options/test_api.py +++ b/coriolis/tests/endpoint_options/test_api.py @@ -10,7 +10,7 @@ "ctxt": "mock_ctxt", "endpoint_id": "mock_endpoint_id", "env": "mock_env", - "option_names": "mock_option_names" + "option_names": "mock_option_names", } @@ -25,52 +25,52 @@ def setUp(self): def test_get_endpoint_source_options(self): result = self.endpoint_options_api.get_endpoint_source_options(**ARGS) - (self.endpoint_options_api._rpc_conductor_client. - get_endpoint_source_options.assert_called_once_with)( - *(ARGS.values()) - ) + ( + self.endpoint_options_api._rpc_conductor_client.get_endpoint_source_options.assert_called_once_with + )(*(ARGS.values())) self.assertEqual( - (self.endpoint_options_api._rpc_conductor_client. - get_endpoint_source_options.return_value), - result + ( + self.endpoint_options_api._rpc_conductor_client.get_endpoint_source_options.return_value + ), + result, ) def test_get_endpoint_destination_options(self): - result = self.endpoint_options_api.get_endpoint_destination_options( - **ARGS) - (self.endpoint_options_api._rpc_conductor_client. - get_endpoint_destination_options.assert_called_once_with)( - *(ARGS.values()) - ) + result = self.endpoint_options_api.get_endpoint_destination_options(**ARGS) + ( + self.endpoint_options_api._rpc_conductor_client.get_endpoint_destination_options.assert_called_once_with + )(*(ARGS.values())) self.assertEqual( - (self.endpoint_options_api._rpc_conductor_client. - get_endpoint_destination_options.return_value), - result + ( + self.endpoint_options_api._rpc_conductor_client.get_endpoint_destination_options.return_value + ), + result, ) def test_get_endpoint_source_minion_pool_options(self): - result = (self.endpoint_options_api. - get_endpoint_source_minion_pool_options)(**ARGS) - (self.endpoint_options_api._rpc_minion_manager_client. - get_endpoint_source_minion_pool_options.assert_called_once_with)( - *(ARGS.values()) + result = (self.endpoint_options_api.get_endpoint_source_minion_pool_options)( + **ARGS ) + ( + self.endpoint_options_api._rpc_minion_manager_client.get_endpoint_source_minion_pool_options.assert_called_once_with + )(*(ARGS.values())) self.assertEqual( - (self.endpoint_options_api._rpc_minion_manager_client. - get_endpoint_source_minion_pool_options.return_value), - result + ( + self.endpoint_options_api._rpc_minion_manager_client.get_endpoint_source_minion_pool_options.return_value + ), + result, ) def test_get_endpoint_destination_minion_pool_options(self): - result = (self.endpoint_options_api. - get_endpoint_destination_minion_pool_options)(**ARGS) - (self.endpoint_options_api._rpc_minion_manager_client. - get_endpoint_destination_minion_pool_options. - assert_called_once_with)( - *(ARGS.values()) - ) + result = ( + self.endpoint_options_api.get_endpoint_destination_minion_pool_options + )(**ARGS) + ( + self.endpoint_options_api._rpc_minion_manager_client.get_endpoint_destination_minion_pool_options.assert_called_once_with + )(*(ARGS.values())) self.assertEqual( - (self.endpoint_options_api._rpc_minion_manager_client. - get_endpoint_destination_minion_pool_options.return_value), - result + ( + self.endpoint_options_api._rpc_minion_manager_client.get_endpoint_destination_minion_pool_options.return_value + ), + result, ) diff --git a/coriolis/tests/endpoint_resources/test_api.py b/coriolis/tests/endpoint_resources/test_api.py index 9504fa3f..dace142d 100644 --- a/coriolis/tests/endpoint_resources/test_api.py +++ b/coriolis/tests/endpoint_resources/test_api.py @@ -6,10 +6,7 @@ from coriolis.endpoint_resources import api from coriolis.tests import test_base -ARGS = { - "ctxt": "mock_ctxt", - "endpoint_id": "mock_endpoint_id" -} +ARGS = {"ctxt": "mock_ctxt", "endpoint_id": "mock_endpoint_id"} class EndpointResourcesAPITestCase(test_base.CoriolisBaseTestCase): @@ -27,69 +24,67 @@ def test_get_endpoint_instances(self): marker=mock.sentinel.marker, limit=mock.sentinel.limit, instance_name_pattern=mock.sentinel.instance_name_pattern, - refresh=mock.sentinel.refresh + refresh=mock.sentinel.refresh, ) - (self.endpoint_resources_api._rpc_client. - get_endpoint_instances.assert_called_once_with)( + ( + self.endpoint_resources_api._rpc_client.get_endpoint_instances.assert_called_once_with + )( *(ARGS.values()), mock.sentinel.source_environment, mock.sentinel.marker, mock.sentinel.limit, mock.sentinel.instance_name_pattern, - refresh=mock.sentinel.refresh + refresh=mock.sentinel.refresh, ) self.assertEqual( - (self.endpoint_resources_api._rpc_client. - get_endpoint_instances.return_value), - result + ( + self.endpoint_resources_api._rpc_client.get_endpoint_instances.return_value + ), + result, ) def test_get_endpoint_instance(self): result = self.endpoint_resources_api.get_endpoint_instance( **ARGS, source_environment=mock.sentinel.source_environment, - instance_name=mock.sentinel.instance_name + instance_name=mock.sentinel.instance_name, ) - (self.endpoint_resources_api._rpc_client. - get_endpoint_instance.assert_called_once_with)( + ( + self.endpoint_resources_api._rpc_client.get_endpoint_instance.assert_called_once_with + )( *(ARGS.values()), mock.sentinel.source_environment, - mock.sentinel.instance_name + mock.sentinel.instance_name, ) self.assertEqual( - (self.endpoint_resources_api._rpc_client. - get_endpoint_instance.return_value), - result + ( + self.endpoint_resources_api._rpc_client.get_endpoint_instance.return_value + ), + result, ) def test_get_endpoint_networks(self): result = self.endpoint_resources_api.get_endpoint_networks( - **ARGS, - env=mock.sentinel.env - ) - (self.endpoint_resources_api._rpc_client. - get_endpoint_networks.assert_called_once_with)( - *(ARGS.values()), - mock.sentinel.env + **ARGS, env=mock.sentinel.env ) + ( + self.endpoint_resources_api._rpc_client.get_endpoint_networks.assert_called_once_with + )(*(ARGS.values()), mock.sentinel.env) self.assertEqual( - (self.endpoint_resources_api._rpc_client. - get_endpoint_networks.return_value), - result + ( + self.endpoint_resources_api._rpc_client.get_endpoint_networks.return_value + ), + result, ) def test_get_endpoint_storage(self): result = self.endpoint_resources_api.get_endpoint_storage( - **ARGS, - env=mock.sentinel.env - ) - (self.endpoint_resources_api._rpc_client. - get_endpoint_storage.assert_called_once_with)( - *(ARGS.values()), - mock.sentinel.env + **ARGS, env=mock.sentinel.env ) + ( + self.endpoint_resources_api._rpc_client.get_endpoint_storage.assert_called_once_with + )(*(ARGS.values()), mock.sentinel.env) self.assertEqual( - (self.endpoint_resources_api._rpc_client. - get_endpoint_storage.return_value), - result + (self.endpoint_resources_api._rpc_client.get_endpoint_storage.return_value), + result, ) diff --git a/coriolis/tests/endpoints/test_api.py b/coriolis/tests/endpoints/test_api.py index c34fd191..e19068f3 100644 --- a/coriolis/tests/endpoints/test_api.py +++ b/coriolis/tests/endpoints/test_api.py @@ -4,8 +4,7 @@ from unittest import mock from coriolis.endpoints import api -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class EndpointsAPITestCase(test_base.CoriolisBaseTestCase): @@ -24,19 +23,15 @@ def test_create(self): "endpoint_type": mock.sentinel.endpoint_type, "description": mock.sentinel.description, "connection_info": mock.sentinel.connection_info, - "mapped_regions": mock.sentinel.mapped_regions + "mapped_regions": mock.sentinel.mapped_regions, } - result = self.endpoints_api.create( - **args - ) - (self.endpoints_api._rpc_conductor_client. - create_endpoint.assert_called_once_with)( - *(args.values()) - ) + result = self.endpoints_api.create(**args) + ( + self.endpoints_api._rpc_conductor_client.create_endpoint.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - create_endpoint.return_value), - result + (self.endpoints_api._rpc_conductor_client.create_endpoint.return_value), + result, ) def test_update(self): @@ -45,169 +40,128 @@ def test_update(self): "endpoint_id": mock.sentinel.endpoint_id, "properties": mock.sentinel.properties, } - result = self.endpoints_api.update( - **args - ) - (self.endpoints_api._rpc_conductor_client. - update_endpoint.assert_called_once_with)( - *(args.values()) - ) + result = self.endpoints_api.update(**args) + ( + self.endpoints_api._rpc_conductor_client.update_endpoint.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - update_endpoint.return_value), - result + (self.endpoints_api._rpc_conductor_client.update_endpoint.return_value), + result, ) def test_delete(self): - args = { - "ctxt": mock.sentinel.ctxt, - "endpoint_id": mock.sentinel.endpoint_id - } - self.endpoints_api.delete( - **args - ) - (self.endpoints_api._rpc_conductor_client. - delete_endpoint.assert_called_once_with)( - *(args.values()) - ) + args = {"ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id} + self.endpoints_api.delete(**args) + ( + self.endpoints_api._rpc_conductor_client.delete_endpoint.assert_called_once_with + )(*(args.values())) def test_get_endpoints(self): - args = { - "ctxt": mock.sentinel.ctxt - } - result = self.endpoints_api.get_endpoints( - **args - ) - (self.endpoints_api._rpc_conductor_client. - get_endpoints.assert_called_once_with)( - *(args.values()) - ) + args = {"ctxt": mock.sentinel.ctxt} + result = self.endpoints_api.get_endpoints(**args) + ( + self.endpoints_api._rpc_conductor_client.get_endpoints.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - get_endpoints.return_value), - result + (self.endpoints_api._rpc_conductor_client.get_endpoints.return_value), + result, ) def test_get_endpoint(self): - args = { - "ctxt": mock.sentinel.ctxt, - "endpoint_id": mock.sentinel.endpoint_id - } - result = self.endpoints_api.get_endpoint( - **args - ) - (self.endpoints_api._rpc_conductor_client. - get_endpoint.assert_called_once_with)( + args = {"ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id} + result = self.endpoints_api.get_endpoint(**args) + (self.endpoints_api._rpc_conductor_client.get_endpoint.assert_called_once_with)( *(args.values()) ) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - get_endpoint.return_value), - result + (self.endpoints_api._rpc_conductor_client.get_endpoint.return_value), result ) def test_validate_connection(self): - args = { - "ctxt": mock.sentinel.ctxt, - "endpoint_id": mock.sentinel.endpoint_id - } - result = self.endpoints_api.validate_connection( - **args - ) - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_connection.assert_called_once_with)( - *(args.values()) - ) + args = {"ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id} + result = self.endpoints_api.validate_connection(**args) + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_connection.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_connection.return_value), - result + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_connection.return_value + ), + result, ) def test_validate_target_environment(self): args = { "ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id, - "target_env": mock.sentinel.target_env + "target_env": mock.sentinel.target_env, } result = testutils.get_wrapped_function( - self.endpoints_api.validate_target_environment)( - self.endpoints_api, - **args - ) - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_target_environment.assert_called_once_with)( - *(args.values()) - ) + self.endpoints_api.validate_target_environment + )(self.endpoints_api, **args) + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_target_environment.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_target_environment.return_value), - result + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_target_environment.return_value + ), + result, ) def test_validate_source_environment(self): args = { "ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id, - "source_env": mock.sentinel.source_env + "source_env": mock.sentinel.source_env, } result = testutils.get_wrapped_function( - self.endpoints_api.validate_source_environment)( - self.endpoints_api, - **args - ) - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_source_environment.assert_called_once_with)( - *(args.values()) - ) + self.endpoints_api.validate_source_environment + )(self.endpoints_api, **args) + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_source_environment.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_conductor_client. - validate_endpoint_source_environment.return_value), - result + ( + self.endpoints_api._rpc_conductor_client.validate_endpoint_source_environment.return_value + ), + result, ) def test_validate_endpoint_source_minion_pool_options(self): args = { "ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id, - "pool_environment": mock.sentinel.pool_environment + "pool_environment": mock.sentinel.pool_environment, } result = testutils.get_wrapped_function( - self.endpoints_api.validate_endpoint_source_minion_pool_options)( - self.endpoints_api, - **args - ) - (self.endpoints_api._rpc_minion_manager_client. - validate_endpoint_source_minion_pool_options. - assert_called_once_with)( - *(args.values()) - ) + self.endpoints_api.validate_endpoint_source_minion_pool_options + )(self.endpoints_api, **args) + ( + self.endpoints_api._rpc_minion_manager_client.validate_endpoint_source_minion_pool_options.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_minion_manager_client. - validate_endpoint_source_minion_pool_options.return_value), - result + ( + self.endpoints_api._rpc_minion_manager_client.validate_endpoint_source_minion_pool_options.return_value + ), + result, ) def test_validate_endpoint_destination_minion_pool_options(self): args = { "ctxt": mock.sentinel.ctxt, "endpoint_id": mock.sentinel.endpoint_id, - "pool_environment": mock.sentinel.pool_environment + "pool_environment": mock.sentinel.pool_environment, } result = testutils.get_wrapped_function( - self.endpoints_api. - validate_endpoint_destination_minion_pool_options)( - self.endpoints_api, - **args - ) - (self.endpoints_api._rpc_minion_manager_client. - validate_endpoint_destination_minion_pool_options. - assert_called_once_with)( - *(args.values()) - ) + self.endpoints_api.validate_endpoint_destination_minion_pool_options + )(self.endpoints_api, **args) + ( + self.endpoints_api._rpc_minion_manager_client.validate_endpoint_destination_minion_pool_options.assert_called_once_with + )(*(args.values())) self.assertEqual( - (self.endpoints_api._rpc_minion_manager_client. - validate_endpoint_destination_minion_pool_options. - return_value), - result + ( + self.endpoints_api._rpc_minion_manager_client.validate_endpoint_destination_minion_pool_options.return_value + ), + result, ) diff --git a/coriolis/tests/integration/base.py b/coriolis/tests/integration/base.py index 077d680c..62457c90 100644 --- a/coriolis/tests/integration/base.py +++ b/coriolis/tests/integration/base.py @@ -18,21 +18,20 @@ import unittest from unittest import mock +import oslo_messaging as messaging from coriolisclient import client as coriolis_client from keystoneauth1 import session as ks_session from keystoneauth1 import token_endpoint from oslo_config import cfg from oslo_db.sqlalchemy import models from oslo_log import log as logging -import oslo_messaging as messaging -from coriolis import constants -from coriolis import context +from coriolis import constants, context from coriolis.db import api as db_api from coriolis.providers import factory as providers_factory +from coriolis.tests import test_base from coriolis.tests.integration import harness from coriolis.tests.integration import utils as test_utils -from coriolis.tests import test_base CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -95,8 +94,7 @@ def _call(self, ctxt, method, **kwargs): return original_call(self, ctxt, method, **kwargs) - patcher = mock.patch.object( - messaging.RPCClient, method, _call) + patcher = mock.patch.object(messaging.RPCClient, method, _call) patcher.start() self.addCleanup(patcher.stop) @@ -126,8 +124,14 @@ def _create_endpoint(cls, **kwargs): return endpoint def _create_transfer( - self, src_id, dst_id, instances, source_environment=None, - destination_environment=None, **kwargs): + self, + src_id, + dst_id, + instances, + source_environment=None, + destination_environment=None, + **kwargs, + ): """Create a Replica transfer object and return its ID.""" transfer = self._client.transfers.create( @@ -148,8 +152,7 @@ def _create_transfer( return transfer @classmethod - def _create_pool( - cls, endpoint_id, name="test-pool", skip_allocation=True): + def _create_pool(cls, endpoint_id, name="test-pool", skip_allocation=True): pool = cls._client.minion_pools.create( name=name, endpoint=endpoint_id, @@ -160,7 +163,8 @@ def _create_pool( maximum_minions=1, minion_max_idle_time=3600, minion_retention_strategy=( - constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE), + constants.MINION_POOL_MACHINE_RETENTION_STRATEGY_DELETE + ), skip_allocation=skip_allocation, ) cls.addClassCleanup(cls._safe_delete_pool, pool.id) @@ -177,8 +181,7 @@ def _safe_delete_pool(cls, pool_id): return if pool.status not in MINION_DEALLOCATED_TERMINAL: - cls._client.minion_pools.deallocate_minion_pool( - pool_id, force=True) + cls._client.minion_pools.deallocate_minion_pool(pool_id, force=True) cls._wait_for_pool(pool_id, MINION_DEALLOCATED_TERMINAL) with mock.patch("coriolis.keystone.delete_trust"): @@ -218,6 +221,7 @@ def _get_db_context(): @staticmethod def _ignoreExc(func, ignored_exc=Exception): """Wrap the given function, ignoring exceptions.""" + def f(*args, **kwargs): try: return func(*args, **kwargs) @@ -228,7 +232,6 @@ def f(*args, **kwargs): class ReplicaIntegrationTestBase(CoriolisIntegrationTestBase): - _CREATE_MINION_POOLS = False _SCSI_DEBUG_SIZE_MB = 16 @@ -244,7 +247,8 @@ def setUpClass(cls): "Docker image not found; build it with: " "docker build -t %s " "coriolis/tests/integration/dockerfiles/data-minion/" - % test_utils.DATA_MINION_IMAGE) + % test_utils.DATA_MINION_IMAGE + ) super().setUpClass() @@ -272,7 +276,8 @@ def setUpClass(cls): cls._pool_id = None if cls._CREATE_MINION_POOLS: pool = cls._create_pool( - cls._dst_endpoint.id, "transfer-pool", skip_allocation=False) + cls._dst_endpoint.id, "transfer-pool", skip_allocation=False + ) cls._pool_id = pool.id pool_obj = cls._wait_for_pool(pool.id, MINION_ALLOCATED_TERMINAL) @@ -326,11 +331,11 @@ def setUp(self): def _execute_and_wait(self, transfer_id, timeout=300): """Trigger one execution of *transfer_id* and wait for completion.""" execution = self._client.transfer_executions.create( - transfer_id, shutdown_instances=False) + transfer_id, shutdown_instances=False + ) self.assertExecutionCompleted(execution.id, timeout=timeout) - def wait_for_execution(self, execution_id, timeout=300, - desired_statuses=None): + def wait_for_execution(self, execution_id, timeout=300, desired_statuses=None): """Block until *execution_id* reaches a terminal state. Polls the DB directly and yields on each iteration so in-process @@ -368,7 +373,8 @@ def assertExecutionCompleted(self, execution_id, timeout=300): [ (t.task_type, t.status, t.exception_details) for t in execution.tasks - if t.status not in ( + if t.status + not in ( constants.TASK_STATUS_COMPLETED, constants.TASK_STATUS_CANCELED, ) @@ -389,8 +395,7 @@ def assertExecutionErrored(self, execution_id, timeout=300): % (execution_id, execution.status), ) - def wait_for_deployment(self, deployment_id, timeout=300, - desired_statuses=None): + def wait_for_deployment(self, deployment_id, timeout=300, desired_statuses=None): """Block until *deployment_id* reaches any terminal state. Returns the finalised deployment ORM object. @@ -431,7 +436,10 @@ def _slow_call(*args, **kwargs): self._execute_and_wait(self._transfer.id) patcher = mock.patch.object( - obj, method_name, side_effect=_slow_call, autospec=True, + obj, + method_name, + side_effect=_slow_call, + autospec=True, ) patcher.start() self.addCleanup(patcher.stop) @@ -452,13 +460,11 @@ def setUpClass(cls): imp_types = available.get(cls._imp_platform, {}).get("types", []) if constants.PROVIDER_TYPE_DESTINATION_MINION_POOL not in imp_types: raise unittest.SkipTest( - "Import provider '%s' does not support minion pools" - % cls._imp_platform + "Import provider '%s' does not support minion pools" % cls._imp_platform ) -class MinionPoolReplicaTestBase( - MinionPoolTestBase, ReplicaIntegrationTestBase): +class MinionPoolReplicaTestBase(MinionPoolTestBase, ReplicaIntegrationTestBase): """Base class for replica integration tests using minion pools. Extends the assertions to also verify that the minions in the pool have @@ -512,6 +518,5 @@ def assertMachinesAvailable(self, pool_id): self.assertIsNotNone( machine.last_used_at, "Machine %s in pool %s has no last_used_at; " - "it may not have been used by the transfer" - % (machine.id, pool_id), + "it may not have been used by the transfer" % (machine.id, pool_id), ) diff --git a/coriolis/tests/integration/deployments/test_deployment.py b/coriolis/tests/integration/deployments/test_deployment.py index de6d293c..4542e527 100644 --- a/coriolis/tests/integration/deployments/test_deployment.py +++ b/coriolis/tests/integration/deployments/test_deployment.py @@ -19,15 +19,14 @@ class ReplicaDeploymentIntegrationTest(base.ReplicaIntegrationTestBase): - def _make_deployment(self, **kwargs): # replica execution self._execute_and_wait(self._transfer.id) deployment = self._client.deployments.create_from_transfer( - self._transfer.id, **kwargs) - self.addCleanup( - self._ignoreExc(self._client.deployments.delete), deployment.id) + self._transfer.id, **kwargs + ) + self.addCleanup(self._ignoreExc(self._client.deployments.delete), deployment.id) return deployment @@ -55,25 +54,25 @@ def test_replica_deployment_crud(self): def test_replica_deployment_without_disk_cloning(self): # clone_disks=False reuses the already-transferred replica disks # directly instead of cloning them first. - deployment = self._make_deployment( - clone_disks=False, skip_os_morphing=True) + deployment = self._make_deployment(clone_disks=False, skip_os_morphing=True) self.assertDeploymentCompleted(deployment.id) def test_cancel_deployment(self): # Artificially bump the execution time of a deployment. self._patch_add_delay( - imp.TestImportProvider, - "finalize_replica_instance_deployment" + imp.TestImportProvider, "finalize_replica_instance_deployment" ) deployment = self._client.deployments.create_from_transfer( - self._transfer.id, skip_os_morphing=True) + self._transfer.id, skip_os_morphing=True + ) self.addCleanup(self._client.deployments.delete, deployment.id) # Wait until the deployment is RUNNING before issuing the cancel. self.wait_for_deployment( - deployment.id, 30, [constants.EXECUTION_STATUS_RUNNING]) + deployment.id, 30, [constants.EXECUTION_STATUS_RUNNING] + ) # Cancel the deployment. self._client.deployments.cancel(deployment.id) @@ -89,5 +88,6 @@ def test_cancel_deployment(self): class MinionPoolReplicaDeploymentTests( - base.MinionPoolReplicaTestBase, ReplicaDeploymentIntegrationTest): + base.MinionPoolReplicaTestBase, ReplicaDeploymentIntegrationTest +): """Replica deployment that uses a pre-allocated destination minion pool.""" diff --git a/coriolis/tests/integration/deployments/test_osmorphing.py b/coriolis/tests/integration/deployments/test_osmorphing.py index 54384b72..761c4af7 100644 --- a/coriolis/tests/integration/deployments/test_osmorphing.py +++ b/coriolis/tests/integration/deployments/test_osmorphing.py @@ -12,7 +12,6 @@ class OsMorphingDeploymentTest(integration_base.ReplicaIntegrationTestBase): - # NOTE(claudiub): Size must be high enough to contain the tested OS and # any new packages to be added during OS morphing. _SCSI_DEBUG_SIZE_MB = 256 diff --git a/coriolis/tests/integration/harness.py b/coriolis/tests/integration/harness.py index 00f44e00..880769f0 100644 --- a/coriolis/tests/integration/harness.py +++ b/coriolis/tests/integration/harness.py @@ -27,37 +27,34 @@ import tempfile import uuid -from cheroot.workers import threadpool as cheroot_threadpool +import webob.dec from cheroot import wsgi as cheroot_wsgi +from cheroot.workers import threadpool as cheroot_threadpool from oslo_config import cfg from oslo_log import log as logging from oslo_middleware import request_id as request_id_middleware from oslo_service import wsgi as base_wsgi -import webob.dec from coriolis import api as api_module +from coriolis import conf as coriolis_conf +from coriolis import constants, context, exception, service +from coriolis import policy as policy_module +from coriolis import rpc as rpc_module +from coriolis import utils as coriolis_utils +from coriolis.api import wsgi as api_wsgi from coriolis.api.middleware import fault as fault_middleware from coriolis.api.v1 import router as api_v1_router -from coriolis.api import wsgi as api_wsgi from coriolis.conductor.rpc import server as conductor_rpc_server -from coriolis import conf as coriolis_conf -from coriolis import constants -from coriolis import context from coriolis.db import api as db_api from coriolis.db.sqlalchemy import api as sqlalchemy_api from coriolis.db.sqlalchemy import migration as db_migration from coriolis.deployer_manager.rpc import server as deployer_manager_rpc_server -from coriolis import exception from coriolis.minion_manager.rpc import server as minion_manager_rpc_server -from coriolis import policy as policy_module -from coriolis import rpc as rpc_module from coriolis.scheduler.rpc import server as scheduler_rpc_server -from coriolis import service from coriolis.taskflow import runner as taskflow_runner from coriolis.tasks import factory as task_runners_factory from coriolis.tests.integration import utils as test_utils from coriolis.transfer_cron.rpc import server as transfer_cron_rpc_server -from coriolis import utils as coriolis_utils from coriolis.worker.rpc import server as worker_rpc_server CONF = cfg.CONF @@ -148,8 +145,10 @@ def _setup_routes(self, mapper, ext_mgr): ('endpoint_actions', '/endpoints/{id}/actions'), ('deployment_actions', '/deployments/{id}/actions'), ('transfer_actions', '/transfers/{id}/actions'), - ('transfer_tasks_execution_actions', - '/transfers/{transfer_id}/executions/{id}/actions'), + ( + 'transfer_tasks_execution_actions', + '/transfers/{transfer_id}/executions/{id}/actions', + ), ] for action, url in action_url_pairs: mapper.connect( @@ -170,26 +169,31 @@ class _InProcessWorkerServerEndpoint(worker_rpc_server.WorkerServerEndpoint): """ def _exec_task_process( - self, ctxt, task_id, task_type, origin, destination, instance, - task_info, report_to_conductor=True): + self, + ctxt, + task_id, + task_type, + origin, + destination, + instance, + task_info, + report_to_conductor=True, + ): result_q = queue.Queue() if report_to_conductor: - self._rpc_conductor_client.set_task_host( - ctxt, task_id, self._server) - self._rpc_conductor_client.set_task_process( - ctxt, task_id, os.getpid()) + self._rpc_conductor_client.set_task_host(ctxt, task_id, self._server) + self._rpc_conductor_client.set_task_process(ctxt, task_id, os.getpid()) def _run(): try: - task_runner = task_runners_factory.get_task_runner_class( - task_type)() - event_handler = ( - worker_rpc_server._get_event_handler_for_task_type( - task_type, ctxt, task_id)) + task_runner = task_runners_factory.get_task_runner_class(task_type)() + event_handler = worker_rpc_server._get_event_handler_for_task_type( + task_type, ctxt, task_id + ) task_result = task_runner.run( - ctxt, instance, origin, destination, task_info, - event_handler) + ctxt, instance, origin, destination, task_info, event_handler + ) coriolis_utils.is_serializable(task_result) result_q.put(task_result) except Exception as ex: @@ -224,7 +228,8 @@ def run_flow_in_background(self, flow, store=None): class _InProcessMinionManagerServerEndpoint( - minion_manager_rpc_server.MinionManagerServerEndpoint): + minion_manager_rpc_server.MinionManagerServerEndpoint +): """Minion manager endpoint that runs pool task flows in-thread.""" @property @@ -257,45 +262,59 @@ def __init__(self): self.lock_path = os.path.join(self.workdir, "locks") os.makedirs(self.lock_path) - self._mysql_container_name = "coriolis-test-mysql-%s" % str( - uuid.uuid4()).split("-")[0] + self._mysql_container_name = ( + "coriolis-test-mysql-%s" % str(uuid.uuid4()).split("-")[0] + ) self._mysql_username = "root" self._mysql_password = "coriolis" self._mysql_database = "coriolis" self.ssh_key_path = os.path.join(self.workdir, "id_rsa") subprocess.run( - ["ssh-keygen", "-t", "rsa", "-b", "2048", - "-f", self.ssh_key_path, "-N", ""], + [ + "ssh-keygen", + "-t", + "rsa", + "-b", + "2048", + "-f", + self.ssh_key_path, + "-N", + "", + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) coriolis_conf.init_common_opts() - cfg.CONF([], project='coriolis', version='1.0.0', - default_config_files=[], default_config_dirs=[]) + cfg.CONF( + [], + project='coriolis', + version='1.0.0', + default_config_files=[], + default_config_dirs=[], + ) cfg.CONF.set_override('messaging_transport_url', 'fake://') cfg.CONF.set_override( - 'providers', [_TEST_EXPORT_PROVIDER, _TEST_IMPORT_PROVIDER]) - db_url = ('mysql+pymysql://%(user)s:%(password)s' - '@localhost:13306/%(database)s') % { + 'providers', [_TEST_EXPORT_PROVIDER, _TEST_IMPORT_PROVIDER] + ) + db_url = ( + 'mysql+pymysql://%(user)s:%(password)s@localhost:13306/%(database)s' + ) % { "user": self._mysql_username, "password": self._mysql_password, "database": self._mysql_database, } - cfg.CONF.set_override( - 'connection', db_url, group='database') - cfg.CONF.set_override( - 'retry_interval', 1, group='database') - cfg.CONF.set_override( - 'lock_path', self.lock_path, group='oslo_concurrency') + cfg.CONF.set_override('connection', db_url, group='database') + cfg.CONF.set_override('retry_interval', 1, group='database') + cfg.CONF.set_override('lock_path', self.lock_path, group='oslo_concurrency') # Disable automatic pool-refresh cron jobs (they would try to contact # Keystone for trust maintenance). cfg.CONF.set_override( - 'minion_pool_default_refresh_period_minutes', 0, - group='minion_manager') + 'minion_pool_default_refresh_period_minutes', 0, group='minion_manager' + ) coriolis_utils.setup_logging() @@ -340,10 +359,13 @@ def _start_db_container(self): self._mysql_container_name, "-e", f"MYSQL_ROOT_PASSWORD={self._mysql_password}", - "-e", f"MYSQL_DATABASE={self._mysql_database}", - "-p", "13306:3306", + "-e", + f"MYSQL_DATABASE={self._mysql_database}", + "-p", + "13306:3306", "mariadb:10-jammy", - ]) + ] + ) def _start_coriolis_services(self): """Start conductor, scheduler, worker, and API in-process.""" @@ -465,25 +487,20 @@ def _teardown(self): LOG.info("Teardown initiated.") try: - coriolis_utils.exec_process( - [ - "docker", - "stop", - self._mysql_container_name - ]) - coriolis_utils.exec_process( - [ - "docker", - "rm", - self._mysql_container_name - ]) + coriolis_utils.exec_process(["docker", "stop", self._mysql_container_name]) + coriolis_utils.exec_process(["docker", "rm", self._mysql_container_name]) except Exception: pass - for svc in [self._worker_host_svc, self._worker_svc, - self._minion_manager_svc, self._deployer_manager_svc, - self._transfer_cron_svc, self._scheduler_svc, - self._conductor_svc]: + for svc in [ + self._worker_host_svc, + self._worker_svc, + self._minion_manager_svc, + self._deployer_manager_svc, + self._transfer_cron_svc, + self._scheduler_svc, + self._conductor_svc, + ]: if not svc: continue try: diff --git a/coriolis/tests/integration/management/test_diagnostics.py b/coriolis/tests/integration/management/test_diagnostics.py index 9bf6f83e..ac7cbc01 100644 --- a/coriolis/tests/integration/management/test_diagnostics.py +++ b/coriolis/tests/integration/management/test_diagnostics.py @@ -6,22 +6,21 @@ Exercises diagnostics.get() via the Coriolis REST API. """ -import netifaces import socket -from coriolis.tests.integration import base +import netifaces + from coriolis import utils +from coriolis.tests.integration import base class DiagnosticsTest(base.CoriolisIntegrationTestBase): - def test_get_diagnostics(self): diag_list = self._client.diagnostics.get() # Returns a list of Diagnostics resources, one per service. self.assertIsInstance(diag_list, list) - self.assertTrue( - len(diag_list) > 0, "Expected at least one diagnostics entry") + self.assertTrue(len(diag_list) > 0, "Expected at least one diagnostics entry") diag = diag_list[0] diag_ip_addr = diag.ip_addresses[0] diff --git a/coriolis/tests/integration/management/test_providers.py b/coriolis/tests/integration/management/test_providers.py index 65499e86..c5b53c53 100644 --- a/coriolis/tests/integration/management/test_providers.py +++ b/coriolis/tests/integration/management/test_providers.py @@ -12,7 +12,6 @@ class ProvidersTest(base.CoriolisIntegrationTestBase): - def test_list_providers(self): providers = self._client.providers.list() @@ -23,7 +22,8 @@ def test_list_providers(self): def test_schemas_list(self): # PROVIDER_TYPE_ENDPOINT returns a "connection_info_schema" key. schemas = self._client.providers.schemas_list( - self._imp_platform, constants.PROVIDER_TYPE_ENDPOINT) + self._imp_platform, constants.PROVIDER_TYPE_ENDPOINT + ) schema_types = [s["type"] for s in schemas.provider_schemas] self.assertIn("connection_info_schema", schema_types) @@ -31,7 +31,8 @@ def test_schemas_list(self): # PROVIDER_TYPE_TRANSFER_IMPORT returns a # "destination_environment_schema" key. schemas = self._client.providers.schemas_list( - self._imp_platform, constants.PROVIDER_TYPE_TRANSFER_IMPORT) + self._imp_platform, constants.PROVIDER_TYPE_TRANSFER_IMPORT + ) schema_types = [s["type"] for s in schemas.provider_schemas] self.assertIn("destination_environment_schema", schema_types) @@ -39,7 +40,8 @@ def test_schemas_list(self): # PROVIDER_TYPE_TRANSFER_EXPORT returns a "source_environment_schema" # key. schemas = self._client.providers.schemas_list( - self._exp_platform, constants.PROVIDER_TYPE_TRANSFER_EXPORT) + self._exp_platform, constants.PROVIDER_TYPE_TRANSFER_EXPORT + ) schema_types = [s["type"] for s in schemas.provider_schemas] self.assertIn("source_environment_schema", schema_types) diff --git a/coriolis/tests/integration/management/test_region.py b/coriolis/tests/integration/management/test_region.py index a975839f..9067b8e5 100644 --- a/coriolis/tests/integration/management/test_region.py +++ b/coriolis/tests/integration/management/test_region.py @@ -10,18 +10,17 @@ class RegionTests(base.CoriolisIntegrationTestBase): - def _create_region(self, name, **kwargs): region = self._client.regions.create(name, **kwargs) - self.addCleanup( - self._ignoreExc(self._client.regions.delete), region.id) + self.addCleanup(self._ignoreExc(self._client.regions.delete), region.id) return region def test_region_crud(self): # Create. region = self._create_region( - "test-region", description="integration test region") + "test-region", description="integration test region" + ) # Get. fetched = self._client.regions.get(region.id) @@ -34,8 +33,7 @@ def test_region_crud(self): self.assertIn(region.id, ids) # Update. - updated = self._client.regions.update( - region.id, {"description": "updated"}) + updated = self._client.regions.update(region.id, {"description": "updated"}) self.assertEqual("updated", updated.description) # Delete. diff --git a/coriolis/tests/integration/management/test_service.py b/coriolis/tests/integration/management/test_service.py index 4ad2ae2b..7b4a6576 100644 --- a/coriolis/tests/integration/management/test_service.py +++ b/coriolis/tests/integration/management/test_service.py @@ -12,10 +12,10 @@ class ServiceTests(base.CoriolisIntegrationTestBase): - def _create_service(self, host, binary, topic): svc = self._client.services.create( - host=host, binary=binary, topic=topic, regions=[]) + host=host, binary=binary, topic=topic, regions=[] + ) self.addCleanup(self._ignoreExc(self._client.services.delete), svc.id) @@ -26,13 +26,11 @@ def test_service_crud(self): # the conductor; at least one service record should exist. services = self._client.services.list() - self.assertTrue( - len(services) > 0, "Expected at least one registered service") + self.assertTrue(len(services) > 0, "Expected at least one registered service") # Create. hostname = socket.gethostname() - svc = self._create_service( - hostname, "foo-binary", "coriolis_worker") + svc = self._create_service(hostname, "foo-binary", "coriolis_worker") # Get. fetched = self._client.services.get(svc.id) diff --git a/coriolis/tests/integration/test_endpoints.py b/coriolis/tests/integration/test_endpoints.py index d7c1b3c2..c249d955 100644 --- a/coriolis/tests/integration/test_endpoints.py +++ b/coriolis/tests/integration/test_endpoints.py @@ -17,7 +17,6 @@ class EndpointCapabilitiesTest(base.CoriolisIntegrationTestBase): - def setUp(self): super().setUp() self._src_endpoint = self._create_endpoint( @@ -37,11 +36,13 @@ def setUp(self): def test_validate_connection(self): valid, message = self._client.endpoints.validate_connection( - self._src_endpoint.id) + self._src_endpoint.id + ) self.assertTrue(valid, f"source: {message}") valid, message = self._client.endpoints.validate_connection( - self._dst_endpoint.id) + self._dst_endpoint.id + ) self.assertTrue(valid, f"destination: {message}") def test_validate_connection_failure(self): @@ -52,8 +53,7 @@ def test_validate_connection_failure(self): "pkey_path": "/root/.ssh/coriolis-no-such-key", }, ) - valid, message = self._client.endpoints.validate_connection( - bad_endpoint.id) + valid, message = self._client.endpoints.validate_connection(bad_endpoint.id) self.assertFalse(valid) self.assertIsNotNone(message) @@ -73,34 +73,27 @@ def test_list_networks(self): def test_list_storage(self): storage = self._client.endpoint_storage.list(self._dst_endpoint.id) - self.assertTrue( - len(storage) > 0, "Expected at least one storage backend") + self.assertTrue(len(storage) > 0, "Expected at least one storage backend") first = storage[0] self.assertIn("id", first._info) # The test provider's get_storage() does not set a config_default so # the value is expected to be None. - default = self._client.endpoint_storage.get_default( - self._dst_endpoint.id) + default = self._client.endpoint_storage.get_default(self._dst_endpoint.id) self.assertIsNone(default) def test_list_source_options(self): - options = self._client.endpoint_source_options.list( - self._src_endpoint.id) + options = self._client.endpoint_source_options.list(self._src_endpoint.id) self.assertIsInstance(options, list) - self.assertTrue( - len(options) > 0, "Expected at least one source option") + self.assertTrue(len(options) > 0, "Expected at least one source option") def test_list_destination_options(self): - options = self._client.endpoint_destination_options.list( - self._dst_endpoint.id) + options = self._client.endpoint_destination_options.list(self._dst_endpoint.id) self.assertIsInstance(options, list) - self.assertTrue( - len(options) > 0, "Expected at least one destination option") + self.assertTrue(len(options) > 0, "Expected at least one destination option") def test_list_instances(self): - instances = self._client.endpoint_instances.list( - self._src_endpoint.id, env={}) + instances = self._client.endpoint_instances.list(self._src_endpoint.id, env={}) self.assertIsInstance(instances, list) self.assertTrue(len(instances) > 0, "Expected at least one instance") @@ -109,16 +102,17 @@ def test_list_instances(self): self.assertIn("name", first._info) instances = self._client.endpoint_instances.list( - self._src_endpoint.id, env={}, name="null") + self._src_endpoint.id, env={}, name="null" + ) self.assertIsInstance(instances, list) def test_get_instance(self): - instances = self._client.endpoint_instances.list( - self._src_endpoint.id, env={}) + instances = self._client.endpoint_instances.list(self._src_endpoint.id, env={}) self.assertTrue(len(instances) > 0) instance = self._client.endpoint_instances.get( - self._src_endpoint.id, instances[0].name, env={}) + self._src_endpoint.id, instances[0].name, env={} + ) self.assertEqual(instances[0].name, instance.name) diff --git a/coriolis/tests/integration/test_failure_recovery.py b/coriolis/tests/integration/test_failure_recovery.py index 86ca4874..a361cf16 100644 --- a/coriolis/tests/integration/test_failure_recovery.py +++ b/coriolis/tests/integration/test_failure_recovery.py @@ -32,5 +32,6 @@ def test_error_status_on_provider_failure(self): side_effect=injected_error, ): execution = self._client.transfer_executions.create( - self._transfer.id, shutdown_instances=False) + self._transfer.id, shutdown_instances=False + ) self.assertExecutionErrored(execution.id) diff --git a/coriolis/tests/integration/test_minion_pools.py b/coriolis/tests/integration/test_minion_pools.py index 433b0ffe..de57dc70 100644 --- a/coriolis/tests/integration/test_minion_pools.py +++ b/coriolis/tests/integration/test_minion_pools.py @@ -15,7 +15,6 @@ class MinionPoolLifecycleTest(base.MinionPoolTestBase): - def setUp(self): super().setUp() @@ -33,8 +32,7 @@ def test_minion_pool_crud(self): pool = self._create_pool(self._endpoint.id) self.assertEqual("test-pool", pool.name) - self.assertEqual( - constants.MINION_POOL_STATUS_DEALLOCATED, pool.status) + self.assertEqual(constants.MINION_POOL_STATUS_DEALLOCATED, pool.status) # List pools = self._client.minion_pools.list() @@ -49,8 +47,7 @@ def test_minion_pool_crud(self): self.assertEqual("test-pool", fetched.name) # Update - updated = self._client.minion_pools.update( - pool.id, {"notes": "updated notes"}) + updated = self._client.minion_pools.update(pool.id, {"notes": "updated notes"}) self.assertEqual("updated notes", updated.notes) @@ -62,8 +59,7 @@ def test_minion_pool_crud(self): def test_allocate_deallocate(self): pool = self._create_pool(self._endpoint.id) - self.assertEqual( - constants.MINION_POOL_STATUS_DEALLOCATED, pool.status) + self.assertEqual(constants.MINION_POOL_STATUS_DEALLOCATED, pool.status) # Allocate self._client.minion_pools.allocate_minion_pool(pool.id) diff --git a/coriolis/tests/integration/test_pagination.py b/coriolis/tests/integration/test_pagination.py index fefa125a..89f66c90 100755 --- a/coriolis/tests/integration/test_pagination.py +++ b/coriolis/tests/integration/test_pagination.py @@ -9,11 +9,10 @@ from oslo_utils import timeutils -from coriolis import constants +from coriolis import constants, exception from coriolis import context as coriolis_context from coriolis.db import api as db_api from coriolis.db.sqlalchemy import models -from coriolis import exception from coriolis.tests.integration import base @@ -30,8 +29,8 @@ def setUpClass(cls): super().setUpClass() cls._ctx = coriolis_context.RequestContext( - user=cls.FAKE_USER_ID, - project_id=cls.FAKE_PROJECT_ID) + user=cls.FAKE_USER_ID, project_id=cls.FAKE_PROJECT_ID + ) cls._admin_ctx = coriolis_context.get_admin_context() cls._setup_mocks() @@ -48,13 +47,16 @@ def _create_db_transfer( kwargs["instances"] = instances or [] kwargs["origin_endpoint_id"] = origin_endpoint_id kwargs["destination_endpoint_id"] = destination_endpoint_id - kwargs["info"] = {instance: { - 'volumes_info': []} for instance in kwargs["instances"]} + kwargs["info"] = { + instance: {'volumes_info': []} for instance in kwargs["instances"] + } transfer = models.Transfer(**kwargs) db_api.add_transfer(cls._ctx, transfer) cls.addClassCleanup( cls._ignoreExc(db_api.delete_transfer, exception.NotFound), - cls._admin_ctx, transfer.id) + cls._admin_ctx, + transfer.id, + ) return transfer @classmethod @@ -64,21 +66,18 @@ def _create_db_execution( **kwargs, ) -> models.TasksExecution: kwargs["action_id"] = transfer.id - kwargs["status"] = kwargs.get( - "status", - constants.EXECUTION_STATUS_UNEXECUTED) - kwargs["type"] = kwargs.get( - "type", - constants.EXECUTION_TYPE_TRANSFER_EXECUTION) + kwargs["status"] = kwargs.get("status", constants.EXECUTION_STATUS_UNEXECUTED) + kwargs["type"] = kwargs.get("type", constants.EXECUTION_TYPE_TRANSFER_EXECUTION) execution = models.TasksExecution(**kwargs) # "add_transfer_tasks_execution" expects "action" to be set, # despite not being declared by the model. execution.action = transfer db_api.add_transfer_tasks_execution(cls._admin_ctx, execution) cls.addClassCleanup( - cls._ignoreExc(db_api.delete_transfer_tasks_execution, - exception.NotFound), - cls._admin_ctx, execution.id) + cls._ignoreExc(db_api.delete_transfer_tasks_execution, exception.NotFound), + cls._admin_ctx, + execution.id, + ) return execution @classmethod @@ -90,12 +89,13 @@ def _create_db_endpoint( kwargs["id"] = endpoint_id kwargs["name"] = kwargs.get("name", f"test-endpoint-{endpoint_id}") kwargs["type"] = kwargs.get("type", "openstack") - endpoint = models.Endpoint( - **kwargs) + endpoint = models.Endpoint(**kwargs) db_api.add_endpoint(cls._ctx, endpoint) cls.addClassCleanup( cls._ignoreExc(db_api.delete_endpoint, exception.NotFound), - cls._admin_ctx, endpoint.id) + cls._admin_ctx, + endpoint.id, + ) return endpoint @classmethod @@ -110,12 +110,13 @@ def _create_db_deployment( kwargs["transfer_id"] = transfer_id kwargs["origin_endpoint_id"] = origin_endpoint_id kwargs["destination_endpoint_id"] = destination_endpoint_id - deployment = models.Deployment( - **kwargs) + deployment = models.Deployment(**kwargs) db_api.add_deployment(cls._ctx, deployment) cls.addClassCleanup( cls._ignoreExc(db_api.delete_deployment, exception.NotFound), - cls._admin_ctx, deployment.id) + cls._admin_ctx, + deployment.id, + ) return deployment @classmethod @@ -136,16 +137,18 @@ def _setup_mocks(cls): transfer = cls._create_db_transfer( origin_endpoint_id=cls._src_endpoint.id, destination_endpoint_id=cls._dst_endpoint.id, - created_at=timeutils.utcnow() + datetime.timedelta( - seconds=transfer_idx)) + created_at=timeutils.utcnow() + + datetime.timedelta(seconds=transfer_idx), + ) cls._transfers.append(transfer) cls._executions[transfer.id] = [] for execution_idx in range(cls.EXECUTIONS_PER_TRANSFER): execution = cls._create_db_execution( transfer=transfer, - created_at=timeutils.utcnow() + datetime.timedelta( - seconds=execution_idx)) + created_at=timeutils.utcnow() + + datetime.timedelta(seconds=execution_idx), + ) cls._executions[transfer.id].append(execution) cls._deployments[transfer.id] = [] @@ -154,8 +157,9 @@ def _setup_mocks(cls): transfer_id=transfer.id, origin_endpoint_id=cls._src_endpoint.id, destination_endpoint_id=cls._dst_endpoint.id, - created_at=timeutils.utcnow() + datetime.timedelta( - seconds=deployment_idx)) + created_at=timeutils.utcnow() + + datetime.timedelta(seconds=deployment_idx), + ) cls._deployments[transfer.id].append(deployment) cls._all_deployments.append(deployment) @@ -176,35 +180,29 @@ def _get_record_summary(record): } def test_transfer_execution_list(self): - executions = self._client.transfer_executions.list( - self._transfers[0].id) + executions = self._client.transfer_executions.list(self._transfers[0].id) ret_exec_summary = [self._get_record_summary(e) for e in executions] exp_exec = self._executions[self._transfers[0].id] sorted_exp_exec = sorted( - exp_exec, - key=operator.attrgetter('created_at'), - reverse=True) - exp_sorted_exec_summary = [ - self._get_record_summary(e) for e in sorted_exp_exec] + exp_exec, key=operator.attrgetter('created_at'), reverse=True + ) + exp_sorted_exec_summary = [self._get_record_summary(e) for e in sorted_exp_exec] self.assertEqual(exp_sorted_exec_summary, ret_exec_summary) def test_transfer_execution_list_pagination(self): # Get the first 2 entries, sorted by ID in ascending order. executions = self._client.transfer_executions.list( - self._transfers[0].id, - limit=2, - sort_keys=['id'], - sort_dirs=['asc']) + self._transfers[0].id, limit=2, sort_keys=['id'], sort_dirs=['asc'] + ) ret_exec_summary = [self._get_record_summary(e) for e in executions] exp_exec = self._executions[self._transfers[0].id] - sorted_exp_exec = sorted( - exp_exec, - key=operator.attrgetter('id')) + sorted_exp_exec = sorted(exp_exec, key=operator.attrgetter('id')) exp_sorted_exec_summary = [ - self._get_record_summary(e) for e in sorted_exp_exec][:2] + self._get_record_summary(e) for e in sorted_exp_exec + ][:2] self.assertEqual(exp_sorted_exec_summary, ret_exec_summary) # Get the next 2 entries. @@ -213,13 +211,13 @@ def test_transfer_execution_list_pagination(self): limit=2, marker=executions[-1].id, sort_keys=['id'], - sort_dirs=['asc']) - ret_exec_summary = [ - self._get_record_summary(e) - for e in next_executions] + sort_dirs=['asc'], + ) + ret_exec_summary = [self._get_record_summary(e) for e in next_executions] exp_sorted_exec_summary = [ - self._get_record_summary(e) for e in sorted_exp_exec][2:4] + self._get_record_summary(e) for e in sorted_exp_exec + ][2:4] self.assertEqual(exp_sorted_exec_summary, ret_exec_summary) def test_deployment_list(self): @@ -227,26 +225,26 @@ def test_deployment_list(self): ret_depl_summary = [self._get_record_summary(d) for d in deployments] exp_sorted_depl_summary = [ - self._get_record_summary(d) for d in self._all_deployments] + self._get_record_summary(d) for d in self._all_deployments + ] exp_sorted_depl_summary = sorted( exp_sorted_depl_summary, key=lambda x: (x["created_at"], x["id"]), - reverse=True) + reverse=True, + ) self.assertEqual(exp_sorted_depl_summary, ret_depl_summary) def test_deployment_list_pagination(self): # Get the first 2 entries, sorted by ID in ascending order. deployments = self._client.deployments.list( - limit=2, - sort_keys=['id'], - sort_dirs=['asc']) + limit=2, sort_keys=['id'], sort_dirs=['asc'] + ) ret_depl_summary = [self._get_record_summary(d) for d in deployments] exp_sorted_depl_summary = [ - self._get_record_summary(d) for d in self._all_deployments] - exp_sorted_depl_summary = sorted( - exp_sorted_depl_summary, - key=lambda x: x["id"]) + self._get_record_summary(d) for d in self._all_deployments + ] + exp_sorted_depl_summary = sorted(exp_sorted_depl_summary, key=lambda x: x["id"]) self.assertEqual(exp_sorted_depl_summary[:2], ret_depl_summary) # Get the next 2 entries. @@ -264,34 +262,33 @@ def test_transfer_list(self): ret_transfer_summary = [self._get_record_summary(t) for t in transfers] exp_sorted_transfer_summary = [ - self._get_record_summary(d) for d in self._transfers] + self._get_record_summary(d) for d in self._transfers + ] exp_sorted_transfer_summary = sorted( exp_sorted_transfer_summary, key=lambda x: (x["created_at"], x["id"]), - reverse=True) + reverse=True, + ) self.assertEqual(exp_sorted_transfer_summary, ret_transfer_summary) def test_transfer_list_pagination(self): # Get the first 2 entries, sorted by ID in ascending order. transfers = self._client.transfers.list( - limit=2, - sort_keys=['id'], - sort_dirs=['asc']) + limit=2, sort_keys=['id'], sort_dirs=['asc'] + ) ret_transfer_summary = [self._get_record_summary(t) for t in transfers] exp_sorted_transfer_summary = [ - self._get_record_summary(d) for d in self._transfers] + self._get_record_summary(d) for d in self._transfers + ] exp_sorted_transfer_summary = sorted( - exp_sorted_transfer_summary, - key=lambda x: x["id"]) + exp_sorted_transfer_summary, key=lambda x: x["id"] + ) self.assertEqual(exp_sorted_transfer_summary[:2], ret_transfer_summary) # Get the next 2 entries. transfers = self._client.transfers.list( - limit=2, - sort_keys=['id'], - sort_dirs=['asc'], - marker=transfers[-1].id) + limit=2, sort_keys=['id'], sort_dirs=['asc'], marker=transfers[-1].id + ) ret_transfer_summary = [self._get_record_summary(t) for t in transfers] - self.assertEqual( - exp_sorted_transfer_summary[2:4], ret_transfer_summary) + self.assertEqual(exp_sorted_transfer_summary[2:4], ret_transfer_summary) diff --git a/coriolis/tests/integration/test_provider/exp.py b/coriolis/tests/integration/test_provider/exp.py index e5124410..1941c83e 100644 --- a/coriolis/tests/integration/test_provider/exp.py +++ b/coriolis/tests/integration/test_provider/exp.py @@ -11,18 +11,20 @@ import os import uuid +import paramiko from oslo_config import cfg from oslo_log import log as logging -import paramiko from coriolis import events from coriolis.providers import backup_writers -from coriolis.providers.base import BaseEndpointInstancesProvider -from coriolis.providers.base import BaseEndpointSourceOptionsProvider -from coriolis.providers.base import BaseReplicaExportProvider -from coriolis.providers.base import BaseReplicaExportValidationProvider -from coriolis.providers.base import BaseUpdateSourceReplicaProvider from coriolis.providers import replicator as replicator_module +from coriolis.providers.base import ( + BaseEndpointInstancesProvider, + BaseEndpointSourceOptionsProvider, + BaseReplicaExportProvider, + BaseReplicaExportValidationProvider, + BaseUpdateSourceReplicaProvider, +) from coriolis.tests.integration import utils as test_utils CONF = cfg.CONF @@ -30,11 +32,12 @@ class TestExportProvider( - BaseEndpointInstancesProvider, - BaseEndpointSourceOptionsProvider, - BaseUpdateSourceReplicaProvider, - BaseReplicaExportProvider, - BaseReplicaExportValidationProvider): + BaseEndpointInstancesProvider, + BaseEndpointSourceOptionsProvider, + BaseUpdateSourceReplicaProvider, + BaseReplicaExportProvider, + BaseReplicaExportValidationProvider, +): """Source-side provider backed by a local `scsi_debug` block device. ``connection_info`` (the source endpoint's connection info) has the form:: @@ -72,7 +75,8 @@ def _make_replicator(self, conn_info, event_mgr, volumes_info, repl_state): "pkey": pkey, } return replicator_module.Replicator( - repl_conn_info, event_mgr, volumes_info, repl_state) + repl_conn_info, event_mgr, volumes_info, repl_state + ) # BaseProvider / BaseEndpointProvider @@ -103,13 +107,19 @@ def get_source_environment_schema(self): # BaseEndpointInstancesProvider - def get_instances(self, ctxt, connection_info, source_environment, - limit=None, last_seen_id=None, - instance_name_pattern=None, refresh=False): + def get_instances( + self, + ctxt, + connection_info, + source_environment, + limit=None, + last_seen_id=None, + instance_name_pattern=None, + refresh=False, + ): return [self._instance_info(source_environment)] - def get_instance(self, ctxt, connection_info, source_environment, - instance_name): + def get_instance(self, ctxt, connection_info, source_environment, instance_name): return self._instance_info(source_environment) def _instance_info(self, source_environment): @@ -136,7 +146,8 @@ def _instance_info(self, source_environment): # BaseEndpointSourceOptionsProvider def get_source_environment_options( - self, ctxt, connection_info, env=None, option_names=None): + self, ctxt, connection_info, env=None, option_names=None + ): return [ { "name": "source_opt", @@ -148,8 +159,8 @@ def get_source_environment_options( # BaseUpdateSourceReplicaProvider def check_update_source_environment_params( - self, ctxt, connection_info, instance_name, volumes_info, - old_params, new_params): + self, ctxt, connection_info, instance_name, volumes_info, old_params, new_params + ): return volumes_info def get_os_morphing_tools(self, os_type, osmorphing_info): @@ -158,7 +169,8 @@ def get_os_morphing_tools(self, os_type, osmorphing_info): # BaseReplicaExportProvider def get_replica_instance_info( - self, ctxt, connection_info, source_environment, instance_name): + self, ctxt, connection_info, source_environment, instance_name + ): """Return minimal export info describing the source block device.""" block_device_path = source_environment["block_device_path"] size_bytes = _get_block_device_size(block_device_path) @@ -189,7 +201,8 @@ def get_replica_instance_info( } def deploy_replica_source_resources( - self, ctxt, connection_info, export_info, source_environment): + self, ctxt, connection_info, export_info, source_environment + ): block_device_path = source_environment["block_device_path"] pkey_path = connection_info["pkey_path"] @@ -213,7 +226,8 @@ def deploy_replica_source_resources( "pkey_path": pkey_path, } replicator = self._make_replicator( - src_conn_info, self._event_manager(), [], None) + src_conn_info, self._event_manager(), [], None + ) replicator.init_replicator() disk_id = os.path.basename(block_device_path) @@ -229,20 +243,29 @@ def deploy_replica_source_resources( raise def delete_replica_source_resources( - self, ctxt, connection_info, source_environment, - migr_resources_dict): + self, ctxt, connection_info, source_environment, migr_resources_dict + ): container_id = (migr_resources_dict or {}).get("container_id") if container_id: test_utils.remove_container(container_id) def replicate_disks( - self, ctxt, connection_info, source_environment, instance_name, - source_resources, source_conn_info, target_conn_info, - volumes_info, incremental): + self, + ctxt, + connection_info, + source_environment, + instance_name, + source_resources, + source_conn_info, + target_conn_info, + volumes_info, + incremental, + ): repl_state = _extract_repl_state(volumes_info) if incremental else None replicator = self._make_replicator( - source_conn_info, self._event_manager(), volumes_info, repl_state) + source_conn_info, self._event_manager(), volumes_info, repl_state + ) replicator.init_replicator() replicator.wait_for_chunks() @@ -256,25 +279,29 @@ def replicate_disks( ] backup_writer = backup_writers.BackupWritersFactory( - target_conn_info, volumes_info).get_writer() + target_conn_info, volumes_info + ).get_writer() replicator.replicate_disks(source_volumes_info, backup_writer) return volumes_info def delete_replica_source_snapshots( - self, ctxt, connection_info, source_environment, volumes_info): + self, ctxt, connection_info, source_environment, volumes_info + ): # scsi_debug devices have no snapshots. return volumes_info def shutdown_instance( - self, ctxt, connection_info, source_environment, instance_name): + self, ctxt, connection_info, source_environment, instance_name + ): # Nothing to shut down for a block device. pass # BaseReplicaExportValidationProvider def validate_replica_export_input( - self, ctxt, connection_info, instance_name, source_environment): + self, ctxt, connection_info, instance_name, source_environment + ): return {} diff --git a/coriolis/tests/integration/test_provider/imp.py b/coriolis/tests/integration/test_provider/imp.py index b37d4c78..29664359 100644 --- a/coriolis/tests/integration/test_provider/imp.py +++ b/coriolis/tests/integration/test_provider/imp.py @@ -12,21 +12,23 @@ import os import uuid -from oslo_log import log as logging import paramiko +from oslo_log import log as logging +from coriolis import utils as coriolis_utils from coriolis.providers import backup_writers -from coriolis.providers.base import BaseDestinationMinionPoolProvider -from coriolis.providers.base import BaseEndpointDestinationOptionsProvider -from coriolis.providers.base import BaseEndpointNetworksProvider -from coriolis.providers.base import BaseEndpointProvider -from coriolis.providers.base import BaseEndpointStorageProvider -from coriolis.providers.base import BaseReplicaImportProvider -from coriolis.providers.base import BaseReplicaImportValidationProvider -from coriolis.providers.base import BaseUpdateDestinationReplicaProvider -from coriolis.tests.integration.test_provider import osmorphing +from coriolis.providers.base import ( + BaseDestinationMinionPoolProvider, + BaseEndpointDestinationOptionsProvider, + BaseEndpointNetworksProvider, + BaseEndpointProvider, + BaseEndpointStorageProvider, + BaseReplicaImportProvider, + BaseReplicaImportValidationProvider, + BaseUpdateDestinationReplicaProvider, +) from coriolis.tests.integration import utils as test_utils -from coriolis import utils as coriolis_utils +from coriolis.tests.integration.test_provider import osmorphing LOG = logging.getLogger(__name__) @@ -35,14 +37,15 @@ class TestImportProvider( - BaseEndpointProvider, - BaseEndpointDestinationOptionsProvider, - BaseEndpointNetworksProvider, - BaseEndpointStorageProvider, - BaseUpdateDestinationReplicaProvider, - BaseReplicaImportProvider, - BaseReplicaImportValidationProvider, - BaseDestinationMinionPoolProvider): + BaseEndpointProvider, + BaseEndpointDestinationOptionsProvider, + BaseEndpointNetworksProvider, + BaseEndpointStorageProvider, + BaseUpdateDestinationReplicaProvider, + BaseReplicaImportProvider, + BaseReplicaImportValidationProvider, + BaseDestinationMinionPoolProvider, +): """Destination-side provider backed by a local `scsi_debug` block device. ``connection_info`` (the destination endpoint's connection info) has the @@ -98,7 +101,8 @@ def get_target_environment_schema(self): # BaseEndpointDestinationOptionsProvider def get_target_environment_options( - self, ctxt, connection_info, env=None, option_names=None): + self, ctxt, connection_info, env=None, option_names=None + ): return [ { "name": "dest_opt", @@ -122,15 +126,21 @@ def get_storage(self, ctxt, connection_info, target_environment): # BaseUpdateDestinationReplicaProvider def check_update_destination_environment_params( - self, ctxt, connection_info, export_info, volumes_info, - old_params, new_params): + self, ctxt, connection_info, export_info, volumes_info, old_params, new_params + ): return volumes_info # BaseReplicaImportProvider def deploy_replica_disks( - self, ctxt, connection_info, target_environment, instance_name, - export_info, volumes_info): + self, + ctxt, + connection_info, + target_environment, + instance_name, + export_info, + volumes_info, + ): """Map each source disk in export_info to a destination device. Returns a volumes_info list where each entry has ``disk_id`` (from @@ -147,18 +157,20 @@ def deploy_replica_disks( result = [] for i, disk in enumerate(src_disks): - result.append({ - "disk_id": disk["id"], - "volume_dev": dest_devices[i], - }) + result.append( + { + "disk_id": disk["id"], + "volume_dev": dest_devices[i], + } + ) return result def deploy_replica_target_resources( - self, ctxt, connection_info, target_environment, volumes_info): + self, ctxt, connection_info, target_environment, volumes_info + ): devices = [vol["volume_dev"] for vol in volumes_info] - result = self._create_minion( - "coriolis-writer", connection_info, devices) + result = self._create_minion("coriolis-writer", connection_info, devices) return { "volumes_info": volumes_info, @@ -167,8 +179,14 @@ def deploy_replica_target_resources( } def _create_minion( - self, name_prefix, connection_info, devices=None, volumes=None, - device_cgroup_rules=None, setup_writer=True): + self, + name_prefix, + connection_info, + devices=None, + volumes=None, + device_cgroup_rules=None, + setup_writer=True, + ): pkey_path = connection_info["pkey_path"] container_name = "%s-%s" % (name_prefix, uuid.uuid4().hex[:8]) @@ -200,7 +218,8 @@ def _create_minion( } if setup_writer: bootstrapper = backup_writers.HTTPBackupWriterBootstrapper( - ssh_conn_info, WRITER_TEST_PORT) + ssh_conn_info, WRITER_TEST_PORT + ) writer_conn_details = bootstrapper.setup_writer() info["backup_writer_connection_info"] = { "backend": "http_backup_writer", @@ -213,38 +232,49 @@ def _create_minion( raise def delete_replica_target_resources( - self, ctxt, connection_info, target_environment, - migr_resources_dict): + self, ctxt, connection_info, target_environment, migr_resources_dict + ): container_id = (migr_resources_dict or {}).get("container_id") if container_id: test_utils.remove_container(container_id) def delete_replica_disks( - self, ctxt, connection_info, target_environment, volumes_info): + self, ctxt, connection_info, target_environment, volumes_info + ): # scsi_debug devices are managed externally; nothing to delete here. return volumes_info def create_replica_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, ctxt, connection_info, target_environment, volumes_info + ): # scsi_debug has no snapshot support. return volumes_info def delete_replica_target_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, ctxt, connection_info, target_environment, volumes_info + ): return volumes_info def restore_replica_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, ctxt, connection_info, target_environment, volumes_info + ): return volumes_info def deploy_replica_instance( - self, ctxt, connection_info, target_environment, instance_name, - export_info, volumes_info, clone_disks): + self, + ctxt, + connection_info, + target_environment, + instance_name, + export_info, + volumes_info, + clone_disks, + ): return {"instance_deployment_info": {}} def finalize_replica_instance_deployment( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, ctxt, connection_info, target_environment, instance_deployment_info + ): return { "id": "test-instance", "name": "test-instance", @@ -263,8 +293,8 @@ def finalize_replica_instance_deployment( } def cleanup_failed_replica_instance_deployment( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, ctxt, connection_info, target_environment, instance_deployment_info + ): pass # BaseInstanceProvider @@ -275,24 +305,25 @@ def get_os_morphing_tools(self, os_type, osmorphing_info): # BaseImportInstanceProvider def deploy_os_morphing_resources( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, ctxt, connection_info, target_environment, instance_deployment_info + ): devices = list(target_environment.get("devices", [])) # lsblk inside the container sees all the host block devices because # Docker containers share the host kernel's sysfs (/sys/block/). # Populate ignore_devices with every host disk except the target # so osmorphing only considers the devices we actually attached. - ignore_devices = list( - test_utils.get_host_disk_devices() - set(devices) - ) + ignore_devices = list(test_utils.get_host_disk_devices() - set(devices)) # Mount the host's /lib/modules tree so that modprobe can # resolve built-in modules. volumes = ["/lib/modules:/lib/modules:ro"] result = self._create_minion( - "coriolis-osmorphing", connection_info, devices, - volumes, setup_writer=False, + "coriolis-osmorphing", + connection_info, + devices, + volumes, + setup_writer=False, ) return { @@ -305,8 +336,8 @@ def deploy_os_morphing_resources( } def delete_os_morphing_resources( - self, ctxt, connection_info, target_environment, - os_morphing_resources): + self, ctxt, connection_info, target_environment, os_morphing_resources + ): if os_morphing_resources: container_id = os_morphing_resources.get("container_id") if container_id: @@ -315,12 +346,19 @@ def delete_os_morphing_resources( # BaseReplicaImportValidationProvider def validate_replica_import_input( - self, ctxt, connection_info, target_environment, export_info, - check_os_morphing_resources=False, check_final_vm_params=False): + self, + ctxt, + connection_info, + target_environment, + export_info, + check_os_morphing_resources=False, + check_final_vm_params=False, + ): return {} def validate_replica_deployment_input( - self, ctxt, connection_info, target_environment, export_info): + self, ctxt, connection_info, target_environment, export_info + ): return {} # BaseDestinationMinionPoolProvider @@ -329,38 +367,52 @@ def get_minion_pool_environment_schema(self): return self.get_target_environment_schema() def get_minion_pool_options( - self, ctxt, connection_info, env=None, option_names=None): + self, ctxt, connection_info, env=None, option_names=None + ): return self.get_target_environment_options( - ctxt, connection_info, env, option_names) + ctxt, connection_info, env, option_names + ) def validate_minion_compatibility_for_transfer( - self, ctxt, connection_info, export_info, environment_options, - minion_properties): + self, ctxt, connection_info, export_info, environment_options, minion_properties + ): pass def validate_minion_pool_environment_options( - self, ctxt, connection_info, environment_options): + self, ctxt, connection_info, environment_options + ): pass def set_up_pool_shared_resources( - self, ctxt, connection_info, environment_options, pool_identifier): + self, ctxt, connection_info, environment_options, pool_identifier + ): return {} def tear_down_pool_shared_resources( - self, ctxt, connection_info, environment_options, - pool_shared_resources): + self, ctxt, connection_info, environment_options, pool_shared_resources + ): pass def create_minion( - self, ctxt, connection_info, environment_options, pool_identifier, - pool_os_type, pool_shared_resources, new_minion_identifier): + self, + ctxt, + connection_info, + environment_options, + pool_identifier, + pool_os_type, + pool_shared_resources, + new_minion_identifier, + ): # Devices are hotplugged after container creation via mknod / nsenter. # We must pre-authorize all block devices through the # --device-cgroup-rule option, otherwise any device added will be # inaccessible ("operation not permitted" error on open). result = self._create_minion( - "coriolis-pool-minion", connection_info, [], - device_cgroup_rules=["b *:* rwm"]) + "coriolis-pool-minion", + connection_info, + [], + device_cgroup_rules=["b *:* rwm"], + ) backup_writer_conn_info = result["backup_writer_connection_info"] return { @@ -387,8 +439,13 @@ def start_minion(self, ctxt, connection_info, minion_properties): test_utils.start_container(container_id) def attach_volumes_to_minion( - self, ctxt, connection_info, minion_properties, - minion_connection_info, volumes_info): + self, + ctxt, + connection_info, + minion_properties, + minion_connection_info, + volumes_info, + ): container_id = minion_properties["container_id"] for vol in volumes_info: device_path = vol["volume_dev"] @@ -400,13 +457,18 @@ def attach_volumes_to_minion( } def detach_volumes_from_minion( - self, ctxt, connection_info, minion_properties, - minion_connection_info, volumes_info): + self, + ctxt, + connection_info, + minion_properties, + minion_connection_info, + volumes_info, + ): container_id = (minion_properties or {}).get("container_id") if not container_id: return - for vol in (volumes_info or []): + for vol in volumes_info or []: dev_path = vol.get("volume_dev") if not dev_path: continue @@ -419,8 +481,8 @@ def detach_volumes_from_minion( } def healthcheck_minion( - self, ctxt, connection_info, minion_properties, - minion_connection_info): + self, ctxt, connection_info, minion_properties, minion_connection_info + ): ip = minion_connection_info.get("ip") port = minion_connection_info.get("port", 22) username = minion_connection_info.get("username", "root") @@ -430,11 +492,11 @@ def healthcheck_minion( client.close() def validate_osmorphing_minion_compatibility_for_transfer( - self, ctxt, connection_info, export_info, environment_options, - minion_properties): + self, ctxt, connection_info, export_info, environment_options, minion_properties + ): pass def get_additional_os_morphing_info( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, ctxt, connection_info, target_environment, instance_deployment_info + ): return {} diff --git a/coriolis/tests/integration/test_provider/osmorphing/__init__.py b/coriolis/tests/integration/test_provider/osmorphing/__init__.py index 13ddcd8f..7987f5a0 100644 --- a/coriolis/tests/integration/test_provider/osmorphing/__init__.py +++ b/coriolis/tests/integration/test_provider/osmorphing/__init__.py @@ -4,7 +4,6 @@ from coriolis.osmorphing import base from coriolis.tests.integration.test_provider.osmorphing import ubuntu - OS_MORPHERS: list[base.BaseLinuxOSMorphingTools] = [ ubuntu.TestUbuntuOSMorphingTools, ] diff --git a/coriolis/tests/integration/test_smoke.py b/coriolis/tests/integration/test_smoke.py index 313d3655..3fc721fa 100644 --- a/coriolis/tests/integration/test_smoke.py +++ b/coriolis/tests/integration/test_smoke.py @@ -50,7 +50,9 @@ def test_transfer_create(self): instances = ["smoke-instance"] transfer = self._create_transfer( - src.id, dst.id, instances=instances, + src.id, + dst.id, + instances=instances, destination_environment={"devices": ["foo"]}, ) diff --git a/coriolis/tests/integration/transfers/test_executions.py b/coriolis/tests/integration/transfers/test_executions.py index 2f754205..1b9db51c 100644 --- a/coriolis/tests/integration/transfers/test_executions.py +++ b/coriolis/tests/integration/transfers/test_executions.py @@ -11,7 +11,6 @@ class TransferExecutionsTests(base.ReplicaIntegrationTestBase): - def test_executions(self): # We didn't start the execution yet. executions = self._client.transfer_executions.list(self._transfer.id) @@ -20,10 +19,13 @@ def test_executions(self): # Start the execution. execution = self._client.transfer_executions.create( - self._transfer.id, shutdown_instances=False) + self._transfer.id, shutdown_instances=False + ) self.addCleanup( self._ignoreExc(self._client.transfer_executions.delete), - self._transfer.id, execution.id) + self._transfer.id, + execution.id, + ) self.assertExecutionCompleted(execution.id) executions = self._client.transfer_executions.list(self._transfer.id) @@ -31,13 +33,11 @@ def test_executions(self): self.assertIn(execution.id, ids) # Get the execution. - fetched = self._client.transfer_executions.get( - self._transfer.id, execution.id) + fetched = self._client.transfer_executions.get(self._transfer.id, execution.id) self.assertEqual(execution.id, fetched.id) # Delete the execution. - self._client.transfer_executions.delete( - self._transfer.id, execution.id) + self._client.transfer_executions.delete(self._transfer.id, execution.id) executions = self._client.transfer_executions.list(self._transfer.id) ids = [e.id for e in executions] @@ -46,10 +46,11 @@ def test_executions(self): def test_shutdown_instances(self): # shutdown_instances=True calls provider.shutdown_instance(). execution = self._client.transfer_executions.create( - self._transfer.id, shutdown_instances=True) + self._transfer.id, shutdown_instances=True + ) self.addCleanup( - self._client.transfer_executions.delete, - self._transfer.id, execution.id) + self._client.transfer_executions.delete, self._transfer.id, execution.id + ) self.assertExecutionCompleted(execution.id) @@ -60,8 +61,8 @@ def test_execution_auto_deploy(self): auto_deploy=True, ) self.addCleanup( - self._client.transfer_executions.delete, - self._transfer.id, execution.id) + self._client.transfer_executions.delete, self._transfer.id, execution.id + ) self.assertExecutionCompleted(execution.id) deployments = self._client.deployments.list() @@ -69,8 +70,7 @@ def test_execution_auto_deploy(self): d for d in deployments if d.transfer_id == self._transfer.id ] self.assertEqual(1, len(transfer_deployments)) - self.addCleanup( - self._client.deployments.delete, transfer_deployments[0].id) + self.addCleanup(self._client.deployments.delete, transfer_deployments[0].id) def test_cancel_running_execution(self): self._test_cancel_running_execution(False) @@ -91,18 +91,19 @@ def _test_cancel_running_execution(self, force): ) execution = self._client.transfer_executions.create( - self._transfer.id, shutdown_instances=False) + self._transfer.id, shutdown_instances=False + ) self.addCleanup( - self._client.transfer_executions.delete, - self._transfer.id, execution.id) + self._client.transfer_executions.delete, self._transfer.id, execution.id + ) # Wait until the execution is RUNNING before issuing the cancel. - self.wait_for_execution( - execution.id, 30, [constants.EXECUTION_STATUS_RUNNING]) + self.wait_for_execution(execution.id, 30, [constants.EXECUTION_STATUS_RUNNING]) # Cancel the execution. self._client.transfer_executions.cancel( - self._transfer.id, execution.id, force=force) + self._transfer.id, execution.id, force=force + ) final = self.wait_for_execution(execution.id) expected_statuses = [ @@ -113,11 +114,11 @@ def _test_cancel_running_execution(self, force): self.assertIn( final.status, expected_statuses, - "Expected a canceled/error status after cancel, got %s" - % final.status, + "Expected a canceled/error status after cancel, got %s" % final.status, ) class MinionPoolTransferExecutionsTests( - base.MinionPoolReplicaTestBase, TransferExecutionsTests): + base.MinionPoolReplicaTestBase, TransferExecutionsTests +): """Transfer executions that use a pre-allocated destination minion pool.""" diff --git a/coriolis/tests/integration/transfers/test_schedules.py b/coriolis/tests/integration/transfers/test_schedules.py index 6591e88f..61cd55f5 100644 --- a/coriolis/tests/integration/transfers/test_schedules.py +++ b/coriolis/tests/integration/transfers/test_schedules.py @@ -16,7 +16,6 @@ class _TransferScheduleTestBase(base.ReplicaIntegrationTestBase): - def _create_schedule(self, **overrides): defaults = { "transfer": self._transfer.id, @@ -30,20 +29,20 @@ def _create_schedule(self, **overrides): sched = self._client.transfer_schedules.create(**defaults) self.addCleanup( self._ignoreExc(self._client.transfer_schedules.delete), - self._transfer.id, sched.id) + self._transfer.id, + sched.id, + ) return sched class TransferScheduleBasicTests(_TransferScheduleTestBase): - def test_schedule_crud(self): # Create. sched = self._create_schedule() # Get. - fetched = self._client.transfer_schedules.get( - self._transfer.id, sched.id) + fetched = self._client.transfer_schedules.get(self._transfer.id, sched.id) self.assertEqual(sched.id, fetched.id) self.assertTrue(fetched.enabled) @@ -54,7 +53,8 @@ def test_schedule_crud(self): # Update. updated = self._client.transfer_schedules.update( - self._transfer.id, sched.id, {"enabled": False}) + self._transfer.id, sched.id, {"enabled": False} + ) self.assertFalse(updated.enabled) # Delete. @@ -66,7 +66,6 @@ def test_schedule_crud(self): class TransferScheduleTests(_TransferScheduleTestBase): - def test_scheduled_execution(self): """Verify that a schedule triggers a transfer execution. @@ -88,15 +87,14 @@ def test_scheduled_execution(self): deadline = time.monotonic() + 180 execution = None while time.monotonic() < deadline: - executions = self._client.transfer_executions.list( - self._transfer.id) + executions = self._client.transfer_executions.list(self._transfer.id) if executions: execution = executions[0] break time.sleep(2) self.assertIsNotNone( - execution, - "No transfer execution was triggered within 180s by the schedule") + execution, "No transfer execution was triggered within 180s by the schedule" + ) self.assertExecutionCompleted(execution.id) diff --git a/coriolis/tests/integration/transfers/test_transfer.py b/coriolis/tests/integration/transfers/test_transfer.py index 23e8625b..105c72cd 100644 --- a/coriolis/tests/integration/transfers/test_transfer.py +++ b/coriolis/tests/integration/transfers/test_transfer.py @@ -24,7 +24,8 @@ def test_transfer(self): # Update the transfer execution = self._client.transfers.update( - self._transfer.id, {"notes": "updated by integration test"}) + self._transfer.id, {"notes": "updated by integration test"} + ) self.assertExecutionCompleted(execution.id) updated = self._client.transfers.get(self._transfer.id) @@ -76,7 +77,8 @@ def test_incremental_replica_transfer(self): class MinionPoolTransferTest( - base.MinionPoolReplicaTestBase, ReplicaTransferIntegrationTest): + base.MinionPoolReplicaTestBase, ReplicaTransferIntegrationTest +): """Transfer execution that uses a pre-allocated destination minion pool.""" def test_transfer(self): diff --git a/coriolis/tests/integration/utils.py b/coriolis/tests/integration/utils.py index d954e53d..b071dec7 100644 --- a/coriolis/tests/integration/utils.py +++ b/coriolis/tests/integration/utils.py @@ -12,8 +12,8 @@ import tempfile import time -from oslo_log import log as logging import paramiko +from oslo_log import log as logging from coriolis import utils as coriolis_utils @@ -43,10 +43,7 @@ def _lsblk_disk_names() -> set: return set() data = json.loads(result.stdout) - return { - d["name"] for d in data.get("blockdevices", []) - if d["type"] == "disk" - } + return {d["name"] for d in data.get("blockdevices", []) if d["type"] == "disk"} def _poll_for_new_disks(before, count, timeout=_SETTLE_TIMEOUT): @@ -76,13 +73,15 @@ def init_scsi_debug(size_mb=16): knob gets its own independent backing store, so devices never share storage. """ - _run([ - "modprobe", - "scsi_debug", - "per_host_store=1", - "num_tgts=1", - f"dev_size_mb={size_mb}", - ]) + _run( + [ + "modprobe", + "scsi_debug", + "per_host_store=1", + "num_tgts=1", + f"dev_size_mb={size_mb}", + ] + ) def destroy_scsi_debug(): @@ -127,8 +126,13 @@ def write_test_pattern(device_path, chunk_size=4096): try: _run( - ["dd", "if=%s" % tmp_path, "of=%s" % device_path, - "bs=%d" % chunk_size, "conv=notrunc"], + [ + "dd", + "if=%s" % tmp_path, + "of=%s" % device_path, + "bs=%d" % chunk_size, + "conv=notrunc", + ], ) _run(["sync"]) finally: @@ -175,15 +179,17 @@ def wait_for_ssh(host, port, username, pkey_path, timeout=30): while time.monotonic() < deadline: try: client = coriolis_utils.connect_ssh( - host, port, username, pkey=pkey, connect_timeout=5) + host, port, username, pkey=pkey, connect_timeout=5 + ) client.close() return except (paramiko.SSHException, socket.error, OSError) as exc: last_exc = exc time.sleep(1) raise AssertionError( - "SSH %s@%s:%d not ready after %ds: %s" % ( - username, host, port, timeout, last_exc)) + "SSH %s@%s:%d not ready after %ds: %s" + % (username, host, port, timeout, last_exc) + ) # Docker utils @@ -204,8 +210,14 @@ def _run_container(image, name, extra_args=None): def run_container( - image, name, is_systemd=False, ssh_key=None, volumes=None, devices=None, - device_cgroup_rules=None, extra_args=None, + image, + name, + is_systemd=False, + ssh_key=None, + volumes=None, + devices=None, + device_cgroup_rules=None, + extra_args=None, ): """Start a detached Docker container and return its container ID. @@ -277,16 +289,20 @@ def get_container_ip(container_id): :returns: IP address string """ result = _run( - ["docker", "inspect", "--format", - "{{.NetworkSettings.IPAddress}}", - container_id]) + [ + "docker", + "inspect", + "--format", + "{{.NetworkSettings.IPAddress}}", + container_id, + ] + ) return result.stdout.decode().strip() def _get_container_pid(container_id): """Return the host PID of the init process of *container_id*.""" - result = _run( - ["docker", "inspect", "--format", "{{.State.Pid}}", container_id]) + result = _run(["docker", "inspect", "--format", "{{.State.Pid}}", container_id]) return int(result.stdout.decode().strip()) @@ -297,19 +313,38 @@ def hotplug_device_to_container(container_id, device_path): major = os.major(stat_result.st_rdev) minor = os.minor(stat_result.st_rdev) - _run([ - "nsenter", "--target", str(pid), "--mount", "--", - "mknod", device_path, "b", str(major), str(minor), - ]) + _run( + [ + "nsenter", + "--target", + str(pid), + "--mount", + "--", + "mknod", + device_path, + "b", + str(major), + str(minor), + ] + ) def unplug_device_from_container(container_id, device_path): """Remove a device node from *container_id*'s mount namespace.""" pid = _get_container_pid(container_id) - _run([ - "nsenter", "--target", str(pid), "--mount", "--", - "rm", "-f", device_path, - ], check=False) + _run( + [ + "nsenter", + "--target", + str(pid), + "--mount", + "--", + "rm", + "-f", + device_path, + ], + check=False, + ) # OS Morphing utils diff --git a/coriolis/tests/licensing/test_client.py b/coriolis/tests/licensing/test_client.py index 3cd0d0c6..af64932a 100644 --- a/coriolis/tests/licensing/test_client.py +++ b/coriolis/tests/licensing/test_client.py @@ -7,8 +7,7 @@ from coriolis import exception from coriolis.licensing import client as licensing_module -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class LicensingClientTestCase(test_base.CoriolisBaseTestCase): @@ -16,8 +15,7 @@ class LicensingClientTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(LicensingClientTestCase, self).setUp() - self.client = licensing_module.LicensingClient( - "https://10.7.2.3:37667/v1") + self.client = licensing_module.LicensingClient("https://10.7.2.3:37667/v1") self.resource = "licences" self.body = {'test_key': 'test_value'} self.reservation_id = "reservation_id" @@ -25,9 +23,14 @@ def setUp(self): licensing_module.CONF = mock.Mock() # Helper function to setup a mock response - def setup_mock_response(self, mock_request, ok=True, - status_code=200, json_return_value=None, - side_effect=None): + def setup_mock_response( + self, + mock_request, + ok=True, + status_code=200, + json_return_value=None, + side_effect=None, + ): mock_resp = mock.MagicMock() mock_resp.ok = ok mock_resp.status_code = status_code @@ -42,8 +45,9 @@ def setup_mock_response(self, mock_request, ok=True, @mock.patch.object(licensing_module.LicensingClient, 'get_appliances') @mock.patch.object(licensing_module.LicensingClient, 'create_appliance') @mock.patch.object(licensing_module.LicensingClient, 'get_licence_status') - def test_from_env(self, mock_get_licence_status, mock_create_appliance, - mock_get_appliances): + def test_from_env( + self, mock_get_licence_status, mock_create_appliance, mock_get_appliances + ): os.environ["LICENSING_SERVER_BASE_URL"] = "https://10.7.2.3:37667/v1" mock_get_appliances.return_value = [{"id": "appliance_id1"}] mock_create_appliance.return_value = {"id": "appliance_id2"} @@ -57,8 +61,7 @@ def test_from_env(self, mock_get_licence_status, mock_create_appliance, def test_from_env_no_base_url(self): os.environ["LICENSING_SERVER_BASE_URL"] = "" - with self.assertLogs('coriolis.licensing.client', - level=logging.WARN): + with self.assertLogs('coriolis.licensing.client', level=logging.WARN): result = licensing_module.LicensingClient.from_env() self.assertIsNone(result) @@ -66,18 +69,20 @@ def test_from_env_no_base_url(self): def test_from_env_multiple_appliances(self, mock_get_appliances): os.environ["LICENSING_SERVER_BASE_URL"] = "https://10.7.2.3:37667/v1" mock_get_appliances.return_value = [ - {"id": "appliance_id1"}, {"id": "appliance_id2"} + {"id": "appliance_id1"}, + {"id": "appliance_id2"}, ] - self.assertRaises(exception.CoriolisException, - licensing_module.LicensingClient.from_env) + self.assertRaises( + exception.CoriolisException, licensing_module.LicensingClient.from_env + ) @mock.patch.object(licensing_module.LicensingClient, 'get_appliances') @mock.patch.object(licensing_module.LicensingClient, 'create_appliance') @mock.patch.object(licensing_module.LicensingClient, 'get_licence_status') - def test_from_env_no_appliances(self, mock_get_licence_status, - mock_create_appliance, - mock_get_appliances): + def test_from_env_no_appliances( + self, mock_get_licence_status, mock_create_appliance, mock_get_appliances + ): os.environ["LICENSING_SERVER_BASE_URL"] = "https://10.7.2.3:37667/v1" mock_get_appliances.return_value = [] mock_create_appliance.return_value = {"id": "new_appliance_id"} @@ -87,9 +92,7 @@ def test_from_env_no_appliances(self, mock_get_licence_status, mock_get_appliances.assert_called_once() mock_create_appliance.assert_called_once() mock_get_licence_status.assert_called_once() - self.assertEqual( - result._appliance_id, mock_create_appliance.return_value["id"] - ) + self.assertEqual(result._appliance_id, mock_create_appliance.return_value["id"]) def test__get_url_for_resource(self): result = self.client._get_url_for_resource(self.resource) @@ -98,8 +101,7 @@ def test__get_url_for_resource(self): def test__get_url_for_appliance_resource(self): result = self.client._get_url_for_appliance_resource(self.resource) self.assertEqual( - result, - "https://10.7.2.3:37667/v1/appliances/appliance_id/licences" + result, "https://10.7.2.3:37667/v1/appliances/appliance_id/licences" ) def test_raise_response_error(self): @@ -113,12 +115,11 @@ def test_raise_response_error(self): def test_raise_response_error_conflict(self): mock_response = mock.Mock() mock_response.json.return_value = { - 'error': {'code': 409, 'message': 'test_message'}} + 'error': {'code': 409, 'message': 'test_message'} + } self.assertRaises( - exception.Conflict, - self.client._raise_response_error, - mock_response + exception.Conflict, self.client._raise_response_error, mock_response ) mock_response.raise_for_status.assert_not_called() @@ -140,24 +141,22 @@ def test_raise_response_error_json_exception(self): mock_response.raise_for_status.assert_called_once() - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch('requests.post') - def test__do_req( - self, - mock_post, - mock_raise_response_error - ): + def test__do_req(self, mock_post, mock_raise_response_error): mock_response = self.setup_mock_response( - mock_post, json_return_value={'test_key': 'test_value'}) + mock_post, json_return_value={'test_key': 'test_value'} + ) original_do_req = testutils.get_wrapped_function(self.client._do_req) with self.assertLogs('coriolis.licensing.client', level=logging.DEBUG): result = original_do_req( - self.client, 'POST', self.resource, + self.client, + 'POST', + self.resource, body=self.body, - response_key='test_key' + response_key='test_key', ) self.assertEqual(result, 'test_value') @@ -165,7 +164,7 @@ def test__do_req( self.client._get_url_for_appliance_resource(self.resource), verify=True, timeout=licensing_module.CONF.default_requests_timeout, - data=licensing_module.json.dumps(self.body) + data=licensing_module.json.dumps(self.body), ) mock_response.json.assert_called_once() mock_raise_response_error.assert_not_called() @@ -174,75 +173,60 @@ def test__do_req_invalid_method(self): original_do_req = testutils.get_wrapped_function(self.client._do_req) self.assertRaises( - ValueError, original_do_req, self.client, 'INVALID_METHOD', - self.resource + ValueError, original_do_req, self.client, 'INVALID_METHOD', self.resource ) - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch('requests.get') - def test__do_req_raw_response( - self, - mock_get, - mock_raise_response_error - ): + def test__do_req_raw_response(self, mock_get, mock_raise_response_error): mock_response = self.setup_mock_response(mock_get) original_do_req = testutils.get_wrapped_function(self.client._do_req) - result = original_do_req( - self.client, 'GET', self.resource, raw_response=True - ) + result = original_do_req(self.client, 'GET', self.resource, raw_response=True) self.assertEqual(result, mock_get.return_value) mock_get.assert_called_once_with( self.client._get_url_for_appliance_resource(self.resource), verify=True, - timeout=licensing_module.CONF.default_requests_timeout + timeout=licensing_module.CONF.default_requests_timeout, ) mock_response.json.assert_not_called() mock_raise_response_error.assert_not_called() - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch('requests.get') - def test__do_req_response_not_ok( - self, - mock_get, - mock_raise_response_error - ): + def test__do_req_response_not_ok(self, mock_get, mock_raise_response_error): mock_response = self.setup_mock_response( - mock_get, ok=False, status_code=409, - json_return_value={ - 'error': {'code': 409, 'message': 'test_message'}} + mock_get, + ok=False, + status_code=409, + json_return_value={'error': {'code': 409, 'message': 'test_message'}}, ) original_do_req = testutils.get_wrapped_function(self.client._do_req) result = original_do_req(self.client, 'GET', self.resource) - self.assertEqual( - result, - {'error': {'code': 409, 'message': 'test_message'}} - ) + self.assertEqual(result, {'error': {'code': 409, 'message': 'test_message'}}) mock_raise_response_error.assert_called_once_with(mock_response) - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch('requests.get') - def test__do_req_no_response_key( - self, - mock_get, - mock_raise_response_error - ): + def test__do_req_no_response_key(self, mock_get, mock_raise_response_error): mock_response = self.setup_mock_response( - mock_get, json_return_value={'test_key': 'test_value'}) + mock_get, json_return_value={'test_key': 'test_value'} + ) original_do_req = testutils.get_wrapped_function(self.client._do_req) self.assertRaises( - ValueError, original_do_req, self.client, 'GET', self.resource, - response_key='nonexistent_key' + ValueError, + original_do_req, + self.client, + 'GET', + self.resource, + response_key='nonexistent_key', ) mock_response.json.assert_called_once() @@ -263,8 +247,11 @@ def test__post(self, mock_do_req): self.assertEqual(result, mock_do_req.return_value) mock_do_req.assert_called_once_with( - 'POST', self.resource, body=self.body, - response_key=None, appliance_scoped=True + 'POST', + self.resource, + body=self.body, + response_key=None, + appliance_scoped=True, ) @mock.patch.object(licensing_module.LicensingClient, '_do_req') @@ -273,8 +260,11 @@ def test__put(self, mock_do_req): self.assertEqual(result, mock_do_req.return_value) mock_do_req.assert_called_once_with( - 'PUT', self.resource, body=self.body, - response_key=None, appliance_scoped=True + 'PUT', + self.resource, + body=self.body, + response_key=None, + appliance_scoped=True, ) @mock.patch.object(licensing_module.LicensingClient, '_do_req') @@ -283,8 +273,11 @@ def test__delete(self, mock_do_req): self.assertEqual(result, mock_do_req.return_value) mock_do_req.assert_called_once_with( - 'DELETE', self.resource, body=self.body, - response_key=None, appliance_scoped=True + 'DELETE', + self.resource, + body=self.body, + response_key=None, + appliance_scoped=True, ) @mock.patch.object(licensing_module.LicensingClient, '_get') @@ -304,7 +297,7 @@ def test_get_appliance(self, mock_get): mock_get.assert_called_once_with( f'/appliances/{self.client._appliance_id}', response_key='appliance', - appliance_scoped=False + appliance_scoped=False, ) @mock.patch.object(licensing_module.LicensingClient, '_post') @@ -313,8 +306,7 @@ def test_create_appliance(self, mock_post): self.assertEqual(result, mock_post.return_value) mock_post.assert_called_once_with( - '/appliances', body=None, response_key='appliance', - appliance_scoped=False + '/appliances', body=None, response_key='appliance', appliance_scoped=False ) @mock.patch.object(licensing_module.LicensingClient, '_get') @@ -322,52 +314,44 @@ def test_get_licence_status(self, mock_get): result = self.client.get_licence_status() self.assertEqual(result, mock_get.return_value) - mock_get.assert_called_once_with( - '/status', 'appliance_licence_status' - ) + mock_get.assert_called_once_with('/status', 'appliance_licence_status') @mock.patch.object(licensing_module.LicensingClient, '_get') def test_get_licences(self, mock_get): result = self.client.get_licences() self.assertEqual(result, mock_get.return_value) - mock_get.assert_called_once_with( - '/licences', response_key='licences' - ) + mock_get.assert_called_once_with('/licences', response_key='licences') @mock.patch.object(licensing_module.LicensingClient, '_post') def test_add_licence(self, mock_post): - licence_data = ( - "-----BEGIN CERTIFICATE-----\n\ + licence_data = "-----BEGIN CERTIFICATE-----\n\ MIIDXTCCAkWgAwIBAgIJAJC1HiE2u90vMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV\n\ -----END CERTIFICATE-----" - ) result = self.client.add_licence(licence_data) self.assertEqual(result, mock_post.return_value) - mock_post.assert_called_once_with( - '/licences', licence_data - ) + mock_post.assert_called_once_with('/licences', licence_data) @mock.patch.object(licensing_module.LicensingClient, '_post') def test_add_reservation(self, mock_post): result = self.client.add_reservation( - licensing_module.RESERVATION_TYPE_REPLICA, 2) + licensing_module.RESERVATION_TYPE_REPLICA, 2 + ) self.assertEqual(result, mock_post.return_value) mock_post.assert_called_once_with( '/reservations', - { - 'type': licensing_module.RESERVATION_TYPE_REPLICA, - 'count': 2 - }, - response_key='reservation' + {'type': licensing_module.RESERVATION_TYPE_REPLICA, 'count': 2}, + response_key='reservation', ) def test_add_reservation_invalid_type(self): self.assertRaises( - ValueError, self.client.add_reservation, - 'invalid_reservation_type', mock.sentinel.num_vms + ValueError, + self.client.add_reservation, + 'invalid_reservation_type', + mock.sentinel.num_vms, ) @mock.patch.object(licensing_module.LicensingClient, 'add_reservation') @@ -393,9 +377,7 @@ def test_get_reservations(self, mock_get): result = self.client.get_reservations() self.assertEqual(result, mock_get.return_value) - mock_get.assert_called_once_with( - '/reservations', response_key='reservations' - ) + mock_get.assert_called_once_with('/reservations', response_key='reservations') @mock.patch.object(licensing_module.LicensingClient, '_get') def test_get_reservation(self, mock_get): @@ -413,85 +395,62 @@ def test_check_refresh_reservation(self, mock_post): mock_post.assert_called_once_with( f'/reservations/{self.reservation_id}/refresh', - None, response_key='reservation' + None, + response_key='reservation', ) - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch.object(licensing_module.LicensingClient, '_do_req') def test_delete_reservation_404_status_code( - self, - mock_do_req, - mock_raise_response_error + self, mock_do_req, mock_raise_response_error ): self.setup_mock_response(mock_do_req, ok=False, status_code=404) with self.assertLogs('coriolis.licensing.client', level=logging.WARN): - self.client.delete_reservation( - self.reservation_id, raise_on_404=False) + self.client.delete_reservation(self.reservation_id, raise_on_404=False) mock_do_req.assert_called_once_with( - 'delete', '/reservations/%s' % self.reservation_id, - raw_response=True + 'delete', '/reservations/%s' % self.reservation_id, raw_response=True ) mock_raise_response_error.assert_not_called() - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch.object(licensing_module.LicensingClient, '_do_req') def test_delete_reservation_404_status_code_raise( - self, - mock_do_req, - mock_raise_response_error + self, mock_do_req, mock_raise_response_error ): - mock_response = self.setup_mock_response( - mock_do_req, ok=False, status_code=404) + mock_response = self.setup_mock_response(mock_do_req, ok=False, status_code=404) with self.assertLogs('coriolis.licensing.client', level=logging.WARN): - self.client.delete_reservation( - self.reservation_id, raise_on_404=True) + self.client.delete_reservation(self.reservation_id, raise_on_404=True) mock_do_req.assert_called_once_with( - 'delete', '/reservations/%s' % self.reservation_id, - raw_response=True + 'delete', '/reservations/%s' % self.reservation_id, raw_response=True ) mock_raise_response_error.assert_called_once_with(mock_response) - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch.object(licensing_module.LicensingClient, '_do_req') def test_delete_reservation_status_code_raise( - self, - mock_do_req, - mock_raise_response_error + self, mock_do_req, mock_raise_response_error ): - mock_response = self.setup_mock_response( - mock_do_req, ok=False, status_code=400) + mock_response = self.setup_mock_response(mock_do_req, ok=False, status_code=400) - self.client.delete_reservation( - self.reservation_id, raise_on_404=True) + self.client.delete_reservation(self.reservation_id, raise_on_404=True) mock_do_req.assert_called_once_with( - 'delete', '/reservations/%s' % self.reservation_id, - raw_response=True + 'delete', '/reservations/%s' % self.reservation_id, raw_response=True ) mock_raise_response_error.assert_called_once_with(mock_response) - @mock.patch.object(licensing_module.LicensingClient, - '_raise_response_error') + @mock.patch.object(licensing_module.LicensingClient, '_raise_response_error') @mock.patch.object(licensing_module.LicensingClient, '_do_req') - def test_delete_reservation( - self, - mock_do_req, - mock_raise_response_error - ): - mock_response = self.setup_mock_response( - mock_do_req, ok=False, status_code=200) + def test_delete_reservation(self, mock_do_req, mock_raise_response_error): + mock_response = self.setup_mock_response(mock_do_req, ok=False, status_code=200) self.client.delete_reservation(self.reservation_id, raise_on_404=True) mock_do_req.assert_called_once_with( - 'delete', '/reservations/%s' % self.reservation_id, - raw_response=True + 'delete', '/reservations/%s' % self.reservation_id, raw_response=True ) mock_raise_response_error.assert_called_once_with(mock_response) diff --git a/coriolis/tests/minion_manager/rpc/test_client.py b/coriolis/tests/minion_manager/rpc/test_client.py index ac7c9159..8c2fac6b 100644 --- a/coriolis/tests/minion_manager/rpc/test_client.py +++ b/coriolis/tests/minion_manager/rpc/test_client.py @@ -6,7 +6,6 @@ from coriolis.minion_manager.rpc import client from coriolis.tests import test_base - CREATE_MINION_POOL_ARGS = { "name": "pool_name", "endpoint_id": "endpoint_id", @@ -16,32 +15,29 @@ "minimum_minions": 1, "maximum_minions": 2, "minion_max_idle_time": 3, - "minion_retention_strategy": "strategy" + "minion_retention_strategy": "strategy", } UPDATE_MINION_POOL_PROGRESS_UPDATE_ARGS = { "minion_pool_id": "pool_id", "progress_update_index": 1, - "new_current_step": 2 + "new_current_step": 2, } ADD_MINION_POOL_EVENT_ARGS = { "minion_pool_id": "pool_id", "level": "INFO", - "message": "test_message" + "message": "test_message", } ENDPOINT_OPT_ARGS = { "endpoint_id": "endpoint_id", "env": "env", - "option_names": ["opt1", "opt2"] + "option_names": ["opt1", "opt2"], } -POOL_VALIDATION_ARGS = { - "endpoint_id": "endpoint_id", - "pool_environment": "pool_env" -} +POOL_VALIDATION_ARGS = {"endpoint_id": "endpoint_id", "pool_environment": "pool_env"} class MinionManagerClientTestCase(test_base.CoriolisRPCClientTestCase): @@ -64,7 +60,8 @@ def test__init__(self, mock_target, mock_conf): result = client.MinionManagerClient() mock_target.assert_called_once_with( - topic='coriolis_minion_manager', version=client.VERSION) + topic='coriolis_minion_manager', version=client.VERSION + ) self.assertEqual(result._target, mock_target.return_value) self.assertEqual(result._timeout, expected_timeout) @@ -81,9 +78,7 @@ def test_add_minion_pool_progress_update(self): "total_steps": 2, } with mock.patch.object(self.client, '_cast') as mock_cast: - self.client.add_minion_pool_progress_update( - self.ctxt, **args - ) + self.client.add_minion_pool_progress_update(self.ctxt, **args) mock_cast.assert_called_once_with( self.ctxt, 'add_minion_pool_progress_update', **args ) @@ -100,16 +95,18 @@ def test_update_minion_pool_progress_update(self): args = { **UPDATE_MINION_POOL_PROGRESS_UPDATE_ARGS, "new_total_steps": None, - "new_message": None + "new_message": None, } self._test( - self.client.update_minion_pool_progress_update, args, + self.client.update_minion_pool_progress_update, + args, rpc_op='_cast', ) def test_add_minion_pool_event(self): self._test( - self.client.add_minion_pool_event, ADD_MINION_POOL_EVENT_ARGS, + self.client.add_minion_pool_event, + ADD_MINION_POOL_EVENT_ARGS, rpc_op='_cast', ) @@ -118,132 +115,114 @@ def test_get_diagnostics(self): def test_validate_minion_pool_selections_for_action(self): args = {"action": "test_action"} - self._test( - self.client.validate_minion_pool_selections_for_action, args - ) + self._test(self.client.validate_minion_pool_selections_for_action, args) def test_allocate_minion_machines_for_transfer(self): args = {"transfer": "test_transfer"} self._test( - self.client.allocate_minion_machines_for_transfer, args, + self.client.allocate_minion_machines_for_transfer, + args, rpc_op='_cast', - server_fun_name='allocate_minion_machines_for_transfer' + server_fun_name='allocate_minion_machines_for_transfer', ) def test_allocate_minion_machines_for_deployment(self): args = { "deployment": "test_deployment", "include_transfer_minions": True, - "include_osmorphing_minions": True + "include_osmorphing_minions": True, } self._test( - self.client.allocate_minion_machines_for_deployment, args, + self.client.allocate_minion_machines_for_deployment, + args, rpc_op='_cast', - server_fun_name='allocate_minion_machines_for_deployment' + server_fun_name='allocate_minion_machines_for_deployment', ) def test_deallocate_minion_machine(self): args = {"minion_machine_id": "test_id"} self._test( - self.client.deallocate_minion_machine, args, + self.client.deallocate_minion_machine, + args, rpc_op='_cast', ) def test_deallocate_minion_machines_for_action(self): args = {"action_id": "test_id"} self._test( - self.client.deallocate_minion_machines_for_action, args, + self.client.deallocate_minion_machines_for_action, + args, rpc_op='_cast', ) def test_create_minion_pool(self): - args = { - **CREATE_MINION_POOL_ARGS, - "notes": None, - "skip_allocation": False - } + args = {**CREATE_MINION_POOL_ARGS, "notes": None, "skip_allocation": False} self._test(self.client.create_minion_pool, args) def test_set_up_shared_minion_pool_resources(self): - args = { - "minion_pool_id": self.minion_pool_id - } + args = {"minion_pool_id": self.minion_pool_id} self._test( - self.client.set_up_shared_minion_pool_resources, args, + self.client.set_up_shared_minion_pool_resources, + args, ) def test_tear_down_shared_minion_pool_resources(self): - args = { - "minion_pool_id": self.minion_pool_id, - "force": False - } + args = {"minion_pool_id": self.minion_pool_id, "force": False} self._test( - self.client.tear_down_shared_minion_pool_resources, args, + self.client.tear_down_shared_minion_pool_resources, + args, ) def test_allocate_minion_pool(self): - args = { - "minion_pool_id": self.minion_pool_id - } + args = {"minion_pool_id": self.minion_pool_id} self._test(self.client.allocate_minion_pool, args) def test_refresh_minion_pool(self): - args = { - "minion_pool_id": self.minion_pool_id - } + args = {"minion_pool_id": self.minion_pool_id} self._test(self.client.refresh_minion_pool, args) def test_deallocate_minion_pool(self): - args = { - "minion_pool_id": self.minion_pool_id, - "force": False - } + args = {"minion_pool_id": self.minion_pool_id, "force": False} self._test(self.client.deallocate_minion_pool, args) def test_get_minion_pools(self): self._test(self.client.get_minion_pools, {}) def test_get_minion_pool(self): - args = { - "minion_pool_id": self.minion_pool_id - } + args = {"minion_pool_id": self.minion_pool_id} self._test(self.client.get_minion_pool, args) def test_update_minion_pool(self): args = { "minion_pool_id": self.minion_pool_id, - "updated_values": {"opt1": "value1"} + "updated_values": {"opt1": "value1"}, } self._test(self.client.update_minion_pool, args) def test_delete_minion_pool(self): - args = { - "minion_pool_id": self.minion_pool_id - } + args = {"minion_pool_id": self.minion_pool_id} self._test(self.client.delete_minion_pool, args) def test_get_endpoint_source_minion_pool_options(self): self._test( - self.client.get_endpoint_source_minion_pool_options, - ENDPOINT_OPT_ARGS + self.client.get_endpoint_source_minion_pool_options, ENDPOINT_OPT_ARGS ) def test_get_endpoint_destination_minion_pool_options(self): self._test( - self.client.get_endpoint_destination_minion_pool_options, - ENDPOINT_OPT_ARGS + self.client.get_endpoint_destination_minion_pool_options, ENDPOINT_OPT_ARGS ) def test_validate_endpoint_source_minion_pool_options(self): self._test( self.client.validate_endpoint_source_minion_pool_options, - POOL_VALIDATION_ARGS + POOL_VALIDATION_ARGS, ) def test_validate_endpoint_destination_minion_pool_options(self): self._test( self.client.validate_endpoint_destination_minion_pool_options, - POOL_VALIDATION_ARGS + POOL_VALIDATION_ARGS, ) @@ -256,8 +235,7 @@ def setUp(self): self.pool_id = mock.sentinel.pool_id self.message = mock.sentinel.message - self.client = client.MinionManagerPoolRpcEventHandler( - self.ctxt, self.pool_id) + self.client = client.MinionManagerPoolRpcEventHandler(self.ctxt, self.pool_id) @mock.patch.object(client, 'MinionManagerClient') def test__rpc_minion_manager_client(self, mock_client): @@ -278,49 +256,50 @@ def test__rpc_minion_manager_client_instantiated(self, mock_client): def test_get_progress_update_identifier(self): progress_update = {"index": 2} - result = client.MinionManagerPoolRpcEventHandler.\ - get_progress_update_identifier(progress_update) + result = client.MinionManagerPoolRpcEventHandler.get_progress_update_identifier( + progress_update + ) self.assertEqual(result, progress_update['index']) - @mock.patch.object( - client.MinionManagerClient, 'add_minion_pool_progress_update' - ) + @mock.patch.object(client.MinionManagerClient, 'add_minion_pool_progress_update') def test_add_progress_update(self, mock_add_minion_pool_progress_update): result = self.client.add_progress_update( self.message, initial_step=1, total_steps=3, return_event=False ) mock_add_minion_pool_progress_update.assert_called_once_with( - self.ctxt, self.pool_id, self.message, initial_step=1, - total_steps=3, return_event=False - ) - self.assertEqual( - result, mock_add_minion_pool_progress_update.return_value + self.ctxt, + self.pool_id, + self.message, + initial_step=1, + total_steps=3, + return_event=False, ) + self.assertEqual(result, mock_add_minion_pool_progress_update.return_value) - @mock.patch.object( - client.MinionManagerClient, 'update_minion_pool_progress_update' - ) - def test_update_progress_update(self, - mock_update_minion_pool_progress_update): + @mock.patch.object(client.MinionManagerClient, 'update_minion_pool_progress_update') + def test_update_progress_update(self, mock_update_minion_pool_progress_update): self.client.update_progress_update( mock.sentinel.update_identifier, mock.sentinel.new_current_step ) mock_update_minion_pool_progress_update.assert_called_once_with( - self.ctxt, self.pool_id, mock.sentinel.update_identifier, - mock.sentinel.new_current_step, new_total_steps=None, - new_message=None + self.ctxt, + self.pool_id, + mock.sentinel.update_identifier, + mock.sentinel.new_current_step, + new_total_steps=None, + new_message=None, ) @mock.patch.object(client.MinionManagerClient, 'add_minion_pool_event') def test_add_event(self, mock_add_minion_pool_event): result = self.client.add_event( - self.message, level=client.constants.TASK_EVENT_INFO) + self.message, level=client.constants.TASK_EVENT_INFO + ) mock_add_minion_pool_event.assert_called_once_with( - self.ctxt, self.pool_id, client.constants.TASK_EVENT_INFO, - self.message + self.ctxt, self.pool_id, client.constants.TASK_EVENT_INFO, self.message ) self.assertIsNone(result) diff --git a/coriolis/tests/minion_manager/rpc/test_server.py b/coriolis/tests/minion_manager/rpc/test_server.py index 7e505451..1b820410 100644 --- a/coriolis/tests/minion_manager/rpc/test_server.py +++ b/coriolis/tests/minion_manager/rpc/test_server.py @@ -1,18 +1,16 @@ # Copyright 2023 Cloudbase Solutions Srl # All Rights Reserved. +import datetime +import uuid from unittest import mock -import datetime import ddt -import uuid -from coriolis import constants +from coriolis import constants, exception from coriolis.db import api as db_api -from coriolis import exception from coriolis.minion_manager.rpc import server -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils @ddt.ddt @@ -24,19 +22,17 @@ def setUp(self, *_, **__): self.server = server.MinionManagerServerEndpoint() @mock.patch.object( - server.MinionManagerServerEndpoint, - '_check_keys_for_action_dict') - @mock.patch.object(db_api, "get_minion_pools") - @ddt.file_data( - "data/validate_minion_pool_selections_for_action_config.yaml" + server.MinionManagerServerEndpoint, '_check_keys_for_action_dict' ) + @mock.patch.object(db_api, "get_minion_pools") + @ddt.file_data("data/validate_minion_pool_selections_for_action_config.yaml") @ddt.unpack def test_validate_minion_pool_selections_for_action( - self, - mock_get_minion_pools, - mock_check_keys_for_action_dict, - config, - expected_exception, + self, + mock_get_minion_pools, + mock_check_keys_for_action_dict, + config, + expected_exception, ): action = config.get("action") minion_pools = config.get("minion_pools", []) @@ -44,7 +40,8 @@ def test_validate_minion_pool_selections_for_action( mock_get_minion_pools.return_value = [ mock.MagicMock( **pool, - ) for pool in minion_pools + ) + for pool in minion_pools ] if expected_exception: @@ -63,35 +60,31 @@ def test_validate_minion_pool_selections_for_action( ) mock_check_keys_for_action_dict.assert_called_once_with( - action, - mock.ANY, - operation="minion pool selection validation") + action, mock.ANY, operation="minion pool selection validation" + ) mock_get_minion_pools.assert_called_once_with( mock.sentinel.context, include_machines=False, include_events=False, include_progress_updates=False, - to_dict=False) + to_dict=False, + ) @mock.patch.object(uuid, "uuid4", return_value="new_machine") - @mock.patch.object( - server.MinionManagerServerEndpoint, - "_add_minion_pool_event") + @mock.patch.object(server.MinionManagerServerEndpoint, "_add_minion_pool_event") @mock.patch.object(db_api, "add_minion_machine") @mock.patch.object(db_api, "set_minion_machines_allocation_statuses") - @ddt.file_data( - "data/make_minion_machine_allocation_subflow_for_action.yaml" - ) + @ddt.file_data("data/make_minion_machine_allocation_subflow_for_action.yaml") @ddt.unpack def test_make_minion_machine_allocation_subflow_for_action( - self, - mock_set_minion_machines_allocation_statuses, - mock_add_minion_machine, - mock_add_minion_pool_event, - mock_uuid4, - config, - expect + self, + mock_set_minion_machines_allocation_statuses, + mock_add_minion_machine, + mock_add_minion_pool_event, + mock_uuid4, + config, + expect, ): expected_exception = expect.get("exception") expected_result = expect.get("result") @@ -112,12 +105,11 @@ def test_make_minion_machine_allocation_subflow_for_action( self.assertRaises( exception_type, self.server._make_minion_machine_allocation_subflow_for_action, - *args) + *args, + ) return - result = self.server\ - ._make_minion_machine_allocation_subflow_for_action( - *args) + result = self.server._make_minion_machine_allocation_subflow_for_action(*args) mappings = expected_result.get("mappings") exptected_flow_tasks = expected_result.get("flow_allocations", []) @@ -125,21 +117,19 @@ def test_make_minion_machine_allocation_subflow_for_action( num_new_machines = list(mappings.values()).count("new_machine") # db_api.add_minion_machine is called once for each new machine - self.assertEqual( - num_new_machines, - mock_add_minion_machine.call_count) + self.assertEqual(num_new_machines, mock_add_minion_machine.call_count) num_non_new_machines = len(mappings) - num_new_machines if num_non_new_machines: # db_api.set_minion_machines_allocation_statuses is called once # with the non-new machines - mock_set_minion_machines_allocation_statuses\ - .assert_called_once_with( - mock.sentinel.context, - list(mappings.values())[:num_non_new_machines], - mock.sentinel.action_id, - constants.MINION_MACHINE_STATUS_RESERVED, - refresh_allocation_time=True) + mock_set_minion_machines_allocation_statuses.assert_called_once_with( + mock.sentinel.context, + list(mappings.values())[:num_non_new_machines], + mock.sentinel.action_id, + constants.MINION_MACHINE_STATUS_RESERVED, + refresh_allocation_time=True, + ) # _add_minion_pool_event is called once if there're new machines, # twice if there's both new and non-new machines @@ -148,9 +138,7 @@ def test_make_minion_machine_allocation_subflow_for_action( else: add_event_count = 1 - self.assertEqual( - add_event_count, - mock_add_minion_pool_event.call_count) + self.assertEqual(add_event_count, mock_add_minion_pool_event.call_count) add_event_args = [ mock.sentinel.context, minion_pool.id, @@ -158,51 +146,49 @@ def test_make_minion_machine_allocation_subflow_for_action( mock.ANY, ] if add_event_count == 1: - mock_add_minion_pool_event.assert_called_once_with( - *add_event_args) + mock_add_minion_pool_event.assert_called_once_with(*add_event_args) else: - mock_add_minion_pool_event.assert_has_calls([ - mock.call(*add_event_args), - mock.call(*add_event_args), - ]) + mock_add_minion_pool_event.assert_has_calls( + [ + mock.call(*add_event_args), + mock.call(*add_event_args), + ] + ) - flow_allocations = [ - node.name for node, _ in result.get("flow").iter_nodes()] + flow_allocations = [node.name for node, _ in result.get("flow").iter_nodes()] - self.assertEqual( - exptected_flow_tasks, - flow_allocations) + self.assertEqual(exptected_flow_tasks, flow_allocations) self.assertEqual( - mappings, - result.get("action_instance_minion_allocation_mappings")) + mappings, result.get("action_instance_minion_allocation_mappings") + ) @mock.patch.object(db_api, "delete_minion_machine") @mock.patch.object(uuid, "uuid4", return_value="new_machine") - @mock.patch.object( - server.MinionManagerServerEndpoint, - "_add_minion_pool_event") + @mock.patch.object(server.MinionManagerServerEndpoint, "_add_minion_pool_event") @mock.patch.object(db_api, "add_minion_machine") @mock.patch.object(db_api, "set_minion_machines_allocation_statuses") def test_make_minion_machine_allocation_subflow_for_action_delete( - self, - mock_set_minion_machines_allocation_statuses, - mock_add_minion_machine, - mock_add_minion_pool_event, - mock_uuid4, - mock_delete_minion_machine, + self, + mock_set_minion_machines_allocation_statuses, + mock_add_minion_machine, + mock_add_minion_pool_event, + mock_uuid4, + mock_delete_minion_machine, ): # This test is a special case of the test above, where the added # minion machines are deleted when there's an exception trying to add # the last one. - minion_pool = testutils.DictToObject({ - "id": "minion_pool_1", - "maximum_minions": 5, - "minion_machines": [ - {"id": "machine_1", "allocation_status": "AVAILABLE"}, - ] - }) + minion_pool = testutils.DictToObject( + { + "id": "minion_pool_1", + "maximum_minions": 5, + "minion_machines": [ + {"id": "machine_1", "allocation_status": "AVAILABLE"}, + ], + } + ) action_instances = { "instance_1": {"name": "Instance 1"}, "instance_2": {"name": "Instance 2"}, @@ -212,7 +198,10 @@ def test_make_minion_machine_allocation_subflow_for_action_delete( # two machines are added, but the third one fails mock_add_minion_machine.side_effect = [ - mock.Mock(), mock.Mock(), exception.CoriolisException] + mock.Mock(), + mock.Mock(), + exception.CoriolisException, + ] self.assertRaises( exception.CoriolisException, @@ -221,7 +210,7 @@ def test_make_minion_machine_allocation_subflow_for_action_delete( minion_pool, mock.sentinel.action_id, action_instances, - mock.sentinel.subflow_name + mock.sentinel.subflow_name, ) # two machines are deleted @@ -235,40 +224,36 @@ def test_make_minion_machine_allocation_subflow_for_action_delete( ["machine_1"], mock.sentinel.action_id, constants.MINION_MACHINE_STATUS_RESERVED, - refresh_allocation_time=True), + refresh_allocation_time=True, + ), mock.call( mock.sentinel.context, ["machine_1"], None, constants.MINION_MACHINE_STATUS_AVAILABLE, - refresh_allocation_time=False), - ]) + refresh_allocation_time=False, + ), + ] + ) @mock.patch.object(db_api, "set_minion_machine_allocation_status") - @mock.patch.object( - server.MinionManagerServerEndpoint, - "_add_minion_pool_event") - @mock.patch.object( - server.MinionManagerServerEndpoint, - "_get_minion_pool") - @ddt.file_data( - "data/get_minion_pool_refresh_flow.yaml" - ) + @mock.patch.object(server.MinionManagerServerEndpoint, "_add_minion_pool_event") + @mock.patch.object(server.MinionManagerServerEndpoint, "_get_minion_pool") + @ddt.file_data("data/get_minion_pool_refresh_flow.yaml") @ddt.unpack def test_get_minion_pool_refresh_flow( - self, - mock_get_minion_pool, - mock_add_minion_pool_event, - mock_set_minion_machine_allocation_status, - config, - expect, + self, + mock_get_minion_pool, + mock_add_minion_pool_event, + mock_set_minion_machine_allocation_status, + config, + expect, ): minion_pool_dict = config.get("minion_pool", {}) if minion_pool_dict: for minion in minion_pool_dict.get("minion_machines", []): if minion.get("last_used_at"): - minion["last_used_at"] = datetime.datetime( - *minion["last_used_at"]) + minion["last_used_at"] = datetime.datetime(*minion["last_used_at"]) minion_pool = testutils.DictToObject(minion_pool_dict, {}) expected_result = expect.get("result", {}) @@ -308,28 +293,27 @@ def test_get_minion_pool_refresh_flow( # Test the flow returned by the function flow_tasks = [node.name for node, _ in flow.iter_nodes()] - self.assertEqual( - exptected_flow_tasks, - flow_tasks) + self.assertEqual(exptected_flow_tasks, flow_tasks) # Test DB calls that should be made for call in expected_db_calls_include: for method, args in call.items(): if method == "set_minion_machine_allocation_status": - mock_set_minion_machine_allocation_status\ - .assert_any_call( - mock.sentinel.context, - args.get("id"), - args.get("allocation_status")) + mock_set_minion_machine_allocation_status.assert_any_call( + mock.sentinel.context, + args.get("id"), + args.get("allocation_status"), + ) # Test DB calls that should not be made for call in expected_db_calls_exclude: for method, args in call.items(): if method == "set_minion_machine_allocation_status": - assert mock.call( - mock.sentinel.context, - args.get("id"), - args.get("allocation_status"))\ - not in mock_set_minion_machine_allocation_status\ - .mock_calls, f"Unexpected call to {method}, " \ - f"args: {args}" + assert ( + mock.call( + mock.sentinel.context, + args.get("id"), + args.get("allocation_status"), + ) + not in mock_set_minion_machine_allocation_status.mock_calls + ), f"Unexpected call to {method}, args: {args}" diff --git a/coriolis/tests/minion_manager/rpc/test_tasks.py b/coriolis/tests/minion_manager/rpc/test_tasks.py index 42990f8c..392bac93 100644 --- a/coriolis/tests/minion_manager/rpc/test_tasks.py +++ b/coriolis/tests/minion_manager/rpc/test_tasks.py @@ -5,10 +5,10 @@ from taskflow.types import failure -from coriolis.conductor.rpc.client import ConductorClient from coriolis import exception -from coriolis.minion_manager.rpc.client import MinionManagerClient +from coriolis.conductor.rpc.client import ConductorClient from coriolis.minion_manager.rpc import tasks +from coriolis.minion_manager.rpc.client import MinionManagerClient from coriolis.taskflow import base from coriolis.tests import test_base @@ -28,169 +28,190 @@ def setUp(self): def test__conductor_client(self): with mock.patch( 'coriolis.conductor.rpc.client.ConductorClient', - return_value=mock.MagicMock(spec=ConductorClient)) as \ - mock_Conductor_client: + return_value=mock.MagicMock(spec=ConductorClient), + ) as mock_Conductor_client: result = self.task._conductor_client - self.assertEqual( - result, mock_Conductor_client.return_value - ) + self.assertEqual(result, mock_Conductor_client.return_value) mock_Conductor_client.assert_called_once_with() def test__conductor_client_already_set(self): with mock.patch( - 'coriolis.conductor.rpc.client.ConductorClient') as \ - mock_Conductor_client: - self.task._conductor_client_instance = mock.MagicMock( - spec=ConductorClient) + 'coriolis.conductor.rpc.client.ConductorClient' + ) as mock_Conductor_client: + self.task._conductor_client_instance = mock.MagicMock(spec=ConductorClient) result = self.task._conductor_client - self.assertEqual( - result, self.task._conductor_client_instance - ) + self.assertEqual(result, self.task._conductor_client_instance) mock_Conductor_client.assert_not_called() def test__minion_manager_client(self): with mock.patch( 'coriolis.minion_manager.rpc.client.MinionManagerClient', - return_value=mock.MagicMock(spec=MinionManagerClient)) as \ - mock_Minion_Manager_Client: + return_value=mock.MagicMock(spec=MinionManagerClient), + ) as mock_Minion_Manager_Client: result = self.task._minion_manager_client - self.assertEqual( - result, mock_Minion_Manager_Client.return_value - ) + self.assertEqual(result, mock_Minion_Manager_Client.return_value) mock_Minion_Manager_Client.assert_called_once_with() def test__minion_manager_client_already_set(self): with mock.patch( - 'coriolis.minion_manager.rpc.client.MinionManagerClient') as \ - mock_Minion_Manager_Client: + 'coriolis.minion_manager.rpc.client.MinionManagerClient' + ) as mock_Minion_Manager_Client: self.task._minion_manager_client_instance = mock.MagicMock( - spec=MinionManagerClient) + spec=MinionManagerClient + ) result = self.task._minion_manager_client - self.assertEqual( - result, self.task._minion_manager_client_instance - ) + self.assertEqual(result, self.task._minion_manager_client_instance) mock_Minion_Manager_Client.assert_not_called() @mock.patch.object(tasks.db_api, 'add_minion_pool_event') def test__add_minion_pool_event(self, mock_add_minion_pool_event): result = self.task._add_minion_pool_event( - mock.sentinel.context, mock.sentinel.message, - level=tasks.constants.TASK_EVENT_INFO) + mock.sentinel.context, + mock.sentinel.message, + level=tasks.constants.TASK_EVENT_INFO, + ) self.assertIsNone(result) mock_add_minion_pool_event.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - tasks.constants.TASK_EVENT_INFO, mock.sentinel.message + mock.sentinel.context, + mock.sentinel.minion_pool_id, + tasks.constants.TASK_EVENT_INFO, + mock.sentinel.message, ) @mock.patch.object(tasks.db_api, 'get_minion_machine') def test__get_minion_machine(self, mock_get_minion_machine): result = self.task._get_minion_machine( - mock.sentinel.context, mock.sentinel.minion_machine_id) + mock.sentinel.context, mock.sentinel.minion_machine_id + ) self.assertEqual(result, mock_get_minion_machine.return_value) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_machine_id) + mock.sentinel.context, mock.sentinel.minion_machine_id + ) @mock.patch.object(tasks.db_api, 'get_minion_machine') def test__get_minion_machine_not_found(self, mock_get_minion_machine): mock_get_minion_machine.return_value = None self.assertRaises( - tasks.exception.NotFound, self.task._get_minion_machine, - mock.sentinel.context, mock.sentinel.minion_machine_id, - raise_if_not_found=True) + tasks.exception.NotFound, + self.task._get_minion_machine, + mock.sentinel.context, + mock.sentinel.minion_machine_id, + raise_if_not_found=True, + ) @mock.patch.object(tasks.db_api, 'set_minion_pool_status') def test__set_minion_pool_status(self, mock_set_minion_pool_status): with mock.patch.object( - tasks.minion_manager_utils, 'get_minion_pool_lock') as \ - mock_get_minion_pool_lock: + tasks.minion_manager_utils, 'get_minion_pool_lock' + ) as mock_get_minion_pool_lock: result = self.task._set_minion_pool_status( - mock.sentinel.context, mock.sentinel.minion_pool_id, - mock.sentinel.new_status) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + mock.sentinel.new_status, + ) self.assertIsNone(result) mock_get_minion_pool_lock.assert_called_once_with( - mock.sentinel.minion_pool_id, external=True) + mock.sentinel.minion_pool_id, external=True + ) mock_set_minion_pool_status.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - mock.sentinel.new_status) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + mock.sentinel.new_status, + ) @mock.patch.object(tasks.db_api, 'update_minion_machine') def test__update_minion_machine(self, mock_update_minion_machine): with mock.patch.object( - tasks.minion_manager_utils, 'get_minion_pool_lock') as \ - mock_get_minion_pool_lock: + tasks.minion_manager_utils, 'get_minion_pool_lock' + ) as mock_get_minion_pool_lock: result = self.task._update_minion_machine( - mock.sentinel.ctxt, mock.sentinel.minion_pool_id, - mock.sentinel.minion_machine_id, mock.sentinel.updated_values) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_id, + mock.sentinel.minion_machine_id, + mock.sentinel.updated_values, + ) self.assertIsNone(result) mock_get_minion_pool_lock.assert_called_once_with( - mock.sentinel.minion_pool_id, external=True) + mock.sentinel.minion_pool_id, external=True + ) mock_update_minion_machine.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.minion_machine_id, - mock.sentinel.updated_values) + mock.sentinel.ctxt, + mock.sentinel.minion_machine_id, + mock.sentinel.updated_values, + ) @mock.patch.object(tasks.db_api, 'set_minion_machine_allocation_status') def test__set_minion_machine_allocation_status( - self, mock_set_minion_machine_allocation_status): + self, mock_set_minion_machine_allocation_status + ): with mock.patch.object( - tasks.minion_manager_utils, 'get_minion_pool_lock') as \ - mock_get_minion_pool_lock: + tasks.minion_manager_utils, 'get_minion_pool_lock' + ) as mock_get_minion_pool_lock: result = self.task._set_minion_machine_allocation_status( - mock.sentinel.ctxt, mock.sentinel.minion_pool_id, - mock.sentinel.minion_machine_id, mock.sentinel.new_status) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_id, + mock.sentinel.minion_machine_id, + mock.sentinel.new_status, + ) self.assertIsNone(result) mock_get_minion_pool_lock.assert_called_once_with( - mock.sentinel.minion_pool_id, external=True) + mock.sentinel.minion_pool_id, external=True + ) mock_set_minion_machine_allocation_status.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.minion_machine_id, - mock.sentinel.new_status) + mock.sentinel.ctxt, + mock.sentinel.minion_machine_id, + mock.sentinel.new_status, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_update_minion_machine' - ) - def test__set_minion_machine_power_status( - self, mock_update_minion_machine): + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_update_minion_machine') + def test__set_minion_machine_power_status(self, mock_update_minion_machine): result = self.task._set_minion_machine_power_status( - mock.sentinel.ctxt, mock.sentinel.minion_pool_id, - mock.sentinel.minion_machine_id, mock.sentinel.new_status) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_id, + mock.sentinel.minion_machine_id, + mock.sentinel.new_status, + ) self.assertIsNone(result) mock_update_minion_machine.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.minion_pool_id, + mock.sentinel.ctxt, + mock.sentinel.minion_pool_id, mock.sentinel.minion_machine_id, - {'power_status': mock.sentinel.new_status} + {'power_status': mock.sentinel.new_status}, ) class BaseReportMinionAllocationFailureForActionTaskTestCase( - test_base.CoriolisBaseTestCase): + test_base.CoriolisBaseTestCase +): @mock.patch.object( tasks._BaseReportMinionAllocationFailureForActionTask, - '__abstractmethods__', set() + '__abstractmethods__', + set(), ) @mock.patch.object( tasks._BaseReportMinionAllocationFailureForActionTask, '_get_task_name' ) def setUp(self, mock_get_task_name): - super( - BaseReportMinionAllocationFailureForActionTaskTestCase, - self).setUp() + super(BaseReportMinionAllocationFailureForActionTaskTestCase, self).setUp() self.task = tasks._BaseReportMinionAllocationFailureForActionTask( - mock.sentinel.action_id) + mock.sentinel.action_id + ) def test_execute(self): result = self.task.execute(mock.sentinel.context) @@ -198,47 +219,46 @@ def test_execute(self): @mock.patch.object( tasks._BaseReportMinionAllocationFailureForActionTask, - '_report_machine_allocation_failure' - ) - @mock.patch.object( - MinionManagerClient, 'deallocate_minion_machines_for_action' - ) - def test_revert(self, mock_deallocate_minion_machines_for_action, - mock_report_machine_allocation_failure): - result = self.task.revert( - mock.sentinel.context, mock.sentinel.flow_failures) + '_report_machine_allocation_failure', + ) + @mock.patch.object(MinionManagerClient, 'deallocate_minion_machines_for_action') + def test_revert( + self, + mock_deallocate_minion_machines_for_action, + mock_report_machine_allocation_failure, + ): + result = self.task.revert(mock.sentinel.context, mock.sentinel.flow_failures) self.assertIsNone(result) mock_deallocate_minion_machines_for_action.assert_called_once_with( - mock.sentinel.context, mock.sentinel.action_id) + mock.sentinel.context, mock.sentinel.action_id + ) mock_report_machine_allocation_failure.assert_called_once_with( - mock.sentinel.context, mock.sentinel.action_id, - "No flow failures provided.") + mock.sentinel.context, mock.sentinel.action_id, "No flow failures provided." + ) class ReportMinionAllocationFailureForMigrationTaskTestCase( - test_base.CoriolisBaseTestCase): + test_base.CoriolisBaseTestCase +): def setUp(self): - super(ReportMinionAllocationFailureForMigrationTaskTestCase, - self).setUp() + super(ReportMinionAllocationFailureForMigrationTaskTestCase, self).setUp() self.action_id = mock.sentinel.action_id - self.task = tasks.ReportMinionAllocationFailureForMigrationTask( - self.action_id) + self.task = tasks.ReportMinionAllocationFailureForMigrationTask(self.action_id) def test_get_task_name(self): result = self.task._get_task_name(self.action_id) self.assertEqual( - result, - f"migration-{self.action_id}-minion-allocation-failure" + result, f"migration-{self.action_id}-minion-allocation-failure" ) - @mock.patch.object( - ConductorClient, 'report_deployment_minions_allocation_error' - ) + @mock.patch.object(ConductorClient, 'report_deployment_minions_allocation_error') def test__report_machine_allocation_failure( - self, mock_report_depl_minions_alloation_error): + self, mock_report_depl_minions_alloation_error + ): result = self.task._report_machine_allocation_failure( - mock.sentinel.context, self.action_id, mock.sentinel.failure_str) + mock.sentinel.context, self.action_id, mock.sentinel.failure_str + ) self.assertIsNone(result) mock_report_depl_minions_alloation_error.assert_called_once_with( @@ -247,28 +267,24 @@ def test__report_machine_allocation_failure( class ReportMinionAllocationFailureForReplicaTaskTestCase( - test_base.CoriolisBaseTestCase): + test_base.CoriolisBaseTestCase +): def setUp(self): - super(ReportMinionAllocationFailureForReplicaTaskTestCase, - self).setUp() + super(ReportMinionAllocationFailureForReplicaTaskTestCase, self).setUp() self.action_id = mock.sentinel.action_id - self.task = tasks.ReportMinionAllocationFailureForReplicaTask( - self.action_id) + self.task = tasks.ReportMinionAllocationFailureForReplicaTask(self.action_id) def test_get_task_name(self): result = self.task._get_task_name(self.action_id) - self.assertEqual( - result, - f"replica-{self.action_id}-minion-allocation-failure" - ) + self.assertEqual(result, f"replica-{self.action_id}-minion-allocation-failure") - @mock.patch.object( - ConductorClient, 'report_transfer_minions_allocation_error' - ) + @mock.patch.object(ConductorClient, 'report_transfer_minions_allocation_error') def test__report_machine_allocation_failure( - self, mock_report_transfer_minions_allocation_error): + self, mock_report_transfer_minions_allocation_error + ): result = self.task._report_machine_allocation_failure( - mock.sentinel.context, self.action_id, mock.sentinel.failure_str) + mock.sentinel.context, self.action_id, mock.sentinel.failure_str + ) self.assertIsNone(result) mock_report_transfer_minions_allocation_error.assert_called_once_with( @@ -276,12 +292,9 @@ def test__report_machine_allocation_failure( ) -class BaseConfirmMinionAllocationForActionTaskTestCase( - test_base.CoriolisBaseTestCase): - +class BaseConfirmMinionAllocationForActionTaskTestCase(test_base.CoriolisBaseTestCase): @mock.patch.object( - tasks._BaseConfirmMinionAllocationForActionTask, - '__abstractmethods__', set() + tasks._BaseConfirmMinionAllocationForActionTask, '__abstractmethods__', set() ) @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, '_get_task_name' @@ -293,9 +306,11 @@ def setUp(self, mock_get_task_name): self.minion_machine.pool_id = mock.sentinel.pool_id self.minion_machine.allocated_action = mock.sentinel.action_id self.minion_machine.allocation_status = ( - tasks.constants.MINION_MACHINE_STATUS_IN_USE) + tasks.constants.MINION_MACHINE_STATUS_IN_USE + ) self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON + ) self.allocated_machine_id_mappings = { mock.sentinel.instance_id: { @@ -306,28 +321,38 @@ def setUp(self, mock_get_task_name): } self.task = tasks._BaseConfirmMinionAllocationForActionTask( - mock.sentinel.action_id, self.allocated_machine_id_mappings) + mock.sentinel.action_id, self.allocated_machine_id_mappings + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, - '_confirm_machine_allocation_for_action' + '_confirm_machine_allocation_for_action', ) def test_execute(self, mock_confirm_allocation, mock_get_minion_machine): mock_get_minion_machine.return_value = self.minion_machine result = self.task.execute(mock.sentinel.context) - mock_get_minion_machine.assert_has_calls([ - mock.call(mock.sentinel.context, mock.sentinel.origin_minion_id, - raise_if_not_found=True), - mock.call(mock.sentinel.context, - mock.sentinel.destination_minion_id, - raise_if_not_found=True), - mock.call(mock.sentinel.context, - mock.sentinel.osmorphing_minion_id, - raise_if_not_found=True)], any_order=True) + mock_get_minion_machine.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + mock.sentinel.origin_minion_id, + raise_if_not_found=True, + ), + mock.call( + mock.sentinel.context, + mock.sentinel.destination_minion_id, + raise_if_not_found=True, + ), + mock.call( + mock.sentinel.context, + mock.sentinel.osmorphing_minion_id, + raise_if_not_found=True, + ), + ], + any_order=True, + ) expected_machine_allocations = { mock.sentinel.instance_id: { @@ -338,115 +363,125 @@ def test_execute(self, mock_confirm_allocation, mock_get_minion_machine): } mock_confirm_allocation.assert_called_once_with( - mock.sentinel.context, mock.sentinel.action_id, - expected_machine_allocations + mock.sentinel.context, mock.sentinel.action_id, expected_machine_allocations ) self.assertIsNone(result) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_raises_exception_when_allocation_status_is_not_in_use( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.allocation_status = 'DEALLOCATED' self.assertRaises( exception.InvalidMinionMachineState, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin_minion_id, - raise_if_not_found=True) + mock.sentinel.context, + mock.sentinel.origin_minion_id, + raise_if_not_found=True, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_raises_exception_when_allocated_action_is_not_correct( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.allocated_action = 'ANOTHER_ACTION' self.assertRaises( exception.InvalidMinionMachineState, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin_minion_id, - raise_if_not_found=True) + mock.sentinel.context, + mock.sentinel.origin_minion_id, + raise_if_not_found=True, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_raises_exception_when_power_status_is_not_powered_on( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.power_status = 'POWERED_OFF' self.assertRaises( exception.InvalidMinionMachineState, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin_minion_id, - raise_if_not_found=True) + mock.sentinel.context, + mock.sentinel.origin_minion_id, + raise_if_not_found=True, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, - '_confirm_machine_allocation_for_action' + '_confirm_machine_allocation_for_action', ) def test_execute_no_machine_allocation( - self, mock_confirm_allocation, mock_get_minion_machine): + self, mock_confirm_allocation, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine - self.allocated_machine_id_mappings[ - mock.sentinel.instance_id] = {} + self.allocated_machine_id_mappings[mock.sentinel.instance_id] = {} result = self.task.execute(mock.sentinel.context) self.assertIsNone(result) mock_get_minion_machine.assert_not_called() mock_confirm_allocation.assert_called_once_with( - mock.sentinel.context, mock.sentinel.action_id, {}) + mock.sentinel.context, mock.sentinel.action_id, {} + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_raises_exception_when_required_properties_are_missing( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.provider_properties = None self.assertRaises( exception.InvalidMinionMachineState, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin_minion_id, - raise_if_not_found=True) + mock.sentinel.context, + mock.sentinel.origin_minion_id, + raise_if_not_found=True, + ) @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, - '_confirm_machine_allocation_for_action' + '_confirm_machine_allocation_for_action', ) @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, '_get_action_label' ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_with_exception_not_found( - self, mock_get_minion_machine, mock_get_action_label, - mock_confirm_allocation): + self, mock_get_minion_machine, mock_get_action_label, mock_confirm_allocation + ): mock_get_minion_machine.return_value = self.minion_machine mock_confirm_allocation.side_effect = exception.NotFound() self.assertRaises( exception.MinionMachineAllocationFailure, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_action_label.assert_called_once_with() @@ -455,35 +490,33 @@ def test_execute_with_exception_not_found( ) @mock.patch.object( tasks._BaseConfirmMinionAllocationForActionTask, - '_confirm_machine_allocation_for_action' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' + '_confirm_machine_allocation_for_action', ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_raises_exception_when_invalid_migration_state( - self, mock_get_minion_machine, mock_confirm_allocation, - mock_get_action_label): + self, mock_get_minion_machine, mock_confirm_allocation, mock_get_action_label + ): mock_get_minion_machine.return_value = self.minion_machine mock_confirm_allocation.side_effect = [ - exception.InvalidTransferState(reason='Invalid state')] + exception.InvalidTransferState(reason='Invalid state') + ] self.assertRaises( exception.MinionMachineAllocationFailure, - self.task.execute, mock.sentinel.context) + self.task.execute, + mock.sentinel.context, + ) mock_get_action_label.assert_called_once_with() -class ConfirmMinionAllocationForMigrationTaskTestCase( - test_base.CoriolisBaseTestCase): - +class ConfirmMinionAllocationForMigrationTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ConfirmMinionAllocationForMigrationTaskTestCase, self).setUp() self.action_id = mock.sentinel.action_id self.task = tasks.ConfirmMinionAllocationForMigrationTask( - mock.sentinel.action_id, - mock.sentinel.allocated_machine_id_mappings + mock.sentinel.action_id, mock.sentinel.allocated_machine_id_mappings ) def test__get_action_label(self): @@ -495,33 +528,28 @@ def test_get_task_name(self): result = self.task._get_task_name(self.action_id) self.assertEqual( - result, - f"migration-{self.action_id}-minion-allocation-confirmation" + result, f"migration-{self.action_id}-minion-allocation-confirmation" ) - @mock.patch.object( - ConductorClient, 'confirm_deployment_minions_allocation' - ) + @mock.patch.object(ConductorClient, 'confirm_deployment_minions_allocation') def test__confirm_machine_allocation_for_action( - self, mock_confirm_deployment_minions_allocation): + self, mock_confirm_deployment_minions_allocation + ): result = self.task._confirm_machine_allocation_for_action( - mock.sentinel.context, self.action_id, - mock.sentinel.machine_allocations) + mock.sentinel.context, self.action_id, mock.sentinel.machine_allocations + ) self.assertIsNone(result) mock_confirm_deployment_minions_allocation.assert_called_once_with( - mock.sentinel.context, self.action_id, - mock.sentinel.machine_allocations) - + mock.sentinel.context, self.action_id, mock.sentinel.machine_allocations + ) -class ConfirmMinionAllocationForReplicaTaskTestCase( - test_base.CoriolisBaseTestCase): +class ConfirmMinionAllocationForReplicaTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ConfirmMinionAllocationForReplicaTaskTestCase, self).setUp() self.task = tasks.ConfirmMinionAllocationForReplicaTask( - mock.sentinel.action_id, - mock.sentinel.allocate_machine_id_mappings + mock.sentinel.action_id, mock.sentinel.allocate_machine_id_mappings ) def test__get_action_label(self): @@ -533,23 +561,25 @@ def test_get_task_name(self): result = self.task._get_task_name(mock.sentinel.action_id) self.assertEqual( - result, - f"replica-{mock.sentinel.action_id}-minion-allocation-confirmation" + result, f"replica-{mock.sentinel.action_id}-minion-allocation-confirmation" ) - @mock.patch.object( - ConductorClient, 'confirm_transfer_minions_allocation' - ) + @mock.patch.object(ConductorClient, 'confirm_transfer_minions_allocation') def test__confirm_machine_allocation_for_action( - self, mock_confirm_transfer_minions_allocation): + self, mock_confirm_transfer_minions_allocation + ): result = self.task._confirm_machine_allocation_for_action( - mock.sentinel.context, mock.sentinel.action_id, - mock.sentinel.machine_allocations) + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.machine_allocations, + ) self.assertIsNone(result) mock_confirm_transfer_minions_allocation.assert_called_once_with( - mock.sentinel.context, mock.sentinel.action_id, - mock.sentinel.machine_allocations) + mock.sentinel.context, + mock.sentinel.action_id, + mock.sentinel.machine_allocations, + ) class UpdateMinionPoolStatusTaskTestCase(test_base.CoriolisBaseTestCase): @@ -560,36 +590,42 @@ def setUp(self): self.status_to_revert_to = mock.sentinel.status_to_revert_to self.task = tasks.UpdateMinionPoolStatusTask( - mock.sentinel.minion_pool_id, mock.sentinel.status, - status_to_revert_to=self.status_to_revert_to) + mock.sentinel.minion_pool_id, + mock.sentinel.status, + status_to_revert_to=self.status_to_revert_to, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status') @mock.patch.object(tasks.db_api, 'get_minion_pool') - def test_execute(self, mock_get_minion_pool, mock_set_minion_pool_status, - mock_add_minion_pool_event): + def test_execute( + self, + mock_get_minion_pool, + mock_set_minion_pool_status, + mock_add_minion_pool_event, + ): result = self.task.execute(mock.sentinel.context) self.assertEqual(result, self.task._target_status) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - include_machines=False, include_events=False, - include_progress_updates=False) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ) mock_set_minion_pool_status.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - self.task._target_status) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + self.task._target_status, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status') @mock.patch.object(tasks.db_api, 'get_minion_pool') def test_execute_when_previous_status_equals_target_status( - self, mock_get_minion_pool, mock_set_minion_pool_status): + self, mock_get_minion_pool, mock_set_minion_pool_status + ): self.task._previous_status = self.task._target_status result = self.task.execute(mock.sentinel.context) @@ -598,35 +634,39 @@ def test_execute_when_previous_status_equals_target_status( mock_get_minion_pool.assert_not_called() mock_set_minion_pool_status.assert_not_called() - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status') @mock.patch.object(tasks.db_api, 'get_minion_pool') - def test_revert(self, mock_get_minion_pool, mock_set_minion_pool_status, - mock_add_minion_pool_event): + def test_revert( + self, + mock_get_minion_pool, + mock_set_minion_pool_status, + mock_add_minion_pool_event, + ): mock_get_minion_pool.return_value.status = 'DEALLOCATED' result = self.task.revert(mock.sentinel.context) self.assertIsNone(result) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - include_machines=False, include_events=False, - include_progress_updates=False) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ) mock_set_minion_pool_status.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - self.status_to_revert_to) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + self.status_to_revert_to, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status') @mock.patch.object(tasks.db_api, 'get_minion_pool') - def test_revert_no_minion_pool(self, mock_get_minion_pool, - mock_set_minion_pool_status): + def test_revert_no_minion_pool( + self, mock_get_minion_pool, mock_set_minion_pool_status + ): mock_get_minion_pool.return_value = None result = self.task.revert(mock.sentinel.context) @@ -634,21 +674,23 @@ def test_revert_no_minion_pool(self, mock_get_minion_pool, mock_set_minion_pool_status.assert_not_called() - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_set_minion_pool_status') @mock.patch.object(tasks.db_api, 'get_minion_pool') def test_revert_when_previous_status_equals_target_status( - self, mock_get_minion_pool, mock_set_minion_pool_status): + self, mock_get_minion_pool, mock_set_minion_pool_status + ): mock_get_minion_pool.return_value.status = self.status_to_revert_to result = self.task.revert(mock.sentinel.context) self.assertIsNone(result) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, mock.sentinel.minion_pool_id, - include_machines=False, include_events=False, - include_progress_updates=False) + mock.sentinel.context, + mock.sentinel.minion_pool_id, + include_machines=False, + include_events=False, + include_progress_updates=False, + ) mock_set_minion_pool_status.assert_not_called() @@ -656,40 +698,49 @@ def test_revert_when_previous_status_equals_target_status( class BaseMinionManangerTaskTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis BaseMinionManangerTask class.""" - @mock.patch.object( - tasks.BaseMinionManangerTask, '__abstractmethods__', set() - ) + @mock.patch.object(tasks.BaseMinionManangerTask, '__abstractmethods__', set()) @mock.patch.object(tasks.BaseMinionManangerTask, '_get_task_name') def setUp(self, mock_get_task_name): super(BaseMinionManangerTaskTestCase, self).setUp() self.task = tasks.BaseMinionManangerTask( - mock.sentinel.minion_pool_id, mock.sentinel.minion_machine_id, - mock.sentinel.main_task_runner_type) + mock.sentinel.minion_pool_id, + mock.sentinel.minion_machine_id, + mock.sentinel.main_task_runner_type, + ) @mock.patch.object(base.BaseRunWorkerTask, 'execute') def test_execute(self, mock_execute): result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertEqual(result, mock_execute.return_value) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(base.BaseRunWorkerTask, 'revert') def test_revert(self, mock_revert, mock_add_minion_pool_event): result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertIsNone(result) mock_revert.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, ) @@ -702,55 +753,63 @@ def setUp(self): self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE self.task = tasks.ValidateMinionPoolOptionsTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.ValidateMinionPoolOptionsTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS + tasks.constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS, ) def test__get_task_name(self): result = self.task._get_task_name( - self.minion_pool_id, mock.sentinel.minion_machine_id) + self.minion_pool_id, mock.sentinel.minion_machine_id + ) self.assertEqual( - result, - tasks.MINION_POOL_VALIDATION_TASK_NAME_FORMAT % self.minion_pool_id + result, tasks.MINION_POOL_VALIDATION_TASK_NAME_FORMAT % self.minion_pool_id ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') def test_execute(self, mock_execute, mock_add_minion_pool_event): result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertIsNone(result) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, ) @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') def test_revert(self, mock_revert): result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertIsNone(result) mock_revert.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, ) @@ -761,87 +820,101 @@ def setUp(self): super(AllocateSharedPoolResourcesTaskTestCase, self).setUp() self.minion_pool_id = 'test_pool_id' self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE - self.task_info = { - 'pool_shared_resources': True - } + self.task_info = {'pool_shared_resources': True} self.task = tasks.AllocateSharedPoolResourcesTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.AllocateSharedPoolResourcesTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES + tasks.constants.TASK_TYPE_SET_UP_DESTINATION_POOL_SHARED_RESOURCES, ) self.assertEqual( task._cleanup_task_runner_type, - tasks.constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES # noqa: E501 + tasks.constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES, # noqa: E501 ) def test__get_task_name(self): result = self.task._get_task_name( - self.minion_pool_id, mock.sentinel.minion_machine_id) + self.minion_pool_id, mock.sentinel.minion_machine_id + ) self.assertEqual( result, - tasks.MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % - self.minion_pool_id + tasks.MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT + % self.minion_pool_id, ) @mock.patch.object(tasks.db_api, 'get_minion_pool') - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object(tasks.db_api, 'update_minion_pool') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') - def test_execute(self, mock_get_minion_pool_lock, mock_update_minion_pool, - mock_execute, mock_add_minion_pool_event, - mock_get_minion_pool): - mock_execute.return_value = { - 'pool_shared_resources': {"resource1": "id1"} - } + def test_execute( + self, + mock_get_minion_pool_lock, + mock_update_minion_pool, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_pool, + ): + mock_execute.return_value = {'pool_shared_resources': {"resource1": "id1"}} mock_get_minion_pool.return_value.shared_resources = {} result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) - self.assertEqual( - result, {'pool_shared_resources': {"resource1": "id1"}}) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) + self.assertEqual(result, {'pool_shared_resources': {"resource1": "id1"}}) - mock_get_minion_pool_lock.assert_called_with( - self.minion_pool_id, external=True) + mock_get_minion_pool_lock.assert_called_with(self.minion_pool_id, external=True) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id) + mock.sentinel.context, self.minion_pool_id + ) mock_update_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, - {'shared_resources': {"resource1": "id1"}}) + mock.sentinel.context, + self.minion_pool_id, + {'shared_resources': {"resource1": "id1"}}, + ) @mock.patch.object(tasks.db_api, 'get_minion_pool') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object(tasks.db_api, 'update_minion_pool') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') def test_execute_shared_resources_already_allocated( - self, mock_get_minion_pool_lock, mock_update_minion_pool, - mock_execute, mock_get_minion_pool): + self, + mock_get_minion_pool_lock, + mock_update_minion_pool, + mock_execute, + mock_get_minion_pool, + ): mock_get_minion_pool.return_value.shared_resources = True self.assertRaises( - exception.InvalidMinionPoolState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionPoolState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id) + mock.sentinel.context, self.minion_pool_id + ) mock_execute.assert_not_called() mock_update_minion_pool.assert_not_called() @@ -850,50 +923,65 @@ def test_execute_shared_resources_already_allocated( @mock.patch.object(tasks.db_api, 'update_minion_pool') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') def test_execute_no_minion_pool( - self, mock_get_minion_pool_lock, mock_update_minion_pool, - mock_execute, mock_get_minion_pool): + self, + mock_get_minion_pool_lock, + mock_update_minion_pool, + mock_execute, + mock_get_minion_pool, + ): mock_get_minion_pool.return_value = None self.assertRaises( - exception.InvalidMinionPoolSelection, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionPoolSelection, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_get_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id) + mock.sentinel.context, self.minion_pool_id + ) mock_execute.assert_not_called() mock_update_minion_pool.assert_not_called() @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @mock.patch.object(tasks.db_api, 'update_minion_pool') - def test_revert(self, mock_update_minion_pool, mock_get_minion_pool_lock, - mock_revert): + def test_revert( + self, mock_update_minion_pool, mock_get_minion_pool_lock, mock_revert + ): self.task_info = {} - self.update_values = { - 'pool_shared_resources': None - } + self.update_values = {'pool_shared_resources': None} result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertIsNone(result) self.assertEqual(self.task_info['pool_shared_resources'], {}) mock_revert.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_update_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.update_values) + mock.sentinel.context, self.minion_pool_id, self.update_values + ) -class DeallocateSharedPoolResourcesTaskTestCase( - test_base.CoriolisBaseTestCase): +class DeallocateSharedPoolResourcesTaskTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis DeallocateSharedPoolResourcesTask class.""" def setUp(self): @@ -901,71 +989,80 @@ def setUp(self): self.minion_pool_id = 'test_pool_id' self.task_info = { 'pool_shared_resources': {"resource1": "id1"}, - 'pool_environment_options': {'option1': 'value1'} + 'pool_environment_options': {'option1': 'value1'}, } self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE self.task = tasks.DeallocateSharedPoolResourcesTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.DeallocateSharedPoolResourcesTask( - self.minion_pool_id, mock.sentinel.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, mock.sentinel.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES # noqa: E501 + tasks.constants.TASK_TYPE_TEAR_DOWN_DESTINATION_POOL_SHARED_RESOURCES, # noqa: E501 ) def test__get_task_name(self): result = self.task._get_task_name( - self.minion_pool_id, mock.sentinel.minion_machine_id) + self.minion_pool_id, mock.sentinel.minion_machine_id + ) self.assertEqual( result, - tasks.MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % - self.minion_pool_id + tasks.MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT + % self.minion_pool_id, ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.db_api, 'update_minion_pool') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') - def test_execute(self, mock_execute, mock_update_minion_pool, - mock_add_minion_pool_event): + def test_execute( + self, mock_execute, mock_update_minion_pool, mock_add_minion_pool_event + ): self.update_values = { 'shared_resources': None, } result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_update_minion_pool.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.update_values) + mock.sentinel.context, self.minion_pool_id, self.update_values + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.db_api, 'update_minion_pool') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') def test_execute_no_pool_shared_resources( - self, mock_execute, mock_update_minion_pool, - mock_add_minion_pool_event): + self, mock_execute, mock_update_minion_pool, mock_add_minion_pool_event + ): self.task_info = {} self.assertRaises( - exception.InvalidInput, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidInput, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_execute.assert_not_called() mock_update_minion_pool.assert_not_called() @@ -980,7 +1077,8 @@ def setUp(self): self.minion_machine.id = 'test_machine_id' self.minion_machine.pool_id = 'test_pool_id' self.minion_machine.allocation_status = ( - tasks.constants.MINION_MACHINE_STATUS_UNINITIALIZED) + tasks.constants.MINION_MACHINE_STATUS_UNINITIALIZED + ) self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE self.minion_machine.allocated_action = mock.sentinel.allocation_action self.minion_pool_id = 'test_pool_id' @@ -994,8 +1092,8 @@ def setUp(self): self.mock_failure = mock.MagicMock(spec=failure.Failure) self.task = tasks.AllocateMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.task._allocate_to_action = self.minion_machine.allocated_action @@ -1003,205 +1101,246 @@ def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.AllocateMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE + tasks.constants.TASK_TYPE_CREATE_DESTINATION_MINION_MACHINE, ) self.assertEqual( task._cleanup_task_runner_type, - tasks.constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE + tasks.constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE, ) def test__get_task_name(self): - result = self.task._get_task_name( - self.minion_pool_id, self.minion_machine_id) + result = self.task._get_task_name(self.minion_pool_id, self.minion_machine_id) self.assertEqual( result, - tasks.MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id), ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_update_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_update_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.timeutils, 'utcnow') @mock.patch.object(tasks.models, 'MinionMachine') @mock.patch.object(tasks.db_api, 'add_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') def test_execute( - self, mock_execute, mock_add_minion_machine, mock_minion_machine, - mock_utcnow, mock_set_minion_machine_allocation, - mock_add_minion_pool_event, mock_update_minion_machine, - mock_get_minion_machine): + self, + mock_execute, + mock_add_minion_machine, + mock_minion_machine, + mock_utcnow, + mock_set_minion_machine_allocation, + mock_add_minion_pool_event, + mock_update_minion_machine, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine mock_execute.return_value = { 'minion_connection_info': 'test_connection_info', 'minion_provider_properties': 'test_provider_properties', 'minion_backup_writer_connection_info': ( - 'test_backup_writer_connection_info'), + 'test_backup_writer_connection_info' + ), } expected_updated_values = { 'last_used_at': mock_utcnow.return_value, - 'allocation_status': ( - tasks.constants.MINION_MACHINE_STATUS_IN_USE), - 'power_status': ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON), + 'allocation_status': (tasks.constants.MINION_MACHINE_STATUS_IN_USE), + 'power_status': (tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON), 'connection_info': 'test_connection_info', 'provider_properties': 'test_provider_properties', - 'backup_writer_connection_info': ( - 'test_backup_writer_connection_info'), - 'allocated_action': self.task._allocate_to_action + 'backup_writer_connection_info': ('test_backup_writer_connection_info'), + 'allocated_action': self.task._allocate_to_action, } result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_ALLOCATING) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_ALLOCATING, + ) mock_minion_machine.assert_not_called() mock_add_minion_machine.assert_not_called() mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_update_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - expected_updated_values) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + expected_updated_values, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_when_allocation_status_is_not_uninitialized( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.allocation_status = 'ALLOCATING' self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_when_minion_machine_belongs_to_different_pool( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.pool_id = 'DIFFERENT_POOL_ID' self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_when_minion_machine_already_allocated( - self, mock_get_minion_machine): + self, mock_get_minion_machine + ): mock_get_minion_machine.return_value = self.minion_machine self.task._allocate_to_action = 'ANOTHER_ACTION' self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_update_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_update_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.models, 'MinionMachine') @mock.patch.object(tasks.db_api, 'add_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') def test_execute_add_new_minion_machine( - self, mock_execute, mock_add_minion_machine, mock_minion_machine, - mock_set_minion_machine_allocation, mock_add_minion_pool_event, - mock_update_minion_machine, mock_get_minion_machine): + self, + mock_execute, + mock_add_minion_machine, + mock_minion_machine, + mock_set_minion_machine_allocation, + mock_add_minion_pool_event, + mock_update_minion_machine, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = None mock_minion_machine.return_value = self.minion_machine mock_execute.side_effect = CoriolisTestException - self.assertRaises(CoriolisTestException, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + self.assertRaises( + CoriolisTestException, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_minion_machine.assert_called_once_with() mock_add_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine) + mock.sentinel.context, self.minion_machine + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_ERROR_DEPLOYING, + ) mock_update_minion_machine.assert_not_called() @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @mock.patch.object(tasks.db_api, 'get_minion_machine') @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') - def test_revert(self, mock_revert, mock_delete_minion_machine, - mock_get_minion_machine, mock_get_minion_pool_lock): + def test_revert( + self, + mock_revert, + mock_delete_minion_machine, + mock_get_minion_machine, + mock_get_minion_pool_lock, + ): self.task_info['minion_provider_properties'] = ( - mock_get_minion_machine.return_value.provider_properties) + mock_get_minion_machine.return_value.provider_properties + ) mock_get_minion_machine.return_value = self.minion_machine self.mock_failure.traceback_str = 'test_traceback' kwargs = {'result': self.mock_failure} result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info, **kwargs) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + **kwargs, + ) self.assertIsNone(result) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_revert.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info, **kwargs + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + **kwargs, ) @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @@ -1209,20 +1348,29 @@ def test_revert(self, mock_revert, mock_delete_minion_machine, @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') def test_revert_failed_deletion( - self, mock_revert, mock_delete_minion_machine, - mock_get_minion_machine, mock_get_minion_pool_lock): + self, + mock_revert, + mock_delete_minion_machine, + mock_get_minion_machine, + mock_get_minion_pool_lock, + ): mock_get_minion_machine.return_value = None mock_delete_minion_machine.side_effect = CoriolisTestException result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertIsNone(result) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_revert.assert_not_called() @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @@ -1230,27 +1378,42 @@ def test_revert_failed_deletion( @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') def test_revert_unexpected_result_type( - self, mock_revert, mock_delete_minion_machine, - mock_get_minion_machine, mock_get_minion_pool_lock): + self, + mock_revert, + mock_delete_minion_machine, + mock_get_minion_machine, + mock_get_minion_pool_lock, + ): self.task_info['minion_provider_properties'] = ( - mock_get_minion_machine.return_value.provider_properties) + mock_get_minion_machine.return_value.provider_properties + ) mock_get_minion_machine.return_value = self.minion_machine kwargs = {'result': 'test_result'} result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info, **kwargs) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + **kwargs, + ) self.assertIsNone(result) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_revert.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info, **kwargs + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + **kwargs, ) @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @@ -1258,45 +1421,66 @@ def test_revert_unexpected_result_type( @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') def test_revert_raises_unhandled_exception( - self, mock_revert, mock_delete_minion_machine, - mock_get_minion_machine, mock_get_minion_pool_lock): + self, + mock_revert, + mock_delete_minion_machine, + mock_get_minion_machine, + mock_get_minion_pool_lock, + ): mock_get_minion_machine.return_value = self.minion_machine mock_revert.side_effect = CoriolisTestException self.assertRaises( - CoriolisTestException, self.task.revert, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + CoriolisTestException, + self.task.revert, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') @mock.patch.object(tasks.db_api, 'get_minion_machine') @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'revert') def test_revert_no_raise_on_cleanup_failure( - self, mock_revert, mock_delete_minion_machine, - mock_get_minion_machine, mock_get_minion_pool_lock): + self, + mock_revert, + mock_delete_minion_machine, + mock_get_minion_machine, + mock_get_minion_pool_lock, + ): mock_get_minion_machine.return_value = self.minion_machine self.task._raise_on_cleanup_failure = False mock_revert.side_effect = CoriolisTestException result = self.task.revert( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertIsNone(result) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) class DeallocateMinionMachineTaskTestCase(test_base.CoriolisBaseTestCase): @@ -1308,7 +1492,8 @@ def setUp(self): self.minion_machine.id = 'test_machine_id' self.minion_machine.pool_id = 'test_pool_id' self.minion_machine.allocation_status = ( - tasks.constants.MINION_MACHINE_STATUS_IN_USE) + tasks.constants.MINION_MACHINE_STATUS_IN_USE + ) self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE self.minion_machine.allocated_action = mock.sentinel.allocation_action self.minion_pool_id = 'test_pool_id' @@ -1319,151 +1504,178 @@ def setUp(self): self.mock_failure = mock.MagicMock(spec=failure.Failure) self.task = tasks.DeallocateMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) - self.task._deallocate_from_action = ( - self.minion_machine.allocated_action) + self.task._deallocate_from_action = self.minion_machine.allocated_action def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.DeallocateMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE + tasks.constants.TASK_TYPE_DELETE_DESTINATION_MINION_MACHINE, ) def test__get_task_name(self): - result = self.task._get_task_name( - self.minion_pool_id, self.minion_machine_id) + result = self.task._get_task_name(self.minion_pool_id, self.minion_machine_id) self.assertEqual( result, - tasks.MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id), ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') def test_execute( - self, mock_get_minion_pool_lock, mock_execute, - mock_delete_minion_machine, mock_set_minion_machine_allocation, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_get_minion_pool_lock, + mock_execute, + mock_delete_minion_machine, + mock_set_minion_machine_allocation, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.provider_properties = None result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_set_minion_machine_allocation.assert_not_called() mock_execute.assert_not_called() mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_no_machine(self, mock_get_minion_machine): mock_get_minion_machine.return_value = None result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') def test_execute_with_provider_properties( - self, mock_get_minion_pool_lock, mock_execute, - mock_delete_minion_machine, mock_set_minion_machine_allocation, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_get_minion_pool_lock, + mock_execute, + mock_delete_minion_machine, + mock_set_minion_machine_allocation, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine self.task_info['minion_provider_properties'] = ( - mock_get_minion_machine.return_value.provider_properties) + mock_get_minion_machine.return_value.provider_properties + ) result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_DEALLOCATING) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_DEALLOCATING, + ) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_delete_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_get_minion_pool_lock.assert_called_once_with( - self.minion_pool_id, external=True) + self.minion_pool_id, external=True + ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.db_api, 'delete_minion_machine') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object(tasks.minion_manager_utils, 'get_minion_pool_lock') def test_execute_with_exception( - self, mock_get_minion_pool_lock, mock_execute, - mock_delete_minion_machine, mock_set_minion_machine_allocation, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_get_minion_pool_lock, + mock_execute, + mock_delete_minion_machine, + mock_set_minion_machine_allocation, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine mock_execute.side_effect = CoriolisTestException self.assertRaises( - CoriolisTestException, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + CoriolisTestException, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id) + mock.sentinel.context, self.minion_machine_id + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_DEALLOCATING) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_DEALLOCATING, + ) mock_delete_minion_machine.assert_not_called() mock_get_minion_pool_lock.assert_not_called() @@ -1481,224 +1693,254 @@ def setUp(self): self.minion_pool_id = 'test_pool_id' self.minion_machine_id = 'test_machine_id' self.task = tasks.HealthcheckMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, self.minion_pool_type, - machine_status_on_success=( - tasks.constants.MINION_MACHINE_STATUS_AVAILABLE)) + self.minion_pool_id, + self.minion_machine_id, + self.minion_pool_type, + machine_status_on_success=(tasks.constants.MINION_MACHINE_STATUS_AVAILABLE), + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.HealthcheckMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION + tasks.constants.TASK_TYPE_HEALTHCHECK_DESTINATION_MINION, ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute( - self, mock_set_minion_machine_allocation, mock_execute, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_set_minion_machine_allocation, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine execution_info = { 'minion_provider_properties': ( - mock_get_minion_machine.return_value.provider_properties), + mock_get_minion_machine.return_value.provider_properties + ), 'minion_connection_info': ( - mock_get_minion_machine.return_value.connection_info), + mock_get_minion_machine.return_value.connection_info + ), } result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertEqual(result['healthy'], True) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, execution_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + execution_info, + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_AVAILABLE) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_AVAILABLE, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_no_machine( - self, mock_set_minion_machine_allocation, mock_execute, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_set_minion_machine_allocation, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = None result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertEqual(result['healthy'], False) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_set_minion_machine_allocation.assert_not_called() mock_execute.assert_not_called() - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') def test_execute_no_machine_with_fail_on_error( - self, mock_add_minion_pool_event, mock_get_minion_machine): + self, mock_add_minion_pool_event, mock_get_minion_machine + ): self.task._fail_on_error = True mock_get_minion_machine.return_value = None self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_with_allocation_status( - self, mock_set_minion_machine_allocation, mock_execute, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_set_minion_machine_allocation, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): self.minion_machine.allocation_status = ( - tasks.constants.MINION_MACHINE_STATUS_ERROR) + tasks.constants.MINION_MACHINE_STATUS_ERROR + ) mock_get_minion_machine.return_value = self.minion_machine result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertEqual(result['healthy'], False) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_set_minion_machine_allocation.assert_not_called() mock_execute.assert_not_called() - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') def test_execute_with_allocation_status_and_fail_on_error( - self, mock_add_minion_pool_event, mock_get_minion_machine): + self, mock_add_minion_pool_event, mock_get_minion_machine + ): self.minion_machine.allocation_status = ( - tasks.constants.MINION_MACHINE_STATUS_ERROR) + tasks.constants.MINION_MACHINE_STATUS_ERROR + ) self.task._fail_on_error = True mock_get_minion_machine.return_value = self.minion_machine self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_with_exception( - self, mock_set_minion_machine_allocation, mock_execute, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_set_minion_machine_allocation, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine mock_execute.side_effect = CoriolisTestException result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) self.assertEqual(result['healthy'], False) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_ERROR) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_ERROR, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_with_exception_and_fail_on_error( - self, mock_set_minion_machine_allocation, mock_execute, - mock_add_minion_pool_event, mock_get_minion_machine): + self, + mock_set_minion_machine_allocation, + mock_execute, + mock_add_minion_pool_event, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine mock_execute.side_effect = CoriolisTestException self.task._fail_on_error = True self.assertRaises( - CoriolisTestException, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info) + CoriolisTestException, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=False) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=False + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_ERROR) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_ERROR, + ) def test__get_task_name(self): - result = self.task._get_task_name( - self.minion_pool_id, self.minion_machine_id) + result = self.task._get_task_name(self.minion_pool_id, self.minion_machine_id) self.assertEqual( result, - tasks.MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id), ) @@ -1710,13 +1952,13 @@ def setUp(self): self.minion_pool_id = 'test_pool_id' self.minion_machine_id = 'test_machine_id' self.healthcheck_task_name = ( - tasks.MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id) ) self.task = tasks.MinionMachineHealtchcheckDecider( - self.minion_pool_id, self.minion_machine_id, - on_successful_healthcheck=True) + self.minion_pool_id, self.minion_machine_id, on_successful_healthcheck=True + ) def test_call_with_empty_history(self): history = {} @@ -1725,21 +1967,13 @@ def test_call_with_empty_history(self): self.assertFalse(result) def test_call_with_healthy_result(self): - history = { - self.healthcheck_task_name: { - "healthy": True - } - } + history = {self.healthcheck_task_name: {"healthy": True}} result = self.task.__call__(history) self.assertEqual(result, self.task._on_success) def test_call_with_unhealthy_result(self): - history = { - self.healthcheck_task_name: { - "healthy": False - } - } + history = {self.healthcheck_task_name: {"healthy": False}} result = self.task.__call__(history) self.assertEqual(result, not self.task._on_success) @@ -1761,140 +1995,162 @@ def setUp(self): } self.minion_pool_type = tasks.constants.PROVIDER_PLATFORM_SOURCE self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF + ) self.task = tasks.PowerOnMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.PowerOnMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_POWER_ON_DESTINATION_MINION + tasks.constants.TASK_TYPE_POWER_ON_DESTINATION_MINION, ) def test__get_task_name(self): - result = self.task._get_task_name( - self.minion_pool_id, self.minion_machine_id) + result = self.task._get_task_name(self.minion_pool_id, self.minion_machine_id) self.assertEqual( result, - tasks.MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id), ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( tasks.MinionManagerTaskEventMixin, '_set_minion_machine_power_status' ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') - def test_execute(self, mock_execute, mock_add_minion_pool_event, - mock_set_minion_machine_power_status, - mock_get_minion_machine): + def test_execute( + self, + mock_execute, + mock_add_minion_pool_event, + mock_set_minion_machine_power_status, + mock_get_minion_machine, + ): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF + ) self.task_info['minion_provider_properties'] = ( - mock_get_minion_machine.return_value.provider_properties) + mock_get_minion_machine.return_value.provider_properties + ) result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) - mock_set_minion_machine_power_status.assert_has_calls([ - mock.call( - mock.sentinel.context, self.minion_pool_id, - self.minion_machine_id, - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERING_ON - ), - mock.call( - mock.sentinel.context, self.minion_pool_id, - self.minion_machine_id, - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON) - ]) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) + mock_set_minion_machine_power_status.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERING_ON, + ), + mock.call( + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON, + ), + ] + ) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_power_on(self, mock_get_minion_machine): self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON + ) mock_get_minion_machine.return_value = self.minion_machine result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_invalid_power_status(self, mock_get_minion_machine): mock_get_minion_machine.return_value = self.minion_machine self.minion_machine.power_status = 'INVALID_POWER_STATUS' self.assertRaises( - exception.InvalidMinionMachineState, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.InvalidMinionMachineState, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) @mock.patch.object( tasks.MinionManagerTaskEventMixin, '_set_minion_machine_power_status' ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_with_exception( - self, mock_set_minion_machine_allocation, - mock_get_minion_machine, mock_add_minion_pool_event, - mock_set_minion_machine_power_status): + self, + mock_set_minion_machine_allocation, + mock_get_minion_machine, + mock_add_minion_pool_event, + mock_set_minion_machine_power_status, + ): mock_get_minion_machine.return_value = self.minion_machine - mock_set_minion_machine_power_status.side_effect = ( - exception.CoriolisException) + mock_set_minion_machine_power_status.side_effect = exception.CoriolisException self.assertRaises( - exception.CoriolisException, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.CoriolisException, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_POWER_ERROR) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_POWER_ERROR, + ) class PowerOffMinionMachineTaskTestCase(test_base.CoriolisBaseTestCase): @@ -1913,126 +2169,148 @@ def setUp(self): 'minion_provider_properties': None, } self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_ON + ) self.task = tasks.PowerOffMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) def test__init__with_different_minion_pool_type(self): self.minion_pool_type = "destination" task = tasks.PowerOffMinionMachineTask( - self.minion_pool_id, self.minion_machine_id, - self.minion_pool_type) + self.minion_pool_id, self.minion_machine_id, self.minion_pool_type + ) self.assertEqual( task._main_task_runner_type, - tasks.constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION + tasks.constants.TASK_TYPE_POWER_OFF_DESTINATION_MINION, ) def test__get_task_name(self): - result = self.task._get_task_name( - self.minion_pool_id, self.minion_machine_id) + result = self.task._get_task_name(self.minion_pool_id, self.minion_machine_id) self.assertEqual( result, - tasks.MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT % - (self.minion_pool_id, self.minion_machine_id) + tasks.MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT + % (self.minion_pool_id, self.minion_machine_id), ) @mock.patch.object( tasks.MinionManagerTaskEventMixin, '_set_minion_machine_power_status' ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) @mock.patch.object(tasks.BaseMinionManangerTask, 'execute') - def test_execute(self, mock_execute, mock_set_minion_machine_allocation, - mock_get_minion_machine, mock_add_minion_pool_event, - mock_set_minion_machine_power_status): + def test_execute( + self, + mock_execute, + mock_set_minion_machine_allocation, + mock_get_minion_machine, + mock_add_minion_pool_event, + mock_set_minion_machine_power_status, + ): mock_get_minion_machine.return_value = self.minion_machine self.task_info['minion_provider_properties'] = ( - mock_get_minion_machine.return_value.provider_properties) + mock_get_minion_machine.return_value.provider_properties + ) result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_execute.assert_called_once_with( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) - mock_set_minion_machine_power_status.assert_has_calls([ - mock.call( - mock.sentinel.context, self.minion_pool_id, - self.minion_machine_id, - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF - ), - mock.call( - mock.sentinel.context, self.minion_pool_id, - self.minion_machine_id, - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF) - ]) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) + mock_set_minion_machine_power_status.assert_has_calls( + [ + mock.call( + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERING_OFF, + ), + mock.call( + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF, + ), + ] + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_AVAILABLE) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_AVAILABLE, + ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') def test_execute_power_off(self, mock_get_minion_machine): self.minion_machine.power_status = ( - tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF) + tasks.constants.MINION_MACHINE_POWER_STATUS_POWERED_OFF + ) mock_get_minion_machine.return_value = self.minion_machine result = self.task.execute( - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) self.assertEqual(result, self.task_info) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) @mock.patch.object( tasks.MinionManagerTaskEventMixin, '_set_minion_machine_power_status' ) + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event') + @mock.patch.object(tasks.MinionManagerTaskEventMixin, '_get_minion_machine') @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_add_minion_pool_event' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, '_get_minion_machine' - ) - @mock.patch.object( - tasks.MinionManagerTaskEventMixin, - '_set_minion_machine_allocation_status' + tasks.MinionManagerTaskEventMixin, '_set_minion_machine_allocation_status' ) def test_execute_with_exception( - self, mock_set_minion_machine_allocation, - mock_get_minion_machine, mock_add_minion_pool_event, - mock_set_minion_machine_power_status): + self, + mock_set_minion_machine_allocation, + mock_get_minion_machine, + mock_add_minion_pool_event, + mock_set_minion_machine_power_status, + ): mock_get_minion_machine.return_value = self.minion_machine - mock_set_minion_machine_power_status.side_effect = ( - exception.CoriolisException) + mock_set_minion_machine_power_status.side_effect = exception.CoriolisException self.assertRaises( - exception.CoriolisException, self.task.execute, - mock.sentinel.context, mock.sentinel.origin, - mock.sentinel.destination, self.task_info) + exception.CoriolisException, + self.task.execute, + mock.sentinel.context, + mock.sentinel.origin, + mock.sentinel.destination, + self.task_info, + ) mock_get_minion_machine.assert_called_once_with( - mock.sentinel.context, self.minion_machine_id, - raise_if_not_found=True) + mock.sentinel.context, self.minion_machine_id, raise_if_not_found=True + ) mock_set_minion_machine_allocation.assert_called_once_with( - mock.sentinel.context, self.minion_pool_id, self.minion_machine_id, - tasks.constants.MINION_MACHINE_STATUS_POWER_ERROR) + mock.sentinel.context, + self.minion_pool_id, + self.minion_machine_id, + tasks.constants.MINION_MACHINE_STATUS_POWER_ERROR, + ) diff --git a/coriolis/tests/minion_pools/test_api.py b/coriolis/tests/minion_pools/test_api.py index 38247701..c2258415 100644 --- a/coriolis/tests/minion_pools/test_api.py +++ b/coriolis/tests/minion_pools/test_api.py @@ -30,33 +30,49 @@ def test_create(self): minion_retention_strategy = mock.sentinel.minion_retention_strategy result = self.api.create( - self.ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy) + self.ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + ) self.rpc_client.create_minion_pool.assert_called_once_with( - self.ctxt, name, endpoint_id, pool_platform, pool_os_type, - environment_options, minimum_minions, maximum_minions, - minion_max_idle_time, minion_retention_strategy, notes=None, - skip_allocation=False) - self.assertEqual(result, - self.rpc_client.create_minion_pool.return_value) + self.ctxt, + name, + endpoint_id, + pool_platform, + pool_os_type, + environment_options, + minimum_minions, + maximum_minions, + minion_max_idle_time, + minion_retention_strategy, + notes=None, + skip_allocation=False, + ) + self.assertEqual(result, self.rpc_client.create_minion_pool.return_value) def test_update(self): updated_values = mock.sentinel.updated_values - result = self.api.update(self.ctxt, self.minion_pool_id, - updated_values) + result = self.api.update(self.ctxt, self.minion_pool_id, updated_values) self.rpc_client.update_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id, updated_values=updated_values) - self.assertEqual(result, - self.rpc_client.update_minion_pool.return_value) + self.ctxt, self.minion_pool_id, updated_values=updated_values + ) + self.assertEqual(result, self.rpc_client.update_minion_pool.return_value) def test_delete(self): self.api.delete(self.ctxt, self.minion_pool_id) self.rpc_client.delete_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id) + self.ctxt, self.minion_pool_id + ) def test_get_minion_pools(self): result = self.api.get_minion_pools(self.ctxt) @@ -68,30 +84,32 @@ def test_get_minion_pool(self): result = self.api.get_minion_pool(self.ctxt, self.minion_pool_id) self.rpc_client.get_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id) + self.ctxt, self.minion_pool_id + ) self.assertEqual(result, self.rpc_client.get_minion_pool.return_value) def test_allocate_minion_pool(self): result = self.api.allocate_minion_pool(self.ctxt, self.minion_pool_id) self.rpc_client.allocate_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id) - self.assertEqual(result, - self.rpc_client.allocate_minion_pool.return_value) + self.ctxt, self.minion_pool_id + ) + self.assertEqual(result, self.rpc_client.allocate_minion_pool.return_value) def test_refresh_minion_pool(self): result = self.api.refresh_minion_pool(self.ctxt, self.minion_pool_id) self.rpc_client.refresh_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id) - self.assertEqual(result, - self.rpc_client.refresh_minion_pool.return_value) + self.ctxt, self.minion_pool_id + ) + self.assertEqual(result, self.rpc_client.refresh_minion_pool.return_value) def test_deallocate_minion_pool(self): result = self.api.deallocate_minion_pool( - self.ctxt, self.minion_pool_id, force=False) + self.ctxt, self.minion_pool_id, force=False + ) self.rpc_client.deallocate_minion_pool.assert_called_once_with( - self.ctxt, self.minion_pool_id, force=False) - self.assertEqual(result, - self.rpc_client.deallocate_minion_pool.return_value) + self.ctxt, self.minion_pool_id, force=False + ) + self.assertEqual(result, self.rpc_client.deallocate_minion_pool.return_value) diff --git a/coriolis/tests/osmorphing/netpreserver/test_factory.py b/coriolis/tests/osmorphing/netpreserver/test_factory.py index c4e8b711..c40ca99b 100644 --- a/coriolis/tests/osmorphing/netpreserver/test_factory.py +++ b/coriolis/tests/osmorphing/netpreserver/test_factory.py @@ -1,14 +1,17 @@ # Copyright 2025 Cloudbase Solutions Srl # All Rights Reserved. -import ddt from unittest import mock -from coriolis.osmorphing.netpreserver import factory -from coriolis.osmorphing.netpreserver import ifcfg -from coriolis.osmorphing.netpreserver import interfaces -from coriolis.osmorphing.netpreserver import netplan -from coriolis.osmorphing.netpreserver import nmconnection +import ddt + +from coriolis.osmorphing.netpreserver import ( + factory, + ifcfg, + interfaces, + netplan, + nmconnection, +) from coriolis.tests import test_base @@ -24,25 +27,38 @@ def setUp(self): (False, True, False, False, nmconnection.NmconnectionNetPreserver), (False, False, True, False, ifcfg.IfcfgNetPreserver), (False, False, False, True, interfaces.InterfacesNetPreserver), - (False, False, False, False, None) + (False, False, False, False, None), ) @ddt.unpack - def test_get_net_preserver_first_match(self, return_netplan, - return_nmconnection, return_ifcfg, - return_interfaces, - result_class): - with mock.patch.object(netplan.NetplanNetPreserver, - "check_net_preserver", - return_value=return_netplan), \ - mock.patch.object(nmconnection.NmconnectionNetPreserver, - "check_net_preserver", - return_value=return_nmconnection), \ - mock.patch.object(ifcfg.IfcfgNetPreserver, - "check_net_preserver", - return_value=return_ifcfg), \ - mock.patch.object(interfaces.InterfacesNetPreserver, - "check_net_preserver", - return_value=return_interfaces): - + def test_get_net_preserver_first_match( + self, + return_netplan, + return_nmconnection, + return_ifcfg, + return_interfaces, + result_class, + ): + with ( + mock.patch.object( + netplan.NetplanNetPreserver, + "check_net_preserver", + return_value=return_netplan, + ), + mock.patch.object( + nmconnection.NmconnectionNetPreserver, + "check_net_preserver", + return_value=return_nmconnection, + ), + mock.patch.object( + ifcfg.IfcfgNetPreserver, + "check_net_preserver", + return_value=return_ifcfg, + ), + mock.patch.object( + interfaces.InterfacesNetPreserver, + "check_net_preserver", + return_value=return_interfaces, + ), + ): res = factory.get_net_preserver(self.osmorphing_tool) self.assertEqual(res, result_class) diff --git a/coriolis/tests/osmorphing/netpreserver/test_ifcfg.py b/coriolis/tests/osmorphing/netpreserver/test_ifcfg.py index 0c7cc476..904803b6 100644 --- a/coriolis/tests/osmorphing/netpreserver/test_ifcfg.py +++ b/coriolis/tests/osmorphing/netpreserver/test_ifcfg.py @@ -3,9 +3,8 @@ from unittest import mock -from coriolis.osmorphing import base +from coriolis.osmorphing import base, redhat from coriolis.osmorphing.netpreserver import ifcfg -from coriolis.osmorphing import redhat from coriolis.tests import test_base @@ -27,71 +26,78 @@ def setUp(self): } self.netpreserver = ifcfg.IfcfgNetPreserver( redhat.BaseRedHatMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_dir, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_dir, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout)) + mock.sentinel.operation_timeout, + ) + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_config_file_sudo') @mock.patch.object(ifcfg.IfcfgNetPreserver, '_get_net_config_files') - def test_get_ifcfgs_by_type(self, mock_get_net_config_files, - mock_read_config_file_sudo): + def test_get_ifcfgs_by_type( + self, mock_get_net_config_files, mock_read_config_file_sudo + ): mock_get_net_config_files.return_value = [mock.sentinel.ifcfg_file] mock_read_config_file_sudo.side_effect = [{"TYPE": "Ethernet"}] result = self.netpreserver._get_ifcfgs_by_type( - "Ethernet", self.netpreserver.network_scripts_path) + "Ethernet", self.netpreserver.network_scripts_path + ) - mock_read_config_file_sudo.assert_called_once_with( - mock.sentinel.ifcfg_file) + mock_read_config_file_sudo.assert_called_once_with(mock.sentinel.ifcfg_file) mock_get_net_config_files.assert_called_once_with( - self.netpreserver.network_scripts_path) + self.netpreserver.network_scripts_path + ) - self.assertEqual( - result, [(mock.sentinel.ifcfg_file, {"TYPE": "Ethernet"})]) + self.assertEqual(result, [(mock.sentinel.ifcfg_file, {"TYPE": "Ethernet"})]) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test_get_net_config_files(self, mock_list_dir): mock_list_dir.return_value = ['ifcfg-eth0', 'ifcfg-lo', 'other-file'] result = self.netpreserver._get_net_config_files( - self.netpreserver.network_scripts_path) + self.netpreserver.network_scripts_path + ) expected_result = [ 'etc/sysconfig/network-scripts/ifcfg-eth0', - 'etc/sysconfig/network-scripts/ifcfg-lo' + 'etc/sysconfig/network-scripts/ifcfg-lo', ] - mock_list_dir.assert_called_once_with( - self.netpreserver.network_scripts_path) + mock_list_dir.assert_called_once_with(self.netpreserver.network_scripts_path) self.assertEqual(result, expected_result) @mock.patch.object(ifcfg.IfcfgNetPreserver, '_get_ifcfgs_by_type') @mock.patch.object(ifcfg.IfcfgNetPreserver, '_get_net_config_files') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') - def test_check_net_preserver_True(self, mock_test_path, - mock_get_net_config_files, - mock_get_ifcfgs_by_type) -> None: + def test_check_net_preserver_True( + self, mock_test_path, mock_get_net_config_files, mock_get_ifcfgs_by_type + ) -> None: mock_test_path.return_value = True mock_get_net_config_files.return_value = [ "etc/sysconfig/network-scripts/ifcfg-eth0", - "etc/sysconfig/network-scripts/ifcfg-lo" + "etc/sysconfig/network-scripts/ifcfg-lo", ] mock_get_ifcfgs_by_type.return_value = [ { "TYPE": "Ethernet", "DEVICE": "eth0", "HWADDR": "00:11:22:33:44:55", - "IPADDR": "192.168.1.10" + "IPADDR": "192.168.1.10", }, { "TYPE": "Ethernet", "HWADDR": "AA:BB:CC:DD:EE:FF", - "IPADDR": "192.168.1.11" - }] + "IPADDR": "192.168.1.11", + }, + ] result = self.netpreserver.check_net_preserver() @@ -100,13 +106,13 @@ def test_check_net_preserver_True(self, mock_test_path, @mock.patch.object(ifcfg.IfcfgNetPreserver, '_get_ifcfgs_by_type') @mock.patch.object(ifcfg.IfcfgNetPreserver, '_get_net_config_files') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') - def test_check_net_preserver_no_ifcfg_files(self, mock_test_path, - mock_get_net_config_files, - mock_get_ifcfgs_by_type): + def test_check_net_preserver_no_ifcfg_files( + self, mock_test_path, mock_get_net_config_files, mock_get_ifcfgs_by_type + ): mock_test_path.return_value = True mock_get_net_config_files.return_value = [ "etc/sysconfig/network-scripts/ifcfg-eth0", - "etc/sysconfig/network-scripts/ifcfg-lo" + "etc/sysconfig/network-scripts/ifcfg-lo", ] mock_get_ifcfgs_by_type.return_value = [] @@ -121,25 +127,25 @@ def test_parse_network(self, mock_get_ifcfgs_by_type): "TYPE": "Ethernet", "DEVICE": "eth0", "HWADDR": "00:11:22:33:44:55", - "IPADDR": "192.168.1.10" + "IPADDR": "192.168.1.10", } ifcfg_file_without_device = "etc/sysconfig/network-scripts/ifcfg-eth1" ifcfg_without_device = { "TYPE": "Ethernet", "HWADDR": "AA:BB:CC:DD:EE:FF", - "IPADDR": "192.168.1.11" + "IPADDR": "192.168.1.11", } ifcfg_file_without_hwaddr = "etc/sysconfig/network-scripts/ifcfg-eth2" ifcfg_without_hwaddr = { "TYPE": "Ethernet", "DEVICE": "eth2", - "IPADDR": "192.168.1.12" + "IPADDR": "192.168.1.12", } mock_get_ifcfgs_by_type.return_value = [ (ifcfg_file_with_device, ifcfg_with_device), (ifcfg_file_without_device, ifcfg_without_device), - (ifcfg_file_without_hwaddr, ifcfg_without_hwaddr) + (ifcfg_file_without_hwaddr, ifcfg_without_hwaddr), ] self.netpreserver.parse_network() @@ -147,16 +153,16 @@ def test_parse_network(self, mock_get_ifcfgs_by_type): expected_info = { "eth0": { "mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"] + "ip_addresses": ["192.168.1.10"], }, "eth1": { "mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": ["192.168.1.11"] + "ip_addresses": ["192.168.1.11"], }, "eth2": { "mac_address": None, "ip_addresses": ["192.168.1.12"], - } + }, } self.assertEqual(self.netpreserver.interface_info, expected_info) diff --git a/coriolis/tests/osmorphing/netpreserver/test_interfaces.py b/coriolis/tests/osmorphing/netpreserver/test_interfaces.py index 5ede0fa6..f646407a 100644 --- a/coriolis/tests/osmorphing/netpreserver/test_interfaces.py +++ b/coriolis/tests/osmorphing/netpreserver/test_interfaces.py @@ -29,7 +29,7 @@ def setUp(self): self.event_manager, self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout + mock.sentinel.operation_timeout, ) ) @@ -47,8 +47,7 @@ def test_check_net_preserver_true(self, mock_test_path, mock_list_dir): @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_check_net_preserver_false_no_file(self, mock_test_path, - mock_list_dir): + def test_check_net_preserver_false_no_file(self, mock_test_path, mock_list_dir): mock_test_path.return_value = False mock_list_dir.return_value = [] @@ -59,8 +58,9 @@ def test_check_net_preserver_false_no_file(self, mock_test_path, @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_parse_network_basic(self, mock_test_path, mock_exec_cmd_chroot, - mock_read_file_sudo): + def test_parse_network_basic( + self, mock_test_path, mock_exec_cmd_chroot, mock_read_file_sudo + ): interfaces_content = ( "iface eth0 inet static\n" "hwaddress ether 00:11:22:33:44:55\n" @@ -79,21 +79,18 @@ def test_parse_network_basic(self, mock_test_path, mock_exec_cmd_chroot, expected_info = { "eth0": { "mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"] + "ip_addresses": ["192.168.1.10"], }, - "eth1": { - "mac_address": "", - "ip_addresses": ["192.168.1.20"] - } + "eth1": {"mac_address": "", "ip_addresses": ["192.168.1.20"]}, } self.assertEqual(self.netpreserver.interface_info, expected_info) @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_parse_network_with_source(self, mock_test_path, - mock_exec_cmd_chroot, - mock_read_file_sudo): + def test_parse_network_with_source( + self, mock_test_path, mock_exec_cmd_chroot, mock_read_file_sudo + ): main_content = ( "iface eth0 inet static\n" "hwaddress ether 00:11:22:33:44:55\n" @@ -101,15 +98,13 @@ def test_parse_network_with_source(self, mock_test_path, "source /etc/network/interfaces.d/\n" ) extra_file = "extra_iface" - extra_content = ( - "iface eth1 inet dhcp\n" - "address 192.168.1.20/24\n" - ) + extra_content = "iface eth1 inet dhcp\naddress 192.168.1.20/24\n" mock_test_path.return_value = True mock_exec_cmd_chroot.return_value = extra_file mock_read_file_sudo.side_effect = ( - main_content.encode(), extra_content.encode() + main_content.encode(), + extra_content.encode(), ) self.netpreserver.parse_network() @@ -117,25 +112,20 @@ def test_parse_network_with_source(self, mock_test_path, expected_info = { "eth0": { "mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"] + "ip_addresses": ["192.168.1.10"], }, - "eth1": { - "mac_address": "", - "ip_addresses": ["192.168.1.20"] - } + "eth1": {"mac_address": "", "ip_addresses": ["192.168.1.20"]}, } self.assertEqual(self.netpreserver.interface_info, expected_info) @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_parse_network_with_trailing_hwaddress(self, mock_test_path, - mock_exec_cmd_chroot, - mock_read_file_sudo): + def test_parse_network_with_trailing_hwaddress( + self, mock_test_path, mock_exec_cmd_chroot, mock_read_file_sudo + ): - interfaces_content = ( - "hwaddress ether 00:11:22:33:44:55\n" - ) + interfaces_content = "hwaddress ether 00:11:22:33:44:55\n" mock_read_file_sudo.return_value = interfaces_content.encode() mock_test_path.return_value = True mock_exec_cmd_chroot.return_value = "" diff --git a/coriolis/tests/osmorphing/netpreserver/test_netplan.py b/coriolis/tests/osmorphing/netpreserver/test_netplan.py index 33b9fde3..706a738a 100644 --- a/coriolis/tests/osmorphing/netpreserver/test_netplan.py +++ b/coriolis/tests/osmorphing/netpreserver/test_netplan.py @@ -29,7 +29,7 @@ def setUp(self): self.event_manager, self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout + mock.sentinel.operation_timeout, ) ) @@ -37,8 +37,11 @@ def setUp(self): @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') def test_check_net_preserver_true(self, mock_test_path, mock_list_dir): mock_test_path.return_value = True - mock_list_dir.return_value = ["01-netplan.yaml", "notconfig.txt", - "02-other.yml"] + mock_list_dir.return_value = [ + "01-netplan.yaml", + "notconfig.txt", + "02-other.yml", + ] result = self.netpreserver.check_net_preserver() @@ -48,8 +51,7 @@ def test_check_net_preserver_true(self, mock_test_path, mock_list_dir): @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_check_net_preserver_false_no_file(self, mock_test_path, - mock_list_dir): + def test_check_net_preserver_false_no_file(self, mock_test_path, mock_list_dir): mock_test_path.return_value = True mock_list_dir.return_value = ["config.txt", "README.md"] @@ -59,8 +61,9 @@ def test_check_net_preserver_false_no_file(self, mock_test_path, @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') - def test_check_net_preserver_false_no_directory(self, mock_test_path, - mock_list_dir): + def test_check_net_preserver_false_no_directory( + self, mock_test_path, mock_list_dir + ): mock_test_path.return_value = False mock_list_dir.return_value = [] @@ -111,40 +114,24 @@ def test_parse_network(self, mock_list_dir, mock_read_file_sudo): expected_info = { "eth0": { "mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"] + "ip_addresses": ["192.168.1.10"], }, "eth1": { "mac_address": "", - "ip_addresses": ["192.168.1.20", "192.168.1.21"] + "ip_addresses": ["192.168.1.20", "192.168.1.21"], }, - "eth2": { - "mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": [] - }, - "eth3": { - "mac_address": "", - "ip_addresses": ["192.168.1.30"] - }, - "eth4": { - "mac_address": "", - "ip_addresses": ["192.168.1.31"] - }, - "eth5": { - "mac_address": "", - "ip_addresses": ["192.168.1.32"] - }, - "eth6": { - "mac_address": "", - "ip_addresses": [] - } + "eth2": {"mac_address": "AA:BB:CC:DD:EE:FF", "ip_addresses": []}, + "eth3": {"mac_address": "", "ip_addresses": ["192.168.1.30"]}, + "eth4": {"mac_address": "", "ip_addresses": ["192.168.1.31"]}, + "eth5": {"mac_address": "", "ip_addresses": ["192.168.1.32"]}, + "eth6": {"mac_address": "", "ip_addresses": []}, } self.assertEqual(self.netpreserver.interface_info, expected_info) @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') - def test_parse_network_invalid_file(self, mock_list_dir, - mock_read_file_sudo): + def test_parse_network_invalid_file(self, mock_list_dir, mock_read_file_sudo): mock_list_dir.return_value = ["01-netplan.yaml"] yaml_content = """ network: diff --git a/coriolis/tests/osmorphing/netpreserver/test_nmconnection.py b/coriolis/tests/osmorphing/netpreserver/test_nmconnection.py index 0948a785..794b475c 100644 --- a/coriolis/tests/osmorphing/netpreserver/test_nmconnection.py +++ b/coriolis/tests/osmorphing/netpreserver/test_nmconnection.py @@ -3,9 +3,8 @@ from unittest import mock -from coriolis.osmorphing import base +from coriolis.osmorphing import base, redhat from coriolis.osmorphing.netpreserver import nmconnection -from coriolis.osmorphing import redhat from coriolis.tests import test_base @@ -23,59 +22,68 @@ def setUp(self): } self.netpreserver = nmconnection.NmconnectionNetPreserver( redhat.BaseRedHatMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_dir, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_dir, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test_get_nmconnection_files(self, mock_list_dir): mock_list_dir.return_value = [ - 'eth0.nmconnection', 'eth1.nmconnection', 'other-file'] + 'eth0.nmconnection', + 'eth1.nmconnection', + 'other-file', + ] result = self.netpreserver._get_nmconnection_files( - self.netpreserver.nmconnection_file) + self.netpreserver.nmconnection_file + ) expected_result = [ 'etc/NetworkManager/system-connections/eth0.nmconnection', - 'etc/NetworkManager/system-connections/eth1.nmconnection' + 'etc/NetworkManager/system-connections/eth1.nmconnection', ] - mock_list_dir.assert_called_once_with( - self.netpreserver.nmconnection_file) + mock_list_dir.assert_called_once_with(self.netpreserver.nmconnection_file) self.assertEqual(result, expected_result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_config_file_sudo') - @mock.patch.object(nmconnection.NmconnectionNetPreserver, - '_get_nmconnection_files') - def test_get_keyfiles_by_type(self, mock_get_nmconnection_files, - mock_read_config_file_sudo): + @mock.patch.object(nmconnection.NmconnectionNetPreserver, '_get_nmconnection_files') + def test_get_keyfiles_by_type( + self, mock_get_nmconnection_files, mock_read_config_file_sudo + ): mock_get_nmconnection_files.return_value = [mock.sentinel.nmconn_file] mock_read_config_file_sudo.side_effect = [{"type": "ethernet"}] result = self.netpreserver._get_keyfiles_by_type( - "ethernet", self.netpreserver.nmconnection_file) + "ethernet", self.netpreserver.nmconnection_file + ) mock_get_nmconnection_files.assert_called_once_with( - self.netpreserver.nmconnection_file) - mock_read_config_file_sudo.assert_called_once_with( - mock.sentinel.nmconn_file) - self.assertEqual(result, [(mock.sentinel.nmconn_file, - {"type": "ethernet"})]) + self.netpreserver.nmconnection_file + ) + mock_read_config_file_sudo.assert_called_once_with(mock.sentinel.nmconn_file) + self.assertEqual(result, [(mock.sentinel.nmconn_file, {"type": "ethernet"})]) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') - @mock.patch.object(nmconnection.NmconnectionNetPreserver, - '_get_nmconnection_files') - @mock.patch.object(nmconnection.NmconnectionNetPreserver, - '_get_keyfiles_by_type') - def test_check_net_preserver_True(self, mock_get_keyfiles_by_type, - mock_get_nmconnection_files, - mock_test_path): + @mock.patch.object(nmconnection.NmconnectionNetPreserver, '_get_nmconnection_files') + @mock.patch.object(nmconnection.NmconnectionNetPreserver, '_get_keyfiles_by_type') + def test_check_net_preserver_True( + self, mock_get_keyfiles_by_type, mock_get_nmconnection_files, mock_test_path + ): mock_test_path.return_value = True - mock_get_nmconnection_files.return_value = ["eth0.nmconnection", - "eth1.nmconnection"] + mock_get_nmconnection_files.return_value = [ + "eth0.nmconnection", + "eth1.nmconnection", + ] mock_get_keyfiles_by_type.return_value = [ - (mock.sentinel.nmconn_file, {"type": "ethernet", - "connection": {"id": "eth0"}}) + ( + mock.sentinel.nmconn_file, + {"type": "ethernet", "connection": {"id": "eth0"}}, + ) ] result = self.netpreserver.check_net_preserver() @@ -83,10 +91,10 @@ def test_check_net_preserver_True(self, mock_get_keyfiles_by_type, self.assertTrue(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') - @mock.patch.object(nmconnection.NmconnectionNetPreserver, - '_get_nmconnection_files') - def test_check_net_preserver_no_files(self, mock_get_nmconnection_files, - mock_test_path): + @mock.patch.object(nmconnection.NmconnectionNetPreserver, '_get_nmconnection_files') + def test_check_net_preserver_no_files( + self, mock_get_nmconnection_files, mock_test_path + ): mock_test_path.return_value = True mock_get_nmconnection_files.return_value = [] @@ -94,23 +102,20 @@ def test_check_net_preserver_no_files(self, mock_get_nmconnection_files, self.assertFalse(result) - @mock.patch.object(nmconnection.NmconnectionNetPreserver, - '_get_keyfiles_by_type') + @mock.patch.object(nmconnection.NmconnectionNetPreserver, '_get_keyfiles_by_type') def test_parse_network(self, mock_get_keyfiles_by_type): - nmconn_file_with_id = ( - self.netpreserver.nmconnection_file + "/eth0.nmconnection" - ) + nmconn_file_with_id = self.netpreserver.nmconnection_file + "/eth0.nmconnection" nmconn_with_id = { "id": "eth0", "mac-address": "00:11:22:33:44:55", - "address1": "192.168.1.10/24" + "address1": "192.168.1.10/24", } nmconn_file_without_id = ( self.netpreserver.nmconnection_file + "/eth1.nmconnection" ) nmconn_without_id = { "mac-address": "AA:BB:CC:DD:EE:FF", - "Address2": "192.168.1.20/24, 192.168.1.21/24" + "Address2": "192.168.1.20/24, 192.168.1.21/24", } nmconn_file_without_mac_address = ( self.netpreserver.nmconnection_file + "/eth2.nmconnection" @@ -141,10 +146,12 @@ def test_parse_network(self, mock_get_keyfiles_by_type): (nmconn_file_with_id, nmconn_with_id), (nmconn_file_without_id, nmconn_without_id), (nmconn_file_without_mac_address, nmconn_without_mac_address), - (nmconn_file_without_mac_address_ip_address, - nmconn_without_mac_address_ip_address), + ( + nmconn_file_without_mac_address_ip_address, + nmconn_without_mac_address_ip_address, + ), (nmconn_file_with_space, nmconn_without_mac_address_and_id), - (nmconn_file_with_space, nmconn_with_interface_name) + (nmconn_file_with_space, nmconn_with_interface_name), ] self.netpreserver.interface_info = {} @@ -154,28 +161,16 @@ def test_parse_network(self, mock_get_keyfiles_by_type): expected_info = { "eth0": { "mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"] + "ip_addresses": ["192.168.1.10"], }, "eth1": { "mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": ["192.168.1.20", "192.168.1.21"] - }, - "eth2": { - "mac_address": None, - "ip_addresses": ["192.168.1.30"] - }, - "id_eth3": { - "mac_address": None, - "ip_addresses": ["192.168.1.40"] - }, - "System_ethx": { - "mac_address": None, - "ip_addresses": ["192.168.1.50"] + "ip_addresses": ["192.168.1.20", "192.168.1.21"], }, - "eth4": { - "mac_address": None, - "ip_addresses": ["192.168.1.60"] - } + "eth2": {"mac_address": None, "ip_addresses": ["192.168.1.30"]}, + "id_eth3": {"mac_address": None, "ip_addresses": ["192.168.1.40"]}, + "System_ethx": {"mac_address": None, "ip_addresses": ["192.168.1.50"]}, + "eth4": {"mac_address": None, "ip_addresses": ["192.168.1.60"]}, } self.assertEqual(expected_info, self.netpreserver.interface_info) diff --git a/coriolis/tests/osmorphing/osdetect/test_amazon.py b/coriolis/tests/osmorphing/osdetect/test_amazon.py index 87730419..c7f3bd51 100644 --- a/coriolis/tests/osmorphing/osdetect/test_amazon.py +++ b/coriolis/tests/osmorphing/osdetect/test_amazon.py @@ -4,8 +4,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import amazon -from coriolis.osmorphing.osdetect import base +from coriolis.osmorphing.osdetect import amazon, base from coriolis.tests import test_base @@ -17,19 +16,21 @@ def test_detect_os(self, mock_get_os_release): mock_get_os_release.return_value = { "ID": "amzn", "VERSION": mock.sentinel.version, - "NAME": "Amazon Linux" + "NAME": "Amazon Linux", } expected_info = { "os_type": amazon.constants.OS_TYPE_LINUX, "distribution_name": amazon.AMAZON_DISTRO_NAME, "release_version": mock.sentinel.version, - "friendly_release_name": "Amazon Linux %s" % mock.sentinel.version + "friendly_release_name": "Amazon Linux %s" % mock.sentinel.version, } amazon_os_detect_tools = amazon.AmazonLinuxOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = amazon_os_detect_tools.detect_os() diff --git a/coriolis/tests/osmorphing/osdetect/test_base.py b/coriolis/tests/osmorphing/osdetect/test_base.py index df1a3675..3f95e3b2 100644 --- a/coriolis/tests/osmorphing/osdetect/test_base.py +++ b/coriolis/tests/osmorphing/osdetect/test_base.py @@ -16,20 +16,19 @@ class BaseOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(BaseOSDetectToolsTestCase, self).setUp() self.base_os_detect_tools = base.BaseOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) def test_returned_detected_os_info_fields(self): self.assertRaises( NotImplementedError, - self.base_os_detect_tools.returned_detected_os_info_fields + self.base_os_detect_tools.returned_detected_os_info_fields, ) def test_detect_os(self): - self.assertRaises( - NotImplementedError, - self.base_os_detect_tools.detect_os - ) + self.assertRaises(NotImplementedError, self.base_os_detect_tools.detect_os) def test_set_environment(self): self.base_os_detect_tools.set_environment(mock.sentinel.environment) @@ -42,33 +41,31 @@ def test_set_environment(self): class BaseLinuxOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): """Test suite for the BaseLinuxOSDetectTools class.""" - @mock.patch.object( - base.BaseLinuxOSDetectTools, '__abstractmethods__', set() - ) + @mock.patch.object(base.BaseLinuxOSDetectTools, '__abstractmethods__', set()) def setUp(self): super(BaseLinuxOSDetectToolsTestCase, self).setUp() self.chroot_path = '/mock/chroot/path' self.os_root_dir = '/mock/os/root/dir' self.base_os_detect = base.BaseLinuxOSDetectTools( - mock.sentinel.conn, self.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, self.os_root_dir, mock.sentinel.operation_timeout + ) def test_returned_detected_os_info_fields(self): result = self.base_os_detect.returned_detected_os_info_fields() - self.assertEqual( - result, base.REQUIRED_DETECTED_OS_FIELDS - ) + self.assertEqual(result, base.REQUIRED_DETECTED_OS_FIELDS) @mock.patch.object(base.utils, 'read_ssh_file') def test__read_file(self, mock_read_ssh_file): result = self.base_os_detect._read_file(self.chroot_path) mocked_full_path = os.path.join( - self.base_os_detect._os_root_dir, self.chroot_path) + self.base_os_detect._os_root_dir, self.chroot_path + ) mock_read_ssh_file.assert_called_once_with( - self.base_os_detect._conn, mocked_full_path) + self.base_os_detect._conn, mocked_full_path + ) self.assertEqual(result, mock_read_ssh_file.return_value) @@ -77,10 +74,12 @@ def test__read_config_file(self, mock_read_ssh_ini_config): result = self.base_os_detect._read_config_file(self.chroot_path) mocked_full_path = os.path.join( - self.base_os_detect._os_root_dir, self.chroot_path) + self.base_os_detect._os_root_dir, self.chroot_path + ) mock_read_ssh_ini_config.assert_called_once_with( - self.base_os_detect._conn, mocked_full_path, check_exists=False) + self.base_os_detect._conn, mocked_full_path, check_exists=False + ) self.assertEqual(result, mock_read_ssh_ini_config.return_value) @@ -89,7 +88,8 @@ def test__get_os_release(self, mock_read_config_file): result = self.base_os_detect._get_os_release() mock_read_config_file.assert_called_once_with( - "etc/os-release", check_exists=True) + "etc/os-release", check_exists=True + ) self.assertEqual(result, mock_read_config_file.return_value) @@ -98,9 +98,11 @@ def test__test_path(self, mock_test_ssh_path): result = self.base_os_detect._test_path(self.chroot_path) mocked_full_path = os.path.join( - self.base_os_detect._os_root_dir, self.chroot_path) + self.base_os_detect._os_root_dir, self.chroot_path + ) mock_test_ssh_path.assert_called_once_with( - self.base_os_detect._conn, mocked_full_path) + self.base_os_detect._conn, mocked_full_path + ) self.assertEqual(result, mock_test_ssh_path.return_value) @@ -109,9 +111,12 @@ def test__exec_cmd(self, mock_exec_ssh_cmd): result = self.base_os_detect._exec_cmd(mock.sentinel.cmd, timeout=120) mock_exec_ssh_cmd.assert_called_once_with( - self.base_os_detect._conn, mock.sentinel.cmd, - environment=self.base_os_detect._environment, get_pty=True, - timeout=120) + self.base_os_detect._conn, + mock.sentinel.cmd, + environment=self.base_os_detect._environment, + get_pty=True, + timeout=120, + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @@ -120,9 +125,12 @@ def test__exec_cmd_without_timeout(self, mock_exec_ssh_cmd): result = self.base_os_detect._exec_cmd(mock.sentinel.cmd) mock_exec_ssh_cmd.assert_called_once_with( - self.base_os_detect._conn, mock.sentinel.cmd, - environment=self.base_os_detect._environment, get_pty=True, - timeout=self.base_os_detect._osdetect_operation_timeout) + self.base_os_detect._conn, + mock.sentinel.cmd, + environment=self.base_os_detect._environment, + get_pty=True, + timeout=self.base_os_detect._osdetect_operation_timeout, + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @@ -133,18 +141,21 @@ def test__exec_cmd_with_exception(self, mock_exec_ssh_cmd): self.assertRaises( exception.OSMorphingSSHOperationTimeout, self.base_os_detect._exec_cmd, - mock.sentinel.cmd + mock.sentinel.cmd, ) @mock.patch.object(base.utils, 'exec_ssh_cmd_chroot') def test__exec_cmd_chroot(self, mock_exec_ssh_cmd_chroot): - result = self.base_os_detect._exec_cmd_chroot( - mock.sentinel.cmd, timeout=120) + result = self.base_os_detect._exec_cmd_chroot(mock.sentinel.cmd, timeout=120) mock_exec_ssh_cmd_chroot.assert_called_once_with( - self.base_os_detect._conn, self.base_os_detect._os_root_dir, - mock.sentinel.cmd, environment=self.base_os_detect._environment, - get_pty=True, timeout=120) + self.base_os_detect._conn, + self.base_os_detect._os_root_dir, + mock.sentinel.cmd, + environment=self.base_os_detect._environment, + get_pty=True, + timeout=120, + ) self.assertEqual(result, mock_exec_ssh_cmd_chroot.return_value) @@ -153,20 +164,22 @@ def test__exec_cmd_chroot_without_timeout(self, mock_exec_ssh_cmd_chroot): result = self.base_os_detect._exec_cmd_chroot(mock.sentinel.cmd) mock_exec_ssh_cmd_chroot.assert_called_once_with( - self.base_os_detect._conn, self.base_os_detect._os_root_dir, - mock.sentinel.cmd, environment=self.base_os_detect._environment, + self.base_os_detect._conn, + self.base_os_detect._os_root_dir, + mock.sentinel.cmd, + environment=self.base_os_detect._environment, get_pty=True, - timeout=self.base_os_detect._osdetect_operation_timeout) + timeout=self.base_os_detect._osdetect_operation_timeout, + ) self.assertEqual(result, mock_exec_ssh_cmd_chroot.return_value) @mock.patch.object(base.utils, 'exec_ssh_cmd_chroot') def test__exec_cmd_chroot_with_exception(self, mock_exec_ssh_cmd_chroot): - mock_exec_ssh_cmd_chroot.side_effect = [ - exception.MinionMachineCommandTimeout] + mock_exec_ssh_cmd_chroot.side_effect = [exception.MinionMachineCommandTimeout] self.assertRaises( exception.OSMorphingSSHOperationTimeout, self.base_os_detect._exec_cmd_chroot, - mock.sentinel.cmd + mock.sentinel.cmd, ) diff --git a/coriolis/tests/osmorphing/osdetect/test_centos.py b/coriolis/tests/osmorphing/osdetect/test_centos.py index 8060c1bc..3e68325b 100644 --- a/coriolis/tests/osmorphing/osdetect/test_centos.py +++ b/coriolis/tests/osmorphing/osdetect/test_centos.py @@ -4,8 +4,7 @@ import logging from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import centos +from coriolis.osmorphing.osdetect import base, centos from coriolis.tests import test_base @@ -15,8 +14,10 @@ class CentOSOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(CentOSOSDetectToolsTestCase, self).setUp() self.centos_os_detect_tools = centos.CentOSOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) @mock.patch.object(base.BaseLinuxOSDetectTools, '_test_path') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_file') @@ -28,8 +29,8 @@ def test_detect_os(self, mock_read_file, mock_test_path): "os_type": centos.constants.OS_TYPE_LINUX, "distribution_name": centos.CENTOS_DISTRO_IDENTIFIER, "release_version": '7.9', - "friendly_release_name": "%s Version %s" % ( - centos.CENTOS_DISTRO_IDENTIFIER, '7.9') + "friendly_release_name": "%s Version %s" + % (centos.CENTOS_DISTRO_IDENTIFIER, '7.9'), } result = self.centos_os_detect_tools.detect_os() @@ -49,8 +50,8 @@ def test_detect_os_centos_stream(self, mock_read_file, mock_test_path): "os_type": centos.constants.OS_TYPE_LINUX, "distribution_name": centos.CENTOS_STREAM_DISTRO_IDENTIFIER, "release_version": '8.3', - "friendly_release_name": "%s Version %s" % ( - centos.CENTOS_STREAM_DISTRO_IDENTIFIER, '8.3') + "friendly_release_name": "%s Version %s" + % (centos.CENTOS_STREAM_DISTRO_IDENTIFIER, '8.3'), } result = self.centos_os_detect_tools.detect_os() @@ -66,8 +67,9 @@ def test_detect_os_not_centos(self, mock_read_file, mock_test_path): mock_test_path.return_value = True mock_read_file.return_value = b"dummy release 8.3" - with self.assertLogs('coriolis.osmorphing.osdetect.centos', - level=logging.DEBUG): + with self.assertLogs( + 'coriolis.osmorphing.osdetect.centos', level=logging.DEBUG + ): result = self.centos_os_detect_tools.detect_os() self.assertEqual(result, {}) diff --git a/coriolis/tests/osmorphing/osdetect/test_coreos.py b/coriolis/tests/osmorphing/osdetect/test_coreos.py index 935762dc..2be4ae51 100644 --- a/coriolis/tests/osmorphing/osdetect/test_coreos.py +++ b/coriolis/tests/osmorphing/osdetect/test_coreos.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import coreos +from coriolis.osmorphing.osdetect import base, coreos from coriolis.tests import test_base @@ -15,19 +14,21 @@ class CoreOSOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def test_detect_os(self, mock_get_os_release): mock_get_os_release.return_value = { "ID": "coreos", - "VERSION_ID": mock.sentinel.version + "VERSION_ID": mock.sentinel.version, } expected_info = { "os_type": coreos.constants.OS_TYPE_LINUX, "distribution_name": coreos.COREOS_DISTRO_IDENTIFIER, "release_version": mock.sentinel.version, - "friendly_release_name": "CoreOS Linux %s" % mock.sentinel.version + "friendly_release_name": "CoreOS Linux %s" % mock.sentinel.version, } coreos_os_detect_tools = coreos.CoreOSOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = coreos_os_detect_tools.detect_os() diff --git a/coriolis/tests/osmorphing/osdetect/test_debian.py b/coriolis/tests/osmorphing/osdetect/test_debian.py index c1f42fcf..18544c7a 100644 --- a/coriolis/tests/osmorphing/osdetect/test_debian.py +++ b/coriolis/tests/osmorphing/osdetect/test_debian.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import debian +from coriolis.osmorphing.osdetect import base, debian from coriolis.tests import test_base @@ -14,25 +13,26 @@ class DebianOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(DebianOSDetectToolsTestCase, self).setUp() self.debian_os_detect_tools = debian.DebianOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) @mock.patch.object(base.BaseLinuxOSDetectTools, '_test_path') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_config_file') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_file') - def test_detect_os(self, mock_read_file, mock_read_config_file, - mock_test_path): + def test_detect_os(self, mock_read_file, mock_read_config_file, mock_test_path): mock_test_path.return_value = True mock_read_config_file.return_value = { 'DISTRIB_ID': 'Debian', - 'DISTRIB_RELEASE': '10' + 'DISTRIB_RELEASE': '10', } expected_info = { "os_type": debian.constants.OS_TYPE_LINUX, "distribution_name": debian.DEBIAN_DISTRO_IDENTIFIER, "release_version": '10', - "friendly_release_name": "Debian Linux 10" + "friendly_release_name": "Debian Linux 10", } result = self.debian_os_detect_tools.detect_os() @@ -47,7 +47,8 @@ def test_detect_os(self, mock_read_file, mock_read_config_file, @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_config_file') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_file') def test_detect_os_debian_version( - self, mock_read_file, mock_read_config_file, mock_test_path): + self, mock_read_file, mock_read_config_file, mock_test_path + ): mock_test_path.side_effect = [False, True] mock_read_file.return_value = b"10\n" @@ -55,16 +56,15 @@ def test_detect_os_debian_version( "os_type": debian.constants.OS_TYPE_LINUX, "distribution_name": debian.DEBIAN_DISTRO_IDENTIFIER, "release_version": '10', - "friendly_release_name": "Debian Linux 10" + "friendly_release_name": "Debian Linux 10", } result = self.debian_os_detect_tools.detect_os() mock_read_config_file.assert_not_called() - mock_test_path.assert_has_calls([ - mock.call("etc/lsb-release"), - mock.call("etc/debian_version") - ]) + mock_test_path.assert_has_calls( + [mock.call("etc/lsb-release"), mock.call("etc/debian_version")] + ) mock_read_file.assert_called_once_with("etc/debian_version") self.assertEqual(result, expected_info) @@ -72,17 +72,17 @@ def test_detect_os_debian_version( @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_config_file') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_file') @mock.patch.object(base.BaseLinuxOSDetectTools, '_test_path') - def test_detect_os_no_release(self, mock_test_path, mock_read_file, - mock_read_config_file): + def test_detect_os_no_release( + self, mock_test_path, mock_read_file, mock_read_config_file + ): mock_test_path.return_value = False result = self.debian_os_detect_tools.detect_os() self.assertEqual(result, {}) - mock_test_path.assert_has_calls([ - mock.call("etc/lsb-release"), - mock.call("etc/debian_version") - ]) + mock_test_path.assert_has_calls( + [mock.call("etc/lsb-release"), mock.call("etc/debian_version")] + ) mock_read_file.assert_not_called() mock_read_config_file.assert_not_called() diff --git a/coriolis/tests/osmorphing/osdetect/test_manager.py b/coriolis/tests/osmorphing/osdetect/test_manager.py index c8cdc22a..1b6497dc 100644 --- a/coriolis/tests/osmorphing/osdetect/test_manager.py +++ b/coriolis/tests/osmorphing/osdetect/test_manager.py @@ -4,8 +4,7 @@ from unittest import mock from coriolis import exception -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import manager +from coriolis.osmorphing.osdetect import base, manager from coriolis.tests import test_base @@ -14,10 +13,7 @@ def returned_detected_os_info_fields(self): return ['os_type', 'os_version'] def detect_os(self): - return { - 'os_type': 'mock_os_type', - 'os_version': 'mock_os_version' - } + return {'os_type': 'mock_os_type', 'os_version': 'mock_os_version'} class MockOSDetectToolNoInfo(base.BaseOSDetectTools): @@ -33,10 +29,7 @@ def returned_detected_os_info_fields(self): return ['os_type', 'os_version', 'extra_field'] def detect_os(self): - return { - 'os_type': 'mock_os_type', - 'os_version': 'mock_os_version' - } + return {'os_type': 'mock_os_type', 'os_version': 'mock_os_version'} class MockOSDetectToolExtraFields(MockOSDetectTool): @@ -47,7 +40,7 @@ def detect_os(self): return { 'os_type': 'mock_os_type', 'os_version': 'mock_os_version', - 'extra_field': 'extra_value' + 'extra_field': 'extra_value', } @@ -69,29 +62,35 @@ def test__check_custom_os_detect_tools(self): self.assertTrue(result) def test_check_custom_os_detect_tools_not_list(self): - self.assertRaises(exception.InvalidCustomOSDetectTools, - manager._check_custom_os_detect_tools, "not a list") + self.assertRaises( + exception.InvalidCustomOSDetectTools, + manager._check_custom_os_detect_tools, + "not a list", + ) def test_check_custom_os_detect_tools_invalid_type(self): - self.assertRaises(exception.InvalidCustomOSDetectTools, - manager._check_custom_os_detect_tools, [object()]) + self.assertRaises( + exception.InvalidCustomOSDetectTools, + manager._check_custom_os_detect_tools, + [object()], + ) @mock.patch.object(manager, '_check_custom_os_detect_tools') def test_detect_os_custom_tools(self, mock_check_custom_tools): mock_os_detect_tool = MockOSDetectTool result = manager.detect_os( - self.conn, self.os_type, self.os_root_dir, + self.conn, + self.os_type, + self.os_root_dir, mock.sentinel.operation_timeout, tools_environment=mock.sentinel.tools_environment, - custom_os_detect_tools=[mock_os_detect_tool]) + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) - expected_result = { - 'os_type': 'mock_os_type', - 'os_version': 'mock_os_version' - } + expected_result = {'os_type': 'mock_os_type', 'os_version': 'mock_os_version'} self.assertEqual(result, expected_result) @@ -101,24 +100,29 @@ def test_detect_os_windows(self, mock_check_custom_tools): self.os_type = 'windows' result = manager.detect_os( - self.conn, self.os_type, self.os_root_dir, + self.conn, + self.os_type, + self.os_root_dir, mock.sentinel.operation_timeout, - custom_os_detect_tools=[mock_os_detect_tool]) + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) - expected_result = { - 'os_type': 'mock_os_type', - 'os_version': 'mock_os_version' - } + expected_result = {'os_type': 'mock_os_type', 'os_version': 'mock_os_version'} self.assertEqual(result, expected_result) def test_detect_os_invalid_os_type(self): self.os_type = 'invalid_os_type' - self.assertRaises(exception.OSDetectToolsNotFound, manager.detect_os, - self.conn, self.os_type, self.os_root_dir, - mock.sentinel.operation_timeout) + self.assertRaises( + exception.OSDetectToolsNotFound, + manager.detect_os, + self.conn, + self.os_type, + self.os_root_dir, + mock.sentinel.operation_timeout, + ) @mock.patch.object(manager, '_check_custom_os_detect_tools') @mock.patch.object(manager.rocky.RockyLinuxOSDetectTools, 'detect_os') @@ -126,19 +130,28 @@ def test_detect_os_invalid_os_type(self): @mock.patch.object(manager.centos.CentOSOSDetectTools, 'detect_os') @mock.patch.object(manager.oracle.OracleOSDetectTools, 'detect_os') def test_detect_os_no_detected_info( - self, mock_oracle_detect_os, mock_centos_detect_os, - mock_redhat_detect_os, mock_rocky_detect_os, - mock_check_custom_tools): + self, + mock_oracle_detect_os, + mock_centos_detect_os, + mock_redhat_detect_os, + mock_rocky_detect_os, + mock_check_custom_tools, + ): mock_rocky_detect_os.return_value = None mock_redhat_detect_os.return_value = None mock_centos_detect_os.return_value = None mock_oracle_detect_os.return_value = None mock_os_detect_tool = MockOSDetectToolNoInfo - self.assertRaises(exception.OSDetectToolsNotFound, manager.detect_os, - self.conn, self.os_type, self.os_root_dir, - mock.sentinel.operation_timeout, - custom_os_detect_tools=[mock_os_detect_tool]) + self.assertRaises( + exception.OSDetectToolsNotFound, + manager.detect_os, + self.conn, + self.os_type, + self.os_root_dir, + mock.sentinel.operation_timeout, + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) @mock.patch.object(manager, '_check_custom_os_detect_tools') @@ -147,19 +160,28 @@ def test_detect_os_no_detected_info( @mock.patch.object(manager.centos.CentOSOSDetectTools, 'detect_os') @mock.patch.object(manager.oracle.OracleOSDetectTools, 'detect_os') def test_detect_os_invalid_detected_info( - self, mock_oracle_detect_os, mock_centos_detect_os, - mock_redhat_detect_os, mock_rocky_detect_os, - mock_check_custom_tools): + self, + mock_oracle_detect_os, + mock_centos_detect_os, + mock_redhat_detect_os, + mock_rocky_detect_os, + mock_check_custom_tools, + ): mock_rocky_detect_os.return_value = None mock_redhat_detect_os.return_value = None mock_centos_detect_os.return_value = None mock_oracle_detect_os.return_value = "invalid_detected_info" mock_os_detect_tool = MockOSDetectToolNoInfo - self.assertRaises(exception.InvalidDetectedOSParams, manager.detect_os, - self.conn, self.os_type, self.os_root_dir, - mock.sentinel.operation_timeout, - custom_os_detect_tools=[mock_os_detect_tool]) + self.assertRaises( + exception.InvalidDetectedOSParams, + manager.detect_os, + self.conn, + self.os_type, + self.os_root_dir, + mock.sentinel.operation_timeout, + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) @mock.patch.object(manager, '_check_custom_os_detect_tools') @@ -168,19 +190,28 @@ def test_detect_os_invalid_detected_info( @mock.patch.object(manager.centos.CentOSOSDetectTools, 'detect_os') @mock.patch.object(manager.oracle.OracleOSDetectTools, 'detect_os') def test_detect_os_missing_detected_info_fields( - self, mock_oracle_detect_os, mock_centos_detect_os, - mock_redhat_detect_os, mock_rocky_detect_os, - mock_check_custom_tools): + self, + mock_oracle_detect_os, + mock_centos_detect_os, + mock_redhat_detect_os, + mock_rocky_detect_os, + mock_check_custom_tools, + ): mock_rocky_detect_os.return_value = None mock_redhat_detect_os.return_value = None mock_centos_detect_os.return_value = None mock_oracle_detect_os.return_value = None mock_os_detect_tool = MockOSDetectToolMissingFields - self.assertRaises(exception.InvalidDetectedOSParams, manager.detect_os, - self.conn, self.os_type, self.os_root_dir, - mock.sentinel.operation_timeout, - custom_os_detect_tools=[mock_os_detect_tool]) + self.assertRaises( + exception.InvalidDetectedOSParams, + manager.detect_os, + self.conn, + self.os_type, + self.os_root_dir, + mock.sentinel.operation_timeout, + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) @mock.patch.object(manager, '_check_custom_os_detect_tools') @@ -189,17 +220,26 @@ def test_detect_os_missing_detected_info_fields( @mock.patch.object(manager.centos.CentOSOSDetectTools, 'detect_os') @mock.patch.object(manager.oracle.OracleOSDetectTools, 'detect_os') def test_detect_os_extra_detected_info_fields( - self, mock_oracle_detect_os, mock_centos_detect_os, - mock_redhat_detect_os, mock_rocky_detect_os, - mock_check_custom_tools): + self, + mock_oracle_detect_os, + mock_centos_detect_os, + mock_redhat_detect_os, + mock_rocky_detect_os, + mock_check_custom_tools, + ): mock_rocky_detect_os.return_value = None mock_redhat_detect_os.return_value = None mock_centos_detect_os.return_value = None mock_oracle_detect_os.return_value = None mock_os_detect_tool = MockOSDetectToolExtraFields - self.assertRaises(exception.InvalidDetectedOSParams, manager.detect_os, - self.conn, self.os_type, self.os_root_dir, - mock.sentinel.operation_timeout, - custom_os_detect_tools=[mock_os_detect_tool]) + self.assertRaises( + exception.InvalidDetectedOSParams, + manager.detect_os, + self.conn, + self.os_type, + self.os_root_dir, + mock.sentinel.operation_timeout, + custom_os_detect_tools=[mock_os_detect_tool], + ) mock_check_custom_tools.assert_called_once_with([mock_os_detect_tool]) diff --git a/coriolis/tests/osmorphing/osdetect/test_openwrt.py b/coriolis/tests/osmorphing/osdetect/test_openwrt.py index 788eda3e..f58ac009 100644 --- a/coriolis/tests/osmorphing/osdetect/test_openwrt.py +++ b/coriolis/tests/osmorphing/osdetect/test_openwrt.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import openwrt +from coriolis.osmorphing.osdetect import base, openwrt from coriolis.tests import test_base @@ -16,24 +15,27 @@ def test_detect_os(self, mock_read_config_file): mock_read_config_file.return_value = { "DISTRIB_ID": "OpenWrt", "DISTRIB_DESCRIPTION": "OpenWrt Description", - "DISTRIB_RELEASE": mock.sentinel.version + "DISTRIB_RELEASE": mock.sentinel.version, } expected_info = { "os_type": openwrt.constants.OS_TYPE_LINUX, "distribution_name": openwrt.OPENWRT_DISTRO_IDENTIFIER, "release_version": mock.sentinel.version, - "friendly_release_name": "OpenWrt Description Version %s" % ( - mock.sentinel.version) + "friendly_release_name": "OpenWrt Description Version %s" + % (mock.sentinel.version), } openwrt_os_detect_tools = openwrt.OpenWRTOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = openwrt_os_detect_tools.detect_os() mock_read_config_file.assert_called_once_with( - "etc/openwrt_release", check_exists=True) + "etc/openwrt_release", check_exists=True + ) self.assertEqual(result, expected_info) diff --git a/coriolis/tests/osmorphing/osdetect/test_oracle.py b/coriolis/tests/osmorphing/osdetect/test_oracle.py index 9386057a..063e14b3 100644 --- a/coriolis/tests/osmorphing/osdetect/test_oracle.py +++ b/coriolis/tests/osmorphing/osdetect/test_oracle.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import oracle +from coriolis.osmorphing.osdetect import base, oracle from coriolis.tests import test_base @@ -21,12 +20,14 @@ def test_detect_os(self, mock_read_file, mock_test_path): "os_type": oracle.constants.OS_TYPE_LINUX, "distribution_name": oracle.ORACLE_DISTRO_IDENTIFIER, "release_version": '8.4', - "friendly_release_name": "Oracle Linux Version 8.4" + "friendly_release_name": "Oracle Linux Version 8.4", } oracle_os_detect_tools = oracle.OracleOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = oracle_os_detect_tools.detect_os() mock_test_path.assert_called_once_with("etc/oracle-release") diff --git a/coriolis/tests/osmorphing/osdetect/test_redhat.py b/coriolis/tests/osmorphing/osdetect/test_redhat.py index 350aaa6a..133eabd3 100644 --- a/coriolis/tests/osmorphing/osdetect/test_redhat.py +++ b/coriolis/tests/osmorphing/osdetect/test_redhat.py @@ -4,8 +4,7 @@ import logging from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import redhat +from coriolis.osmorphing.osdetect import base, redhat from coriolis.tests import test_base @@ -15,22 +14,23 @@ class RedHatOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(RedHatOSDetectToolsTestCase, self).setUp() self.redhat_os_detect_tools = redhat.RedHatOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) @mock.patch.object(base.BaseLinuxOSDetectTools, '_test_path') @mock.patch.object(base.BaseLinuxOSDetectTools, '_read_file') def test_detect_os(self, mock_read_file, mock_test_path): mock_test_path.return_value = True - mock_read_file.return_value = ( - b"Red Hat Enterprise Linux release 8.4 (Ootpa)") + mock_read_file.return_value = b"Red Hat Enterprise Linux release 8.4 (Ootpa)" expected_info = { "os_type": redhat.constants.OS_TYPE_LINUX, "distribution_name": redhat.RED_HAT_DISTRO_IDENTIFIER, "release_version": '8.4', - "friendly_release_name": "%s Version %s" % ( - redhat.RED_HAT_DISTRO_IDENTIFIER, '8.4') + "friendly_release_name": "%s Version %s" + % (redhat.RED_HAT_DISTRO_IDENTIFIER, '8.4'), } result = self.redhat_os_detect_tools.detect_os() @@ -46,8 +46,9 @@ def test_detect_os_no_redhat(self, mock_read_file, mock_test_path): mock_test_path.return_value = True mock_read_file.return_value = b"CentOS Linux release 8.4 (Ootpa)" - with self.assertLogs('coriolis.osmorphing.osdetect.redhat', - level=logging.DEBUG): + with self.assertLogs( + 'coriolis.osmorphing.osdetect.redhat', level=logging.DEBUG + ): result = self.redhat_os_detect_tools.detect_os() self.assertEqual(result, {}) diff --git a/coriolis/tests/osmorphing/osdetect/test_rocky.py b/coriolis/tests/osmorphing/osdetect/test_rocky.py index b5942756..074fa9dd 100644 --- a/coriolis/tests/osmorphing/osdetect/test_rocky.py +++ b/coriolis/tests/osmorphing/osdetect/test_rocky.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import rocky +from coriolis.osmorphing.osdetect import base, rocky from coriolis.tests import test_base @@ -21,12 +20,14 @@ def test_detect_os(self, mock_read_file, mock_test_path): "os_type": rocky.constants.OS_TYPE_LINUX, "distribution_name": rocky.ROCKY_LINUX_DISTRO_IDENTIFIER, "release_version": '8.4', - "friendly_release_name": "Rocky Linux Version 8.4" + "friendly_release_name": "Rocky Linux Version 8.4", } rocky_os_detect_tools = rocky.RockyLinuxOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = rocky_os_detect_tools.detect_os() mock_test_path.assert_called_once_with("etc/redhat-release") @@ -40,11 +41,12 @@ def test_detect_os_no_rocky(self, mock_read_file, mock_test_path): mock_test_path.return_value = True mock_read_file.return_value = b"CentOS Linux release 8.4" - with self.assertLogs('coriolis.osmorphing.osdetect.rocky', - level="DEBUG"): + with self.assertLogs('coriolis.osmorphing.osdetect.rocky', level="DEBUG"): rocky_os_detect_tools = rocky.RockyLinuxOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = rocky_os_detect_tools.detect_os() self.assertEqual(result, {}) diff --git a/coriolis/tests/osmorphing/osdetect/test_suse.py b/coriolis/tests/osmorphing/osdetect/test_suse.py index 1d19871d..3e640409 100644 --- a/coriolis/tests/osmorphing/osdetect/test_suse.py +++ b/coriolis/tests/osmorphing/osdetect/test_suse.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import suse +from coriolis.osmorphing.osdetect import base, suse from coriolis.tests import test_base @@ -15,20 +14,22 @@ def setUp(self): super(SUSEOSDetectToolsTestCase, self).setUp() self.suse_os_detect_tools = suse.SUSEOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) - @mock.patch.object( - base.BaseLinuxOSDetectTools, 'returned_detected_os_info_fields') - def test_returned_detected_os_info_fields(self, - mock_detected_os_info_fields): - mock_detected_os_info_fields.return_value = [ - "os_type", "distribution_name"] + @mock.patch.object(base.BaseLinuxOSDetectTools, 'returned_detected_os_info_fields') + def test_returned_detected_os_info_fields(self, mock_detected_os_info_fields): + mock_detected_os_info_fields.return_value = ["os_type", "distribution_name"] result = suse.SUSEOSDetectTools.returned_detected_os_info_fields() - expected_fields = ["os_type", "distribution_name", - suse.DETECTED_SUSE_RELEASE_FIELD_NAME] + expected_fields = [ + "os_type", + "distribution_name", + suse.DETECTED_SUSE_RELEASE_FIELD_NAME, + ] self.assertEqual(result, expected_fields) @@ -36,7 +37,7 @@ def test_returned_detected_os_info_fields(self, def test_detect_os_sles(self, mock_get_os_release): mock_get_os_release.return_value = { "NAME": "SLES", - "VERSION_ID": suse.constants.OS_TYPE_UNKNOWN + "VERSION_ID": suse.constants.OS_TYPE_UNKNOWN, } expected_info = { @@ -44,7 +45,7 @@ def test_detect_os_sles(self, mock_get_os_release): "distribution_name": suse.SLES_DISTRO_IDENTIFIER, suse.DETECTED_SUSE_RELEASE_FIELD_NAME: "SLES", "release_version": suse.constants.OS_TYPE_UNKNOWN, - "friendly_release_name": "SLES %s" % suse.constants.OS_TYPE_UNKNOWN + "friendly_release_name": "SLES %s" % suse.constants.OS_TYPE_UNKNOWN, } result = self.suse_os_detect_tools.detect_os() @@ -57,7 +58,7 @@ def test_detect_os_sles(self, mock_get_os_release): def test_detect_os_opensuse_tumbleweed(self, mock_get_os_release): mock_get_os_release.return_value = { "NAME": "openSUSE tumbleweed", - "VERSION_ID": suse.constants.OS_TYPE_UNKNOWN + "VERSION_ID": suse.constants.OS_TYPE_UNKNOWN, } expected_info = { @@ -65,7 +66,7 @@ def test_detect_os_opensuse_tumbleweed(self, mock_get_os_release): "distribution_name": suse.OPENSUSE_DISTRO_IDENTIFIER, suse.DETECTED_SUSE_RELEASE_FIELD_NAME: "openSUSE tumbleweed", "release_version": suse.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER, - "friendly_release_name": "openSUSE tumbleweed" + "friendly_release_name": "openSUSE tumbleweed", } result = self.suse_os_detect_tools.detect_os() @@ -78,7 +79,7 @@ def test_detect_os_opensuse_tumbleweed(self, mock_get_os_release): def test_detect_os_opensuse(self, mock_get_os_release): mock_get_os_release.return_value = { "NAME": "openSUSE test", - "VERSION_ID": "15.3" + "VERSION_ID": "15.3", } expected_info = { @@ -86,7 +87,7 @@ def test_detect_os_opensuse(self, mock_get_os_release): "distribution_name": suse.OPENSUSE_DISTRO_IDENTIFIER, suse.DETECTED_SUSE_RELEASE_FIELD_NAME: "openSUSE test", "release_version": "15.3", - "friendly_release_name": "openSUSE 15.3" + "friendly_release_name": "openSUSE 15.3", } result = self.suse_os_detect_tools.detect_os() diff --git a/coriolis/tests/osmorphing/osdetect/test_ubuntu.py b/coriolis/tests/osmorphing/osdetect/test_ubuntu.py index b9c1085c..d4a7d420 100644 --- a/coriolis/tests/osmorphing/osdetect/test_ubuntu.py +++ b/coriolis/tests/osmorphing/osdetect/test_ubuntu.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis.osmorphing.osdetect import base -from coriolis.osmorphing.osdetect import ubuntu +from coriolis.osmorphing.osdetect import base, ubuntu from coriolis.tests import test_base @@ -15,24 +14,26 @@ class UbuntuOSDetectToolsTestCase(test_base.CoriolisBaseTestCase): def test_detect_os(self, mock_read_config_file): mock_read_config_file.return_value = { "DISTRIB_ID": "Ubuntu", - "DISTRIB_RELEASE": mock.sentinel.version + "DISTRIB_RELEASE": mock.sentinel.version, } expected_info = { "os_type": ubuntu.constants.OS_TYPE_LINUX, "distribution_name": ubuntu.UBUNTU_DISTRO_IDENTIFIER, "release_version": mock.sentinel.version, - "friendly_release_name": "Ubuntu %s" % ( - mock.sentinel.version) + "friendly_release_name": "Ubuntu %s" % (mock.sentinel.version), } ubuntu_os_detect_tools = ubuntu.UbuntuOSDetectTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.operation_timeout) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.operation_timeout, + ) result = ubuntu_os_detect_tools.detect_os() mock_read_config_file.assert_called_once_with( - "etc/lsb-release", check_exists=True) + "etc/lsb-release", check_exists=True + ) self.assertEqual(result, expected_info) diff --git a/coriolis/tests/osmorphing/osdetect/test_windows.py b/coriolis/tests/osmorphing/osdetect/test_windows.py index 1c20c9fa..6da253f2 100644 --- a/coriolis/tests/osmorphing/osdetect/test_windows.py +++ b/coriolis/tests/osmorphing/osdetect/test_windows.py @@ -11,7 +11,6 @@ from coriolis.osmorphing.osdetect import windows from coriolis.tests import test_base - WIN_VERSION_PS_OUTPUT = """ CurrentVersion : 6.3 CurrentMajorVersionNumber : 10 @@ -51,7 +50,8 @@ def setUp(self): self.conn.EOL = '\n' self.windows_os_detect_tools = windows.WindowsOSDetectTools( - self.conn, self.os_root_dir, mock.sentinel.operation_timeout) + self.conn, self.os_root_dir, mock.sentinel.operation_timeout + ) def test_returned_detected_os_info_fields(self): expected_base_fields = [ @@ -62,32 +62,33 @@ def test_returned_detected_os_info_fields(self): "version_number", "edition_id", "installation_type", - "product_name" + "product_name", ] - result = ( - windows.WindowsOSDetectTools.returned_detected_os_info_fields() - ) + result = windows.WindowsOSDetectTools.returned_detected_os_info_fields() self.assertEqual(result, expected_base_fields) def test__load_registry_hive(self): self.windows_os_detect_tools._load_registry_hive( - mock.sentinel.subkey, mock.sentinel.path) + mock.sentinel.subkey, mock.sentinel.path + ) self.conn.exec_command.assert_called_once_with( - "reg.exe", ["load", mock.sentinel.subkey, mock.sentinel.path]) + "reg.exe", ["load", mock.sentinel.subkey, mock.sentinel.path] + ) def test__unload_registry_hive(self): - self.windows_os_detect_tools._unload_registry_hive( - mock.sentinel.subkey) + self.windows_os_detect_tools._unload_registry_hive(mock.sentinel.subkey) self.conn.exec_command.assert_called_once_with( - "reg.exe", ["unload", mock.sentinel.subkey]) + "reg.exe", ["unload", mock.sentinel.subkey] + ) def test__get_ps_fl_value(self): result = self.windows_os_detect_tools._get_ps_fl_value( - WIN_VERSION_PS_OUTPUT, 'CurrentVersion') + WIN_VERSION_PS_OUTPUT, 'CurrentVersion' + ) self.assertEqual(result, '6.3') @@ -98,7 +99,7 @@ def test__get_ps_fl_value(self): windows.version.LooseVersion("10.0.20348"), "ServerDatacenterEval", "Server", - "Windows Server 2022 Datacenter Evaluation" + "Windows Server 2022 Datacenter Evaluation", ), }, { @@ -107,45 +108,48 @@ def test__get_ps_fl_value(self): "6.3.20348", "ServerDatacenterEval", "Server", - "Windows Server 2022 Datacenter Evaluation" - ) - } + "Windows Server 2022 Datacenter Evaluation", + ), + }, ) @mock.patch.object(windows.WindowsOSDetectTools, '_load_registry_hive') @mock.patch.object(windows.WindowsOSDetectTools, '_unload_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') def test__get_image_version_info( - self, data, mock_uuid4, mock_unload_registry_hive, - mock_load_registry_hive): - self.conn.exec_ps_command.return_value = ( - data["ps_output"].replace('\n', os.linesep) + self, data, mock_uuid4, mock_unload_registry_hive, mock_load_registry_hive + ): + self.conn.exec_ps_command.return_value = data["ps_output"].replace( + '\n', os.linesep ) result = self.windows_os_detect_tools._get_image_version_info() mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SOFTWARE" % self.os_root_dir) + "%sWindows\\System32\\config\\SOFTWARE" % self.os_root_dir, + ) mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) self.assertEqual(result, data["expected_result"]) @mock.patch.object(windows.WindowsOSDetectTools, '_load_registry_hive') @mock.patch.object(windows.WindowsOSDetectTools, '_unload_registry_hive') def test__get_image_version_info_with_exception( - self, mock_unload_registry_hive, - mock_load_registry_hive): + self, mock_unload_registry_hive, mock_load_registry_hive + ): self.conn.exec_ps_command.return_value = ( - WIN_VERSION_PS_OUTPUT_MISSING_FIELDS.replace('\n', os.linesep)) + WIN_VERSION_PS_OUTPUT_MISSING_FIELDS.replace('\n', os.linesep) + ) mock_unload_registry_hive.assert_not_called() mock_load_registry_hive.assert_not_called() self.assertRaises( exception.CoriolisException, - self.windows_os_detect_tools._get_image_version_info + self.windows_os_detect_tools._get_image_version_info, ) @ddt.data( @@ -162,7 +166,7 @@ def test__get_image_version_info_with_exception( 'installation_type': mock.sentinel.installation_type, 'product_name': mock.sentinel.product_name, 'distribution_name': windows.WINDOWS_CLIENT_IDENTIFIER, - } + }, ) @mock.patch.object(windows.WindowsOSDetectTools, '_get_image_version_info') def test_detect_os(self, data, mock_get_image_version_info): @@ -170,7 +174,7 @@ def test_detect_os(self, data, mock_get_image_version_info): data['version_number'], data['edition_id'], data['installation_type'], - data['product_name'] + data['product_name'], ) expected_result = { @@ -181,11 +185,12 @@ def test_detect_os(self, data, mock_get_image_version_info): "os_type": windows.constants.OS_TYPE_WINDOWS, "distribution_name": data['distribution_name'], "release_version": data['product_name'], - "friendly_release_name": "Windows %s" % data['product_name'] + "friendly_release_name": "Windows %s" % data['product_name'], } - with self.assertLogs('coriolis.osmorphing.osdetect.windows', - level=logging.DEBUG): + with self.assertLogs( + 'coriolis.osmorphing.osdetect.windows', level=logging.DEBUG + ): result = self.windows_os_detect_tools.detect_os() self.assertEqual(result, expected_result) @@ -194,9 +199,11 @@ def test_detect_os(self, data, mock_get_image_version_info): def test_detect_os_with_exception(self, mock_get_image_version_info): mock_get_image_version_info.side_effect = exception.CoriolisException - with self.assertLogs('coriolis.osmorphing.osdetect.windows', - level=logging.DEBUG): - self.assertRaises(exception.CoriolisException, - self.windows_os_detect_tools.detect_os) + with self.assertLogs( + 'coriolis.osmorphing.osdetect.windows', level=logging.DEBUG + ): + self.assertRaises( + exception.CoriolisException, self.windows_os_detect_tools.detect_os + ) mock_get_image_version_info.assert_called_once_with() diff --git a/coriolis/tests/osmorphing/osmount/test_base.py b/coriolis/tests/osmorphing/osmount/test_base.py index e4068439..76eea85d 100644 --- a/coriolis/tests/osmorphing/osmount/test_base.py +++ b/coriolis/tests/osmorphing/osmount/test_base.py @@ -6,8 +6,7 @@ from coriolis import exception from coriolis.osmorphing.osmount import base -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class CoriolisTestException(Exception): @@ -22,8 +21,11 @@ def setUp(self): super(BaseOSMountToolsTestCase, self).setUp() self.event_manager = mock.MagicMock() self.base_os_mount_tools = base.BaseOSMountTools( - mock.sentinel.conn, self.event_manager, - mock.sentinel.ignore_devices, mock.sentinel.operation_timeout) + mock.sentinel.conn, + self.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) def test_get_environment(self): result = self.base_os_mount_tools.get_environment() @@ -57,12 +59,15 @@ def setUp(self, mock_connect): "ip": "127.0.0.1", "username": "random_username", "password": "random_password", - "pkey": "random_pkey" + "pkey": "random_pkey", } self.base_os_mount_tools = TestBaseSSHOSMountTools( - self.conn_info, self.event_manager, - mock.sentinel.ignore_devices, mock.sentinel.operation_timeout) + self.conn_info, + self.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) self.base_os_mount_tools._ssh = self.ssh @@ -72,32 +77,36 @@ def setUp(self, mock_connect): @mock.patch.object(base.utils, 'wait_for_port_connectivity') def test__connect(self, mock_wait_for_port_connectivity, mock_ssh_client): base_os_mount_tools = TestBaseSSHOSMountTools( - self.conn_info, self.event_manager, - mock.sentinel.ignore_devices, mock.sentinel.operation_timeout) + self.conn_info, + self.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) mock_ssh_client.return_value = self.ssh - original_connect = testutils.get_wrapped_function( - base_os_mount_tools._connect) + original_connect = testutils.get_wrapped_function(base_os_mount_tools._connect) - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.INFO): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.INFO): original_connect(base_os_mount_tools) - mock_wait_for_port_connectivity.assert_has_calls([ - mock.call(self.conn_info['ip'], 22), - mock.call(self.conn_info['ip'], 22), - ]) + mock_wait_for_port_connectivity.assert_has_calls( + [ + mock.call(self.conn_info['ip'], 22), + mock.call(self.conn_info['ip'], 22), + ] + ) self.ssh.set_missing_host_key_policy.assert_called() self.ssh.connect.assert_called_once_with( - hostname=self.conn_info['ip'], port=22, + hostname=self.conn_info['ip'], + port=22, username=self.conn_info['username'], pkey=self.conn_info['pkey'], - password=self.conn_info['password']) + password=self.conn_info['password'], + ) self.ssh.set_log_channel.assert_called_once_with( - "paramiko.morpher.%s.%s" % ( - self.conn_info['ip'], 22) + "paramiko.morpher.%s.%s" % (self.conn_info['ip'], 22) ) @mock.patch.object(base.BaseSSHOSMountTools, '_allow_ssh_env_vars') @@ -113,8 +122,8 @@ def test__exec_cmd(self, mock_exec_ssh_cmd): result = self.base_os_mount_tools._exec_cmd(self.cmd, timeout=120) mock_exec_ssh_cmd.assert_called_once_with( - self.base_os_mount_tools._ssh, self.cmd, {}, get_pty=True, - timeout=120) + self.base_os_mount_tools._ssh, self.cmd, {}, get_pty=True, timeout=120 + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @@ -123,8 +132,12 @@ def test__exec_cmd_without_timeout(self, mock_exec_ssh_cmd): result = self.base_os_mount_tools._exec_cmd(self.cmd) mock_exec_ssh_cmd.assert_called_once_with( - self.base_os_mount_tools._ssh, self.cmd, {}, get_pty=True, - timeout=self.base_os_mount_tools._osmount_operation_timeout) + self.base_os_mount_tools._ssh, + self.cmd, + {}, + get_pty=True, + timeout=self.base_os_mount_tools._osmount_operation_timeout, + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @@ -134,8 +147,10 @@ def test__exec_cmd_with_exception(self, mock_exec_ssh_cmd): self.assertRaises( exception.OSMorphingSSHOperationTimeout, - self.base_os_mount_tools._exec_cmd, self.cmd, - timeout=self.base_os_mount_tools._osmount_operation_timeout) + self.base_os_mount_tools._exec_cmd, + self.cmd, + timeout=self.base_os_mount_tools._osmount_operation_timeout, + ) class TestBaseLinuxOSMountTools(base.BaseLinuxOSMountTools): @@ -156,8 +171,11 @@ def setUp(self, mock_connect): self.devices = ["/dev/sda", "/dev/sdb"] self.base_os_mount_tools = TestBaseLinuxOSMountTools( - mock.sentinel.conn, self.event_manager, - mock.sentinel.ignore_devices, mock.sentinel.operation_timeout) + mock.sentinel.conn, + self.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) mock_connect.assert_called_once_with() @@ -165,12 +183,10 @@ def setUp(self, mock_connect): @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') def test__get_pvs(self, mock_exec_cmd): - mock_exec_cmd.return_value = ( - "pv1:vg1\nimproper_line\npv2:vg1\n\npv3:vg2") + mock_exec_cmd.return_value = "pv1:vg1\nimproper_line\npv2:vg1\n\npv3:vg2" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): - result = self.base_os_mount_tools._get_pvs() + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): + result = self.base_os_mount_tools._get_pvs() mock_exec_cmd.assert_called_once_with("sudo pvdisplay -c") expected_result = {"vg1": ["pv1", "pv2"], "vg2": ["pv3"]} @@ -181,8 +197,7 @@ def test__get_pvs(self, mock_exec_cmd): def test__get_pvs_improper_output(self, mock_exec_cmd): mock_exec_cmd.return_value = "improper_line" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._get_pvs() mock_exec_cmd.assert_called_once_with("sudo pvdisplay -c") @@ -191,22 +206,17 @@ def test__get_pvs_improper_output(self, mock_exec_cmd): @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') def test__get_vgs(self, mock_exec_cmd): - mock_exec_cmd.return_value = ( - "vg1:pv1:uuid1\nimproper_line\n\n\nvg2:pv3:uuid2") + mock_exec_cmd.return_value = "vg1:pv1:uuid1\nimproper_line\n\n\nvg2:pv3:uuid2" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._get_vgs() mock_exec_cmd.assert_called_once_with( - "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :") + "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :" + ) expected_result = { - "uuid1": { - "name": "vg1", - "pvs": ["pv1"]}, - "uuid2": { - "name": "vg2", - "pvs": ["pv3"]} + "uuid1": {"name": "vg1", "pvs": ["pv1"]}, + "uuid2": {"name": "vg2", "pvs": ["pv3"]}, } self.assertEqual(result, expected_result) @@ -217,27 +227,17 @@ def test__get_vgs_duplicate_vg_names(self, mock_exec_cmd, mock_uuid4): mock_exec_cmd.return_value = "vg1:pv1:uuid1\nvg1:pv2:uuid2" mock_uuid4.return_value = "random_uuid" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.DEBUG): result = self.base_os_mount_tools._get_vgs() - vgs_cmd = ( - "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :" + vgs_cmd = "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :" + mock_exec_cmd.assert_has_calls( + [mock.call(vgs_cmd), mock.call("sudo vgrename uuid2 random_uuid")] ) - mock_exec_cmd.assert_has_calls([ - mock.call(vgs_cmd), - mock.call("sudo vgrename uuid2 random_uuid") - ]) expected_result = { - "uuid1": { - "name": "vg1", - "pvs": ["pv1"] - }, - "uuid2": { - "name": "random_uuid", - "pvs": ["pv2"] - } + "uuid1": {"name": "vg1", "pvs": ["pv1"]}, + "uuid2": {"name": "random_uuid", "pvs": ["pv2"]}, } self.assertEqual(result, expected_result) @@ -246,12 +246,12 @@ def test__get_vgs_duplicate_vg_names(self, mock_exec_cmd, mock_uuid4): def test__get_vgs_improper_output(self, mock_exec_cmd): mock_exec_cmd.return_value = "improper_line" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._get_vgs() mock_exec_cmd.assert_called_once_with( - "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :") + "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :" + ) self.assertEqual(result, {}) @@ -266,8 +266,8 @@ def test__check_vgs_with_exception(self, mock_exec_cmd): mock_exec_cmd.side_effect = Exception() self.assertRaises( - exception.CoriolisException, - self.base_os_mount_tools._check_vgs) + exception.CoriolisException, self.base_os_mount_tools._check_vgs + ) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') def test__get_vgnames(self, mock_exec_cmd): @@ -301,24 +301,25 @@ def test__get_lv_paths(self, mock_exec_cmd): @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') - def test__check_mount_fstab_partitions(self, mock_get_device_file_paths, - mock_exec_cmd, mock_read_ssh_file, - mock_test_ssh_path): + def test__check_mount_fstab_partitions( + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" mock_read_ssh_file.return_value = ( - b"Unparseable line\n" - b"UUID=uuid2 /mnt1 ext4 defaults 0 0" + b"Unparseable line\nUUID=uuid2 /mnt1 ext4 defaults 0 0" ) - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir, mountable_lvm_devs=["/dev/sda1"]) + self.os_root_dir, mountable_lvm_devs=["/dev/sda1"] + ) expected_result = [self.os_root_dir + "/mnt1"] @@ -326,30 +327,32 @@ def test__check_mount_fstab_partitions(self, mock_get_device_file_paths, mock_get_device_file_paths.assert_called_once() mock_exec_cmd.assert_called_once_with( - "sudo mount -t ext4 -o defaults" - " /dev/disk/by-uuid/uuid2 '/root/mnt1'" + "sudo mount -t ext4 -o defaults /dev/disk/by-uuid/uuid2 '/root/mnt1'" ) mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) - mock_test_ssh_path.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, mocked_full_path), - mock.call( - self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid2") - ]) + self.base_os_mount_tools._ssh, mocked_full_path + ) + mock_test_ssh_path.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, mocked_full_path), + mock.call(self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid2"), + ] + ) @mock.patch.object(base.utils, 'test_ssh_path') def test__check_mount_fstab_partitions_no_fstab(self, mock_test_ssh_path): mock_test_ssh_path.return_value = False - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARNING): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARNING): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir) + self.os_root_dir + ) self.assertEqual(result, []) mock_test_ssh_path.assert_called_once_with( - self.base_os_mount_tools._ssh, self.os_root_dir + "/etc/fstab") + self.base_os_mount_tools._ssh, self.os_root_dir + "/etc/fstab" + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @@ -357,202 +360,214 @@ def test__check_mount_fstab_partitions_no_fstab(self, mock_test_ssh_path): @mock.patch.object(base.BaseLinuxOSMountTools, '_get_symlink_target') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_no_device_path( - self, mock_get_device_file_paths, mock_get_symlink_target, - mock_exec_cmd, mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_get_symlink_target, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.side_effect = [True, False] mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] mock_get_symlink_target.return_value = "/dev/sda1" - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" mock_read_ssh_file.return_value = ( - b"Unparseable line\n" - b"UUID=uuid2 /mnt1 ext4 defaults 0 0" + b"Unparseable line\nUUID=uuid2 /mnt1 ext4 defaults 0 0" ) - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir) + self.os_root_dir + ) self.assertEqual(result, []) mock_get_device_file_paths.assert_called_once() mock_get_symlink_target.assert_not_called() mock_exec_cmd.assert_not_called() mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) - mock_test_ssh_path.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, mocked_full_path), - mock.call( - self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid2") - ]) + self.base_os_mount_tools._ssh, mocked_full_path + ) + mock_test_ssh_path.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, mocked_full_path), + mock.call(self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid2"), + ] + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_unsupported_device( - self, mock_get_device_file_paths, mock_exec_cmd, - mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) - mock_read_ssh_file.return_value = ( - b"/dev/sda3 /mnt1 ext4 defaults 0 0" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" + mock_read_ssh_file.return_value = b"/dev/sda3 /mnt1 ext4 defaults 0 0" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir, mountable_lvm_devs=[ - "/dev/sda1", "/dev/sda2"]) + self.os_root_dir, mountable_lvm_devs=["/dev/sda1", "/dev/sda2"] + ) self.assertEqual(result, []) mock_get_device_file_paths.assert_called_once() - mock_exec_cmd.assert_called_once_with( - "readlink -en /dev/sda3") + mock_exec_cmd.assert_called_once_with("readlink -en /dev/sda3") mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) mock_test_ssh_path.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_skip_undesired_mount( - self, mock_get_device_file_paths, mock_exec_cmd, - mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) - mock_read_ssh_file.return_value = ( - b"UUID=uuid2 /mnt1 ext4 defaults 0 0" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" + mock_read_ssh_file.return_value = b"UUID=uuid2 /mnt1 ext4 defaults 0 0" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.DEBUG): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir, skip_mounts=["/mnt1"]) + self.os_root_dir, skip_mounts=["/mnt1"] + ) self.assertEqual(result, []) mock_get_device_file_paths.assert_called_once() mock_exec_cmd.assert_not_called() mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) mock_test_ssh_path.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_skip_filesystems( - self, mock_get_device_file_paths, mock_exec_cmd, - mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) - mock_read_ssh_file.return_value = ( - b"UUID=uuid1 /mnt1 ext4 defaults 0 0" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" + mock_read_ssh_file.return_value = b"UUID=uuid1 /mnt1 ext4 defaults 0 0" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.DEBUG): result = self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir, skip_filesystems=["ext4"]) + self.os_root_dir, skip_filesystems=["ext4"] + ) self.assertEqual(result, []) mock_get_device_file_paths.assert_called_once() mock_exec_cmd.assert_not_called() mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) mock_test_ssh_path.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_duplicate_mounts( - self, mock_get_device_file_paths, mock_exec_cmd, - mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] - mock_exec_cmd.return_value = ( - b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" - ) + mock_exec_cmd.return_value = b"/dev/sda1:UUID=uuid1\n/dev/sda2:UUID=uuid2" mock_read_ssh_file.return_value = ( - b"UUID=uuid1 /mnt1 ext4 defaults 0 0\n" - b"UUID=uuid2 /mnt1 ext4 defaults 0 0" + b"UUID=uuid1 /mnt1 ext4 defaults 0 0\nUUID=uuid2 /mnt1 ext4 defaults 0 0" ) self.assertRaises( exception.CoriolisException, self.base_os_mount_tools._check_mount_fstab_partitions, - self.os_root_dir) + self.os_root_dir, + ) mock_get_device_file_paths.assert_not_called() mock_exec_cmd.assert_not_called() mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) mock_test_ssh_path.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) + self.base_os_mount_tools._ssh, mocked_full_path + ) @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.utils, 'read_ssh_file') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_device_file_paths') def test__check_mount_fstab_partitions_mountcmd_with_exception( - self, mock_get_device_file_paths, mock_exec_cmd, - mock_read_ssh_file, mock_test_ssh_path): + self, + mock_get_device_file_paths, + mock_exec_cmd, + mock_read_ssh_file, + mock_test_ssh_path, + ): mocked_full_path = self.os_root_dir + "/etc/fstab" mock_test_ssh_path.return_value = True mock_get_device_file_paths.return_value = ["/dev/sda1", "/dev/sda2"] mock_exec_cmd.side_effect = Exception() - mock_read_ssh_file.return_value = ( - b"UUID=uuid1 /mnt1 ext4 defaults 0 0" - ) + mock_read_ssh_file.return_value = b"UUID=uuid1 /mnt1 ext4 defaults 0 0" - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARNING): - self.base_os_mount_tools._check_mount_fstab_partitions( - self.os_root_dir) + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARNING): + self.base_os_mount_tools._check_mount_fstab_partitions(self.os_root_dir) mock_get_device_file_paths.assert_called_once() mock_exec_cmd.assert_called() mock_read_ssh_file.assert_called_once_with( - self.base_os_mount_tools._ssh, mocked_full_path) - mock_test_ssh_path.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, mocked_full_path), - mock.call( - self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid1") - ]) + self.base_os_mount_tools._ssh, mocked_full_path + ) + mock_test_ssh_path.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, mocked_full_path), + mock.call(self.base_os_mount_tools._ssh, "/dev/disk/by-uuid/uuid1"), + ] + ) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') def test__get_symlink_target(self, mock_exec_cmd): mock_exec_cmd.return_value = "/dev/sda1" - result = self.base_os_mount_tools._get_symlink_target( - "/dev/sda1") + result = self.base_os_mount_tools._get_symlink_target("/dev/sda1") - mock_exec_cmd.assert_called_once_with( - 'readlink -en %s' % "/dev/sda1") + mock_exec_cmd.assert_called_once_with('readlink -en %s' % "/dev/sda1") self.assertEqual(result, "/dev/sda1") @@ -560,27 +575,23 @@ def test__get_symlink_target(self, mock_exec_cmd): def test__get_symlink_target_with_exception(self, mock_exec_cmd): mock_exec_cmd.side_effect = Exception() - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): - self.base_os_mount_tools._get_symlink_target( - "/dev/sda1") + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): + self.base_os_mount_tools._get_symlink_target("/dev/sda1") @mock.patch.object(base.BaseLinuxOSMountTools, '_get_symlink_target') def test__get_device_file_paths(self, mock_get_symlink_target): symlink_list = ['/dev/GROUP/VOLUME0', '/dev/mapper/GROUP-VOLUME0'] mock_get_symlink_target.side_effect = ['/dev/dm0', None] - result = self.base_os_mount_tools._get_device_file_paths( - symlink_list) + result = self.base_os_mount_tools._get_device_file_paths(symlink_list) expected_result = ['/dev/dm0', '/dev/mapper/GROUP-VOLUME0'] self.assertEqual(result, expected_result) - mock_get_symlink_target.assert_has_calls([ - mock.call('/dev/GROUP/VOLUME0'), - mock.call('/dev/mapper/GROUP-VOLUME0') - ]) + mock_get_symlink_target.assert_has_calls( + [mock.call('/dev/GROUP/VOLUME0'), mock.call('/dev/mapper/GROUP-VOLUME0')] + ) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') def test__get_mounted_devices(self, mock_exec_cmd): @@ -589,40 +600,42 @@ def test__get_mounted_devices(self, mock_exec_cmd): "/dev/sda1", "8:1", "brw-rw---- 1 root disk 8, 1 Jan 1 00:00 sda1 sda2", - "" + "", ] result = self.base_os_mount_tools._get_mounted_devices() - mock_exec_cmd.assert_has_calls([ - mock.call("cat /proc/mounts"), - mock.call("readlink -en /dev/sda1"), - mock.call("mountpoint -x /dev/sda1"), - mock.call("ls -al /dev | grep ^b"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("cat /proc/mounts"), + mock.call("readlink -en /dev/sda1"), + mock.call("mountpoint -x /dev/sda1"), + mock.call("ls -al /dev | grep ^b"), + ] + ) self.assertEqual(result, ['/dev/sda1', '/dev/sda2']) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.utils, 'test_ssh_path') - def test__get_mounted_devices_not_found(self, mock_test_ssh_path, - mock_exec_cmd): + def test__get_mounted_devices_not_found(self, mock_test_ssh_path, mock_exec_cmd): mock_exec_cmd.side_effect = [ "/dev/sda1 on / type ext4 (rw,relatime,errors=remount-ro)\n", "/dev/sda1", - "" + "", ] mock_test_ssh_path.return_value = False - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): result = self.base_os_mount_tools._get_mounted_devices() self.assertEqual(result, []) - mock_exec_cmd.assert_has_calls([ - mock.call("cat /proc/mounts"), - mock.call("readlink -en /dev/sda1"), - mock.call("ls -al /dev | grep ^b") - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("cat /proc/mounts"), + mock.call("readlink -en /dev/sda1"), + mock.call("ls -al /dev | grep ^b"), + ] + ) mock_test_ssh_path.assert_called_once_with( self.base_os_mount_tools._ssh, "/dev/sda1" @@ -662,15 +675,19 @@ def test__find_dev_with_contents(self, mock_list_ssh_dir, mock_exec_cmd): mock_list_ssh_dir.return_value = ["etc", "bin", "sbin", "boot"] result = self.base_os_mount_tools._find_dev_with_contents( - self.devices, all_files=self.all_files) + self.devices, all_files=self.all_files + ) - mock_exec_cmd.assert_has_calls([ - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sda /tmp/tmp_dir"), - mock.call("sudo umount /tmp/tmp_dir"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sda /tmp/tmp_dir"), + mock.call("sudo umount /tmp/tmp_dir"), + ] + ) mock_list_ssh_dir.assert_called_once_with( - self.base_os_mount_tools._ssh, "/tmp/tmp_dir") + self.base_os_mount_tools._ssh, "/tmp/tmp_dir" + ) expected_result = "/dev/sda" self.assertEqual(result, expected_result) @@ -678,41 +695,50 @@ def test__find_dev_with_contents(self, mock_list_ssh_dir, mock_exec_cmd): @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.utils, 'list_ssh_dir') def test__find_dev_with_contents_both_all_and_one_of_files( - self, mock_list_ssh_dir, mock_exec_cmd): + self, mock_list_ssh_dir, mock_exec_cmd + ): self.assertRaises( exception.CoriolisException, self.base_os_mount_tools._find_dev_with_contents, - self.devices, all_files=self.all_files, - one_of_files=self.one_of_files) + self.devices, + all_files=self.all_files, + one_of_files=self.one_of_files, + ) mock_exec_cmd.assert_not_called() mock_list_ssh_dir.assert_not_called() @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.utils, 'list_ssh_dir') - def test__find_dev_with_contents_one_of_files(self, mock_list_ssh_dir, - mock_exec_cmd): + def test__find_dev_with_contents_one_of_files( + self, mock_list_ssh_dir, mock_exec_cmd + ): mock_exec_cmd.return_value = "/tmp/tmp_dir" mock_list_ssh_dir.return_value = ["etc", "bin", "sbin", "boot"] result = self.base_os_mount_tools._find_dev_with_contents( - self.devices, one_of_files=self.one_of_files) + self.devices, one_of_files=self.one_of_files + ) - mock_exec_cmd.assert_has_calls([ - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sda /tmp/tmp_dir"), - mock.call("sudo umount /tmp/tmp_dir"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sda /tmp/tmp_dir"), + mock.call("sudo umount /tmp/tmp_dir"), + ] + ) mock_list_ssh_dir.assert_called_once_with( - self.base_os_mount_tools._ssh, "/tmp/tmp_dir") + self.base_os_mount_tools._ssh, "/tmp/tmp_dir" + ) expected_result = "/dev/sda" self.assertEqual(result, expected_result) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.utils, 'list_ssh_dir') - def test__find_dev_with_contents_missing_all_files(self, mock_list_ssh_dir, - mock_exec_cmd): + def test__find_dev_with_contents_missing_all_files( + self, mock_list_ssh_dir, mock_exec_cmd + ): mock_exec_cmd.return_value = "/tmp/tmp_dir" mock_list_ssh_dir.return_value = ["etc", "bin", "sbin", "boot"] @@ -720,44 +746,57 @@ def test__find_dev_with_contents_missing_all_files(self, mock_list_ssh_dir, all_files = self.all_files + ["missing_file"] result = self.base_os_mount_tools._find_dev_with_contents( - self.devices, all_files=all_files) - - mock_exec_cmd.assert_has_calls([ - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sda /tmp/tmp_dir"), - mock.call("sudo umount /tmp/tmp_dir"), - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sdb /tmp/tmp_dir"), - mock.call("sudo umount /tmp/tmp_dir"), - ]) - mock_list_ssh_dir.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir"), - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir"), - ]) + self.devices, all_files=all_files + ) + + mock_exec_cmd.assert_has_calls( + [ + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sda /tmp/tmp_dir"), + mock.call("sudo umount /tmp/tmp_dir"), + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sdb /tmp/tmp_dir"), + mock.call("sudo umount /tmp/tmp_dir"), + ] + ) + mock_list_ssh_dir.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir"), + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir"), + ] + ) self.assertIsNone(result) @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.utils, 'list_ssh_dir') - def test__find_dev_with_contents_with_exception(self, mock_list_ssh_dir, - mock_exec_cmd): + def test__find_dev_with_contents_with_exception( + self, mock_list_ssh_dir, mock_exec_cmd + ): mock_exec_cmd.side_effect = [ - b"/tmp/tmp_dir", Exception(), None, None, # First device - b"/tmp/tmp_dir", Exception(), None, None # Second device + b"/tmp/tmp_dir", + Exception(), + None, + None, # First device + b"/tmp/tmp_dir", + Exception(), + None, + None, # Second device ] - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): self.base_os_mount_tools._find_dev_with_contents( - self.devices, all_files=self.all_files) + self.devices, all_files=self.all_files + ) mock_list_ssh_dir.assert_not_called() @mock.patch.object(base.utils, 'test_ssh_path') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_find_dev_with_contents') - def test__find_and_mount_root(self, mock_find_dev_with_contents, - mock_exec_cmd, mock_test_ssh_path): + def test__find_and_mount_root( + self, mock_find_dev_with_contents, mock_exec_cmd, mock_test_ssh_path + ): devices = ["/dev/sda", "/dev/sdb"] mock_exec_cmd.return_value = "/tmp/tmp_dir" mock_find_dev_with_contents.return_value = "/dev/sda" @@ -765,21 +804,26 @@ def test__find_and_mount_root(self, mock_find_dev_with_contents, result = self.base_os_mount_tools._find_and_mount_root(devices) - mock_exec_cmd.assert_has_calls([ - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sda /tmp/tmp_dir"), - mock.call("sudo mount -o bind /proc/ /tmp/tmp_dir/proc"), - mock.call("sudo mount -o bind /dev/ /tmp/tmp_dir/dev"), - mock.call("sudo mount -o bind /run/ /tmp/tmp_dir/run"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sda /tmp/tmp_dir"), + mock.call("sudo mount -o bind /proc/ /tmp/tmp_dir/proc"), + mock.call("sudo mount -o bind /dev/ /tmp/tmp_dir/dev"), + mock.call("sudo mount -o bind /run/ /tmp/tmp_dir/run"), + ] + ) mock_find_dev_with_contents.assert_called_once_with( - ["/dev/sdb"], all_files=self.all_files) - mock_test_ssh_path.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/proc"), - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/sys"), - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/dev"), - mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/run"), - ]) + ["/dev/sdb"], all_files=self.all_files + ) + mock_test_ssh_path.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/proc"), + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/sys"), + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/dev"), + mock.call(self.base_os_mount_tools._ssh, "/tmp/tmp_dir/run"), + ] + ) expected_result = ('/tmp/tmp_dir', '/dev/sda') @@ -789,18 +833,21 @@ def test__find_and_mount_root(self, mock_find_dev_with_contents, @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_find_dev_with_contents') def test__find_and_mount_root_with_exception( - self, mock_find_dev_with_contents, mock_exec_cmd, - mock_test_ssh_path): + self, mock_find_dev_with_contents, mock_exec_cmd, mock_test_ssh_path + ): devices = ["/dev/sda", "/dev/sdb"] mock_exec_cmd.return_value = "/tmp/tmp_dir" mock_find_dev_with_contents.return_value = None - self.assertRaises(exception.OperatingSystemNotFound, - self.base_os_mount_tools._find_and_mount_root, - devices) + self.assertRaises( + exception.OperatingSystemNotFound, + self.base_os_mount_tools._find_and_mount_root, + devices, + ) mock_find_dev_with_contents.assert_called_once_with( - devices, all_files=self.all_files) + devices, all_files=self.all_files + ) mock_exec_cmd.assert_not_called() mock_test_ssh_path.assert_not_called() @@ -808,29 +855,31 @@ def test__find_and_mount_root_with_exception( @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMountTools, '_find_dev_with_contents') def test__find_and_mount_root_exec_cmd_exception( - self, mock_find_dev_with_contents, mock_exec_cmd, - mock_test_ssh_path): + self, mock_find_dev_with_contents, mock_exec_cmd, mock_test_ssh_path + ): devices = ["/dev/sda", "/dev/sdb"] mock_exec_cmd.side_effect = ["/tmp/tmp_dir", CoriolisTestException()] mock_find_dev_with_contents.return_value = "/dev/sda" mock_test_ssh_path.return_value = True - with self.assertLogs('coriolis.osmorphing.osmount.base', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.base', level=logging.WARN): self.assertRaises( CoriolisTestException, self.base_os_mount_tools._find_and_mount_root, - devices + devices, ) - mock_exec_cmd.assert_has_calls([ - mock.call("mktemp -d"), - mock.call("sudo mount /dev/sda /tmp/tmp_dir"), - mock.call("sudo umount /tmp/tmp_dir"), - mock.call("sudo rmdir /tmp/tmp_dir"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("mktemp -d"), + mock.call("sudo mount /dev/sda /tmp/tmp_dir"), + mock.call("sudo umount /tmp/tmp_dir"), + mock.call("sudo rmdir /tmp/tmp_dir"), + ] + ) mock_find_dev_with_contents.assert_called_once_with( - devices, all_files=['etc', 'bin', 'sbin', 'boot']) + devices, all_files=['etc', 'bin', 'sbin', 'boot'] + ) mock_test_ssh_path.assert_not_called() @mock.patch.object(base.utils, 'check_fs') @@ -840,22 +889,21 @@ def test__find_and_mount_root_exec_cmd_exception( @mock.patch.object(base.BaseLinuxOSMountTools, '_find_and_mount_root') @mock.patch.object(base.BaseLinuxOSMountTools, '_find_dev_with_contents') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_volume_block_devices') - @mock.patch.object( - base.BaseLinuxOSMountTools, '_check_mount_fstab_partitions' - ) - def test_mount_os(self, mock_check_mount_fstab_partitions, - mock_get_volume_block_devices, - mock_find_dev_with_contents, mock_find_and_mount_root, - mock_get_mounted_devices, mock_get_vgs, mock_exec_cmd, - mock_check_fs): + @mock.patch.object(base.BaseLinuxOSMountTools, '_check_mount_fstab_partitions') + def test_mount_os( + self, + mock_check_mount_fstab_partitions, + mock_get_volume_block_devices, + mock_find_dev_with_contents, + mock_find_and_mount_root, + mock_get_mounted_devices, + mock_get_vgs, + mock_exec_cmd, + mock_check_fs, + ): mock_get_volume_block_devices.return_value = ["/dev/sda", "/dev/sdb"] mock_find_and_mount_root.return_value = ("/tmp/tmp_dir", "/dev/sdb1") - mock_get_vgs.return_value = { - "vgid1": { - "name": "vg1", - "pvs": ["/dev/sda1"] - } - } + mock_get_vgs.return_value = {"vgid1": {"name": "vg1", "pvs": ["/dev/sda1"]}} mock_get_mounted_devices.return_value = ["/dev/sda1"] mock_find_dev_with_contents.return_value = "/dev/sdb1" mock_exec_cmd.side_effect = [ @@ -878,35 +926,41 @@ def test_mount_os(self, mock_check_mount_fstab_partitions, result = self.base_os_mount_tools.mount_os() - mock_exec_cmd.assert_has_calls([ - mock.call('sudo partx -v -a /dev/sda || true'), - mock.call('sudo ls -1 /dev/sda*'), - mock.call('sudo partx -v -a /dev/sdb || true'), - mock.call('sudo ls -1 /dev/sdb*'), - mock.call('sudo vgck'), - mock.call('sudo vgchange -ay -S vg_uuid=vgid1'), - mock.call('sudo vgchange --refresh'), - mock.call('sudo ls -1 /dev/vg1/*'), - mock.call('readlink -en /dev/sda1'), - mock.call('readlink -en /dev/sdb1'), - mock.call('sudo blkid -o value -s TYPE /dev/sdb1 || true'), - mock.call('readlink -en /dev/vg1/lv1'), - mock.call('sudo blkid -o value -s TYPE /dev/vg1/lv1 || true'), - mock.call('sudo mount /dev/sdb1 "/tmp/tmp_dir/boot"'), - mock.call('sudo lvdisplay -c') - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call('sudo partx -v -a /dev/sda || true'), + mock.call('sudo ls -1 /dev/sda*'), + mock.call('sudo partx -v -a /dev/sdb || true'), + mock.call('sudo ls -1 /dev/sdb*'), + mock.call('sudo vgck'), + mock.call('sudo vgchange -ay -S vg_uuid=vgid1'), + mock.call('sudo vgchange --refresh'), + mock.call('sudo ls -1 /dev/vg1/*'), + mock.call('readlink -en /dev/sda1'), + mock.call('readlink -en /dev/sdb1'), + mock.call('sudo blkid -o value -s TYPE /dev/sdb1 || true'), + mock.call('readlink -en /dev/vg1/lv1'), + mock.call('sudo blkid -o value -s TYPE /dev/vg1/lv1 || true'), + mock.call('sudo mount /dev/sdb1 "/tmp/tmp_dir/boot"'), + mock.call('sudo lvdisplay -c'), + ] + ) mock_get_volume_block_devices.assert_called_once() mock_find_dev_with_contents.assert_called_once_with( - ['/dev/sdb1', '/dev/vg1/lv1'], one_of_files=["grub", "grub2"]) + ['/dev/sdb1', '/dev/vg1/lv1'], one_of_files=["grub", "grub2"] + ) mock_get_mounted_devices.assert_called_once() mock_find_and_mount_root.assert_called_once() mock_get_vgs.assert_called_once() mock_check_mount_fstab_partitions.assert_called_once_with( - "/tmp/tmp_dir", mountable_lvm_devs=['ext4']) - mock_check_fs.assert_has_calls([ - mock.call(self.base_os_mount_tools._ssh, "ext4", "/dev/sdb1"), - mock.call(self.base_os_mount_tools._ssh, "ext4", "/dev/vg1/lv1"), - ]) + "/tmp/tmp_dir", mountable_lvm_devs=['ext4'] + ) + mock_check_fs.assert_has_calls( + [ + mock.call(self.base_os_mount_tools._ssh, "ext4", "/dev/sdb1"), + mock.call(self.base_os_mount_tools._ssh, "ext4", "/dev/vg1/lv1"), + ] + ) self.assertEqual(result, ('/tmp/tmp_dir', '/dev/sdb1')) @@ -917,23 +971,21 @@ def test_mount_os(self, mock_check_mount_fstab_partitions, @mock.patch.object(base.BaseLinuxOSMountTools, '_find_and_mount_root') @mock.patch.object(base.BaseLinuxOSMountTools, '_find_dev_with_contents') @mock.patch.object(base.BaseLinuxOSMountTools, '_get_volume_block_devices') - @mock.patch.object( - base.BaseLinuxOSMountTools, '_check_mount_fstab_partitions' - ) - def test_mount_os_run_xfs(self, mock_check_mount_fstab_partitions, - mock_get_volume_block_devices, - mock_find_dev_with_contents, - mock_find_and_mount_root, - mock_get_mounted_devices, mock_get_vgs, - mock_exec_cmd, mock_sleep): + @mock.patch.object(base.BaseLinuxOSMountTools, '_check_mount_fstab_partitions') + def test_mount_os_run_xfs( + self, + mock_check_mount_fstab_partitions, + mock_get_volume_block_devices, + mock_find_dev_with_contents, + mock_find_and_mount_root, + mock_get_mounted_devices, + mock_get_vgs, + mock_exec_cmd, + mock_sleep, + ): mock_get_volume_block_devices.return_value = ["/dev/sda", "/dev/sdb"] mock_find_and_mount_root.return_value = ("/tmp/tmp_dir", "/dev/sdb1") - mock_get_vgs.return_value = { - "vgid1": { - "name": "vg1", - "pvs": ["/dev/sda1"] - } - } + mock_get_vgs.return_value = {"vgid1": {"name": "vg1", "pvs": ["/dev/sda1"]}} mock_get_mounted_devices.return_value = ["/dev/sda1"] mock_exec_cmd.side_effect = [ "", @@ -954,28 +1006,34 @@ def test_mount_os_run_xfs(self, mock_check_mount_fstab_partitions, result = self.base_os_mount_tools.mount_os() - mock_exec_cmd.assert_has_calls([ - mock.call('sudo partx -v -a /dev/sda || true'), - mock.call('sudo ls -1 /dev/sda*'), - mock.call('sudo partx -v -a /dev/sdb || true'), - mock.call('sudo ls -1 /dev/sdb*'), - mock.call('sudo vgck'), - mock.call('readlink -en xfs'), - mock.call('sudo blkid -o value -s TYPE xfs || true'), - mock.call('readlink -en xfs'), - mock.call('sudo blkid -o value -s TYPE xfs || true'), - mock.call('sudo mount %s "/tmp/tmp_dir/boot"' % - mock_find_dev_with_contents.return_value), - mock.call("sudo lvdisplay -c"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call('sudo partx -v -a /dev/sda || true'), + mock.call('sudo ls -1 /dev/sda*'), + mock.call('sudo partx -v -a /dev/sdb || true'), + mock.call('sudo ls -1 /dev/sdb*'), + mock.call('sudo vgck'), + mock.call('readlink -en xfs'), + mock.call('sudo blkid -o value -s TYPE xfs || true'), + mock.call('readlink -en xfs'), + mock.call('sudo blkid -o value -s TYPE xfs || true'), + mock.call( + 'sudo mount %s "/tmp/tmp_dir/boot"' + % mock_find_dev_with_contents.return_value + ), + mock.call("sudo lvdisplay -c"), + ] + ) mock_get_volume_block_devices.assert_called_once() mock_find_dev_with_contents.assert_called_once_with( - ['xfs'], one_of_files=["grub", "grub2"]) + ['xfs'], one_of_files=["grub", "grub2"] + ) mock_get_mounted_devices.assert_called_once() mock_find_and_mount_root.assert_called_once() mock_get_vgs.assert_called_once() mock_check_mount_fstab_partitions.assert_called_once_with( - "/tmp/tmp_dir", mountable_lvm_devs=['ext4']) + "/tmp/tmp_dir", mountable_lvm_devs=['ext4'] + ) self.assertEqual(result, ('/tmp/tmp_dir', '/dev/sdb1')) @@ -984,9 +1042,11 @@ def test_dismount_os(self, mock_exec_cmd): root_dir = "/mnt/root_dir" mock_exec_cmd.side_effect = [ None, - ("/dev/sda1 /mnt/root_dir/sub_dir type ext4\n" - "/dev/sda2 /mnt/root_dir/dev type ext4\n" - "/dev/sda3 /mnt/root_dir type ext4\n"), + ( + "/dev/sda1 /mnt/root_dir/sub_dir type ext4\n" + "/dev/sda2 /mnt/root_dir/dev type ext4\n" + "/dev/sda3 /mnt/root_dir type ext4\n" + ), None, None, None, @@ -994,15 +1054,17 @@ def test_dismount_os(self, mock_exec_cmd): self.base_os_mount_tools.dismount_os(root_dir) - mock_exec_cmd.assert_has_calls([ - mock.call("sudo fuser --kill --mount /mnt/root_dir || true"), - mock.call("cat /proc/mounts"), - mock.call("sudo umount /mnt/root_dir/sub_dir"), - mock.call("mountpoint -q /mnt/root_dir/dev" - " && sudo umount /mnt/root_dir/dev"), - mock.call( - "mountpoint -q /mnt/root_dir && sudo umount /mnt/root_dir"), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("sudo fuser --kill --mount /mnt/root_dir || true"), + mock.call("cat /proc/mounts"), + mock.call("sudo umount /mnt/root_dir/sub_dir"), + mock.call( + "mountpoint -q /mnt/root_dir/dev && sudo umount /mnt/root_dir/dev" + ), + mock.call("mountpoint -q /mnt/root_dir && sudo umount /mnt/root_dir"), + ] + ) @mock.patch.object(base.utils, 'get_url_with_credentials') def test_set_proxy(self, mock_get_url_with_credentials): @@ -1010,25 +1072,23 @@ def test_set_proxy(self, mock_get_url_with_credentials): 'url': "http://127.0.0.1:8080", 'username': "admin", 'password': 'Random-Password-123!', - 'no_proxy': ['cloudbase.it', '127.0.0.1'] + 'no_proxy': ['cloudbase.it', '127.0.0.1'], } self.base_os_mount_tools.set_proxy(proxy_settings) mock_get_url_with_credentials.assert_called_once_with( - proxy_settings['url'], proxy_settings['username'], - proxy_settings['password']) + proxy_settings['url'], + proxy_settings['username'], + proxy_settings['password'], + ) self.assertEqual( - self.base_os_mount_tools._environment['no_proxy'], - 'cloudbase.it.127.0.0.1' + self.base_os_mount_tools._environment['no_proxy'], 'cloudbase.it.127.0.0.1' ) @mock.patch.object(base.utils, 'get_url_with_credentials') def test_set_proxy_no_url(self, mock_get_url_with_credentials): - proxy_settings = { - 'username': "admin", - 'password': 'Random-Password-123!' - } + proxy_settings = {'username': "admin", 'password': 'Random-Password-123!'} result = self.base_os_mount_tools.set_proxy(proxy_settings) self.assertIsNone(result) diff --git a/coriolis/tests/osmorphing/osmount/test_factory.py b/coriolis/tests/osmorphing/osmount/test_factory.py index f9a15406..7eac7ca1 100644 --- a/coriolis/tests/osmorphing/osmount/test_factory.py +++ b/coriolis/tests/osmorphing/osmount/test_factory.py @@ -4,8 +4,7 @@ from unittest import mock from coriolis import exception -from coriolis.osmorphing.osmount import base -from coriolis.osmorphing.osmount import factory +from coriolis.osmorphing.osmount import base, factory from coriolis.tests import test_base @@ -14,30 +13,46 @@ class GetOsMountToolsTestCase(test_base.CoriolisBaseTestCase): def test_get_os_mount_tools_unsupported_os_type(self): self.assertRaises( - exception.CoriolisException, factory.get_os_mount_tools, - "unsupported_os", mock.sentinel.connection_info, - mock.sentinel.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + exception.CoriolisException, + factory.get_os_mount_tools, + "unsupported_os", + mock.sentinel.connection_info, + mock.sentinel.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) @mock.patch.object(base.BaseSSHOSMountTools, '_connect') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') - @mock.patch.object(factory.ubuntu.UbuntuOSMountTools, 'check_os', - return_value=False) - @mock.patch.object(factory.redhat.RedHatOSMountTools, 'check_os', - return_value=False) - @mock.patch.object(factory.suse.SUSEOSMountTools, 'check_os', - return_value=False) - @mock.patch.object(factory.windows.WindowsMountTools, 'check_os', - return_value=False) + @mock.patch.object( + factory.ubuntu.UbuntuOSMountTools, 'check_os', return_value=False + ) + @mock.patch.object( + factory.redhat.RedHatOSMountTools, 'check_os', return_value=False + ) + @mock.patch.object(factory.suse.SUSEOSMountTools, 'check_os', return_value=False) + @mock.patch.object( + factory.windows.WindowsMountTools, 'check_os', return_value=False + ) def test_get_os_mount_tools_no_os_found( - self, mock_windows_check, mock_suse_check, mock_redhat_check, - mock_ubuntu_check, mock_exec_cmd, mock_connect): + self, + mock_windows_check, + mock_suse_check, + mock_redhat_check, + mock_ubuntu_check, + mock_exec_cmd, + mock_connect, + ): mock_exec_cmd.return_value = ("Ubuntu", "") self.assertRaises( - exception.CoriolisException, factory.get_os_mount_tools, - factory.constants.OS_TYPE_LINUX, mock_connect, - mock.sentinel.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + exception.CoriolisException, + factory.get_os_mount_tools, + factory.constants.OS_TYPE_LINUX, + mock_connect, + mock.sentinel.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) mock_redhat_check.assert_called_once_with() mock_ubuntu_check.assert_called_once_with() @@ -46,22 +61,31 @@ def test_get_os_mount_tools_no_os_found( @mock.patch.object(base.BaseSSHOSMountTools, '_connect') @mock.patch.object(base.BaseSSHOSMountTools, '_exec_cmd') - @mock.patch.object(factory.ubuntu.UbuntuOSMountTools, 'check_os', - return_value=True) - @mock.patch.object(factory.redhat.RedHatOSMountTools, 'check_os', - return_value=False) - @mock.patch.object(factory.suse.SUSEOSMountTools, 'check_os', - return_value=False) - @mock.patch.object(factory.windows.WindowsMountTools, 'check_os', - return_value=False) + @mock.patch.object(factory.ubuntu.UbuntuOSMountTools, 'check_os', return_value=True) + @mock.patch.object( + factory.redhat.RedHatOSMountTools, 'check_os', return_value=False + ) + @mock.patch.object(factory.suse.SUSEOSMountTools, 'check_os', return_value=False) + @mock.patch.object( + factory.windows.WindowsMountTools, 'check_os', return_value=False + ) def test_get_os_mount_tools_os_found( - self, mock_windows_check, mock_suse_check, mock_redhat_check, - mock_ubuntu_check, mock_exec_cmd, mock_connect): + self, + mock_windows_check, + mock_suse_check, + mock_redhat_check, + mock_ubuntu_check, + mock_exec_cmd, + mock_connect, + ): mock_exec_cmd.return_value = ("Ubuntu", "") tools = factory.get_os_mount_tools( - factory.constants.OS_TYPE_LINUX, mock_connect, - mock.sentinel.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + factory.constants.OS_TYPE_LINUX, + mock_connect, + mock.sentinel.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) self.assertIsInstance(tools, factory.ubuntu.UbuntuOSMountTools) mock_ubuntu_check.assert_called_once_with() diff --git a/coriolis/tests/osmorphing/osmount/test_redhat.py b/coriolis/tests/osmorphing/osmount/test_redhat.py index 00bac958..556b0486 100644 --- a/coriolis/tests/osmorphing/osmount/test_redhat.py +++ b/coriolis/tests/osmorphing/osmount/test_redhat.py @@ -16,9 +16,11 @@ def setUp(self, mock_connect): self.ssh = mock.MagicMock() self.tools = redhat.RedHatOSMountTools( - self.ssh, mock.sentinel.event_manager, + self.ssh, + mock.sentinel.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) mock_connect.assert_called_once_with() @@ -38,10 +40,12 @@ def test_setup(self, mock_setup, mock_exec_cmd): self.assertIsNone(result) mock_setup.assert_called_once_with() - mock_exec_cmd.assert_has_calls([ - mock.call("sudo -E yum install -y lvm2 psmisc"), - mock.call("sudo modprobe dm-mod") - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("sudo -E yum install -y lvm2 psmisc"), + mock.call("sudo modprobe dm-mod"), + ] + ) @mock.patch.object(redhat.base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(redhat.utils, 'restart_service') @@ -50,5 +54,6 @@ def test__allow_ssh_env_vars(self, mock_restart_service, mock_exec_cmd): self.assertTrue(result) mock_exec_cmd.assert_called_once_with( - 'sudo sed -i -e "\$aAcceptEnv *" /etc/ssh/sshd_config') + 'sudo sed -i -e "\$aAcceptEnv *" /etc/ssh/sshd_config' + ) mock_restart_service.assert_called_once_with(self.ssh, "sshd") diff --git a/coriolis/tests/osmorphing/osmount/test_suse.py b/coriolis/tests/osmorphing/osmount/test_suse.py index 91afd96a..2cf14d92 100644 --- a/coriolis/tests/osmorphing/osmount/test_suse.py +++ b/coriolis/tests/osmorphing/osmount/test_suse.py @@ -16,9 +16,11 @@ def setUp(self, mock_connect): self.ssh = mock.MagicMock() self.tools = suse.SUSEOSMountTools( - self.ssh, mock.sentinel.event_manager, + self.ssh, + mock.sentinel.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) mock_connect.assert_called_once_with() @@ -61,44 +63,48 @@ def test_setup(self, mock_setup, mock_exec_cmd, mock_retry_on_error): self.assertIsNone(result) mock_setup.assert_called_once_with() - mock_retry_on_error.assert_called_once_with( - max_attempts=10, sleep_seconds=30) - mock_exec_cmd.assert_has_calls([ - mock.call( - "sudo -E zypper --non-interactive install lvm2 psmisc"), - mock.call("sudo modprobe dm-mod"), - mock.call("sudo rm -f /etc/lvm/devices/system.devices") - ]) + mock_retry_on_error.assert_called_once_with(max_attempts=10, sleep_seconds=30) + mock_exec_cmd.assert_has_calls( + [ + mock.call("sudo -E zypper --non-interactive install lvm2 psmisc"), + mock.call("sudo modprobe dm-mod"), + mock.call("sudo rm -f /etc/lvm/devices/system.devices"), + ] + ) @mock.patch.object(suse.base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(suse.utils, 'restart_service') @mock.patch.object(suse.utils, 'test_ssh_path', return_value=True) def test__allow_ssh_env_vars( - self, mock_test_ssh_path, mock_restart_service, mock_exec_cmd): + self, mock_test_ssh_path, mock_restart_service, mock_exec_cmd + ): result = self.tools._allow_ssh_env_vars() self.assertTrue(result) - mock_test_ssh_path.assert_called_once_with( - self.ssh, suse.SSHD_CONFIG_PATH) + mock_test_ssh_path.assert_called_once_with(self.ssh, suse.SSHD_CONFIG_PATH) mock_exec_cmd.assert_called_once_with( - 'sudo sed -i -e "\\$aAcceptEnv *" %s' % suse.SSHD_CONFIG_PATH) + 'sudo sed -i -e "\\$aAcceptEnv *" %s' % suse.SSHD_CONFIG_PATH + ) mock_restart_service.assert_called_once_with(self.ssh, "sshd") @mock.patch.object(suse.base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(suse.utils, 'restart_service') @mock.patch.object(suse.utils, 'test_ssh_path', return_value=False) def test__allow_ssh_env_vars_usr_etc( - self, mock_test_ssh_path, mock_restart_service, mock_exec_cmd): + self, mock_test_ssh_path, mock_restart_service, mock_exec_cmd + ): result = self.tools._allow_ssh_env_vars() self.assertTrue(result) - mock_test_ssh_path.assert_called_once_with( - self.ssh, suse.SSHD_CONFIG_PATH) - mock_exec_cmd.assert_has_calls([ - mock.call( - "sudo cp %s %s" % ( - suse.USR_SSHD_CONFIG_PATH, suse.SSHD_CONFIG_PATH)), - mock.call( - 'sudo sed -i -e "\\$aAcceptEnv *" %s' % - suse.SSHD_CONFIG_PATH)]) + mock_test_ssh_path.assert_called_once_with(self.ssh, suse.SSHD_CONFIG_PATH) + mock_exec_cmd.assert_has_calls( + [ + mock.call( + "sudo cp %s %s" % (suse.USR_SSHD_CONFIG_PATH, suse.SSHD_CONFIG_PATH) + ), + mock.call( + 'sudo sed -i -e "\\$aAcceptEnv *" %s' % suse.SSHD_CONFIG_PATH + ), + ] + ) mock_restart_service.assert_called_once_with(self.ssh, "sshd") diff --git a/coriolis/tests/osmorphing/osmount/test_ubuntu.py b/coriolis/tests/osmorphing/osmount/test_ubuntu.py index c12cc52a..a555806c 100644 --- a/coriolis/tests/osmorphing/osmount/test_ubuntu.py +++ b/coriolis/tests/osmorphing/osmount/test_ubuntu.py @@ -16,8 +16,11 @@ def setUp(self, mock_connect): self.ssh = mock.MagicMock() self.tools = ubuntu.UbuntuOSMountTools( - self.ssh, mock.sentinel.event_manager, - mock.sentinel.ignore_devices, mock.sentinel.operation_timeout) + self.ssh, + mock.sentinel.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) mock_connect.assert_called_once_with() @@ -37,12 +40,15 @@ def test_setup(self, mock_setup, mock_exec_cmd): self.assertIsNone(result) mock_setup.assert_called_once_with() - mock_exec_cmd.assert_has_calls([ - mock.call("sudo -E apt-get update -y"), - mock.call("sudo -E apt-get -o DPkg::Lock::Timeout=600 " - "install lvm2 psmisc -y"), - mock.call("sudo modprobe dm-mod") - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call("sudo -E apt-get update -y"), + mock.call( + "sudo -E apt-get -o DPkg::Lock::Timeout=600 install lvm2 psmisc -y" + ), + mock.call("sudo modprobe dm-mod"), + ] + ) @mock.patch.object(ubuntu.base.BaseSSHOSMountTools, '_exec_cmd') @mock.patch.object(ubuntu.utils, 'restart_service') @@ -51,5 +57,6 @@ def test__allow_ssh_env_vars(self, mock_restart_service, mock_exec_cmd): self.assertTrue(result) mock_exec_cmd.assert_called_once_with( - 'sudo sed -i -e "\$aAcceptEnv *" /etc/ssh/sshd_config') + 'sudo sed -i -e "\$aAcceptEnv *" /etc/ssh/sshd_config' + ) mock_restart_service.assert_called_once_with(self.ssh, "sshd") diff --git a/coriolis/tests/osmorphing/osmount/test_windows.py b/coriolis/tests/osmorphing/osmount/test_windows.py index 56dd8343..93ca632e 100644 --- a/coriolis/tests/osmorphing/osmount/test_windows.py +++ b/coriolis/tests/osmorphing/osmount/test_windows.py @@ -31,13 +31,16 @@ def setUp(self, mock_wsman_connection): "ip": "127.0.0.1", "username": "random_username", "password": "random_password", - "pkey": "random_pkey" + "pkey": "random_pkey", } self.status = "Online" self.service_script_with_id_fmt = "SELECT DISK %s\r\nONLINE DISK" self.tools = windows.WindowsMountTools( - self.conn_info, self.event_manager, mock.sentinel.ignore_devices, - mock.sentinel.operation_timeout) + self.conn_info, + self.event_manager, + mock.sentinel.ignore_devices, + mock.sentinel.operation_timeout, + ) self.tools._conn = mock_wsman_connection @mock.patch.object(windows.wsman.WSManConnection, 'from_connection_info') @@ -46,34 +49,37 @@ def test__connect(self, mock_from_connection_info): self.assertIsNone(result) mock_from_connection_info.assert_called_once_with( - self.conn_info, mock.sentinel.operation_timeout) + self.conn_info, mock.sentinel.operation_timeout + ) def test_get_connection(self): result = self.tools.get_connection() self.assertEqual(result, self.tools._conn) def test_check_os(self): - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.DEBUG): + with self.assertLogs( + 'coriolis.osmorphing.osmount.windows', level=logging.DEBUG + ): result = self.tools.check_os() self.assertTrue(result) self.tools._conn.exec_ps_command.assert_called_once_with( - "(get-ciminstance Win32_OperatingSystem).Caption") + "(get-ciminstance Win32_OperatingSystem).Caption" + ) def test_check_os_not_authorized(self): - self.tools._conn.exec_ps_command.side_effect = ( - windows.exception.NotAuthorized) + self.tools._conn.exec_ps_command.side_effect = windows.exception.NotAuthorized - self.assertRaises( - windows.exception.NotAuthorized, self.tools.check_os) + self.assertRaises(windows.exception.NotAuthorized, self.tools.check_os) def test_check_os_with_exception(self): self.tools._conn.exec_ps_command.side_effect = ( - windows.exception.CoriolisException) + windows.exception.CoriolisException + ) - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.DEBUG): + with self.assertLogs( + 'coriolis.osmorphing.osmount.windows', level=logging.DEBUG + ): result = self.tools.check_os() self.assertFalse(result) @@ -84,18 +90,22 @@ def test__run_diskpart_script(self, mock_uuid4): mocked_file_path = r"%s\%s.txt" % ( self.tools._conn.exec_ps_command.return_value, - mock_uuid4.return_value) + mock_uuid4.return_value, + ) result = self.tools._run_diskpart_script(script) self.assertEqual(result, 'random_tempdir') self.tools._conn.write_file.assert_called_once_with( - mocked_file_path, b"random_script") + mocked_file_path, b"random_script" + ) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call("$env:TEMP"), - mock.call("diskpart.exe /s '%s'" % mocked_file_path) - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call("$env:TEMP"), + mock.call("diskpart.exe /s '%s'" % mocked_file_path), + ] + ) @mock.patch.object(windows.WindowsMountTools, '_run_diskpart_script') def test__service_disks_with_status(self, mock_run_script): @@ -103,20 +113,22 @@ def test__service_disks_with_status(self, mock_run_script): mock_run_script.side_effect = [ " Disk 0 Online\n Disk 1 Online\n Disk 2 Online\n", " Disk 0 Online\n Disk 1 Online\n Disk 2 Online\n", - Exception() + Exception(), ] - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.windows', level=logging.WARN): self.tools._service_disks_with_status( - self.status, self.service_script_with_id_fmt, + self.status, + self.service_script_with_id_fmt, skip_on_error=True, logmsg_fmt="Operating on disk with index '%s'", - disk_ids_to_skip=disk_ids_to_skip) + disk_ids_to_skip=disk_ids_to_skip, + ) def test__service_disks_with_status_no_skip_ids(self): result = self.tools._service_disks_with_status( - self.status, mock.sentinel.service_script_with_id_fmt) + self.status, mock.sentinel.service_script_with_id_fmt + ) self.assertIsNone(result) @mock.patch.object(windows.WindowsMountTools, '_run_diskpart_script') @@ -125,16 +137,17 @@ def test__service_disks_with_status_disk_skipped(self, mock_run_script): mock_run_script.side_effect = [ " Disk 0 Online\n Disk 1 Online\n Disk 2 Online\n", " Disk 0 Online\n Disk 1 Online\n Disk 2 Online\n", - CoriolisTestException() + CoriolisTestException(), ] - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.osmount.windows', level=logging.WARN): self.tools._service_disks_with_status( - self.status, self.service_script_with_id_fmt, + self.status, + self.service_script_with_id_fmt, skip_on_error=False, logmsg_fmt="Operating on disk with index '%s'", - disk_ids_to_skip=disk_ids_to_skip) + disk_ids_to_skip=disk_ids_to_skip, + ) @mock.patch.object(windows.WindowsMountTools, '_service_disks_with_status') def test__set_foreign_disks_rw_mode(self, mock_service_disks_with_status): @@ -144,7 +157,7 @@ def test__set_foreign_disks_rw_mode(self, mock_service_disks_with_status): mock_service_disks_with_status.assert_called_once_with( "Foreign", "SELECT DISK %s\r\nATTRIBUTES DISK CLEAR READONLY\r\nEXIT", - logmsg_fmt="Clearing R/O flag on foreign disk with ID '%s'." + logmsg_fmt="Clearing R/O flag on foreign disk with ID '%s'.", ) @mock.patch.object(windows.WindowsMountTools, '_service_disks_with_status') @@ -155,7 +168,7 @@ def test__import_foreign_disks(self, mock_service_disks_with_status): mock_service_disks_with_status.assert_called_once_with( "Foreign", "SELECT DISK %s\r\nIMPORT\r\nEXIT", - logmsg_fmt="Importing foreign disk with ID '%s'." + logmsg_fmt="Importing foreign disk with ID '%s'.", ) def test__bring_disks_online(self): @@ -163,54 +176,69 @@ def test__bring_disks_online(self): result = self.tools._bring_disks_online() self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call( - "(Get-Disk | Where-Object { $_.IsOffline -eq $True }).Number"), - mock.call("Set-Disk -IsOffline $False 1"), - mock.call("Set-Disk -IsOffline $False 2"), - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "(Get-Disk | Where-Object { $_.IsOffline -eq $True }).Number" + ), + mock.call("Set-Disk -IsOffline $False 1"), + mock.call("Set-Disk -IsOffline $False 2"), + ] + ) def test__bring_disks_online_with_exception(self): self.tools._conn.exec_ps_command.side_effect = [ - "1\n2", None, windows.exception.CoriolisException] - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.WARNING): + "1\n2", + None, + windows.exception.CoriolisException, + ] + with self.assertLogs( + 'coriolis.osmorphing.osmount.windows', level=logging.WARNING + ): self.tools._bring_disks_online() def test__bring_disks_online_disk_nums(self): result = self.tools._bring_disks_online(disk_nums=['1', 2]) self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call("Set-Disk -IsOffline $False 1"), - mock.call("Set-Disk -IsOffline $False 2"), - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call("Set-Disk -IsOffline $False 1"), + mock.call("Set-Disk -IsOffline $False 2"), + ] + ) def test__set_basic_disks_rw_mode(self): self.tools._conn.exec_ps_command.return_value = "1\n2" result = self.tools._set_basic_disks_rw_mode() self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call("(Get-Disk | " - "Where-Object { $_.IsReadOnly -eq $True }).Number"), - mock.call("Set-Disk -IsReadOnly $False 1"), - mock.call("Set-Disk -IsReadOnly $False 2"), - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "(Get-Disk | Where-Object { $_.IsReadOnly -eq $True }).Number" + ), + mock.call("Set-Disk -IsReadOnly $False 1"), + mock.call("Set-Disk -IsReadOnly $False 2"), + ] + ) def test__set_basic_disks_rw_mode_with_exception(self): self.tools._conn.exec_ps_command.side_effect = [ - "1\n2", None, windows.exception.CoriolisException] - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.WARNING): + "1\n2", + None, + windows.exception.CoriolisException, + ] + with self.assertLogs( + 'coriolis.osmorphing.osmount.windows', level=logging.WARNING + ): self.tools._set_basic_disks_rw_mode() def test__get_system_drive(self): result = self.tools._get_system_drive() self.assertEqual(result, self.tools._conn.exec_ps_command.return_value) - self.tools._conn.exec_ps_command.assert_called_once_with( - "$env:SystemDrive") + self.tools._conn.exec_ps_command.assert_called_once_with("$env:SystemDrive") def test__get_fs_roots(self): self.tools._conn.exec_ps_command.return_value = "C:\\\nD:\\\nE:\\" @@ -220,12 +248,15 @@ def test__get_fs_roots(self): self.assertEqual(result, ["C:\\", "D:\\", "E:\\"]) self.tools._conn.exec_ps_command.assert_called_once_with( - "(get-psdrive -PSProvider FileSystem).Root") + "(get-psdrive -PSProvider FileSystem).Root" + ) def test__get_fs_roots_with_exception(self): self.assertRaises( - windows.exception.CoriolisException, self.tools._get_fs_roots, - fail_if_empty=True) + windows.exception.CoriolisException, + self.tools._get_fs_roots, + fail_if_empty=True, + ) def test__bring_nonboot_disks_offline(self): self.tools._conn.exec_ps_command.return_value = "1\n2\n3" @@ -233,13 +264,14 @@ def test__bring_nonboot_disks_offline(self): result = self.tools._bring_nonboot_disks_offline() self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call( - "(Get-Disk | Where-Object { $_.IsBoot -eq $False }).Number"), - mock.call("Set-Disk -IsOffline $True 1"), - mock.call("Set-Disk -IsOffline $True 2"), - mock.call("Set-Disk -IsOffline $True 3") - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call("(Get-Disk | Where-Object { $_.IsBoot -eq $False }).Number"), + mock.call("Set-Disk -IsOffline $True 1"), + mock.call("Set-Disk -IsOffline $True 2"), + mock.call("Set-Disk -IsOffline $True 3"), + ] + ) def test__bring_nonboot_disks_offline_with_exception(self): self.tools._conn.exec_ps_command.side_effect = [ @@ -249,24 +281,27 @@ def test__bring_nonboot_disks_offline_with_exception(self): None, ] - with self.assertLogs('coriolis.osmorphing.osmount.windows', - level=logging.WARNING): + with self.assertLogs( + 'coriolis.osmorphing.osmount.windows', level=logging.WARNING + ): self.tools._bring_nonboot_disks_offline() def test__bring_nonboot_disks_offline_disk_nums(self): result = self.tools._bring_nonboot_disks_offline(disk_nums=['1', 2]) self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call('Set-Disk -IsOffline $True 1'), - mock.call('Set-Disk -IsOffline $True 2'), - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call('Set-Disk -IsOffline $True 1'), + mock.call('Set-Disk -IsOffline $True 2'), + ] + ) - @mock.patch.object(windows.WindowsMountTools, - '_bring_nonboot_disks_offline') + @mock.patch.object(windows.WindowsMountTools, '_bring_nonboot_disks_offline') @mock.patch.object(windows.WindowsMountTools, '_bring_disks_online') - def test__rebring_disks_online(self, bring_disks_online_mock, - bring_disks_offline_mock): + def test__rebring_disks_online( + self, bring_disks_online_mock, bring_disks_offline_mock + ): result = self.tools._rebring_disks_online() self.assertIsNone(result) bring_disks_offline_mock.assert_called() @@ -278,26 +313,33 @@ def test__set_volumes_drive_letter(self, rebring_disks_mock): result = self.tools._set_volumes_drive_letter() self.assertIsNone(result) - self.tools._conn.exec_ps_command.assert_has_calls([ - mock.call( - 'Get-Partition | Where-Object { $_.Type -eq "Basic" -and ' - '$_.NoDefaultDriveLetter -eq $True } | Select-Object -Property' - ' DiskNumber,PartitionNumber'), - mock.call( - 'Set-Partition -NoDefaultDriveLetter $False -DiskNumber 2 ' - '-PartitionNumber 2'), - mock.call( - 'Set-Partition -NoDefaultDriveLetter $False -DiskNumber 3 ' - '-PartitionNumber 4'), - ]) + self.tools._conn.exec_ps_command.assert_has_calls( + [ + mock.call( + 'Get-Partition | Where-Object { $_.Type -eq "Basic" -and ' + '$_.NoDefaultDriveLetter -eq $True } | Select-Object -Property' + ' DiskNumber,PartitionNumber' + ), + mock.call( + 'Set-Partition -NoDefaultDriveLetter $False -DiskNumber 2 ' + '-PartitionNumber 2' + ), + mock.call( + 'Set-Partition -NoDefaultDriveLetter $False -DiskNumber 3 ' + '-PartitionNumber 4' + ), + ] + ) rebring_disks_mock.assert_called_once_with(disk_nums=['2', '3']) @mock.patch.object(windows.WindowsMountTools, '_rebring_disks_online') def test__set_volumes_drive_letter_exception(self, rebring_disks_mock): self.tools._conn.exec_ps_command.side_effect = [ - GET_PARTITION_OUTPUT, None, exception.CoriolisException] - with self.assertLogs( - 'coriolis.osmorphing.osmount.windows', logging.WARNING): + GET_PARTITION_OUTPUT, + None, + exception.CoriolisException, + ] + with self.assertLogs('coriolis.osmorphing.osmount.windows', logging.WARNING): self.tools._set_volumes_drive_letter() rebring_disks_mock.assert_called_once_with(disk_nums=['2']) @@ -306,9 +348,14 @@ def test__set_volumes_drive_letter_exception(self, rebring_disks_mock): @mock.patch.object(windows.WindowsMountTools, '_set_volumes_drive_letter') @mock.patch.object(windows.WindowsMountTools, '_set_basic_disks_rw_mode') @mock.patch.object(windows.WindowsMountTools, '_bring_disks_online') - def test_mount_os(self, mock_bring_disks_online, mock_set_rw_mode, - mock_set_drive_letters, mock_get_fs_roots, - mock_get_system_drive): + def test_mount_os( + self, + mock_bring_disks_online, + mock_set_rw_mode, + mock_set_drive_letters, + mock_get_fs_roots, + mock_get_system_drive, + ): mock_get_fs_roots.return_value = ["C:\\", "D:\\", "E:\\"] mock_get_system_drive.return_value = "C:" self.tools._conn.test_path.side_effect = [True, False] @@ -325,8 +372,9 @@ def test_mount_os_no_root_partition(self): self.tools._conn.exec_ps_command.return_value = "C:\\\nD:\\\nE:\\" self.tools._conn.EOL = "\n" - self.assertRaises(windows.exception.OperatingSystemNotFound, - self.tools.mount_os) + self.assertRaises( + windows.exception.OperatingSystemNotFound, self.tools.mount_os + ) def test_dismount_os(self): root_drive = "C:\\" @@ -335,4 +383,5 @@ def test_dismount_os(self): self.assertIsNone(result) self.tools._conn.exec_ps_command.assert_called_once_with( - '(Get-Disk | Where-Object { $_.IsBoot -eq $False }).Number') + '(Get-Disk | Where-Object { $_.IsBoot -eq $False }).Number' + ) diff --git a/coriolis/tests/osmorphing/test_amazon.py b/coriolis/tests/osmorphing/test_amazon.py index 5ac604c8..87f74691 100644 --- a/coriolis/tests/osmorphing/test_amazon.py +++ b/coriolis/tests/osmorphing/test_amazon.py @@ -12,11 +12,12 @@ class BaseAmazonLinuxOSMorphingToolsTestCase(test_base.CoriolisBaseTestCase): def test_check_os_supported(self): detected_os_info = { "distribution_name": amazon.AMAZON_DISTRO_NAME_IDENTIFIER, - "release_version": "2" + "release_version": "2", } result = amazon.BaseAmazonLinuxOSMorphingTools.check_os_supported( - detected_os_info) + detected_os_info + ) self.assertTrue(result) @@ -25,6 +26,7 @@ def test_check_os_not_supported(self): "distribution_name": 'unsupported', } result = amazon.BaseAmazonLinuxOSMorphingTools.check_os_supported( - detected_os_info) + detected_os_info + ) self.assertFalse(result) diff --git a/coriolis/tests/osmorphing/test_base.py b/coriolis/tests/osmorphing/test_base.py index 2d121a7b..e37102cf 100644 --- a/coriolis/tests/osmorphing/test_base.py +++ b/coriolis/tests/osmorphing/test_base.py @@ -19,74 +19,67 @@ class BaseOSMorphingToolsTestBase(test_base.CoriolisBaseTestCase): """Test suite for the BaseOSMorphingTools class.""" @mock.patch.object(base.BaseOSMorphingTools, '__abstractmethods__', set()) - @mock.patch.object( - base.BaseOSMorphingTools, 'get_required_detected_os_info_fields' - ) + @mock.patch.object(base.BaseOSMorphingTools, 'get_required_detected_os_info_fields') def setUp(self, mock_get_required_fields): super(BaseOSMorphingToolsTestBase, self).setUp() - mock_get_required_fields.return_value = [ - 'distribution_name', 'release_version' - ] + mock_get_required_fields.return_value = ['distribution_name', 'release_version'] self.detected_os_info = { 'distribution_name': mock.sentinel.distribution_name, 'release_version': mock.sentinel.release_version, } self.os_morphing_tools = base.BaseOSMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_device, mock.sentinel.hypervisor, - mock.sentinel.event_manager, self.detected_os_info, + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_device, + mock.sentinel.hypervisor, + mock.sentinel.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) def test_get_required_detected_os_info_fields(self): self.assertRaises( NotImplementedError, - base.BaseOSMorphingTools.get_required_detected_os_info_fields + base.BaseOSMorphingTools.get_required_detected_os_info_fields, ) - @mock.patch.object( - base.BaseOSMorphingTools, 'get_required_detected_os_info_fields' - ) + @mock.patch.object(base.BaseOSMorphingTools, 'get_required_detected_os_info_fields') def test_check_detected_os_info_parameters(self, mock_get_required_fields): - mock_get_required_fields.return_value = [ - 'distribution_name', 'release_version' - ] + mock_get_required_fields.return_value = ['distribution_name', 'release_version'] result = base.BaseOSMorphingTools.check_detected_os_info_parameters( - self.detected_os_info) + self.detected_os_info + ) mock_get_required_fields.assert_called_once_with() self.assertTrue(result) - @mock.patch.object( - base.BaseOSMorphingTools, 'get_required_detected_os_info_fields' - ) + @mock.patch.object(base.BaseOSMorphingTools, 'get_required_detected_os_info_fields') def test_check_detected_os_info_parameters_missing_os_info_fields( - self, mock_get_required_fields): - mock_get_required_fields.return_value = [ - 'distribution_name', 'release_version' - ] + self, mock_get_required_fields + ): + mock_get_required_fields.return_value = ['distribution_name', 'release_version'] # Remove the release_version field in order to trigger the exception. self.detected_os_info.pop('release_version') self.assertRaises( exception.InvalidDetectedOSParams, base.BaseOSMorphingTools.check_detected_os_info_parameters, - self.detected_os_info + self.detected_os_info, ) mock_get_required_fields.assert_called_once_with() - @mock.patch.object( - base.BaseOSMorphingTools, 'get_required_detected_os_info_fields' - ) + @mock.patch.object(base.BaseOSMorphingTools, 'get_required_detected_os_info_fields') def test_check_detected_os_info_parameters_missing_extra_os_info_fields( - self, mock_get_required_fields): + self, mock_get_required_fields + ): # Add an extra field in the detected OS info in order to trigger the # exception. self.detected_os_info['extra_field'] = mock.sentinel.extra_field self.assertRaises( exception.InvalidDetectedOSParams, base.BaseOSMorphingTools.check_detected_os_info_parameters, - self.detected_os_info + self.detected_os_info, ) mock_get_required_fields.assert_called_once_with() @@ -94,13 +87,12 @@ def test_check_os_supported_not_implemented(self): self.assertRaises( NotImplementedError, base.BaseOSMorphingTools.check_os_supported, - self.detected_os_info + self.detected_os_info, ) def test_set_environment(self): self.os_morphing_tools.set_environment(mock.sentinel.environment) - self.assertEqual( - self.os_morphing_tools._environment, mock.sentinel.environment) + self.assertEqual(self.os_morphing_tools._environment, mock.sentinel.environment) # This class is used to test the BaseLinuxOSMorphingTools class since it is @@ -144,47 +136,60 @@ def setUp(self): self.osmorphing_parameters = mock.sentinel.osmorphing_parameters self.operation_timeout = mock.sentinel.operation_timeout self.os_morphing_tools = TestLinuxOSMorphingTools( - self.conn, self.os_root_dir, self.os_root_device, self.hypervisor, - self.event_manager, self.detected_os_info, - self.osmorphing_parameters, self.operation_timeout) + self.conn, + self.os_root_dir, + self.os_root_device, + self.hypervisor, + self.event_manager, + self.detected_os_info, + self.osmorphing_parameters, + self.operation_timeout, + ) @ddt.data( (None, None, None, False), ("1.0", 2.0, None, False), ("2.0", 2.0, 2.0, True), ("3.0", 2.0, 2.5, False), - ("2.5", 2.0, 3.0, True) + ("2.5", 2.0, 3.0, True), ) @ddt.unpack - def test__version_supported_util(self, version, min_version, max_version, - expected_result): + def test__version_supported_util( + self, version, min_version, max_version, expected_result + ): result = self.os_morphing_tools._version_supported_util( - version, min_version, max_version) + version, min_version, max_version + ) self.assertEqual(result, expected_result) @ddt.data( (1.0, 2.0, ValueError), ) @ddt.unpack - def test__version_supported_util_exceptions(self, version, minimum, - expected_exception): + def test__version_supported_util_exceptions( + self, version, minimum, expected_exception + ): self.assertRaises( expected_exception, - self.os_morphing_tools._version_supported_util, version, minimum) + self.os_morphing_tools._version_supported_util, + version, + minimum, + ) def test_version_supported_util_warnings_no_match(self): version = "no match" minimum = 1.0 with self.assertLogs('coriolis.osmorphing.base', level=logging.WARN): result = base.BaseLinuxOSMorphingTools._version_supported_util( - version, minimum) + version, minimum + ) self.assertFalse(result) def test_get_packages(self): self.os_morphing_tools._packages = { None: [('pkg1', False), ('pkg2', True)], 'hypervisor1': [('pkg3', False)], - 'hypervisor2': [('pkg4', True)] + 'hypervisor2': [('pkg4', True)], } self.os_morphing_tools._hypervisor = 'hypervisor1' @@ -194,9 +199,7 @@ def test_get_packages(self): self.assertEqual(remove, ['pkg2', 'pkg4']) def test_get_packages_no_hypervisor(self): - self.os_morphing_tools._packages = { - None: [('pkg1', False), ('pkg2', True)] - } + self.os_morphing_tools._packages = {None: [('pkg1', False), ('pkg2', True)]} self.os_morphing_tools._hypervisor = None add, remove = self.os_morphing_tools.get_packages() @@ -206,8 +209,7 @@ def test_get_packages_no_hypervisor(self): @mock.patch.object(base.utils, 'write_ssh_file') @mock.patch.object(base.utils, 'exec_ssh_cmd') - def test_run_user_script_empty_script(self, mock_exec_ssh_cmd, - mock_write_ssh_file): + def test_run_user_script_empty_script(self, mock_exec_ssh_cmd, mock_write_ssh_file): result = self.os_morphing_tools.run_user_script('') self.assertIsNone(result) mock_write_ssh_file.assert_not_called() @@ -220,31 +222,39 @@ def test_run_user_script(self, mock_exec_ssh_cmd, mock_write_ssh_file): script_path = '/tmp/coriolis_user_script' self.os_morphing_tools.run_user_script(user_script) - mock_write_ssh_file.assert_called_once_with( - self.conn, script_path, user_script) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.conn, "sudo chmod +x %s" % script_path, - get_pty=True), - mock.call(self.conn, 'sudo "%s" "%s"' % ( - script_path, self.os_morphing_tools._os_root_dir), - get_pty=True)]) + mock_write_ssh_file.assert_called_once_with(self.conn, script_path, user_script) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call(self.conn, "sudo chmod +x %s" % script_path, get_pty=True), + mock.call( + self.conn, + 'sudo "%s" "%s"' + % (script_path, self.os_morphing_tools._os_root_dir), + get_pty=True, + ), + ] + ) @mock.patch.object(base.utils, 'write_ssh_file') @mock.patch.object(base.utils, 'exec_ssh_cmd') - def test_run_user_script_with_exception(self, mock_exec_ssh_cmd, - mock_write_ssh_file): + def test_run_user_script_with_exception( + self, mock_exec_ssh_cmd, mock_write_ssh_file + ): user_script = 'echo "Hello, World!"' mock_write_ssh_file.side_effect = exception.CoriolisException self.assertRaises( exception.CoriolisException, - self.os_morphing_tools.run_user_script, user_script) + self.os_morphing_tools.run_user_script, + user_script, + ) mock_exec_ssh_cmd.assert_not_called() @mock.patch.object(base.utils, 'write_ssh_file') @mock.patch.object(base.utils, 'exec_ssh_cmd') - def test_run_user_script_with_exception_on_chmod(self, mock_exec_ssh_cmd, - mock_write_ssh_file): + def test_run_user_script_with_exception_on_chmod( + self, mock_exec_ssh_cmd, mock_write_ssh_file + ): user_script = 'echo "Hello, World!"' script_path = '/tmp/coriolis_user_script' @@ -252,10 +262,11 @@ def test_run_user_script_with_exception_on_chmod(self, mock_exec_ssh_cmd, self.assertRaises( exception.CoriolisException, - self.os_morphing_tools.run_user_script, user_script) + self.os_morphing_tools.run_user_script, + user_script, + ) - mock_write_ssh_file.assert_called_once_with( - self.conn, script_path, user_script) + mock_write_ssh_file.assert_called_once_with(self.conn, script_path, user_script) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_copy_resolv_conf') def test_pre_packages_install(self, mock_copy_resolv_conf): @@ -264,32 +275,31 @@ def test_pre_packages_install(self, mock_copy_resolv_conf): @mock.patch.object(base.BaseLinuxOSMorphingTools, '_restore_resolv_conf') def test_post_packages_install(self, mock_restore_resolv_conf): - self.os_morphing_tools.post_packages_install( - mock.sentinel.package_name) + self.os_morphing_tools.post_packages_install(mock.sentinel.package_name) mock_restore_resolv_conf.assert_called_once_with() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_copy_resolv_conf') def test_pre_packages_uninstall(self, mock_copy_resolv_conf): - self.os_morphing_tools.pre_packages_uninstall( - mock.sentinel.package_name) + self.os_morphing_tools.pre_packages_uninstall(mock.sentinel.package_name) mock_copy_resolv_conf.assert_called_once_with() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_restore_resolv_conf') def test_post_packages_uninstall(self, mock_restore_resolv_conf): - self.os_morphing_tools.post_packages_uninstall( - mock.sentinel.package_name) + self.os_morphing_tools.post_packages_uninstall(mock.sentinel.package_name) mock_restore_resolv_conf.assert_called_once_with() def test_get_update_grub2_command(self): - self.assertRaises(NotImplementedError, - self.os_morphing_tools.get_update_grub2_command) + self.assertRaises( + NotImplementedError, self.os_morphing_tools.get_update_grub2_command + ) @mock.patch.object(base.utils, 'test_ssh_path') def test__test_path(self, mock_test_ssh_path): result = self.os_morphing_tools._test_path(self.chroot_path) mock_test_ssh_path.assert_called_once_with( - self.os_morphing_tools._ssh, self.joined_chroot_path) + self.os_morphing_tools._ssh, self.joined_chroot_path + ) self.assertEqual(result, mock_test_ssh_path.return_value) @mock.patch.object(base.utils, 'read_ssh_file') @@ -297,35 +307,38 @@ def test__read_file(self, mock_read_ssh_file): result = self.os_morphing_tools._read_file(self.chroot_path) mock_read_ssh_file.assert_called_once_with( - self.os_morphing_tools._ssh, self.joined_chroot_path) + self.os_morphing_tools._ssh, self.joined_chroot_path + ) self.assertEqual(result, mock_read_ssh_file.return_value) @mock.patch.object(base.utils, 'write_ssh_file') def test__write_file(self, mock_write_ssh_file): - self.os_morphing_tools._write_file( - self.chroot_path, mock.sentinel.content) + self.os_morphing_tools._write_file(self.chroot_path, mock.sentinel.content) mock_write_ssh_file.assert_called_once_with( - self.os_morphing_tools._ssh, self.joined_chroot_path, - mock.sentinel.content) + self.os_morphing_tools._ssh, self.joined_chroot_path, mock.sentinel.content + ) @mock.patch.object(base.utils, 'list_ssh_dir') def test__list_dir(self, mock_list_ssh_dir): result = self.os_morphing_tools._list_dir(self.chroot_path) mock_list_ssh_dir.assert_called_once_with( - self.os_morphing_tools._ssh, self.joined_chroot_path) + self.os_morphing_tools._ssh, self.joined_chroot_path + ) self.assertEqual(result, mock_list_ssh_dir.return_value) @mock.patch.object(base.utils, 'exec_ssh_cmd') def test__exec_cmd(self, mock_exec_ssh_cmd): - result = self.os_morphing_tools._exec_cmd( - mock.sentinel.cmd, timeout=120) + result = self.os_morphing_tools._exec_cmd(mock.sentinel.cmd, timeout=120) mock_exec_ssh_cmd.assert_called_once_with( - self.os_morphing_tools._ssh, mock.sentinel.cmd, - environment=self.os_morphing_tools._environment, get_pty=True, - timeout=120) + self.os_morphing_tools._ssh, + mock.sentinel.cmd, + environment=self.os_morphing_tools._environment, + get_pty=True, + timeout=120, + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @@ -334,9 +347,12 @@ def test__exec_cmd_without_timeout(self, mock_exec_ssh_cmd): result = self.os_morphing_tools._exec_cmd(mock.sentinel.cmd) mock_exec_ssh_cmd.assert_called_once_with( - self.os_morphing_tools._ssh, mock.sentinel.cmd, - environment=self.os_morphing_tools._environment, get_pty=True, - timeout=self.os_morphing_tools._osmorphing_operation_timeout) + self.os_morphing_tools._ssh, + mock.sentinel.cmd, + environment=self.os_morphing_tools._environment, + get_pty=True, + timeout=self.os_morphing_tools._osmorphing_operation_timeout, + ) self.assertEqual(result, mock_exec_ssh_cmd.return_value) @mock.patch.object(base.utils, 'exec_ssh_cmd') @@ -345,17 +361,22 @@ def test__exec_cmd_with_exception(self, mock_exec_ssh_cmd): self.assertRaises( exception.OSMorphingSSHOperationTimeout, - self.os_morphing_tools._exec_cmd, mock.sentinel.cmd) + self.os_morphing_tools._exec_cmd, + mock.sentinel.cmd, + ) @mock.patch.object(base.utils, 'exec_ssh_cmd_chroot') def test__exec_cmd_chroot(self, mock_exec_ssh_cmd_chroot): - result = self.os_morphing_tools._exec_cmd_chroot( - mock.sentinel.cmd, timeout=120) + result = self.os_morphing_tools._exec_cmd_chroot(mock.sentinel.cmd, timeout=120) mock_exec_ssh_cmd_chroot.assert_called_once_with( - self.os_morphing_tools._ssh, self.os_morphing_tools._os_root_dir, - mock.sentinel.cmd, environment=self.os_morphing_tools._environment, - get_pty=True, timeout=120) + self.os_morphing_tools._ssh, + self.os_morphing_tools._os_root_dir, + mock.sentinel.cmd, + environment=self.os_morphing_tools._environment, + get_pty=True, + timeout=120, + ) self.assertEqual(result, mock_exec_ssh_cmd_chroot.return_value) @mock.patch.object(base.utils, 'exec_ssh_cmd_chroot') @@ -363,121 +384,132 @@ def test__exec_cmd_chroot_without_timeout(self, mock_exec_ssh_cmd_chroot): result = self.os_morphing_tools._exec_cmd_chroot(mock.sentinel.cmd) mock_exec_ssh_cmd_chroot.assert_called_once_with( - self.os_morphing_tools._ssh, self.os_morphing_tools._os_root_dir, - mock.sentinel.cmd, environment=self.os_morphing_tools._environment, + self.os_morphing_tools._ssh, + self.os_morphing_tools._os_root_dir, + mock.sentinel.cmd, + environment=self.os_morphing_tools._environment, get_pty=True, - timeout=self.os_morphing_tools._osmorphing_operation_timeout) + timeout=self.os_morphing_tools._osmorphing_operation_timeout, + ) self.assertEqual(result, mock_exec_ssh_cmd_chroot.return_value) @mock.patch.object(base.utils, 'exec_ssh_cmd_chroot') def test__exec_cmd_chroot_with_exception(self, mock_exec_ssh_cmd_chroot): - mock_exec_ssh_cmd_chroot.side_effect = [ - exception.MinionMachineCommandTimeout()] + mock_exec_ssh_cmd_chroot.side_effect = [exception.MinionMachineCommandTimeout()] self.assertRaises( exception.OSMorphingSSHOperationTimeout, - self.os_morphing_tools._exec_cmd_chroot, mock.sentinel.cmd) + self.os_morphing_tools._exec_cmd_chroot, + mock.sentinel.cmd, + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__check_user_exists(self, mock_exec_cmd_chroot): - result = self.os_morphing_tools._check_user_exists( - mock.sentinel.username) + result = self.os_morphing_tools._check_user_exists(mock.sentinel.username) mock_exec_cmd_chroot.assert_called_once_with( - 'id -u %s' % mock.sentinel.username) + 'id -u %s' % mock.sentinel.username + ) self.assertTrue(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__check_user_exists_with_exception(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = CoriolisTestException() - result = self.os_morphing_tools._check_user_exists( - mock.sentinel.username) + result = self.os_morphing_tools._check_user_exists(mock.sentinel.username) self.assertFalse(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.uuid, 'uuid4') @mock.patch.object(base.utils, 'exec_ssh_cmd') - def test__write_file_sudo(self, mock_exec_ssh_cmd, mock_uuid, - mock_exec_cmd, mock_write_file): + def test__write_file_sudo( + self, mock_exec_ssh_cmd, mock_uuid, mock_exec_cmd, mock_write_file + ): self.os_morphing_tools._write_file_sudo( - mock.sentinel.chroot_path, mock.sentinel.content) + mock.sentinel.chroot_path, mock.sentinel.content + ) mock_write_file.assert_called_once_with( - 'tmp/%s' % mock_uuid.return_value, mock.sentinel.content) - mock_exec_cmd.assert_has_calls([ - mock.call('cp /tmp/%s /%s' % ( - mock_uuid.return_value, mock.sentinel.chroot_path)), - mock.call('rm /tmp/%s' % mock_uuid.return_value)]) + 'tmp/%s' % mock_uuid.return_value, mock.sentinel.content + ) + mock_exec_cmd.assert_has_calls( + [ + mock.call( + 'cp /tmp/%s /%s' + % (mock_uuid.return_value, mock.sentinel.chroot_path) + ), + mock.call('rm /tmp/%s' % mock_uuid.return_value), + ] + ) mock_exec_ssh_cmd.assert_called_once_with( - self.os_morphing_tools._ssh, 'sudo sync', - self.os_morphing_tools._environment, get_pty=True) + self.os_morphing_tools._ssh, + 'sudo sync', + self.os_morphing_tools._environment, + get_pty=True, + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__enable_systemd_service(self, mock_exec_cmd_chroot): - self.os_morphing_tools._enable_systemd_service( - mock.sentinel.service_name) + self.os_morphing_tools._enable_systemd_service(mock.sentinel.service_name) mock_exec_cmd_chroot.assert_called_once_with( - 'systemctl enable %s.service' % mock.sentinel.service_name) + 'systemctl enable %s.service' % mock.sentinel.service_name + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__disable_systemd_service(self, mock_exec_cmd_chroot): - self.os_morphing_tools._disable_systemd_service( - mock.sentinel.service_name) + self.os_morphing_tools._disable_systemd_service(mock.sentinel.service_name) mock_exec_cmd_chroot.assert_called_once_with( - 'systemctl disable %s.service' % mock.sentinel.service_name) + 'systemctl disable %s.service' % mock.sentinel.service_name + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__disable_upstart_service(self, mock_exec_cmd_chroot): - self.os_morphing_tools._disable_upstart_service( - mock.sentinel.service_name) + self.os_morphing_tools._disable_upstart_service(mock.sentinel.service_name) mock_exec_cmd_chroot.assert_called_once_with( - 'echo manual | tee /etc/init/%s.override' % - mock.sentinel.service_name) + 'echo manual | tee /etc/init/%s.override' % mock.sentinel.service_name + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_config_file') def test__get_os_release(self, mock_read_config_file): result = self.os_morphing_tools._get_os_release() mock_read_config_file.assert_called_once_with( - 'etc/os-release', check_exists=True) + 'etc/os-release', check_exists=True + ) self.assertEqual(result, mock_read_config_file.return_value) @mock.patch.object(base.utils, 'read_ssh_ini_config_file') def test__read_config_file(self, mock_read_ssh_ini): result = self.os_morphing_tools._read_config_file( - self.chroot_path, check_exists=False) + self.chroot_path, check_exists=False + ) mock_read_ssh_ini.assert_called_once_with( - self.os_morphing_tools._ssh, self.joined_chroot_path, - check_exists=False) + self.os_morphing_tools._ssh, self.joined_chroot_path, check_exists=False + ) self.assertEqual(result, mock_read_ssh_ini.return_value) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') - def test__read_config_file_sudo( - self, mock_read_file_sudo, mock_test_path_chroot): + def test__read_config_file_sudo(self, mock_read_file_sudo, mock_test_path_chroot): mock_test_path_chroot.return_value = True mock_read_file_sudo.return_value = '[connection]\ntype=ethernet' - result = self.os_morphing_tools._read_config_file_sudo( - self.chroot_path) + result = self.os_morphing_tools._read_config_file_sudo(self.chroot_path) mock_read_file_sudo.assert_called_once_with(self.chroot_path) self.assertEqual(result, {'type': 'ethernet'}) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') - def test__read_config_file_none( - self, mock_read_file_sudo, mock_test_path_chroot): + def test__read_config_file_none(self, mock_read_file_sudo, mock_test_path_chroot): mock_test_path_chroot.return_value = False - result = self.os_morphing_tools._read_config_file_sudo( - self.chroot_path) + result = self.os_morphing_tools._read_config_file_sudo(self.chroot_path) mock_read_file_sudo.assert_not_called() self.assertEqual(result, {}) @@ -485,14 +517,16 @@ def test__read_config_file_none( @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') def test__read_config_file_sudo_raises( - self, mock_read_file_sudo, mock_test_path_chroot): + self, mock_read_file_sudo, mock_test_path_chroot + ): mock_test_path_chroot.return_value = False self.assertRaises( IOError, self.os_morphing_tools._read_config_file_sudo, self.chroot_path, - check_exists=True) + check_exists=True, + ) mock_read_file_sudo.assert_not_called() @@ -508,11 +542,17 @@ def test__copy_resolv_conf(self, mock_exec_cmd, mock_test_path): self.os_morphing_tools._copy_resolv_conf() mock_test_path.assert_called_once_with(mocked_resolv_conf) - mock_exec_cmd.assert_has_calls([ - mock.call('sudo mv -f %s %s' % ( - mocked_full_path, mocked_full_path_old)), - mock.call('sudo cp -L --remove-destination /etc/resolv.conf %s' % - mocked_full_path)]) + mock_exec_cmd.assert_has_calls( + [ + mock.call( + 'sudo mv -f %s %s' % (mocked_full_path, mocked_full_path_old) + ), + mock.call( + 'sudo cp -L --remove-destination /etc/resolv.conf %s' + % mocked_full_path + ), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @@ -527,98 +567,104 @@ def test__restore_resolv_conf(self, mock_exec_cmd, mock_test_path): mock_test_path.assert_called_once_with(mocked_resolv_conf_old) mock_exec_cmd.assert_called_once_with( - 'sudo mv -f %s %s' % (mocked_full_path_old, mocked_full_path)) + 'sudo mv -f %s %s' % (mocked_full_path_old, mocked_full_path) + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__replace_fstab_entries_device_prefix( - self, mock_exec_cmd_chroot, mock_write_file, mock_read_file): + self, mock_exec_cmd_chroot, mock_write_file, mock_read_file + ): fstab_chroot_path = "etc/fstab" current_prefix = "/dev/sd" new_prefix = "/dev/vd" mock_read_file.return_value = ( - b"/dev/sda1 / ext4 defaults 0 0\n" - b"/dev/sdb1 /home ext4 defaults 0 0") + b"/dev/sda1 / ext4 defaults 0 0\n/dev/sdb1 /home ext4 defaults 0 0" + ) self.os_morphing_tools._replace_fstab_entries_device_prefix( - current_prefix, new_prefix) + current_prefix, new_prefix + ) mock_read_file.assert_called_once_with(fstab_chroot_path) mock_exec_cmd_chroot.assert_called_once_with( - "mv -f /%s /%s.bak" % (fstab_chroot_path, fstab_chroot_path)) + "mv -f /%s /%s.bak" % (fstab_chroot_path, fstab_chroot_path) + ) mock_write_file.assert_called_once_with( fstab_chroot_path, - "/dev/vda1 / ext4 defaults 0 0\n/dev/vdb1 /home ext4 defaults 0 0") + "/dev/vda1 / ext4 defaults 0 0\n/dev/vdb1 /home ext4 defaults 0 0", + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__set_selinux_autorelabel(self, mock_exec_cmd_chroot): self.os_morphing_tools._set_selinux_autorelabel() - mock_exec_cmd_chroot.assert_called_once_with( - 'touch /.autorelabel') + mock_exec_cmd_chroot.assert_called_once_with('touch /.autorelabel') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - def test__set_selinux_autorelabel_with_exception(self, - mock_exec_cmd_chroot): + def test__set_selinux_autorelabel_with_exception(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = CoriolisTestException() - with self.assertLogs('coriolis.osmorphing.base', - level=logging.WARNING): + with self.assertLogs('coriolis.osmorphing.base', level=logging.WARNING): self.os_morphing_tools._set_selinux_autorelabel() @mock.patch.object(base.BaseLinuxOSMorphingTools, "_write_file_sudo") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__write_cloud_init_mods_config( - self, mock__test_path, mock__exec_cmd_chroot, - mock__write_file_sudo): + self, mock__test_path, mock__exec_cmd_chroot, mock__write_file_sudo + ): mock__test_path.return_value = True cloud_cfg = { "mock_key1": {"mock_key2": "mock_value1"}, - "mock_key3": "mock_value2"} + "mock_key3": "mock_value2", + } self.os_morphing_tools._write_cloud_init_mods_config(cloud_cfg) mock__exec_cmd_chroot.assert_not_called() mock__write_file_sudo.assert_called_once_with( "/etc/cloud/cloud.cfg.d/99_coriolis.cfg", - 'mock_key1:\n mock_key2: mock_value1\nmock_key3: mock_value2\n') + 'mock_key1:\n mock_key2: mock_value1\nmock_key3: mock_value2\n', + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, "_write_file_sudo") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__write_cloud_init_mods_config_no_directory( - self, mock__test_path, mock__exec_cmd_chroot, - mock__write_file_sudo): + self, mock__test_path, mock__exec_cmd_chroot, mock__write_file_sudo + ): mock__test_path.return_value = False cloud_cfg = {} self.os_morphing_tools._write_cloud_init_mods_config(cloud_cfg) - mock__exec_cmd_chroot.assert_called_once_with( - "mkdir -p /etc/cloud/cloud.cfg.d") + mock__exec_cmd_chroot.assert_called_once_with("mkdir -p /etc/cloud/cloud.cfg.d") mock__write_file_sudo.assert_called_once_with( - "/etc/cloud/cloud.cfg.d/99_coriolis.cfg", - '{}\n') + "/etc/cloud/cloud.cfg.d/99_coriolis.cfg", '{}\n' + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__disable_installer_cloud_config( - self, mock__test_path, mock__exec_cmd_chroot): + self, mock__test_path, mock__exec_cmd_chroot + ): mock__test_path.return_value = True self.os_morphing_tools._disable_installer_cloud_config() mock__exec_cmd_chroot.assert_called_once_with( "mv /etc/cloud/cloud.cfg.d/99-installer.cfg " - "/etc/cloud/cloud.cfg.d/99-installer.cfg.bak") + "/etc/cloud/cloud.cfg.d/99-installer.cfg.bak" + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__disable_installer_cloud_config_no_file( - self, mock__test_path, mock__exec_cmd_chroot): + self, mock__test_path, mock__exec_cmd_chroot + ): mock__test_path.return_value = False self.os_morphing_tools._disable_installer_cloud_config() @@ -627,27 +673,40 @@ def test__disable_installer_cloud_config_no_file( @ddt.data( ((False, False, False), [], False), - ((True, True, False), [ - "rm /etc/cloud/cloud-init.disabled", - "sed -i '/cloud-init=disabled/d' /etc/systemd/system.conf", - ], False), - ((False, False, True), [ - "sed -i '/cloud-init=disabled/d' /etc/default/grub", - ], True) + ( + (True, True, False), + [ + "rm /etc/cloud/cloud-init.disabled", + "sed -i '/cloud-init=disabled/d' /etc/systemd/system.conf", + ], + False, + ), + ( + (False, False, True), + [ + "sed -i '/cloud-init=disabled/d' /etc/default/grub", + ], + True, + ), ) @ddt.unpack @mock.patch.object(base.BaseLinuxOSMorphingTools, "_execute_update_grub") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__ensure_cloud_init_not_disabled( - self, test_path_results, expected_cmds, updates_grub, - mock__test_path, mock__exec_cmd_chroot, mock__execute_update_grub): + self, + test_path_results, + expected_cmds, + updates_grub, + mock__test_path, + mock__exec_cmd_chroot, + mock__execute_update_grub, + ): mock__test_path.side_effect = test_path_results self.os_morphing_tools._ensure_cloud_init_not_disabled() - called_cmds = [ - call.args[0] for call in mock__exec_cmd_chroot.call_args_list] + called_cmds = [call.args[0] for call in mock__exec_cmd_chroot.call_args_list] self.assertEqual(called_cmds, expected_cmds) if updates_grub: mock__execute_update_grub.assert_called_once() @@ -658,22 +717,29 @@ def test__ensure_cloud_init_not_disabled( def test__reset_cloud_init_run(self, mock__exec_cmd_chroot): self.os_morphing_tools._reset_cloud_init_run() - mock__exec_cmd_chroot.assert_called_once_with( - "cloud-init clean --logs") + mock__exec_cmd_chroot.assert_called_once_with("cloud-init clean --logs") @ddt.data( (False, None, None), - (True, "other_modules:\n - not_update_etc_hosts\n" - "cloud_init_modules:\n - set_hostname\n - update_etc_hosts\n", - ["set_hostname", "update_etc_hosts"]), - (True, "other_modules:\n - update_etc_hosts\n", []) + ( + True, + "other_modules:\n - not_update_etc_hosts\n" + "cloud_init_modules:\n - set_hostname\n - update_etc_hosts\n", + ["set_hostname", "update_etc_hosts"], + ), + (True, "other_modules:\n - update_etc_hosts\n", []), ) @ddt.unpack @mock.patch.object(base.BaseLinuxOSMorphingTools, "_read_file_sudo") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__get_cloud_init_modules( - self, test_path_result, file_content, - expected_result, mock__test_path, mock__read_file_sudo): + self, + test_path_result, + file_content, + expected_result, + mock__test_path, + mock__read_file_sudo, + ): mock__test_path.return_value = test_path_result mock__read_file_sudo.return_value = file_content @@ -683,16 +749,20 @@ def test__get_cloud_init_modules( @ddt.data( (False, None, base.DEFAULT_CLOUD_USER), - (True, "system_info:\n default_user:\n name: mock_user\n", - "mock_user"), + (True, "system_info:\n default_user:\n name: mock_user\n", "mock_user"), (True, "{}", base.DEFAULT_CLOUD_USER), ) @ddt.unpack @mock.patch.object(base.BaseLinuxOSMorphingTools, "_read_file_sudo") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_test_path") def test__get_default_cloud_user( - self, test_path_result, file_content, expected_user, - mock__test_path, mock__read_file_sudo): + self, + test_path_result, + file_content, + expected_user, + mock__test_path, + mock__read_file_sudo, + ): mock__test_path.return_value = test_path_result mock__read_file_sudo.return_value = file_content @@ -702,11 +772,13 @@ def test__get_default_cloud_user( @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_check_user_exists") - @mock.patch.object(base.BaseLinuxOSMorphingTools, - "_get_default_cloud_user") + @mock.patch.object(base.BaseLinuxOSMorphingTools, "_get_default_cloud_user") def test__create_cloudinit_user( - self, mock__get_default_cloud_user, - mock__check_user_exists, mock__exec_cmd_chroot): + self, + mock__get_default_cloud_user, + mock__check_user_exists, + mock__exec_cmd_chroot, + ): mock__get_default_cloud_user.return_value = "mock_user" mock__check_user_exists.return_value = False @@ -716,11 +788,13 @@ def test__create_cloudinit_user( @mock.patch.object(base.BaseLinuxOSMorphingTools, "_exec_cmd_chroot") @mock.patch.object(base.BaseLinuxOSMorphingTools, "_check_user_exists") - @mock.patch.object(base.BaseLinuxOSMorphingTools, - "_get_default_cloud_user") + @mock.patch.object(base.BaseLinuxOSMorphingTools, "_get_default_cloud_user") def test__create_cloudinit_user_already_exists( - self, mock__get_default_cloud_user, - mock__check_user_exists, mock__exec_cmd_chroot): + self, + mock__get_default_cloud_user, + mock__check_user_exists, + mock__exec_cmd_chroot, + ): mock__get_default_cloud_user.return_value = "mock_user" mock__check_user_exists.return_value = True @@ -729,14 +803,7 @@ def test__create_cloudinit_user_already_exists( mock__exec_cmd_chroot.assert_not_called() @ddt.data( - ( - ["vim"], - {}, - False, - [], - None, - False - ), + (["vim"], {}, False, [], None, False), ( ["cloud-init"], {"retain_user_credentials": True, "set_dhcp": False}, @@ -748,7 +815,7 @@ def test__create_cloudinit_user_already_exists( "users": None, "network": {"config": "disabled"}, }, - True + True, ), ( ["cloud-init", "vim"], @@ -756,7 +823,7 @@ def test__create_cloudinit_user_already_exists( True, [], {}, - False + False, ), ( ["cloud-init", "vim"], @@ -765,35 +832,38 @@ def test__create_cloudinit_user_already_exists( ["update_etc_hosts", "set_hostname", "write_files"], { "network": {"config": "disabled"}, - "cloud_init_modules": ["set_hostname", "write_files"] + "cloud_init_modules": ["set_hostname", "write_files"], }, - False + False, ), ) @ddt.unpack - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_enable_systemd_service') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_enable_systemd_service') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_has_systemd_chroot') - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_get_cloud_init_modules') - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_write_cloud_init_mods_config') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_get_cloud_init_modules') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_cloud_init_mods_config') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_create_cloudinit_user') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_reset_cloud_init_run') - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_ensure_cloud_init_not_disabled') - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_disable_installer_cloud_config') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_ensure_cloud_init_not_disabled') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_disable_installer_cloud_config') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'get_packages') def test__configure_cloud_init( - self, returned_packages, osmorphing_params, creates_cloudinit_user, - cloud_init_modules, - expected_result, has_systemd_chroot, mock_get_packages, - mock__disable_installer_cloud_config, - mock__ensure_cloud_init_not_disabled, mock__reset_cloud_init_run, - mock__create_cloudinit_user, mock__write_cloud_init_mods_config, - mock__get_cloud_init_modules, - mock__has_systemd_chroot, mock__enable_systemd_service + self, + returned_packages, + osmorphing_params, + creates_cloudinit_user, + cloud_init_modules, + expected_result, + has_systemd_chroot, + mock_get_packages, + mock__disable_installer_cloud_config, + mock__ensure_cloud_init_not_disabled, + mock__reset_cloud_init_run, + mock__create_cloudinit_user, + mock__write_cloud_init_mods_config, + mock__get_cloud_init_modules, + mock__has_systemd_chroot, + mock__enable_systemd_service, ): mock_get_packages.return_value = returned_packages self.os_morphing_tools._osmorphing_parameters = osmorphing_params @@ -805,15 +875,13 @@ def test__configure_cloud_init( if expected_result is not None: mock__ensure_cloud_init_not_disabled.assert_called_once() mock__reset_cloud_init_run.assert_called_once() - mock__write_cloud_init_mods_config.assert_called_once_with( - expected_result) + mock__write_cloud_init_mods_config.assert_called_once_with(expected_result) if creates_cloudinit_user: mock__create_cloudinit_user.assert_called_once() else: mock__create_cloudinit_user.assert_not_called() if has_systemd_chroot: - mock__enable_systemd_service.assert_called_once_with( - "cloud-init") + mock__enable_systemd_service.assert_called_once_with("cloud-init") else: mock__enable_systemd_service.assert_not_called() else: @@ -827,7 +895,8 @@ def test__test_path_chroot(self, mock_exec_cmd_chroot): result = self.os_morphing_tools._test_path_chroot(path) mock_exec_cmd_chroot.assert_called_once_with( - '[ -f "%s" ] && echo 1 || echo 0' % path) + '[ -f "%s" ] && echo 1 || echo 0' % path + ) self.assertTrue(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @@ -838,7 +907,8 @@ def test__test_path_chroot_no_leading_slash(self, mock_exec_cmd_chroot): result = self.os_morphing_tools._test_path_chroot(path) mock_exec_cmd_chroot.assert_called_once_with( - '[ -f "/%s" ] && echo 1 || echo 0' % path) + '[ -f "/%s" ] && echo 1 || echo 0' % path + ) self.assertTrue(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @@ -861,8 +931,9 @@ def test__read_file_sudo_no_leading_slash(self, mock_exec_cmd_chroot): @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__read_grub_config_file_exists(self, mock_test_path_chroot, - mock_read_file_sudo): + def test__read_grub_config_file_exists( + self, mock_test_path_chroot, mock_read_file_sudo + ): config = mock.sentinel.config file_contents = 'key1="value1"\n#comment\nkey2="value2"\ninvalid_line' @@ -878,11 +949,13 @@ def test__read_grub_config_file_exists(self, mock_test_path_chroot, @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') def test__read_grub_config_file_not_exists( - self, mock_test_path_chroot, mock_read_file_sudo): + self, mock_test_path_chroot, mock_read_file_sudo + ): mock_test_path_chroot.return_value = False - self.assertRaises(IOError, self.os_morphing_tools._read_grub_config, - mock.sentinel.config) + self.assertRaises( + IOError, self.os_morphing_tools._read_grub_config, mock.sentinel.config + ) mock_read_file_sudo.assert_not_called() @@ -890,29 +963,30 @@ def test__read_grub_config_file_not_exists( @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') def test__get_grub_config_obj_file_exists( - self, mock_test_path_chroot, mock_exec_cmd_chroot, - mock_read_grub_config): + self, mock_test_path_chroot, mock_exec_cmd_chroot, mock_read_grub_config + ): grub_conf = "/etc/default/grub" tmp_file = "/tmp/tmp_file" mock_test_path_chroot.return_value = True mock_exec_cmd_chroot.side_effect = [tmp_file, None] - mock_read_grub_config.return_value = ( - mock_exec_cmd_chroot.return_value) + mock_read_grub_config.return_value = mock_exec_cmd_chroot.return_value result = self.os_morphing_tools._get_grub_config_obj(grub_conf) mock_test_path_chroot.assert_called_once_with(grub_conf) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call('mktemp'), - mock.call('/bin/cp -fp %s %s' % (grub_conf, tmp_file)) - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call('mktemp'), + mock.call('/bin/cp -fp %s %s' % (grub_conf, tmp_file)), + ] + ) mock_read_grub_config.assert_called_once_with(tmp_file) expected_result = { 'source': grub_conf, 'location': tmp_file, - 'contents': mock_read_grub_config.return_value + 'contents': mock_read_grub_config.return_value, } self.assertEqual(result, expected_result) @@ -921,14 +995,15 @@ def test__get_grub_config_obj_file_exists( @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') def test__get_grub_config_obj_file_not_exists( - self, mock_test_path_chroot, mock_exec_cmd_chroot, - mock_read_grub_config): + self, mock_test_path_chroot, mock_exec_cmd_chroot, mock_read_grub_config + ): grub_conf = "/etc/default/grub" mock_test_path_chroot.return_value = False self.assertRaises( - IOError, self.os_morphing_tools._get_grub_config_obj, grub_conf) + IOError, self.os_morphing_tools._get_grub_config_obj, grub_conf + ) mock_exec_cmd_chroot.assert_not_called() mock_read_grub_config.assert_not_called() @@ -936,15 +1011,15 @@ def test__get_grub_config_obj_file_not_exists( def test__validate_grub_config_obj_not_dict(self): config_obj = "invalid config_obj" - self.assertRaises(ValueError, - self.os_morphing_tools._validate_grub_config_obj, - config_obj) + self.assertRaises( + ValueError, self.os_morphing_tools._validate_grub_config_obj, config_obj + ) def test__validate_grub_config_obj_valid(self): config_obj = { 'location': mock.sentinel.location, 'source': mock.sentinel.source, - 'contents': mock.sentinel.contents + 'contents': mock.sentinel.contents, } # Should not raise any exceptions @@ -953,22 +1028,19 @@ def test__validate_grub_config_obj_valid(self): def test__validate_grub_config_obj_missing_keys(self): config_obj = {'location': mock.sentinel.location} - self.assertRaises(ValueError, - self.os_morphing_tools._validate_grub_config_obj, - config_obj) + self.assertRaises( + ValueError, self.os_morphing_tools._validate_grub_config_obj, config_obj + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - def test_set_grub_value_append(self, mock_exec_cmd_chroot, - mock_read_file_sudo): + def test_set_grub_value_append(self, mock_exec_cmd_chroot, mock_read_file_sudo): option = 'option' value = 'value' config_obj = { 'location': '/tmp/tmp_file', 'source': '/etc/default/grub', - 'contents': { - 'GRUB_DEFAULT': '0' - }, + 'contents': {'GRUB_DEFAULT': '0'}, } cfg = 'cfg' @@ -977,15 +1049,13 @@ def test_set_grub_value_append(self, mock_exec_cmd_chroot, self.os_morphing_tools.set_grub_value(option, value, config_obj) mock_exec_cmd_chroot.assert_called_once_with( - 'sed -ie \'$a%s="%s"\' %s' % ( - option, value, config_obj['location']) + 'sed -ie \'$a%s="%s"\' %s' % (option, value, config_obj['location']) ) mock_read_file_sudo.assert_called_once_with(config_obj['location']) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - def test_set_grub_value_replace(self, mock_exec_cmd_chroot, - mock_read_file_sudo): + def test_set_grub_value_replace(self, mock_exec_cmd_chroot, mock_read_file_sudo): option = 'option' value = 'value' config_obj = { @@ -1000,8 +1070,8 @@ def test_set_grub_value_replace(self, mock_exec_cmd_chroot, self.os_morphing_tools.set_grub_value(option, value, config_obj) mock_exec_cmd_chroot.assert_called_once_with( - 'sed -i \'s|^%s=.*|%s="%s"|g\' %s' % (option, option, value, - config_obj['location']) + 'sed -i \'s|^%s=.*|%s="%s"|g\' %s' + % (option, option, value, config_obj['location']) ) mock_read_file_sudo.assert_called_once_with(config_obj['location']) @@ -1015,11 +1085,11 @@ def test__set_grub2_cmdline_clobber(self, mock_set_grub_value): } options = ['option1', 'option2'] - self.os_morphing_tools._set_grub2_cmdline(config_obj, options, - clobber=True) + self.os_morphing_tools._set_grub2_cmdline(config_obj, options, clobber=True) mock_set_grub_value.assert_called_once_with( - 'GRUB_CMDLINE_LINUX', ' '.join(options), config_obj, replace=True) + 'GRUB_CMDLINE_LINUX', ' '.join(options), config_obj, replace=True + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, 'set_grub_value') def test__set_grub2_cmdline_add_options(self, mock_set_grub_value): @@ -1031,12 +1101,14 @@ def test__set_grub2_cmdline_add_options(self, mock_set_grub_value): } options = ['option1', 'option2'] - self.os_morphing_tools._set_grub2_cmdline(config_obj, options, - clobber=False) + self.os_morphing_tools._set_grub2_cmdline(config_obj, options, clobber=False) mock_set_grub_value.assert_called_once_with( - 'GRUB_CMDLINE_LINUX', 'quiet_linux option1 option2', config_obj, - replace=True) + 'GRUB_CMDLINE_LINUX', + 'quiet_linux option1 option2', + config_obj, + replace=True, + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, 'set_grub_value') def test__set_grub2_cmdline_no_options_to_add(self, mock_set_grub_value): @@ -1048,16 +1120,14 @@ def test__set_grub2_cmdline_no_options_to_add(self, mock_set_grub_value): } options = ['option1'] - self.os_morphing_tools._set_grub2_cmdline(config_obj, options, - clobber=False) + self.os_morphing_tools._set_grub2_cmdline(config_obj, options, clobber=False) mock_set_grub_value.assert_not_called() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - @mock.patch.object( - base.BaseLinuxOSMorphingTools, 'get_update_grub2_command' - ) - def test__execute_update_grub(self, mock_get_update_grub2_command, - mock_exec_cmd_chroot): + @mock.patch.object(base.BaseLinuxOSMorphingTools, 'get_update_grub2_command') + def test__execute_update_grub( + self, mock_get_update_grub2_command, mock_exec_cmd_chroot + ): self.os_morphing_tools._execute_update_grub() mock_get_update_grub2_command.assert_called_once_with() @@ -1067,18 +1137,14 @@ def test__execute_update_grub(self, mock_get_update_grub2_command, @mock.patch.object(base.BaseLinuxOSMorphingTools, '_execute_update_grub') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - def test__apply_grub2_config(self, mock_exec_cmd_chroot, - mock_execute_update_grub): + def test__apply_grub2_config(self, mock_exec_cmd_chroot, mock_execute_update_grub): config_obj = { 'location': mock.sentinel.location, 'source': mock.sentinel.source, - 'contents': { - 'GRUB_DEFAULT': '0' - }, + 'contents': {'GRUB_DEFAULT': '0'}, } - self.os_morphing_tools._apply_grub2_config(config_obj, - execute_update_grub=True) + self.os_morphing_tools._apply_grub2_config(config_obj, execute_update_grub=True) mock_exec_cmd_chroot.assert_called_once_with( 'mv -f %s %s' % (config_obj['location'], config_obj['source']) @@ -1087,18 +1153,18 @@ def test__apply_grub2_config(self, mock_exec_cmd_chroot, @mock.patch.object(base.BaseLinuxOSMorphingTools, '_execute_update_grub') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') - def test__apply_grub2_config_no_update_grub(self, mock_exec_cmd_chroot, - mock_execute_update_grub): + def test__apply_grub2_config_no_update_grub( + self, mock_exec_cmd_chroot, mock_execute_update_grub + ): config_obj = { 'location': mock.sentinel.location, 'source': mock.sentinel.source, - 'contents': { - 'GRUB_DEFAULT': '0' - }, + 'contents': {'GRUB_DEFAULT': '0'}, } - self.os_morphing_tools._apply_grub2_config(config_obj, - execute_update_grub=False) + self.os_morphing_tools._apply_grub2_config( + config_obj, execute_update_grub=False + ) mock_exec_cmd_chroot.assert_called_once_with( 'mv -f %s %s' % (config_obj['location'], config_obj['source']) @@ -1109,20 +1175,27 @@ def test__set_grub2_console_settings_invalid_parity(self): self.assertRaises( ValueError, self.os_morphing_tools._set_grub2_console_settings, - parity='invalid_parity') + parity='invalid_parity', + ) def test__set_grub2_console_settings_invalid_consoles(self): self.assertRaises( - ValueError, self.os_morphing_tools._set_grub2_console_settings, - consoles='invalid_consoles') + ValueError, + self.os_morphing_tools._set_grub2_console_settings, + consoles='invalid_consoles', + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_apply_grub2_config') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_set_grub2_cmdline') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'set_grub_value') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_get_grub_config_obj') def test__set_grub2_console_settings_all_params( - self, mock_get_grub_config_obj, mock_set_grub_value, - mock_set_grub2_cmdline, mock_apply_grub2_config): + self, + mock_get_grub_config_obj, + mock_set_grub_value, + mock_set_grub2_cmdline, + mock_apply_grub2_config, + ): consoles = ['tty0', 'ttyS0'] speed = 9600 parity = 'odd' @@ -1134,49 +1207,54 @@ def test__set_grub2_console_settings_all_params( serial_cmd = base.GRUB2_SERIAL % (speed, parity) self.os_morphing_tools._set_grub2_console_settings( - consoles, speed, parity, grub_conf, - execute_update_grub=False) + consoles, speed, parity, grub_conf, execute_update_grub=False + ) mock_get_grub_config_obj.assert_called_once_with(grub_conf) mock_set_grub_value.assert_called_once_with( - 'GRUB_SERIAL_COMMAND', serial_cmd, config_obj) + 'GRUB_SERIAL_COMMAND', serial_cmd, config_obj + ) mock_set_grub2_cmdline.assert_called_once_with( - config_obj, ['console=tty0', 'console=ttyS0']) - mock_apply_grub2_config.assert_called_once_with( - config_obj, False) + config_obj, ['console=tty0', 'console=ttyS0'] + ) + mock_apply_grub2_config.assert_called_once_with(config_obj, False) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_apply_grub2_config') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_set_grub2_cmdline') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'set_grub_value') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_get_grub_config_obj') def test__set_grub2_console_settings_default_params( - self, mock_get_grub_config_obj, mock_set_grub_value, - mock_set_grub2_cmdline, mock_apply_grub2_config): + self, + mock_get_grub_config_obj, + mock_set_grub_value, + mock_set_grub2_cmdline, + mock_apply_grub2_config, + ): grub_conf = '/etc/default/grub' config_obj = {'location': grub_conf} mock_get_grub_config_obj.return_value = config_obj self.os_morphing_tools._set_grub2_console_settings( - grub_conf=grub_conf, execute_update_grub=True) + grub_conf=grub_conf, execute_update_grub=True + ) mock_get_grub_config_obj.assert_called_once_with(grub_conf) mock_set_grub_value.assert_called_once_with( 'GRUB_SERIAL_COMMAND', 'serial --word=8 --stop=1 --speed=115200 --parity=no --unit=0', - config_obj) + config_obj, + ) mock_set_grub2_cmdline.assert_called_once_with( - config_obj, ['console=tty0', 'console=ttyS0']) + config_obj, ['console=tty0', 'console=ttyS0'] + ) mock_apply_grub2_config.assert_called_once_with(config_obj, True) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file_sudo') def test__add_net_udev_rules(self, mock_write_file_sudo, mock_test_path): mock_test_path.return_value = False - net_ifaces_info = { - "eth0": "AA:BB:CC:DD:EE:FF", - "eth1": "FF:EE:DD:CC:BB:AA" - } + net_ifaces_info = {"eth0": "AA:BB:CC:DD:EE:FF", "eth1": "FF:EE:DD:CC:BB:AA"} content = ( 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' 'ATTR{address}=="aa:bb:cc:dd:ee:ff", NAME="eth0"\n' @@ -1193,127 +1271,122 @@ def test__add_net_udev_rules(self, mock_write_file_sudo, mock_test_path): @ddt.data( # (nics_info, interface_info, expected_net_ifaces) ( - [ - {"mac_address": None, - "ip_addresses": []} - ], + [{"mac_address": None, "ip_addresses": []}], { - "eth0": {"mac_address": None, - "ip_addresses": []}, + "eth0": {"mac_address": None, "ip_addresses": []}, }, - {} + {}, ), ( [ - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, + {"mac_address": "00:11:22:33:44:55", "ip_addresses": ["192.168.1.10"]}, ], { - "eth0": {"mac_address": None, - "ip_addresses": None}, + "eth0": {"mac_address": None, "ip_addresses": None}, }, - {} + {}, ), ( [ - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, + {"mac_address": "00:11:22:33:44:55", "ip_addresses": ["192.168.1.10"]}, ], { - "eth0": {"mac_address": "00:11:22:33:44:55", - "ip_addresses": []}, + "eth0": {"mac_address": "00:11:22:33:44:55", "ip_addresses": []}, }, - { - "eth0": "00:11:22:33:44:55" - } + {"eth0": "00:11:22:33:44:55"}, ), ( [ - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, + {"mac_address": "00:11:22:33:44:55", "ip_addresses": ["192.168.1.10"]}, ], { - "eth0": {"mac_address": None, - "ip_addresses": ["192.168.1.10"]}, + "eth0": {"mac_address": None, "ip_addresses": ["192.168.1.10"]}, }, - { - "eth0": "00:11:22:33:44:55" - } + {"eth0": "00:11:22:33:44:55"}, ), ( [ - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, + {"mac_address": "00:11:22:33:44:55", "ip_addresses": ["192.168.1.10"]}, ], { - "eth0": {"mac_address": None, - "ip_addresses": ["192.168.1.20", "192.168.1.10"]}, + "eth0": { + "mac_address": None, + "ip_addresses": ["192.168.1.20", "192.168.1.10"], + }, }, - { - "eth0": "00:11:22:33:44:55" - } + {"eth0": "00:11:22:33:44:55"}, ), ( [ - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.20", "192.168.1.10"]}, + { + "mac_address": "00:11:22:33:44:55", + "ip_addresses": ["192.168.1.20", "192.168.1.10"], + }, ], { - "eth0": {"mac_address": None, - "ip_addresses": ["192.168.1.30", "192.168.1.10"]}, + "eth0": { + "mac_address": None, + "ip_addresses": ["192.168.1.30", "192.168.1.10"], + }, }, - { - "eth0": "00:11:22:33:44:55" - } + {"eth0": "00:11:22:33:44:55"}, ), ( [ - {"mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": ["192.168.4.10"]}, + {"mac_address": "AA:BB:CC:DD:EE:FF", "ip_addresses": ["192.168.4.10"]}, ], { - "eth0": {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, - "eth1": {"mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": ["192.168.2.20"]}, - "eth2": {"mac_address": "11:22:33:44:55:66", - "ip_addresses": ["192.168.3.30"]}, + "eth0": { + "mac_address": "00:11:22:33:44:55", + "ip_addresses": ["192.168.1.10"], + }, + "eth1": { + "mac_address": "AA:BB:CC:DD:EE:FF", + "ip_addresses": ["192.168.2.20"], + }, + "eth2": { + "mac_address": "11:22:33:44:55:66", + "ip_addresses": ["192.168.3.30"], + }, }, - { - "eth1": "AA:BB:CC:DD:EE:FF" - } + {"eth1": "AA:BB:CC:DD:EE:FF"}, ), ( [ - {"mac_address": None, - "ip_addresses": ["192.168.3.10"]}, - {"mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": []}, - {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.11"]}, - {"mac_address": "FF:FF:FF:FF:FF:FF", - "ip_addresses": ["192.168.2.10", "192.168.2.20"]}, + {"mac_address": None, "ip_addresses": ["192.168.3.10"]}, + {"mac_address": "AA:BB:CC:DD:EE:FF", "ip_addresses": []}, + {"mac_address": "00:11:22:33:44:55", "ip_addresses": ["192.168.1.11"]}, + { + "mac_address": "FF:FF:FF:FF:FF:FF", + "ip_addresses": ["192.168.2.10", "192.168.2.20"], + }, ], { - "eth3": {"mac_address": None, - "ip_addresses": ["192.168.3.10"]}, - "eth0": {"mac_address": "00:11:22:33:44:55", - "ip_addresses": ["192.168.1.10"]}, - "eth1": {"mac_address": "AA:BB:CC:DD:EE:FF", - "ip_addresses": ["192.168.1.20"]}, - "eth2": {"mac_address": "11:22:33:44:55:66", - "ip_addresses": ["192.168.2.20"]}, + "eth3": {"mac_address": None, "ip_addresses": ["192.168.3.10"]}, + "eth0": { + "mac_address": "00:11:22:33:44:55", + "ip_addresses": ["192.168.1.10"], + }, + "eth1": { + "mac_address": "AA:BB:CC:DD:EE:FF", + "ip_addresses": ["192.168.1.20"], + }, + "eth2": { + "mac_address": "11:22:33:44:55:66", + "ip_addresses": ["192.168.2.20"], + }, }, { "eth0": "00:11:22:33:44:55", "eth1": "AA:BB:CC:DD:EE:FF", "eth2": "FF:FF:FF:FF:FF:FF", - } + }, ), ) @ddt.unpack def test__setup_network_preservation( - self, nics_info, interface_info, expected_net_ifaces): + self, nics_info, interface_info, expected_net_ifaces + ): class FakeNetPreserver: def __init__(self, tool): self.tool = tool @@ -1324,16 +1397,16 @@ def parse_network(self): with mock.patch( "coriolis.osmorphing.netpreserver.factory.get_net_preserver", - return_value=FakeNetPreserver) as mock_get_np: - + return_value=FakeNetPreserver, + ) as mock_get_np: self.os_morphing_tools._add_net_udev_rules = mock.MagicMock() - with self.assertLogs( - 'coriolis.osmorphing.base', level=logging.INFO): + with self.assertLogs('coriolis.osmorphing.base', level=logging.INFO): self.os_morphing_tools._setup_network_preservation(nics_info) result_net_ifaces = dict( - self.os_morphing_tools._add_net_udev_rules.call_args[0][0]) + self.os_morphing_tools._add_net_udev_rules.call_args[0][0] + ) mock_get_np.assert_called_once_with(self.os_morphing_tools) self.os_morphing_tools._add_net_udev_rules.assert_called_once() diff --git a/coriolis/tests/osmorphing/test_centos.py b/coriolis/tests/osmorphing/test_centos.py index 0f3fb717..3a66c301 100644 --- a/coriolis/tests/osmorphing/test_centos.py +++ b/coriolis/tests/osmorphing/test_centos.py @@ -12,11 +12,10 @@ class BaseCentOSMorphingToolsTestCase(test_base.CoriolisBaseTestCase): def test_check_os_supported(self): detected_os_info = { "distribution_name": centos.CENTOS_DISTRO_IDENTIFIER, - "release_version": "6" + "release_version": "6", } - result = centos.BaseCentOSMorphingTools.check_os_supported( - detected_os_info) + result = centos.BaseCentOSMorphingTools.check_os_supported(detected_os_info) self.assertTrue(result) @@ -24,7 +23,6 @@ def test_check_os_not_supported(self): detected_os_info = { "distribution_name": 'unsupported', } - result = centos.BaseCentOSMorphingTools.check_os_supported( - detected_os_info) + result = centos.BaseCentOSMorphingTools.check_os_supported(detected_os_info) self.assertFalse(result) diff --git a/coriolis/tests/osmorphing/test_coreos.py b/coriolis/tests/osmorphing/test_coreos.py index 05c1fb9e..1cf879ed 100644 --- a/coriolis/tests/osmorphing/test_coreos.py +++ b/coriolis/tests/osmorphing/test_coreos.py @@ -10,17 +10,11 @@ class BaseCoreOSMorphingToolsTestCase(test_base.CoriolisBaseTestCase): """Test suite for the BaseCoreOSMorphingTools class.""" def test_check_os_supported(self): - detected_os_info = { - "distribution_name": coreos_detect.COREOS_DISTRO_IDENTIFIER - } - result = coreos.BaseCoreOSMorphingTools.check_os_supported( - detected_os_info) + detected_os_info = {"distribution_name": coreos_detect.COREOS_DISTRO_IDENTIFIER} + result = coreos.BaseCoreOSMorphingTools.check_os_supported(detected_os_info) self.assertTrue(result) def test_check_os_not_supported(self): - detected_os_info = { - "distribution_name": "unsupported" - } - result = coreos.BaseCoreOSMorphingTools.check_os_supported( - detected_os_info) + detected_os_info = {"distribution_name": "unsupported"} + result = coreos.BaseCoreOSMorphingTools.check_os_supported(detected_os_info) self.assertFalse(result) diff --git a/coriolis/tests/osmorphing/test_debian.py b/coriolis/tests/osmorphing/test_debian.py index b9ce5394..1790db5f 100644 --- a/coriolis/tests/osmorphing/test_debian.py +++ b/coriolis/tests/osmorphing/test_debian.py @@ -3,8 +3,7 @@ from unittest import mock from coriolis import exception -from coriolis.osmorphing import base -from coriolis.osmorphing import debian +from coriolis.osmorphing import base, debian from coriolis.tests import test_base @@ -25,14 +24,20 @@ def setUp(self): self._event_manager = mock.MagicMock() self.morpher = debian.BaseDebianMorphingTools( - mock.sentinel.conn, self.os_root_dir, mock.sentinel.os_root_dev, - mock.sentinel.hypervisor, self._event_manager, - self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.osmorphing_config) + mock.sentinel.conn, + self.os_root_dir, + mock.sentinel.os_root_dev, + mock.sentinel.hypervisor, + self._event_manager, + self.detected_os_info, + mock.sentinel.osmorphing_parameters, + mock.sentinel.osmorphing_config, + ) def test_check_os_supported(self): result = debian.BaseDebianMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertTrue(result) @@ -40,7 +45,8 @@ def test_check_os_not_supported(self): self.detected_os_info['distribution_name'] = 'unsupported' result = debian.BaseDebianMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertFalse(result) @@ -50,39 +56,43 @@ def test_check_os_not_supported(self): @mock.patch.object(debian.BaseDebianMorphingTools, '_write_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') def test_disable_predictable_nic_names( - self, mock_read_file_sudo, mock_write_file_sudo, - mock_exec_cmd_chroot, mock_test_path_chroot, - mock_grub2_cfg_editor): + self, + mock_read_file_sudo, + mock_write_file_sudo, + mock_exec_cmd_chroot, + mock_test_path_chroot, + mock_grub2_cfg_editor, + ): mock_test_path_chroot.return_value = True self.morpher.disable_predictable_nic_names() mock_test_path_chroot.assert_called_once_with('etc/default/grub') - mock_grub2_cfg_editor.assert_called_once_with( - mock_read_file_sudo.return_value) + mock_grub2_cfg_editor.assert_called_once_with(mock_read_file_sudo.return_value) mock_grub2_cfg_editor.return_value.append_to_option.assert_has_calls( [ mock.call( "GRUB_CMDLINE_LINUX_DEFAULT", - {"opt_type": "key_val", "opt_key": "net.ifnames", - "opt_val": 0}), + {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}, + ), mock.call( "GRUB_CMDLINE_LINUX_DEFAULT", - {"opt_type": "key_val", "opt_key": "biosdevname", - "opt_val": 0}), + {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}, + ), mock.call( "GRUB_CMDLINE_LINUX", - {"opt_type": "key_val", "opt_key": "net.ifnames", - "opt_val": 0}), + {"opt_type": "key_val", "opt_key": "net.ifnames", "opt_val": 0}, + ), mock.call( "GRUB_CMDLINE_LINUX", - {"opt_type": "key_val", "opt_key": "biosdevname", - "opt_val": 0}) + {"opt_type": "key_val", "opt_key": "biosdevname", "opt_val": 0}, + ), ] ) mock_read_file_sudo.assert_called_once_with('etc/default/grub') mock_write_file_sudo.assert_called_once_with( - "etc/default/grub", mock_grub2_cfg_editor.return_value.dump()) + "etc/default/grub", mock_grub2_cfg_editor.return_value.dump() + ) mock_exec_cmd_chroot.assert_called_once_with("/usr/sbin/update-grub") @mock.patch('coriolis.utils.Grub2ConfigEditor') @@ -91,8 +101,13 @@ def test_disable_predictable_nic_names( @mock.patch.object(debian.BaseDebianMorphingTools, '_read_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path_chroot') def test_disable_predictable_nic_names_no_test_path_chroot( - self, mock_test_path_chroot, mock_read_file_sudo, - mock_write_file_sudo, mock_exec_cmd_chroot, mock_grub2_cfg_editor): + self, + mock_test_path_chroot, + mock_read_file_sudo, + mock_write_file_sudo, + mock_exec_cmd_chroot, + mock_grub2_cfg_editor, + ): mock_test_path_chroot.return_value = False @@ -130,12 +145,7 @@ def test__compose_netplan_cfg(self): "network": { "version": 2, "ethernets": { - "lo": { - "match": { - "name": "lo" - }, - "addresses": ["127.0.0.1/8"] - }, + "lo": {"match": {"name": "lo"}, "addresses": ["127.0.0.1/8"]}, "eth0": { "dhcp4": True, "dhcp6": True, @@ -143,15 +153,14 @@ def test__compose_netplan_cfg(self): "eth1": { "dhcp4": True, "dhcp6": True, - } - } + }, + }, } } result = self.morpher._compose_netplan_cfg(self.nics_info) - expected_result = debian.yaml.dump( - expected_cfg, default_flow_style=False) + expected_result = debian.yaml.dump(expected_cfg, default_flow_style=False) self.assertEqual(result, expected_result) @@ -165,12 +174,15 @@ def test__has_systemd_chroot(self, mock__test_path): @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') - @mock.patch.object( - debian.BaseDebianMorphingTools, 'disable_predictable_nic_names' - ) + @mock.patch.object(debian.BaseDebianMorphingTools, 'disable_predictable_nic_names') def test_set_net_config( - self, mock_disable_predictable_nic_names, mock_list_dir, - mock_test_path, mock_exec_cmd_chroot, mock_write_file_sudo): + self, + mock_disable_predictable_nic_names, + mock_list_dir, + mock_test_path, + mock_exec_cmd_chroot, + mock_write_file_sudo, + ): dhcp = True mock_test_path.return_value = True mock_list_dir.return_value = ['file1.yaml', 'file2.yml'] @@ -178,33 +190,44 @@ def test_set_net_config( self.morpher.set_net_config(self.nics_info, dhcp) netplan_cfg = self.morpher._compose_netplan_cfg(self.nics_info) - interfaces_config = self.morpher._compose_interfaces_config( - self.nics_info) + interfaces_config = self.morpher._compose_interfaces_config(self.nics_info) mock_disable_predictable_nic_names.assert_called_once() - mock_test_path.assert_has_calls([ - mock.call('etc/network'), - mock.call('etc/network/interfaces'), - mock.call('etc/netplan')]) + mock_test_path.assert_has_calls( + [ + mock.call('etc/network'), + mock.call('etc/network/interfaces'), + mock.call('etc/netplan'), + ] + ) mock_list_dir.assert_called_once_with('etc/netplan') mock_write_file_sudo.assert_has_calls( - [mock.call('etc/network/interfaces', interfaces_config), - mock.call('etc/netplan/coriolis_netplan.yaml', netplan_cfg)]) + [ + mock.call('etc/network/interfaces', interfaces_config), + mock.call('etc/netplan/coriolis_netplan.yaml', netplan_cfg), + ] + ) mock_exec_cmd_chroot.assert_has_calls( - [mock.call('cp etc/network/interfaces etc/network/interfaces.bak'), - mock.call('mv etc/netplan/file1.yaml etc/netplan/file1.yaml.bak'), - mock.call('mv etc/netplan/file2.yml etc/netplan/file2.yml.bak')]) + [ + mock.call('cp etc/network/interfaces etc/network/interfaces.bak'), + mock.call('mv etc/netplan/file1.yaml etc/netplan/file1.yaml.bak'), + mock.call('mv etc/netplan/file2.yml etc/netplan/file2.yml.bak'), + ] + ) @mock.patch.object(debian.BaseDebianMorphingTools, '_write_file_sudo') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') @mock.patch.object(debian.BaseDebianMorphingTools, '_test_path') @mock.patch.object(debian.BaseDebianMorphingTools, '_list_dir') - @mock.patch.object( - debian.BaseDebianMorphingTools, 'disable_predictable_nic_names' - ) + @mock.patch.object(debian.BaseDebianMorphingTools, 'disable_predictable_nic_names') def test_set_net_config_no_dhcp( - self, mock_disable_predictable_nic_names, mock_list_dir, - mock_test_path, mock_exec_cmd_chroot, mock_write_file_sudo): + self, + mock_disable_predictable_nic_names, + mock_list_dir, + mock_test_path, + mock_exec_cmd_chroot, + mock_write_file_sudo, + ): dhcp = False mock_test_path.return_value = True @@ -219,12 +242,10 @@ def test_get_installed_packages(self, mock_exec_cmd_chroot): self.morpher.get_installed_packages() - self.assertEqual( - self.morpher.installed_packages, - ['package1', 'package2'] - ) + self.assertEqual(self.morpher.installed_packages, ['package1', 'package2']) mock_exec_cmd_chroot.assert_called_once_with( - "dpkg-query -f '${binary:Package}\\n' -W") + "dpkg-query -f '${binary:Package}\\n' -W" + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_get_installed_packages_none(self, mock_exec_cmd_chroot): @@ -232,42 +253,44 @@ def test_get_installed_packages_none(self, mock_exec_cmd_chroot): self.morpher.get_installed_packages() - self.assertEqual( - self.morpher.installed_packages, - [] - ) + self.assertEqual(self.morpher.installed_packages, []) mock_exec_cmd_chroot.assert_called_once_with( - "dpkg-query -f '${binary:Package}\\n' -W") + "dpkg-query -f '${binary:Package}\\n' -W" + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, 'pre_packages_install') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') - def test_pre_packages_install(self, mock_exec_cmd_chroot, - mock_pre_packages_install): + def test_pre_packages_install( + self, mock_exec_cmd_chroot, mock_pre_packages_install + ): self.morpher.pre_packages_install(self.package_names) mock_pre_packages_install.assert_called_once_with(self.package_names) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call('apt-get clean'), - mock.call('apt-get update -y') - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call('apt-get clean'), mock.call('apt-get update -y')] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, 'pre_packages_install') @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') - def test_pre_packages_install_with_exception(self, mock_exec_cmd_chroot, - mock_pre_packages_install): + def test_pre_packages_install_with_exception( + self, mock_exec_cmd_chroot, mock_pre_packages_install + ): mock_exec_cmd_chroot.side_effect = Exception() - self.assertRaises(exception.PackageManagerOperationException, - self.morpher.pre_packages_install, - self.package_names) + self.assertRaises( + exception.PackageManagerOperationException, + self.morpher.pre_packages_install, + self.package_names, + ) mock_pre_packages_install.assert_called_once_with(self.package_names) @mock.patch.object(debian.BaseDebianMorphingTools, '_configure_cloud_init') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'post_packages_install') def test_post_packages_install( - self, mock_post_packages_install, mock__configure_cloud_init): + self, mock_post_packages_install, mock__configure_cloud_init + ): self.morpher.post_packages_install(self.package_names) @@ -281,34 +304,41 @@ def test_install_packages(self, mock_exec_cmd_chroot): apt_get_cmd = ( '/bin/bash -c "DEBIAN_FRONTEND=noninteractive ' 'apt-get install %s -y ' - '-o Dpkg::Options::=\'--force-confdef\'"' % ( - " ".join(self.package_names))) + '-o Dpkg::Options::=\'--force-confdef\'"' % (" ".join(self.package_names)) + ) deb_reconfigure_cmd = "dpkg --configure --force-confold -a" - mock_exec_cmd_chroot.assert_has_calls([ - mock.call(deb_reconfigure_cmd), - mock.call(apt_get_cmd) - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call(deb_reconfigure_cmd), mock.call(apt_get_cmd)] + ) @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') def test_install_packages_with_exception(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = Exception() - self.assertRaises(exception.FailedPackageInstallationException, - self.morpher.install_packages, self.package_names) + self.assertRaises( + exception.FailedPackageInstallationException, + self.morpher.install_packages, + self.package_names, + ) @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') def test_uninstall_packages(self, mock_exec_cmd_chroot): self.morpher.uninstall_packages(self.package_names) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call('apt-get remove %s -y || true' % self.package_names[0]), - mock.call('apt-get remove %s -y || true' % self.package_names[1]) - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call('apt-get remove %s -y || true' % self.package_names[0]), + mock.call('apt-get remove %s -y || true' % self.package_names[1]), + ] + ) @mock.patch.object(debian.BaseDebianMorphingTools, '_exec_cmd_chroot') def test_uninstall_packages_with_exception(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = exception.CoriolisException() - self.assertRaises(exception.FailedPackageUninstallationException, - self.morpher.uninstall_packages, self.package_names) + self.assertRaises( + exception.FailedPackageUninstallationException, + self.morpher.uninstall_packages, + self.package_names, + ) diff --git a/coriolis/tests/osmorphing/test_manager.py b/coriolis/tests/osmorphing/test_manager.py index 2012e413..299b2a8a 100644 --- a/coriolis/tests/osmorphing/test_manager.py +++ b/coriolis/tests/osmorphing/test_manager.py @@ -15,10 +15,7 @@ class ManagerTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ManagerTestCase, self).setUp() - self.osmorphing_info = { - 'os_type': 'linux', - 'ignore_devices': [] - } + self.osmorphing_info = {'os_type': 'linux', 'ignore_devices': []} manager.CONF.proxy.url = "http://127.0.0.1:8080" manager.CONF.proxy.username = "admin" manager.CONF.proxy.password = "Random-Password-123!" @@ -52,31 +49,37 @@ def test_run_os_detect(self, mock_detect_os): "release_version": "22.04", "friendly_release_name": "Ubuntu 22.04", } - self.provider.get_custom_os_detect_tools.return_value = [ - mock.sentinel.os_type] + self.provider.get_custom_os_detect_tools.return_value = [mock.sentinel.os_type] self.destination_provider.get_custom_os_detect_tools.return_value = [ - mock.sentinel.os_type] + mock.sentinel.os_type + ] result = manager.run_os_detect( - self.provider, self.destination_provider, self.worker_connection, - mock.sentinel.os_type, mock.sentinel.os_root_dir, - mock.sentinel.osmorphing_info, tools_environment={}) + self.provider, + self.destination_provider, + self.worker_connection, + mock.sentinel.os_type, + mock.sentinel.os_root_dir, + mock.sentinel.osmorphing_info, + tools_environment={}, + ) self.assertEqual(result, mock_detect_os.return_value) - self.provider.get_custom_os_detect_tools.\ - assert_called_once_with(mock.sentinel.os_type, - mock.sentinel.osmorphing_info) - self.destination_provider.get_custom_os_detect_tools.\ - assert_called_once_with(mock.sentinel.os_type, - mock.sentinel.osmorphing_info) + self.provider.get_custom_os_detect_tools.assert_called_once_with( + mock.sentinel.os_type, mock.sentinel.osmorphing_info + ) + self.destination_provider.get_custom_os_detect_tools.assert_called_once_with( + mock.sentinel.os_type, mock.sentinel.osmorphing_info + ) mock_detect_os.assert_called_once_with( - self.worker_connection, mock.sentinel.os_type, + self.worker_connection, + mock.sentinel.os_type, mock.sentinel.os_root_dir, 60, tools_environment={}, - custom_os_detect_tools=[mock.sentinel.os_type, - mock.sentinel.os_type]) + custom_os_detect_tools=[mock.sentinel.os_type, mock.sentinel.os_type], + ) def test_get_osmorphing_tools_class_for_provider(self): class MockToolsClass(base_osmorphing.BaseOSMorphingTools): @@ -95,8 +98,11 @@ def check_os_supported(cls, detected_os_info): self.provider.get_os_morphing_tools.return_value = [MockToolsClass] result = manager.get_osmorphing_tools_class_for_provider( - self.provider, mock.sentinel.detected_os_info, - mock.sentinel.os_type, mock.sentinel.osmorphing_info) + self.provider, + mock.sentinel.detected_os_info, + mock.sentinel.os_type, + mock.sentinel.osmorphing_info, + ) self.assertEqual(result, MockToolsClass) @@ -104,13 +110,16 @@ def test_get_osmorphing_tools_class_for_provider_invalid_tools(self): class MockInvalidToolsClass: pass - self.provider.get_os_morphing_tools.return_value = [ - MockInvalidToolsClass] + self.provider.get_os_morphing_tools.return_value = [MockInvalidToolsClass] - self.assertRaises(exception.InvalidOSMorphingTools, - manager.get_osmorphing_tools_class_for_provider, - self.provider, mock.sentinel.detected_os_info, - mock.sentinel.os_type, mock.sentinel.osmorphing_info) + self.assertRaises( + exception.InvalidOSMorphingTools, + manager.get_osmorphing_tools_class_for_provider, + self.provider, + mock.sentinel.detected_os_info, + mock.sentinel.os_type, + mock.sentinel.osmorphing_info, + ) def test_get_osmorphing_tools_class_for_provider_invalid_os_params(self): class MockToolsClass(base_osmorphing.BaseOSMorphingTools): @@ -124,11 +133,13 @@ def check_detected_os_info_parameters(cls, detected_os_info): self.provider.get_os_morphing_tools.return_value = [MockToolsClass] - with self.assertLogs( - 'coriolis.osmorphing.manager', level=logging.WARN): + with self.assertLogs('coriolis.osmorphing.manager', level=logging.WARN): result = manager.get_osmorphing_tools_class_for_provider( - self.provider, mock.sentinel.detected_os_info, - mock.sentinel.os_type, mock.sentinel.osmorphing_info) + self.provider, + mock.sentinel.detected_os_info, + mock.sentinel.os_type, + mock.sentinel.osmorphing_info, + ) self.assertIsNone(result) @@ -144,11 +155,13 @@ def check_os_supported(cls, detected_os_info): self.provider.get_os_morphing_tools.return_value = [MockToolsClass] - with self.assertLogs( - 'coriolis.osmorphing.manager', level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.manager', level=logging.DEBUG): result = manager.get_osmorphing_tools_class_for_provider( - self.provider, self.detected_os_info, - mock.sentinel.os_type, mock.sentinel.osmorphing_info) + self.provider, + self.detected_os_info, + mock.sentinel.os_type, + mock.sentinel.osmorphing_info, + ) self.assertIsNone(result) @@ -196,31 +209,42 @@ def uninstall_packages(self, packages_remove): @mock.patch.object(manager, 'run_os_detect') @mock.patch.object(manager, 'get_osmorphing_tools_class_for_provider') def test_morph_image( - self, mock_get_osmorphing_tools_class, mock_run_os_detect, - mock_EventManager, mock_get_os_mount_tools): + self, + mock_get_osmorphing_tools_class, + mock_run_os_detect, + mock_EventManager, + mock_get_os_mount_tools, + ): mock_EventManager.return_value = self.event_manager mock_get_os_mount_tools.return_value = self.os_mount_tools - self.os_mount_tools.mount_os.return_value = ( - 'os_root_dir', 'os_root_dev') + self.os_mount_tools.mount_os.return_value = ('os_root_dir', 'os_root_dev') mock_run_os_detect.return_value = {'friendly_release_name': 'mock_os'} - mock_get_osmorphing_tools_class.return_value = ( - self.MockOSMorphingToolsClass) + mock_get_osmorphing_tools_class.return_value = self.MockOSMorphingToolsClass manager.morph_image( - mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock.sentinel.connection_info, self.osmorphing_info, - mock.sentinel.user_script, self.event_handler) + mock.sentinel.origin_provider, + mock.sentinel.destination_provider, + mock.sentinel.connection_info, + self.osmorphing_info, + mock.sentinel.user_script, + self.event_handler, + ) expected_calls = [ mock.call( - mock.sentinel.origin_provider, mock_run_os_detect.return_value, - self.osmorphing_info.get('os_type'), self.osmorphing_info), + mock.sentinel.origin_provider, + mock_run_os_detect.return_value, + self.osmorphing_info.get('os_type'), + self.osmorphing_info, + ), mock.call( mock.sentinel.destination_provider, mock_run_os_detect.return_value, - self.osmorphing_info.get('os_type'), self.osmorphing_info) + self.osmorphing_info.get('os_type'), + self.osmorphing_info, + ), ] mock_get_osmorphing_tools_class.assert_has_calls(expected_calls) @@ -229,7 +253,8 @@ def test_morph_image( self.os_mount_tools.dismount_os.assert_called_once() mock_get_os_mount_tools.assert_called_once_with( - 'linux', mock.sentinel.connection_info, self.event_manager, [], 60) + 'linux', mock.sentinel.connection_info, self.event_manager, [], 60 + ) mock_EventManager.assert_called_once_with(self.event_handler) self.os_mount_tools.dismount_os.assert_called_once() @@ -238,8 +263,11 @@ def test_morph_image( @mock.patch.object(manager.osmount_factory, 'get_os_mount_tools') @mock.patch.object(manager.events, 'EventManager') def test_morph_image_failed_os_mount_setup( - self, mock_EventManager, mock_get_os_mount_tools, - mock_get_osmorphing_tools_class): + self, + mock_EventManager, + mock_get_os_mount_tools, + mock_get_osmorphing_tools_class, + ): mock_EventManager.return_value = self.event_manager mock_get_os_mount_tools.return_value = self.os_mount_tools @@ -247,14 +275,18 @@ def test_morph_image_failed_os_mount_setup( self.assertRaises( exception.CoriolisException, - manager.morph_image, mock.sentinel.origin_provider, + manager.morph_image, + mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock.sentinel.connection_info, self.osmorphing_info, + mock.sentinel.connection_info, + self.osmorphing_info, mock.sentinel.user_script, - self.event_handler) + self.event_handler, + ) mock_get_os_mount_tools.assert_called_once_with( - 'linux', mock.sentinel.connection_info, self.event_manager, [], 60) + 'linux', mock.sentinel.connection_info, self.event_manager, [], 60 + ) mock_EventManager.assert_called_once_with(self.event_handler) mock_get_osmorphing_tools_class.assert_not_called() @@ -266,49 +298,61 @@ def test_morph_image_failed_os_mount_setup( @mock.patch.object(manager, 'run_os_detect') @mock.patch.object(manager, 'get_osmorphing_tools_class_for_provider') def test_morph_image_no_import_os_morphing_tools_cls( - self, mock_get_osmorphing_tools_class, mock_run_os_detect, - mock_EventManager, mock_get_os_mount_tools): + self, + mock_get_osmorphing_tools_class, + mock_run_os_detect, + mock_EventManager, + mock_get_os_mount_tools, + ): mock_EventManager.return_value = self.event_manager mock_get_os_mount_tools.return_value = self.os_mount_tools - self.os_mount_tools.mount_os.return_value = ( - 'os_root_dir', 'os_root_dev') + self.os_mount_tools.mount_os.return_value = ('os_root_dir', 'os_root_dev') mock_run_os_detect.return_value = {'friendly_release_name': 'mock_os'} mock_get_osmorphing_tools_class.return_value = None - with self.assertLogs( - 'coriolis.osmorphing.manager', level=logging.ERROR): + with self.assertLogs('coriolis.osmorphing.manager', level=logging.ERROR): self.assertRaises( exception.OSMorphingToolsNotFound, - manager.morph_image, mock.sentinel.origin_provider, + manager.morph_image, + mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock.sentinel.connection_info, self.osmorphing_info, - mock.sentinel.user_script, self.event_handler) + mock.sentinel.connection_info, + self.osmorphing_info, + mock.sentinel.user_script, + self.event_handler, + ) @mock.patch.object(manager.osmount_factory, 'get_os_mount_tools') @mock.patch.object(manager.events, 'EventManager') @mock.patch.object(manager, 'run_os_detect') @mock.patch.object(manager, 'get_osmorphing_tools_class_for_provider') def test_morph_image_no_user_script( - self, mock_get_osmorphing_tools_class, mock_run_os_detect, - mock_EventManager, mock_get_os_mount_tools): + self, + mock_get_osmorphing_tools_class, + mock_run_os_detect, + mock_EventManager, + mock_get_os_mount_tools, + ): mock_user_script = None mock_EventManager.return_value = self.event_manager mock_get_os_mount_tools.return_value = self.os_mount_tools - self.os_mount_tools.mount_os.return_value = ( - 'os_root_dir', 'os_root_dev') + self.os_mount_tools.mount_os.return_value = ('os_root_dir', 'os_root_dev') mock_run_os_detect.return_value = {'friendly_release_name': 'mock_os'} - mock_get_osmorphing_tools_class.return_value = ( - self.MockOSMorphingToolsClass) + mock_get_osmorphing_tools_class.return_value = self.MockOSMorphingToolsClass manager.morph_image( - mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock.sentinel.connection_info, self.osmorphing_info, - mock_user_script, self.event_handler) + mock.sentinel.origin_provider, + mock.sentinel.destination_provider, + mock.sentinel.connection_info, + self.osmorphing_info, + mock_user_script, + self.event_handler, + ) mock_get_osmorphing_tools_class.run_user_script.assert_not_called() @@ -317,23 +361,29 @@ def test_morph_image_no_user_script( @mock.patch.object(manager, 'run_os_detect') @mock.patch.object(manager, 'get_osmorphing_tools_class_for_provider') def test_morph_image_dismount_os_exception( - self, mock_get_osmorphing_tools_class, mock_run_os_detect, - mock_EventManager, mock_get_os_mount_tools): + self, + mock_get_osmorphing_tools_class, + mock_run_os_detect, + mock_EventManager, + mock_get_os_mount_tools, + ): mock_EventManager.return_value = self.event_manager mock_get_os_mount_tools.return_value = self.os_mount_tools - self.os_mount_tools.mount_os.return_value = ( - 'os_root_dir', 'os_root_dev') + self.os_mount_tools.mount_os.return_value = ('os_root_dir', 'os_root_dev') mock_run_os_detect.return_value = {'friendly_release_name': 'mock_os'} - mock_get_osmorphing_tools_class.return_value = ( - self.MockOSMorphingToolsClass) + mock_get_osmorphing_tools_class.return_value = self.MockOSMorphingToolsClass self.os_mount_tools.dismount_os.side_effect = Exception() self.assertRaises( exception.CoriolisException, - manager.morph_image, mock.sentinel.origin_provider, + manager.morph_image, + mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock.sentinel.connection_info, self.osmorphing_info, - mock.sentinel.user_script, self.event_handler) + mock.sentinel.connection_info, + self.osmorphing_info, + mock.sentinel.user_script, + self.event_handler, + ) diff --git a/coriolis/tests/osmorphing/test_openwrt.py b/coriolis/tests/osmorphing/test_openwrt.py index bcd72e75..85316acc 100644 --- a/coriolis/tests/osmorphing/test_openwrt.py +++ b/coriolis/tests/osmorphing/test_openwrt.py @@ -23,24 +23,17 @@ class BaseOpenWRTMorphingToolsTestCase(test_base.CoriolisBaseTestCase): """Test case for the BaseOpenWRTMorphingTools class.""" def test_check_os_supported(self): - detected_os_info = { - 'distribution_name': openwrt.OPENWRT_DISTRO_IDENTIFIER - } - result = openwrt.BaseOpenWRTMorphingTools.check_os_supported( - detected_os_info) + detected_os_info = {'distribution_name': openwrt.OPENWRT_DISTRO_IDENTIFIER} + result = openwrt.BaseOpenWRTMorphingTools.check_os_supported(detected_os_info) self.assertTrue(result) def test_check_os_not_supported(self): - detected_os_info = { - 'distribution_name': 'unsupported' - } - result = openwrt.BaseOpenWRTMorphingTools.check_os_supported( - detected_os_info) + detected_os_info = {'distribution_name': 'unsupported'} + result = openwrt.BaseOpenWRTMorphingTools.check_os_supported(detected_os_info) self.assertFalse(result) def test_get_update_grub2_command(self): morphing_tools = MockOpenWRTMorphingTools() - self.assertRaises(NotImplementedError, - morphing_tools.get_update_grub2_command) + self.assertRaises(NotImplementedError, morphing_tools.get_update_grub2_command) diff --git a/coriolis/tests/osmorphing/test_oracle.py b/coriolis/tests/osmorphing/test_oracle.py index 66455ea8..616f9a1c 100644 --- a/coriolis/tests/osmorphing/test_oracle.py +++ b/coriolis/tests/osmorphing/test_oracle.py @@ -3,10 +3,8 @@ from unittest import mock -from coriolis.osmorphing import base -from coriolis.osmorphing import oracle +from coriolis.osmorphing import base, oracle, redhat from coriolis.osmorphing.osdetect import oracle as oracle_detect -from coriolis.osmorphing import redhat from coriolis.tests import test_base @@ -22,14 +20,19 @@ def setUp(self): 'friendly_release_name': mock.sentinel.friendly_release_name, } self.oracle_morphing_tools = oracle.BaseOracleMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_dir, mock.sentinel.hypervisor, - mock.sentinel.event_manager, self.detected_os_info, - mock.sentinel.osmorphing_parameters) + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_dir, + mock.sentinel.hypervisor, + mock.sentinel.event_manager, + self.detected_os_info, + mock.sentinel.osmorphing_parameters, + ) def test_check_os_supported(self): result = oracle.BaseOracleMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertTrue(result) @@ -37,32 +40,39 @@ def test_check_os_not_supported(self): self.detected_os_info['distribution_name'] = 'unsupported' result = oracle.BaseOracleMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertFalse(result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_find_yum_repos') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_install') - def test__get_oracle_repos(self, mock_yum_install, mock_find_yum_repos, - mock_exec_cmd_chroot): + def test__get_oracle_repos( + self, mock_yum_install, mock_find_yum_repos, mock_exec_cmd_chroot + ): self.oracle_morphing_tools._version = '10' result = self.oracle_morphing_tools._get_oracle_repos() self.assertEqual(result, mock_find_yum_repos.return_value) - mock_find_yum_repos.assert_has_calls([ - mock.call(["ol10_baseos_latest"]), - mock.call([ - "ol10_baseos_latest", - "ol10_appstream", - "ol10_addons", - "ol10_UEKR8" - ]), - ]) + mock_find_yum_repos.assert_has_calls( + [ + mock.call(["ol10_baseos_latest"]), + mock.call( + [ + "ol10_baseos_latest", + "ol10_appstream", + "ol10_addons", + "ol10_UEKR8", + ] + ), + ] + ) mock_yum_install.assert_called_once_with( - ['oraclelinux-release-el10'], mock_find_yum_repos.return_value) + ['oraclelinux-release-el10'], mock_find_yum_repos.return_value + ) mock_exec_cmd_chroot.assert_not_called() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @@ -70,23 +80,25 @@ def test__get_oracle_repos(self, mock_yum_install, mock_find_yum_repos, @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_install') @mock.patch.object(oracle.uuid, 'uuid4') def test__get_oracle_repos_major_version_lt_8( - self, mock_uuid4, mock_yum_install, mock_find_yum_repos, - mock_exec_cmd_chroot): + self, mock_uuid4, mock_yum_install, mock_find_yum_repos, mock_exec_cmd_chroot + ): result = self.oracle_morphing_tools._get_oracle_repos() self.assertEqual(result, mock_find_yum_repos.return_value) - mock_find_yum_repos.assert_called_once_with([ - "ol6_software_collections", - "ol6_addons", - "ol6_UEKR", - "ol6_latest" - ]) + mock_find_yum_repos.assert_called_once_with( + ["ol6_software_collections", "ol6_addons", "ol6_UEKR", "ol6_latest"] + ) mock_yum_install.assert_not_called() - mock_exec_cmd_chroot.assert_has_calls([ - mock.call( - "curl -L http://public-yum.oracle.com/public-yum-ol6.repo " - "-o /etc/yum.repos.d/%s.repo" % mock_uuid4.return_value), - mock.call('sed -i "s/^enabled=1$/enabled=0/g" %s' % ( - "/etc/yum.repos.d/%s.repo" % mock_uuid4.return_value)) - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call( + "curl -L http://public-yum.oracle.com/public-yum-ol6.repo " + "-o /etc/yum.repos.d/%s.repo" % mock_uuid4.return_value + ), + mock.call( + 'sed -i "s/^enabled=1$/enabled=0/g" %s' + % ("/etc/yum.repos.d/%s.repo" % mock_uuid4.return_value) + ), + ] + ) diff --git a/coriolis/tests/osmorphing/test_redhat.py b/coriolis/tests/osmorphing/test_redhat.py index f3c1e459..ca9f6339 100644 --- a/coriolis/tests/osmorphing/test_redhat.py +++ b/coriolis/tests/osmorphing/test_redhat.py @@ -7,8 +7,7 @@ import ddt from coriolis import exception -from coriolis.osmorphing import base -from coriolis.osmorphing import redhat +from coriolis.osmorphing import base, redhat from coriolis.tests import test_base @@ -28,16 +27,21 @@ def setUp(self): self.enable_repos = ['repo1', 'repo2'] self.event_manager = mock.MagicMock() self.morphing_tools = redhat.BaseRedHatMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_dir, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_dir, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) self.morphing_tools._os_root_dir = '/root' def test_check_os_supported(self): result = redhat.BaseRedHatMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertTrue(result) @@ -45,7 +49,8 @@ def test_check_os_not_supported(self): self.detected_os_info['distribution_name'] = 'unsupported' result = redhat.BaseRedHatMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertFalse(result) @@ -53,75 +58,76 @@ def test_check_os_not_supported(self): def test_disable_predictable_nic_names(self, mock_exec_cmd_chroot): self.morphing_tools.disable_predictable_nic_names() mock_exec_cmd_chroot.assert_called_once_with( - 'grubby --update-kernel=ALL --args="net.ifnames=0 biosdevname=0"') + 'grubby --update-kernel=ALL --args="net.ifnames=0 biosdevname=0"' + ) - @mock.patch.object( - redhat.BaseRedHatMorphingTools, '_get_grub2_cfg_location' - ) + @mock.patch.object(redhat.BaseRedHatMorphingTools, '_get_grub2_cfg_location') def test_get_update_grub2_command(self, mock_get_grub2_cfg_location): result = self.morphing_tools.get_update_grub2_command() mock_get_grub2_cfg_location.assert_called_once() self.assertEqual( - result, - 'grub2-mkconfig -o %s' % mock_get_grub2_cfg_location.return_value + result, 'grub2-mkconfig -o %s' % mock_get_grub2_cfg_location.return_value ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_bios(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_bios( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.return_value = True result = self.morphing_tools._get_grub2_cfg_location() self.assertEqual(result, '/boot/grub2/grub.cfg') - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_called_once_with( - '/boot/grub2/grub.cfg') + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_called_once_with('/boot/grub2/grub.cfg') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_uefi(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_uefi( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.side_effect = [False, True] result = self.morphing_tools._get_grub2_cfg_location() self.assertEqual(result, '/boot/efi/EFI/redhat/grub.cfg') - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_has_calls([ - mock.call('/boot/grub2/grub.cfg'), - mock.call('/boot/efi/EFI/redhat/grub.cfg') - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_has_calls( + [ + mock.call('/boot/grub2/grub.cfg'), + mock.call('/boot/efi/EFI/redhat/grub.cfg'), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_unknown(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_unknown( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.return_value = False self.assertRaisesRegex( Exception, "could not determine grub location. boot partition not mounted?", - self.morphing_tools._get_grub2_cfg_location + self.morphing_tools._get_grub2_cfg_location, ) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_has_calls([ - mock.call('/boot/grub2/grub.cfg'), - mock.call('/boot/efi/EFI/redhat/grub.cfg') - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_has_calls( + [ + mock.call('/boot/grub2/grub.cfg'), + mock.call('/boot/efi/EFI/redhat/grub.cfg'), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__has_systemd(self, mock_exec_cmd_chroot): @@ -140,17 +146,18 @@ def test__has_systemd_with_exception(self, mock_exec_cmd_chroot): @mock.patch.object(redhat.BaseRedHatMorphingTools, '_write_config_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_config_file') - def test__set_dhcp_net_config(self, mock_read_config_file, - mock_write_config_file): + def test__set_dhcp_net_config(self, mock_read_config_file, mock_write_config_file): ifcfgs_ethernet = [ - (mock.sentinel.ifcfg_file, - { - "BOOTPROTO": "none", - "IPADDR": mock.sentinel.ip_address, - "GATEWAY": mock.sentinel.gateway, - "NETMASK": mock.sentinel.netmask, - "NETWORK": mock.sentinel.network, - }), + ( + mock.sentinel.ifcfg_file, + { + "BOOTPROTO": "none", + "IPADDR": mock.sentinel.ip_address, + "GATEWAY": mock.sentinel.gateway, + "NETMASK": mock.sentinel.netmask, + "NETWORK": mock.sentinel.network, + }, + ), ] mock_read_config_file.return_value = { "GATEWAY": mock.sentinel.gateway, @@ -159,114 +166,136 @@ def test__set_dhcp_net_config(self, mock_read_config_file, self.morphing_tools._set_dhcp_net_config(ifcfgs_ethernet) mock_read_config_file.assert_called_once_with( - "etc/sysconfig/network", check_exists=True) - mock_write_config_file.assert_has_calls([ - mock.call(mock.sentinel.ifcfg_file, - {"BOOTPROTO": "dhcp", "UUID": mock.ANY}), - mock.call("etc/sysconfig/network", - mock_read_config_file.return_value) - ]) + "etc/sysconfig/network", check_exists=True + ) + mock_write_config_file.assert_has_calls( + [ + mock.call( + mock.sentinel.ifcfg_file, {"BOOTPROTO": "dhcp", "UUID": mock.ANY} + ), + mock.call("etc/sysconfig/network", mock_read_config_file.return_value), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') def test_write_nic_configs( - self, mock_test_path, mock_exec_cmd_chroot, mock_write_file_sudo): + self, mock_test_path, mock_exec_cmd_chroot, mock_write_file_sudo + ): nics_info = [{'name': 'eth0'}, {'name': 'eth1'}] mock_test_path.return_value = True self.morphing_tools._write_nic_configs(nics_info) - mock_write_file_sudo.assert_has_calls([ - mock.call( - "etc/sysconfig/network-scripts/ifcfg-eth0", - redhat.IFCFG_TEMPLATE % {"device_name": "eth0"}, - ), - mock.call( - "etc/sysconfig/network-scripts/ifcfg-eth1", - redhat.IFCFG_TEMPLATE % {"device_name": "eth1"}, - ) - ]) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("cp etc/sysconfig/network-scripts/ifcfg-eth0 " - "etc/sysconfig/network-scripts/ifcfg-eth0.bak"), - mock.call("cp etc/sysconfig/network-scripts/ifcfg-eth1 " - "etc/sysconfig/network-scripts/ifcfg-eth1.bak") - ]) + mock_write_file_sudo.assert_has_calls( + [ + mock.call( + "etc/sysconfig/network-scripts/ifcfg-eth0", + redhat.IFCFG_TEMPLATE % {"device_name": "eth0"}, + ), + mock.call( + "etc/sysconfig/network-scripts/ifcfg-eth1", + redhat.IFCFG_TEMPLATE % {"device_name": "eth1"}, + ), + ] + ) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call( + "cp etc/sysconfig/network-scripts/ifcfg-eth0 " + "etc/sysconfig/network-scripts/ifcfg-eth0.bak" + ), + mock.call( + "cp etc/sysconfig/network-scripts/ifcfg-eth1 " + "etc/sysconfig/network-scripts/ifcfg-eth1.bak" + ), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @mock.patch.object(redhat.utils, 'list_ssh_dir') - def test__comment_keys_from_ifcfg_files( - self, mock_list_ssh_dir, mock_exec_cmd): + def test__comment_keys_from_ifcfg_files(self, mock_list_ssh_dir, mock_exec_cmd): keys = ['KEY1', 'KEY2'] interfaces = ['eth0', 'eth1'] mock_list_ssh_dir.return_value = [ - 'ifcfg-eth0', 'ifcfg-eth1', 'unknown-file', 'ifcfg-eth2'] + 'ifcfg-eth0', + 'ifcfg-eth1', + 'unknown-file', + 'ifcfg-eth2', + ] - self.morphing_tools._comment_keys_from_ifcfg_files( - keys, interfaces=interfaces) + self.morphing_tools._comment_keys_from_ifcfg_files(keys, interfaces=interfaces) mock_list_ssh_dir.assert_called_once_with( - mock.sentinel.conn, '/root/etc/sysconfig/network-scripts') - mock_exec_cmd.assert_has_calls([ - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth0" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth0" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth1" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth1" - ), - ]) + mock.sentinel.conn, '/root/etc/sysconfig/network-scripts' + ) + mock_exec_cmd.assert_has_calls( + [ + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth0" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth0" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth1" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth1" + ), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @mock.patch.object(redhat.utils, 'list_ssh_dir') def test__comment_keys_from_ifcfg_files_no_interfaces( - self, mock_list_ssh_dir, mock_exec_cmd): + self, mock_list_ssh_dir, mock_exec_cmd + ): keys = ['KEY1', 'KEY2'] mock_list_ssh_dir.return_value = ['ifcfg-eth0', 'ifcfg-eth1'] self.morphing_tools._comment_keys_from_ifcfg_files(keys) mock_list_ssh_dir.assert_called_once_with( - mock.sentinel.conn, '/root/etc/sysconfig/network-scripts') + mock.sentinel.conn, '/root/etc/sysconfig/network-scripts' + ) - mock_exec_cmd.assert_has_calls([ - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth0" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth0" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth1" - ), - mock.call( - "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " - "/root/etc/sysconfig/network-scripts/ifcfg-eth1" - ), - ]) + mock_exec_cmd.assert_has_calls( + [ + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth0" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth0" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY1=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth1" + ), + mock.call( + "sudo sed -i.bak -E -e 's/^(KEY2=.*)$/# \\1/g' " + "/root/etc/sysconfig/network-scripts/ifcfg-eth1" + ), + ] + ) - @mock.patch.object( - redhat.BaseRedHatMorphingTools, 'disable_predictable_nic_names' - ) + @mock.patch.object(redhat.BaseRedHatMorphingTools, 'disable_predictable_nic_names') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_write_nic_configs') - def test_set_net_config_dhcp(self, mock_write_nic_configs, - mock_disable_predictable_nic_names): - nics_info = [{ - 'mac_address': mock.sentinel.mac_address, - }] + def test_set_net_config_dhcp( + self, mock_write_nic_configs, mock_disable_predictable_nic_names + ): + nics_info = [ + { + 'mac_address': mock.sentinel.mac_address, + } + ] dhcp = True self.morphing_tools.set_net_config(nics_info, dhcp) @@ -274,20 +303,22 @@ def test_set_net_config_dhcp(self, mock_write_nic_configs, mock_disable_predictable_nic_names.assert_called_once() mock_write_nic_configs.assert_called_once_with(nics_info) - @mock.patch.object( - redhat.BaseRedHatMorphingTools, 'disable_predictable_nic_names' - ) + @mock.patch.object(redhat.BaseRedHatMorphingTools, 'disable_predictable_nic_names') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_write_nic_configs') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_add_net_udev_rules') - @mock.patch.object(base.BaseLinuxOSMorphingTools, - '_setup_network_preservation') + @mock.patch.object(base.BaseLinuxOSMorphingTools, '_setup_network_preservation') def test_set_net_config_no_dhcp( - self, mock_setup_network_preservation, mock_add_net_udev_rules, - mock_write_nic_configs, - mock_disable_predictable_nic_names): - nics_info = [{ - 'mac_address': mock.sentinel.mac_address, - }] + self, + mock_setup_network_preservation, + mock_add_net_udev_rules, + mock_write_nic_configs, + mock_disable_predictable_nic_names, + ): + nics_info = [ + { + 'mac_address': mock.sentinel.mac_address, + } + ] dhcp = False self.morphing_tools.set_net_config(nics_info, dhcp) @@ -303,34 +334,26 @@ def test_get_installed_packages(self, mock_exec_cmd_chroot): self.morphing_tools.get_installed_packages() self.assertEqual( - self.morphing_tools.installed_packages, - ['package1', 'package2'] + self.morphing_tools.installed_packages, ['package1', 'package2'] ) - mock_exec_cmd_chroot.assert_called_once_with( - 'rpm -qa --qf "%{NAME}\\n"') + mock_exec_cmd_chroot.assert_called_once_with('rpm -qa --qf "%{NAME}\\n"') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_get_installed_packages_none(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = exception.CoriolisException() - with self.assertLogs( - 'coriolis.osmorphing.redhat', level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.redhat', level=logging.DEBUG): self.morphing_tools.get_installed_packages() - self.assertEqual( - self.morphing_tools.installed_packages, - [] - ) - mock_exec_cmd_chroot.assert_called_once_with( - 'rpm -qa --qf "%{NAME}\\n"') + self.assertEqual(self.morphing_tools.installed_packages, []) + mock_exec_cmd_chroot.assert_called_once_with('rpm -qa --qf "%{NAME}\\n"') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__yum_install(self, mock_exec_cmd_chroot): self.morphing_tools._yum_install(self.package_names, self.enable_repos) mock_exec_cmd_chroot.assert_called_once_with( - "yum install package1 package2 -y " - "--enablerepo=repo1 --enablerepo=repo2" + "yum install package1 package2 -y --enablerepo=repo1 --enablerepo=repo2" ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @@ -340,17 +363,17 @@ def test__yum_install_with_exception(self, mock_exec_cmd_chroot): self.assertRaises( exception.FailedPackageInstallationException, self.morphing_tools._yum_install, - self.package_names, self.enable_repos + self.package_names, + self.enable_repos, ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__yum_uninstall(self, mock_exec_cmd_chroot): self.morphing_tools._yum_uninstall(self.package_names) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("yum remove package1 -y"), - mock.call("yum remove package2 -y") - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("yum remove package1 -y"), mock.call("yum remove package2 -y")] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__yum_uninstall_with_exception(self, mock_exec_cmd_chroot): @@ -358,7 +381,8 @@ def test__yum_uninstall_with_exception(self, mock_exec_cmd_chroot): self.assertRaises( exception.FailedPackageUninstallationException, - self.morphing_tools._yum_uninstall, self.package_names + self.morphing_tools._yum_uninstall, + self.package_names, ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @@ -369,15 +393,13 @@ def test__yum_clean_all(self, mock_test_path, mock_exec_cmd_chroot): self.morphing_tools._yum_clean_all() mock_test_path.assert_called_once_with("var/cache/yum") - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("yum clean all"), - mock.call("rm -rf /var/cache/yum") - ]) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("yum clean all"), mock.call("rm -rf /var/cache/yum")] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path') - def test__yum_clean_all_path_not_exists(self, mock_test_path, - mock_exec_cmd_chroot): + def test__yum_clean_all_path_not_exists(self, mock_test_path, mock_exec_cmd_chroot): mock_test_path.return_value = False self.morphing_tools._yum_clean_all() @@ -393,17 +415,18 @@ def test__find_yum_repos_found(self, mock_read_file_sudo, mock_list_dir): result = self.morphing_tools._find_yum_repos(repos_to_enable) - mock_read_file_sudo.assert_has_calls([ - mock.call('etc/yum.repos.d/file1.repo'), - mock.call('etc/yum.repos.d/file2.repo') - ]) + mock_read_file_sudo.assert_has_calls( + [ + mock.call('etc/yum.repos.d/file1.repo'), + mock.call('etc/yum.repos.d/file2.repo'), + ] + ) self.assertEqual(result, ['repo1']) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file_sudo') - def test__find_yum_repos_not_found(self, mock_read_file_sudo, - mock_list_dir): + def test__find_yum_repos_not_found(self, mock_read_file_sudo, mock_list_dir): mock_list_dir.return_value = ['file1.repo', 'file2.repo'] mock_read_file_sudo.return_value = '[repo1]\n[repo2]' repos_to_enable = ['repo3'] @@ -414,8 +437,9 @@ def test__find_yum_repos_not_found(self, mock_read_file_sudo, @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_install') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_clean_all') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'pre_packages_install') - def test_pre_packages_install(self, mock_pre_packages_install, - mock_yum_clean_all, mock_yum_install): + def test_pre_packages_install( + self, mock_pre_packages_install, mock_yum_clean_all, mock_yum_install + ): self.morphing_tools.installed_packages = [] self.morphing_tools.pre_packages_install(self.package_names) @@ -428,7 +452,8 @@ def test_pre_packages_install(self, mock_pre_packages_install, @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_clean_all') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'pre_packages_install') def test_pre_packages_install_has_grubby( - self, mock_pre_packages_install, mock_yum_clean_all, mock_yum_install): + self, mock_pre_packages_install, mock_yum_clean_all, mock_yum_install + ): self.morphing_tools.installed_packages = ['grubby'] self.morphing_tools.pre_packages_install(self.package_names) @@ -441,8 +466,8 @@ def test_pre_packages_install_has_grubby( @mock.patch.object(redhat.BaseRedHatMorphingTools, '_run_dracut') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'post_packages_install') def test_post_packages_install( - self, mock_post_packages_install, mock__run_dracut, - mock__configure_cloud_init): + self, mock_post_packages_install, mock__run_dracut, mock__configure_cloud_init + ): self.morphing_tools.post_packages_install(self.package_names) @@ -452,13 +477,13 @@ def test_post_packages_install( @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_install') @mock.patch.object(redhat.BaseRedHatMorphingTools, '_get_repos_to_enable') - def test_install_packages(self, mock_get_repos_to_enable, - mock_yum_install): + def test_install_packages(self, mock_get_repos_to_enable, mock_yum_install): self.morphing_tools.install_packages(self.package_names) mock_get_repos_to_enable.assert_called_once() mock_yum_install.assert_called_once_with( - self.package_names, mock_get_repos_to_enable.return_value) + self.package_names, mock_get_repos_to_enable.return_value + ) @mock.patch.object(redhat.BaseRedHatMorphingTools, '_yum_uninstall') def test_uninstall_packages(self, mock_yum_uninstall): @@ -470,41 +495,38 @@ def test_uninstall_packages(self, mock_yum_uninstall): def test__run_dracut(self, mock_exec_cmd_chroot): self.morphing_tools._run_dracut() - mock_exec_cmd_chroot.assert_called_once_with( - "dracut --regenerate-all -f") + mock_exec_cmd_chroot.assert_called_once_with("dracut --regenerate-all -f") @mock.patch.object(redhat.BaseRedHatMorphingTools, '_write_config_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_config_file') def test__set_network_nozeroconf_config( - self, mock_read_config_file, mock_write_config_file): + self, mock_read_config_file, mock_write_config_file + ): self.morphing_tools._set_network_nozeroconf_config() mock_read_config_file.assert_called_once_with( - "etc/sysconfig/network", check_exists=True) + "etc/sysconfig/network", check_exists=True + ) mock_write_config_file.assert_called_once_with( - "etc/sysconfig/network", mock_read_config_file.return_value) + "etc/sysconfig/network", mock_read_config_file.return_value + ) - @mock.patch.object( - redhat.BaseRedHatMorphingTools, '_get_config_file_content' - ) + @mock.patch.object(redhat.BaseRedHatMorphingTools, '_get_config_file_content') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file_sudo') - def test__write_config_file(self, mock_write_file_sudo, - mock_get_config_file_content): + def test__write_config_file( + self, mock_write_file_sudo, mock_get_config_file_content + ): self.morphing_tools._write_config_file( - mock.sentinel.chroot_path, mock.sentinel.config_data) + mock.sentinel.chroot_path, mock.sentinel.config_data + ) - mock_get_config_file_content.assert_called_once_with( - mock.sentinel.config_data) + mock_get_config_file_content.assert_called_once_with(mock.sentinel.config_data) mock_write_file_sudo.assert_called_once_with( - mock.sentinel.chroot_path, - mock_get_config_file_content.return_value + mock.sentinel.chroot_path, mock_get_config_file_content.return_value ) def test__get_config_file_content(self): - config = { - 'key1': 'value1', - 'key2': 'value2' - } + config = {'key1': 'value1', 'key2': 'value2'} result = self.morphing_tools._get_config_file_content(config) self.assertEqual(result, 'key1="value1"\nkey2="value2"\n') diff --git a/coriolis/tests/osmorphing/test_rocky.py b/coriolis/tests/osmorphing/test_rocky.py index 3a384688..79662432 100644 --- a/coriolis/tests/osmorphing/test_rocky.py +++ b/coriolis/tests/osmorphing/test_rocky.py @@ -12,11 +12,9 @@ class BaseRockyLinuxMorphingToolsTestCase(test_base.CoriolisBaseTestCase): def test_check_os_supported(self): detected_os_info = { "distribution_name": rocky.ROCKY_LINUX_DISTRO_IDENTIFIER, - "release_version": "8" + "release_version": "8", } - result = rocky.BaseRockyLinuxMorphingTools.check_os_supported( - detected_os_info - ) + result = rocky.BaseRockyLinuxMorphingTools.check_os_supported(detected_os_info) self.assertTrue(result) @@ -24,8 +22,6 @@ def test_check_os_not_supported(self): detected_os_info = { "distribution_name": "unsupported", } - result = rocky.BaseRockyLinuxMorphingTools.check_os_supported( - detected_os_info - ) + result = rocky.BaseRockyLinuxMorphingTools.check_os_supported(detected_os_info) self.assertFalse(result) diff --git a/coriolis/tests/osmorphing/test_suse.py b/coriolis/tests/osmorphing/test_suse.py index fd526cda..a40f7877 100644 --- a/coriolis/tests/osmorphing/test_suse.py +++ b/coriolis/tests/osmorphing/test_suse.py @@ -5,8 +5,7 @@ from unittest import mock from coriolis import exception -from coriolis.osmorphing import base -from coriolis.osmorphing import suse +from coriolis.osmorphing import base, suse from coriolis.tests import test_base @@ -21,58 +20,59 @@ def setUp(self): "distribution_name": suse.SLES_DISTRO_IDENTIFIER, "release_version": "12", 'friendly_release_name': mock.sentinel.friendly_release_name, - 'suse_release_name': 'test release' + 'suse_release_name': 'test release', } self.package_names = ['package1', 'package2'] self.morphing_tools = suse.BaseSUSEMorphingTools( - mock.sentinel.conn, mock.sentinel.os_root_dir, - mock.sentinel.os_root_dir, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + mock.sentinel.conn, + mock.sentinel.os_root_dir, + mock.sentinel.os_root_dir, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) - - def test_get_required_detected_os_info_fields(self): - result = ( - suse.BaseSUSEMorphingTools.get_required_detected_os_info_fields() + mock.sentinel.operation_timeout, ) - base_fields = ['os_type', 'distribution_name', 'release_version', - 'friendly_release_name'] + def test_get_required_detected_os_info_fields(self): + result = suse.BaseSUSEMorphingTools.get_required_detected_os_info_fields() + + base_fields = [ + 'os_type', + 'distribution_name', + 'release_version', + 'friendly_release_name', + ] expected_result = base_fields + [suse.DETECTED_SUSE_RELEASE_FIELD_NAME] self.assertEqual(expected_result, result) def test_check_os_supported(self): - result = suse.BaseSUSEMorphingTools.check_os_supported( - self.detected_os_info) + result = suse.BaseSUSEMorphingTools.check_os_supported(self.detected_os_info) self.assertTrue(result) def test_check_os_supported_opensuse_tumbleweed(self): - self.detected_os_info[ - 'distribution_name'] = suse.OPENSUSE_DISTRO_IDENTIFIER - self.detected_os_info[ - 'release_version'] = suse.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER + self.detected_os_info['distribution_name'] = suse.OPENSUSE_DISTRO_IDENTIFIER + self.detected_os_info['release_version'] = ( + suse.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER + ) - result = suse.BaseSUSEMorphingTools.check_os_supported( - self.detected_os_info) + result = suse.BaseSUSEMorphingTools.check_os_supported(self.detected_os_info) self.assertTrue(result) def test_check_os_supported_opensuse_unsupported_version(self): - self.detected_os_info[ - 'distribution_name'] = suse.OPENSUSE_DISTRO_IDENTIFIER + self.detected_os_info['distribution_name'] = suse.OPENSUSE_DISTRO_IDENTIFIER self.detected_os_info['release_version'] = 'unsupported' - result = suse.BaseSUSEMorphingTools.check_os_supported( - self.detected_os_info) + result = suse.BaseSUSEMorphingTools.check_os_supported(self.detected_os_info) self.assertFalse(result) def test_check_os_not_supported(self): self.detected_os_info['distribution_name'] = 'unsupported' - result = suse.BaseSUSEMorphingTools.check_os_supported( - self.detected_os_info) + result = suse.BaseSUSEMorphingTools.check_os_supported(self.detected_os_info) self.assertFalse(result) @@ -83,106 +83,94 @@ def test_get_installed_packages(self, mock_exec_cmd_chroot): self.morphing_tools.get_installed_packages() self.assertEqual( - self.morphing_tools.installed_packages, - ['package1', 'package2'] + self.morphing_tools.installed_packages, ['package1', 'package2'] ) - mock_exec_cmd_chroot.assert_called_once_with( - 'rpm -qa --qf "%{NAME}\\n"') + mock_exec_cmd_chroot.assert_called_once_with('rpm -qa --qf "%{NAME}\\n"') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_get_installed_packages_none(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = exception.CoriolisException() - with self.assertLogs( - 'coriolis.osmorphing.suse', level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.suse', level=logging.DEBUG): self.morphing_tools.get_installed_packages() - self.assertEqual( - self.morphing_tools.installed_packages, - [] - ) - mock_exec_cmd_chroot.assert_called_once_with( - 'rpm -qa --qf "%{NAME}\\n"') + self.assertEqual(self.morphing_tools.installed_packages, []) + mock_exec_cmd_chroot.assert_called_once_with('rpm -qa --qf "%{NAME}\\n"') - @mock.patch.object( - suse.BaseSUSEMorphingTools, '_get_grub2_cfg_location' - ) + @mock.patch.object(suse.BaseSUSEMorphingTools, '_get_grub2_cfg_location') def test_get_update_grub2_command(self, mock_get_grub2_cfg_location): result = self.morphing_tools.get_update_grub2_command() mock_get_grub2_cfg_location.assert_called_once_with() self.assertEqual( - result, - "grub2-mkconfig -o %s" % mock_get_grub2_cfg_location.return_value + result, "grub2-mkconfig -o %s" % mock_get_grub2_cfg_location.return_value ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_uefi(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_uefi( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.return_value = True result = self.morphing_tools._get_grub2_cfg_location() self.assertEqual(result, '/boot/efi/EFI/suse/grub.cfg') - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_called_once_with( - '/boot/efi/EFI/suse/grub.cfg') + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_called_once_with('/boot/efi/EFI/suse/grub.cfg') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_bios(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_bios( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.side_effect = [False, True] result = self.morphing_tools._get_grub2_cfg_location() - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_called_with( - '/boot/grub2/grub.cfg') + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_called_with('/boot/grub2/grub.cfg') self.assertEqual(result, '/boot/grub2/grub.cfg') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_test_path_chroot') - def test__get_grub2_cfg_location_not_found(self, mock_test_path_chroot, - mock_exec_cmd_chroot): + def test__get_grub2_cfg_location_not_found( + self, mock_test_path_chroot, mock_exec_cmd_chroot + ): mock_test_path_chroot.return_value = False self.assertRaisesRegex( Exception, "could not determine grub location. boot partition not mounted?", - self.morphing_tools._get_grub2_cfg_location + self.morphing_tools._get_grub2_cfg_location, + ) + mock_exec_cmd_chroot.assert_has_calls( + [mock.call("mount /boot || true"), mock.call("mount /boot/efi || true")] + ) + mock_test_path_chroot.assert_has_calls( + [ + mock.call('/boot/efi/EFI/suse/grub.cfg'), + mock.call('/boot/grub2/grub.cfg'), + ] ) - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("mount /boot || true"), - mock.call("mount /boot/efi || true") - ]) - mock_test_path_chroot.assert_has_calls([ - mock.call('/boot/efi/EFI/suse/grub.cfg'), - mock.call('/boot/grub2/grub.cfg') - ]) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__run_dracut(self, mock_exec_cmd_chroot): self.morphing_tools._run_dracut() - mock_exec_cmd_chroot.assert_called_once_with( - "dracut --regenerate-all -f") + mock_exec_cmd_chroot.assert_called_once_with("dracut --regenerate-all -f") @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__run_mkinitrd_success(self, mock_exec_cmd_chroot): self.morphing_tools._run_mkinitrd() - mock_exec_cmd_chroot.assert_called_once_with( - "mkinitrd") + mock_exec_cmd_chroot.assert_called_once_with("mkinitrd") @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__run_mkinitrd_with_exception(self, mock_exec_cmd_chroot): @@ -203,8 +191,7 @@ def test__rebuild_initrds(self, mock_run_dracut, mock_run_mkinitrd): @mock.patch.object(suse.BaseSUSEMorphingTools, '_run_mkinitrd') @mock.patch.object(suse.BaseSUSEMorphingTools, '_run_dracut') - def test__rebuild_initrds_old_version(self, mock_run_dracut, - mock_run_mkinitrd): + def test__rebuild_initrds_old_version(self, mock_run_dracut, mock_run_mkinitrd): self.morphing_tools._rebuild_initrds() mock_run_mkinitrd.assert_not_called() @@ -229,8 +216,8 @@ def test__has_systemd_with_exception(self, mock_exec_cmd_chroot): @mock.patch.object(suse.BaseSUSEMorphingTools, '_run_dracut') @mock.patch.object(base.BaseLinuxOSMorphingTools, 'post_packages_install') def test_post_packages_install( - self, mock_post_packages_install, mock__run_dracut, - mock__configure_cloud_init): + self, mock_post_packages_install, mock__run_dracut, mock__configure_cloud_init + ): self.morphing_tools.post_packages_install(self.package_names) @@ -244,41 +231,40 @@ def test__enable_sles_module(self, mock_exec_cmd_chroot): self.morphing_tools._enable_sles_module("module2") - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("SUSEConnect --list-extensions"), - mock.call("cp /etc/zypp/zypp.conf /etc/zypp/zypp.conf.tmp"), - mock.call( - "sed -i -e 's/^gpgcheck.*//g' -e '$ a\\gpgcheck = off' " - "/etc/zypp/zypp.conf" - ), - mock.call( - 'SUSEConnect -p %s' % 'module2' - ), - mock.call('mv -f /etc/zypp/zypp.conf.tmp /etc/zypp/zypp.conf'), - mock.call('zypper --non-interactive --no-gpg-checks refresh') - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call("SUSEConnect --list-extensions"), + mock.call("cp /etc/zypp/zypp.conf /etc/zypp/zypp.conf.tmp"), + mock.call( + "sed -i -e 's/^gpgcheck.*//g' -e '$ a\\gpgcheck = off' " + "/etc/zypp/zypp.conf" + ), + mock.call('SUSEConnect -p %s' % 'module2'), + mock.call('mv -f /etc/zypp/zypp.conf.tmp /etc/zypp/zypp.conf'), + mock.call('zypper --non-interactive --no-gpg-checks refresh'), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__enable_sles_module_with_exception(self, mock_exec_cmd_chroot): - mock_exec_cmd_chroot.side_effect = [ - "module output", None, None, Exception()] + mock_exec_cmd_chroot.side_effect = ["module output", None, None, Exception()] - self.assertRaises(exception.CoriolisException, - self.morphing_tools._enable_sles_module, - mock.sentinel.module) + self.assertRaises( + exception.CoriolisException, + self.morphing_tools._enable_sles_module, + mock.sentinel.module, + ) @mock.patch.object(suse.BaseSUSEMorphingTools, '_add_repo') def test_add_cloud_tools_repo(self, mock_add_repo): self.morphing_tools._add_cloud_tools_repo() - expected_repo = suse.CLOUD_TOOLS_REPO_URI_FORMAT % ( - 'test_release', '_12') + expected_repo = suse.CLOUD_TOOLS_REPO_URI_FORMAT % ('test_release', '_12') mock_add_repo.assert_called_once_with(expected_repo, 'Cloud-Tools') @mock.patch.object(suse.BaseSUSEMorphingTools, '_add_repo') def test_add_cloud_tools_repo_with_tumbleweed_version(self, mock_add_repo): - self.morphing_tools._version = ( - suse.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER) + self.morphing_tools._version = suse.OPENSUSE_TUMBLEWEED_VERSION_IDENTIFIER self.morphing_tools._add_cloud_tools_repo() @@ -298,70 +284,74 @@ def test_add_cloud_tools_repo_version_16(self, mock_add_repo): def test_add_cloud_tools_repo_add_repo_failure(self, mock_add_repo): mock_add_repo.side_effect = Exception("connection error") - with self.assertLogs( - 'coriolis.osmorphing.suse', level=logging.WARNING): + with self.assertLogs('coriolis.osmorphing.suse', level=logging.WARNING): self.morphing_tools._add_cloud_tools_repo() - expected_repo = suse.CLOUD_TOOLS_REPO_URI_FORMAT % ( - 'test_release', '_12') + expected_repo = suse.CLOUD_TOOLS_REPO_URI_FORMAT % ('test_release', '_12') mock_add_repo.assert_called_once_with(expected_repo, 'Cloud-Tools') self.event_manager.progress_update.assert_called_once() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test__get_repos(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.return_value = ( - "repo1 http://repo1.com\nrepo2 http://repo2.com") + "repo1 http://repo1.com\nrepo2 http://repo2.com" + ) result = self.morphing_tools._get_repos() mock_exec_cmd_chroot.assert_called_once_with( - "zypper repos -u | awk -F '|' '/^\\s[0-9]+/ {print $2 $7}'") + "zypper repos -u | awk -F '|' '/^\\s[0-9]+/ {print $2 $7}'" + ) - expected_result = { - 'repo1': 'http://repo1.com', 'repo2': 'http://repo2.com'} + expected_result = {'repo1': 'http://repo1.com', 'repo2': 'http://repo2.com'} self.assertEqual(result, expected_result) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(suse.BaseSUSEMorphingTools, '_get_repos') - def test__add_repo_existing_same_uri(self, mock_get_repos, - mock_exec_cmd_chroot): + def test__add_repo_existing_same_uri(self, mock_get_repos, mock_exec_cmd_chroot): mock_get_repos.return_value = {'alias': 'http://repo.com'} with self.assertLogs('coriolis.osmorphing.suse', level=logging.DEBUG): self.morphing_tools._add_repo('http://repo.com', 'alias') mock_get_repos.assert_called_once() - mock_exec_cmd_chroot.assert_has_calls([ - mock.call("zypper --non-interactive modifyrepo -e alias"), - mock.call("zypper --non-interactive --no-gpg-checks refresh") - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call("zypper --non-interactive modifyrepo -e alias"), + mock.call("zypper --non-interactive --no-gpg-checks refresh"), + ] + ) @mock.patch.object(suse.uuid, 'uuid4') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(suse.BaseSUSEMorphingTools, '_get_repos') - def test__add_repo_new(self, mock_get_repos, mock_exec_cmd_chroot, - mock_uuid4): + def test__add_repo_new(self, mock_get_repos, mock_exec_cmd_chroot, mock_uuid4): mock_get_repos.return_value = {'alias': 'http://oldrepo.com'} self.morphing_tools._add_repo('http://newrepo.com', 'alias') mock_get_repos.assert_called_once() - mock_exec_cmd_chroot.assert_has_calls([ - mock.call( - "zypper --non-interactive addrepo -f http://newrepo.com alias" - "%s" % mock_uuid4.return_value), - mock.call("zypper --non-interactive --no-gpg-checks refresh") - ]) + mock_exec_cmd_chroot.assert_has_calls( + [ + mock.call( + "zypper --non-interactive addrepo -f http://newrepo.com alias" + "%s" % mock_uuid4.return_value + ), + mock.call("zypper --non-interactive --no-gpg-checks refresh"), + ] + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') @mock.patch.object(suse.BaseSUSEMorphingTools, '_get_repos') - def test__add_repo_with_exception(self, mock_get_repos, - mock_exec_cmd_chroot): + def test__add_repo_with_exception(self, mock_get_repos, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = Exception() - self.assertRaises(exception.CoriolisException, - self.morphing_tools._add_repo, - 'http://repo.com', 'alias') + self.assertRaises( + exception.CoriolisException, + self.morphing_tools._add_repo, + 'http://repo.com', + 'alias', + ) mock_get_repos.assert_called_once() @@ -370,22 +360,26 @@ def test_install_packages(self, mock_exec_cmd_chroot): self.morphing_tools.install_packages(self.package_names) mock_exec_cmd_chroot.assert_called_once_with( - 'zypper --non-interactive install package1 package2') + 'zypper --non-interactive install package1 package2' + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_install_packages_with_exception(self, mock_exec_cmd_chroot): mock_exec_cmd_chroot.side_effect = exception.CoriolisException() - self.assertRaises(exception.FailedPackageInstallationException, - self.morphing_tools.install_packages, - self.package_names) + self.assertRaises( + exception.FailedPackageInstallationException, + self.morphing_tools.install_packages, + self.package_names, + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_uninstall_packages(self, mock_exec_cmd_chroot): self.morphing_tools.uninstall_packages(self.package_names) mock_exec_cmd_chroot.assert_called_once_with( - 'zypper --non-interactive remove package1 package2') + 'zypper --non-interactive remove package1 package2' + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd_chroot') def test_uninstall_packages_with_exception(self, mock_exec_cmd_chroot): diff --git a/coriolis/tests/osmorphing/test_ubuntu.py b/coriolis/tests/osmorphing/test_ubuntu.py index 4c8327f0..dbe2ee47 100644 --- a/coriolis/tests/osmorphing/test_ubuntu.py +++ b/coriolis/tests/osmorphing/test_ubuntu.py @@ -6,8 +6,7 @@ import ddt -from coriolis.osmorphing import base -from coriolis.osmorphing import ubuntu +from coriolis.osmorphing import base, ubuntu from coriolis.tests import test_base @@ -26,17 +25,22 @@ def setUp(self): self.event_manager = mock.MagicMock() self.os_root_dir = '/root' self.morphing_tools = ubuntu.BaseUbuntuMorphingTools( - mock.sentinel.conn, self.os_root_dir, - mock.sentinel.os_root_dev, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + mock.sentinel.conn, + self.os_root_dir, + mock.sentinel.os_root_dev, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) def test_check_os_supported_not_supported(self): self.detected_os_info['distribution_name'] = 'unsupported' result = ubuntu.BaseUbuntuMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertFalse(result) @@ -52,7 +56,8 @@ def test_check_os_supported_lts_release(self, release_version, expected): self.detected_os_info['release_version'] = release_version result = ubuntu.BaseUbuntuMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertEqual(expected, result) @@ -61,57 +66,54 @@ def test_check_os_supported_lts_release(self, release_version, expected): @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test__set_netplan_ethernet_configs( - self, mock_list_dir, mock_read_file, mock_write_file_sudo, - mock_exec_cmd): + self, mock_list_dir, mock_read_file, mock_write_file_sudo, mock_exec_cmd + ): mock_list_dir.return_value = ['file1', 'file2.yaml'] mock_read_file.return_value = ( - b'network: {version: 2, ethernets: {eth0: {dhcp4: true}}}') + b'network: {version: 2, ethernets: {eth0: {dhcp4: true}}}' + ) nics_info = [ {'name': 'eth0', 'mac_address': '00:00:00:00:00:00'}, - {'name': 'eth1', 'mac_address': '00:00:00:00:00:01'}] + {'name': 'eth1', 'mac_address': '00:00:00:00:00:01'}, + ] config_data = { 'network': { - 'ethernets': { - 'test0': { - 'dhcp4': True, - 'dhcp6': True - } - }, - 'version': 2 + 'ethernets': {'test0': {'dhcp4': True, 'dhcp6': True}}, + 'version': 2, } } self.morphing_tools._set_netplan_ethernet_configs( - nics_info, dhcp=True, iface_name_prefix='test') + nics_info, dhcp=True, iface_name_prefix='test' + ) mock_exec_cmd.assert_called_once_with( "sudo cp '%s/etc/netplan/file2.yaml' " - "'%s/etc/netplan/file2.yaml.bak'" % ( - self.os_root_dir, self.os_root_dir)) + "'%s/etc/netplan/file2.yaml.bak'" % (self.os_root_dir, self.os_root_dir) + ) mock_read_file.assert_called_once_with('etc/netplan/file2.yaml') mock_write_file_sudo.assert_called_once_with( - 'etc/netplan/file2.yaml', ubuntu.yaml.dump(config_data)) + 'etc/netplan/file2.yaml', ubuntu.yaml.dump(config_data) + ) @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_write_file_sudo') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test__set_netplan_ethernet_configs_no_network_data( - self, mock_list_dir, mock_read_file, mock_write_file_sudo, - mock_exec_cmd): + self, mock_list_dir, mock_read_file, mock_write_file_sudo, mock_exec_cmd + ): mock_list_dir.return_value = ['file1.yaml'] mock_read_file.return_value = b'{}' - with self.assertLogs('coriolis.osmorphing.ubuntu', - level=logging.DEBUG): - self.morphing_tools._set_netplan_ethernet_configs( - mock.sentinel.nics_info) + with self.assertLogs('coriolis.osmorphing.ubuntu', level=logging.DEBUG): + self.morphing_tools._set_netplan_ethernet_configs(mock.sentinel.nics_info) mock_read_file.assert_called_once_with('etc/netplan/file1.yaml') mock_exec_cmd.assert_called_once_with( "sudo cp '%s/etc/netplan/file1.yaml' " - "'%s/etc/netplan/file1.yaml.bak'" % ( - self.os_root_dir, self.os_root_dir)) + "'%s/etc/netplan/file1.yaml.bak'" % (self.os_root_dir, self.os_root_dir) + ) mock_write_file_sudo.assert_not_called() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @@ -119,21 +121,19 @@ def test__set_netplan_ethernet_configs_no_network_data( @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test__set_netplan_ethernet_configs_incompatible_version( - self, mock_list_dir, mock_read_file, mock_write_file_sudo, - mock_exec_cmd): + self, mock_list_dir, mock_read_file, mock_write_file_sudo, mock_exec_cmd + ): mock_list_dir.return_value = ['file1.yaml'] mock_read_file.return_value = b'network: {version: 4}' - with self.assertLogs('coriolis.osmorphing.ubuntu', - level=logging.DEBUG): - self.morphing_tools._set_netplan_ethernet_configs( - mock.sentinel.nics_info) + with self.assertLogs('coriolis.osmorphing.ubuntu', level=logging.DEBUG): + self.morphing_tools._set_netplan_ethernet_configs(mock.sentinel.nics_info) mock_read_file.assert_called_once_with('etc/netplan/file1.yaml') mock_exec_cmd.assert_called_once_with( "sudo cp '%s/etc/netplan/file1.yaml' " - "'%s/etc/netplan/file1.yaml.bak'" % ( - self.os_root_dir, self.os_root_dir)) + "'%s/etc/netplan/file1.yaml.bak'" % (self.os_root_dir, self.os_root_dir) + ) mock_write_file_sudo.assert_not_called() @mock.patch.object(base.BaseLinuxOSMorphingTools, '_exec_cmd') @@ -141,16 +141,16 @@ def test__set_netplan_ethernet_configs_incompatible_version( @mock.patch.object(base.BaseLinuxOSMorphingTools, '_read_file') @mock.patch.object(base.BaseLinuxOSMorphingTools, '_list_dir') def test__set_netplan_ethernet_configs_no_ethernet_configs( - self, mock_list_dir, mock_read_file, mock_write_file_sudo, - mock_exec_cmd): + self, mock_list_dir, mock_read_file, mock_write_file_sudo, mock_exec_cmd + ): mock_list_dir.return_value = ['file1.yaml'] mock_read_file.return_value = ( - b'network: {version: 2, ethernets: {eth0: {dhcp4: true}}}') + b'network: {version: 2, ethernets: {eth0: {dhcp4: true}}}' + ) nics_info = [{'name': 'eth0', 'mac_address': '00:00:00:00:00:00'}] - with self.assertLogs('coriolis.osmorphing.ubuntu', - level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.ubuntu', level=logging.DEBUG): self.morphing_tools._set_netplan_ethernet_configs(nics_info) config_data = { @@ -160,13 +160,14 @@ def test__set_netplan_ethernet_configs_no_ethernet_configs( 'dhcp4': True, } }, - 'version': 2 + 'version': 2, } } mock_read_file.assert_called_once_with('etc/netplan/file1.yaml') mock_exec_cmd.assert_called_once_with( "sudo cp '%s/etc/netplan/file1.yaml' " - "'%s/etc/netplan/file1.yaml.bak'" % ( - self.os_root_dir, self.os_root_dir)) + "'%s/etc/netplan/file1.yaml.bak'" % (self.os_root_dir, self.os_root_dir) + ) mock_write_file_sudo.assert_called_once_with( - 'etc/netplan/file1.yaml', ubuntu.yaml.dump(config_data)) + 'etc/netplan/file1.yaml', ubuntu.yaml.dump(config_data) + ) diff --git a/coriolis/tests/osmorphing/test_windows.py b/coriolis/tests/osmorphing/test_windows.py index f635561e..abbc095c 100644 --- a/coriolis/tests/osmorphing/test_windows.py +++ b/coriolis/tests/osmorphing/test_windows.py @@ -44,26 +44,30 @@ def setUp(self): self.event_manager = mock.MagicMock() self.os_root_dir = 'C:\\' self.morphing_tools = windows.BaseWindowsMorphingTools( - self.conn, self.os_root_dir, - mock.sentinel.os_root_dev, mock.sentinel.hypervisor, - self.event_manager, self.detected_os_info, + self.conn, + self.os_root_dir, + mock.sentinel.os_root_dev, + mock.sentinel.hypervisor, + self.event_manager, + self.detected_os_info, mock.sentinel.osmorphing_parameters, - mock.sentinel.operation_timeout) + mock.sentinel.operation_timeout, + ) def test_get_required_detected_os_info_fields(self): - result = windows.BaseWindowsMorphingTools.\ - get_required_detected_os_info_fields() + result = windows.BaseWindowsMorphingTools.get_required_detected_os_info_fields() expected_fields = ( - windows.base.REQUIRED_DETECTED_OS_FIELDS + - windows.REQUIRED_DETECTED_WINDOWS_OS_FIELDS + windows.base.REQUIRED_DETECTED_OS_FIELDS + + windows.REQUIRED_DETECTED_WINDOWS_OS_FIELDS ) self.assertEqual(result, expected_fields) def test_check_os_supported(self): result = windows.BaseWindowsMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertTrue(result) @@ -71,7 +75,8 @@ def test_check_os_supported_not_supported(self): self.detected_os_info['os_type'] = 'unsupported' result = windows.BaseWindowsMorphingTools.check_os_supported( - self.detected_os_info) + self.detected_os_info + ) self.assertFalse(result) @@ -79,13 +84,12 @@ def test__get_worker_os_drive_path(self): result = self.morphing_tools._get_worker_os_drive_path() self.conn.exec_ps_command.assert_called_once_with( - "(Get-WmiObject Win32_OperatingSystem).SystemDrive") + "(Get-WmiObject Win32_OperatingSystem).SystemDrive" + ) self.assertEqual(result, self.conn.exec_ps_command.return_value) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path') def test__get_dism_path(self, mock_get_worker_os_drive_path): mock_get_worker_os_drive_path.return_value = 'C:' @@ -93,10 +97,7 @@ def test__get_dism_path(self, mock_get_worker_os_drive_path): mock_get_worker_os_drive_path.assert_called_once_with() - self.assertEqual( - result, - "C:\\Windows\\System32\\dism.exe" - ) + self.assertEqual(result, "C:\\Windows\\System32\\dism.exe") def test__get_sid(self): result = self.morphing_tools._get_sid() @@ -110,96 +111,97 @@ def test__get_sid(self): def test__grant_permissions(self): self.morphing_tools._grant_permissions( - mock.sentinel.path, mock.sentinel.user, perm="(OI)(CI)F") + mock.sentinel.path, mock.sentinel.user, perm="(OI)(CI)F" + ) self.conn.exec_command.assert_called_once_with( - 'icacls.exe', - [mock.sentinel.path, '/grant', 'sentinel.user:(OI)(CI)F']) + 'icacls.exe', [mock.sentinel.path, '/grant', 'sentinel.user:(OI)(CI)F'] + ) def test__revoke_permissions(self): - self.morphing_tools._revoke_permissions( - mock.sentinel.path, mock.sentinel.user) + self.morphing_tools._revoke_permissions(mock.sentinel.path, mock.sentinel.user) self.conn.exec_command.assert_called_once_with( - 'icacls.exe', - [mock.sentinel.path, '/remove', mock.sentinel.user]) + 'icacls.exe', [mock.sentinel.path, '/remove', mock.sentinel.user] + ) def test__load_registry_hive(self): self.morphing_tools._load_registry_hive( - mock.sentinel.subkey, mock.sentinel.path) + mock.sentinel.subkey, mock.sentinel.path + ) self.conn.exec_command.assert_called_once_with( - 'reg.exe', - ['load', mock.sentinel.subkey, mock.sentinel.path]) + 'reg.exe', ['load', mock.sentinel.subkey, mock.sentinel.path] + ) def test__unload_registry_hive(self): self.morphing_tools._unload_registry_hive(mock.sentinel.subkey) self.conn.exec_command.assert_called_once_with( - 'reg.exe', - ['unload', mock.sentinel.subkey]) + 'reg.exe', ['unload', mock.sentinel.subkey] + ) def test__get_ps_fl_value(self): result = self.morphing_tools._get_ps_fl_value( - WIN_VERSION_PS_OUTPUT, 'InstallationType') + WIN_VERSION_PS_OUTPUT, 'InstallationType' + ) self.assertEqual(result, 'Server') - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path') def test__add_dism_driver(self, mock_get_worker_os_drive_path): mock_get_worker_os_drive_path.return_value = "C:" - result = self.morphing_tools._add_dism_driver( - mock.sentinel.driver_path) + result = self.morphing_tools._add_dism_driver(mock.sentinel.driver_path) self.assertEqual(result, self.conn.exec_command.return_value) self.conn.exec_command.assert_called_once_with( "C:\\Windows\\System32\\dism.exe", - ['/add-driver', '/image:%s' % self.os_root_dir, - '/driver:"%s"' % mock.sentinel.driver_path, - '/recurse', '/forceunsigned']) + [ + '/add-driver', + '/image:%s' % self.os_root_dir, + '/driver:"%s"' % mock.sentinel.driver_path, + '/recurse', + '/forceunsigned', + ], + ) mock_get_worker_os_drive_path.assert_called_once_with() - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path') def test__add_dism_driver_exception(self, mock_get_worker_os_drive_path): mock_get_worker_os_drive_path.return_value = "C:" self.conn.test_path.return_value = True self.conn.exec_command.side_effect = CoriolisTestException - with self.assertLogs('coriolis.osmorphing.windows', - level=logging.ERROR): - self.assertRaises(CoriolisTestException, - self.morphing_tools._add_dism_driver, - mock.sentinel.driver_path) + with self.assertLogs('coriolis.osmorphing.windows', level=logging.ERROR): + self.assertRaises( + CoriolisTestException, + self.morphing_tools._add_dism_driver, + mock.sentinel.driver_path, + ) mock_get_worker_os_drive_path.assert_called() - self.conn.test_path.assert_called_once_with( - "C:\\Windows\\Logs\\DISM\\dism.log") + self.conn.test_path.assert_called_once_with("C:\\Windows\\Logs\\DISM\\dism.log") - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_worker_os_drive_path') def test__add_dism_driver_not_found(self, mock_get_worker_os_drive_path): self.conn.test_path.return_value = False self.conn.exec_command.side_effect = CoriolisTestException - with self.assertLogs('coriolis.osmorphing.windows', - level=logging.WARN): - self.assertRaises(CoriolisTestException, - self.morphing_tools._add_dism_driver, - mock.sentinel.driver_path) + with self.assertLogs('coriolis.osmorphing.windows', level=logging.WARN): + self.assertRaises( + CoriolisTestException, + self.morphing_tools._add_dism_driver, + mock.sentinel.driver_path, + ) mock_get_worker_os_drive_path.assert_called() def test__mount_disk_image(self): - result = self.morphing_tools._mount_disk_image( - mock.sentinel.path) + result = self.morphing_tools._mount_disk_image(mock.sentinel.path) self.conn.exec_ps_command.assert_called_once_with( - "(Mount-DiskImage '%s' -PassThru | Get-Volume).DriveLetter" % - mock.sentinel.path) + "(Mount-DiskImage '%s' -PassThru | Get-Volume).DriveLetter" + % mock.sentinel.path + ) self.assertEqual(result, self.conn.exec_ps_command.return_value) @@ -207,16 +209,15 @@ def test__dismount_disk_image(self): self.morphing_tools._dismount_disk_image(mock.sentinel.path) self.conn.exec_ps_command.assert_called_once_with( - "Dismount-DiskImage '%s'" % mock.sentinel.path, ignore_stdout=True) + "Dismount-DiskImage '%s'" % mock.sentinel.path, ignore_stdout=True + ) def test__expand_archive(self): self.conn.test_path.return_value = True destination = 'C:\\Windows\\destination' - with self.assertLogs('coriolis.osmorphing.windows', - level=logging.WARN): - self.morphing_tools._expand_archive( - mock.sentinel.archive_path, destination) + with self.assertLogs('coriolis.osmorphing.windows', level=logging.WARN): + self.morphing_tools._expand_archive(mock.sentinel.archive_path, destination) self.conn.exec_ps_command.assert_called_once() @@ -225,29 +226,33 @@ def test__expand_archive_remove_destination(self): destination = 'C:\\test\\destination' - self.morphing_tools._expand_archive( - mock.sentinel.archive_path, destination) - - self.conn.exec_ps_command.assert_has_calls([ - mock.call("rm -recurse -force %s" % destination), - mock.call( - "Expand-Archive -LiteralPath '%(path)s' " - "-DestinationPath '%(destination)s' -Force" % - {"path": mock.sentinel.archive_path, - "destination": destination}, - ignore_stdout=True) - ]) + self.morphing_tools._expand_archive(mock.sentinel.archive_path, destination) + + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call("rm -recurse -force %s" % destination), + mock.call( + "Expand-Archive -LiteralPath '%(path)s' " + "-DestinationPath '%(destination)s' -Force" + % {"path": mock.sentinel.archive_path, "destination": destination}, + ignore_stdout=True, + ), + ] + ) def test__set_service_start_mode(self): self.morphing_tools._set_service_start_mode( - mock.sentinel.key_name, mock.sentinel.service_name, - mock.sentinel.start_mode) + mock.sentinel.key_name, mock.sentinel.service_name, mock.sentinel.start_mode + ) registry_path = windows.SERVICE_PATH_FORMAT % ( - mock.sentinel.key_name, mock.sentinel.service_name) - expected_command = ( - "Set-ItemProperty -Path '%s' -Name 'Start' -Value %s" % - (registry_path, mock.sentinel.start_mode)) + mock.sentinel.key_name, + mock.sentinel.service_name, + ) + expected_command = "Set-ItemProperty -Path '%s' -Name 'Start' -Value %s" % ( + registry_path, + mock.sentinel.start_mode, + ) self.conn.exec_ps_command.assert_called_once_with(expected_command) @@ -256,13 +261,20 @@ def test__create_service(self): depends_on = [mock.sentinel.depends_on] self.morphing_tools._create_service( - mock.sentinel.key_name, mock.sentinel.service_name, - mock.sentinel.image_path, mock.sentinel.display_name, - mock.sentinel.description, windows.SERVICE_START_AUTO, - service_account, depends_on) + mock.sentinel.key_name, + mock.sentinel.service_name, + mock.sentinel.image_path, + mock.sentinel.display_name, + mock.sentinel.description, + windows.SERVICE_START_AUTO, + service_account, + depends_on, + ) registry_path = windows.SERVICE_PATH_FORMAT % ( - mock.sentinel.key_name, mock.sentinel.service_name) + mock.sentinel.key_name, + mock.sentinel.service_name, + ) depends_on_ps = "@('%s')" % mock.sentinel.depends_on expected_commands = ( @@ -283,23 +295,26 @@ def test__create_service(self): "New-ItemProperty -Path '%(path)s' -Name 'Type' -Value " "16 -Type DWord -Force;" "New-ItemProperty -Path '%(path)s' -Name 'ErrorControl' -Value " - "0 -Type DWord -Force" % - {"path": registry_path, "image_path": mock.sentinel.image_path, - "display_name": mock.sentinel.display_name, - "description": mock.sentinel.description, - "depends_on": depends_on_ps, "service_account": service_account, - "start_mode": windows.SERVICE_START_AUTO} + "0 -Type DWord -Force" + % { + "path": registry_path, + "image_path": mock.sentinel.image_path, + "display_name": mock.sentinel.display_name, + "description": mock.sentinel.description, + "depends_on": depends_on_ps, + "service_account": service_account, + "start_mode": windows.SERVICE_START_AUTO, + } ) self.conn.exec_ps_command.assert_called_once_with( - expected_commands, ignore_stdout=True) + expected_commands, ignore_stdout=True + ) def test__delete_startup_entry(self): - self.morphing_tools._delete_startup_entry( - "mock_key_name", "mock_service_name") + self.morphing_tools._delete_startup_entry("mock_key_name", "mock_service_name") - registry_path = ("HKLM:\\mock_key_name\\\Microsoft\\" - "Windows\\CurrentVersion\\Run") + registry_path = "HKLM:\\mock_key_name\\\Microsoft\\Windows\\CurrentVersion\\Run" expected_commands = ( "$ErrorActionPreference = 'Stop';" @@ -309,14 +324,17 @@ def test__delete_startup_entry(self): ) self.conn.exec_ps_command.assert_called_once_with( - expected_commands, ignore_stdout=True) + expected_commands, ignore_stdout=True + ) def test__delete_uninstall_entry(self): self.morphing_tools._delete_uninstall_entry( - "mock_key_name", "mock_service_name") + "mock_key_name", "mock_service_name" + ) - registry_path = ("HKLM:\\mock_key_name\\\Microsoft\\" - "Windows\\CurrentVersion\\Uninstall\\*") + registry_path = ( + "HKLM:\\mock_key_name\\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\*" + ) expected_commands = ( "$ErrorActionPreference = 'Stop';" @@ -326,7 +344,8 @@ def test__delete_uninstall_entry(self): ) self.conn.exec_ps_command.assert_called_once_with( - expected_commands, ignore_stdout=True) + expected_commands, ignore_stdout=True + ) @mock.patch.object(windows.utils, 'write_winrm_file') def test_run_user_script(self, mock_write_winrm_file): @@ -336,13 +355,17 @@ def test_run_user_script(self, mock_write_winrm_file): self.morphing_tools.run_user_script(user_script) mock_write_winrm_file.assert_called_once_with( - self.conn, script_path, user_script) + self.conn, script_path, user_script + ) self.conn.exec_ps_command.assert_called_once_with( - ('$ErrorActionPreference = "Stop"; powershell.exe ' - '-NonInteractive -ExecutionPolicy RemoteSigned ' - '-File "%(script)s" "%(os_root_dir)s"') % { - "script": script_path, - "os_root_dir": self.os_root_dir, + ( + '$ErrorActionPreference = "Stop"; powershell.exe ' + '-NonInteractive -ExecutionPolicy RemoteSigned ' + '-File "%(script)s" "%(os_root_dir)s"' + ) + % { + "script": script_path, + "os_root_dir": self.os_root_dir, } ) @@ -356,19 +379,23 @@ def test_run_user_script_empty_script(self, mock_write_winrm_file): @mock.patch.object(windows.utils, 'write_winrm_file') def test_run_user_script_raises_exception_on_write_winrm_file( - self, mock_write_winrm_file): + self, mock_write_winrm_file + ): user_script = 'echo "Hello, World!"' mock_write_winrm_file.side_effect = exception.CoriolisException self.assertRaises( exception.CoriolisException, - self.morphing_tools.run_user_script, user_script) + self.morphing_tools.run_user_script, + user_script, + ) self.conn.exec_ps_command.assert_not_called() @mock.patch.object(windows.utils, 'write_winrm_file') def test_run_user_script_raises_exception_on_exec_ps_command( - self, mock_write_winrm_file): + self, mock_write_winrm_file + ): user_script = 'echo "Hello, World!"' script_path = "$env:TMP\\coriolis_user_script.ps1" @@ -376,76 +403,90 @@ def test_run_user_script_raises_exception_on_exec_ps_command( self.assertRaises( exception.CoriolisException, - self.morphing_tools.run_user_script, user_script) + self.morphing_tools.run_user_script, + user_script, + ) mock_write_winrm_file.assert_called_once_with( - self.conn, script_path, user_script) + self.conn, script_path, user_script + ) @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_set_service_start_mode' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_set_service_start_mode') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') def test__disable_cloudbase_init( - self, mock_uuid4, mock_unload_registry_hive, - mock_set_service_start_mode, mock_check_cloudbase_init_exists, - mock_load_registry_hive): + self, + mock_uuid4, + mock_unload_registry_hive, + mock_set_service_start_mode, + mock_check_cloudbase_init_exists, + mock_load_registry_hive, + ): self.morphing_tools._disable_cloudbase_init() mock_check_cloudbase_init_exists.return_value = True mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir, + ) mock_set_service_start_mode.assert_called_once_with( - str(mock_uuid4.return_value), windows.CLOUDBASEINIT_SERVICE_NAME, - windows.SERVICE_START_DISABLED) + str(mock_uuid4.return_value), + windows.CLOUDBASEINIT_SERVICE_NAME, + windows.SERVICE_START_DISABLED, + ) mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) def test__check_cloudbase_init_exists(self): self.conn.exec_ps_command.return_value = "True" result = self.morphing_tools._check_cloudbase_init_exists( - mock.sentinel.key_name) + mock.sentinel.key_name + ) self.conn.exec_ps_command.assert_called_once_with( - "Test-Path %s" % (windows.SERVICE_PATH_FORMAT % ( - mock.sentinel.key_name, windows.CLOUDBASEINIT_SERVICE_NAME))) + "Test-Path %s" + % ( + windows.SERVICE_PATH_FORMAT + % (mock.sentinel.key_name, windows.CLOUDBASEINIT_SERVICE_NAME) + ) + ) self.assertTrue(result) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_set_service_start_mode' - ) - def test__setup_existing_cbslinit_service( - self, mock_set_service_start_mode): + @mock.patch.object(windows.BaseWindowsMorphingTools, '_set_service_start_mode') + def test__setup_existing_cbslinit_service(self, mock_set_service_start_mode): self.morphing_tools._setup_existing_cbslinit_service( - mock.sentinel.key_name, mock.sentinel.image_path) + mock.sentinel.key_name, mock.sentinel.image_path + ) reg_service_path = windows.SERVICE_PATH_FORMAT % ( - mock.sentinel.key_name, windows.CLOUDBASEINIT_SERVICE_NAME) - - self.conn.exec_ps_command.assert_has_calls([ - mock.call( - "Set-ItemProperty -Path '%s' -Name 'ImagePath' -Value '%s' " - "-Force" % (reg_service_path, mock.sentinel.image_path), - ), - mock.call( - "Set-ItemProperty -Path '%s' -Name 'ObjectName' " - "-Value 'LocalSystem' -Force" % reg_service_path - ), - ]) + mock.sentinel.key_name, + windows.CLOUDBASEINIT_SERVICE_NAME, + ) + + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "Set-ItemProperty -Path '%s' -Name 'ImagePath' -Value '%s' " + "-Force" % (reg_service_path, mock.sentinel.image_path), + ), + mock.call( + "Set-ItemProperty -Path '%s' -Name 'ObjectName' " + "-Value 'LocalSystem' -Force" % reg_service_path + ), + ] + ) mock_set_service_start_mode.assert_called_once_with( - mock.sentinel.key_name, windows.CLOUDBASEINIT_SERVICE_NAME, - windows.SERVICE_START_AUTO) + mock.sentinel.key_name, + windows.CLOUDBASEINIT_SERVICE_NAME, + windows.SERVICE_START_AUTO, + ) def test__get_cbslinit_base_dir(self): result = self.morphing_tools._get_cbslinit_base_dir() @@ -462,36 +503,39 @@ def test__write_local_script(self, mock_write_winrm_file): script_path = "/script/path/script.sh" with mock.patch('builtins.open', mock.mock_open()) as mock_open: - self.morphing_tools._write_local_script( - 'C:', script_path, priority=50) + self.morphing_tools._write_local_script('C:', script_path, priority=50) mock_open.assert_called_once_with(script_path, 'r') mock_write_winrm_file.assert_called_once_with( - self.conn, - 'C:\\LocalScripts\\50-script.sh', - '') + self.conn, 'C:\\LocalScripts\\50-script.sh', '' + ) @mock.patch.object(windows.utils, 'write_winrm_file') @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_local_script') def test__write_cloudbase_init_conf( - self, mock_write_local_script, mock_write_winrm_file): + self, mock_write_local_script, mock_write_winrm_file + ): local_base_dir = "C:\\LocalBaseDir" mocked_full_path = windows.os.path.join( - windows.utils.get_resources_bin_dir(), 'bring-disks-online.ps1') + windows.utils.get_resources_bin_dir(), 'bring-disks-online.ps1' + ) self.morphing_tools._write_cloudbase_init_conf( - 'C:\\Cloudbase-Init', local_base_dir, com_port='COM1') - - self.conn.exec_ps_command.assert_has_calls([ - mock.call( - "mkdir 'C:\\Cloudbase-Init\\conf' -Force", - ignore_stdout=True), - mock.call( - "mkdir 'C:\\Cloudbase-Init\\LocalScripts' -Force", - ignore_stdout=True), - ]) - - conf_file_path = ( - "C:\\Cloudbase-Init\\conf\\cloudbase-init.conf") + 'C:\\Cloudbase-Init', local_base_dir, com_port='COM1' + ) + + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "mkdir 'C:\\Cloudbase-Init\\conf' -Force", ignore_stdout=True + ), + mock.call( + "mkdir 'C:\\Cloudbase-Init\\LocalScripts' -Force", + ignore_stdout=True, + ), + ] + ) + + conf_file_path = "C:\\Cloudbase-Init\\conf\\cloudbase-init.conf" conf_content = ( "[DEFAULT]\r\n" "username = Admin\r\n" @@ -510,46 +554,50 @@ def test__write_cloudbase_init_conf( "debug = true\r\n" "san_policy = OnlineAll\r\n" "metadata_services = %(metadata_services)s\r\n" - "logging_serial_port_settings = %(com_port)s,9600,N,8\r\n" % - {"bin_path": "%s\\Bin" % local_base_dir, - "log_path": "%s\\Log" % local_base_dir, - "scripts_path": "%s\\LocalScripts" % local_base_dir, - "com_port": 'COM1', - "metadata_services": ",".join( - windows.CLOUDBASE_INIT_DEFAULT_METADATA_SVCS), - "plugins": ",".join(windows.CLOUDBASE_INIT_DEFAULT_PLUGINS)}) + "logging_serial_port_settings = %(com_port)s,9600,N,8\r\n" + % { + "bin_path": "%s\\Bin" % local_base_dir, + "log_path": "%s\\Log" % local_base_dir, + "scripts_path": "%s\\LocalScripts" % local_base_dir, + "com_port": 'COM1', + "metadata_services": ",".join( + windows.CLOUDBASE_INIT_DEFAULT_METADATA_SVCS + ), + "plugins": ",".join(windows.CLOUDBASE_INIT_DEFAULT_PLUGINS), + } + ) mock_write_winrm_file.assert_called_once_with( - self.conn, conf_file_path, conf_content) + self.conn, conf_file_path, conf_content + ) mock_write_local_script.assert_called_once_with( - 'C:\\Cloudbase-Init', mocked_full_path, priority=99) + 'C:\\Cloudbase-Init', mocked_full_path, priority=99 + ) @mock.patch.object(windows.utils, 'write_winrm_file') @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_local_script') def test__write_cloudbase_init_conf_with_exception( - self, mock_write_local_script, mock_write_winrm_file): + self, mock_write_local_script, mock_write_winrm_file + ): plugins = "invalid plugins" self.assertRaises( exception.CoriolisException, self.morphing_tools._write_cloudbase_init_conf, - mock.sentinel.cloudbaseinit_base_dir, mock.sentinel.local_base_dir, - com_port='COM1', plugins=plugins) + mock.sentinel.cloudbaseinit_base_dir, + mock.sentinel.local_base_dir, + com_port='COM1', + plugins=plugins, + ) self.conn.exec_ps_command.assert_not_called() mock_write_winrm_file.assert_not_called() mock_write_local_script.assert_not_called() - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_write_cloudbase_init_conf' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_cloudbase_init_conf') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists') @mock.patch.object( windows.BaseWindowsMorphingTools, '_setup_existing_cbslinit_service' ) @@ -558,11 +606,16 @@ def test__write_cloudbase_init_conf_with_exception( @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') def test__install_cloudbase_init( - self, mock_uuid4, mock_load_registry_hive, - mock_expand_archive, mock_create_service, - mock_setup_existing_cbslinit_service, - mock_check_cloudbase_init_exists, mock_write_cloudbase_init_conf, - mock_unload_registry_hive): + self, + mock_uuid4, + mock_load_registry_hive, + mock_expand_archive, + mock_create_service, + mock_setup_existing_cbslinit_service, + mock_check_cloudbase_init_exists, + mock_write_cloudbase_init_conf, + mock_unload_registry_hive, + ): cloudbaseinit_zip_path = 'c:\\cloudbaseinit.zip' cloudbaseinit_base_dir = "C:\\Cloudbase-Init" local_base_dir = "C%s" % cloudbaseinit_base_dir[1:] @@ -570,33 +623,41 @@ def test__install_cloudbase_init( mock_check_cloudbase_init_exists.return_value = False result = self.morphing_tools._install_cloudbase_init( - mock.sentinel.download_url, com_port='COM1') + mock.sentinel.download_url, com_port='COM1' + ) mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir, + ) mock_expand_archive.assert_called_once_with( - cloudbaseinit_zip_path, cloudbaseinit_base_dir, - overwrite=False) + cloudbaseinit_zip_path, cloudbaseinit_base_dir, overwrite=False + ) self.conn.exec_ps_command.assert_called_once_with( - "mkdir '%s' -Force" % - ("%s\\Log" % cloudbaseinit_base_dir), - ignore_stdout=True) + "mkdir '%s' -Force" % ("%s\\Log" % cloudbaseinit_base_dir), + ignore_stdout=True, + ) mock_write_cloudbase_init_conf.assert_called_once_with( - cloudbaseinit_base_dir, local_base_dir, - metadata_services=None, plugins=None, com_port='COM1') + cloudbaseinit_base_dir, + local_base_dir, + metadata_services=None, + plugins=None, + com_port='COM1', + ) mock_check_cloudbase_init_exists.assert_called_once_with( - str(mock_uuid4.return_value)) + str(mock_uuid4.return_value) + ) expected_image_path = ( '"%(path)s\\Bin\\OpenStackService.exe" cloudbase-init ' '"%(path)s\\Python\\Python.exe" ' '"%(path)s\\Python\\Scripts\\cloudbase-init.exe" ' - '--config-file "%(path)s\\conf\\cloudbase-init.conf"' % { - 'path': local_base_dir}) + '--config-file "%(path)s\\conf\\cloudbase-init.conf"' + % {'path': local_base_dir} + ) mock_setup_existing_cbslinit_service.assert_not_called() mock_create_service.assert_called_once_with( @@ -608,65 +669,75 @@ def test__install_cloudbase_init( ) mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) self.assertEqual(result, cloudbaseinit_base_dir) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_check_cloudbase_init_exists') @mock.patch.object( windows.BaseWindowsMorphingTools, '_setup_existing_cbslinit_service' ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_write_cloudbase_init_conf' - ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_cloudbase_init_conf') @mock.patch.object(windows.BaseWindowsMorphingTools, '_create_service') @mock.patch.object(windows.BaseWindowsMorphingTools, '_expand_archive') @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') def test_install_cloudbase_init_existing_service( - self, mock_uuid4, mock_load_registry_hive, mock_expand_archive, - mock_create_service, mock_write_cloudbase_init_conf, - mock_setup_existing_cbslinit_service, - mock_check_cloudbase_init_exists, mock_unload_registry_hive): + self, + mock_uuid4, + mock_load_registry_hive, + mock_expand_archive, + mock_create_service, + mock_write_cloudbase_init_conf, + mock_setup_existing_cbslinit_service, + mock_check_cloudbase_init_exists, + mock_unload_registry_hive, + ): cloudbaseinit_zip_path = 'c:\\cloudbaseinit.zip' cloudbaseinit_base_dir = "C:\\Cloudbase-Init" local_base_dir = "C%s" % cloudbaseinit_base_dir[1:] mock_check_cloudbase_init_exists.return_value = True - self.morphing_tools._install_cloudbase_init(mock.sentinel.download_url, - com_port='COM1') + self.morphing_tools._install_cloudbase_init( + mock.sentinel.download_url, com_port='COM1' + ) expected_image_path = ( '"%(path)s\\Bin\\OpenStackService.exe" cloudbase-init ' '"%(path)s\\Python\\Python.exe" ' '"%(path)s\\Python\\Scripts\\cloudbase-init.exe" ' - '--config-file "%(path)s\\conf\\cloudbase-init.conf"' % { - 'path': local_base_dir}) + '--config-file "%(path)s\\conf\\cloudbase-init.conf"' + % {'path': local_base_dir} + ) mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir, + ) mock_expand_archive.assert_called_once_with( - cloudbaseinit_zip_path, cloudbaseinit_base_dir, - overwrite=False) + cloudbaseinit_zip_path, cloudbaseinit_base_dir, overwrite=False + ) self.conn.exec_ps_command.assert_called_once_with( - "mkdir '%s' -Force" % - ("%s\\Log" % cloudbaseinit_base_dir), - ignore_stdout=True) + "mkdir '%s' -Force" % ("%s\\Log" % cloudbaseinit_base_dir), + ignore_stdout=True, + ) mock_write_cloudbase_init_conf.assert_called_once_with( - cloudbaseinit_base_dir, local_base_dir, - metadata_services=None, plugins=None, com_port='COM1') + cloudbaseinit_base_dir, + local_base_dir, + metadata_services=None, + plugins=None, + com_port='COM1', + ) mock_setup_existing_cbslinit_service.assert_called_once_with( - str(mock_uuid4.return_value), expected_image_path) + str(mock_uuid4.return_value), expected_image_path + ) mock_create_service.assert_not_called() mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) def test__compile_static_ip_conf_from_registry(self): self.conn.exec_ps_command.side_effect = [ @@ -678,41 +749,52 @@ def test__compile_static_ip_conf_from_registry(self): '255.255.255.0', '1', ] - interfaces_reg_path = (windows.INTERFACES_PATH_FORMAT % - mock.sentinel.key_name) + interfaces_reg_path = windows.INTERFACES_PATH_FORMAT % mock.sentinel.key_name result = self.morphing_tools._compile_static_ip_conf_from_registry( - mock.sentinel.key_name) - - self.conn.exec_ps_command.assert_has_calls([ - mock.call( - "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " - "'[^\\\\]+$').Matches).Value" % interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').DefaultGateway" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').NameServer" - % interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').IPAddress" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').SubnetMask" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface2').EnableDHCP" - % interfaces_reg_path), - ]) + mock.sentinel.key_name + ) + + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " + "'[^\\\\]+$').Matches).Value" % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').DefaultGateway" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').NameServer" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').IPAddress" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').SubnetMask" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface2').EnableDHCP" + % interfaces_reg_path + ), + ] + ) expected_ips_info = [ - {"ip_addresses": ['192.168.1.1'], - "prefix_lengths": [24], - "default_gateway": '192.168.1.254', - "dns_addresses": '8.8.8.8'} + { + "ip_addresses": ['192.168.1.1'], + "prefix_lengths": [24], + "default_gateway": '192.168.1.254', + "dns_addresses": '8.8.8.8', + } ] self.assertEqual(result, expected_ips_info) @@ -726,50 +808,64 @@ def test_compile_static_ip_conf_from_registry_no_ip_or_subnet(self): None, ] - interfaces_reg_path = (windows.INTERFACES_PATH_FORMAT % - mock.sentinel.key_name) - with self.assertLogs('coriolis.osmorphing.windows', - level=logging.WARNING): + interfaces_reg_path = windows.INTERFACES_PATH_FORMAT % mock.sentinel.key_name + with self.assertLogs('coriolis.osmorphing.windows', level=logging.WARNING): self.morphing_tools._compile_static_ip_conf_from_registry( - mock.sentinel.key_name) - - self.conn.exec_ps_command.assert_has_calls([ - mock.call( - "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " - "'[^\\\\]+$').Matches).Value" % interfaces_reg_path), - mock.call("(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').DefaultGateway" % - interfaces_reg_path), - mock.call( - "(Get-ItemProperty -Path '%s\\interface1').NameServer" - % interfaces_reg_path), - mock.call("(Get-ItemProperty -Path '%s\\interface1').IPAddress" % ( - interfaces_reg_path)), - mock.call("(Get-ItemProperty -Path '%s\\interface1').SubnetMask" % - interfaces_reg_path), - ]) + mock.sentinel.key_name + ) + + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " + "'[^\\\\]+$').Matches).Value" % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').DefaultGateway" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').NameServer" + % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').IPAddress" + % (interfaces_reg_path) + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').SubnetMask" + % interfaces_reg_path + ), + ] + ) def test_compile_static_ip_conf_from_registry_no_static_ip(self): self.conn.exec_ps_command.side_effect = [ 'interface1', '1', ] - interfaces_reg_path = (windows.INTERFACES_PATH_FORMAT % - mock.sentinel.key_name) + interfaces_reg_path = windows.INTERFACES_PATH_FORMAT % mock.sentinel.key_name - with self.assertLogs('coriolis.osmorphing.windows', - level=logging.DEBUG): + with self.assertLogs('coriolis.osmorphing.windows', level=logging.DEBUG): self.morphing_tools._compile_static_ip_conf_from_registry( - mock.sentinel.key_name) - self.conn.exec_ps_command.assert_has_calls([ - mock.call( - "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " - "'[^\\\\]+$').Matches).Value" % interfaces_reg_path), - mock.call("(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" % - interfaces_reg_path), - ]) + mock.sentinel.key_name + ) + self.conn.exec_ps_command.assert_has_calls( + [ + mock.call( + "(((Get-ChildItem -Path '%s').Name | Select-String -Pattern " + "'[^\\\\]+$').Matches).Value" % interfaces_reg_path + ), + mock.call( + "(Get-ItemProperty -Path '%s\\interface1').EnableDHCP" + % interfaces_reg_path + ), + ] + ) def test__get_static_nics_info(self): static_ipv4 = "10.0.0.16" @@ -777,126 +873,117 @@ def test__get_static_nics_info(self): dynamic_ipv4 = "10.0.1.16" dynamic_ipv6 = "fe81::728a:688:1a92:baec" # detected static IPs - ips_info = [{"ip_addresses": [static_ipv4]}, - {"ip_addresses": [static_ipv6]}] + ips_info = [{"ip_addresses": [static_ipv4]}, {"ip_addresses": [static_ipv6]}] nics_info = [ # no IP addresses on NIC {"ip_addresses": [], "mac_address": "00:50:56:92:91:42"}, # dynamic ipv6 IP - {"ip_addresses": [static_ipv4, dynamic_ipv6], - "mac_address": "00:50:56:92:91:43"}, + { + "ip_addresses": [static_ipv4, dynamic_ipv6], + "mac_address": "00:50:56:92:91:43", + }, # both IPs dynamic - {"ip_addresses": [dynamic_ipv4, dynamic_ipv6], - "mac_address": "00:50:56:92:91:44"}, + { + "ip_addresses": [dynamic_ipv4, dynamic_ipv6], + "mac_address": "00:50:56:92:91:44", + }, # dynamic ipv4 IP - {"ip_addresses": [dynamic_ipv4, static_ipv6], - "mac_address": "00:50:56:92:91:45"}] + { + "ip_addresses": [dynamic_ipv4, static_ipv6], + "mac_address": "00:50:56:92:91:45", + }, + ] expected_result = [ - {"mac_address": "00:50:56:92:91:43", - "ip_addresses": ["10.0.0.16"]}, - {"mac_address": "00:50:56:92:91:45", - "ip_addresses": ["fe80::728a:688:1a92:baec"]}] + {"mac_address": "00:50:56:92:91:43", "ip_addresses": ["10.0.0.16"]}, + { + "mac_address": "00:50:56:92:91:45", + "ip_addresses": ["fe80::728a:688:1a92:baec"], + }, + ] self.assertEqual( self.morphing_tools._get_static_nics_info(nics_info, ips_info), - expected_result) + expected_result, + ) @mock.patch.object(windows.utils, 'write_winrm_file') def test__write_static_ip_script(self, mock_write_winrm_file): - nics_info = [ - {'ip_addresses': ["10.1.10.10"]} - ] - ips_info = [ - {'ip_address': "10.1.10.10"} - ] - self.morphing_tools._write_static_ip_script( - 'C:', nics_info, ips_info) + nics_info = [{'ip_addresses': ["10.1.10.10"]}] + ips_info = [{'ip_address': "10.1.10.10"}] + self.morphing_tools._write_static_ip_script('C:', nics_info, ips_info) contents = windows.STATIC_IP_SCRIPT_TEMPLATE % { - 'nics_info': base64.b64encode( - json.dumps(nics_info).encode()).decode(), - 'ips_info': base64.b64encode( - json.dumps(ips_info).encode()).decode(), + 'nics_info': base64.b64encode(json.dumps(nics_info).encode()).decode(), + 'ips_info': base64.b64encode(json.dumps(ips_info).encode()).decode(), } mock_write_winrm_file.assert_called_once_with( self.morphing_tools._conn, "C:\\LocalScripts\\01-static-ip-config.ps1", - contents.encode()) + contents.encode(), + ) @mock.patch.object( - windows.BaseWindowsMorphingTools, - '_compile_static_ip_conf_from_registry' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_write_static_ip_script' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_static_nics_info') - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_load_registry_hive' + windows.BaseWindowsMorphingTools, '_compile_static_ip_conf_from_registry' ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_static_ip_script') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_static_nics_info') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') - def test_set_net_config(self, mock_uuid4, mock_load_registry_hive, - mock_unload_registry_hive, - mock_get_static_nics_info, - mock_write_static_ip_script, - mock_compile_static_ip_conf_from_registry): + def test_set_net_config( + self, + mock_uuid4, + mock_load_registry_hive, + mock_unload_registry_hive, + mock_get_static_nics_info, + mock_write_static_ip_script, + mock_compile_static_ip_conf_from_registry, + ): dhcp = False - nics_info = [ - {'ip_addresses': ["10.1.10.10"]} - ] - ips_info = [ - {'ip_address': "10.1.10.10"} - ] + nics_info = [{'ip_addresses': ["10.1.10.10"]}] + ips_info = [{'ip_address': "10.1.10.10"}] mock_compile_static_ip_conf_from_registry.return_value = ips_info self.morphing_tools.set_net_config(nics_info, dhcp=dhcp) mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir, + ) mock_compile_static_ip_conf_from_registry.assert_called_once_with( - str(mock_uuid4.return_value)) + str(mock_uuid4.return_value) + ) mock_get_static_nics_info.assert_called_once_with(nics_info, ips_info) mock_write_static_ip_script.assert_called_once_with( "C:\\Cloudbase-Init", mock_get_static_nics_info.return_value, - mock_compile_static_ip_conf_from_registry.return_value) + mock_compile_static_ip_conf_from_registry.return_value, + ) mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) @mock.patch.object( - windows.BaseWindowsMorphingTools, - '_compile_static_ip_conf_from_registry' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_write_static_ip_script' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_static_nics_info') - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_load_registry_hive' + windows.BaseWindowsMorphingTools, '_compile_static_ip_conf_from_registry' ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_static_ip_script') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_static_nics_info') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') @mock.patch.object(windows.uuid, 'uuid4') def test_set_net_config_no_static_info( - self, mock_uuid4, mock_load_registry_hive, - mock_unload_registry_hive, mock_get_static_nics_info, - mock_write_static_ip_script, - mock_compile_static_ip_conf_from_registry): + self, + mock_uuid4, + mock_load_registry_hive, + mock_unload_registry_hive, + mock_get_static_nics_info, + mock_write_static_ip_script, + mock_compile_static_ip_conf_from_registry, + ): dhcp = False - nics_info = [ - {'ip_addresses': ["10.1.10.10"]} - ] - ips_info = [ - {'ip_address': "10.1.10.10"} - ] + nics_info = [{'ip_addresses': ["10.1.10.10"]}] + ips_info = [{'ip_address': "10.1.10.10"}] mock_compile_static_ip_conf_from_registry.return_value = ips_info mock_get_static_nics_info.return_value = [] @@ -904,41 +991,37 @@ def test_set_net_config_no_static_info( mock_load_registry_hive.assert_called_once_with( "HKLM\\%s" % mock_uuid4.return_value, - "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir) + "%sWindows\\System32\\config\\SYSTEM" % self.os_root_dir, + ) mock_compile_static_ip_conf_from_registry.assert_called_once_with( - str(mock_uuid4.return_value)) + str(mock_uuid4.return_value) + ) mock_get_static_nics_info.assert_called_once_with(nics_info, ips_info) mock_write_static_ip_script.assert_not_called() mock_unload_registry_hive.assert_called_once_with( - "HKLM\\%s" % mock_uuid4.return_value) + "HKLM\\%s" % mock_uuid4.return_value + ) + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_cbslinit_base_dir') @mock.patch.object( - windows.BaseWindowsMorphingTools, '_get_cbslinit_base_dir' + windows.BaseWindowsMorphingTools, '_compile_static_ip_conf_from_registry' ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, - '_compile_static_ip_conf_from_registry' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_write_static_ip_script' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_unload_registry_hive' - ) - @mock.patch.object( - windows.BaseWindowsMorphingTools, '_load_registry_hive' - ) - @mock.patch.object(windows.BaseWindowsMorphingTools, - '_get_static_nics_info') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_static_ip_script') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_unload_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_load_registry_hive') + @mock.patch.object(windows.BaseWindowsMorphingTools, '_get_static_nics_info') def test_set_net_config_with_dhcp( - self, mock_check_ips_info, mock_load_registry_hive, - mock_unload_registry_hive, mock_write_static_ip_script, - mock_get_cbslinit_base_dir, - mock_compile_static_ip_conf_from_registry): + self, + mock_check_ips_info, + mock_load_registry_hive, + mock_unload_registry_hive, + mock_write_static_ip_script, + mock_get_cbslinit_base_dir, + mock_compile_static_ip_conf_from_registry, + ): dhcp = True - result = self.morphing_tools.set_net_config( - mock.sentinel.nics_info, dhcp=dhcp) + result = self.morphing_tools.set_net_config(mock.sentinel.nics_info, dhcp=dhcp) mock_load_registry_hive.assert_not_called() mock_get_cbslinit_base_dir.assert_not_called() diff --git a/coriolis/tests/providers/base.py b/coriolis/tests/providers/base.py index 26c3cbd0..380e22b4 100644 --- a/coriolis/tests/providers/base.py +++ b/coriolis/tests/providers/base.py @@ -2,12 +2,10 @@ from unittest import mock -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class ProvidersBaseTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(ProvidersBaseTestCase, self).setUp() @@ -18,7 +16,8 @@ def setUp(self): self._mock_morph = mock.MagicMock() osmorphing_patcher = mock.patch( - 'coriolis.osmorphing.manager.morph_image', new=self._mock_morph) + 'coriolis.osmorphing.manager.morph_image', new=self._mock_morph + ) osmorphing_patcher.start() # NOTE: declare utils.retry_on_error mock; '_patch_utils_retry' should @@ -27,6 +26,7 @@ def setUp(self): def retry_dec(*args, **kwargs): return self._mock_utils_retry_inner + self._mock_utils_retry = mock.MagicMock(side_effect=retry_dec) # NOTE: declare various utils disk helpers mocks; @@ -38,19 +38,20 @@ def retry_dec(*args, **kwargs): def _patch_utils_retry(self): retry_patcher = mock.patch( - 'coriolis.utils.retry_on_error', self._mock_utils_retry) + 'coriolis.utils.retry_on_error', self._mock_utils_retry + ) retry_patcher.start() def _patch_utils_disk_functions(self): utils_patcher = mock.patch.multiple( 'coriolis.utils', get_disk_info=self._mock_utils_get_disk_info, - convert_disk_format=self._mock_utils_convert_disk) + convert_disk_format=self._mock_utils_convert_disk, + ) utils_patcher.start() class ImportProviderTestCase(ProvidersBaseTestCase): - @property def _platform(self): raise NotImplementedError("Missing platform type.") @@ -65,17 +66,19 @@ def setUp(self): self._test_target_env = mock.sentinel.target_environment self._test_export_info = mock.sentinel.export_info - def _test_morphing_called(self, os_type="", nics_info=None, - ignore_devs=[]): + def _test_morphing_called(self, os_type="", nics_info=None, ignore_devs=[]): self._mock_morph.morph_image.assert_called_once_with( - self._mock_conn_info, os_type, - self._hypervisor, self._platform, - nics_info, self._mock_event_manager, - ignore_devices=ignore_devs) + self._mock_conn_info, + os_type, + self._hypervisor, + self._platform, + nics_info, + self._mock_event_manager, + ignore_devices=ignore_devs, + ) class ExportProviderTestCase(ProvidersBaseTestCase): - def setUp(self): super(ExportProviderTestCase, self).setUp() diff --git a/coriolis/tests/providers/test_api.py b/coriolis/tests/providers/test_api.py index 0e2dc1c2..1f03f30c 100644 --- a/coriolis/tests/providers/test_api.py +++ b/coriolis/tests/providers/test_api.py @@ -20,16 +20,15 @@ def setUp(self): def test_get_available_providers(self): result = self.api.get_available_providers(self.ctxt) - self.rpc_client.get_available_providers.assert_called_once_with( - self.ctxt) - self.assertEqual(result, - self.rpc_client.get_available_providers.return_value) + self.rpc_client.get_available_providers.assert_called_once_with(self.ctxt) + self.assertEqual(result, self.rpc_client.get_available_providers.return_value) def test_get_provider_schemas(self): result = self.api.get_provider_schemas( - self.ctxt, mock.sentinel.platform_name, '1') + self.ctxt, mock.sentinel.platform_name, '1' + ) self.rpc_client.get_provider_schemas.assert_called_once_with( - self.ctxt, mock.sentinel.platform_name, 1) - self.assertEqual( - result, self.rpc_client.get_provider_schemas.return_value) + self.ctxt, mock.sentinel.platform_name, 1 + ) + self.assertEqual(result, self.rpc_client.get_provider_schemas.return_value) diff --git a/coriolis/tests/providers/test_backup_writers.py b/coriolis/tests/providers/test_backup_writers.py index 7ac97c09..a20bb656 100644 --- a/coriolis/tests/providers/test_backup_writers.py +++ b/coriolis/tests/providers/test_backup_writers.py @@ -8,10 +8,8 @@ from unittest import mock from coriolis import exception -from coriolis.providers import backup_writers -from coriolis.providers import provider_utils -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.providers import backup_writers, provider_utils +from coriolis.tests import test_base, testutils class CoriolisTestException(Exception): @@ -27,8 +25,7 @@ def setUp(self): @mock.patch('coriolis.utils.test_ssh_path') @mock.patch('coriolis.utils.exec_ssh_cmd') - def test__disable_lvm2_lvmetad(self, mock_exec_ssh_cmd, - mock_test_ssh_path): + def test__disable_lvm2_lvmetad(self, mock_exec_ssh_cmd, mock_test_ssh_path): cfg = "/etc/lvm/lvm.conf" mock_test_ssh_path.return_value = True @@ -38,32 +35,36 @@ def test__disable_lvm2_lvmetad(self, mock_exec_ssh_cmd, expected_calls = [ mock.call( self.mock_ssh, - 'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" %s' % - cfg, get_pty=True), - mock.call(self.mock_ssh, - 'sudo service lvm2-lvmetad stop', get_pty=True), - mock.call(self.mock_ssh, 'sudo vgchange -an', get_pty=True)] + 'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" %s' % cfg, + get_pty=True, + ), + mock.call(self.mock_ssh, 'sudo service lvm2-lvmetad stop', get_pty=True), + mock.call(self.mock_ssh, 'sudo vgchange -an', get_pty=True), + ] mock_exec_ssh_cmd.assert_has_calls(expected_calls) @mock.patch('coriolis.utils.test_ssh_path') @mock.patch('coriolis.utils.exec_ssh_cmd') - def test__disable_lvm_metad_udev_rule(self, mock_exec_ssh_cmd, - mock_test_ssh_path): - rule_path = ["/lib/udev/rules.d/69-lvm-metad.rules", - "/lib/udev/rules.d/69-dm-lvm.rules"] + def test__disable_lvm_metad_udev_rule(self, mock_exec_ssh_cmd, mock_test_ssh_path): + rule_path = [ + "/lib/udev/rules.d/69-lvm-metad.rules", + "/lib/udev/rules.d/69-dm-lvm.rules", + ] mock_test_ssh_path.return_value = True backup_writers._disable_lvm_metad_udev_rule(self.mock_ssh) mock_test_ssh_path.assert_has_calls( - [mock.call(self.mock_ssh, rule_path[0]), - mock.call(self.mock_ssh, rule_path[1])]) + [ + mock.call(self.mock_ssh, rule_path[0]), + mock.call(self.mock_ssh, rule_path[1]), + ] + ) expected_calls = [ - mock.call(self.mock_ssh, 'sudo rm %s' % rule_path[0], - get_pty=True), - mock.call(self.mock_ssh, 'sudo rm %s' % rule_path[1], - get_pty=True)] + mock.call(self.mock_ssh, 'sudo rm %s' % rule_path[0], get_pty=True), + mock.call(self.mock_ssh, 'sudo rm %s' % rule_path[1], get_pty=True), + ] mock_exec_ssh_cmd.assert_has_calls(expected_calls) def test__check_deserialize_key(self): @@ -74,25 +75,26 @@ def test__check_deserialize_key(self): self.assertEqual(result, mock_rsa_key) @mock.patch('coriolis.utils.deserialize_key') - def test__check_deserialize_key_with_pem_string( - self, mock_deserialize_key): + def test__check_deserialize_key_with_pem_string(self, mock_deserialize_key): mock_key = 'mock_key' - mock_deserialized_key = backup_writers.paramiko.RSAKey.generate( - bits=2048) + mock_deserialized_key = backup_writers.paramiko.RSAKey.generate(bits=2048) mock_deserialize_key.return_value = mock_deserialized_key backup_writers._check_deserialize_key(mock_key) mock_deserialize_key.assert_called_once_with( - mock_key, backup_writers.CONF.serialization.temp_keypair_password) + mock_key, backup_writers.CONF.serialization.temp_keypair_password + ) @mock.patch('coriolis.utils.deserialize_key') def test__check_deserialize_key_with_exception(self, mock_deserialize_key): mock_deserialize_key.side_effect = exception.CoriolisException() - self.assertRaises(exception.CoriolisException, - backup_writers._check_deserialize_key, - mock.sentinel.key) + self.assertRaises( + exception.CoriolisException, + backup_writers._check_deserialize_key, + mock.sentinel.key, + ) class BackupWritersFactoryTestCase(test_base.CoriolisBaseTestCase): @@ -112,7 +114,8 @@ def test_get_writer_with_ssh(self, mock_ssh_backup_writer): result = factory.get_writer() mock_ssh_backup_writer.assert_called_with( - factory._conn_info, factory._volumes_info) + factory._conn_info, factory._volumes_info + ) self.assertEqual(result, mock_ssh_backup_writer.return_value) @@ -122,7 +125,8 @@ def test_get_writer_with_http(self, mock_http_backup_writer): result = factory.get_writer() mock_http_backup_writer.assert_called_with( - factory._conn_info, factory._volumes_info) + factory._conn_info, factory._volumes_info + ) self.assertEqual(result, mock_http_backup_writer.return_value) @@ -132,52 +136,55 @@ def test_get_writer_with_file(self, mock_file_backup_writer): result = factory.get_writer() mock_file_backup_writer.assert_called_with( - factory._conn_info, factory._volumes_info) + factory._conn_info, factory._volumes_info + ) self.assertEqual(result, mock_file_backup_writer.return_value) def test_get_writer_with_exception(self): - with mock.patch.object(backup_writers.BackupWritersFactory, - '_validate_info') as mock_validate_info: + with mock.patch.object( + backup_writers.BackupWritersFactory, '_validate_info' + ) as mock_validate_info: mock_validate_info.return_value = None factory = self._get_writer("invalid backup writer type") self.assertRaises(exception.CoriolisException, factory.get_writer) def test__validate_info_with_non_dict(self): - self.assertRaises(exception.CoriolisException, - self._get_factory, "not a dict") + self.assertRaises(exception.CoriolisException, self._get_factory, "not a dict") def test__validate_info_with_missing_backend(self): - self.assertRaises(exception.CoriolisException, - self._get_factory, {"connection_details": {}}) + self.assertRaises( + exception.CoriolisException, self._get_factory, {"connection_details": {}} + ) def test__validate_info_with_invalid_backend(self): - self.assertRaises(exception.CoriolisException, - self._get_factory, - {"backend": "invalid", "connection_details": {}}) + self.assertRaises( + exception.CoriolisException, + self._get_factory, + {"backend": "invalid", "connection_details": {}}, + ) def test__validate_info_with_missing_connection_details(self): backup_writers.BackupWritersFactory._conn_info = None - self.assertRaises(exception.CoriolisException, - self._get_factory, {"backend": "ssh"}) + self.assertRaises( + exception.CoriolisException, self._get_factory, {"backend": "ssh"} + ) class BaseBackupWriterTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis BaseBackupWriter class.""" - @mock.patch.object( - backup_writers.BaseBackupWriter, '__abstractmethods__', set() - ) + @mock.patch.object(backup_writers.BaseBackupWriter, '__abstractmethods__', set()) def setUp(self): super(BaseBackupWriterTestCase, self).setUp() self.writer = backup_writers.BaseBackupWriter() @mock.patch.object(backup_writers.BaseBackupWriter, '_get_impl') def test_open(self, mock_get_impl): - with self.writer.open( - mock.sentinel.path, mock.sentinel.disk_id) as impl: + with self.writer.open(mock.sentinel.path, mock.sentinel.disk_id) as impl: mock_get_impl.assert_called_once_with( - mock.sentinel.path, mock.sentinel.disk_id) + mock.sentinel.path, mock.sentinel.disk_id + ) impl._open.assert_called_once() impl.close.assert_called_once() @@ -204,7 +211,8 @@ class FileBackupWriterImplTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(FileBackupWriterImplTestCase, self).setUp() self.writer = backup_writers.FileBackupWriterImpl( - mock.sentinel.path, mock.sentinel.disk_id) + mock.sentinel.path, mock.sentinel.disk_id + ) self.mock_file = mock.MagicMock() self.writer._file = self.mock_file @@ -212,9 +220,11 @@ def setUp(self): def test__open(self, mock_open): self.writer._open() - calls = [mock.call(self.writer._path, 'ab+'), - mock.call().close(), - mock.call(self.writer._path, 'rb+')] + calls = [ + mock.call(self.writer._path, 'ab+'), + mock.call().close(), + mock.call(self.writer._path, 'rb+'), + ] mock_open.assert_has_calls(calls) def test_seek(self): @@ -247,16 +257,17 @@ def setUp(self): @mock.patch.object(backup_writers, 'FileBackupWriterImpl') def test__get_impl(self, mock_file_backup_writer_impl): - result = self.writer._get_impl( - mock.sentinel.path, mock.sentinel.disk_id) + result = self.writer._get_impl(mock.sentinel.path, mock.sentinel.disk_id) mock_file_backup_writer_impl.assert_called_once_with( - mock.sentinel.path, mock.sentinel.disk_id) + mock.sentinel.path, mock.sentinel.disk_id + ) self.assertEqual(result, mock_file_backup_writer_impl.return_value) def test_from_connection_info(self): result = backup_writers.FileBackupWriter.from_connection_info( - mock.sentinel.connection_info, mock.sentinel.volume_info) + mock.sentinel.connection_info, mock.sentinel.volume_info + ) self.assertIsInstance(result, backup_writers.FileBackupWriter) @@ -267,7 +278,8 @@ class SSHBackupWriterImplTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(SSHBackupWriterImplTestCase, self).setUp() self.writer = backup_writers.SSHBackupWriterImpl( - mock.sentinel.path, mock.sentinel.path) + mock.sentinel.path, mock.sentinel.path + ) self._ssh = mock.MagicMock() self._stdin = mock.MagicMock() self._stdout = mock.MagicMock() @@ -283,25 +295,29 @@ def test__set_ssh_client(self): self.assertEqual(self.writer._ssh, self._ssh) def test__exec_helper_cmd(self): - self._ssh.exec_command.return_value = ( - self._stdin, self._stdout, self._stderr) + self._ssh.exec_command.return_value = (self._stdin, self._stdout, self._stderr) original_exec_helper_cmd = testutils.get_wrapped_function( - self.writer._exec_helper_cmd) + self.writer._exec_helper_cmd + ) original_exec_helper_cmd(self.writer) self._ssh.exec_command.assert_called_once_with( - "chmod +x write_data && sudo ./write_data") + "chmod +x write_data && sudo ./write_data" + ) @mock.patch('coriolis.data_transfer.encode_data') def test__encode_data(self, mock_encode_data): - result = self.writer._encode_data( - 'test_content', mock.sentinel.offset, 456) + result = self.writer._encode_data('test_content', mock.sentinel.offset, 456) mock_encode_data.assert_called_once_with( - 456, mock.sentinel.path, mock.sentinel.offset, 'test_content', - compress=self.writer._compress_transfer) + 456, + mock.sentinel.path, + mock.sentinel.offset, + 'test_content', + compress=self.writer._compress_transfer, + ) self.assertEqual(result, mock_encode_data.return_value) @@ -317,8 +333,7 @@ def test__encode_eod(self, mock_encode_eod): def test__send_msg(self): self._stdout.channel.exit_status_ready.return_value = False - original_send_msg = testutils.get_wrapped_function( - self.writer._send_msg) + original_send_msg = testutils.get_wrapped_function(self.writer._send_msg) original_send_msg(self.writer, mock.sentinel.data) @@ -331,24 +346,33 @@ def test__send_msg_with_exception(self): self._stdout.channel.exit_status_ready.return_value = True self._stdout.channel.recv_exit_status.return_value = 1 - original_send_msg = testutils.get_wrapped_function( - self.writer._send_msg) + original_send_msg = testutils.get_wrapped_function(self.writer._send_msg) - self.assertRaises(exception.CoriolisException, original_send_msg, - self.writer, mock.sentinel.data) + self.assertRaises( + exception.CoriolisException, + original_send_msg, + self.writer, + mock.sentinel.data, + ) @mock.patch('coriolis.utils.start_thread') @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_encoder') @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_sender') @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_exec_helper_cmd') - def test__open(self, mock_exec_helper_cmd, mock_sender, mock_encoder, - mock_start_thread): + def test__open( + self, mock_exec_helper_cmd, mock_sender, mock_encoder, mock_start_thread + ): self.writer._open() mock_exec_helper_cmd.assert_called_once() mock_start_thread.assert_has_calls( - [mock.call(mock_sender), mock.call(mock_encoder), - mock.call(mock_encoder), mock.call(mock_encoder)]) + [ + mock.call(mock_sender), + mock.call(mock_encoder), + mock.call(mock_encoder), + mock.call(mock_encoder), + ] + ) self.assertEqual(len(self.writer._encoder_threads), 3) @@ -358,14 +382,12 @@ def test_seek(self): @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_send_msg') def test__sender(self, mock_send_msg): - mock_send_msg.side_effect = [ - mock.sentinel.data, CoriolisTestException()] + mock_send_msg.side_effect = [mock.sentinel.data, CoriolisTestException()] self.writer._sender_q = self._sender_queue self.assertRaises(CoriolisTestException, self.writer._sender) - mock_send_msg.assert_called_with( - self.writer._sender_q.get.return_value) + mock_send_msg.assert_called_with(self.writer._sender_q.get.return_value) self.assertEqual(self._sender_queue.task_done.call_count, 2) @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_encode_data') @@ -374,23 +396,28 @@ def test__encoder(self, mock_encode_data): self.writer._sender_q = self._sender_queue mock_encode_data.side_effect = [ - {"data": mock.sentinel.data, - "offset": mock.sentinel.offset, - "msg_id": mock.sentinel.msg_id}, - CoriolisTestException()] + { + "data": mock.sentinel.data, + "offset": mock.sentinel.offset, + "msg_id": mock.sentinel.msg_id, + }, + CoriolisTestException(), + ] self.assertRaises(CoriolisTestException, self.writer._encoder) mock_encode_data.assert_called_with( self.writer._enc_q.get.return_value["data"], self.writer._enc_q.get.return_value["offset"], - self.writer._enc_q.get.return_value["msg_id"] + self.writer._enc_q.get.return_value["msg_id"], ) self._sender_queue.put.assert_called_once_with( - {"data": mock.sentinel.data, - "offset": mock.sentinel.offset, - "msg_id": mock.sentinel.msg_id} + { + "data": mock.sentinel.data, + "offset": mock.sentinel.offset, + "msg_id": mock.sentinel.msg_id, + } ) self.assertEqual(self._enc_queue.task_done.call_count, 2) @@ -404,24 +431,22 @@ def test_write(self): self.writer.write('test_data') - expected_payload = { - "offset": 0, - "data": 'test_data', - "msg_id": 0 - } + expected_payload = {"offset": 0, "data": 'test_data', "msg_id": 0} self._enc_queue.put.assert_called_once_with(expected_payload) self.assertEqual(self.writer._offset, len('test_data')) self.assertEqual(self.writer._msg_id, 1) def test_write_with_closing(self): self.writer._closing = True - self.assertRaises(exception.CoriolisException, self.writer.write, - mock.sentinel.data) + self.assertRaises( + exception.CoriolisException, self.writer.write, mock.sentinel.data + ) def test_write_with_exception(self): self.writer._exception = exception.CoriolisException() - self.assertRaises(exception.CoriolisException, self.writer.write, - mock.sentinel.data) + self.assertRaises( + exception.CoriolisException, self.writer.write, mock.sentinel.data + ) def test__wait_for_queues(self): self.writer._enc_q = mock.MagicMock() @@ -434,12 +459,10 @@ def test__wait_for_queues(self): start_time = datetime.datetime.now() timeout_time = start_time + datetime.timedelta(seconds=600) - with mock.patch("datetime.datetime") as \ - mock_datetime, mock.patch("time.sleep"): + with mock.patch("datetime.datetime") as mock_datetime, mock.patch("time.sleep"): mock_datetime.now.side_effect = [start_time, timeout_time] - self.assertRaises(exception.CoriolisException, - self.writer._wait_for_queues) + self.assertRaises(exception.CoriolisException, self.writer._wait_for_queues) def test__wait_for_queues_no_unfinished_tasks(self): self.writer._enc_q = mock.MagicMock() @@ -460,8 +483,7 @@ def test_close_with_exception(self, mock_wait_for_queues): @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_wait_for_queues') @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_send_msg') @mock.patch.object(backup_writers.SSHBackupWriterImpl, '_encode_eod') - def test_close_with_ssh(self, mock_encode_eod, mock_send_msg, - mock_wait_for_queues): + def test_close_with_ssh(self, mock_encode_eod, mock_send_msg, mock_wait_for_queues): self.writer._ssh = self._ssh mock_wait_for_queues.return_value = None self.writer._exception = None @@ -497,8 +519,9 @@ def test__handle_exception_with_exit_status(self): self.writer._stdout.channel.recv_exit_status.return_value = 1 self.writer._ssh = self._ssh - self.assertRaises(exception.CoriolisException, - self.writer._handle_exception, Exception()) + self.assertRaises( + exception.CoriolisException, self.writer._handle_exception, Exception() + ) self._ssh.close.assert_called_once() self.assertIsNone(self.writer._ssh) @@ -507,8 +530,9 @@ def test__handle_exception_without_exit_status(self): self.writer._stdout.channel.exit_status_ready.return_value = False self.writer._ssh = self._ssh - self.assertRaises(exception.CoriolisException, - self.writer._handle_exception, Exception()) + self.assertRaises( + exception.CoriolisException, self.writer._handle_exception, Exception() + ) self._ssh.close.assert_called_once() self.assertIsNone(self.writer._ssh) @@ -538,15 +562,21 @@ def setUp(self): "password": mock.sentinel.password, } self.writer = backup_writers.SSHBackupWriter( - self.ip, self.port, mock.sentinel.username, self.pkey, - mock.sentinel.password, self.volume_info) + self.ip, + self.port, + mock.sentinel.username, + self.pkey, + mock.sentinel.password, + self.volume_info, + ) @mock.patch.object(backup_writers, '_check_deserialize_key') def test_from_connection_info(self, mock_deserialize_key): self.conn_info["pkey"] = mock.sentinel.pkey result = backup_writers.SSHBackupWriter.from_connection_info( - self.conn_info, mock.sentinel.volume_info) + self.conn_info, mock.sentinel.volume_info + ) mock_deserialize_key.assert_called_once_with(mock.sentinel.pkey) @@ -556,39 +586,53 @@ def test_from_connection_info_missing_required_field(self): # Remove the IP from the connection info to test the missing required. self.conn_info.pop("ip") - self.assertRaises(exception.CoriolisException, - backup_writers.SSHBackupWriter.from_connection_info, - self.conn_info, self.volume_info) + self.assertRaises( + exception.CoriolisException, + backup_writers.SSHBackupWriter.from_connection_info, + self.conn_info, + self.volume_info, + ) def test_from_connection_info_missing_password(self): self.conn_info.pop("password") - self.assertRaises(exception.CoriolisException, - backup_writers.SSHBackupWriter.from_connection_info, - self.conn_info, self.volume_info) + self.assertRaises( + exception.CoriolisException, + backup_writers.SSHBackupWriter.from_connection_info, + self.conn_info, + self.volume_info, + ) @mock.patch.object(backup_writers.SSHBackupWriter, '_connect_ssh') @mock.patch.object(backup_writers, '_disable_lvm_metad_udev_rule') @mock.patch.object(backup_writers, '_disable_lvm2_lvmetad') @mock.patch.object(backup_writers, 'SSHBackupWriterImpl') @mock.patch.object(backup_writers.SSHBackupWriter, '_copy_helper_cmd') - def test__get_impl(self, mock_copy_helper_cmd, mock_ssh_backup_writer_impl, - mock_disable_lvm2_lvmetad, - mock_disable_lvm_metad_udev_rule, mock_connect_ssh): + def test__get_impl( + self, + mock_copy_helper_cmd, + mock_ssh_backup_writer_impl, + mock_disable_lvm2_lvmetad, + mock_disable_lvm_metad_udev_rule, + mock_connect_ssh, + ): mock_connect_ssh.return_value = self._ssh self.writer._volumes_info = [self.volume_info] result = self.writer._get_impl( - self.volume_info["path"], self.volume_info["disk_id"]) + self.volume_info["path"], self.volume_info["disk_id"] + ) mock_connect_ssh.assert_called_once_with() mock_disable_lvm_metad_udev_rule.assert_called_once_with(self._ssh) mock_disable_lvm2_lvmetad.assert_called_once_with(self._ssh) mock_copy_helper_cmd.assert_called_once_with(self._ssh) - mock_ssh_backup_writer_impl.return_value.\ - _set_ssh_client.assert_called_once_with(self._ssh) + mock_ssh_backup_writer_impl.return_value._set_ssh_client.assert_called_once_with( + self._ssh + ) mock_ssh_backup_writer_impl.assert_called_once_with( - mock.sentinel.volume_dev, mock.sentinel.disk_id) + mock.sentinel.volume_dev, mock.sentinel.disk_id + ) self.assertEqual(result, mock_ssh_backup_writer_impl.return_value) @@ -596,24 +640,38 @@ def test__get_impl(self, mock_copy_helper_cmd, mock_ssh_backup_writer_impl, @mock.patch.object(backup_writers, '_disable_lvm_metad_udev_rule') @mock.patch.object(backup_writers, '_disable_lvm2_lvmetad') def test__get_impl_no_matching_disk_id( - self, mock_disable_lvm2_lvmetad, mock_disable_lvm_metad_udev_rule, - mock_connect_ssh): + self, + mock_disable_lvm2_lvmetad, + mock_disable_lvm_metad_udev_rule, + mock_connect_ssh, + ): self.writer._volumes_info = [self.volume_info] - self.assertRaises(exception.CoriolisException, self.writer._get_impl, - mock.sentinel.path, "invalid_disk_id") + self.assertRaises( + exception.CoriolisException, + self.writer._get_impl, + mock.sentinel.path, + "invalid_disk_id", + ) @mock.patch.object(backup_writers.SSHBackupWriter, '_connect_ssh') @mock.patch.object(backup_writers, '_disable_lvm_metad_udev_rule') @mock.patch.object(backup_writers, '_disable_lvm2_lvmetad') def test__get_impl_multiple_matching_disks( - self, mock_disable_lvm2_lvmetad, mock_disable_lvm_metad_udev_rule, - mock_connect_ssh): + self, + mock_disable_lvm2_lvmetad, + mock_disable_lvm_metad_udev_rule, + mock_connect_ssh, + ): mock_connect_ssh.return_value = self._ssh self.writer._volumes_info = [self.volume_info, self.volume_info] - self.assertRaises(exception.CoriolisException, self.writer._get_impl, - mock.sentinel.path, mock.sentinel.disk_id) + self.assertRaises( + exception.CoriolisException, + self.writer._get_impl, + mock.sentinel.path, + mock.sentinel.disk_id, + ) @mock.patch('paramiko.SSHClient') def test__copy_helper_cmd(self, mock_ssh_client): @@ -621,7 +679,8 @@ def test__copy_helper_cmd(self, mock_ssh_client): mock_ssh.open_sftp.return_value = self._sftp original_copy_helper_cmd = testutils.get_wrapped_function( - self.writer._copy_helper_cmd) + self.writer._copy_helper_cmd + ) original_copy_helper_cmd(self.writer, mock_ssh) @@ -632,17 +691,19 @@ def test__copy_helper_cmd(self, mock_ssh_client): @mock.patch.object(backup_writers.utils, 'get_resources_bin_dir') @mock.patch('paramiko.SSHClient') def test__copy_helper_cmd_file_does_not_exist( - self, mock_ssh_client, mock_get_resources_bin_dir): + self, mock_ssh_client, mock_get_resources_bin_dir + ): mock_ssh = mock_ssh_client.return_value mock_ssh.open_sftp.return_value = self._sftp - local_path = os.path.join( - mock_get_resources_bin_dir.return_value, 'write_data') + local_path = os.path.join(mock_get_resources_bin_dir.return_value, 'write_data') self._sftp.stat.side_effect = IOError( - backup_writers.errno.ENOENT, 'No such file or directory') + backup_writers.errno.ENOENT, 'No such file or directory' + ) original_copy_helper_cmd = testutils.get_wrapped_function( - self.writer._copy_helper_cmd) + self.writer._copy_helper_cmd + ) original_copy_helper_cmd(self.writer, mock_ssh) @@ -656,10 +717,10 @@ def test__copy_helper_cmd_stat_error(self, mock_ssh_client): self._sftp.stat.side_effect = IOError() original_copy_helper_cmd = testutils.get_wrapped_function( - self.writer._copy_helper_cmd) + self.writer._copy_helper_cmd + ) - self.assertRaises( - IOError, original_copy_helper_cmd, self.writer, mock_ssh) + self.assertRaises(IOError, original_copy_helper_cmd, self.writer, mock_ssh) self._sftp.put.assert_not_called() self._sftp.close.assert_called_once() @@ -668,8 +729,7 @@ def test__copy_helper_cmd_stat_error(self, mock_ssh_client): def test__connect_ssh(self, mock_ssh_client): mock_ssh_client.return_value = self._ssh - original_connect_ssh = testutils.get_wrapped_function( - self.writer._connect_ssh) + original_connect_ssh = testutils.get_wrapped_function(self.writer._connect_ssh) result = original_connect_ssh(self.writer) @@ -679,7 +739,7 @@ def test__connect_ssh(self, mock_ssh_client): port=self.writer._port, username=self.writer._username, pkey=self.writer._pkey, - password=self.writer._password + password=self.writer._password, ) self.assertEqual(result, self._ssh) @@ -688,11 +748,9 @@ def test__connect_ssh_with_exception(self, mock_ssh_client): mock_ssh_client.return_value = self._ssh self._ssh.connect.side_effect = CoriolisTestException() - original_connect_ssh = testutils.get_wrapped_function( - self.writer._connect_ssh) + original_connect_ssh = testutils.get_wrapped_function(self.writer._connect_ssh) - self.assertRaises(CoriolisTestException, original_connect_ssh, - self.writer) + self.assertRaises(CoriolisTestException, original_connect_ssh, self.writer) self._ssh.close.assert_called_once() @@ -710,10 +768,9 @@ def setUp(self): "client_crt": mock.sentinel.client_crt, "client_key": mock.sentinel.client_key, "ca_crt": mock.sentinel.ca_crt, - "id": mock.sentinel.id + "id": mock.sentinel.id, } - self.writer = backup_writers.HTTPBackupWriterImpl( - self.path, self.disk_id) + self.writer = backup_writers.HTTPBackupWriterImpl(self.path, self.disk_id) def test__set_info(self): self.writer._set_info(self.info) @@ -725,8 +782,7 @@ def test__set_info(self): self.assertEqual(self.writer._id, self.info["id"]) def test__set_info_missing_info(self): - self.assertRaises(exception.CoriolisException, - self.writer._set_info, {}) + self.assertRaises(exception.CoriolisException, self.writer._set_info, {}) def test__uri(self): self.writer._ip = self.info["ip"] @@ -735,15 +791,19 @@ def test__uri(self): result = self.writer._uri - self.assertEqual(result, "https://%s:%s/api/v2/device/%s" % ( - self.writer._ip, self.writer._port, "cGF0aC90ZXN0X3BhdGg=")) + self.assertEqual( + result, + "https://%s:%s/api/v2/device/%s" + % (self.writer._ip, self.writer._port, "cGF0aC90ZXN0X3BhdGg="), + ) @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri') @mock.patch.object(backup_writers, 'CONF') @mock.patch('requests.Session') - def test__acquire(self, mock_session_class, mock_conf, mock_uri, - mock_ensure_session): + def test__acquire( + self, mock_session_class, mock_conf, mock_uri, mock_ensure_session + ): mock_response = mock.MagicMock() mock_response.status_code = 200 mock_response.content = 'OK' @@ -761,15 +821,17 @@ def test__acquire(self, mock_session_class, mock_conf, mock_uri, mock_session.post.assert_called_once_with( f"{mock_uri}/acquire", headers={"X-Client-Token": self.writer._id}, - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_response.raise_for_status.assert_called_once() @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri') @mock.patch.object(backup_writers, 'CONF') @mock.patch('requests.Session') - def test__release(self, mock_session_class, mock_conf, mock_uri, - mock_ensure_session): + def test__release( + self, mock_session_class, mock_conf, mock_uri, mock_ensure_session + ): mock_response = mock.Mock() mock_response.status_code = 200 mock_response.content = 'OK' @@ -787,7 +849,8 @@ def test__release(self, mock_session_class, mock_conf, mock_uri, mock_session.post.assert_called_once_with( f"{mock_uri}/release", headers={"X-Client-Token": self.writer._id}, - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_response.raise_for_status.assert_called_once() @mock.patch.object(provider_utils, 'ProviderSession') @@ -803,8 +866,9 @@ def test__init_session(self, mock_session_class): self.writer._session.close.assert_not_called() self.assertEqual(self.writer._session, mock_session_class.return_value) - self.assertEqual(self.writer._session.cert, - (self.writer._crt, self.writer._key)) + self.assertEqual( + self.writer._session.cert, (self.writer._crt, self.writer._key) + ) self.assertEqual(self.writer._session.verify, self.writer._ca) @mock.patch.object(provider_utils, 'ProviderSession') @@ -820,8 +884,9 @@ def test__init_session_exists(self, mock_session_class): mock_session_class.assert_called_once_with() self.assertEqual(self.writer._session, mock_session_class.return_value) - self.assertEqual(self.writer._session.cert, - (self.writer._crt, self.writer._key)) + self.assertEqual( + self.writer._session.cert, (self.writer._crt, self.writer._key) + ) self.assertEqual(self.writer._session.verify, self.writer._ca) @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_init_session') @@ -835,8 +900,8 @@ def test__open(self, mock_start_thread, mock_acquire, mock_init_session): mock_init_session.assert_called_once() mock_acquire.assert_called_once() mock_start_thread.assert_has_calls( - [mock.call(self.writer._sender), - mock.call(self.writer._compressor)]) + [mock.call(self.writer._sender), mock.call(self.writer._compressor)] + ) self.assertEqual(self.writer._compressor_count, 1) @@ -844,14 +909,15 @@ def test__open(self, mock_start_thread, mock_acquire, mock_init_session): @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_acquire') @mock.patch('coriolis.utils.start_thread') def test__open_compressor_count_not_none_or_zero( - self, mock_start_thread, mock_acquire, mock_init_session): + self, mock_start_thread, mock_acquire, mock_init_session + ): self.writer._compressor_count = 2 self.writer._open() self.assertEqual( - len(self.writer._compressor_threads), - self.writer._compressor_count) + len(self.writer._compressor_threads), self.writer._compressor_count + ) self.assertEqual(mock_start_thread.call_count, 3) def test_seek(self): @@ -881,22 +947,28 @@ def test__compressor(self, mock_compression_proxy): self.writer._comp_q.get.return_value = { "offset": mock.sentinel.offset, - "data": mock.sentinel.data} + "data": mock.sentinel.data, + } mock_compression_proxy.side_effect = [ (mock.sentinel.data, True), - CoriolisTestException()] + CoriolisTestException(), + ] self.assertRaises(CoriolisTestException, self.writer._compressor) self.assertEqual(self.writer._comp_q.get.call_count, 2) mock_compression_proxy.assert_called_with( self.writer._comp_q.get.return_value["data"], - backup_writers.constants.COMPRESSION_FORMAT_GZIP) + backup_writers.constants.COMPRESSION_FORMAT_GZIP, + ) self.writer._sender_q.put.assert_called_once_with( - {'encoding': 'gzip', - 'offset': mock.sentinel.offset, - 'chunk': mock.sentinel.data}) + { + 'encoding': 'gzip', + 'offset': mock.sentinel.offset, + 'chunk': mock.sentinel.data, + } + ) self.assertEqual(self.writer._comp_q.task_done.call_count, 2) @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') @@ -909,7 +981,7 @@ def test__sender(self, mock_conf, mock_uri, mock_ensure_session): "offset": mock.sentinel.offset, "data": mock.sentinel.data, "chunk": 'compressed_data', - "encoding": "gzip" + "encoding": "gzip", } mock_response = mock.MagicMock() @@ -918,8 +990,7 @@ def test__sender(self, mock_conf, mock_uri, mock_ensure_session): mock_response.raise_for_status.side_effect = [None, BaseException()] self.writer._session.post.return_value = mock_response - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.ERROR): + with self.assertLogs('coriolis.providers.backup_writers', level=logging.ERROR): self.assertRaises(BaseException, self.writer._sender) expected_headers = { @@ -932,7 +1003,8 @@ def test__sender(self, mock_conf, mock_uri, mock_ensure_session): mock_uri, headers=expected_headers, data='compressed_data', - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_ensure_session.assert_called() mock_response.raise_for_status.assert_called() @@ -944,8 +1016,9 @@ def test__sender(self, mock_conf, mock_uri, mock_ensure_session): @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri') @mock.patch.object(backup_writers, 'CONF') - def test__sender_with_exception(self, mock_conf, mock_uri, - mock_ensure_session, mock_sleep): + def test__sender_with_exception( + self, mock_conf, mock_uri, mock_ensure_session, mock_sleep + ): self.writer._session = mock.MagicMock() self.writer._sender_q = mock.MagicMock() mock_response = mock.MagicMock() @@ -958,8 +1031,9 @@ def test__sender_with_exception(self, mock_conf, mock_uri, "chunk": 'compressed_data', } - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.WARNING): + with self.assertLogs( + 'coriolis.providers.backup_writers', level=logging.WARNING + ): self.assertRaises(CoriolisTestException, self.writer._sender) expected_headers = { @@ -971,7 +1045,8 @@ def test__sender_with_exception(self, mock_conf, mock_uri, mock_uri, headers=expected_headers, data='compressed_data', - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_ensure_session.assert_called() self.writer._sender_q.task_done.assert_called_once() @@ -1001,16 +1076,18 @@ def test_write_with_closing(self): original_write = testutils.get_wrapped_function(self.writer.write) - self.assertRaises(exception.CoriolisException, original_write, - self.writer, mock.sentinel.data) + self.assertRaises( + exception.CoriolisException, original_write, self.writer, mock.sentinel.data + ) def test_write_with_exception(self): self.writer._exception = exception.CoriolisException() original_write = testutils.get_wrapped_function(self.writer.write) - self.assertRaises(exception.CoriolisException, original_write, - self.writer, mock.sentinel.data) + self.assertRaises( + exception.CoriolisException, original_write, self.writer, mock.sentinel.data + ) @mock.patch('time.sleep') def test__wait_for_queues(self, mock_sleep): @@ -1051,16 +1128,15 @@ def test_close_with_exception(self, mock_release, mock_wait_for_queues): self.writer._exception = exception.CoriolisException() mock_release.side_effect = Exception() - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.ERROR): + with self.assertLogs('coriolis.providers.backup_writers', level=logging.ERROR): self.assertRaises(exception.CoriolisException, self.writer.close) @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri', - new_callable=mock.PropertyMock) + @mock.patch.object( + backup_writers.HTTPBackupWriterImpl, '_uri', new_callable=mock.PropertyMock + ) @mock.patch.object(backup_writers, 'CONF') - def test__create_checksum_job(self, mock_conf, mock_uri, - mock_ensure_session): + def test__create_checksum_job(self, mock_conf, mock_uri, mock_ensure_session): self.writer._set_info(self.info) mock_uri.return_value = "https://host:port/api/v2/device/b64path" self.writer._session = mock.MagicMock() @@ -1074,17 +1150,17 @@ def test__create_checksum_job(self, mock_conf, mock_uri, self.writer._session.post.assert_called_once_with( "https://host:port/api/v2/device/b64path/checksumJob", headers={"X-Client-Token": self.info["id"]}, - json={"start_offset": 0, "end_offset": 0, - "checksum_algorithm": "sha256"}, - timeout=mock_conf.default_requests_timeout) + json={"start_offset": 0, "end_offset": 0, "checksum_algorithm": "sha256"}, + timeout=mock_conf.default_requests_timeout, + ) mock_resp.raise_for_status.assert_called_once() @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri', - new_callable=mock.PropertyMock) + @mock.patch.object( + backup_writers.HTTPBackupWriterImpl, '_uri', new_callable=mock.PropertyMock + ) @mock.patch.object(backup_writers, 'CONF') - def test__get_checksum_job_status(self, mock_conf, mock_uri, - mock_ensure_session): + def test__get_checksum_job_status(self, mock_conf, mock_uri, mock_ensure_session): self.writer._set_info(self.info) mock_uri.return_value = "https://host:port/api/v2/device/b64path" self.writer._session = mock.MagicMock() @@ -1096,15 +1172,16 @@ def test__get_checksum_job_status(self, mock_conf, mock_uri, self.assertEqual(result, mock_resp.json.return_value) self.writer._session.get.assert_called_once_with( "https://host:port/api/v2/device/b64path/checksumJob/test-job-id", - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_resp.raise_for_status.assert_called_once() @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_ensure_session') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_uri', - new_callable=mock.PropertyMock) + @mock.patch.object( + backup_writers.HTTPBackupWriterImpl, '_uri', new_callable=mock.PropertyMock + ) @mock.patch.object(backup_writers, 'CONF') - def test__delete_checksum_job(self, mock_conf, mock_uri, - mock_ensure_session): + def test__delete_checksum_job(self, mock_conf, mock_uri, mock_ensure_session): self.writer._set_info(self.info) mock_uri.return_value = "https://host:port/api/v2/device/b64path" self.writer._session = mock.MagicMock() @@ -1115,18 +1192,15 @@ def test__delete_checksum_job(self, mock_conf, mock_uri, self.writer._session.delete.assert_called_once_with( "https://host:port/api/v2/device/b64path/checksumJob/test-job-id", - timeout=mock_conf.default_requests_timeout) + timeout=mock_conf.default_requests_timeout, + ) mock_resp.raise_for_status.assert_called_once() - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_delete_checksum_job') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_get_checksum_job_status') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_create_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_delete_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_get_checksum_job_status') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_create_checksum_job') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_wait_for_queues') - def test_get_disk_checksum(self, mock_wait, mock_create, mock_status, - mock_delete): + def test_get_disk_checksum(self, mock_wait, mock_create, mock_status, mock_delete): self.writer._set_info(self.info) mock_create.return_value = "test-job-id" mock_status.return_value = { @@ -1150,15 +1224,13 @@ def test_get_disk_checksum(self, mock_wait, mock_create, mock_status, ) mock_delete.assert_called_once_with("test-job-id") - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_delete_checksum_job') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_get_checksum_job_status') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_create_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_delete_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_get_checksum_job_status') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_create_checksum_job') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_wait_for_queues') - def test_get_disk_checksum_job_failed(self, mock_wait, mock_create, - mock_status, mock_delete): + def test_get_disk_checksum_job_failed( + self, mock_wait, mock_create, mock_status, mock_delete + ): self.writer._set_info(self.info) mock_create.return_value = "test-job-id" mock_status.return_value = { @@ -1167,38 +1239,38 @@ def test_get_disk_checksum_job_failed(self, mock_wait, mock_create, } self.assertRaises( - exception.CoriolisException, - self.writer.get_disk_checksum, "sha256") + exception.CoriolisException, self.writer.get_disk_checksum, "sha256" + ) mock_delete.assert_called_once_with("test-job-id") - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_delete_checksum_job') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_create_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_delete_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_create_checksum_job') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_wait_for_queues') - def test_get_disk_checksum_write_error(self, mock_wait, mock_create, - mock_delete): + def test_get_disk_checksum_write_error(self, mock_wait, mock_create, mock_delete): self.writer._set_info(self.info) self.writer._exception = exception.CoriolisException("write failed") self.assertRaises( - exception.CoriolisException, - self.writer.get_disk_checksum, "sha256") + exception.CoriolisException, self.writer.get_disk_checksum, "sha256" + ) mock_create.assert_not_called() mock_delete.assert_not_called() @mock.patch('coriolis.providers.backup_writers.time.sleep') @mock.patch('coriolis.providers.backup_writers.time.monotonic') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_delete_checksum_job') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_get_checksum_job_status') - @mock.patch.object(backup_writers.HTTPBackupWriterImpl, - '_create_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_delete_checksum_job') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_get_checksum_job_status') + @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_create_checksum_job') @mock.patch.object(backup_writers.HTTPBackupWriterImpl, '_wait_for_queues') - def test_get_disk_checksum_timeout(self, mock_wait, mock_create, - mock_status, mock_delete, - mock_monotonic, mock_sleep): + def test_get_disk_checksum_timeout( + self, + mock_wait, + mock_create, + mock_status, + mock_delete, + mock_monotonic, + mock_sleep, + ): self.writer._set_info(self.info) mock_create.return_value = "test-job-id" mock_status.return_value = {"execution_status": "running"} @@ -1206,8 +1278,8 @@ def test_get_disk_checksum_timeout(self, mock_wait, mock_create, mock_monotonic.side_effect = [0, 3601] self.assertRaises( - exception.CoriolisException, - self.writer.get_disk_checksum, "sha256") + exception.CoriolisException, self.writer.get_disk_checksum, "sha256" + ) mock_delete.assert_called_once_with("test-job-id") @@ -1228,41 +1300,43 @@ def setUp(self, mock_ssh_client): self._ssh = mock_ssh_client.return_value self.bootstrapper = backup_writers.HTTPBackupWriterBootstrapper( - self.ssh_conn_info, self.writer_port) + self.ssh_conn_info, self.writer_port + ) @mock.patch.object(backup_writers, '_check_deserialize_key') - @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh' - ) - def test__init__missing_required_params(self, mock_connect_ssh, - mock_deserialize_key): + @mock.patch.object(backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh') + def test__init__missing_required_params( + self, mock_connect_ssh, mock_deserialize_key + ): # Remove the writer_port to test the missing required. self.ssh_conn_info.pop("ip") - self.assertRaises(exception.CoriolisException, - backup_writers.HTTPBackupWriterBootstrapper, - self.ssh_conn_info, self.writer_port) + self.assertRaises( + exception.CoriolisException, + backup_writers.HTTPBackupWriterBootstrapper, + self.ssh_conn_info, + self.writer_port, + ) @mock.patch.object(backup_writers, '_check_deserialize_key') - @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh' - ) - def test__init__missing_password(self, mock_connect_ssh, - mock_deserialize_key): + @mock.patch.object(backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh') + def test__init__missing_password(self, mock_connect_ssh, mock_deserialize_key): self.ssh_conn_info.pop("password") - self.assertRaises(exception.CoriolisException, - backup_writers.HTTPBackupWriterBootstrapper, - self.ssh_conn_info, self.writer_port) + self.assertRaises( + exception.CoriolisException, + backup_writers.HTTPBackupWriterBootstrapper, + self.ssh_conn_info, + self.writer_port, + ) @mock.patch.object(backup_writers, '_check_deserialize_key') - @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh' - ) + @mock.patch.object(backup_writers.HTTPBackupWriterBootstrapper, '_connect_ssh') def test__init__with_pkey(self, mock_connect_ssh, mock_deserialize_key): self.ssh_conn_info["pkey"] = mock.sentinel.pkey writer = backup_writers.HTTPBackupWriterBootstrapper( - self.ssh_conn_info, self.writer_port) + self.ssh_conn_info, self.writer_port + ) mock_deserialize_key.assert_called_once_with(mock.sentinel.pkey) self.assertEqual(writer._pkey, mock_deserialize_key.return_value) @@ -1272,7 +1346,8 @@ def test__connect_ssh(self, mock_ssh_client): mock_ssh = mock_ssh_client.return_value original_connect_ssh = testutils.get_wrapped_function( - self.bootstrapper._connect_ssh) + self.bootstrapper._connect_ssh + ) result = original_connect_ssh(self.bootstrapper) @@ -1282,7 +1357,7 @@ def test__connect_ssh(self, mock_ssh_client): port=self.bootstrapper._port, username=self.bootstrapper._username, pkey=self.bootstrapper._pkey, - password=self.bootstrapper._password + password=self.bootstrapper._password, ) self.assertEqual(result, mock_ssh) @@ -1292,7 +1367,8 @@ def test__connect_ssh_with_exception(self, mock_ssh_client): self._ssh.connect.side_effect = [None, CoriolisTestException()] bootstrapper = backup_writers.HTTPBackupWriterBootstrapper( - self.ssh_conn_info, self.writer_port) + self.ssh_conn_info, self.writer_port + ) original = testutils.get_wrapped_function(bootstrapper._connect_ssh) @@ -1308,16 +1384,16 @@ def test__inject_dport_allow_rule(self, mock_exec_ssh_cmd): self._ssh, "sudo nft insert rule ip filter INPUT tcp dport %(port)s counter " "accept || " - "sudo iptables -I INPUT -p tcp --dport %(port)s -j ACCEPT" % { - "port": self.writer_port}, - get_pty=True) + "sudo iptables -I INPUT -p tcp --dport %(port)s -j ACCEPT" + % {"port": self.writer_port}, + get_pty=True, + ) @mock.patch('coriolis.utils.exec_ssh_cmd') def test__inject_dport_allow_rule_with_exception(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.side_effect = exception.CoriolisException() - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.WARN): + with self.assertLogs('coriolis.providers.backup_writers', level=logging.WARN): self.bootstrapper._inject_dport_allow_rule(self._ssh) @mock.patch('coriolis.utils.exec_ssh_cmd') @@ -1326,15 +1402,15 @@ def test__add_firewalld_port(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.assert_called_once_with( self._ssh, - "sudo firewall-cmd --add-port=%s/tcp" % - self.writer_port, get_pty=True) + "sudo firewall-cmd --add-port=%s/tcp" % self.writer_port, + get_pty=True, + ) @mock.patch('coriolis.utils.exec_ssh_cmd') def test__add_firewalld_port_with_exception(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.side_effect = exception.CoriolisException() - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.WARN): + with self.assertLogs('coriolis.providers.backup_writers', level=logging.WARN): self.bootstrapper._add_firewalld_port(self._ssh) @mock.patch('coriolis.utils.exec_ssh_cmd') @@ -1342,15 +1418,14 @@ def test__change_binary_se_context(self, mock_exec_ssh_cmd): self.bootstrapper._change_binary_se_context(self._ssh) mock_exec_ssh_cmd.assert_called_once_with( - self._ssh, - 'sudo chcon -t bin_t /usr/bin/coriolis-writer', get_pty=True) + self._ssh, 'sudo chcon -t bin_t /usr/bin/coriolis-writer', get_pty=True + ) @mock.patch('coriolis.utils.exec_ssh_cmd') def test__change_binary_se_context_with_exception(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.side_effect = exception.CoriolisException() - with self.assertLogs('coriolis.providers.backup_writers', - level=logging.WARN): + with self.assertLogs('coriolis.providers.backup_writers', level=logging.WARN): self.bootstrapper._change_binary_se_context(self._ssh) @mock.patch('coriolis.utils.exec_ssh_cmd') @@ -1359,7 +1434,8 @@ def test__copy_writer_file(self, mock_exec_ssh_cmd): self._ssh.open_sftp.return_value = mock_sftp original_copy_writer_file = testutils.get_wrapped_function( - self.bootstrapper._copy_writer) + self.bootstrapper._copy_writer + ) original_copy_writer_file(self.bootstrapper, self._ssh) @@ -1370,34 +1446,42 @@ def test__copy_writer_file(self, mock_exec_ssh_cmd): @mock.patch.object(backup_writers.utils, 'get_resources_bin_dir') @mock.patch('coriolis.utils.exec_ssh_cmd') def test__copy_writer_file_does_not_exist( - self, mock_exec_ssh_cmd, mock_get_resources_bin_dir): + self, mock_exec_ssh_cmd, mock_get_resources_bin_dir + ): local_path = os.path.join( mock_get_resources_bin_dir.return_value, - backup_writers._CORIOLIS_HTTP_WRITER_CMD) + backup_writers._CORIOLIS_HTTP_WRITER_CMD, + ) - remote_tmp_path = os.path.join( - "/tmp", backup_writers._CORIOLIS_HTTP_WRITER_CMD) + remote_tmp_path = os.path.join("/tmp", backup_writers._CORIOLIS_HTTP_WRITER_CMD) mock_sftp = mock.MagicMock() self._ssh.open_sftp.return_value = mock_sftp mock_sftp.stat.side_effect = IOError( - backup_writers.errno.ENOENT, 'No such file or directory') + backup_writers.errno.ENOENT, 'No such file or directory' + ) original_copy_writer_file = testutils.get_wrapped_function( - self.bootstrapper._copy_writer) + self.bootstrapper._copy_writer + ) original_copy_writer_file(self.bootstrapper, self._ssh) mock_sftp.put.assert_called_once_with(local_path, remote_tmp_path) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call( - self._ssh, "sudo mv %s %s" % ( - remote_tmp_path, self.bootstrapper._writer_cmd), - get_pty=True - ), - mock.call( - self._ssh, "sudo chmod +x %s" % self.bootstrapper._writer_cmd, - get_pty=True)]) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self._ssh, + "sudo mv %s %s" % (remote_tmp_path, self.bootstrapper._writer_cmd), + get_pty=True, + ), + mock.call( + self._ssh, + "sudo chmod +x %s" % self.bootstrapper._writer_cmd, + get_pty=True, + ), + ] + ) mock_sftp.close.assert_called_once() @mock.patch('coriolis.utils.exec_ssh_cmd') @@ -1405,12 +1489,15 @@ def test__copy_writer_stat_error(self, mock_exec_ssh_cmd): mock_sftp = mock.MagicMock() self._ssh.open_sftp.return_value = mock_sftp mock_sftp.stat.side_effect = IOError( - backup_writers.errno.EACCES, 'Permission denied') + backup_writers.errno.EACCES, 'Permission denied' + ) original_copy_writer_file = testutils.get_wrapped_function( - self.bootstrapper._copy_writer) - self.assertRaises(IOError, original_copy_writer_file, - self.bootstrapper, self._ssh) + self.bootstrapper._copy_writer + ) + self.assertRaises( + IOError, original_copy_writer_file, self.bootstrapper, self._ssh + ) mock_sftp.put.assert_not_called() mock_exec_ssh_cmd.assert_not_called() @@ -1421,14 +1508,16 @@ def test__copy_writer_stat_error(self, mock_exec_ssh_cmd): def test__fetch_remote_file(self, mock_exec_ssh_cmd, mock_read_ssh_file): with mock.patch('builtins.open', mock.mock_open()) as data: self.bootstrapper._fetch_remote_file( - self._ssh, mock.sentinel.remote_file, mock.sentinel.local_file) + self._ssh, mock.sentinel.remote_file, mock.sentinel.local_file + ) data.assert_called_once_with(mock.sentinel.local_file, 'wb') mock_exec_ssh_cmd.assert_called_once_with( - self._ssh, "sudo chmod +r %s" % mock.sentinel.remote_file, - get_pty=True) + self._ssh, "sudo chmod +r %s" % mock.sentinel.remote_file, get_pty=True + ) data.return_value.write.assert_called_once_with( - mock_read_ssh_file.return_value) + mock_read_ssh_file.return_value + ) @mock.patch('coriolis.utils.test_ssh_path') @mock.patch('coriolis.utils.exec_ssh_cmd') @@ -1440,7 +1529,7 @@ def test__setup_certificates(self, mock_exec_ssh_cmd, mock_test_ssh_path): "srv_key": "/etc/coriolis-writer/srv-key.pem", "ca_crt": "/etc/coriolis-writer/ca-cert.pem", "client_crt": "/etc/coriolis-writer/client-cert.pem", - "client_key": "/etc/coriolis-writer/client-key.pem" + "client_key": "/etc/coriolis-writer/client-key.pem", } result = self.bootstrapper._setup_certificates(self._ssh) @@ -1451,37 +1540,38 @@ def test__setup_certificates(self, mock_exec_ssh_cmd, mock_test_ssh_path): @mock.patch('coriolis.utils.test_ssh_path') @mock.patch('coriolis.utils.exec_ssh_cmd') def test__setup_certificates_no_files_exist( - self, mock_exec_ssh_cmd, mock_test_ssh_path): + self, mock_exec_ssh_cmd, mock_test_ssh_path + ): mock_test_ssh_path.return_value = False self.bootstrapper._setup_certificates(self._ssh) mock_exec_ssh_cmd.assert_any_call( - self._ssh, "sudo mkdir -p /etc/coriolis-writer", get_pty=True) + self._ssh, "sudo mkdir -p /etc/coriolis-writer", get_pty=True + ) mock_exec_ssh_cmd.assert_any_call( self._ssh, "sudo %(writer_cmd)s generate-certificates -output-dir " - "%(cert_dir)s -certificate-hosts %(extra_hosts)s" % { + "%(cert_dir)s -certificate-hosts %(extra_hosts)s" + % { "writer_cmd": self.bootstrapper._writer_cmd, "cert_dir": "/etc/coriolis-writer", "extra_hosts": self.bootstrapper._ip, }, - get_pty=True) + get_pty=True, + ) @mock.patch('coriolis.utils.exec_ssh_cmd') def test__read_remote_file_sudo(self, mock_exec_ssh_cmd): - result = self.bootstrapper._read_remote_file_sudo( - mock.sentinel.remote_path) + result = self.bootstrapper._read_remote_file_sudo(mock.sentinel.remote_path) mock_exec_ssh_cmd.assert_called_once_with( - self._ssh, 'sudo cat "%s"' % mock.sentinel.remote_path, - get_pty=True) - self.assertEqual( - result, mock_exec_ssh_cmd.return_value) + self._ssh, 'sudo cat "%s"' % mock.sentinel.remote_path, get_pty=True + ) + self.assertEqual(result, mock_exec_ssh_cmd.return_value) @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, - '_change_binary_se_context' + backup_writers.HTTPBackupWriterBootstrapper, '_change_binary_se_context' ) @mock.patch.object( backup_writers.HTTPBackupWriterBootstrapper, '_inject_dport_allow_rule' @@ -1491,8 +1581,12 @@ def test__read_remote_file_sudo(self, mock_exec_ssh_cmd): ) @mock.patch('coriolis.utils.create_service') def test__init_writer( - self, mock_create_service, mock_add_firewalld_port, - mock_inject_dport_allow_rule, mock_change_binary_se_context): + self, + mock_create_service, + mock_add_firewalld_port, + mock_inject_dport_allow_rule, + mock_change_binary_se_context, + ): cert_paths = { "ca_crt": mock.sentinel.ca_crt, "srv_key": mock.sentinel.srv_key, @@ -1504,7 +1598,8 @@ def test__init_writer( mock_change_binary_se_context.assert_called_once_with(self._ssh) mock_create_service.assert_called_once_with( self._ssh, - "%s run -ca-cert %s -key %s -cert %s -listen-port %s" % ( + "%s run -ca-cert %s -key %s -cert %s -listen-port %s" + % ( self.bootstrapper._writer_cmd, cert_paths["ca_crt"], cert_paths["srv_key"], @@ -1520,21 +1615,22 @@ def test__init_writer( @mock.patch.object( backup_writers.HTTPBackupWriterBootstrapper, '_read_remote_file_sudo' ) - @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, '_init_writer' - ) + @mock.patch.object(backup_writers.HTTPBackupWriterBootstrapper, '_init_writer') @mock.patch.object( backup_writers.HTTPBackupWriterBootstrapper, '_setup_certificates' ) - @mock.patch.object( - backup_writers.HTTPBackupWriterBootstrapper, '_copy_writer' - ) + @mock.patch.object(backup_writers.HTTPBackupWriterBootstrapper, '_copy_writer') @mock.patch.object(backup_writers, '_disable_lvm2_lvmetad') @mock.patch.object(backup_writers, '_disable_lvm_metad_udev_rule') - def test_setup_writer(self, mock_disable_lvm_metad_udev, - mock_disable_lvm2_lvmetad, mock_copy_writer, - mock_setup_certificates, mock_init_writer, - mock_read_remote_file_sudo): + def test_setup_writer( + self, + mock_disable_lvm_metad_udev, + mock_disable_lvm2_lvmetad, + mock_copy_writer, + mock_setup_certificates, + mock_init_writer, + mock_read_remote_file_sudo, + ): cert_paths = { "client_crt": mock.sentinel.client_crt, "client_key": mock.sentinel.client_key, @@ -1557,7 +1653,7 @@ def test_setup_writer(self, mock_disable_lvm_metad_udev, "client_crt": mock_read_remote_file_sudo.return_value, "client_key": mock_read_remote_file_sudo.return_value, "ca_crt": mock_read_remote_file_sudo.return_value, - } + }, } self.assertEqual(result, expected_result) @@ -1569,9 +1665,9 @@ def setUp(self): super(HTTPBackupWriterTestCase, self).setUp() self.ip = mock.sentinel.ip self.port = mock.sentinel.port - self.volumes_info = [{ - "disk_id": mock.sentinel.disk_id, - "volume_dev": mock.sentinel.volume_dev}] + self.volumes_info = [ + {"disk_id": mock.sentinel.disk_id, "volume_dev": mock.sentinel.volume_dev} + ] self.certificates = { "client_crt": mock.sentinel.client_crt, "client_key": mock.sentinel.client_key, @@ -1579,17 +1675,23 @@ def setUp(self): } self.writer = backup_writers.HTTPBackupWriter( - self.ip, self.port, self.volumes_info, self.certificates) + self.ip, self.port, self.volumes_info, self.certificates + ) def test__init__no_certificates(self): self.assertRaises( - exception.CoriolisException, backup_writers.HTTPBackupWriter, - self.ip, self.port, mock.sentinel.volumes_info, None) + exception.CoriolisException, + backup_writers.HTTPBackupWriter, + self.ip, + self.port, + mock.sentinel.volumes_info, + None, + ) def test_from_connection_info(self): - volumes_info = [{ - "disk_id": mock.sentinel.disk_id, - "volume_dev": mock.sentinel.volume_dev}] + volumes_info = [ + {"disk_id": mock.sentinel.disk_id, "volume_dev": mock.sentinel.volume_dev} + ] certificates = { "client_crt": mock.sentinel.client_crt, "client_key": mock.sentinel.client_key, @@ -1602,10 +1704,11 @@ def test_from_connection_info(self): "client_crt": certificates["client_crt"], "client_key": certificates["client_key"], "ca_crt": certificates["ca_crt"], - } + }, } result = backup_writers.HTTPBackupWriter.from_connection_info( - conn_info, volumes_info) + conn_info, volumes_info + ) self.assertIsInstance(result, backup_writers.HTTPBackupWriter) self.assertEqual(result._ip, conn_info["ip"]) self.assertEqual(result._port, conn_info["port"]) @@ -1621,12 +1724,14 @@ def test_from_connection_info_missing_required_fields(self): self.assertRaises( exception.CoriolisException, backup_writers.HTTPBackupWriter.from_connection_info, - conn_info, mock.sentinel.volumes_info) + conn_info, + mock.sentinel.volumes_info, + ) def test_from_connection_info_missing_cert_options(self): - volumes_info = [{ - "disk_id": mock.sentinel.disk_id, - "volume_dev": mock.sentinel.volume_dev}] + volumes_info = [ + {"disk_id": mock.sentinel.disk_id, "volume_dev": mock.sentinel.volume_dev} + ] certificates = { "client_crt": mock.sentinel.client_crt, "client_key": mock.sentinel.client_key, @@ -1638,13 +1743,15 @@ def test_from_connection_info_missing_cert_options(self): "certificates": { "client_crt": certificates.get("client_crt"), "client_key": certificates.get("client_key"), - } + }, } self.assertRaises( exception.CoriolisException, backup_writers.HTTPBackupWriter.from_connection_info, - conn_info, volumes_info) + conn_info, + volumes_info, + ) def test__del__(self): # Create a temporary directory to test the deletion @@ -1671,13 +1778,13 @@ def test__wait_for_conn(self, mock_wait_for_port_connectivity): self.writer._wait_for_conn() mock_wait_for_port_connectivity.assert_called_once_with( - self.writer._ip, self.writer._port) + self.writer._ip, self.writer._port + ) def test__write_cert_files_no_certificates(self): self.writer._certificates = None - self.assertRaises( - exception.CoriolisException, self.writer._write_cert_files) + self.assertRaises(exception.CoriolisException, self.writer._write_cert_files) def test__write_cert_files_exists(self): self.writer._cert_paths = { @@ -1695,7 +1802,8 @@ def test__write_cert_files_write_new_certs(self, mock_mkstemp, mock_open): mock_mkstemp.side_effect = [ (0, mock.sentinel.client_crt), (1, mock.sentinel.client_key), - (2, mock.sentinel.ca_crt)] + (2, mock.sentinel.ca_crt), + ] self.writer._certificates = self.certificates result = self.writer._write_cert_files() @@ -1707,33 +1815,40 @@ def test__write_cert_files_write_new_certs(self, mock_mkstemp, mock_open): } self.assertEqual(result, expected_cert_paths) - mock_open.assert_has_calls([ - mock.call(self.writer._cert_paths["client_crt"], "w"), - mock.call(self.writer._cert_paths["client_key"], "w"), - mock.call(self.writer._cert_paths["ca_crt"], "w")], any_order=True) + mock_open.assert_has_calls( + [ + mock.call(self.writer._cert_paths["client_crt"], "w"), + mock.call(self.writer._cert_paths["client_key"], "w"), + mock.call(self.writer._cert_paths["ca_crt"], "w"), + ], + any_order=True, + ) @mock.patch.object(backup_writers, 'HTTPBackupWriterImpl') @mock.patch.object(backup_writers.HTTPBackupWriter, '_wait_for_conn') @mock.patch.object(backup_writers.HTTPBackupWriter, '_write_cert_files') - def test__get_impl(self, mock_write_cert_files, mock_wait_for_conn, - mock_http_backup_writer_impl): - result = self.writer._get_impl(mock.sentinel.volume_dev, - mock.sentinel.disk_id) + def test__get_impl( + self, mock_write_cert_files, mock_wait_for_conn, mock_http_backup_writer_impl + ): + result = self.writer._get_impl(mock.sentinel.volume_dev, mock.sentinel.disk_id) self.assertEqual(result, mock_http_backup_writer_impl.return_value) mock_write_cert_files.assert_called_once_with() mock_wait_for_conn.assert_called_once_with() mock_http_backup_writer_impl.assert_called_once_with( - mock.sentinel.volume_dev, mock.sentinel.disk_id, + mock.sentinel.volume_dev, + mock.sentinel.disk_id, compressor_count=self.writer._compressor_count, - compress_transfer=backup_writers.CONF.compress_transfers) - mock_http_backup_writer_impl.return_value._set_info.\ - assert_called_once_with({ + compress_transfer=backup_writers.CONF.compress_transfers, + ) + mock_http_backup_writer_impl.return_value._set_info.assert_called_once_with( + { "ip": self.ip, "port": self.port, "client_crt": mock_write_cert_files.return_value["client_crt"], "client_key": mock_write_cert_files.return_value["client_key"], "ca_crt": mock_write_cert_files.return_value["ca_crt"], "id": self.writer._id, - }) + } + ) diff --git a/coriolis/tests/providers/test_factory.py b/coriolis/tests/providers/test_factory.py index b0c43c1b..8c201628 100644 --- a/coriolis/tests/providers/test_factory.py +++ b/coriolis/tests/providers/test_factory.py @@ -25,20 +25,16 @@ class ProviderClass: # We mocked the PROVIDER_TYPE_MAP to avoid having to import all the # providers - with mock.patch.dict('coriolis.providers.factory.PROVIDER_TYPE_MAP', - {'type1': ProviderClass, - 'type2': ProviderClass}): - expected_result = { - mock.sentinel.platform: { - 'types': ['type1', 'type2'] - } - } + with mock.patch.dict( + 'coriolis.providers.factory.PROVIDER_TYPE_MAP', + {'type1': ProviderClass, 'type2': ProviderClass}, + ): + expected_result = {mock.sentinel.platform: {'types': ['type1', 'type2']}} result = factory.get_available_providers() - mock_load_class.assert_has_calls([ - mock.call(mock_conf.providers[0]), - mock.call(mock_conf.providers[1])] + mock_load_class.assert_has_calls( + [mock.call(mock_conf.providers[0]), mock.call(mock_conf.providers[1])] ) self.assertEqual(result, expected_result) @@ -52,12 +48,14 @@ def test_get_provider(self, mock_conf, mock_load_class): mock_load_class.return_value = mock_class result = factory.get_provider( - mock.sentinel.platform, mock.sentinel.provider_type, - mock.sentinel.event_handler, raise_if_not_found=False) + mock.sentinel.platform, + mock.sentinel.provider_type, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) - mock_load_class.assert_has_calls([ - mock.call(mock_conf.providers[0]), - mock.call(mock_conf.providers[1])] + mock_load_class.assert_has_calls( + [mock.call(mock_conf.providers[0]), mock.call(mock_conf.providers[1])] ) self.assertEqual(result, None) @@ -79,20 +77,28 @@ def __init__(self, event_handler=None): mock_load_class.return_value = ProviderClass - with mock.patch.dict('coriolis.providers.factory.PROVIDER_TYPE_MAP', - {mock.sentinel.provider_type: BaseProviderType}): + with mock.patch.dict( + 'coriolis.providers.factory.PROVIDER_TYPE_MAP', + {mock.sentinel.provider_type: BaseProviderType}, + ): result = factory.get_provider( - mock.sentinel.platform, mock.sentinel.provider_type, - mock.sentinel.event_handler, raise_if_not_found=False) + mock.sentinel.platform, + mock.sentinel.provider_type, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) - mock_load_class.assert_has_calls([ - mock.call(mock_conf.providers[0])]) + mock_load_class.assert_has_calls([mock.call(mock_conf.providers[0])]) self.assertIsInstance(result, ProviderClass) @mock.patch.dict('coriolis.providers.factory.PROVIDER_TYPE_MAP', {}) def test_get_provider_with_exception(self): self.assertRaises( - exception.NotFound, factory.get_provider, mock.sentinel.platform, - mock.sentinel.provider_type, mock.sentinel.event_handler, - raise_if_not_found=True) + exception.NotFound, + factory.get_provider, + mock.sentinel.platform, + mock.sentinel.provider_type, + mock.sentinel.event_handler, + raise_if_not_found=True, + ) diff --git a/coriolis/tests/providers/test_provider_utils.py b/coriolis/tests/providers/test_provider_utils.py index d77dee2b..049f5497 100644 --- a/coriolis/tests/providers/test_provider_utils.py +++ b/coriolis/tests/providers/test_provider_utils.py @@ -14,37 +14,39 @@ class ProviderUtilsTestCase(test_base.CoriolisBaseTestCase): def test_get_storage_mapping_for_disk(self): storage_mappings = { 'disk_mappings': [{'disk_id': '1', 'destination': 'backend1'}], - 'backend_mappings': [{'source': 'backend1', - 'destination': 'backend2'}], - 'default': 'default_backend' - } - disk_info = { - 'id': '1', - 'storage_backend_identifier': 'backend1' + 'backend_mappings': [{'source': 'backend1', 'destination': 'backend2'}], + 'default': 'default_backend', } + disk_info = {'id': '1', 'storage_backend_identifier': 'backend1'} storage_backends = [{'name': 'backend1'}, {'name': 'backend2'}] expected_result = 'backend1' result = provider_utils.get_storage_mapping_for_disk( - storage_mappings, disk_info, storage_backends, + storage_mappings, + disk_info, + storage_backends, config_default='default_backend', - error_on_missing_mapping=False, error_on_backend_not_found=False) + error_on_missing_mapping=False, + error_on_backend_not_found=False, + ) self.assertEqual(expected_result, result) def test_get_storage_mapping_for_disk_backend_mapping_found(self): storage_mappings = { - "backend_mappings": [{ - "source": "source1", "destination": "backend1"}] + "backend_mappings": [{"source": "source1", "destination": "backend1"}] } disk_info = {"id": "1", "storage_backend_identifier": "source1"} storage_backends = [{"name": "backend1"}] with self.assertLogs(level=logging.DEBUG): result = provider_utils.get_storage_mapping_for_disk( - storage_mappings, disk_info, storage_backends, + storage_mappings, + disk_info, + storage_backends, error_on_backend_not_found=False, - error_on_missing_mapping=False) + error_on_missing_mapping=False, + ) self.assertEqual("backend1", result) @@ -54,24 +56,31 @@ def test_get_storage_mapping_for_disk_no_mapping_found(self): storage_backends = [{"name": "backend1"}] with self.assertLogs(level=logging.WARN): - self.assertRaises(exception.DiskStorageMappingNotFound, - provider_utils.get_storage_mapping_for_disk, - storage_mappings, disk_info, storage_backends, - error_on_backend_not_found=False, - error_on_missing_mapping=True) + self.assertRaises( + exception.DiskStorageMappingNotFound, + provider_utils.get_storage_mapping_for_disk, + storage_mappings, + disk_info, + storage_backends, + error_on_backend_not_found=False, + error_on_missing_mapping=True, + ) def test_get_storage_mapping_for_disk_backend_not_found(self): storage_mappings = {"default": "backend1"} disk_info = {"id": "1", "storage_backend_identifier": "source_backend"} storage_backends = [{"name": "backend2"}] - with self.assertLogs( - 'coriolis.providers.provider_utils', level=logging.WARN): - self.assertRaises(exception.StorageBackendNotFound, - provider_utils.get_storage_mapping_for_disk, - storage_mappings, disk_info, storage_backends, - error_on_backend_not_found=True, - error_on_missing_mapping=False) + with self.assertLogs('coriolis.providers.provider_utils', level=logging.WARN): + self.assertRaises( + exception.StorageBackendNotFound, + provider_utils.get_storage_mapping_for_disk, + storage_mappings, + disk_info, + storage_backends, + error_on_backend_not_found=True, + error_on_missing_mapping=False, + ) def test_get_storage_mapping_for_disk_default_backend(self): storage_mappings = {'default': 'default_backend'} @@ -79,181 +88,186 @@ def test_get_storage_mapping_for_disk_default_backend(self): storage_backends = [{'name': 'default_backend'}, {'name': 'backend2'}] result = provider_utils.get_storage_mapping_for_disk( - storage_mappings, disk_info, storage_backends, + storage_mappings, + disk_info, + storage_backends, config_default='default_backend', - error_on_missing_mapping=False, error_on_backend_not_found=False) + error_on_missing_mapping=False, + error_on_backend_not_found=False, + ) self.assertEqual(result, 'default_backend') def test_check_changed_storage_mappings_empty_volumes_info(self): old_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} + 'disk_mappings': [{'key': 'value'}], + } new_storage_mappings = { 'backend_mappings': [{'key': 'value2'}], - 'disk_mappings': [{'key': 'value'}]} - self.assertIsNone(provider_utils.check_changed_storage_mappings( - [], old_storage_mappings, new_storage_mappings)) + 'disk_mappings': [{'key': 'value'}], + } + self.assertIsNone( + provider_utils.check_changed_storage_mappings( + [], old_storage_mappings, new_storage_mappings + ) + ) def test_check_changed_storage_mappings_no_change(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} + 'disk_mappings': [{'key': 'value'}], + } new_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} - self.assertIsNone(provider_utils.check_changed_storage_mappings( - volumes_info, old_storage_mappings, new_storage_mappings)) + 'disk_mappings': [{'key': 'value'}], + } + self.assertIsNone( + provider_utils.check_changed_storage_mappings( + volumes_info, old_storage_mappings, new_storage_mappings + ) + ) def test_check_changed_storage_mappings(self): old_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} + 'disk_mappings': [{'key': 'value'}], + } new_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} - self.assertIsNone(provider_utils.check_changed_storage_mappings( - None, old_storage_mappings, new_storage_mappings)) + 'disk_mappings': [{'key': 'value'}], + } + self.assertIsNone( + provider_utils.check_changed_storage_mappings( + None, old_storage_mappings, new_storage_mappings + ) + ) def test_check_changed_storage_mappings_with_exception(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { 'backend_mappings': [{'key': 'value'}], - 'disk_mappings': [{'key': 'value'}]} + 'disk_mappings': [{'key': 'value'}], + } new_storage_mappings = { 'backend_mappings': [{'key': 'value2'}], - 'disk_mappings': [{'key': 'value2'}]} + 'disk_mappings': [{'key': 'value2'}], + } self.assertRaises( exception.CoriolisException, provider_utils.check_changed_storage_mappings, - volumes_info, old_storage_mappings, - new_storage_mappings) + volumes_info, + old_storage_mappings, + new_storage_mappings, + ) def test_check_changed_storage_mappings_adding_new_mappings_allowed(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } new_storage_mappings = { 'backend_mappings': [ {'source': 'backend1', 'destination': 'dest1'}, - {'source': 'backend2', 'destination': 'dest2'} + {'source': 'backend2', 'destination': 'dest2'}, ], 'disk_mappings': [ {'disk_id': '1', 'destination': 'dest1'}, - {'disk_id': '2', 'destination': 'dest2'} - ] + {'disk_id': '2', 'destination': 'dest2'}, + ], } - self.assertIsNone(provider_utils.check_changed_storage_mappings( - volumes_info, old_storage_mappings, new_storage_mappings)) + self.assertIsNone( + provider_utils.check_changed_storage_mappings( + volumes_info, old_storage_mappings, new_storage_mappings + ) + ) def test_check_changed_storage_mappings_backend_mappings_changed(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } new_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest2'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest2'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } self.assertRaises( exception.CoriolisException, provider_utils.check_changed_storage_mappings, - volumes_info, old_storage_mappings, - new_storage_mappings) + volumes_info, + old_storage_mappings, + new_storage_mappings, + ) def test_check_changed_storage_mappings_missing_backend_mapping(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { 'backend_mappings': [ {'source': 'backend1', 'destination': 'dest1'}, - {'source': 'backend2', 'destination': 'dest2'} + {'source': 'backend2', 'destination': 'dest2'}, ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } new_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } self.assertRaises( exception.CoriolisException, provider_utils.check_changed_storage_mappings, - volumes_info, old_storage_mappings, - new_storage_mappings) + volumes_info, + old_storage_mappings, + new_storage_mappings, + ) def test_check_changed_storage_mappings_missing_disk_mapping(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], 'disk_mappings': [ {'disk_id': '1', 'destination': 'dest1'}, - {'disk_id': '2', 'destination': 'dest2'} - ] + {'disk_id': '2', 'destination': 'dest2'}, + ], } new_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } self.assertRaises( exception.CoriolisException, provider_utils.check_changed_storage_mappings, - volumes_info, old_storage_mappings, - new_storage_mappings) + volumes_info, + old_storage_mappings, + new_storage_mappings, + ) def test_check_changed_storage_mappings_empty_old_mappings(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = {} new_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } - self.assertIsNone(provider_utils.check_changed_storage_mappings( - volumes_info, old_storage_mappings, new_storage_mappings)) + self.assertIsNone( + provider_utils.check_changed_storage_mappings( + volumes_info, old_storage_mappings, new_storage_mappings + ) + ) def test_check_changed_storage_mappings_empty_new_mappings(self): volumes_info = [{'volume_id': '1'}] old_storage_mappings = { - 'backend_mappings': [ - {'source': 'backend1', 'destination': 'dest1'} - ], - 'disk_mappings': [ - {'disk_id': '1', 'destination': 'dest1'} - ] + 'backend_mappings': [{'source': 'backend1', 'destination': 'dest1'}], + 'disk_mappings': [{'disk_id': '1', 'destination': 'dest1'}], } new_storage_mappings = {} self.assertRaises( exception.CoriolisException, provider_utils.check_changed_storage_mappings, - volumes_info, old_storage_mappings, - new_storage_mappings) + volumes_info, + old_storage_mappings, + new_storage_mappings, + ) diff --git a/coriolis/tests/providers/test_replicator.py b/coriolis/tests/providers/test_replicator.py index 7c45b6a3..a5612356 100644 --- a/coriolis/tests/providers/test_replicator.py +++ b/coriolis/tests/providers/test_replicator.py @@ -10,8 +10,7 @@ from coriolis import exception from coriolis.providers import provider_utils from coriolis.providers import replicator as replicator_module -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis.tests import test_base, testutils class ClientTestCase(test_base.CoriolisBaseTestCase): @@ -27,10 +26,7 @@ def setUp(self): 'username': mock.sentinel.username, 'password': mock.sentinel.password, } - self.chunk = { - "offset": 0, - "length": 11 - } + self.chunk = {"offset": 0, "length": 11} self.device = mock.sentinel.device self.disk = mock.sentinel.disk self.ssh_conn_info = { @@ -43,13 +39,21 @@ def setUp(self): self.event_handler = mock.MagicMock() self.mock_response = mock.MagicMock() - with mock.patch.object(replicator_module.Client, '_get_session', - return_value=None), \ - mock.patch.object(replicator_module.Client, '_test_connection', - return_value=None): + with ( + mock.patch.object( + replicator_module.Client, '_get_session', return_value=None + ), + mock.patch.object( + replicator_module.Client, '_test_connection', return_value=None + ), + ): self.client = replicator_module.Client( - self.ip, self.port, self.credentials, self.ssh_conn_info, - self.event_handler) + self.ip, + self.port, + self.credentials, + self.ssh_conn_info, + self.event_handler, + ) self.client._cli = mock.MagicMock() @@ -76,16 +80,17 @@ def test_repl_port_no_port_via_tunnel(self): def test_base_uri(self): self.client._ip_via_tunnel = self.ip_via_tunnel self.client._port_via_tunnel = self.port_via_tunnel - expected_uri_via_tunnel = "https://%s:%s" % (self.ip_via_tunnel, - self.port_via_tunnel) + expected_uri_via_tunnel = "https://%s:%s" % ( + self.ip_via_tunnel, + self.port_via_tunnel, + ) result = self.client._base_uri self.assertEqual(result, expected_uri_via_tunnel) @mock.patch.object(replicator_module.Client, '_get_ssh_tunnel') def test__setup_tunnel_connection(self, mock_get_ssh_tunnel): mock_tunnel = mock.MagicMock() - mock_tunnel.local_bind_address = ( - self.ip_via_tunnel, self.port_via_tunnel) + mock_tunnel.local_bind_address = (self.ip_via_tunnel, self.port_via_tunnel) mock_get_ssh_tunnel.return_value = mock_tunnel self.client._setup_tunnel_connection() @@ -102,25 +107,25 @@ def test__test_connection_no_tunnel_no_exception(self, mock_wait_for_port): self.client._test_connection() - mock_wait_for_port.assert_called_once_with( - self.ip, self.port, max_wait=30) + mock_wait_for_port.assert_called_once_with(self.ip, self.port, max_wait=30) @mock.patch.object(replicator_module.utils, 'wait_for_port_connectivity') @mock.patch.object(replicator_module.Client, '_setup_tunnel_connection') def test__test_connection_with_tunnel_no_exception( - self, mock_setup_tunnel, mock_wait_for_port): + self, mock_setup_tunnel, mock_wait_for_port + ): self.client._use_tunnel = True self.client._test_connection() mock_setup_tunnel.assert_called_once() - mock_wait_for_port.assert_called_once_with( - self.ip, self.port, max_wait=30) + mock_wait_for_port.assert_called_once_with(self.ip, self.port, max_wait=30) @mock.patch.object(replicator_module.utils, 'wait_for_port_connectivity') @mock.patch.object(replicator_module.Client, '_setup_tunnel_connection') def test_test_connection_no_tunnel_with_exception( - self, mock_setup_tunnel, mock_wait_for_port): + self, mock_setup_tunnel, mock_wait_for_port + ): self.client._use_tunnel = False mock_wait_for_port.side_effect = [BaseException(), None] @@ -131,14 +136,14 @@ def test_test_connection_no_tunnel_with_exception( @mock.patch.object(replicator_module.utils, 'wait_for_port_connectivity') @mock.patch.object(replicator_module.Client, '_setup_tunnel_connection') def test_test_connection_with_tunnel_with_exception( - self, mock_setup_tunnel, mock_wait_for_port): + self, mock_setup_tunnel, mock_wait_for_port + ): mock_tunnel = mock.MagicMock() self.client._tunnel = mock_tunnel self.client._use_tunnel = True mock_wait_for_port.side_effect = BaseException() - with self.assertLogs('coriolis.providers.replicator', - logging.WARNING): + with self.assertLogs('coriolis.providers.replicator', logging.WARNING): self.assertRaises(BaseException, self.client._test_connection) mock_setup_tunnel.assert_called_once() @@ -148,13 +153,13 @@ def test_test_connection_with_tunnel_with_exception( def test__get_ssh_tunnel(self, mock_SSHTunnelForwarder): result = self.client._get_ssh_tunnel() - mock_SSHTunnelForwarder.assert_called_once_with(( - self.ssh_conn_info["hostname"], self.ssh_conn_info["port"]), + mock_SSHTunnelForwarder.assert_called_once_with( + (self.ssh_conn_info["hostname"], self.ssh_conn_info["port"]), ssh_username=self.ssh_conn_info["username"], ssh_pkey=self.ssh_conn_info["pkey"], ssh_password=self.ssh_conn_info["password"], remote_bind_address=("127.0.0.1", self.port), - local_bind_address=("127.0.0.1", 0) + local_bind_address=("127.0.0.1", 0), ) self.assertEqual(result, mock_SSHTunnelForwarder.return_value) @@ -165,8 +170,7 @@ def test__get_ssh_tunnel_no_credentials(self): "port": self.ssh_conn_info["port"], "username": self.ssh_conn_info["username"], } - self.assertRaises(exception.CoriolisException, - self.client._get_ssh_tunnel) + self.assertRaises(exception.CoriolisException, self.client._get_ssh_tunnel) def test_raw_disk_uri(self): result = self.client.raw_disk_uri(self.device) @@ -185,19 +189,18 @@ def test__get_session(self, mock_Session): mock_Session.assert_called_once() self.assertEqual(result, mock_Session.return_value) - self.assertEqual(result.cert, ( - self.client._creds["client_cert"], - self.client._creds["client_key"])) + self.assertEqual( + result.cert, + (self.client._creds["client_cert"], self.client._creds["client_key"]), + ) self.assertEqual(result.verify, self.client._creds["ca_cert"]) def test_get_status(self): self.client._cli.get.return_value = self.mock_response - original_get_status = testutils.get_wrapped_function( - self.client.get_status) + original_get_status = testutils.get_wrapped_function(self.client.get_status) - result = original_get_status(self.client, device=self.device, - brief=True) + result = original_get_status(self.client, device=self.device, brief=True) self.assertEqual(result, self.mock_response.json.return_value) self.client._cli.get.assert_called_once_with( @@ -210,8 +213,7 @@ def test_get_status(self): def test_get_status_no_device(self): self.client._cli.get.return_value = self.mock_response - original_get_status = testutils.get_wrapped_function( - self.client.get_status) + original_get_status = testutils.get_wrapped_function(self.client.get_status) result = original_get_status(self.client, device=None, brief=True) @@ -225,16 +227,13 @@ def test_get_status_no_device(self): def test_get_chunks(self): self.client._cli.get.return_value = self.mock_response - original_get_chunks = testutils.get_wrapped_function( - self.client.get_chunks) + original_get_chunks = testutils.get_wrapped_function(self.client.get_chunks) - result = original_get_chunks(self.client, self.device, - skip_zeros=False) + result = original_get_chunks(self.client, self.device, skip_zeros=False) self.assertEqual(result, self.mock_response.json.return_value) self.client._cli.get.assert_called_once_with( - "https://%s:%s/api/v1/dev/%s/chunks/" % ( - self.ip, self.port, self.device), + "https://%s:%s/api/v1/dev/%s/chunks/" % (self.ip, self.port, self.device), params={"skipZeros": False}, timeout=replicator_module.CONF.replicator.default_requests_timeout, ) @@ -243,15 +242,14 @@ def test_get_chunks(self): def test_get_changes(self): self.client._cli.get.return_value = self.mock_response - original_get_changes = testutils.get_wrapped_function( - self.client.get_changes) + original_get_changes = testutils.get_wrapped_function(self.client.get_changes) result = original_get_changes(self.client, self.device) self.assertEqual(result, self.mock_response.json.return_value) self.client._cli.get.assert_called_once_with( - "https://%s:%s/api/v1/dev/%s/chunks/changes/" % ( - self.ip, self.port, self.device), + "https://%s:%s/api/v1/dev/%s/chunks/changes/" + % (self.ip, self.port, self.device), timeout=replicator_module.CONF.replicator.default_requests_timeout, ) self.mock_response.raise_for_status.assert_called_once() @@ -260,12 +258,12 @@ def test_get_disk_size(self): self.client._cli.head.return_value = self.mock_response original_get_disk_size = testutils.get_wrapped_function( - self.client.get_disk_size) + self.client.get_disk_size + ) result = original_get_disk_size(self.client, self.device) - self.assertEqual( - result, int(self.mock_response.headers["Content-Length"])) + self.assertEqual(result, int(self.mock_response.headers["Content-Length"])) self.client._cli.head.assert_called_once_with( "https://%s:%s/device/%s" % (self.ip, self.port, self.device), timeout=replicator_module.CONF.replicator.default_requests_timeout, @@ -277,7 +275,8 @@ def test_download_chunk(self): self.client._cli.get.return_value = self.mock_response original_download_chunk = testutils.get_wrapped_function( - self.client.download_chunk) + self.client.download_chunk + ) result = original_download_chunk(self.client, self.disk, self.chunk) @@ -297,17 +296,22 @@ def test_download_chunk_no_compression(self): self.chunk["length"] = 200 original_download_chunk = testutils.get_wrapped_function( - self.client.download_chunk) + self.client.download_chunk + ) result = original_download_chunk(self.client, self.disk, self.chunk) self.assertEqual(result, self.mock_response.content) self.client._cli.get.assert_called_once_with( "https://%s:%s/device/%s" % (self.ip, self.port, self.disk), - headers={"Range": "bytes=%s-%s" % ( - self.chunk["offset"], - self.chunk["offset"] + self.chunk["length"] - 1), - "Accept-encoding": "identity"}, + headers={ + "Range": "bytes=%s-%s" + % ( + self.chunk["offset"], + self.chunk["offset"] + self.chunk["length"] - 1, + ), + "Accept-encoding": "identity", + }, timeout=replicator_module.CONF.replicator.default_requests_timeout, ) self.mock_response.raise_for_status.assert_called_once() @@ -316,14 +320,14 @@ def test_get_disk_checksum(self): self.client._cli.get.return_value = self.mock_response original_get_disk_checksum = testutils.get_wrapped_function( - self.client.get_disk_checksum) + self.client.get_disk_checksum + ) result = original_get_disk_checksum(self.client, self.device) self.assertEqual(result, self.mock_response.json.return_value) self.client._cli.get.assert_called_once_with( - "https://%s:%s/api/v1/dev/%s/checksum" % ( - self.ip, self.port, self.device), + "https://%s:%s/api/v1/dev/%s/checksum" % (self.ip, self.port, self.device), timeout=replicator_module.CONF.replicator.default_requests_timeout, ) self.mock_response.raise_for_status.assert_called_once() @@ -340,10 +344,12 @@ def setUp(self): 'password': mock.sentinel.password, 'port': mock.sentinel.port, } - self.volumes_info = [{ - 'disk_id': 'test_disk', - 'other_key': mock.sentinel.other_key, - }] + self.volumes_info = [ + { + 'disk_id': 'test_disk', + 'other_key': mock.sentinel.other_key, + } + ] self.replica_state = 'replica_state' self._ssh = mock.MagicMock() self._credentials = mock.MagicMock() @@ -355,15 +361,17 @@ def setUp(self): self.path = '/path/to/disk' self.size = 1000 - with mock.patch.object(replicator_module, 'CONF', - return_value=mock.MagicMock()), \ - mock.patch.object(replicator_module.Replicator, '_setup_ssh', - return_value=None): + with ( + mock.patch.object(replicator_module, 'CONF', return_value=mock.MagicMock()), + mock.patch.object( + replicator_module.Replicator, '_setup_ssh', return_value=None + ), + ): self.replicator = replicator_module.Replicator( self.conn_info, self.event_manager, self.volumes_info, - self.replica_state + self.replica_state, ) self.replicator._cli = mock.MagicMock() self.replicator._event_manager = mock.MagicMock() @@ -377,16 +385,15 @@ def test__del__(self, mock_rmtree): mock_rmtree.assert_called_once_with('/tmp/cert_dir') - @mock.patch.object( - replicator_module.Replicator, '_parse_source_ssh_conn_info' - ) - @mock.patch.object( - replicator_module.Replicator, '_parse_replicator_conn_info' - ) + @mock.patch.object(replicator_module.Replicator, '_parse_source_ssh_conn_info') + @mock.patch.object(replicator_module.Replicator, '_parse_replicator_conn_info') @mock.patch.object(replicator_module, 'Client') def test__init_replicator_client( - self, mock_Client, mock_parse_replicator_conn_info, - mock_parse_source_ssh_conn_info): + self, + mock_Client, + mock_parse_replicator_conn_info, + mock_parse_source_ssh_conn_info, + ): mock_parse_replicator_conn_info.return_value = self.conn_info mock_parse_source_ssh_conn_info.return_value = self.conn_info mock_Client.return_value = None @@ -394,27 +401,31 @@ def test__init_replicator_client( self.replicator._init_replicator_client(self.conn_info) mock_parse_source_ssh_conn_info.assert_called_once_with( - self.replicator._conn_info) + self.replicator._conn_info + ) mock_parse_replicator_conn_info.assert_called_once_with( - self.replicator._conn_info) + self.replicator._conn_info + ) mock_Client.assert_called_once_with( - self.conn_info['ip'], self.conn_info['port'], - self.conn_info, self.conn_info, self.replicator._event_manager, + self.conn_info['ip'], + self.conn_info['port'], + self.conn_info, + self.conn_info, + self.replicator._event_manager, use_compression=self.replicator._use_compression, - use_tunnel=self.replicator._use_tunnel) + use_tunnel=self.replicator._use_tunnel, + ) @mock.patch.object(replicator_module.Replicator, '_get_ssh_client') - @mock.patch.object( - replicator_module.Replicator, '_parse_source_ssh_conn_info' - ) - def test__setup_ssh(self, mock_parse_source_ssh_conn_info, - mock_get_ssh_client): + @mock.patch.object(replicator_module.Replicator, '_parse_source_ssh_conn_info') + def test__setup_ssh(self, mock_parse_source_ssh_conn_info, mock_get_ssh_client): mock_parse_source_ssh_conn_info.return_value = self.conn_info result = self.replicator._setup_ssh() mock_parse_source_ssh_conn_info.assert_called_once_with( - self.replicator._conn_info) + self.replicator._conn_info + ) mock_get_ssh_client.assert_called_once_with(self.conn_info) self.assertEqual(result, mock_get_ssh_client.return_value) @@ -432,8 +443,7 @@ def test__reconnect_ssh(self, mock_setup_ssh): @mock.patch.object(replicator_module.Replicator, '_setup_replicator') @mock.patch.object(replicator_module.Replicator, '_init_replicator_client') - def test_init_replicator(self, mock_init_replicator_client, - mock_setup_replicator): + def test_init_replicator(self, mock_init_replicator_client, mock_setup_replicator): self.replicator._ssh = self._ssh mock_setup_replicator.return_value = self._credentials @@ -455,13 +465,17 @@ def test_attach_new_disk(self, mock_get_status): self.replicator._cli = mock_get_status mock_return_value_before = [ - {'device-path': '/dev/xvdf', 'disk_id': 'existing_disk'}] + {'device-path': '/dev/xvdf', 'disk_id': 'existing_disk'} + ] mock_return_value_after = [ {'device-path': '/dev/xvdf', 'disk_id': 'existing_disk'}, - {'device-path': '/dev/xvdg', 'disk_id': 'test_disk'}] + {'device-path': '/dev/xvdg', 'disk_id': 'test_disk'}, + ] mock_get_status.get_status.side_effect = [ - mock_return_value_before, mock_return_value_after] + mock_return_value_before, + mock_return_value_after, + ] result = self.replicator.attach_new_disk(self.disk, self._attachf) @@ -470,44 +484,59 @@ def test_attach_new_disk(self, mock_get_status): expected_return_value = { 'disk_id': 'test_disk', 'disk_path': '/dev/xvdg', - 'other_key': self.volumes_info[0]['other_key'] + 'other_key': self.volumes_info[0]['other_key'], } self.assertEqual(result, expected_return_value) def test_attach_new_disk_no_matching_volumes(self): self.replicator._volumes_info = [ - {'disk_id': 'other_disk', 'other_key': 'other_value'}] + {'disk_id': 'other_disk', 'other_key': 'other_value'} + ] - self.assertRaises(exception.CoriolisException, - self.replicator.attach_new_disk, self.disk, - self._attachf) + self.assertRaises( + exception.CoriolisException, + self.replicator.attach_new_disk, + self.disk, + self._attachf, + ) def test_attach_new_disk_multiple_matching_volumes(self): self.replicator._volumes_info = [ {'disk_id': 'test_disk', 'other_key': 'other_value'}, - {'disk_id': 'test_disk', 'other_key': 'other_value'}] + {'disk_id': 'test_disk', 'other_key': 'other_value'}, + ] - self.assertRaises(exception.CoriolisException, - self.replicator.attach_new_disk, self.disk, - self._attachf) + self.assertRaises( + exception.CoriolisException, + self.replicator.attach_new_disk, + self.disk, + self._attachf, + ) @mock.patch.object(replicator_module.Client, 'get_status') def test_attach_new_disk_multiple_new_device_paths(self, mock_get_status): self.replicator._cli = mock_get_status mock_return_value_before = [ - {'device-path': '/dev/xvdf', 'disk_id': 'test_disk'}] + {'device-path': '/dev/xvdf', 'disk_id': 'test_disk'} + ] mock_return_value_after = [ {'device-path': '/dev/xvdf', 'disk_id': 'test_disk'}, {'device-path': '/dev/xvdf2', 'disk_id': 'new_disk'}, - {'device-path': '/dev/xvdf3', 'disk_id': 'new_disk'}] + {'device-path': '/dev/xvdf3', 'disk_id': 'new_disk'}, + ] mock_get_status.get_status.side_effect = [ - mock_return_value_before, mock_return_value_after] + mock_return_value_before, + mock_return_value_after, + ] - self.assertRaises(exception.CoriolisException, - self.replicator.attach_new_disk, self.disk, - self._attachf) + self.assertRaises( + exception.CoriolisException, + self.replicator.attach_new_disk, + self.disk, + self._attachf, + ) @mock.patch.object(replicator_module.Client, 'get_status') def test_attach_new_disk_missing_device_paths(self, mock_get_status): @@ -515,16 +544,19 @@ def test_attach_new_disk_missing_device_paths(self, mock_get_status): mock_return_value_before = [ {'device-path': '/dev/xvdf', 'disk_id': 'test_disk'}, - {'device-path': '/dev/xvdg', 'disk_id': 'old_disk'}] + {'device-path': '/dev/xvdg', 'disk_id': 'old_disk'}, + ] mock_return_value_after = [ {'device-path': '/dev/xvdf', 'disk_id': 'test_disk'}, - {'device-path': '/dev/xvdh', 'disk_id': 'new_disk'}] + {'device-path': '/dev/xvdh', 'disk_id': 'new_disk'}, + ] mock_get_status.get_status.side_effect = [ - mock_return_value_before, mock_return_value_after] + mock_return_value_before, + mock_return_value_after, + ] - with self.assertLogs('coriolis.providers.replicator', - logging.WARN): + with self.assertLogs('coriolis.providers.replicator', logging.WARN): self.replicator.attach_new_disk(self.disk, self._attachf) @mock.patch.object(replicator_module.Client, 'get_status') @@ -533,41 +565,44 @@ def test_attach_new_disk_no_new_device_paths(self, _, mock_get_status): self.replicator._cli = mock_get_status mock_return_value_before = [{'device-path': 'path1'}] - self.replicator._volumes_info = [ - {'disk_id': 'disk1', 'disk_path': 'path1'}] + self.replicator._volumes_info = [{'disk_id': 'disk1', 'disk_path': 'path1'}] mock_get_status.get_status.side_effect = [ - mock_return_value_before, mock_return_value_before] + mock_return_value_before, + mock_return_value_before, + ] def attachf(): pass - self.assertRaises(exception.CoriolisException, - self.replicator.attach_new_disk, - 'disk1', attachf, retry_count=1) + self.assertRaises( + exception.CoriolisException, + self.replicator.attach_new_disk, + 'disk1', + attachf, + retry_count=1, + ) @mock.patch.object(replicator_module.Client, 'get_status') @mock.patch('time.sleep') - def test_attach_new_disk_new_device_paths_after_retry(self, _, - mock_get_status): + def test_attach_new_disk_new_device_paths_after_retry(self, _, mock_get_status): self.replicator._cli = mock_get_status mock_return_value_before = [{'device-path': 'path1'}] - mock_return_value_after = [ - {'device-path': 'path1'}, {'device-path': 'path2'}] + mock_return_value_after = [{'device-path': 'path1'}, {'device-path': 'path2'}] - self.replicator._volumes_info = [ - {'disk_id': 'disk1', 'disk_path': 'path1'}] + self.replicator._volumes_info = [{'disk_id': 'disk1', 'disk_path': 'path1'}] mock_get_status.get_status.side_effect = [ - mock_return_value_before, mock_return_value_before, - mock_return_value_after] + mock_return_value_before, + mock_return_value_before, + mock_return_value_after, + ] def attachf(): pass - self.replicator.attach_new_disk('disk1', attachf, retry_count=2, - retry_period=0) + self.replicator.attach_new_disk('disk1', attachf, retry_count=2, retry_period=0) @mock.patch.object(replicator_module.Client, 'get_status') def test_wait_for_chunks(self, mock_get_status): @@ -576,49 +611,54 @@ def test_wait_for_chunks(self, mock_get_status): { "device-path": mock.sentinel.disk, "size": 1024 * units.Mi, - "checksum-status": {"percentage": 100} + "checksum-status": {"percentage": 100}, }, { "device-path": mock.sentinel.disk, "size": 1024 * units.Mi, - "checksum-status": {"percentage": 100} - } + "checksum-status": {"percentage": 100}, + }, ] - self.replicator._event_manager.add_percentage_step.\ - return_value = mock.sentinel.perc_step + self.replicator._event_manager.add_percentage_step.return_value = ( + mock.sentinel.perc_step + ) self.replicator.wait_for_chunks() mock_get_status.get_status.assert_called_once() self.replicator._event_manager.add_percentage_step.assert_called_once() - self.replicator._event_manager.set_percentage_step.assert_has_calls([ - mock.call(mock.sentinel.perc_step, 100), - mock.call(mock.sentinel.perc_step, 100) - ]) + self.replicator._event_manager.set_percentage_step.assert_has_calls( + [ + mock.call(mock.sentinel.perc_step, 100), + mock.call(mock.sentinel.perc_step, 100), + ] + ) def test_wait_for_chunks_not_initialized(self): self.replicator._cli = None - self.assertRaises(exception.CoriolisException, - self.replicator.wait_for_chunks) + self.assertRaises(exception.CoriolisException, self.replicator.wait_for_chunks) @mock.patch.object(replicator_module.utils, 'start_service') def test_start(self, mock_start_service): self.replicator.start() mock_start_service.assert_called_once_with( - self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME) + self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME + ) @mock.patch.object(replicator_module.utils, 'stop_service') def test_stop(self, mock_stop_service): self.replicator.stop() mock_stop_service.assert_called_once_with( - self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME) + self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME + ) @mock.patch.object(replicator_module.utils, 'restart_service') def test_restart(self, mock_stop_service): self.replicator.restart() mock_stop_service.assert_called_once_with( - self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME) + self.replicator._ssh, replicator_module.REPLICATOR_SVC_NAME + ) @mock.patch('builtins.open') @mock.patch.object(replicator_module.json, 'dump') @@ -627,8 +667,14 @@ def test_restart(self, mock_stop_service): @mock.patch.object(replicator_module.Replicator, 'restart') @mock.patch.object(replicator_module.Client, '_test_connection') def test_update_state( - self, mock_test_connection, mock_restart, mock_copy_file, - mock_mkstemp, mock_dump, mock_open): + self, + mock_test_connection, + mock_restart, + mock_copy_file, + mock_mkstemp, + mock_dump, + mock_open, + ): mock_mkstemp.return_value = (None, mock.sentinel.state) mock_dump.return_value = None @@ -637,8 +683,10 @@ def test_update_state( mock_mkstemp.assert_called_once() mock_open.assert_called_once_with(mock.sentinel.state, 'w') mock_copy_file.assert_called_once_with( - self.replicator._ssh, mock_mkstemp.return_value[1], - replicator_module.REPLICATOR_STATE) + self.replicator._ssh, + mock_mkstemp.return_value[1], + replicator_module.REPLICATOR_STATE, + ) mock_restart.assert_not_called() mock_test_connection.assert_not_called() @@ -649,8 +697,14 @@ def test_update_state( @mock.patch.object(replicator_module.Replicator, 'restart') @mock.patch.object(replicator_module.Client, '_test_connection') def test_update_state_with_restart( - self, mock_test_connection, mock_restart, mock_copy_file, - mock_mkstemp, mock_dump, mock_open): + self, + mock_test_connection, + mock_restart, + mock_copy_file, + mock_mkstemp, + mock_dump, + mock_open, + ): mock_mkstemp.return_value = (None, mock.sentinel.state) mock_dump.return_value = None @@ -661,8 +715,10 @@ def test_update_state_with_restart( mock_mkstemp.assert_called_once() mock_open.assert_called_once_with(mock.sentinel.state, 'w') mock_copy_file.assert_called_once_with( - self.replicator._ssh, mock_mkstemp.return_value[1], - replicator_module.REPLICATOR_STATE) + self.replicator._ssh, + mock_mkstemp.return_value[1], + replicator_module.REPLICATOR_STATE, + ) mock_restart.assert_called_once() mock_test_connection.assert_called_once() @@ -671,7 +727,8 @@ def test__get_ssh_client(self, mock_ssh_client): self._ssh = mock_ssh_client.return_value original_get_ssh_client = testutils.get_wrapped_function( - self.replicator._get_ssh_client) + self.replicator._get_ssh_client + ) arg = { "hostname": self.conn_info["ip"], @@ -680,13 +737,13 @@ def test__get_ssh_client(self, mock_ssh_client): "password": self.conn_info["password"], "pkey": None, "banner_timeout": ( - replicator_module.CONF.replicator.default_requests_timeout), + replicator_module.CONF.replicator.default_requests_timeout + ), } result = original_get_ssh_client(self.replicator, arg) mock_ssh_client.assert_called_once() - self._ssh.set_missing_host_key_policy.assert_called_once_with( - mock.ANY) + self._ssh.set_missing_host_key_policy.assert_called_once_with(mock.ANY) self._ssh.connect.assert_called_once_with(**arg) self.assertEqual(result, mock_ssh_client.return_value) @@ -695,10 +752,12 @@ def test__get_ssh_client(self, mock_ssh_client): def test__get_ssh_client_exception(self, mock_ssh_client): self._ssh = mock_ssh_client.return_value self._ssh.connect.side_effect = ( - replicator_module.paramiko.ssh_exception.SSHException()) + replicator_module.paramiko.ssh_exception.SSHException() + ) original_get_ssh_client = testutils.get_wrapped_function( - self.replicator._get_ssh_client) + self.replicator._get_ssh_client + ) arg = { "hostname": self.conn_info["ip"], @@ -707,10 +766,12 @@ def test__get_ssh_client_exception(self, mock_ssh_client): "password": self.conn_info["password"], "pkey": None, "banner_timeout": ( - replicator_module.CONF.replicator.default_requests_timeout), + replicator_module.CONF.replicator.default_requests_timeout + ), } - self.assertRaises(exception.CoriolisException, original_get_ssh_client, - self.replicator, arg) + self.assertRaises( + exception.CoriolisException, original_get_ssh_client, self.replicator, arg + ) def test__parse_source_ssh_conn_info(self): expected_arg = { @@ -720,7 +781,8 @@ def test__parse_source_ssh_conn_info(self): "password": self.conn_info["password"], "pkey": None, "banner_timeout": ( - replicator_module.CONF.replicator.default_requests_timeout), + replicator_module.CONF.replicator.default_requests_timeout + ), } result = self.replicator._parse_source_ssh_conn_info(self.conn_info) @@ -732,26 +794,31 @@ def test__parse_source_ssh_conn_info_missing_required_field(self): # required. self.conn_info.pop("username") - self.assertRaises(exception.CoriolisException, - self.replicator._parse_source_ssh_conn_info, - self.conn_info) + self.assertRaises( + exception.CoriolisException, + self.replicator._parse_source_ssh_conn_info, + self.conn_info, + ) def test__parse_source_ssh_conn_info_missing_password(self): self.conn_info.pop("password") - self.assertRaises(exception.CoriolisException, - self.replicator._parse_source_ssh_conn_info, - self.conn_info) + self.assertRaises( + exception.CoriolisException, + self.replicator._parse_source_ssh_conn_info, + self.conn_info, + ) @mock.patch.object(replicator_module.utils, 'deserialize_key') @mock.patch.object(replicator_module, 'CONF') def test__parse_source_ssh_conn_info_with_string_pkey( - self, mock_CONF, mock_deserialize_key): + self, mock_CONF, mock_deserialize_key + ): self.conn_info["pkey"] = "test_pkey" self.replicator._parse_source_ssh_conn_info(self.conn_info) mock_deserialize_key.assert_called_once_with( - "test_pkey", - mock_CONF.serialization.temp_keypair_password) + "test_pkey", mock_CONF.serialization.temp_keypair_password + ) @mock.patch.object(replicator_module.tempfile, 'mkstemp') @mock.patch('builtins.open') @@ -778,26 +845,29 @@ def test__parse_replicator_conn_info(self): @mock.patch.object(replicator_module.tempfile, 'mktemp') @mock.patch('paramiko.SFTPClient.from_transport') @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') - def test__copy_file(self, mock_exec_ssh_cmd, mock_from_transport, - mock_mktemp): + def test__copy_file(self, mock_exec_ssh_cmd, mock_from_transport, mock_mktemp): mock_sftp = mock.MagicMock() mock_from_transport.return_value = mock_sftp - original_copy_file = testutils.get_wrapped_function( - self.replicator._copy_file) + original_copy_file = testutils.get_wrapped_function(self.replicator._copy_file) - original_copy_file(self.replicator, self._ssh, mock.sentinel.localPath, - mock.sentinel.remotePath) + original_copy_file( + self.replicator, + self._ssh, + mock.sentinel.localPath, + mock.sentinel.remotePath, + ) mock_mktemp.assert_called_once_with(dir='/tmp') - mock_from_transport.assert_called_once_with( - self._ssh.get_transport()) + mock_from_transport.assert_called_once_with(self._ssh.get_transport()) mock_sftp.put.assert_called_once_with( - mock.sentinel.localPath, mock_mktemp.return_value) + mock.sentinel.localPath, mock_mktemp.return_value + ) mock_exec_ssh_cmd.assert_called_once_with( - self._ssh, "sudo mv %s %s" % ( - mock_mktemp.return_value, mock.sentinel.remotePath), - get_pty=True) + self._ssh, + "sudo mv %s %s" % (mock_mktemp.return_value, mock.sentinel.remotePath), + get_pty=True, + ) mock_sftp.close.assert_called_once() @mock.patch.object(os.path, 'join') @@ -805,23 +875,26 @@ def test__copy_file(self, mock_exec_ssh_cmd, mock_from_transport, @mock.patch.object(replicator_module.Replicator, '_copy_file') @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') def test__copy_replicator_cmd( - self, mock_exec_ssh_cmd, mock_copy_file, - mock_get_resources_bin_dir, mock_join): + self, mock_exec_ssh_cmd, mock_copy_file, mock_get_resources_bin_dir, mock_join + ): original_copy_replicator_cmd = testutils.get_wrapped_function( - self.replicator._copy_replicator_cmd) + self.replicator._copy_replicator_cmd + ) original_copy_replicator_cmd(self.replicator, self._ssh) mock_get_resources_bin_dir.assert_called_once() mock_join.assert_called_once_with( - mock_get_resources_bin_dir.return_value, 'replicator') + mock_get_resources_bin_dir.return_value, 'replicator' + ) mock_copy_file.assert_called_once_with( - self._ssh, mock_join.return_value, - replicator_module.REPLICATOR_PATH) + self._ssh, mock_join.return_value, replicator_module.REPLICATOR_PATH + ) mock_exec_ssh_cmd.assert_called_once_with( - self._ssh, "sudo chmod +x %s" % - replicator_module.REPLICATOR_PATH, - get_pty=True) + self._ssh, + "sudo chmod +x %s" % replicator_module.REPLICATOR_PATH, + get_pty=True, + ) @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') def test_setup_replicator_group(self, mock_exec_ssh_cmd): @@ -831,8 +904,9 @@ def test_setup_replicator_group(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.assert_called_once_with( self._ssh, - "getent group %s > /dev/null && echo 1 || echo 0" % ( - replicator_module.REPLICATOR_GROUP_NAME)) + "getent group %s > /dev/null && echo 1 || echo 0" + % (replicator_module.REPLICATOR_GROUP_NAME), + ) self.assertEqual(result, True) @@ -842,18 +916,28 @@ def test__setup_replicator_no_group(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.return_value = 0 result = self.replicator._setup_replicator_group( - self._ssh, group_name=group_name) - - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self._ssh, - "getent group %s > /dev/null && " - "echo 1 || echo 0" % - replicator_module.REPLICATOR_GROUP_NAME), - mock.call(self._ssh, - "sudo groupadd %s" % group_name, get_pty=True), - mock.call(self._ssh, "sudo usermod -aG %s %s" % ( - replicator_module.REPLICATOR_GROUP_NAME, - self.conn_info["username"]), get_pty=True)]) + self._ssh, group_name=group_name + ) + + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self._ssh, + "getent group %s > /dev/null && " + "echo 1 || echo 0" % replicator_module.REPLICATOR_GROUP_NAME, + ), + mock.call(self._ssh, "sudo groupadd %s" % group_name, get_pty=True), + mock.call( + self._ssh, + "sudo usermod -aG %s %s" + % ( + replicator_module.REPLICATOR_GROUP_NAME, + self.conn_info["username"], + ), + get_pty=True, + ), + ] + ) self.assertFalse(result) @@ -865,8 +949,9 @@ def test__setup_replicator_user(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.assert_called_once_with( self._ssh, - "getent passwd %s > /dev/null && echo 1 || echo 0" % - replicator_module.REPLICATOR_USERNAME) + "getent passwd %s > /dev/null && echo 1 || echo 0" + % replicator_module.REPLICATOR_USERNAME, + ) @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') def test__setup_replicator_user_no_user(self, mock_exec_ssh_cmd): @@ -874,19 +959,29 @@ def test__setup_replicator_user_no_user(self, mock_exec_ssh_cmd): self.replicator._setup_replicator_user(self._ssh) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self._ssh, - "getent passwd %s > /dev/null && " - "echo 1 || echo 0" % - replicator_module.REPLICATOR_USERNAME), - mock.call(self._ssh, - "sudo useradd -m -s /bin/bash -g %s %s" % - (replicator_module.REPLICATOR_USERNAME, - replicator_module.REPLICATOR_GROUP_NAME), - get_pty=True), - mock.call(self._ssh, - "sudo usermod -aG disk %s" % - replicator_module.REPLICATOR_USERNAME, get_pty=True)]) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self._ssh, + "getent passwd %s > /dev/null && " + "echo 1 || echo 0" % replicator_module.REPLICATOR_USERNAME, + ), + mock.call( + self._ssh, + "sudo useradd -m -s /bin/bash -g %s %s" + % ( + replicator_module.REPLICATOR_USERNAME, + replicator_module.REPLICATOR_GROUP_NAME, + ), + get_pty=True, + ), + mock.call( + self._ssh, + "sudo usermod -aG disk %s" % replicator_module.REPLICATOR_USERNAME, + get_pty=True, + ), + ] + ) @mock.patch.object(replicator_module.utils, 'create_service') def test__exec_replicator_cmd(self, mock_create_service): @@ -896,25 +991,34 @@ def test__exec_replicator_cmd(self, mock_create_service): "srv_key": mock.sentinel.srv_key, } original_exec_replicator = testutils.get_wrapped_function( - self.replicator._exec_replicator) + self.replicator._exec_replicator + ) original_exec_replicator( - self.replicator, self._ssh, self.conn_info['port'], certs, - mock.sentinel.state) + self.replicator, + self._ssh, + self.conn_info['port'], + certs, + mock.sentinel.state, + ) mock_create_service.assert_called_once_with( - self._ssh, mock.ANY, + self._ssh, + mock.ANY, replicator_module.REPLICATOR_SVC_NAME, - run_as=replicator_module.REPLICATOR_USERNAME) + run_as=replicator_module.REPLICATOR_USERNAME, + ) @mock.patch.object(replicator_module.utils, 'read_ssh_file') def test__fetch_remote_file(self, mock_read_ssh_file): with mock.patch('builtins.open', mock.mock_open()) as data: self.replicator._fetch_remote_file( - self._ssh, mock.sentinel.remote_file, mock.sentinel.local_file) + self._ssh, mock.sentinel.remote_file, mock.sentinel.local_file + ) data.assert_called_once_with(mock.sentinel.local_file, 'wb') data.return_value.write.assert_called_once_with( - mock_read_ssh_file.return_value) + mock_read_ssh_file.return_value + ) @mock.patch.object(os.path, 'isfile') @mock.patch.object(replicator_module.utils, 'test_ssh_path') @@ -923,20 +1027,23 @@ def test__setup_certificates(self, mock_test_ssh_path, mock_isfile): mock_isfile.return_value = True original_setup_certificates = testutils.get_wrapped_function( - self.replicator._setup_certificates) + self.replicator._setup_certificates + ) - result = original_setup_certificates( - self.replicator, self._ssh, self.conn_info) + result = original_setup_certificates(self.replicator, self._ssh, self.conn_info) expected_result = { - "local": - {"ca_cert": self.replicator._cert_dir + "/ca-cert.pem", - "client_cert": self.replicator._cert_dir + "/client-cert.pem", - "client_key": self.replicator._cert_dir + "/client-key.pem"}, - "remote": - {"ca_crt": replicator_module.REPLICATOR_DIR + "/ca-cert.pem", - "srv_crt": replicator_module.REPLICATOR_DIR + "/srv-cert.pem", - "srv_key": replicator_module.REPLICATOR_DIR + "/srv-key.pem"}} + "local": { + "ca_cert": self.replicator._cert_dir + "/ca-cert.pem", + "client_cert": self.replicator._cert_dir + "/client-cert.pem", + "client_key": self.replicator._cert_dir + "/client-key.pem", + }, + "remote": { + "ca_crt": replicator_module.REPLICATOR_DIR + "/ca-cert.pem", + "srv_crt": replicator_module.REPLICATOR_DIR + "/srv-cert.pem", + "srv_key": replicator_module.REPLICATOR_DIR + "/srv-key.pem", + }, + } self.assertEqual(result, expected_result) @@ -944,43 +1051,54 @@ def test__setup_certificates(self, mock_test_ssh_path, mock_isfile): @mock.patch.object(replicator_module.utils, 'test_ssh_path') @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') def test_setup_certificates_no_files_exist( - self, mock_exec_ssh_cmd, mock_test_ssh_path, - mock_fetch_remote_file): + self, mock_exec_ssh_cmd, mock_test_ssh_path, mock_fetch_remote_file + ): mock_test_ssh_path.return_value = False original_setup_certificates = testutils.get_wrapped_function( - self.replicator._setup_certificates) + self.replicator._setup_certificates + ) - original_setup_certificates( - self.replicator, self._ssh, self.conn_info) + original_setup_certificates(self.replicator, self._ssh, self.conn_info) expected_calls = [ - mock.call(self._ssh, "sudo mkdir -p %s" % - replicator_module.REPLICATOR_DIR, get_pty=True), - mock.call(self._ssh, "sudo %s gen-certs -output-dir" - % replicator_module.REPLICATOR_PATH + - " %s -certificate-hosts 127.0.0.1,%s" % - (replicator_module.REPLICATOR_DIR, self.conn_info['ip']), - get_pty=True), - mock.call(self._ssh, "sudo chown -R %s:%s %s" % - (replicator_module.REPLICATOR_USERNAME, - replicator_module.REPLICATOR_GROUP_NAME, - replicator_module.REPLICATOR_DIR), get_pty=True), - mock.call(self._ssh, "sudo chmod -R g+r %s" % - replicator_module.REPLICATOR_DIR, get_pty=True)] + mock.call( + self._ssh, + "sudo mkdir -p %s" % replicator_module.REPLICATOR_DIR, + get_pty=True, + ), + mock.call( + self._ssh, + "sudo %s gen-certs -output-dir" % replicator_module.REPLICATOR_PATH + + " %s -certificate-hosts 127.0.0.1,%s" + % (replicator_module.REPLICATOR_DIR, self.conn_info['ip']), + get_pty=True, + ), + mock.call( + self._ssh, + "sudo chown -R %s:%s %s" + % ( + replicator_module.REPLICATOR_USERNAME, + replicator_module.REPLICATOR_GROUP_NAME, + replicator_module.REPLICATOR_DIR, + ), + get_pty=True, + ), + mock.call( + self._ssh, + "sudo chmod -R g+r %s" % replicator_module.REPLICATOR_DIR, + get_pty=True, + ), + ] mock_exec_ssh_cmd.assert_has_calls(expected_calls) self.assertEqual(mock_fetch_remote_file.call_count, 3) - @mock.patch.object( - replicator_module.Replicator, '_get_replicator_state_file' - ) + @mock.patch.object(replicator_module.Replicator, '_get_replicator_state_file') @mock.patch.object(replicator_module.Replicator, '_copy_file') @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') @mock.patch.object(replicator_module.os, 'remove') - @mock.patch.object( - replicator_module.Replicator, '_parse_replicator_conn_info' - ) + @mock.patch.object(replicator_module.Replicator, '_parse_replicator_conn_info') @mock.patch.object(replicator_module.Replicator, '_copy_replicator_cmd') @mock.patch.object(replicator_module.Replicator, '_setup_replicator_group') @mock.patch.object(replicator_module.Replicator, '_reconnect_ssh') @@ -989,16 +1107,25 @@ def test_setup_certificates_no_files_exist( @mock.patch.object(replicator_module.Replicator, '_exec_replicator') @mock.patch.object(replicator_module.Replicator, 'start') def test__setup_replicator( - self, mock_start, mock_exec_replicator, mock_setup_certificates, - mock_setup_replicator_user, mock_reconnect_ssh, - mock_setup_replicator_group, mock_copy_replicator_cmd, - mock_parse_replicator_conn_info, mock_os_remove, - mock_exec_ssh_cmd, mock_copy_file, - mock_get_replicator_state_file): + self, + mock_start, + mock_exec_replicator, + mock_setup_certificates, + mock_setup_replicator_user, + mock_reconnect_ssh, + mock_setup_replicator_group, + mock_copy_replicator_cmd, + mock_parse_replicator_conn_info, + mock_os_remove, + mock_exec_ssh_cmd, + mock_copy_file, + mock_get_replicator_state_file, + ): mock_setup_replicator_group.return_value = True original_setup_replicator = testutils.get_wrapped_function( - self.replicator._setup_replicator) + self.replicator._setup_replicator + ) result = original_setup_replicator(self.replicator, self._ssh) @@ -1006,37 +1133,41 @@ def test__setup_replicator( mock_copy_file.assert_called_once_with( self._ssh, mock_get_replicator_state_file.return_value, - replicator_module.REPLICATOR_STATE) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call( - self._ssh, - "sudo chmod 755 %s" % replicator_module.REPLICATOR_STATE, - get_pty=True - ), - mock.call( - self._ssh, - "sudo chcon -t bin_t /usr/bin/replicator", - get_pty=True - ), - ]) + replicator_module.REPLICATOR_STATE, + ) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self._ssh, + "sudo chmod 755 %s" % replicator_module.REPLICATOR_STATE, + get_pty=True, + ), + mock.call( + self._ssh, "sudo chcon -t bin_t /usr/bin/replicator", get_pty=True + ), + ] + ) mock_os_remove.assert_called_once_with( - mock_get_replicator_state_file.return_value) + mock_get_replicator_state_file.return_value + ) mock_parse_replicator_conn_info.assert_called_once_with( - self.replicator._conn_info) + self.replicator._conn_info + ) mock_copy_replicator_cmd.assert_called_once_with(self._ssh) mock_setup_replicator_group.assert_called_once_with( - self._ssh, - group_name=replicator_module.REPLICATOR_GROUP_NAME) + self._ssh, group_name=replicator_module.REPLICATOR_GROUP_NAME + ) mock_reconnect_ssh.assert_not_called() mock_setup_replicator_user.assert_called_once_with(self._ssh) mock_setup_certificates.assert_called_once_with( - self._ssh, - mock_parse_replicator_conn_info.return_value) + self._ssh, mock_parse_replicator_conn_info.return_value + ) mock_exec_replicator.assert_called_once_with( self._ssh, mock_parse_replicator_conn_info.return_value['port'], mock_setup_certificates.return_value['remote'], - replicator_module.REPLICATOR_STATE) + replicator_module.REPLICATOR_STATE, + ) mock_start.assert_called_once() self.assertEqual(result, mock_setup_certificates.return_value['local']) @@ -1048,25 +1179,21 @@ def test__change_binary_se_context(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.assert_called_once_with( self._ssh, "sudo chcon -t bin_t %s" % replicator_module.REPLICATOR_PATH, - get_pty=True) + get_pty=True, + ) @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') def test__change_binary_se_context_with_exception(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.side_effect = exception.CoriolisException() - with self.assertLogs('coriolis.providers.replicator', - level=logging.WARN): + with self.assertLogs('coriolis.providers.replicator', level=logging.WARN): self.replicator._change_binary_se_context(self._ssh) - @mock.patch.object( - replicator_module.Replicator, '_get_replicator_state_file' - ) + @mock.patch.object(replicator_module.Replicator, '_get_replicator_state_file') @mock.patch.object(replicator_module.Replicator, '_copy_file') @mock.patch.object(replicator_module.utils, 'exec_ssh_cmd') @mock.patch.object(replicator_module.os, 'remove') - @mock.patch.object( - replicator_module.Replicator, '_parse_replicator_conn_info' - ) + @mock.patch.object(replicator_module.Replicator, '_parse_replicator_conn_info') @mock.patch.object(replicator_module.Replicator, '_copy_replicator_cmd') @mock.patch.object(replicator_module.Replicator, '_setup_replicator_group') @mock.patch.object(replicator_module.Replicator, '_reconnect_ssh') @@ -1075,12 +1202,20 @@ def test__change_binary_se_context_with_exception(self, mock_exec_ssh_cmd): @mock.patch.object(replicator_module.Replicator, '_exec_replicator') @mock.patch.object(replicator_module.Replicator, 'start') def test__setup_replicator_group_not_existed( - self, mock_start, mock_exec_replicator, mock_setup_certificates, - mock_setup_replicator_user, mock_reconnect_ssh, - mock_setup_replicator_group, mock_copy_replicator_cmd, - mock_parse_replicator_conn_info, mock_os_remove, - mock_exec_ssh_cmd, mock_copy_file, - mock_get_replicator_state_file): + self, + mock_start, + mock_exec_replicator, + mock_setup_certificates, + mock_setup_replicator_user, + mock_reconnect_ssh, + mock_setup_replicator_group, + mock_copy_replicator_cmd, + mock_parse_replicator_conn_info, + mock_os_remove, + mock_exec_ssh_cmd, + mock_copy_file, + mock_get_replicator_state_file, + ): mock_setup_replicator_group.return_value = False mock_reconnect_ssh.return_value = self._ssh @@ -1099,26 +1234,33 @@ def test__get_size_from_chunks(self): def test__find_vol_state(self): state = [ - {"device-name": mock.sentinel.device_name, - "other-data": mock.sentinel.other_data}, - {"device-name": mock.sentinel.device_name2, - "other-data": mock.sentinel.other_data2}, + { + "device-name": mock.sentinel.device_name, + "other-data": mock.sentinel.other_data, + }, + { + "device-name": mock.sentinel.device_name2, + "other-data": mock.sentinel.other_data2, + }, ] expected_result = { "device-name": mock.sentinel.device_name, - "other-data": mock.sentinel.other_data + "other-data": mock.sentinel.other_data, } - result = self.replicator._find_vol_state( - mock.sentinel.device_name, state) + result = self.replicator._find_vol_state(mock.sentinel.device_name, state) self.assertEqual(expected_result, result) def test__find_vol_state_not_found(self): state = [ - {"device-name": mock.sentinel.device_name, - "other-data": mock.sentinel.other_data}, - {"device-name": mock.sentinel.device_name2, - "other-data": mock.sentinel.other_data2}, + { + "device-name": mock.sentinel.device_name, + "other-data": mock.sentinel.other_data, + }, + { + "device-name": mock.sentinel.device_name2, + "other-data": mock.sentinel.other_data2, + }, ] result = self.replicator._find_vol_state("nonexistent_device", state) @@ -1136,35 +1278,50 @@ def test__verify_disk_checksum(self): mock_get_disk_size = self.replicator._cli.get_disk_size mock_get_disk_size.assert_called_once_with("sdb") mock_destination.get_disk_checksum.assert_called_once_with( - "sha256", end_offset=mock_get_disk_size.return_value) + "sha256", end_offset=mock_get_disk_size.return_value + ) def test__verify_disk_checksum_value_mismatch(self): self.replicator._cli.get_disk_checksum.return_value = { - "checksum": "abc123", "algorithm": "sha256"} + "checksum": "abc123", + "algorithm": "sha256", + } mock_destination = mock.MagicMock() mock_destination.get_disk_checksum.return_value = { - "checksum": "different", "algorithm": "sha256"} + "checksum": "different", + "algorithm": "sha256", + } self.assertRaises( exception.CoriolisException, self.replicator._verify_disk_checksum, - "sdb", mock_destination) + "sdb", + mock_destination, + ) def test__verify_disk_checksum_algorithm_mismatch(self): self.replicator._cli.get_disk_checksum.return_value = { - "checksum": "abc123", "algorithm": "sha256"} + "checksum": "abc123", + "algorithm": "sha256", + } mock_destination = mock.MagicMock() mock_destination.get_disk_checksum.return_value = { - "checksum": "abc123", "algorithm": "xxhash"} + "checksum": "abc123", + "algorithm": "xxhash", + } self.assertRaises( exception.CoriolisException, self.replicator._verify_disk_checksum, - "sdb", mock_destination) + "sdb", + mock_destination, + ) def test__verify_disk_checksum_not_supported(self): self.replicator._cli.get_disk_checksum.return_value = { - "checksum": "abc123", "algorithm": "sha256"} + "checksum": "abc123", + "algorithm": "sha256", + } mock_destination = mock.MagicMock() mock_destination.get_disk_checksum.return_value = None @@ -1174,26 +1331,23 @@ def test__verify_disk_checksum_not_supported(self): mock_get_disk_size = self.replicator._cli.get_disk_size mock_get_disk_size.assert_called_once_with("sdb") mock_destination.get_disk_checksum.assert_called_once_with( - "sha256", end_offset=mock_get_disk_size.return_value) + "sha256", end_offset=mock_get_disk_size.return_value + ) @mock.patch.object(replicator_module.Replicator, '_verify_disk_checksum') @mock.patch.object(replicator_module, 'Client') - def test_replicate_disks_calls_verify_checksum( - self, mock_Client, mock_verify): + def test_replicate_disks_calls_verify_checksum(self, mock_Client, mock_verify): self.replicator._cli = mock_Client.return_value - self.replicator._cli.get_changes.return_value = [ - {'length': 100, 'offset': 0}] + self.replicator._cli.get_changes.return_value = [{'length': 100, 'offset': 0}] self.replicator._volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb"}] - source_volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb"}] + {"disk_id": "test_disk", "disk_path": "/dev/sdb"} + ] + source_volumes_info = [{"disk_id": "test_disk", "disk_path": "/dev/sdb"}] self.replicator._repl_state = ['non-empty'] mock_destination = mock.MagicMock(spec=['seek', 'write']) - self.backup_writer.open.return_value.__enter__.return_value = ( - mock_destination) + self.backup_writer.open.return_value.__enter__.return_value = mock_destination - self.replicator.replicate_disks( - source_volumes_info, self.backup_writer, True) + self.replicator.replicate_disks(source_volumes_info, self.backup_writer, True) mock_verify.assert_called_once_with("sdb", mock_destination) @@ -1202,21 +1356,29 @@ def test_replicate_disks_calls_verify_checksum( def test_replicate_disks(self, mock_Client, mock_verify): self.replicator._cli = mock_Client.return_value self.replicator._cli.get_changes.return_value = [ - {'length': 100, 'offset': 0}, {'length': 200, 'offset': 100}] + {'length': 100, 'offset': 0}, + {'length': 200, 'offset': 100}, + ] self.replicator._volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True}] + {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True} + ] source_volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True}] + {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True} + ] self.replicator._repl_state = ['non-empty'] result = self.replicator.replicate_disks( - source_volumes_info, self.backup_writer) + source_volumes_info, self.backup_writer + ) self.backup_writer.open.assert_called_with("", "test_disk") - self.replicator._cli.download_chunk.assert_has_calls([ - mock.call("sdb", {"length": 100, "offset": 0}), - mock.call("sdb", {"length": 200, "offset": 100})]) + self.replicator._cli.download_chunk.assert_has_calls( + [ + mock.call("sdb", {"length": 100, "offset": 0}), + mock.call("sdb", {"length": 200, "offset": 100}), + ] + ) self.assertEqual(result, self.replicator._repl_state) self.assertEqual(result, self.replicator._cli.get_status.return_value) @@ -1227,46 +1389,49 @@ def test_replicate_disks_with_nonexistent_volume(self): {"disk_id": "vol2", "disk_path": "/dev/sdb2"}, ] source_volumes_info = [ - {"disk_id": "vol4", "disk_path": "/dev/sdb4", - "other-data": "data4"}, + {"disk_id": "vol4", "disk_path": "/dev/sdb4", "other-data": "data4"}, ] - self.assertRaises(exception.CoriolisException, - self.replicator.replicate_disks, - source_volumes_info, self.backup_writer) + self.assertRaises( + exception.CoriolisException, + self.replicator.replicate_disks, + source_volumes_info, + self.backup_writer, + ) @mock.patch.object(replicator_module.Replicator, '_verify_disk_checksum') @mock.patch.object(replicator_module.Replicator, '_find_vol_state') @mock.patch.object(replicator_module, 'Client') - def test_replicate_disks_initial_sync(self, mock_Client, - mock_find_vol_state, mock_verify): + def test_replicate_disks_initial_sync( + self, mock_Client, mock_find_vol_state, mock_verify + ): self.replicator._cli = mock_Client.return_value self.replicator._cli.get_changes.return_value = [ - {'length': 100, 'offset': 0}, {'length': 200, 'offset': 100}] - self.replicator._volumes_info = [ - {"disk_id": "test_disk", "zeroed": True}] + {'length': 100, 'offset': 0}, + {'length': 200, 'offset': 100}, + ] + self.replicator._volumes_info = [{"disk_id": "test_disk", "zeroed": True}] source_volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True}] + {"disk_id": "test_disk", "disk_path": "/dev/sdb", "zeroed": True} + ] self.replicator._repl_state = None - self.replicator.replicate_disks( - source_volumes_info, self.backup_writer) + self.replicator.replicate_disks(source_volumes_info, self.backup_writer) mock_find_vol_state.assert_called_with( - "sdb", self.replicator._cli.get_status.return_value) - self.replicator._cli.get_chunks.assert_called_with( - "sdb", skip_zeros=True) + "sdb", self.replicator._cli.get_status.return_value + ) + self.replicator._cli.get_chunks.assert_called_with("sdb", skip_zeros=True) def test_replicate_disks_no_chunks(self): self.replicator._cli.get_changes.return_value = [] - self.replicator._volumes_info = [ - {"disk_id": "test_disk", "zeroed": False}] - source_volumes_info = [ - {"disk_id": "test_disk", "disk_path": "/dev/sdb"}] + self.replicator._volumes_info = [{"disk_id": "test_disk", "zeroed": False}] + source_volumes_info = [{"disk_id": "test_disk", "disk_path": "/dev/sdb"}] result = self.replicator.replicate_disks( - source_volumes_info, self.backup_writer) + source_volumes_info, self.backup_writer + ) self.backup_writer.open.assert_not_called() self.assertEqual(result, self.replicator._repl_state) @@ -1276,17 +1441,18 @@ def test_replicate_disks_no_chunks(self): def test__download_full_disk(self, mock_Client, mock_open): self.replicator._cli = mock_Client.return_value self.replicator._cli.raw_disk_uri.return_value = mock.sentinel.uri - self.replicator._cli.get_chunks.return_value = [ - {'offset': 0, 'length': 100} + self.replicator._cli.get_chunks.return_value = [{'offset': 0, 'length': 100}] + self.replicator._cli._cli.get.return_value.__enter__.return_value.iter_content.return_value = [ + b'chunk1', + b'chunk2', + b'chunk3', ] - self.replicator._cli._cli.get.return_value.__enter__.\ - return_value.iter_content.return_value = [ - b'chunk1', b'chunk2', b'chunk3'] self.replicator._repl_state = { self.disk: {'disk_path': '/dev/sdb', 'disk_id': self.disk} } - self.replicator._event_manager.add_percentage_step.\ - return_value = mock.sentinel.perc_step + self.replicator._event_manager.add_percentage_step.return_value = ( + mock.sentinel.perc_step + ) mock_open.return_value.write.side_effect = lambda chunk: len(chunk) self.replicator._download_full_disk(self.disk, self.path) @@ -1295,45 +1461,50 @@ def test__download_full_disk(self, mock_Client, mock_open): mock_open.assert_called_once_with(self.path, 'wb') file_handle = mock_open.return_value - file_handle.write.assert_has_calls([ - mock.call(b'chunk1'), - mock.call(b'chunk2'), - mock.call(b'chunk3')] + file_handle.write.assert_has_calls( + [mock.call(b'chunk1'), mock.call(b'chunk2'), mock.call(b'chunk3')] + ) + self.replicator._event_manager.set_percentage_step.assert_has_calls( + [ + mock.call(mock.sentinel.perc_step, 6), + mock.call(mock.sentinel.perc_step, 12), + mock.call(mock.sentinel.perc_step, 18), + ] ) - self.replicator._event_manager.set_percentage_step.assert_has_calls([ - mock.call(mock.sentinel.perc_step, 6), - mock.call(mock.sentinel.perc_step, 12), - mock.call(mock.sentinel.perc_step, 18)]) @mock.patch('builtins.open', new_callable=mock.mock_open) @mock.patch.object(replicator_module, 'Client') @mock.patch.object(replicator_module.Replicator, '_get_size_from_chunks') def test__download_sparse_disk( - self, mock_get_size_from_chunks, mock_Client, mock_open): + self, mock_get_size_from_chunks, mock_Client, mock_open + ): self.replicator._cli = mock_Client.return_value chunks = [{'offset': 0, 'length': 100}] - self.replicator._download_sparse_disk( - self.disk, mock.sentinel.path, chunks) + self.replicator._download_sparse_disk(self.disk, mock.sentinel.path, chunks) mock_open.assert_called_once_with(mock.sentinel.path, 'wb') file_handle = mock_open.return_value file_handle.truncate.assert_called_once_with( - self.replicator._cli.get_disk_size.return_value) + self.replicator._cli.get_disk_size.return_value + ) file_handle.seek.assert_called_once_with(0) file_handle.write.assert_called_once_with( - self.replicator._cli.download_chunk.return_value) + self.replicator._cli.download_chunk.return_value + ) self.replicator._event_manager.add_percentage_step.assert_called_once() self.replicator._event_manager.set_percentage_step.assert_called_with( - self.replicator._event_manager.add_percentage_step.return_value, 1) + self.replicator._event_manager.add_percentage_step.return_value, 1 + ) @mock.patch.object(replicator_module.Replicator, '_download_full_disk') @mock.patch.object(replicator_module.Replicator, '_download_sparse_disk') @mock.patch.object(replicator_module, 'Client') - def test_download_disk(self, mock_Client, mock_download_sparse_disk, - mock_download_full_disk): + def test_download_disk( + self, mock_Client, mock_download_sparse_disk, mock_download_full_disk + ): self.replicator._cli = mock_Client.return_value disk = "/dev/test_disk" self.replicator._cli.get_chunks.return_value = [] @@ -1341,27 +1512,28 @@ def test_download_disk(self, mock_Client, mock_download_sparse_disk, self.replicator.download_disk(disk, self.path) self.replicator._cli.get_chunks.assert_called_once_with( - device=self.disk, skip_zeros=True) - mock_download_full_disk.assert_called_once_with( - self.disk, self.path) + device=self.disk, skip_zeros=True + ) + mock_download_full_disk.assert_called_once_with(self.disk, self.path) mock_download_sparse_disk.assert_not_called() @mock.patch.object(replicator_module, 'Client') @mock.patch.object(replicator_module.Replicator, '_download_full_disk') @mock.patch.object(replicator_module.Replicator, '_download_sparse_disk') def test_download_disk_with_chunks( - self, mock_download_sparse_disk, mock_download_full_disk, - mock_Client): + self, mock_download_sparse_disk, mock_download_full_disk, mock_Client + ): self.replicator._cli = mock_Client.return_value disk = "/dev/test_disk" - self.replicator._cli.get_chunks.return_value = [ - {'offset': 0, 'length': 100}] + self.replicator._cli.get_chunks.return_value = [{'offset': 0, 'length': 100}] self.replicator.download_disk(disk, self.path) self.replicator._cli.get_chunks.assert_called_once_with( - device=self.disk, skip_zeros=True) + device=self.disk, skip_zeros=True + ) mock_download_full_disk.assert_not_called() mock_download_sparse_disk.assert_called_once_with( - self.disk, self.path, self.replicator._cli.get_chunks.return_value) + self.disk, self.path, self.replicator._cli.get_chunks.return_value + ) diff --git a/coriolis/tests/regions/test_api.py b/coriolis/tests/regions/test_api.py index 42815728..c591cc09 100644 --- a/coriolis/tests/regions/test_api.py +++ b/coriolis/tests/regions/test_api.py @@ -22,12 +22,11 @@ def setUp(self): def test_create(self): description = mock.sentinel.description - result = self.api.create(self.ctxt, self.region_name, description, - enabled=True) + result = self.api.create(self.ctxt, self.region_name, description, enabled=True) self.rpc_client.create_region.assert_called_once_with( - self.ctxt, self.region_name, description=description, - enabled=True) + self.ctxt, self.region_name, description=description, enabled=True + ) self.assertEqual(result, self.rpc_client.create_region.return_value) def test_update(self): @@ -36,14 +35,14 @@ def test_update(self): result = self.api.update(self.ctxt, self.region_id, updated_values) self.rpc_client.update_region.assert_called_once_with( - self.ctxt, self.region_id, updated_values=updated_values) + self.ctxt, self.region_id, updated_values=updated_values + ) self.assertEqual(result, self.rpc_client.update_region.return_value) def test_delete(self): self.api.delete(self.ctxt, self.region_id) - self.rpc_client.delete_region.assert_called_once_with( - self.ctxt, self.region_id) + self.rpc_client.delete_region.assert_called_once_with(self.ctxt, self.region_id) def test_get_regions(self): result = self.api.get_regions(self.ctxt) @@ -54,6 +53,5 @@ def test_get_regions(self): def test_get_region(self): result = self.api.get_region(self.ctxt, self.region_id) - self.rpc_client.get_region.assert_called_once_with( - self.ctxt, self.region_id) + self.rpc_client.get_region.assert_called_once_with(self.ctxt, self.region_id) self.assertEqual(result, self.rpc_client.get_region.return_value) diff --git a/coriolis/tests/scheduler/filters/test_base.py b/coriolis/tests/scheduler/filters/test_base.py index c4463618..a7acc1a6 100644 --- a/coriolis/tests/scheduler/filters/test_base.py +++ b/coriolis/tests/scheduler/filters/test_base.py @@ -19,34 +19,26 @@ def test_is_service_acceptable(self): self.service_filter.rate_service = mock.Mock() self.service_filter.rate_service.return_value = 50 - result = self.service_filter.is_service_acceptable( - mock.sentinel.service) + result = self.service_filter.is_service_acceptable(mock.sentinel.service) - self.service_filter.rate_service.assert_called_once_with( - mock.sentinel.service - ) + self.service_filter.rate_service.assert_called_once_with(mock.sentinel.service) self.assertTrue(result) def test_is_service_acceptable_false(self): self.service_filter.rate_service = mock.Mock() self.service_filter.rate_service.return_value = 0 - result = self.service_filter.is_service_acceptable( - mock.sentinel.service) + result = self.service_filter.is_service_acceptable(mock.sentinel.service) - self.service_filter.rate_service.assert_called_once_with( - mock.sentinel.service - ) + self.service_filter.rate_service.assert_called_once_with(mock.sentinel.service) self.assertFalse(result) def test_filter_services(self): self.service_filter.is_service_acceptable = mock.Mock() - self.service_filter.is_service_acceptable.side_effect = [ - True, False, True] + self.service_filter.is_service_acceptable.side_effect = [True, False, True] - result = self.service_filter.filter_services([mock.sentinel.service1, - mock.sentinel.service2, - mock.sentinel.service3]) + result = self.service_filter.filter_services( + [mock.sentinel.service1, mock.sentinel.service2, mock.sentinel.service3] + ) - self.assertEqual(result, [mock.sentinel.service1, - mock.sentinel.service3]) + self.assertEqual(result, [mock.sentinel.service1, mock.sentinel.service3]) diff --git a/coriolis/tests/scheduler/filters/test_trivial_filters.py b/coriolis/tests/scheduler/filters/test_trivial_filters.py index eac435cd..05020719 100644 --- a/coriolis/tests/scheduler/filters/test_trivial_filters.py +++ b/coriolis/tests/scheduler/filters/test_trivial_filters.py @@ -18,32 +18,48 @@ def setUp(self): self.regions = [mock.sentinel.region1, mock.sentinel.region2] self.any_region = False self.regions_filter = trivial_filters.RegionsFilter( - self.regions, self.any_region) + self.regions, self.any_region + ) def test__repr__(self): result = repr(self.regions_filter) - self.assertEqual(result, "" % (self.regions, - self.any_region)) + self.assertEqual( + result, + "" % (self.regions, self.any_region), + ) @ddt.data( (None, [mock.sentinel.region1, mock.sentinel.region2], 100, False), - ([mock.sentinel.region1, mock.sentinel.region2], - [mock.sentinel.region1, mock.sentinel.region2], 100, False), + ( + [mock.sentinel.region1, mock.sentinel.region2], + [mock.sentinel.region1, mock.sentinel.region2], + 100, + False, + ), ([mock.sentinel.region1, mock.sentinel.region2], [], 0, False), - ([mock.sentinel.region1, mock.sentinel.region2], - [mock.sentinel.region1], 0, False), - ([mock.sentinel.region1, mock.sentinel.region2], - [mock.sentinel.region1], 100, True) + ( + [mock.sentinel.region1, mock.sentinel.region2], + [mock.sentinel.region1], + 0, + False, + ), + ( + [mock.sentinel.region1, mock.sentinel.region2], + [mock.sentinel.region1], + 100, + True, + ), ) @ddt.unpack def test_rate_service(self, regions, mapped_regions, expected, any_region): self.regions_filter._regions = regions self.regions_filter._any_region = any_region mock_service = mock.Mock() - mock_service.mapped_regions = [mock.Mock(id=region) for region in - mapped_regions] + mock_service.mapped_regions = [ + mock.Mock(id=region) for region in mapped_regions + ] result = self.regions_filter.rate_service(mock_service) @@ -109,27 +125,29 @@ class ProviderTypesFilterTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ProviderTypesFilterTestCase, self).setUp() self.service = mock.Mock() - self.provider_types = [mock.sentinel.provider_type1, - mock.sentinel.provider_type2] + self.provider_types = [ + mock.sentinel.provider_type1, + mock.sentinel.provider_type2, + ] self.provider_types_filter = trivial_filters.ProviderTypesFilter( - self.provider_types) + self.provider_types + ) def test__repr__(self): result = repr(self.provider_types_filter) - self.assertEqual(result, - "" % - self.provider_types) + self.assertEqual( + result, + "" % self.provider_types, + ) @ddt.data( ({'platform1': ['type1']}, {'platform1': {'types': ['type1']}}, 100), ({'platform1': ['type1']}, {'platform2': {'types': ['type1']}}, 0), - ({'platform1': ['type1', 'type2']}, - {'platform1': {'types': ['type1']}}, 0) + ({'platform1': ['type1', 'type2']}, {'platform1': {'types': ['type1']}}, 0), ) @ddt.unpack def test_rate_service(self, provider_requirements, providers, expected): - self.provider_types_filter.\ - _provider_requirements = provider_requirements + self.provider_types_filter._provider_requirements = provider_requirements self.service.providers = providers result = self.provider_types_filter.rate_service(self.service) diff --git a/coriolis/tests/scheduler/rpc/test_client.py b/coriolis/tests/scheduler/rpc/test_client.py index ddfb5c41..af49d297 100644 --- a/coriolis/tests/scheduler/rpc/test_client.py +++ b/coriolis/tests/scheduler/rpc/test_client.py @@ -6,8 +6,7 @@ import oslo_messaging -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception from coriolis.scheduler.rpc import client from coriolis.tasks import factory as tasks_factory from coriolis.tests import test_base @@ -23,12 +22,12 @@ def setUp(self): self.origin_endpoint = { 'id': 'origin_id', 'mapped_regions': [{'id': 'region1'}, {'id': 'region2'}], - 'type': 'origin_type' + 'type': 'origin_type', } self.destination_endpoint = { 'id': 'destination_id', 'mapped_regions': [{'id': 'region3'}, {'id': 'region4'}], - 'type': 'destination_type' + 'type': 'destination_type', } @mock.patch('coriolis.scheduler.rpc.client.CONF') @@ -39,7 +38,8 @@ def test__init__(self, mock_target, mock_conf): result = client.SchedulerClient() mock_target.assert_called_once_with( - topic='coriolis_scheduler', version=client.VERSION) + topic='coriolis_scheduler', version=client.VERSION + ) self.assertEqual(result._target, mock_target.return_value) self.assertEqual(result._timeout, expected_timeout) @@ -68,18 +68,26 @@ def test_get_workers_for_specs(self, mock_call): enabled = mock.sentinel.enabled result = self.client.get_workers_for_specs( - ctxt, provider_requirements=provider_requirements, - region_sets=region_sets, enabled=enabled) + ctxt, + provider_requirements=provider_requirements, + region_sets=region_sets, + enabled=enabled, + ) mock_call.assert_called_once_with( - ctxt, 'get_workers_for_specs', region_sets=region_sets, - enabled=enabled, provider_requirements=provider_requirements) + ctxt, + 'get_workers_for_specs', + region_sets=region_sets, + enabled=enabled, + provider_requirements=provider_requirements, + ) self.assertEqual(result, mock_call.return_value) @mock.patch('random.choice') @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') - def test_get_any_worker_service(self, mock_get_workers_for_specs, - mock_random_choice): + def test_get_any_worker_service( + self, mock_get_workers_for_specs, mock_random_choice + ): ctxt = mock.sentinel.ctxt raise_if_none = mock.sentinel.raise_if_none mock_service = mock.MagicMock() @@ -87,18 +95,19 @@ def test_get_any_worker_service(self, mock_get_workers_for_specs, mock_random_choice.return_value = mock_service result = self.client.get_any_worker_service( - ctxt, raise_if_none=raise_if_none, random_choice=True) + ctxt, raise_if_none=raise_if_none, random_choice=True + ) mock_get_workers_for_specs.assert_called_once_with(ctxt) mock_random_choice.assert_called_once_with(mock_service) self.assertEqual(result, mock_service) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') - def test_get_any_worker_service_no_services_no_raise(self, - mock_get_workers): + def test_get_any_worker_service_no_services_no_raise(self, mock_get_workers): mock_get_workers.return_value = [] result = self.client.get_any_worker_service( - mock.sentinel.ctxt, raise_if_none=False) + mock.sentinel.ctxt, raise_if_none=False + ) self.assertIsNone(result) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') @@ -106,7 +115,9 @@ def test_get_any_worker_service_no_services(self, mock_get_workers): mock_get_workers.return_value = [] self.assertRaises( exception.NoWorkerServiceError, - self.client.get_any_worker_service, mock.sentinel.ctxt) + self.client.get_any_worker_service, + mock.sentinel.ctxt, + ) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') def test_get_any_worker_service_random_choice(self, mock_get_workers): @@ -115,7 +126,8 @@ def test_get_any_worker_service_random_choice(self, mock_get_workers): mock_get_workers.return_value = [service_mock1, service_mock2] result = self.client.get_any_worker_service( - mock.sentinel.ctxt, random_choice=True) + mock.sentinel.ctxt, random_choice=True + ) self.assertIsInstance(result, dict) @@ -128,20 +140,27 @@ def test_get_worker_service_for_specs(self, mock_call): raise_on_no_matches = mock.sentinel.raise_on_no_matches self.client.get_worker_service_for_specs( - ctxt, provider_requirements=provider_requirements, - region_sets=region_sets, enabled=enabled, - raise_on_no_matches=raise_on_no_matches) + ctxt, + provider_requirements=provider_requirements, + region_sets=region_sets, + enabled=enabled, + raise_on_no_matches=raise_on_no_matches, + ) mock_call.assert_called_once_with( - ctxt, 'get_workers_for_specs', region_sets=region_sets, - enabled=enabled, provider_requirements=provider_requirements) + ctxt, + 'get_workers_for_specs', + region_sets=region_sets, + enabled=enabled, + provider_requirements=provider_requirements, + ) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') - def test_get_worker_service_for_specs_no_services_no_raise( - self, mock_get_workers): + def test_get_worker_service_for_specs_no_services_no_raise(self, mock_get_workers): mock_get_workers.return_value = [] result = self.client.get_worker_service_for_specs( - mock.sentinel.ctxt, raise_on_no_matches=False) + mock.sentinel.ctxt, raise_on_no_matches=False + ) self.assertIsNone(result) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') @@ -149,60 +168,77 @@ def test_get_worker_service_for_specs_no_services(self, mock_get_workers): mock_get_workers.return_value = [] self.assertRaises( exception.NoSuitableWorkerServiceError, - self.client.get_worker_service_for_specs, mock.sentinel.ctxt) + self.client.get_worker_service_for_specs, + mock.sentinel.ctxt, + ) @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') @mock.patch('random.choice') def test_get_worker_service_for_specs_random_choice( - self, mock_random_choice, mock_get_workers): + self, mock_random_choice, mock_get_workers + ): service_mock1 = {'id': 'test_id1'} service_mock2 = {'id': 'test_id2'} mock_get_workers.return_value = [service_mock1, service_mock2] mock_random_choice.return_value = service_mock1 result = self.client.get_worker_service_for_specs( - mock.sentinel.ctxt, random_choice=True) + mock.sentinel.ctxt, random_choice=True + ) - mock_random_choice.assert_called_once_with([ - service_mock1, service_mock2]) + mock_random_choice.assert_called_once_with([service_mock1, service_mock2]) mock_get_workers.assert_called_once_with( - mock.sentinel.ctxt, provider_requirements=None, region_sets=None, - enabled=True) + mock.sentinel.ctxt, + provider_requirements=None, + region_sets=None, + enabled=True, + ) self.assertEqual(result, service_mock1) @mock.patch('random.choice') @mock.patch.object(client.SchedulerClient, 'get_workers_for_specs') def test_get_worker_service_for_specs_no_random_choice( - self, get_workers_for_specs, mock_random_choice): + self, get_workers_for_specs, mock_random_choice + ): mock_service = mock.MagicMock() get_workers_for_specs.return_value = [mock_service] result = self.client.get_worker_service_for_specs( - mock.sentinel.ctxt, random_choice=False) + mock.sentinel.ctxt, random_choice=False + ) mock_random_choice.assert_not_called() get_workers_for_specs.assert_called_once_with( - mock.sentinel.ctxt, provider_requirements=None, region_sets=None, - enabled=True) + mock.sentinel.ctxt, + provider_requirements=None, + region_sets=None, + enabled=True, + ) self.assertEqual(result, mock_service) @mock.patch.object(client.SchedulerClient, 'get_worker_service_for_specs') @mock.patch.object(tasks_factory, 'get_task_runner_class') def test_get_worker_service_for_task_different_platforms( - self, mock_get_task_runner_class, get_worker_service_for_specs): - for platform in [constants.TASK_PLATFORM_SOURCE, - constants.TASK_PLATFORM_DESTINATION, - constants.TASK_PLATFORM_BILATERAL]: - mock_get_task_runner_class.return_value.get_required_platform.\ - return_value = platform - mock_get_task_runner_class.return_value.\ - get_required_provider_types.return_value = { - constants.PROVIDER_PLATFORM_SOURCE: 'provider_type'} + self, mock_get_task_runner_class, get_worker_service_for_specs + ): + for platform in [ + constants.TASK_PLATFORM_SOURCE, + constants.TASK_PLATFORM_DESTINATION, + constants.TASK_PLATFORM_BILATERAL, + ]: + mock_get_task_runner_class.return_value.get_required_platform.return_value = platform + mock_get_task_runner_class.return_value.get_required_provider_types.return_value = { + constants.PROVIDER_PLATFORM_SOURCE: 'provider_type' + } get_worker_service_for_specs.return_value = {'id': 'test_id'} result = self.client.get_worker_service_for_task( - mock.sentinel.ctxt, self.task, self.origin_endpoint, - self.destination_endpoint, retry_period=0) + mock.sentinel.ctxt, + self.task, + self.origin_endpoint, + self.destination_endpoint, + retry_period=0, + ) self.assertEqual(result, {'id': 'test_id'}) self.assertIsInstance(result, dict) @@ -210,33 +246,45 @@ def test_get_worker_service_for_task_different_platforms( @mock.patch.object(client.SchedulerClient, 'get_worker_service_for_specs') @mock.patch.object(tasks_factory, 'get_task_runner_class') def test_get_worker_service_for_task_retry( - self, mock_get_task_runner_class, mock_get_worker_service): - mock_get_task_runner_class.return_value.get_required_platform.\ - return_value = constants.TASK_PLATFORM_SOURCE - mock_get_task_runner_class.return_value.get_required_provider_types.\ - return_value = {constants.PROVIDER_PLATFORM_DESTINATION: - 'provider_type'} + self, mock_get_task_runner_class, mock_get_worker_service + ): + mock_get_task_runner_class.return_value.get_required_platform.return_value = ( + constants.TASK_PLATFORM_SOURCE + ) + mock_get_task_runner_class.return_value.get_required_provider_types.return_value = { + constants.PROVIDER_PLATFORM_DESTINATION: 'provider_type' + } mock_get_worker_service.side_effect = [Exception(), {'id': 'test_id'}] - with self.assertLogs('coriolis.scheduler.rpc.client', - level=logging.WARN): + with self.assertLogs('coriolis.scheduler.rpc.client', level=logging.WARN): self.client.get_worker_service_for_task( - mock.sentinel.ctxt, self.task, self.origin_endpoint, - self.destination_endpoint, retry_period=0) + mock.sentinel.ctxt, + self.task, + self.origin_endpoint, + self.destination_endpoint, + retry_period=0, + ) @mock.patch.object(client.SchedulerClient, 'get_worker_service_for_specs') @mock.patch.object(tasks_factory, 'get_task_runner_class') def test_get_worker_service_for_task_no_suitable_worker( - self, mock_get_task_runner_class, mock_get_worker_service): - mock_get_task_runner_class.return_value.get_required_platform.\ - return_value = constants.TASK_PLATFORM_SOURCE - mock_get_task_runner_class.return_value.get_required_provider_types.\ - return_value = {constants.PROVIDER_PLATFORM_SOURCE: 'type'} - mock_get_worker_service.side_effect = [ - exception.NoSuitableWorkerServiceError()] + self, mock_get_task_runner_class, mock_get_worker_service + ): + mock_get_task_runner_class.return_value.get_required_platform.return_value = ( + constants.TASK_PLATFORM_SOURCE + ) + mock_get_task_runner_class.return_value.get_required_provider_types.return_value = { + constants.PROVIDER_PLATFORM_SOURCE: 'type' + } + mock_get_worker_service.side_effect = [exception.NoSuitableWorkerServiceError()] self.assertRaises( exception.NoSuitableWorkerServiceError, - self.client.get_worker_service_for_task, mock.sentinel.ctxt, - self.task, self.origin_endpoint, self.destination_endpoint, - retry_period=0, retry_count=0) + self.client.get_worker_service_for_task, + mock.sentinel.ctxt, + self.task, + self.origin_endpoint, + self.destination_endpoint, + retry_period=0, + retry_count=0, + ) diff --git a/coriolis/tests/scheduler/rpc/test_server.py b/coriolis/tests/scheduler/rpc/test_server.py index bdee12eb..16d8a053 100644 --- a/coriolis/tests/scheduler/rpc/test_server.py +++ b/coriolis/tests/scheduler/rpc/test_server.py @@ -6,14 +6,11 @@ import ddt -from coriolis import constants +from coriolis import constants, exception, utils from coriolis.db import api as db_api -from coriolis import exception from coriolis.scheduler.filters import trivial_filters from coriolis.scheduler.rpc import server -from coriolis.tests import test_base -from coriolis.tests import testutils -from coriolis import utils +from coriolis.tests import test_base, testutils @ddt.ddt @@ -33,20 +30,22 @@ def test_get_diagnostics(self, mock_get_diagnostics_info): @mock.patch.object(trivial_filters, 'TopicFilter', autospec=True) @mock.patch.object(db_api, 'get_services') - def test_get_all_worker_services(self, mock_get_services, - mock_topic_filter_cls): + def test_get_all_worker_services(self, mock_get_services, mock_topic_filter_cls): mock_get_services.return_value = mock.sentinel.services - mock_topic_filter_cls.return_value.filter_services.return_value = \ + mock_topic_filter_cls.return_value.filter_services.return_value = ( mock.sentinel.filtered_services + ) result = self.server._get_all_worker_services(mock.sentinel.context) mock_get_services.assert_called_once_with(mock.sentinel.context) mock_topic_filter_cls.assert_called_once_with( - constants.WORKER_MAIN_MESSAGING_TOPIC) - mock_topic_filter_cls.return_value.filter_services.\ - assert_called_once_with(mock.sentinel.services) + constants.WORKER_MAIN_MESSAGING_TOPIC + ) + mock_topic_filter_cls.return_value.filter_services.assert_called_once_with( + mock.sentinel.services + ) self.assertEqual(result, mock.sentinel.filtered_services) @@ -54,19 +53,19 @@ def test_get_all_worker_services(self, mock_get_services, def test_get_all_worker_services_no_services(self, mock_get_services): mock_get_services.return_value = [] - self.assertRaises(exception.NoWorkerServiceError, - self.server._get_all_worker_services, - mock.sentinel.context) + self.assertRaises( + exception.NoWorkerServiceError, + self.server._get_all_worker_services, + mock.sentinel.context, + ) mock_get_services.assert_called_once_with(mock.sentinel.context) def test_get_weighted_filtered_services_no_filters(self): services = [mock.Mock(id=1), mock.Mock(id=2)] - with self.assertLogs('coriolis.scheduler.rpc.server', - level=logging.WARN): - result = self.server._get_weighted_filtered_services(services, - None) + with self.assertLogs('coriolis.scheduler.rpc.server', level=logging.WARN): + result = self.server._get_weighted_filtered_services(services, None) expected_result = [(services[0], 100), (services[1], 100)] self.assertEqual(result, expected_result) @@ -76,9 +75,12 @@ def test_get_weighted_filtered_services_with_filters_reject(self): filters[0].rate_service.return_value = 50 filters[1].rate_service.return_value = 0 - self.assertRaises(exception.NoSuitableWorkerServiceError, - self.server._get_weighted_filtered_services, - services, filters) + self.assertRaises( + exception.NoSuitableWorkerServiceError, + self.server._get_weighted_filtered_services, + services, + filters, + ) def test_get_weighted_filtered_services_with_filters_accept(self): services = [mock.Mock(id=1), mock.Mock(id=2)] @@ -86,8 +88,7 @@ def test_get_weighted_filtered_services_with_filters_accept(self): filters[0].rate_service.return_value = 50 filters[1].rate_service.return_value = 100 - result = self.server._get_weighted_filtered_services(services, - filters) + result = self.server._get_weighted_filtered_services(services, filters) expected_result = [(services[0], 150), (services[1], 150)] self.assertEqual(result, expected_result) @@ -99,8 +100,7 @@ def test__filter_regions_check_all_exist_false(self, mock_get_regions): ] region_ids = ['region1', 'region2'] - result = self.server._filter_regions(None, region_ids, - check_all_exist=False) + result = self.server._filter_regions(None, region_ids, check_all_exist=False) self.assertEqual(result, mock_get_regions.return_value) @@ -117,8 +117,7 @@ def test__filter_regions_all_disabled(self, mock_get_regions): self.assertEqual(result, mock_get_regions.return_value) @mock.patch.object(db_api, 'get_regions') - def test__filter_regions_some_enabled_some_disabled(self, - mock_get_regions): + def test__filter_regions_some_enabled_some_disabled(self, mock_get_regions): mock_get_regions.return_value = [ mock.Mock(id='region1', enabled=True), mock.Mock(id='region2', enabled=False), @@ -137,8 +136,9 @@ def test__filter_regions_some_missing(self, mock_get_regions): ] region_ids = ['region1', 'region2', 'region3'] - self.assertRaises(exception.RegionNotFound, - self.server._filter_regions, None, region_ids) + self.assertRaises( + exception.RegionNotFound, self.server._filter_regions, None, region_ids + ) @mock.patch.object(trivial_filters, 'ProviderTypesFilter', autospec=True) @mock.patch.object(trivial_filters, 'RegionsFilter', autospec=True) @@ -146,25 +146,21 @@ def test__filter_regions_some_missing(self, mock_get_regions): @mock.patch.object( server.SchedulerServerEndpoint, '_get_weighted_filtered_services' ) - @mock.patch.object( - server.SchedulerServerEndpoint, '_filter_regions' - ) - @mock.patch.object( - server.SchedulerServerEndpoint, '_get_all_worker_services' - ) + @mock.patch.object(server.SchedulerServerEndpoint, '_filter_regions') + @mock.patch.object(server.SchedulerServerEndpoint, '_get_all_worker_services') @ddt.file_data("data/get_workers_for_specs_config.yaml") @ddt.unpack def test_get_workers_for_specs( - self, - mock_get_all_worker_services, - mock_filter_regions, - mock_get_weighted_filtered_services, - mock_enabled_filter_cls, - mock_regions_filter_cls, - mock_provider_types_filter_cls, - config, - expected_result, - expected_exception, + self, + mock_get_all_worker_services, + mock_filter_regions, + mock_get_weighted_filtered_services, + mock_enabled_filter_cls, + mock_regions_filter_cls, + mock_provider_types_filter_cls, + config, + expected_result, + expected_exception, ): enabled = config.get("enabled", None) @@ -174,15 +170,13 @@ def test_get_workers_for_specs( # Convert the config dict to an object, skipping the providers # as it's the only field used as dict in the code config_obj = testutils.DictToObject(config, skip_attrs=["providers"]) - mock_get_all_worker_services.return_value = ( - config_obj.services_db or [] - ) + mock_get_all_worker_services.return_value = config_obj.services_db or [] mock_filter_regions.return_value = config_obj.regions_db or [] - mock_get_weighted_filtered_services.return_value = \ - [] if expected_result is None else [ - (mock.Mock(id=expected_id), 100) - for expected_id in expected_result - ] + mock_get_weighted_filtered_services.return_value = ( + [] + if expected_result is None + else [(mock.Mock(id=expected_id), 100) for expected_id in expected_result] + ) kwargs = { "enabled": enabled, @@ -195,25 +189,24 @@ def test_get_workers_for_specs( exception_type, self.server.get_workers_for_specs, mock.sentinel.context, - **kwargs + **kwargs, ) return - result = self.server.get_workers_for_specs( - mock.sentinel.context, - **kwargs - ) + result = self.server.get_workers_for_specs(mock.sentinel.context, **kwargs) - mock_get_all_worker_services.assert_called_once_with( - mock.sentinel.context) + mock_get_all_worker_services.assert_called_once_with(mock.sentinel.context) if region_sets: - calls = [mock.call( - mock.sentinel.context, - region_set, - enabled=True, - check_all_exist=True) - for region_set in region_sets] + calls = [ + mock.call( + mock.sentinel.context, + region_set, + enabled=True, + check_all_exist=True, + ) + for region_set in region_sets + ] mock_filter_regions.assert_has_calls(calls, any_order=True) mock_get_weighted_filtered_services.assert_called_once_with( @@ -228,9 +221,11 @@ def test_get_workers_for_specs( if enabled is not None: mock_enabled_filter_cls.assert_called_once_with(enabled=enabled) if region_sets: - calls = [mock.call(region_set, any_region=True) - for region_set in region_sets] + calls = [ + mock.call(region_set, any_region=True) for region_set in region_sets + ] mock_regions_filter_cls.assert_has_calls(calls, any_order=True) if provider_requirements: mock_provider_types_filter_cls.assert_called_once_with( - provider_requirements) + provider_requirements + ) diff --git a/coriolis/tests/scheduler/test_scheduler_utils.py b/coriolis/tests/scheduler/test_scheduler_utils.py index 1ba07f9f..88b3874e 100644 --- a/coriolis/tests/scheduler/test_scheduler_utils.py +++ b/coriolis/tests/scheduler/test_scheduler_utils.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception from coriolis.scheduler import scheduler_utils from coriolis.tests import test_base @@ -27,7 +26,7 @@ def test_get_rpc_client_for_service(self): with mock.patch.dict( scheduler_utils.RPC_TOPIC_TO_CLIENT_CLASS_MAP, {constants.WORKER_MAIN_MESSAGING_TOPIC: self.rpc_client_class}, - clear=True + clear=True, ): self.service.topic = constants.WORKER_MAIN_MESSAGING_TOPIC self.service.host = 'test_host' @@ -35,7 +34,8 @@ def test_get_rpc_client_for_service(self): result = scheduler_utils.get_rpc_client_for_service(self.service) self.rpc_client_class.assert_called_once_with( - topic='coriolis_worker.test_host') + topic='coriolis_worker.test_host' + ) self.assertEqual(result, self.rpc_client_class.return_value) @@ -43,15 +43,14 @@ def test_get_rpc_client_for_service_different_topic(self): with mock.patch.dict( scheduler_utils.RPC_TOPIC_TO_CLIENT_CLASS_MAP, {mock.sentinel.topic: self.rpc_client_class}, - clear=True + clear=True, ): self.service.topic = mock.sentinel.topic self.service.host = 'host' result = scheduler_utils.get_rpc_client_for_service(self.service) - self.rpc_client_class.assert_called_once_with( - topic=mock.sentinel.topic) + self.rpc_client_class.assert_called_once_with(topic=mock.sentinel.topic) self.assertEqual(result, self.rpc_client_class.return_value) @@ -59,48 +58,53 @@ def test_get_rpc_client_for_service_with_exception(self): self.service.topic = 'non-existent-topic' self.service.host = 'host' - self.assertRaises(exception.NotFound, - scheduler_utils.get_rpc_client_for_service, - self.service) + self.assertRaises( + exception.NotFound, scheduler_utils.get_rpc_client_for_service, self.service + ) def test_get_any_worker_service_no_services(self): self.scheduler_client.get_workers_for_specs.return_value = [] - self.assertRaises(exception.NoWorkerServiceError, - scheduler_utils.get_any_worker_service, - self.scheduler_client, self.ctxt) + self.assertRaises( + exception.NoWorkerServiceError, + scheduler_utils.get_any_worker_service, + self.scheduler_client, + self.ctxt, + ) @mock.patch('coriolis.scheduler.scheduler_utils.db_api.get_service') @mock.patch('random.choice') - def test_get_any_worker_service_random_choice(self, mock_random_choice, - get_service_mock): + def test_get_any_worker_service_random_choice( + self, mock_random_choice, get_service_mock + ): service_mock1 = {'id': 'test_id1'} service_mock2 = {'id': 'test_id2'} self.scheduler_client.get_workers_for_specs.return_value = [ - service_mock1, service_mock2] + service_mock1, + service_mock2, + ] get_service_mock.return_value = [service_mock1, service_mock2] mock_random_choice.return_value = service_mock1 result = scheduler_utils.get_any_worker_service( - self.scheduler_client, self.ctxt, random_choice=True) + self.scheduler_client, self.ctxt, random_choice=True + ) - mock_random_choice.assert_called_once_with([ - service_mock1, service_mock2]) - get_service_mock.assert_called_once_with( - self.ctxt, service_mock1['id']) + mock_random_choice.assert_called_once_with([service_mock1, service_mock2]) + get_service_mock.assert_called_once_with(self.ctxt, service_mock1['id']) self.assertEqual(result, get_service_mock.return_value) @mock.patch('coriolis.scheduler.scheduler_utils.db_api.get_service') def test_get_any_worker_service_raw_dict(self, get_service_mock): service_mock = {'id': 'test_id'} - self.scheduler_client.get_workers_for_specs.return_value = [ - service_mock] + self.scheduler_client.get_workers_for_specs.return_value = [service_mock] result = scheduler_utils.get_any_worker_service( - self.scheduler_client, self.ctxt, raw_dict=True) + self.scheduler_client, self.ctxt, raw_dict=True + ) get_service_mock.assert_not_called() self.assertEqual(result, service_mock) @@ -109,17 +113,18 @@ def test_get_worker_rpc_for_host(self): with mock.patch.dict( scheduler_utils.RPC_TOPIC_TO_CLIENT_CLASS_MAP, {constants.WORKER_MAIN_MESSAGING_TOPIC: self.rpc_client_class}, - clear=True + clear=True, ): host = 'test_host' client_args = ('arg1', 'arg2') client_kwargs = {'key1': 'value1', 'key2': 'value2'} result = scheduler_utils.get_worker_rpc_for_host( - host, *client_args, **client_kwargs) + host, *client_args, **client_kwargs + ) self.rpc_client_class.assert_called_once_with( - *client_args, topic='coriolis_worker.test_host', - **client_kwargs) + *client_args, topic='coriolis_worker.test_host', **client_kwargs + ) self.assertEqual(result, self.rpc_client_class.return_value) diff --git a/coriolis/tests/services/test_api.py b/coriolis/tests/services/test_api.py index fa4d9e6a..8ea459a0 100644 --- a/coriolis/tests/services/test_api.py +++ b/coriolis/tests/services/test_api.py @@ -18,20 +18,20 @@ def setUp(self): self.api._rpc_client = self.rpc_client_mock def test_create(self): - self.api.create('ctxt', 'host', 'binary', 'topic', 'mapped_regions', - True) + self.api.create('ctxt', 'host', 'binary', 'topic', 'mapped_regions', True) self.rpc_client_mock.register_service.assert_called_once_with( - 'ctxt', 'host', 'binary', 'topic', True, 'mapped_regions') + 'ctxt', 'host', 'binary', 'topic', True, 'mapped_regions' + ) def test_update(self): self.api.update('ctxt', 'service_id', 'updated_values') self.rpc_client_mock.update_service.assert_called_once_with( - 'ctxt', 'service_id', 'updated_values') + 'ctxt', 'service_id', 'updated_values' + ) def test_delete(self): self.api.delete('ctxt', 'region_id') - self.rpc_client_mock.delete_service.assert_called_once_with( - 'ctxt', 'region_id') + self.rpc_client_mock.delete_service.assert_called_once_with('ctxt', 'region_id') def test_get_services(self): self.api.get_services('ctxt') @@ -39,5 +39,4 @@ def test_get_services(self): def test_get_service(self): self.api.get_service('ctxt', 'service_id') - self.rpc_client_mock.get_service.assert_called_once_with( - 'ctxt', 'service_id') + self.rpc_client_mock.get_service.assert_called_once_with('ctxt', 'service_id') diff --git a/coriolis/tests/taskflow/test_base.py b/coriolis/tests/taskflow/test_base.py index cde2df46..6e54ebe6 100644 --- a/coriolis/tests/taskflow/test_base.py +++ b/coriolis/tests/taskflow/test_base.py @@ -37,18 +37,19 @@ def test_get_error_str_for_flow_failures(self): self.task_failure.traceback_str = "Traceback" result = self.task._get_error_str_for_flow_failures( - {"task1": self.task_failure}, full_tracebacks=True) + {"task1": self.task_failure}, full_tracebacks=True + ) self.assertEqual(result, " Traceback for task 'task1': Traceback") def test_get_error_str_for_flow_no_failures(self): - result = self.task._get_error_str_for_flow_failures( - None, full_tracebacks=False) + result = self.task._get_error_str_for_flow_failures(None, full_tracebacks=False) self.assertEqual(result, "No flow failures provided.") def test_get_error_str_for_flow_failures_no_items(self): self.flow_failures.items.return_value = False result = self.task._get_error_str_for_flow_failures( - self.flow_failures, full_tracebacks=False) + self.flow_failures, full_tracebacks=False + ) self.assertEqual(result, "No flow failures present.") def test_get_error_str_for_flow_failures_task_process_exception(self): @@ -57,7 +58,8 @@ def test_get_error_str_for_flow_failures_task_process_exception(self): self.task_failure.exception = exception.TaskProcessException() result = self.task._get_error_str_for_flow_failures( - {"task1": self.task_failure}, full_tracebacks=False) + {"task1": self.task_failure}, full_tracebacks=False + ) self.assertEqual(result, " Error message for task 'task1': Traceback") def test_get_error_str_for_flow_failures_exception_multiline(self): @@ -66,7 +68,8 @@ def test_get_error_str_for_flow_failures_exception_multiline(self): self.task_failure.exception = exception.TaskProcessException() result = self.task._get_error_str_for_flow_failures( - {"task1": self.task_failure}, full_tracebacks=False) + {"task1": self.task_failure}, full_tracebacks=False + ) self.assertEqual(result, " Error message for task 'task1': Line 2") def test_revert(self): @@ -86,8 +89,8 @@ class BaseRunWorkerTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(BaseRunWorkerTaskTestCase, self).setUp() self.task = base.BaseRunWorkerTask( - 'test_task', 'test_task_id', 'test-task_instance', - 'test_runner_type') + 'test_task', 'test_task_id', 'test-task_instance', 'test_runner_type' + ) self.kwargs = {'test_task_id': 'test_task_id'} self.depends_on = ['test_task_id'] self.mock_task_runner = mock.Mock() @@ -97,50 +100,49 @@ def setUp(self): def test_scheduler_client_property(self): mock_scheduler_client = mock.Mock(spec=SchedulerClient) - with mock.patch('coriolis.scheduler.rpc.client.SchedulerClient', - return_value=mock_scheduler_client): - - self.assertEqual(self.task._scheduler_client, - mock_scheduler_client) + with mock.patch( + 'coriolis.scheduler.rpc.client.SchedulerClient', + return_value=mock_scheduler_client, + ): + self.assertEqual(self.task._scheduler_client, mock_scheduler_client) def test_scheduler_client_already_set(self): mock_scheduler_client = mock.Mock(spec=SchedulerClient) self.task._scheduler_client_instance = mock_scheduler_client - with mock.patch('coriolis.scheduler.rpc.client.SchedulerClient') as \ - mock_SchedulerClient: - self.assertEqual( - self.task._scheduler_client, mock_scheduler_client) + with mock.patch( + 'coriolis.scheduler.rpc.client.SchedulerClient' + ) as mock_SchedulerClient: + self.assertEqual(self.task._scheduler_client, mock_scheduler_client) mock_SchedulerClient.assert_not_called() def test_set_provides_for_dependencies(self): self.task._set_provides_for_dependencies(self.kwargs) - self.assertEqual(self.kwargs, - {'test_task_id': 'test_task_id', - 'provides': ['task-test_task-result']}) + self.assertEqual( + self.kwargs, + {'test_task_id': 'test_task_id', 'provides': ['task-test_task-result']}, + ) def test_set_provides_for_dependencies_with_provides_key(self): kwargs = {'provides': ['test']} self.task._set_provides_for_dependencies(kwargs) - self.assertEqual(kwargs, {'provides': - ['test', 'task-test_task-result']}) + self.assertEqual(kwargs, {'provides': ['test', 'task-test_task-result']}) def test_set_requires_for_dependencies(self): self.task._set_requires_for_dependencies(self.kwargs, self.depends_on) - self.assertEqual(self.kwargs, - {'test_task_id': 'test_task_id', - 'requires': ['task-test_task_id-result']}) + self.assertEqual( + self.kwargs, + {'test_task_id': 'test_task_id', 'requires': ['task-test_task_id-result']}, + ) def test_set_requires_for_dependencies_with_requires_key(self): kwargs = {'requires': ['test']} self.task._set_requires_for_dependencies(kwargs, self.depends_on) - self.assertEqual(kwargs, - {'requires': ['test', 'task-test_task_id-result']}) + self.assertEqual(kwargs, {'requires': ['test', 'task-test_task_id-result']}) @mock.patch('coriolis.tasks.factory.get_task_runner_class') def test_set_requires_for_task_info_fields(self, mock_get_task_runner): - self.mock_task_runner.get_required_task_info_properties.\ - return_value = ['prop1'] + self.mock_task_runner.get_required_task_info_properties.return_value = ['prop1'] mock_get_task_runner.return_value = self.mock_task_runner kwargs = {'requires': ['test']} @@ -148,31 +150,45 @@ def test_set_requires_for_task_info_fields(self, mock_get_task_runner): self.assertEqual(kwargs, {'requires': ['test', 'prop1']}) @mock.patch('coriolis.tasks.factory.get_task_runner_class') - def test_set_requires_for_task_info_fields_with_cleanup( - self, mock_get_task_runner): - self.mock_task_runner.get_required_task_info_properties.\ - return_value = ['main_task_dep1', 'main_task_dep2'] - self.mock_task_runner.get_returned_task_info_properties.\ - return_value = ['main_task_dep1'] - - self.mock_cleanup_task_runner.get_required_task_info_properties.\ - return_value = ['cleanup_task_dep1', 'cleanup_task_dep2'] - - mock_get_task_runner.side_effect = [self.mock_task_runner, - self.mock_cleanup_task_runner] + def test_set_requires_for_task_info_fields_with_cleanup(self, mock_get_task_runner): + self.mock_task_runner.get_required_task_info_properties.return_value = [ + 'main_task_dep1', + 'main_task_dep2', + ] + self.mock_task_runner.get_returned_task_info_properties.return_value = [ + 'main_task_dep1' + ] + + self.mock_cleanup_task_runner.get_required_task_info_properties.return_value = [ + 'cleanup_task_dep1', + 'cleanup_task_dep2', + ] + + mock_get_task_runner.side_effect = [ + self.mock_task_runner, + self.mock_cleanup_task_runner, + ] self.task._cleanup_task_runner_type = 'cleanup_runner_type' kwargs = {'requires': ['test']} self.task._set_requires_for_task_info_fields(kwargs) - self.assertEqual(sorted(kwargs['requires']), - sorted(['test', 'main_task_dep1', 'main_task_dep2', - 'cleanup_task_dep1', 'cleanup_task_dep2'])) + self.assertEqual( + sorted(kwargs['requires']), + sorted( + [ + 'test', + 'main_task_dep1', + 'main_task_dep2', + 'cleanup_task_dep1', + 'cleanup_task_dep2', + ] + ), + ) @mock.patch('coriolis.tasks.factory.get_task_runner_class') def test_set_provides_for_task_info_fields(self, mock_get_task_runner): - self.mock_task_runner.get_returned_task_info_properties.\ - return_value = ['prop1'] + self.mock_task_runner.get_returned_task_info_properties.return_value = ['prop1'] mock_get_task_runner.return_value = self.mock_task_runner kwargs = {'provides': ['test']} @@ -180,92 +196,136 @@ def test_set_provides_for_task_info_fields(self, mock_get_task_runner): self.assertEqual(kwargs, {'provides': ['test', 'prop1']}) @mock.patch('coriolis.tasks.factory.get_task_runner_class') - def test_set_provides_for_task_info_fields_with_cleanup( - self, mock_get_task_runner): - self.mock_task_runner.get_returned_task_info_properties.\ - return_value = ['main_task_dep1', 'main_task_dep2'] - self.mock_task_runner.get_required_task_info_properties.\ - return_value = ['main_task_dep1'] - - self.mock_cleanup_task_runner.get_returned_task_info_properties.\ - return_value = ['cleanup_task_dep1', 'cleanup_task_dep2'] - - mock_get_task_runner.side_effect = [self.mock_task_runner, - self.mock_cleanup_task_runner] + def test_set_provides_for_task_info_fields_with_cleanup(self, mock_get_task_runner): + self.mock_task_runner.get_returned_task_info_properties.return_value = [ + 'main_task_dep1', + 'main_task_dep2', + ] + self.mock_task_runner.get_required_task_info_properties.return_value = [ + 'main_task_dep1' + ] + + self.mock_cleanup_task_runner.get_returned_task_info_properties.return_value = [ + 'cleanup_task_dep1', + 'cleanup_task_dep2', + ] + + mock_get_task_runner.side_effect = [ + self.mock_task_runner, + self.mock_cleanup_task_runner, + ] self.task._cleanup_task_runner_type = 'cleanup_runner_type' kwargs = {'provides': ['test']} self.task._set_provides_for_task_info_fields(kwargs) - self.assertEqual(sorted(kwargs['provides']), - sorted(['test', 'main_task_dep1', 'main_task_dep2', - 'cleanup_task_dep1', 'cleanup_task_dep2'])) + self.assertEqual( + sorted(kwargs['provides']), + sorted( + [ + 'test', + 'main_task_dep1', + 'main_task_dep2', + 'cleanup_task_dep1', + 'cleanup_task_dep2', + ] + ), + ) @mock.patch.object(rpc_client.WorkerClient, 'from_service_definition') @mock.patch.object(SchedulerClient, 'get_worker_service_for_task') - def test_get_worker_service_rpc_for_task(self, mock_get_worker_service, - mock_from_service_definition): + def test_get_worker_service_rpc_for_task( + self, mock_get_worker_service, mock_from_service_definition + ): mock_get_worker_service.return_value = {'id': 'worker_service_id'} mock_from_service_definition.return_value = 'worker_client' worker_client = self.task._get_worker_service_rpc_for_task( - 'ctxt', 'task_id', 'task_type', 'origin', 'destination') + 'ctxt', 'task_id', 'task_type', 'origin', 'destination' + ) self.assertEqual(worker_client, 'worker_client') mock_get_worker_service.assert_called_once_with( - 'ctxt', {'id': 'task_id', 'task_type': 'task_type'}, 'origin', - 'destination', retry_count=5, retry_period=2, random_choice=True) - mock_from_service_definition.assert_called_once_with({ - 'id': 'worker_service_id'}, - timeout=base.CONF.taskflow.worker_task_execution_timeout) - - @mock.patch.object( - base.BaseRunWorkerTask, - '_get_worker_service_rpc_for_task' - ) + 'ctxt', + {'id': 'task_id', 'task_type': 'task_type'}, + 'origin', + 'destination', + retry_count=5, + retry_period=2, + random_choice=True, + ) + mock_from_service_definition.assert_called_once_with( + {'id': 'worker_service_id'}, + timeout=base.CONF.taskflow.worker_task_execution_timeout, + ) + + @mock.patch.object(base.BaseRunWorkerTask, '_get_worker_service_rpc_for_task') def test_execute_task(self, mock_get_worker_rpc): mock_get_worker_rpc.return_value = self.mock_worker_rpc - result = self.task._execute_task('ctxt', 'task_id', 'task_type', - 'origin', 'destination', 'task_info') + result = self.task._execute_task( + 'ctxt', 'task_id', 'task_type', 'origin', 'destination', 'task_info' + ) self.assertEqual(result, self.mock_worker_rpc.run_task.return_value) mock_get_worker_rpc.assert_called_once_with( - 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination') + 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination' + ) self.mock_worker_rpc.run_task.assert_called_once_with( - 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination', - 'test-task_instance', 'task_info') - - @mock.patch.object( - base.BaseRunWorkerTask, - '_get_worker_service_rpc_for_task' - ) + 'ctxt', + 'test_task_id', + 'task_type', + 'origin', + 'destination', + 'test-task_instance', + 'task_info', + ) + + @mock.patch.object(base.BaseRunWorkerTask, '_get_worker_service_rpc_for_task') def test_execute_task_exception(self, mock_get_worker_rpc): mock_get_worker_rpc.return_value = self.mock_worker_rpc self.mock_worker_rpc.run_task.side_effect = CoriolisTestException() - self.assertRaises(CoriolisTestException, self.task._execute_task, - 'ctxt', 'task_id', 'task_type', 'origin', - 'destination', 'task_info') + self.assertRaises( + CoriolisTestException, + self.task._execute_task, + 'ctxt', + 'task_id', + 'task_type', + 'origin', + 'destination', + 'task_info', + ) mock_get_worker_rpc.assert_called_once_with( - 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination') + 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination' + ) self.mock_worker_rpc.run_task.assert_called_once_with( - 'ctxt', 'test_task_id', 'task_type', 'origin', 'destination', - 'test-task_instance', 'task_info') + 'ctxt', + 'test_task_id', + 'task_type', + 'origin', + 'destination', + 'test-task_instance', + 'task_info', + ) @mock.patch.object(base.BaseRunWorkerTask, '_execute_task') def test_execute(self, mock_execute_task): - result = self.task.execute('ctxt', 'origin', 'destination', - 'task_info') + result = self.task.execute('ctxt', 'origin', 'destination', 'task_info') self.assertEqual(result, mock_execute_task.return_value) mock_execute_task.assert_called_once_with( - 'ctxt', 'test_task_id', 'test_runner_type', 'origin', - 'destination', 'task_info') + 'ctxt', + 'test_task_id', + 'test_runner_type', + 'origin', + 'destination', + 'task_info', + ) @mock.patch.object(base.BaseRunWorkerTask, '_execute_task') def test_revert(self, mock_execute_task): @@ -276,8 +336,13 @@ def test_revert(self, mock_execute_task): self.assertEqual(result, None) mock_execute_task.assert_called_once_with( - 'ctxt', 'test_task_id', 'cleanup_task_type', 'origin', - 'destination', 'task_info') + 'ctxt', + 'test_task_id', + 'cleanup_task_type', + 'origin', + 'destination', + 'task_info', + ) def test_revert_no_cleanup_task_runner_type(self): result = self.task.revert('ctxt', 'origin', 'destination', 'task_info') @@ -293,16 +358,28 @@ def test_revert_with_exception(self, mock_execute_task): self.task.revert('ctxt', 'origin', 'destination', 'task_info') mock_execute_task.assert_called_once_with( - 'ctxt', 'test_task_id', 'cleanup_task_type', 'origin', - 'destination', 'task_info') + 'ctxt', + 'test_task_id', + 'cleanup_task_type', + 'origin', + 'destination', + 'task_info', + ) @mock.patch.object(base.BaseRunWorkerTask, '_execute_task') def test_revert_with_exception_and_raise_on_cleanup_failure( - self, mock_execute_task): + self, mock_execute_task + ): mock_execute_task.side_effect = CoriolisTestException() self.task._cleanup_task_runner_type = 'cleanup_task_type' self.task._raise_on_cleanup_failure = True - self.assertRaises(CoriolisTestException, self.task.revert, 'ctxt', - 'origin', 'destination', 'task_info') + self.assertRaises( + CoriolisTestException, + self.task.revert, + 'ctxt', + 'origin', + 'destination', + 'task_info', + ) diff --git a/coriolis/tests/taskflow/test_runner.py b/coriolis/tests/taskflow/test_runner.py index 12576a0e..a14579e5 100644 --- a/coriolis/tests/taskflow/test_runner.py +++ b/coriolis/tests/taskflow/test_runner.py @@ -22,8 +22,11 @@ class TaskFlowRunnerTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(TaskFlowRunnerTestCase, self).setUp() self.runner = runner.TaskFlowRunner( - 'svc_name', runner.TASKFLOW_EXECUTION_ORDER_PARALLEL, - runner.TASKFLOW_EXECUTOR_THREADED, 1) + 'svc_name', + runner.TASKFLOW_EXECUTION_ORDER_PARALLEL, + runner.TASKFLOW_EXECUTOR_THREADED, + 1, + ) self.mock_flow = mock.Mock() self.mock_engine = mock.Mock() self.mock_mp_log_q = mock.Mock() @@ -34,7 +37,7 @@ def test_log_flow_transition(self): details = { 'flow_name': 'flow_name', 'flow_uuid': 'flow_uuid', - 'old_state': 'old_state' + 'old_state': 'old_state', } with self.assertLogs('coriolis.taskflow.runner', level=logging.DEBUG): @@ -45,7 +48,7 @@ def test_log_task_transition(self): details = { 'task_name': 'task_name', 'task_uuid': 'task_uuid', - 'old_state': 'old_state' + 'old_state': 'old_state', } with self.assertLogs('coriolis.taskflow.runner', level=logging.DEBUG): @@ -56,16 +59,21 @@ def test_setup_engine_for_flow(self, mock_load): result = self.runner._setup_engine_for_flow(self.mock_flow) mock_load.assert_called_once_with( - self.mock_flow, None, executor=self.runner._executor, + self.mock_flow, + None, + executor=self.runner._executor, engine=self.runner._execution_order, - max_workers=self.runner._max_workers) + max_workers=self.runner._max_workers, + ) engine = mock_load.return_value engine.notifier.register.assert_called_once_with( - mock.ANY, self.runner._log_flow_transition) + mock.ANY, self.runner._log_flow_transition + ) engine.atom_notifier.register.assert_called_once_with( - mock.ANY, self.runner._log_task_transition) + mock.ANY, self.runner._log_task_transition + ) self.assertEqual(engine, result) @@ -91,8 +99,9 @@ def test_run_flow_exception(self, mock_get_exception_details): self.mock_engine.run.side_effect = CoriolisTestException() with self.assertLogs('coriolis.taskflow.runner', level=logging.WARN): - self.assertRaises(CoriolisTestException, self.runner._run_flow, - self.mock_flow) + self.assertRaises( + CoriolisTestException, self.runner._run_flow, self.mock_flow + ) self.mock_engine.compile.assert_called_once() self.mock_engine.prepare.assert_called_once() self.mock_engine.run.assert_called_once() @@ -107,28 +116,32 @@ def test_run_flow(self, mock_run_flow): @mock.patch('coriolis.utils.setup_logging') @mock.patch('oslo_log.log.getLogger') @mock.patch('logging.handlers.QueueHandler') - def test_setup_task_process_logging(self, mock_queue_handler, - mock_get_logger, mock_setup_logging, - mock_conf): + def test_setup_task_process_logging( + self, mock_queue_handler, mock_get_logger, mock_setup_logging, mock_conf + ): mock_handler = mock.Mock() mock_get_logger.return_value.logger.handlers = [mock_handler] self.runner._setup_task_process_logging(self.mock_mp_log_q) - mock_conf.assert_called_once_with(sys.argv[1:], project='coriolis', - version='1.0.0') + mock_conf.assert_called_once_with( + sys.argv[1:], project='coriolis', version='1.0.0' + ) mock_setup_logging.assert_called_once() mock_get_logger.assert_called_once_with(None) mock_queue_handler.assert_called_once_with(self.mock_mp_log_q) - mock_get_logger.return_value.logger.removeHandler.\ - assert_called_once_with(mock_handler) + mock_get_logger.return_value.logger.removeHandler.assert_called_once_with( + mock_handler + ) mock_get_logger.return_value.logger.addHandler.assert_called_once_with( - mock_queue_handler.return_value) + mock_queue_handler.return_value + ) @mock.patch.object(runner.TaskFlowRunner, '_setup_task_process_logging') @mock.patch.object(runner.TaskFlowRunner, '_run_flow') def test_run_flow_in_process(self, mock_run_flow, mock_setup_logging): - self.runner._run_flow_in_process(self.mock_flow, self.mock_mp_log_q, - store=self.store) + self.runner._run_flow_in_process( + self.mock_flow, self.mock_mp_log_q, store=self.store + ) mock_setup_logging.assert_called_once_with(self.mock_mp_log_q) mock_run_flow.assert_called_once_with(self.mock_flow, store=self.store) @@ -136,16 +149,13 @@ def test_run_flow_in_process(self, mock_run_flow, mock_setup_logging): @mock.patch.object(logging, 'getLogger') def test__handle_mp_log_events(self, mock_get_logger): mock_p = mock.Mock() - self.mock_mp_log_q.get.side_effect = [mock.sentinel.record, - queue.Empty, None] + self.mock_mp_log_q.get.side_effect = [mock.sentinel.record, queue.Empty, None] mock_p.is_alive.return_value = True result = self.runner._handle_mp_log_events(mock_p, self.mock_mp_log_q) - mock_get_logger.assert_called_once_with( - mock.sentinel.record.name) + mock_get_logger.assert_called_once_with(mock.sentinel.record.name) self.assertIsNone(result) - mock_get_logger.return_value.handle.assert_called_with( - mock.sentinel.record) + mock_get_logger.return_value.handle.assert_called_with(mock.sentinel.record) def test__handle_mp_log_events_process_dead(self): mock_p = mock.Mock() @@ -164,22 +174,24 @@ def test_spawn_process_flow(self, mock_get_context, mock_spawn): mock_get_context.return_value = mock_mp_ctx self.runner._run_flow_in_process = mock.Mock() - result = self.runner._spawn_process_flow(self.mock_flow, - store=self.store) + result = self.runner._spawn_process_flow(self.mock_flow, store=self.store) self.assertIsNone(result) mock_get_context.assert_called_once_with('spawn') mock_mp_ctx.Process.assert_called_once_with( target=self.runner._run_flow_in_process, - args=(self.mock_flow, mock_mp_ctx.Queue.return_value, self.store)) + args=(self.mock_flow, mock_mp_ctx.Queue.return_value, self.store), + ) mock_process.start.assert_called_once_with() mock_spawn.assert_called_once_with( target=self.runner._handle_mp_log_events, args=(mock_process, mock_mp_ctx.Queue.return_value), - daemon=True) + daemon=True, + ) @mock.patch.object(runner.TaskFlowRunner, '_spawn_process_flow') def test_run_flow_in_background(self, mock_spawn_process_flow): self.runner.run_flow_in_background(self.mock_flow, store=self.store) - mock_spawn_process_flow.assert_called_once_with(self.mock_flow, - store=self.store) + mock_spawn_process_flow.assert_called_once_with( + self.mock_flow, store=self.store + ) diff --git a/coriolis/tests/tasks/test_base.py b/coriolis/tests/tasks/test_base.py index 8b47c716..1481f4bb 100644 --- a/coriolis/tests/tasks/test_base.py +++ b/coriolis/tests/tasks/test_base.py @@ -6,15 +6,13 @@ import ddt import paramiko -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception from coriolis.tasks import base from coriolis.tests import test_base @ddt.ddt class BaseTasksTestCase(test_base.CoriolisBaseTestCase): - @mock.patch.object(base.TaskRunner, '__abstractmethods__', set()) def setUp(self): super(BaseTasksTestCase, self).setUp() @@ -25,8 +23,12 @@ def setUp(self): @ddt.file_data('data/get_shared_libs_for_providers.yml') @ddt.unpack def test_get_shared_libs_for_providers( - self, mock_get_shared_lib_dirs, - mock_get_required_platform, config, expected_result): + self, + mock_get_shared_lib_dirs, + mock_get_required_platform, + config, + expected_result, + ): # shared_libs = {} platform = config.get('platform', '') mock_get_required_platform.return_value = platform @@ -40,19 +42,28 @@ def test_get_shared_libs_for_providers( shared_lib_dirs_side_effect.append(source_lib_dirs) expected_calls.append( mock.call.mock_get_shared_lib_dirs( - mock.sentinel.ctxt, mock.sentinel.origin, - mock.sentinel.event_handler)) + mock.sentinel.ctxt, + mock.sentinel.origin, + mock.sentinel.event_handler, + ) + ) if return_dest_dirs: shared_lib_dirs_side_effect.append(dest_lib_dirs) expected_calls.append( mock.call.mock_get_shared_lib_dirs( - mock.sentinel.ctxt, mock.sentinel.destination, - mock.sentinel.event_handler)) + mock.sentinel.ctxt, + mock.sentinel.destination, + mock.sentinel.event_handler, + ) + ) mock_get_shared_lib_dirs.side_effect = shared_lib_dirs_side_effect result = self.task_runner.get_shared_libs_for_providers( - mock.sentinel.ctxt, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.event_handler, + ) mock_get_required_platform.assert_called_once_with() self.assertEqual(result, expected_result) mock_get_shared_lib_dirs.assert_has_calls(expected_calls) @@ -64,9 +75,14 @@ def test_get_shared_libs_for_providers( @ddt.file_data('data/run.yml') @ddt.unpack def test_run( - self, mock_sanitize_task_info, mock_get_returned_task_info_props, - mock_run, mock_get_required_task_info_props, config, - exception_expected): + self, + mock_sanitize_task_info, + mock_get_returned_task_info_props, + mock_run, + mock_get_required_task_info_props, + config, + exception_expected, + ): required_properties = config.get('required_properties', []) returned_properties = config.get('returned_properties', []) task_info = config.get('task_info', {}) @@ -78,60 +94,83 @@ def test_run( if exception_expected: self.assertRaises( - exception.CoriolisException, self.task_runner.run, - mock.sentinel.ctxt, mock.sentinel.instance, - mock.sentinel.origin, mock.sentinel.destination, - task_info, mock.sentinel.event_handler) + exception.CoriolisException, + self.task_runner.run, + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) if sanitize_check: - mock_sanitize_task_info.assert_called_once_with( - mock_run.return_value) + mock_sanitize_task_info.assert_called_once_with(mock_run.return_value) return result = self.task_runner.run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) mock_get_required_task_info_props.assert_called_once_with() mock_get_returned_task_info_props.assert_called_with() mock_run.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, mock_run.return_value) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') - def test_get_shared_lib_dirs_for_provider(self, mock_get_conn_info, - mock_get_provider): + def test_get_shared_lib_dirs_for_provider( + self, mock_get_conn_info, mock_get_provider + ): endpoint = mock.MagicMock() mock_get_provider.return_value = mock.MagicMock() mock_get_shared_lib_dirs = ( - mock_get_provider.return_value.get_shared_library_directories) + mock_get_provider.return_value.get_shared_library_directories + ) result = base.get_shared_lib_dirs_for_provider( - mock.sentinel.ctxt, endpoint, - mock.sentinel.event_handler) + mock.sentinel.ctxt, endpoint, mock.sentinel.event_handler + ) mock_get_provider.assert_called_once_with( - endpoint['type'], constants.PROVIDER_TYPE_SETUP_LIBS, - mock.sentinel.event_handler, raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, endpoint) + endpoint['type'], + constants.PROVIDER_TYPE_SETUP_LIBS, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, endpoint) mock_get_shared_lib_dirs.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value) + mock.sentinel.ctxt, mock_get_conn_info.return_value + ) self.assertEqual(result, mock_get_shared_lib_dirs.return_value) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') def test_get_shared_lib_dirs_for_providers_no_provider( - self, mock_get_conn_info, mock_get_provider): + self, mock_get_conn_info, mock_get_provider + ): endpoint = mock.MagicMock() mock_get_provider.return_value = None result = base.get_shared_lib_dirs_for_provider( - mock.sentinel.ctxt, endpoint, - mock.sentinel.event_handler) + mock.sentinel.ctxt, endpoint, mock.sentinel.event_handler + ) mock_get_provider.assert_called_once_with( - endpoint['type'], constants.PROVIDER_TYPE_SETUP_LIBS, - mock.sentinel.event_handler, raise_if_not_found=False) + endpoint['type'], + constants.PROVIDER_TYPE_SETUP_LIBS, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_not_called() self.assertEqual(result, []) @@ -146,11 +185,11 @@ def test_get_shared_lib_dirs_for_providers_no_provider( def test_get_connection_info(self, data, mock_get_secret_conn_info): expected_conn_info = data[1] - result = base.get_connection_info( - mock.sentinel.ctxt, data[0]) + result = base.get_connection_info(mock.sentinel.ctxt, data[0]) mock_get_secret_conn_info.assert_called_once_with( - mock.sentinel.ctxt, expected_conn_info) + mock.sentinel.ctxt, expected_conn_info + ) self.assertEqual(result, mock_get_secret_conn_info.return_value) def test_marshal_migr_conn_info_none(self): @@ -174,13 +213,14 @@ def test_marshal_migr_conn_info(self, mock_conf, mock_serialize_key): pkey = mock.MagicMock() custom_pkey_field_name = "custom_key" migr_conn_info = {custom_pkey_field_name: pkey} - expected_conn_info = { - custom_pkey_field_name: mock_serialize_key.return_value} + expected_conn_info = {custom_pkey_field_name: mock_serialize_key.return_value} result = base.marshal_migr_conn_info( - migr_conn_info, private_key_field_name=custom_pkey_field_name) + migr_conn_info, private_key_field_name=custom_pkey_field_name + ) mock_serialize_key.assert_called_once_with( - pkey, mock_conf.serialization.temp_keypair_password) + pkey, mock_conf.serialization.temp_keypair_password + ) self.assertEqual(result, expected_conn_info) def test_unmarshal_migr_conn_info_none(self): @@ -204,11 +244,12 @@ def test_unmarshal_migr_conn_info_serialized(self): def test_unmarshal_migr_conn_info(self, mock_conf, mock_deserialize_key): custom_pkey_field_name = "custom_key" migr_conn_info = {custom_pkey_field_name: "pkey"} - expected_conn_info = { - custom_pkey_field_name: mock_deserialize_key.return_value} + expected_conn_info = {custom_pkey_field_name: mock_deserialize_key.return_value} result = base.unmarshal_migr_conn_info( - migr_conn_info, private_key_field_name=custom_pkey_field_name) + migr_conn_info, private_key_field_name=custom_pkey_field_name + ) mock_deserialize_key.assert_called_once_with( - "pkey", mock_conf.serialization.temp_keypair_password) + "pkey", mock_conf.serialization.temp_keypair_password + ) self.assertEqual(result, expected_conn_info) diff --git a/coriolis/tests/tasks/test_factory.py b/coriolis/tests/tasks/test_factory.py index 1b464c09..f48786ea 100644 --- a/coriolis/tests/tasks/test_factory.py +++ b/coriolis/tests/tasks/test_factory.py @@ -1,21 +1,16 @@ # Copyright 2023 Cloudbase Solutions Srl # All Rights Reserved. -from coriolis import constants -from coriolis import exception -from coriolis.tasks import factory -from coriolis.tasks import minion_pool_tasks +from coriolis import constants, exception +from coriolis.tasks import factory, minion_pool_tasks from coriolis.tests import test_base class TasksFactoryTestCase(test_base.CoriolisBaseTestCase): - def test_get_task_runner(self): - self.assertRaises( - exception.NotFound, - factory.get_task_runner_class, - "invalid") + self.assertRaises(exception.NotFound, factory.get_task_runner_class, "invalid") result = factory.get_task_runner_class( - constants.TASK_TYPE_POWER_ON_SOURCE_MINION) + constants.TASK_TYPE_POWER_ON_SOURCE_MINION + ) self.assertEqual(result, minion_pool_tasks.PowerOnSourceMinionTask) diff --git a/coriolis/tests/tasks/test_migration_tasks.py b/coriolis/tests/tasks/test_migration_tasks.py index 01e9c19c..6c3e3b9f 100644 --- a/coriolis/tests/tasks/test_migration_tasks.py +++ b/coriolis/tests/tasks/test_migration_tasks.py @@ -12,7 +12,6 @@ @ddt.ddt class GetOptimalFlavorTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(GetOptimalFlavorTaskTestCase, self).setUp() self.task_runner = migration_tasks.GetOptimalFlavorTask() @@ -20,12 +19,14 @@ def setUp(self): @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.events.EventManager') - @ddt.data( - None, - {"selected_flavor": "flavor1"} - ) - def test__run(self, instance_deployment_info, mock_event_manager, - mock_get_conn_info, mock_get_provider): + @ddt.data(None, {"selected_flavor": "flavor1"}) + def test__run( + self, + instance_deployment_info, + mock_event_manager, + mock_get_conn_info, + mock_get_provider, + ): destination = mock.MagicMock() provider = mock_get_provider.return_value task_info = { @@ -35,38 +36,48 @@ def test__run(self, instance_deployment_info, mock_event_manager, } expected_result = { "instance_deployment_info": { - "selected_flavor": provider.get_optimal_flavor.return_value}} + "selected_flavor": provider.get_optimal_flavor.return_value + } + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_INSTANCE_FLAVOR, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_INSTANCE_FLAVOR, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) provider.get_optimal_flavor.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - mock.sentinel.target_environment, mock.sentinel.export_info) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + mock.sentinel.target_environment, + mock.sentinel.export_info, + ) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) self.assertEqual(result, expected_result) -class ValidateMigrationDestinationInputsTaskTestCase( - test_base.CoriolisBaseTestCase): +class ValidateMigrationDestinationInputsTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ValidateMigrationDestinationInputsTaskTestCase, self).setUp() - self.task_runner = ( - migration_tasks.ValidateMigrationDestinationInputsTask()) + self.task_runner = migration_tasks.ValidateMigrationDestinationInputsTask() def test__validate_provider_replica_import_input(self): provider = mock.MagicMock() args = [ - mock.sentinel.ctxt, mock.sentinel.conn_info, - mock.sentinel.target_environment, mock.sentinel.export_info] - self.task_runner._validate_provider_replica_import_input( - provider, *args) + mock.sentinel.ctxt, + mock.sentinel.conn_info, + mock.sentinel.target_environment, + mock.sentinel.export_info, + ] + self.task_runner._validate_provider_replica_import_input(provider, *args) provider.validate_replica_import_input.assert_called_once_with( - *args, - check_os_morphing_resources=True, - check_final_vm_params=True) + *args, check_os_morphing_resources=True, check_final_vm_params=True + ) diff --git a/coriolis/tests/tasks/test_minion_pool_tasks.py b/coriolis/tests/tasks/test_minion_pool_tasks.py index 77e54860..5f135fa8 100644 --- a/coriolis/tests/tasks/test_minion_pool_tasks.py +++ b/coriolis/tests/tasks/test_minion_pool_tasks.py @@ -5,14 +5,12 @@ import ddt -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception from coriolis.tasks import minion_pool_tasks as mp_tasks from coriolis.tests import test_base MOCK_MINION_CONN_INFO = {"conn_info1": "value1", "conn_info2": "value2"} -MOCK_MINION_BACKUP_WRITER_CONN_INFO = { - "connection_details": {"user": "username"}} +MOCK_MINION_BACKUP_WRITER_CONN_INFO = {"connection_details": {"user": "username"}} MOCK_MINION_CONN_INFO_FIELD = "destination_minion_connection_info" MOCK_MINION_PROPS_TASK_INFO_FIELD = "destination_minion_provider_properties" MOCK_MARSHALED_INFO = {"marshaled": "info"} @@ -20,7 +18,6 @@ class CoriolisBaseMinionPoolTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(CoriolisBaseMinionPoolTaskTestCase, self).setUp() self.task_runner = None @@ -30,8 +27,7 @@ def _setup_base_mocks(self): self.task_info = mock.MagicMock() self.provider_type = constants.PROVIDER_TYPE_DESTINATION_MINION_POOL - get_provider_patcher = mock.patch( - 'coriolis.providers.factory.get_provider') + get_provider_patcher = mock.patch('coriolis.providers.factory.get_provider') self.mock_get_provider = get_provider_patcher.start() self.addCleanup(get_provider_patcher.stop) @@ -40,9 +36,13 @@ def _setup_base_mocks(self): get_required_provider_types=mock.MagicMock( return_value={ constants.PROVIDER_PLATFORM_DESTINATION: [ - constants.PROVIDER_TYPE_DESTINATION_MINION_POOL]}), + constants.PROVIDER_TYPE_DESTINATION_MINION_POOL + ] + } + ), get_required_platform=mock.MagicMock( - return_value=constants.PROVIDER_PLATFORM_DESTINATION), + return_value=constants.PROVIDER_PLATFORM_DESTINATION + ), ) patcher.start() self.addCleanup(patcher.stop) @@ -50,55 +50,56 @@ def _setup_base_mocks(self): mp_tasks_methods_patches = mock.patch.multiple( mp_tasks, _check_missing_minion_properties=mock.MagicMock(), - _get_minion_conn_info=mock.MagicMock( - return_value=MOCK_MINION_CONN_INFO), + _get_minion_conn_info=mock.MagicMock(return_value=MOCK_MINION_CONN_INFO), _get_minion_backup_writer_conn_info=mock.MagicMock( - return_value=MOCK_MINION_BACKUP_WRITER_CONN_INFO), - _get_platform_to_target=mock.MagicMock( - return_value=self.destination), + return_value=MOCK_MINION_BACKUP_WRITER_CONN_INFO + ), + _get_platform_to_target=mock.MagicMock(return_value=self.destination), ) mp_tasks_methods_patches.start() self.addCleanup(mp_tasks_methods_patches.stop) - get_conn_info_patcher = mock.patch( - 'coriolis.tasks.base.get_connection_info') + get_conn_info_patcher = mock.patch('coriolis.tasks.base.get_connection_info') self.mock_get_conn_info = get_conn_info_patcher.start() self.addCleanup(get_conn_info_patcher.stop) def _setup_vol_attachment_tasks_mocks(self): self.provider_disk_op = mock.MagicMock( - return_value=( - self.mock_get_provider.return_value.attach_volumes_to_minion)) + return_value=(self.mock_get_provider.return_value.attach_volumes_to_minion) + ) patcher = mock.patch.multiple( self.task_runner.__class__, _get_volumes_info_from_task_info=mock.MagicMock( - return_value=self.task_info['volumes_info']), + return_value=self.task_info['volumes_info'] + ), _get_minion_properties_task_info_field=mock.MagicMock( - return_value=MOCK_MINION_PROPS_TASK_INFO_FIELD), + return_value=MOCK_MINION_PROPS_TASK_INFO_FIELD + ), _get_minion_connection_info_task_info_field=mock.MagicMock( - return_value=MOCK_MINION_CONN_INFO_FIELD), + return_value=MOCK_MINION_CONN_INFO_FIELD + ), _get_provider_disk_operation=self.provider_disk_op, _get_minion_task_info_field_mappings=mock.MagicMock( - return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS), + return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS + ), ) patcher.start() self.addCleanup(patcher.stop) def _setup_validation_tasks_mocks(self): - prov_fun = self.mock_get_provider.return_value\ - .validate_minion_compatibility_for_transfer - self.provider_pool_validation_op = mock.MagicMock( - return_value=prov_fun) + prov_fun = self.mock_get_provider.return_value.validate_minion_compatibility_for_transfer + self.provider_pool_validation_op = mock.MagicMock(return_value=prov_fun) patcher = mock.patch.multiple( self.task_runner.__class__, _get_minion_properties_task_info_field=mock.MagicMock( - return_value="destination_minion_provider_properties"), + return_value="destination_minion_provider_properties" + ), _get_minion_task_info_field_mappings=mock.MagicMock( - return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS), - _get_provider_pool_validation_operation=( - self.provider_pool_validation_op), + return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS + ), + _get_provider_pool_validation_operation=(self.provider_pool_validation_op), ) patcher.start() self.addCleanup(patcher.stop) @@ -106,23 +107,25 @@ def _setup_validation_tasks_mocks(self): @ddt.ddt class MinionPoolTasksMethodsTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(MinionPoolTasksMethodsTestCase, self).setUp() marshal_migr_conn_info_patcher = mock.patch( 'coriolis.tasks.base.marshal_migr_conn_info', - return_value=MOCK_MARSHALED_INFO) + return_value=MOCK_MARSHALED_INFO, + ) self.mock_marshal_conn_info = marshal_migr_conn_info_patcher.start() self.addCleanup(marshal_migr_conn_info_patcher.stop) @ddt.data( {"platform_type": "", "expected_exception": NotImplementedError}, - - {"platform_type": constants.PROVIDER_PLATFORM_SOURCE, - "expected_result": constants.PROVIDER_TYPE_SOURCE_MINION_POOL}, - - {"platform_type": constants.PROVIDER_PLATFORM_DESTINATION, - "expected_result": constants.PROVIDER_TYPE_DESTINATION_MINION_POOL}, + { + "platform_type": constants.PROVIDER_PLATFORM_SOURCE, + "expected_result": constants.PROVIDER_TYPE_SOURCE_MINION_POOL, + }, + { + "platform_type": constants.PROVIDER_PLATFORM_DESTINATION, + "expected_result": constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, + }, ) def test__get_required_minion_pool_provider_types_for_platform(self, data): platform_type = data.get('platform_type') @@ -145,8 +148,7 @@ def test__get_required_minion_pool_provider_types_for_platform(self, data): def test__get_platform_to_target(self, data): args = [data[0], mock.sentinel.origin, mock.sentinel.destination] if not data[0]: - self.assertRaises( - data[1], mp_tasks._get_platform_to_target, *args) + self.assertRaises(data[1], mp_tasks._get_platform_to_target, *args) return result = mp_tasks._get_platform_to_target(*args) @@ -166,14 +168,17 @@ def test__check_missing_minion_properties(self, data): def _get_result(): if logs_missing_props: - with self.assertLogs('coriolis.tasks.minion_pool_tasks', - mp_tasks.logging.WARN): + with self.assertLogs( + 'coriolis.tasks.minion_pool_tasks', mp_tasks.logging.WARN + ): return mp_tasks._check_missing_minion_properties( - mock.ANY, minion_properties, - properties_to_check=properties_to_check) + mock.ANY, + minion_properties, + properties_to_check=properties_to_check, + ) return mp_tasks._check_missing_minion_properties( - mock.ANY, minion_properties, - properties_to_check=properties_to_check) + mock.ANY, minion_properties, properties_to_check=properties_to_check + ) self.assertEqual(_get_result(), None) @@ -189,34 +194,39 @@ def test__get_minion_conn_info(self, data): @ddt.data( ({}, {}, False), - ({'backup_writer_connection_info': {"backup": "writer"}}, - {"backup": "writer"}, False), - ({"backup_writer_connection_info": { - "connection_details": {"conn": "info"}, - "backup": "writer"}}, - {"connection_details": MOCK_MARSHALED_INFO, - "backup": "writer"}, - True), + ( + {'backup_writer_connection_info': {"backup": "writer"}}, + {"backup": "writer"}, + False, + ), + ( + { + "backup_writer_connection_info": { + "connection_details": {"conn": "info"}, + "backup": "writer", + } + }, + {"connection_details": MOCK_MARSHALED_INFO, "backup": "writer"}, + True, + ), ) def test__get_minion_backup_writer_conn_info(self, data): minion_properties = data[0] - connection_details = minion_properties.get( - 'backup_writer_connection_info', {}).get( - 'connection_details', {}).copy() + connection_details = ( + minion_properties.get('backup_writer_connection_info', {}) + .get('connection_details', {}) + .copy() + ) expected_result = data[1] marshaled = data[2] - result = mp_tasks._get_minion_backup_writer_conn_info( - minion_properties) + result = mp_tasks._get_minion_backup_writer_conn_info(minion_properties) self.assertEqual(result, expected_result) if marshaled: - self.mock_marshal_conn_info.assert_called_once_with( - connection_details) - + self.mock_marshal_conn_info.assert_called_once_with(connection_details) -class _BaseValidateMinionPoolOptsTaskTestCase( - CoriolisBaseMinionPoolTaskTestCase): +class _BaseValidateMinionPoolOptsTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): def setUp(self): super(_BaseValidateMinionPoolOptsTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseValidateMinionPoolOptionsTask() @@ -224,21 +234,25 @@ def setUp(self): def test__run(self): result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) - self.mock_get_provider.return_value.\ - validate_minion_pool_environment_options.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, - self.task_info['pool_environment_options']) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) + self.mock_get_provider.return_value.validate_minion_pool_environment_options.assert_called_once_with( + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, + self.task_info['pool_environment_options'], + ) class _BaseCreateMinionMachineTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): - def setUp(self): super(_BaseCreateMinionMachineTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseCreateMinionMachineTask() @@ -249,30 +263,35 @@ def test__run(self): expected_result = { "minion_connection_info": MOCK_MINION_CONN_INFO, "minion_backup_writer_connection_info": ( - MOCK_MINION_BACKUP_WRITER_CONN_INFO), - "minion_provider_properties": ( - prov_fun.return_value.get.return_value), + MOCK_MINION_BACKUP_WRITER_CONN_INFO + ), + "minion_provider_properties": (prov_fun.return_value.get.return_value), } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info['environment_options'], self.task_info['pool_identifier'], self.task_info['pool_os_type'], self.task_info['pool_shared_resources'], - mock.sentinel.minion_pool_machine_id) + mock.sentinel.minion_pool_machine_id, + ) class _BaseDeleteMinionMachineTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): - def setUp(self): super(_BaseDeleteMinionMachineTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseDeleteMinionMachineTask() @@ -282,80 +301,88 @@ def test__run(self): prov_fun = self.mock_get_provider.return_value.delete_minion result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, - self.task_info['minion_provider_properties']) - + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, + self.task_info['minion_provider_properties'], + ) -class _BaseSetUpPoolSupportingResourcesTaskTestCase( - CoriolisBaseMinionPoolTaskTestCase): +class _BaseSetUpPoolSupportingResourcesTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): def setUp(self): super(_BaseSetUpPoolSupportingResourcesTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseSetUpPoolSupportingResourcesTask() self._setup_base_mocks() def test__run(self): - prov_fun = self.mock_get_provider.return_value.\ - set_up_pool_shared_resources - expected_result = { - "pool_shared_resources": prov_fun.return_value - } + prov_fun = self.mock_get_provider.return_value.set_up_pool_shared_resources + expected_result = {"pool_shared_resources": prov_fun.return_value} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info['environment_options'], - self.task_info['pool_identifier']) + self.task_info['pool_identifier'], + ) class _BaseTearDownPoolSupportingResourcesTaskTestCase( - CoriolisBaseMinionPoolTaskTestCase): - + CoriolisBaseMinionPoolTaskTestCase +): def setUp(self): super(_BaseTearDownPoolSupportingResourcesTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseTearDownPoolSupportingResourcesTask() self._setup_base_mocks() def test__run(self): - prov_fun = self.mock_get_provider.return_value.\ - tear_down_pool_shared_resources - expected_result = { - "pool_shared_resources": None - } + prov_fun = self.mock_get_provider.return_value.tear_down_pool_shared_resources + expected_result = {"pool_shared_resources": None} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info['environment_options'], - self.task_info['pool_shared_resources']) + self.task_info['pool_shared_resources'], + ) @ddt.ddt -class _BaseVolumesMinionMachineAttachmentTask( - CoriolisBaseMinionPoolTaskTestCase): - +class _BaseVolumesMinionMachineAttachmentTask(CoriolisBaseMinionPoolTaskTestCase): def setUp(self): super(_BaseVolumesMinionMachineAttachmentTask, self).setUp() self.task_runner = mp_tasks._BaseVolumesMinionMachineAttachmentTask() @@ -365,17 +392,18 @@ def setUp(self): def test_get_required_task_info_properties(self): expected_result = [ "destination_minion_provider_properties", - "destination_minion_backup_writer_connection_info"] - result = mp_tasks._BaseVolumesMinionMachineAttachmentTask.\ - get_required_task_info_properties() + "destination_minion_backup_writer_connection_info", + ] + result = mp_tasks._BaseVolumesMinionMachineAttachmentTask.get_required_task_info_properties() self.assertEqual(sorted(result), sorted(expected_result)) def test_get_returned_task_info_properties(self): expected_result = [ - "target_resources", "target_resources_connection_info", - "destination_minion_provider_properties"] - result = mp_tasks._BaseVolumesMinionMachineAttachmentTask.\ - get_returned_task_info_properties() + "target_resources", + "target_resources_connection_info", + "destination_minion_provider_properties", + ] + result = mp_tasks._BaseVolumesMinionMachineAttachmentTask.get_returned_task_info_properties() self.assertEqual(sorted(result), sorted(expected_result)) @ddt.data( @@ -392,76 +420,86 @@ def test__check_missing_disk_op_result_keys(self, data): self.assertRaises( exception.CoriolisException, self.task_runner._check_missing_disk_op_result_keys, - mock.sentinel.platform_to_target, disk_op_res) + mock.sentinel.platform_to_target, + disk_op_res, + ) else: self.assertEqual( self.task_runner._check_missing_disk_op_result_keys( - mock.sentinel.platform_to_target, disk_op_res), - None) + mock.sentinel.platform_to_target, disk_op_res + ), + None, + ) - @mock.patch.object(mp_tasks._BaseVolumesMinionMachineAttachmentTask, - '_check_missing_disk_op_result_keys') + @mock.patch.object( + mp_tasks._BaseVolumesMinionMachineAttachmentTask, + '_check_missing_disk_op_result_keys', + ) def test__run(self, mock_check_missing_disk_op_keys): prov_fun = self.provider_disk_op.return_value prov_fun_result = prov_fun.return_value expected_result = { "volumes_info": prov_fun_result['volumes_info'], - MOCK_MINION_PROPS_TASK_INFO_FIELD: prov_fun_result[ - 'minion_properties'], - "target_resources": ( - prov_fun_result['minion_properties']), + MOCK_MINION_PROPS_TASK_INFO_FIELD: prov_fun_result['minion_properties'], + "target_resources": (prov_fun_result['minion_properties']), "target_resources_connection_info": self.task_info[ - 'target_resources_connection_info'], + 'target_resources_connection_info' + ], } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_check_missing_disk_op_keys.assert_called_once_with( - self.destination, prov_fun_result) + self.destination, prov_fun_result + ) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info[MOCK_MINION_PROPS_TASK_INFO_FIELD], self.task_info[MOCK_MINION_CONN_INFO_FIELD], - self.task_info['volumes_info']) - + self.task_info['volumes_info'], + ) -class _BaseAttachVolumesToTransferMinionTaskTestCase( - test_base.CoriolisBaseTestCase): +class _BaseAttachVolumesToTransferMinionTaskTestCase(test_base.CoriolisBaseTestCase): def test_get_required_task_info_properties(self): mock_super_call = mock.MagicMock(return_value=["field1", "field2"]) with mock.patch.object( - mp_tasks._BaseVolumesMinionMachineAttachmentTask, - 'get_required_task_info_properties', - mock_super_call): - result = mp_tasks._BaseAttachVolumesToTransferMinionTask.\ - get_required_task_info_properties() + mp_tasks._BaseVolumesMinionMachineAttachmentTask, + 'get_required_task_info_properties', + mock_super_call, + ): + result = mp_tasks._BaseAttachVolumesToTransferMinionTask.get_required_task_info_properties() self.assertEqual( - sorted(result), sorted(['field1', 'field2', 'volumes_info'])) + sorted(result), sorted(['field1', 'field2', 'volumes_info']) + ) def test_get_returned_task_info_properties(self): mock_super_call = mock.MagicMock(return_value=["field1", "field2"]) with mock.patch.object( - mp_tasks._BaseVolumesMinionMachineAttachmentTask, - 'get_returned_task_info_properties', - mock_super_call): - result = mp_tasks._BaseAttachVolumesToTransferMinionTask.\ - get_returned_task_info_properties() + mp_tasks._BaseVolumesMinionMachineAttachmentTask, + 'get_returned_task_info_properties', + mock_super_call, + ): + result = mp_tasks._BaseAttachVolumesToTransferMinionTask.get_returned_task_info_properties() self.assertEqual( - sorted(result), sorted(['field1', 'field2', 'volumes_info'])) + sorted(result), sorted(['field1', 'field2', 'volumes_info']) + ) @ddt.ddt -class AttachVolumesToOSMorphingMinionTaskTestCase( - test_base.CoriolisBaseTestCase): - +class AttachVolumesToOSMorphingMinionTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(AttachVolumesToOSMorphingMinionTaskTestCase, self).setUp() self.task_runner = mp_tasks.AttachVolumesToOSMorphingMinionTask() @@ -469,41 +507,42 @@ def setUp(self): def test_get_required_task_info_properties(self): mock_super_call = mock.MagicMock(return_value=["field1", "field2"]) with mock.patch.object( - mp_tasks._BaseVolumesMinionMachineAttachmentTask, - 'get_required_task_info_properties', - mock_super_call): - result = mp_tasks.AttachVolumesToOSMorphingMinionTask.\ - get_required_task_info_properties() + mp_tasks._BaseVolumesMinionMachineAttachmentTask, + 'get_required_task_info_properties', + mock_super_call, + ): + result = mp_tasks.AttachVolumesToOSMorphingMinionTask.get_required_task_info_properties() self.assertEqual( - sorted(result), - sorted(['field1', 'field2', 'instance_deployment_info'])) + sorted(result), sorted(['field1', 'field2', 'instance_deployment_info']) + ) def test_get_returned_task_info_properties(self): mock_super_call = mock.MagicMock(return_value=["field1", "field2"]) with mock.patch.object( - mp_tasks._BaseVolumesMinionMachineAttachmentTask, - 'get_returned_task_info_properties', - mock_super_call): - result = mp_tasks.AttachVolumesToOSMorphingMinionTask.\ - get_returned_task_info_properties() + mp_tasks._BaseVolumesMinionMachineAttachmentTask, + 'get_returned_task_info_properties', + mock_super_call, + ): + result = mp_tasks.AttachVolumesToOSMorphingMinionTask.get_returned_task_info_properties() self.assertEqual( - sorted(result), - sorted(['field1', 'field2', 'instance_deployment_info'])) + sorted(result), sorted(['field1', 'field2', 'instance_deployment_info']) + ) - @mock.patch.object( - mp_tasks._BaseVolumesMinionMachineAttachmentTask, '_run') + @mock.patch.object(mp_tasks._BaseVolumesMinionMachineAttachmentTask, '_run') @ddt.data( { "task_info": {"instance_deployment_info": {}}, "res": {"optional": "values"}, - "expected_result": {"optional": "values"} + "expected_result": {"optional": "values"}, }, { "task_info": {"instance_deployment_info": {}}, "res": {"volumes_info": {"vol1": "id1", "vol2": "id2"}}, "expected_result": { "instance_deployment_info": { - "volumes_info": {"vol1": "id1", "vol2": "id2"}}} + "volumes_info": {"vol1": "id1", "vol2": "id2"} + } + }, }, ) def test__run(self, data, mock_super_run): @@ -512,8 +551,12 @@ def test__run(self, data, mock_super_run): expected_result = data.get('expected_result', {}) mock_super_run.return_value = res args = [ - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler, + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, ] result = self.task_runner._run(*args) @@ -522,30 +565,30 @@ def test__run(self, data, mock_super_run): @ddt.ddt -class _BaseValidateMinionCompatibilityTaskTestCase( - CoriolisBaseMinionPoolTaskTestCase): - +class _BaseValidateMinionCompatibilityTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): def setUp(self): super(_BaseValidateMinionCompatibilityTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseValidateMinionCompatibilityTask() self._setup_base_mocks() self._setup_validation_tasks_mocks() - @mock.patch.object(mp_tasks._BaseValidateMinionCompatibilityTask, - '_get_transfer_properties_task_info_field', - return_value="target_environment") + @mock.patch.object( + mp_tasks._BaseValidateMinionCompatibilityTask, + '_get_transfer_properties_task_info_field', + return_value="target_environment", + ) def test_get_required_task_info_properties(self, _): expected_result = [ - "export_info", "target_environment", + "export_info", + "target_environment", "destination_minion_provider_properties", - "destination_minion_backup_writer_connection_info"] - result = mp_tasks._BaseValidateMinionCompatibilityTask\ - .get_required_task_info_properties() + "destination_minion_backup_writer_connection_info", + ] + result = mp_tasks._BaseValidateMinionCompatibilityTask.get_required_task_info_properties() self.assertEqual(sorted(result), sorted(expected_result)) def test_get_returned_task_info_properties(self): - expected_result = [ - "target_resources", "target_resources_connection_info"] + expected_result = ["target_resources", "target_resources_connection_info"] result = self.task_runner.__class__.get_returned_task_info_properties() self.assertEqual(sorted(result), sorted(expected_result)) @@ -555,44 +598,54 @@ def test_get_returned_task_info_properties(self): ("invalid", None), ) def test__get_transfer_properties_task_info_field(self, data): - test_fun = mp_tasks._BaseValidateMinionCompatibilityTask\ - ._get_transfer_properties_task_info_field - with mock.patch.object(mp_tasks._BaseValidateMinionCompatibilityTask, - 'get_required_platform', return_value=data[0]): + test_fun = mp_tasks._BaseValidateMinionCompatibilityTask._get_transfer_properties_task_info_field + with mock.patch.object( + mp_tasks._BaseValidateMinionCompatibilityTask, + 'get_required_platform', + return_value=data[0], + ): if not data[1]: self.assertRaises(exception.CoriolisException, test_fun) return self.assertEqual(test_fun(), data[1]) - @mock.patch.object(mp_tasks._BaseValidateMinionCompatibilityTask, - '_get_transfer_properties_task_info_field', - return_value="target_environment") + @mock.patch.object( + mp_tasks._BaseValidateMinionCompatibilityTask, + '_get_transfer_properties_task_info_field', + return_value="target_environment", + ) def test__run(self, _): prov_fun = self.provider_pool_validation_op.return_value expected_result = { "target_resources": self.task_info['minion_properties'], "target_resources_connection_info": self.task_info[ - 'target_resources_connection_info'], + 'target_resources_connection_info' + ], } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.minion_pool_machine_id, - mock.sentinel.origin, self.destination, self.task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.minion_pool_machine_id, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) self.mock_get_provider.assert_called_once_with( - self.destination['type'], self.provider_type, - mock.sentinel.event_handler) + self.destination['type'], self.provider_type, mock.sentinel.event_handler + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info["export_info"], self.task_info["target_environment"], - self.task_info[MOCK_MINION_PROPS_TASK_INFO_FIELD]) + self.task_info[MOCK_MINION_PROPS_TASK_INFO_FIELD], + ) class _BaseReleaseMinionTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(_BaseReleaseMinionTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseReleaseMinionTask() @@ -600,22 +653,28 @@ def setUp(self): @mock.patch.object( mp_tasks._BaseReleaseMinionTask, '_get_minion_task_info_field_mappings', - return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS) + return_value=mp_tasks.TARGET_MINION_TASK_INFO_FIELD_MAPPINGS, + ) def test_get_required_task_info_properties(self, _): expected_result = [ - "destination_minion_provider_properties", "target_resources", + "destination_minion_provider_properties", + "target_resources", "destination_minion_backup_writer_connection_info", - "target_resources_connection_info"] + "target_resources_connection_info", + ] result = self.task_runner.__class__.get_required_task_info_properties() self.assertEqual(sorted(result), sorted(expected_result)) - @mock.patch.object(mp_tasks._BaseReleaseMinionTask, - 'get_returned_task_info_properties') + @mock.patch.object( + mp_tasks._BaseReleaseMinionTask, 'get_returned_task_info_properties' + ) def test__run(self, mock_returned_props): mock_returned_props.return_value = [ - "destination_minion_provider_properties", "target_resources", + "destination_minion_provider_properties", + "target_resources", "destination_minion_backup_writer_connection_info", - "target_resources_connection_info"] + "target_resources_connection_info", + ] expected_result = { "destination_minion_provider_properties": None, "target_resources": None, @@ -624,15 +683,18 @@ def test__run(self, mock_returned_props): } mock_event_handler = mock.MagicMock() result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.task_info, - mock_event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.task_info, + mock_event_handler, + ) self.assertEqual(result, expected_result) @ddt.ddt class CollectOSMorphingInfoTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): - def setUp(self): super(CollectOSMorphingInfoTaskTestCase, self).setUp() self.task_runner = mp_tasks.CollectOSMorphingInfoTask() @@ -644,16 +706,19 @@ def setUp(self): ({"osmorphing_info": {"some": "info"}}, False), ) def test__run(self, data): - prov_fun = self.mock_get_provider.return_value\ - .get_additional_os_morphing_info + prov_fun = self.mock_get_provider.return_value.get_additional_os_morphing_info prov_fun.return_value = data[0] raises_exception = data[1] args = [ - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - self.destination, self.task_info, mock.sentinel.event_handler] + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ] if raises_exception: - self.assertRaises( - exception.CoriolisException, self.task_runner._run, *args) + self.assertRaises(exception.CoriolisException, self.task_runner._run, *args) return expected_result = { @@ -662,50 +727,65 @@ def test__run(self, data): result = self.task_runner._run(*args) self.assertEqual(result, expected_result) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info['target_environment'], - self.task_info['instance_deployment_info']) - + self.task_info['instance_deployment_info'], + ) -class _BaseHealthcheckMinionMachineTaskTestCase( - CoriolisBaseMinionPoolTaskTestCase): +class _BaseHealthcheckMinionMachineTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): def setUp(self): super(_BaseHealthcheckMinionMachineTaskTestCase, self).setUp() self.task_runner = mp_tasks._BaseHealthcheckMinionMachineTask() self._setup_base_mocks() - @mock.patch('coriolis.tasks.base.unmarshal_migr_conn_info', - return_value=MOCK_UNMARSHALED_INFO) + @mock.patch( + 'coriolis.tasks.base.unmarshal_migr_conn_info', + return_value=MOCK_UNMARSHALED_INFO, + ) def test__run(self, mock_unmarshal_conn_info): prov_fun = self.mock_get_provider.return_value.healthcheck_minion result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - self.destination, self.task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_unmarshal_conn_info.assert_called_once_with( - self.task_info['minion_connection_info']) + self.task_info['minion_connection_info'] + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, self.task_info['minion_properties'], - mock_unmarshal_conn_info.return_value) + mock_unmarshal_conn_info.return_value, + ) class _BasePowerCycleMinionTaskTestCase(CoriolisBaseMinionPoolTaskTestCase): - def setUp(self): super(_BasePowerCycleMinionTaskTestCase, self).setUp() self.task_runner = mp_tasks._BasePowerCycleMinionTask() self._setup_base_mocks() - @mock.patch.object(mp_tasks._BasePowerCycleMinionTask, - '_get_minion_power_cycle_op') + @mock.patch.object(mp_tasks._BasePowerCycleMinionTask, '_get_minion_power_cycle_op') def test__run(self, mock_get_power_op): prov_fun = mock_get_power_op.return_value result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - self.destination, self.task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + self.destination, + self.task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, self.mock_get_conn_info.return_value, - self.task_info['minion_provider_properties']) + mock.sentinel.ctxt, + self.mock_get_conn_info.return_value, + self.task_info['minion_provider_properties'], + ) diff --git a/coriolis/tests/tasks/test_osmorphing_tasks.py b/coriolis/tests/tasks/test_osmorphing_tasks.py index 13402f45..ddb7a378 100644 --- a/coriolis/tests/tasks/test_osmorphing_tasks.py +++ b/coriolis/tests/tasks/test_osmorphing_tasks.py @@ -5,16 +5,13 @@ import ddt -from coriolis import constants -from coriolis import exception -from coriolis import schemas +from coriolis import constants, exception, schemas from coriolis.tasks import osmorphing_tasks from coriolis.tests import test_base @ddt.ddt class OSMorphingTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(OSMorphingTaskTestCase, self).setUp() self.task_runner = osmorphing_tasks.OSMorphingTask() @@ -24,10 +21,18 @@ def setUp(self): @mock.patch('coriolis.osmorphing.manager.morph_image') @ddt.file_data('data/osmorphing_task_run.yml') @ddt.unpack - def test__run(self, mock_morph_image, mock_unmarshal, mock_get_provider, - config, expected_instance_script): - mock_get_provider.side_effect = [mock.sentinel.origin_provider, - mock.sentinel.destination_provider] + def test__run( + self, + mock_morph_image, + mock_unmarshal, + mock_get_provider, + config, + expected_instance_script, + ): + mock_get_provider.side_effect = [ + mock.sentinel.origin_provider, + mock.sentinel.destination_provider, + ] instance = config['instance'] task_info = config['task_info'] osmorphing_info = task_info.get('osmorphing_info', {}) @@ -35,29 +40,40 @@ def test__run(self, mock_morph_image, mock_unmarshal, mock_get_provider, destination = mock.MagicMock() expected_calls = [ mock.call.mock_get_provider( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler), + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ), mock.call.mock_get_provider( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler), + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ), ] result = self.task_runner._run( - mock.sentinel.ctxt, instance, origin, destination, task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + instance, + origin, + destination, + task_info, + mock.sentinel.event_handler, + ) mock_get_provider.assert_has_calls(expected_calls) - mock_unmarshal.assert_called_once_with( - task_info['osmorphing_connection_info']) + mock_unmarshal.assert_called_once_with(task_info['osmorphing_connection_info']) mock_morph_image.assert_called_once_with( - mock.sentinel.origin_provider, mock.sentinel.destination_provider, - mock_unmarshal.return_value, osmorphing_info, - expected_instance_script, mock.sentinel.event_handler) + mock.sentinel.origin_provider, + mock.sentinel.destination_provider, + mock_unmarshal.return_value, + osmorphing_info, + expected_instance_script, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) @ddt.ddt class DeployOSMorphingResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeployOSMorphingResourcesTaskTestCase, self).setUp() self.task_runner = osmorphing_tasks.DeployOSMorphingResourcesTask() @@ -68,56 +84,73 @@ def setUp(self): @mock.patch('coriolis.tasks.base.marshal_migr_conn_info') @ddt.file_data('data/deploy_osmorphing_task_run.yml') @ddt.unpack - def test__run(self, mock_marshal_conn_info, mock_validate_value, - mock_get_conn_info, mock_get_provider, import_info, - raise_expected, log_expected): + def test__run( + self, + mock_marshal_conn_info, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + import_info, + raise_expected, + log_expected, + ): prov_fun = mock_get_provider.return_value.deploy_os_morphing_resources prov_fun.return_value = import_info destination = mock.MagicMock() task_info = mock.MagicMock() method_args = [ - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler] + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ] def _get_result(*args): if log_expected: - with self.assertLogs('coriolis.tasks.osmorphing_tasks', - level=osmorphing_tasks.logging.WARN): + with self.assertLogs( + 'coriolis.tasks.osmorphing_tasks', + level=osmorphing_tasks.logging.WARN, + ): return self.task_runner._run(*args) return self.task_runner._run(*args) if raise_expected: self.assertRaises( - exception.InvalidTaskResult, - self.task_runner._run, *method_args) + exception.InvalidTaskResult, self.task_runner._run, *method_args + ) mock_marshal_conn_info.assert_not_called() else: expected_result = { - "os_morphing_resources": import_info.get( - 'os_morphing_resources'), - "osmorphing_connection_info": ( - mock_marshal_conn_info.return_value), + "os_morphing_resources": import_info.get('os_morphing_resources'), + "osmorphing_connection_info": (mock_marshal_conn_info.return_value), "osmorphing_info": import_info.get('osmorphing_info', {}), } self.assertEqual(expected_result, _get_result(*method_args)) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_OS_MORPHING, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_OS_MORPHING, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, task_info['target_environment'], - task_info['instance_deployment_info']) + task_info['instance_deployment_info'], + ) mock_validate_value.assert_called_once_with( - import_info, schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA, - raise_on_error=False) + import_info, + schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA, + raise_on_error=False, + ) mock_marshal_conn_info.assert_called_once_with( - import_info.get('osmorphing_connection_info')) + import_info.get('osmorphing_connection_info') + ) class DeleteOSMorphingResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeleteOSMorphingResourcesTaskTestCase, self).setUp() self.task_runner = osmorphing_tasks.DeleteOSMorphingResourcesTask() @@ -137,15 +170,23 @@ def test__run(self, mock_get_conn_info, mock_get_provider): "osmorphing_connection_info": None, } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_OS_MORPHING, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_OS_MORPHING, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, task_info['target_environment'], - task_info['os_morphing_resources']) + task_info['os_morphing_resources'], + ) diff --git a/coriolis/tests/tasks/test_replica_tasks.py b/coriolis/tests/tasks/test_replica_tasks.py index f31e04be..91348f33 100644 --- a/coriolis/tests/tasks/test_replica_tasks.py +++ b/coriolis/tests/tasks/test_replica_tasks.py @@ -6,17 +6,14 @@ import ddt -from coriolis import constants -from coriolis import exception +from coriolis import constants, exception, schemas from coriolis.providers import backup_writers -from coriolis import schemas from coriolis.tasks import replica_tasks from coriolis.tests import test_base @ddt.ddt class ReplicaTasksTestCase(test_base.CoriolisBaseTestCase): - @ddt.data( ({"volumes_info": []}, True), ({"volumes_info": None}, True), @@ -26,8 +23,11 @@ class ReplicaTasksTestCase(test_base.CoriolisBaseTestCase): def test__get_volumes_info(self, data): task_info = data[0] if data[1]: - self.assertRaises(exception.InvalidActionTasksExecutionState, - replica_tasks._get_volumes_info, task_info) + self.assertRaises( + exception.InvalidActionTasksExecutionState, + replica_tasks._get_volumes_info, + task_info, + ) else: result = replica_tasks._get_volumes_info(task_info) self.assertEqual(result, task_info['volumes_info']) @@ -36,8 +36,13 @@ def test__get_volumes_info(self, data): @ddt.file_data("data/check_ensure_volumes_info_ordering.yml") @ddt.unpack def test__check_ensure_volumes_info_ordering( - self, mock_sanitize, export_info, volumes_info, - exception_expected, expected_result): + self, + mock_sanitize, + export_info, + volumes_info, + exception_expected, + expected_result, + ): mock_sanitize.side_effect = lambda x: x expected_calls = [ mock.call({"volumes_info": volumes_info}), @@ -47,31 +52,36 @@ def test__check_ensure_volumes_info_ordering( self.assertRaises( exception.InvalidActionTasksExecutionState, replica_tasks._check_ensure_volumes_info_ordering, - export_info, volumes_info) + export_info, + volumes_info, + ) else: result = replica_tasks._check_ensure_volumes_info_ordering( - export_info, volumes_info) + export_info, volumes_info + ) mock_sanitize.assert_has_calls(expected_calls) self.assertEqual(result, expected_result) @ddt.file_data("data/test_nic_ips_update.yml") @ddt.unpack def test__preserve_old_export_info_nic_ips( - self, old_export_info, new_export_info, expected_export_info): + self, old_export_info, new_export_info, expected_export_info + ): replica_tasks._preserve_old_export_info_nic_ips( - old_export_info, new_export_info) + old_export_info, new_export_info + ) self.assertEqual(new_export_info, expected_export_info) @ddt.file_data("data/test_hostname_update.yml") @ddt.unpack def test__preserve_hostname_info( - self, old_export_info, new_export_info, expected_export_info): + self, old_export_info, new_export_info, expected_export_info + ): replica_tasks._preserve_hostname_info(old_export_info, new_export_info) self.assertEqual(new_export_info, expected_export_info) class GetInstanceInfoTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(GetInstanceInfoTaskTestCase, self).setUp() self.task_runner = replica_tasks.GetInstanceInfoTask() @@ -79,30 +89,39 @@ def setUp(self): @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') - def test__run(self, mock_validate_value, mock_get_conn_info, - mock_get_provider): + def test__run(self, mock_validate_value, mock_get_conn_info, mock_get_provider): origin = mock.MagicMock() task_info = mock.MagicMock() prov_fun = mock_get_provider.return_value.get_replica_instance_info expected_result = {"export_info": prov_fun.return_value} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destiantion, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destiantion, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['source_environment'], mock.sentinel.instance) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['source_environment'], + mock.sentinel.instance, + ) mock_validate_value.assert_called_once_with( - prov_fun.return_value, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA + ) class ShutdownInstanceTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(ShutdownInstanceTaskTestCase, self).setUp() self.task_runner = replica_tasks.ShutdownInstanceTask() @@ -115,20 +134,29 @@ def test__run(self, mock_get_conn_info, mock_get_provider): prov_fun = mock_get_provider.return_value.shutdown_instance result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destiantion, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destiantion, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['source_environment'], mock.sentinel.instance) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['source_environment'], + mock.sentinel.instance, + ) class ReplicateDisksTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(ReplicateDisksTaskTestCase, self).setUp() self.task_runner = replica_tasks.ReplicateDisksTask() @@ -139,58 +167,82 @@ def setUp(self): @mock.patch.object(replica_tasks, '_get_volumes_info') @mock.patch.object(replica_tasks, '_check_ensure_volumes_info_ordering') @mock.patch('coriolis.tasks.base.unmarshal_migr_conn_info') - def test__run(self, mock_unmarshal, mock_check_vol_info, mock_get_vol_info, - mock_validate_value, mock_get_conn_info, mock_get_provider): + def test__run( + self, + mock_unmarshal, + mock_check_vol_info, + mock_get_vol_info, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + ): origin = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [ - task_info['incremental'], task_info['source_resources']] + task_info['incremental'], + task_info['source_resources'], + ] prov_fun = mock_get_provider.return_value.replicate_disks expected_result = {"volumes_info": mock_check_vol_info.return_value} expected_validation_calls = [ mock.call.mock_validate_value( {"volumes_info": mock_get_vol_info.return_value}, - schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA), + schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA, + ), mock.call.mock_validate_value( task_info['source_resources_connection_info'], - schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA), + schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA, + ), mock.call.mock_validate_value( task_info['target_resources_connection_info'], - schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA), + schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA, + ), mock.call.mock_validate_value( - prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA), + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ), ] expected_unmarshal_calls = [ + mock.call.mock_unmarshal(task_info['source_resources_connection_info']), mock.call.mock_unmarshal( - task_info['source_resources_connection_info']), - mock.call.mock_unmarshal( - task_info['target_resources_connection_info'][ - 'connection_details']), + task_info['target_resources_connection_info']['connection_details'] + ), ] result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destiantion, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destiantion, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) mock_get_vol_info.assert_called_once_with(task_info) mock_validate_value.assert_has_calls(expected_validation_calls) mock_unmarshal.assert_has_calls(expected_unmarshal_calls) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['source_environment'], mock.sentinel.instance, - task_info['source_resources'], mock_unmarshal.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['source_environment'], + mock.sentinel.instance, + task_info['source_resources'], + mock_unmarshal.return_value, task_info['target_resources_connection_info'], - mock_get_vol_info.return_value, task_info['incremental']) + mock_get_vol_info.return_value, + task_info['incremental'], + ) mock_check_vol_info.assert_called_once_with( - task_info['export_info'], prov_fun.return_value) + task_info['export_info'], prov_fun.return_value + ) class DeployReplicaDisksTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeployReplicaDisksTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeployReplicaDisksTask() @@ -199,8 +251,13 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') @mock.patch.object(replica_tasks, '_check_ensure_volumes_info_ordering') - def test__run(self, mock_check_vol_info, mock_validate_value, - mock_get_conn_info, mock_get_provider): + def test__run( + self, + mock_check_vol_info, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + ): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [task_info['volumes_info']] @@ -208,27 +265,37 @@ def test__run(self, mock_check_vol_info, mock_validate_value, expected_result = {"volumes_info": mock_check_vol_info.return_value} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock.sentinel.instance, - task_info['export_info'], task_info['volumes_info']) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock.sentinel.instance, + task_info['export_info'], + task_info['volumes_info'], + ) mock_validate_value.assert_called_once_with( - prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) mock_check_vol_info.assert_called_once_with( - task_info['export_info'], prov_fun.return_value) + task_info['export_info'], prov_fun.return_value + ) -class DeleteReplicaSourceDiskSnapshotsTaskTestCase( - test_base.CoriolisBaseTestCase): - +class DeleteReplicaSourceDiskSnapshotsTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(DeleteReplicaSourceDiskSnapshotsTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeleteReplicaSourceDiskSnapshotsTask() @@ -237,27 +304,34 @@ def setUp(self): @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch.object(replica_tasks, '_get_volumes_info') - def test__run(self, mock_get_vol_info, mock_get_conn_info, - mock_get_provider, _): + def test__run(self, mock_get_vol_info, mock_get_conn_info, mock_get_provider, _): origin = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [task_info['volumes_info']] - prov_fun = mock_get_provider.return_value\ - .delete_replica_source_snapshots + prov_fun = mock_get_provider.return_value.delete_replica_source_snapshots expected_result = {"volumes_info": prov_fun.return_value} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, origin) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['source_environment'], mock_get_vol_info.return_value) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['source_environment'], + mock_get_vol_info.return_value, + ) mock_get_vol_info.assert_called_once_with(task_info) @mock.patch('coriolis.events.EventManager') @@ -265,13 +339,17 @@ def test__run_no_volumes_info(self, _): task_info = {} expected_result = {"volumes_info": []} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) class DeleteReplicaDisksTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeleteReplicaDisksTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeleteReplicaDisksTask() @@ -281,8 +359,7 @@ def setUp(self): @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch.object(replica_tasks, '_get_volumes_info') - def test__run(self, mock_get_vol_info, mock_get_conn_info, - mock_get_provider): + def test__run(self, mock_get_vol_info, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [task_info['volumes_info']] @@ -290,17 +367,26 @@ def test__run(self, mock_get_vol_info, mock_get_conn_info, expected_result = {"volumes_info": []} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock_get_vol_info.return_value) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock_get_vol_info.return_value, + ) mock_get_vol_info.assert_called_once_with(task_info) @mock.patch('coriolis.events.EventManager') @@ -308,14 +394,18 @@ def test__run_no_volumes_info(self, _): task_info = {} expected_result = {"volumes_info": []} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) @ddt.ddt class DeployReplicaSourceResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeployReplicaSourceResourcesTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeployReplicaSourceResourcesTask() @@ -326,26 +416,36 @@ def setUp(self): @mock.patch('coriolis.tasks.base.marshal_migr_conn_info') @ddt.file_data('data/deploy_replica_source_resources_task_run.yml') @ddt.unpack - def test__run(self, mock_marshal, mock_validate_value, mock_get_conn_info, - mock_get_provider, log_expected, replica_resources_info, - expected_result): + def test__run( + self, + mock_marshal, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + log_expected, + replica_resources_info, + expected_result, + ): origin = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [task_info['source_environment']] - prov_fun = ( - mock_get_provider.return_value.deploy_replica_source_resources) + prov_fun = mock_get_provider.return_value.deploy_replica_source_resources prov_fun.return_value = replica_resources_info - mock_marshal.return_value = replica_resources_info.get( - 'connection_info', {}) + mock_marshal.return_value = replica_resources_info.get('connection_info', {}) def _get_result(): args = [ - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, - mock.sentinel.event_handler] + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ] if log_expected: - with self.assertLogs('coriolis.tasks.replica_tasks', - level=replica_tasks.logging.WARN): + with self.assertLogs( + 'coriolis.tasks.replica_tasks', level=replica_tasks.logging.WARN + ): result = self.task_runner._run(*args) mock_marshal.assert_not_called() mock_validate_value.assert_not_called() @@ -353,27 +453,32 @@ def _get_result(): result = self.task_runner._run(*args) mock_marshal.assert_called_once_with( - replica_resources_info['connection_info']) + replica_resources_info['connection_info'] + ) mock_validate_value.assert_called_once_with( mock_marshal.return_value, schemas.CORIOLIS_REPLICATION_WORKER_CONN_INFO_SCHEMA, - raise_on_error=False) + raise_on_error=False, + ) return result self.assertEqual(_get_result(), expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, origin) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['export_info'], task_info['source_environment']) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['export_info'], + task_info['source_environment'], + ) @ddt.ddt class DeleteReplicaSourceResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeleteReplicaSourceResourcesTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeleteReplicaSourceResourcesTask() @@ -383,49 +488,65 @@ def setUp(self): def test__run(self, mock_get_conn_info, mock_get_provider): origin = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = ( - mock_get_provider.return_value.delete_replica_source_resources) + prov_fun = mock_get_provider.return_value.delete_replica_source_resources expected_result = { - "source_resources": None, "source_resources_connection_info": None} + "source_resources": None, + "source_resources_connection_info": None, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, origin) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - origin['source_environment'], task_info['source_resources']) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + origin['source_environment'], + task_info['source_resources'], + ) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') def test__run_no_resources(self, mock_get_conn_info, mock_get_provider): origin = mock.MagicMock() task_info = {"source_resources": None} - prov_fun = ( - mock_get_provider.return_value.delete_replica_source_resources) + prov_fun = mock_get_provider.return_value.delete_replica_source_resources expected_result = { - "source_resources": None, "source_resources_connection_info": None} + "source_resources": None, + "source_resources_connection_info": None, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_TRANSFER_EXPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, origin) + origin['type'], + constants.PROVIDER_TYPE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_not_called() @ddt.ddt class DeployReplicaTargetResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeployReplicaTargetResourcesTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeployReplicaTargetResourcesTask() @@ -435,73 +556,95 @@ def setUp(self): @mock.patch('coriolis.schemas.validate_value') @ddt.file_data("data/replica_deployment_conn_info_validation.yml") @ddt.unpack - def test__validate_connection_info(self, mock_validate, mock_marshal, - mock_backup_writer, migr_conn_info, - writer_fails, validates_conn_info, - marshals_conn_info): + def test__validate_connection_info( + self, + mock_validate, + mock_marshal, + mock_backup_writer, + migr_conn_info, + writer_fails, + validates_conn_info, + marshals_conn_info, + ): conn_details = copy.copy(migr_conn_info.get('connection_details')) if writer_fails: mock_backup_writer.side_effect = Exception - with self.assertLogs('coriolis.tasks.replica_tasks', - level=replica_tasks.logging.WARN): + with self.assertLogs( + 'coriolis.tasks.replica_tasks', level=replica_tasks.logging.WARN + ): self.task_runner._validate_connection_info(migr_conn_info) else: self.assertEqual( - None, - self.task_runner._validate_connection_info(migr_conn_info)) + None, self.task_runner._validate_connection_info(migr_conn_info) + ) if validates_conn_info: mock_validate.assert_called_once_with( migr_conn_info, schemas.CORIOLIS_DISK_SYNC_RESOURCES_CONN_INFO_SCHEMA, - raise_on_error=False) + raise_on_error=False, + ) if marshals_conn_info: mock_marshal.assert_called_once_with(conn_details) - @mock.patch.object(replica_tasks.DeployReplicaTargetResourcesTask, - '_validate_connection_info') + @mock.patch.object( + replica_tasks.DeployReplicaTargetResourcesTask, '_validate_connection_info' + ) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.tasks.replica_tasks._get_volumes_info') - @mock.patch( - 'coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') + @mock.patch('coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') @mock.patch('coriolis.schemas.validate_value') @ddt.file_data('data/deploy_replica_target_resources_task_run.yml') @ddt.unpack - def test__run(self, mock_validate_value, mock_check_vol_info, - mock_get_vol_info, mock_get_conn_info, mock_get_provider, - mock_validate_conn_info, replica_resources_info, - expected_result): + def test__run( + self, + mock_validate_value, + mock_check_vol_info, + mock_get_vol_info, + mock_get_conn_info, + mock_get_provider, + mock_validate_conn_info, + replica_resources_info, + expected_result, + ): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = ( - mock_get_provider.return_value.deploy_replica_target_resources) + prov_fun = mock_get_provider.return_value.deploy_replica_target_resources prov_fun.return_value = replica_resources_info - mock_check_vol_info.return_value = ( - replica_resources_info.get('volumes_info')) + mock_check_vol_info.return_value = replica_resources_info.get('volumes_info') mock_get_vol_info.return_value = "task_vol_info" result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_get_vol_info.assert_called_once_with(task_info) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock_get_vol_info.return_value) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock_get_vol_info.return_value, + ) mock_validate_value.assert_called_once_with( replica_resources_info, schemas.CORIOLIS_DISK_SYNC_RESOURCES_INFO_SCHEMA, - raise_on_error=False) + raise_on_error=False, + ) @ddt.ddt class DeleteReplicaTargetResourcesTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(DeleteReplicaTargetResourcesTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeleteReplicaTargetResourcesTask() @@ -517,31 +660,39 @@ def test__run(self, data, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.return_value = data[0] - prov_fun = ( - mock_get_provider.return_value.delete_replica_target_resources) + prov_fun = mock_get_provider.return_value.delete_replica_target_resources expected_result = { - "target_resources": None, "target_resources_connection_info": None} + "target_resources": None, + "target_resources_connection_info": None, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) if data[1]: prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], data[0]) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + data[0], + ) else: prov_fun.assert_not_called() -class DeployReplicaInstanceResourcesTaskTestCase( - test_base.CoriolisBaseTestCase): - +class DeployReplicaInstanceResourcesTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(DeployReplicaInstanceResourcesTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeployReplicaInstanceResourcesTask() @@ -549,125 +700,155 @@ def setUp(self): @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.tasks.replica_tasks._get_volumes_info') - def test__run(self, mock_get_vol_info, mock_get_conn_info, - mock_get_provider): + def test__run(self, mock_get_vol_info, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.return_value = True prov_fun = mock_get_provider.return_value.deploy_replica_instance expected_result = { "instance_deployment_info": ( - prov_fun.return_value['instance_deployment_info'])} + prov_fun.return_value['instance_deployment_info'] + ) + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_get_vol_info.assert_called_once_with(task_info) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock.sentinel.instance, - task_info['export_info'], mock_get_vol_info.return_value, True) - + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock.sentinel.instance, + task_info['export_info'], + mock_get_vol_info.return_value, + True, + ) -class FinalizeReplicaInstanceDeploymentTaskTestCase( - test_base.CoriolisBaseTestCase): +class FinalizeReplicaInstanceDeploymentTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(FinalizeReplicaInstanceDeploymentTaskTestCase, self).setUp() - self.task_runner = ( - replica_tasks.FinalizeReplicaInstanceDeploymentTask()) + self.task_runner = replica_tasks.FinalizeReplicaInstanceDeploymentTask() @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') def test__run(self, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = mock_get_provider.return_value\ - .finalize_replica_instance_deployment + prov_fun = mock_get_provider.return_value.finalize_replica_instance_deployment expected_result = {"transfer_result": prov_fun.return_value} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, task_info['target_environment'], - task_info['instance_deployment_info']) + task_info['instance_deployment_info'], + ) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') def test__run_no_result(self, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = mock_get_provider.return_value\ - .finalize_replica_instance_deployment + prov_fun = mock_get_provider.return_value.finalize_replica_instance_deployment prov_fun.return_value = None expected_result = {"transfer_result": None} - with self.assertLogs('coriolis.tasks.replica_tasks', - level=replica_tasks.logging.WARN): + with self.assertLogs( + 'coriolis.tasks.replica_tasks', level=replica_tasks.logging.WARN + ): result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, - mock.sentinel.origin, destination, task_info, - mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, task_info['target_environment'], - task_info['instance_deployment_info']) + task_info['instance_deployment_info'], + ) class CleanupFailedReplicaInstanceDeploymentTaskTestCase( - test_base.CoriolisBaseTestCase): - + test_base.CoriolisBaseTestCase +): def setUp(self): super(CleanupFailedReplicaInstanceDeploymentTaskTestCase, self).setUp() - self.task_runner = ( - replica_tasks.CleanupFailedReplicaInstanceDeploymentTask()) + self.task_runner = replica_tasks.CleanupFailedReplicaInstanceDeploymentTask() @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') def test__run(self, mock_get_conn_info, mock_get_provider): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = mock_get_provider.return_value\ - .cleanup_failed_replica_instance_deployment + prov_fun = ( + mock_get_provider.return_value.cleanup_failed_replica_instance_deployment + ) expected_result = {"instance_deployment_info": None} result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, task_info['target_environment'], - task_info['instance_deployment_info']) + task_info['instance_deployment_info'], + ) class CreateReplicaDiskSnapshotsTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(CreateReplicaDiskSnapshotsTaskTestCase, self).setUp() self.task_runner = replica_tasks.CreateReplicaDiskSnapshotsTask() @@ -676,40 +857,53 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.tasks.replica_tasks._get_volumes_info') @mock.patch('coriolis.schemas.validate_value') - @mock.patch( - 'coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') - def test__run(self, mock_check_ensure_volumes_ordering, - mock_validate_value, mock_get_vol_info, mock_get_conn_info, - mock_get_provider): + @mock.patch('coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') + def test__run( + self, + mock_check_ensure_volumes_ordering, + mock_validate_value, + mock_get_vol_info, + mock_get_conn_info, + mock_get_provider, + ): destination = mock.MagicMock() task_info = mock.MagicMock() prov_fun = mock_get_provider.return_value.create_replica_disk_snapshots expected_result = { - "volumes_info": mock_check_ensure_volumes_ordering.return_value} + "volumes_info": mock_check_ensure_volumes_ordering.return_value + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_get_vol_info.assert_called_once_with(task_info) mock_validate_value.assert_called_once_with( - prov_fun.return_value, - schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) mock_check_ensure_volumes_ordering.assert_called_once_with( - task_info['export_info'], prov_fun.return_value) + task_info['export_info'], prov_fun.return_value + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock_get_vol_info.return_value) - + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock_get_vol_info.return_value, + ) -class DeleteReplicaTargetDiskSnapshotsTaskTestCase( - test_base.CoriolisBaseTestCase): +class DeleteReplicaTargetDiskSnapshotsTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(DeleteReplicaTargetDiskSnapshotsTaskTestCase, self).setUp() self.task_runner = replica_tasks.DeleteReplicaTargetDiskSnapshotsTask() @@ -718,40 +912,53 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.tasks.replica_tasks._get_volumes_info') @mock.patch('coriolis.schemas.validate_value') - @mock.patch( - 'coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') - def test__run(self, mock_check_ensure_volumes_ordering, - mock_validate_value, mock_get_vol_info, mock_get_conn_info, - mock_get_provider): + @mock.patch('coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') + def test__run( + self, + mock_check_ensure_volumes_ordering, + mock_validate_value, + mock_get_vol_info, + mock_get_conn_info, + mock_get_provider, + ): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = mock_get_provider.return_value\ - .delete_replica_target_disk_snapshots + prov_fun = mock_get_provider.return_value.delete_replica_target_disk_snapshots expected_result = { - "volumes_info": mock_check_ensure_volumes_ordering.return_value} + "volumes_info": mock_check_ensure_volumes_ordering.return_value + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_get_vol_info.assert_called_once_with(task_info) mock_validate_value.assert_called_once_with( - prov_fun.return_value, - schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) mock_check_ensure_volumes_ordering.assert_called_once_with( - task_info['export_info'], prov_fun.return_value) + task_info['export_info'], prov_fun.return_value + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock_get_vol_info.return_value) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock_get_vol_info.return_value, + ) class RestoreReplicaDiskSnapshotsTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(RestoreReplicaDiskSnapshotsTaskTestCase, self).setUp() self.task_runner = replica_tasks.RestoreReplicaDiskSnapshotsTask() @@ -760,260 +967,338 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.tasks.replica_tasks._get_volumes_info') @mock.patch('coriolis.schemas.validate_value') - @mock.patch( - 'coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') - def test__run(self, mock_check_ensure_volumes_ordering, - mock_validate_value, mock_get_vol_info, mock_get_conn_info, - mock_get_provider): + @mock.patch('coriolis.tasks.replica_tasks._check_ensure_volumes_info_ordering') + def test__run( + self, + mock_check_ensure_volumes_ordering, + mock_validate_value, + mock_get_vol_info, + mock_get_conn_info, + mock_get_provider, + ): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = mock_get_provider.return_value\ - .restore_replica_disk_snapshots + prov_fun = mock_get_provider.return_value.restore_replica_disk_snapshots expected_result = { - "volumes_info": mock_check_ensure_volumes_ordering.return_value} + "volumes_info": mock_check_ensure_volumes_ordering.return_value + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_get_provider.assert_called_once_with( - destination['type'], constants.PROVIDER_TYPE_TRANSFER_IMPORT, - mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + destination['type'], + constants.PROVIDER_TYPE_TRANSFER_IMPORT, + mock.sentinel.event_handler, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_get_vol_info.assert_called_once_with(task_info) mock_validate_value.assert_called_once_with( - prov_fun.return_value, - schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) mock_check_ensure_volumes_ordering.assert_called_once_with( - task_info['export_info'], prov_fun.return_value) + task_info['export_info'], prov_fun.return_value + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], mock_get_vol_info.return_value) - + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + mock_get_vol_info.return_value, + ) -class ValidateReplicaExecutionSourceInputsTaskTestCase( - test_base.CoriolisBaseTestCase): +class ValidateReplicaExecutionSourceInputsTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): super(ValidateReplicaExecutionSourceInputsTaskTestCase, self).setUp() - self.task_runner = ( - replica_tasks.ValidateReplicaExecutionSourceInputsTask()) + self.task_runner = replica_tasks.ValidateReplicaExecutionSourceInputsTask() @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') - def test__run(self, mock_get_conn_info, mock_get_provider, - mock_event_manager): + def test__run(self, mock_get_conn_info, mock_get_provider, mock_event_manager): origin = mock.MagicMock() task_info = mock.MagicMock() prov_fun = mock_get_provider.return_value.validate_replica_export_input result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, - mock.sentinel.event_handler, raise_if_not_found=False) + origin['type'], + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, mock.sentinel.instance, - source_environment=task_info['source_environment']) + source_environment=task_info['source_environment'], + ) mock_event_manager.return_value.progress_update.assert_not_called() @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') - def test__run_no_source_provider(self, mock_get_conn_info, - mock_get_provider, mock_event_manager): + def test__run_no_source_provider( + self, mock_get_conn_info, mock_get_provider, mock_event_manager + ): origin = mock.MagicMock() task_info = mock.MagicMock() prov_fun = mock_get_provider.return_value.validate_replica_export_input mock_get_provider.side_effect = [None] result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, - mock.sentinel.event_handler, raise_if_not_found=False) + origin['type'], + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) prov_fun.assert_not_called() mock_event_manager.return_value.progress_update.assert_called_once() class ValidateReplicaExecutionDestinationInputsTaskTestCase( - test_base.CoriolisBaseTestCase): - + test_base.CoriolisBaseTestCase +): def setUp(self): - super( - ValidateReplicaExecutionDestinationInputsTaskTestCase, - self).setUp() - self.task_runner = ( - replica_tasks.ValidateReplicaExecutionDestinationInputsTask()) + super(ValidateReplicaExecutionDestinationInputsTaskTestCase, self).setUp() + self.task_runner = replica_tasks.ValidateReplicaExecutionDestinationInputsTask() def test__validate_provider_replica_import_input(self): provider = mock.MagicMock() result = self.task_runner._validate_provider_replica_import_input( - provider, mock.sentinel.ctxt, mock.sentinel.conn_info, - mock.sentinel.target_environment, mock.sentinel.export_info) + provider, + mock.sentinel.ctxt, + mock.sentinel.conn_info, + mock.sentinel.target_environment, + mock.sentinel.export_info, + ) self.assertEqual(result, None) provider.validate_replica_import_input.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.conn_info, - mock.sentinel.target_environment, mock.sentinel.export_info, - check_os_morphing_resources=False, check_final_vm_params=False) + mock.sentinel.ctxt, + mock.sentinel.conn_info, + mock.sentinel.target_environment, + mock.sentinel.export_info, + check_os_morphing_resources=False, + check_final_vm_params=False, + ) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch.object( replica_tasks.ValidateReplicaExecutionDestinationInputsTask, - '_validate_provider_replica_import_input') - def test__run(self, mock_validate_replica_inputs, mock_get_conn_info, - mock_get_provider): + '_validate_provider_replica_import_input', + ) + def test__run( + self, mock_validate_replica_inputs, mock_get_conn_info, mock_get_provider + ): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [task_info['export_info']] result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, None) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + None, + ) self.assertEqual(result, {}) mock_get_provider.assert_called_once_with( destination['type'], - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, None, - raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, + None, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_replica_inputs.assert_called_once_with( - mock_get_provider.return_value, mock.sentinel.ctxt, - mock_get_conn_info.return_value, task_info['target_environment'], - task_info['export_info']) + mock_get_provider.return_value, + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + task_info['export_info'], + ) @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch.object( replica_tasks.ValidateReplicaExecutionDestinationInputsTask, - '_validate_provider_replica_import_input') + '_validate_provider_replica_import_input', + ) def test__run_no_destination_provider( - self, mock_validate_replica_inputs, mock_get_conn_info, - mock_get_provider): + self, mock_validate_replica_inputs, mock_get_conn_info, mock_get_provider + ): destination = mock.MagicMock() task_info = mock.MagicMock() mock_get_provider.return_value = None result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, None) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + None, + ) self.assertEqual(result, {}) mock_get_provider.assert_called_once_with( destination['type'], - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, None, - raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, + None, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_replica_inputs.assert_not_called() @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch.object( replica_tasks.ValidateReplicaExecutionDestinationInputsTask, - '_validate_provider_replica_import_input') + '_validate_provider_replica_import_input', + ) def test__run_no_export_info( - self, mock_validate_replica_inputs, mock_get_conn_info, - mock_get_provider): + self, mock_validate_replica_inputs, mock_get_conn_info, mock_get_provider + ): destination = mock.MagicMock() task_info = mock.MagicMock() task_info.get.side_effect = [None] self.assertRaises( - exception.InvalidActionTasksExecutionState, self.task_runner._run, - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, None) + exception.InvalidActionTasksExecutionState, + self.task_runner._run, + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + None, + ) mock_get_provider.assert_called_once_with( destination['type'], - constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, None, - raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, + None, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_replica_inputs.assert_not_called() -class ValidateReplicaDeploymentParametersTaskTestCase( - test_base.CoriolisBaseTestCase): - +class ValidateReplicaDeploymentParametersTaskTestCase(test_base.CoriolisBaseTestCase): def setUp(self): - super( - ValidateReplicaDeploymentParametersTaskTestCase, - self).setUp() - self.task_runner = ( - replica_tasks.ValidateReplicaDeploymentParametersTask()) + super(ValidateReplicaDeploymentParametersTaskTestCase, self).setUp() + self.task_runner = replica_tasks.ValidateReplicaDeploymentParametersTask() @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') - def test__run(self, mock_validate_value, mock_get_conn_info, - mock_get_provider, mock_event_manager): + def test__run( + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = ( - mock_get_provider.return_value.validate_replica_deployment_input) + prov_fun = mock_get_provider.return_value.validate_replica_deployment_input result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_value.assert_called_once_with( - task_info['export_info'], schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + task_info['export_info'], schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA + ) mock_get_provider.assert_called_once_with( destination['type'], constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, - mock.sentinel.event_handler, raise_if_not_found=False) + mock.sentinel.event_handler, + raise_if_not_found=False, + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - task_info['target_environment'], task_info['export_info']) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + task_info['target_environment'], + task_info['export_info'], + ) @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_dest_provider( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): destination = mock.MagicMock() task_info = mock.MagicMock() - prov_fun = ( - mock_get_provider.return_value.validate_replica_deployment_input) + prov_fun = mock_get_provider.return_value.validate_replica_deployment_input mock_get_provider.return_value = None result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, {}) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_value.assert_called_once_with( - task_info['export_info'], schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + task_info['export_info'], schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA + ) mock_get_provider.assert_called_once_with( destination['type'], constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT, - mock.sentinel.event_handler, raise_if_not_found=False) + mock.sentinel.event_handler, + raise_if_not_found=False, + ) prov_fun.assert_not_called() class UpdateSourceReplicaTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(UpdateSourceReplicaTaskTestCase, self).setUp() self.task_runner = replica_tasks.UpdateSourceReplicaTask() @@ -1023,22 +1308,32 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_new_source_env( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_source_env = {"old": "opt"} volumes_info = [{"id": "vol_id1"}] origin = mock.MagicMock() origin.get.side_effect = [old_source_env] task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, None] - prov_fun = (mock_get_provider.return_value. - check_update_source_environment_params) + prov_fun = mock_get_provider.return_value.check_update_source_environment_params expected_result = { - "volumes_info": volumes_info, "source_environment": old_source_env} + "volumes_info": volumes_info, + "source_environment": old_source_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) @@ -1052,8 +1347,12 @@ def test__run_no_new_source_env( @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_source_provider( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_source_env = {"opt": "old"} origin = mock.MagicMock() origin.get.side_effect = [old_source_env] @@ -1062,20 +1361,27 @@ def test__run_no_source_provider( new_source_env = {"opt": "new"} task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, new_source_env] - prov_fun = (mock_get_provider.return_value. - check_update_source_environment_params) + prov_fun = mock_get_provider.return_value.check_update_source_environment_params mock_get_provider.return_value = None self.assertRaises( exception.InvalidActionTasksExecutionState, self.task_runner._run, - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) + origin['type'], + constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_not_called() mock_validate_value.assert_not_called() prov_fun.assert_not_called() @@ -1085,8 +1391,12 @@ def test__run_no_source_provider( @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_volumes_info( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_source_env = {"opt": "old"} origin = mock.MagicMock() origin.get.side_effect = [old_source_env] @@ -1094,36 +1404,52 @@ def test__run_no_volumes_info( volumes_info = [{"id": "vol_id1"}] new_source_env = {"opt": "new"} task_info = mock.MagicMock() - task_info.get.side_effect = [ - volumes_info, new_source_env, volumes_info] - prov_fun = (mock_get_provider.return_value. - check_update_source_environment_params) + task_info.get.side_effect = [volumes_info, new_source_env, volumes_info] + prov_fun = mock_get_provider.return_value.check_update_source_environment_params prov_fun.return_value = None - expected_result = {"volumes_info": volumes_info, - "source_environment": new_source_env} + expected_result = { + "volumes_info": volumes_info, + "source_environment": new_source_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) + origin['type'], + constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) mock_validate_value.assert_not_called() prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - mock.sentinel.instance, volumes_info, old_source_env, - new_source_env) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + mock.sentinel.instance, + volumes_info, + old_source_env, + new_source_env, + ) @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_source_env = {"opt": "old"} origin = mock.MagicMock() origin.get.side_effect = [old_source_env] @@ -1132,30 +1458,43 @@ def test__run( new_source_env = {"opt": "new"} task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, new_source_env] - prov_fun = (mock_get_provider.return_value. - check_update_source_environment_params) - expected_result = {"volumes_info": prov_fun.return_value, - "source_environment": new_source_env} + prov_fun = mock_get_provider.return_value.check_update_source_environment_params + expected_result = { + "volumes_info": prov_fun.return_value, + "source_environment": new_source_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, origin, - mock.sentinel.destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + origin, + mock.sentinel.destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( - origin['type'], constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) + origin['type'], + constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE, + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, origin) mock_validate_value.assert_called_once_with( - prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - mock.sentinel.instance, volumes_info, old_source_env, - new_source_env) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + mock.sentinel.instance, + volumes_info, + old_source_env, + new_source_env, + ) class UpdateDestinationReplicaTaskTestCase(test_base.CoriolisBaseTestCase): - def setUp(self): super(UpdateDestinationReplicaTaskTestCase, self).setUp() self.task_runner = replica_tasks.UpdateDestinationReplicaTask() @@ -1165,22 +1504,34 @@ def setUp(self): @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_new_dest_env( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_dest_env = {"old": "opt"} volumes_info = [{"id": "vol_id1"}] destination = mock.MagicMock() destination.get.side_effect = [old_dest_env] task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, None] - prov_fun = (mock_get_provider.return_value. - check_update_destination_environment_params) + prov_fun = ( + mock_get_provider.return_value.check_update_destination_environment_params + ) expected_result = { - "volumes_info": volumes_info, "target_environment": old_dest_env} + "volumes_info": volumes_info, + "target_environment": old_dest_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) @@ -1194,8 +1545,12 @@ def test__run_no_new_dest_env( @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_dest_provider( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_dest_env = {"opt": "old"} destination = mock.MagicMock() destination.get.side_effect = [old_dest_env] @@ -1204,21 +1559,29 @@ def test__run_no_dest_provider( new_dest_env = {"opt": "new"} task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, new_dest_env] - prov_fun = (mock_get_provider.return_value. - check_update_destination_environment_params) + prov_fun = ( + mock_get_provider.return_value.check_update_destination_environment_params + ) mock_get_provider.return_value = None self.assertRaises( exception.InvalidActionTasksExecutionState, self.task_runner._run, - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( destination['type'], constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) + mock.sentinel.event_handler, + raise_if_not_found=False, + ) mock_get_conn_info.assert_not_called() mock_validate_value.assert_not_called() prov_fun.assert_not_called() @@ -1228,8 +1591,12 @@ def test__run_no_dest_provider( @mock.patch('coriolis.tasks.base.get_connection_info') @mock.patch('coriolis.schemas.validate_value') def test__run_no_volumes_info( - self, mock_validate_value, mock_get_conn_info, mock_get_provider, - mock_event_manager): + self, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_dest_env = {"opt": "old"} destination = mock.MagicMock() destination.get.side_effect = [old_dest_env] @@ -1239,28 +1606,46 @@ def test__run_no_volumes_info( export_info = {"export": "info"} task_info = mock.MagicMock() task_info.get.side_effect = [ - volumes_info, new_dest_env, export_info, volumes_info] - prov_fun = (mock_get_provider.return_value. - check_update_destination_environment_params) + volumes_info, + new_dest_env, + export_info, + volumes_info, + ] + prov_fun = ( + mock_get_provider.return_value.check_update_destination_environment_params + ) prov_fun.return_value = None - expected_result = {"volumes_info": volumes_info, - "target_environment": new_dest_env} + expected_result = { + "volumes_info": volumes_info, + "target_environment": new_dest_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( destination['type'], constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + mock.sentinel.event_handler, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_value.assert_not_called() prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - export_info, volumes_info, old_dest_env, new_dest_env) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + export_info, + volumes_info, + old_dest_env, + new_dest_env, + ) @mock.patch('coriolis.events.EventManager') @mock.patch('coriolis.providers.factory.get_provider') @@ -1268,8 +1653,13 @@ def test__run_no_volumes_info( @mock.patch('coriolis.schemas.validate_value') @mock.patch.object(replica_tasks, '_check_ensure_volumes_info_ordering') def test__run( - self, mock_check_vol_ordering, mock_validate_value, - mock_get_conn_info, mock_get_provider, mock_event_manager): + self, + mock_check_vol_ordering, + mock_validate_value, + mock_get_conn_info, + mock_get_provider, + mock_event_manager, + ): old_dest_env = {"opt": "old"} destination = mock.MagicMock() destination.get.side_effect = [old_dest_env] @@ -1279,27 +1669,42 @@ def test__run( export_info = {"name": "instance_name", "id": "instance_id"} task_info = mock.MagicMock() task_info.get.side_effect = [volumes_info, new_dest_env, export_info] - prov_fun = (mock_get_provider.return_value. - check_update_destination_environment_params) + prov_fun = ( + mock_get_provider.return_value.check_update_destination_environment_params + ) expected_result = { "volumes_info": mock_check_vol_ordering.return_value, - "target_environment": new_dest_env} + "target_environment": new_dest_env, + } result = self.task_runner._run( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - destination, task_info, mock.sentinel.event_handler) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + destination, + task_info, + mock.sentinel.event_handler, + ) self.assertEqual(result, expected_result) mock_event_manager.assert_called_once_with(mock.sentinel.event_handler) mock_get_provider.assert_called_once_with( destination['type'], constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE, - mock.sentinel.event_handler, raise_if_not_found=False) - mock_get_conn_info.assert_called_once_with( - mock.sentinel.ctxt, destination) + mock.sentinel.event_handler, + raise_if_not_found=False, + ) + mock_get_conn_info.assert_called_once_with(mock.sentinel.ctxt, destination) mock_validate_value.assert_called_once_with( - prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA) + prov_fun.return_value, schemas.CORIOLIS_VOLUMES_INFO_SCHEMA + ) prov_fun.assert_called_once_with( - mock.sentinel.ctxt, mock_get_conn_info.return_value, - export_info, volumes_info, old_dest_env, new_dest_env) + mock.sentinel.ctxt, + mock_get_conn_info.return_value, + export_info, + volumes_info, + old_dest_env, + new_dest_env, + ) mock_check_vol_ordering.assert_called_once_with( - export_info, prov_fun.return_value) + export_info, prov_fun.return_value + ) diff --git a/coriolis/tests/test_base.py b/coriolis/tests/test_base.py index c66b057e..09be23b0 100644 --- a/coriolis/tests/test_base.py +++ b/coriolis/tests/test_base.py @@ -11,33 +11,27 @@ class CoriolisBaseTestCase(base.BaseTestCase): - def setUp(self): super(CoriolisBaseTestCase, self).setUp() class CoriolisApiViewsTestCase(CoriolisBaseTestCase): - def setUp(self): super(CoriolisApiViewsTestCase, self).setUp() self._single_response = {"key1": "value1", "key2": "value2"} - self._collection_response = [ - self._single_response, self._single_response] + self._collection_response = [self._single_response, self._single_response] self._format_fun = views_utils.format_opt def _single_view_test(self, fun, expected_result_key, keys=None): - format_fun = '%s.%s' % ( - self._format_fun.__module__, self._format_fun.__name__) + format_fun = '%s.%s' % (self._format_fun.__module__, self._format_fun.__name__) with mock.patch(format_fun) as format_mock: result = fun(self._single_response, keys) format_mock.assert_called_once_with(self._single_response, keys) expected_result = {expected_result_key: format_mock.return_value} self.assertEqual(expected_result, result) - def _collection_view_test( - self, fun, expected_result_key, keys=None): - format_fun = '%s.%s' % ( - self._format_fun.__module__, self._format_fun.__name__) + def _collection_view_test(self, fun, expected_result_key, keys=None): + format_fun = '%s.%s' % (self._format_fun.__module__, self._format_fun.__name__) with mock.patch(format_fun) as format_mock: f_opts = [] result = fun(self._collection_response, keys) @@ -53,8 +47,7 @@ def setUp(self): super(CoriolisRPCClientTestCase, self).setUp() self.client = None - def _test(self, fun, args, rpc_op='_call', - server_fun_name=None, custom_args=None): + def _test(self, fun, args, rpc_op='_call', server_fun_name=None, custom_args=None): if server_fun_name is None: server_fun_name = fun.__name__ with mock.patch.object(self.client, rpc_op) as op_mock: diff --git a/coriolis/tests/test_cache.py b/coriolis/tests/test_cache.py index a9dae0a3..a7ab4030 100644 --- a/coriolis/tests/test_cache.py +++ b/coriolis/tests/test_cache.py @@ -3,8 +3,7 @@ from unittest import mock -from coriolis import cache -from coriolis import exception +from coriolis import cache, exception from coriolis.tests import test_base @@ -18,15 +17,13 @@ def test_get_cache_decorator(self, mock_get_memoization_decorator): self.assertEqual(result, mock_get_memoization_decorator.return_value) mock_get_memoization_decorator.assert_called_once_with( - cache.CONF, - cache.cache_region, - provider + cache.CONF, cache.cache_region, provider ) def test_get_cache_decorator_invalid_provider_name(self): invalid_providers = [123, '', None] for provider in invalid_providers: - self.assertRaises(exception.CoriolisException, - cache.get_cache_decorator, - provider) + self.assertRaises( + exception.CoriolisException, cache.get_cache_decorator, provider + ) diff --git a/coriolis/tests/test_context.py b/coriolis/tests/test_context.py index 01a953a1..8caeb1a5 100644 --- a/coriolis/tests/test_context.py +++ b/coriolis/tests/test_context.py @@ -3,10 +3,8 @@ from unittest import mock -from coriolis import context -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils +from coriolis import context, exception +from coriolis.tests import test_base, testutils class RequestContextTestCase(test_base.CoriolisBaseTestCase): @@ -40,8 +38,7 @@ def test__init__with_string_timestamp(self, mock_parse_isotime): ) mock_parse_isotime.assert_called_once_with("2023-11-09T13:31:08Z") - self.assertEqual(new_context.timestamp, - mock_parse_isotime.return_value) + self.assertEqual(new_context.timestamp, mock_parse_isotime.return_value) self.assertEqual(new_context.roles, []) def test_to_dict(self): @@ -57,35 +54,43 @@ def test_can(self, mock_check_policy): result = self.req_context.can('test_action') mock_check_policy.assert_called_once_with( - self.req_context, 'test_action', {'project_id': 'test_project_id', - 'user_id': 'user'}) + self.req_context, + 'test_action', + {'project_id': 'test_project_id', 'user_id': 'user'}, + ) self.assertEqual(result, mock_check_policy.return_value) @mock.patch.object(context.policy, 'check_policy_for_context') def test_can_policy_not_authorized(self, mock_check_policy): mock_check_policy.side_effect = exception.PolicyNotAuthorized( - action='test_action') + action='test_action' + ) - self.assertRaises(exception.PolicyNotAuthorized, - self.req_context.can, 'test_action') + self.assertRaises( + exception.PolicyNotAuthorized, self.req_context.can, 'test_action' + ) mock_check_policy.assert_called_once_with( - self.req_context, 'test_action', {'project_id': 'test_project_id', - 'user_id': 'user'}) + self.req_context, + 'test_action', + {'project_id': 'test_project_id', 'user_id': 'user'}, + ) @mock.patch.object(context.policy, 'check_policy_for_context') def test_can_policy_with_custom_target(self, mock_check_policy): custom_target = {'custom_key': 'custom_value'} - result = self.req_context.can('test_action', custom_target, - fatal=False) + result = self.req_context.can('test_action', custom_target, fatal=False) - expected_target = {'project_id': 'test_project_id', - 'user_id': 'user', - 'custom_key': 'custom_value'} + expected_target = { + 'project_id': 'test_project_id', + 'user_id': 'user', + 'custom_key': 'custom_value', + } mock_check_policy.assert_called_once_with( - self.req_context, 'test_action', expected_target) + self.req_context, 'test_action', expected_target + ) self.assertEqual(result, mock_check_policy.return_value) @@ -97,5 +102,6 @@ def test_get_admin_context(self, mock_request_context): result = context.get_admin_context() mock_request_context.assert_called_once_with( - user=None, project_id=None, is_admin=True, trust_id=None) + user=None, project_id=None, is_admin=True, trust_id=None + ) self.assertEqual(result, mock_request_context.return_value) diff --git a/coriolis/tests/test_data_transfer.py b/coriolis/tests/test_data_transfer.py index 832ae271..d077c4da 100644 --- a/coriolis/tests/test_data_transfer.py +++ b/coriolis/tests/test_data_transfer.py @@ -9,8 +9,7 @@ import requests import requests_unixsocket -from coriolis import data_transfer -from coriolis import exception +from coriolis import data_transfer, exception from coriolis.tests import test_base @@ -36,8 +35,7 @@ def test_get_session_and_address(self, mock_conf): result = data_transfer._get_session_and_address() self.assertIsInstance(result[0], requests.Session) - self.assertEqual(result[1], "http://%s/" % - mock_conf.compressor_address) + self.assertEqual(result[1], "http://%s/" % mock_conf.compressor_address) def test_get_session_and_address_no_compressor_address(self): result = data_transfer._get_session_and_address() @@ -53,112 +51,122 @@ def test_get_session_and_address_unix_socket(self, mock_conf, mock_stat): result = data_transfer._get_session_and_address() self.assertIsInstance(result[0], requests_unixsocket.Session) - self.assertEqual(result[1], "http+unix://%s/" % parse.quote_plus( - mock_conf.compressor_address)) + self.assertEqual( + result[1], + "http+unix://%s/" % parse.quote_plus(mock_conf.compressor_address), + ) @mock.patch.object(data_transfer, 'CONF') def test_get_session_and_address_nonexistent_path(self, mock_conf): mock_conf.compressor_address = '/var/run/compressor.sock' - self.assertRaises(exception.CoriolisException, - data_transfer._get_session_and_address) + self.assertRaises( + exception.CoriolisException, data_transfer._get_session_and_address + ) @mock.patch.object(data_transfer.os, 'stat') @mock.patch.object(data_transfer, 'CONF') - def test_get_session_and_address_invalid_unix_socket(self, mock_conf, - mock_stat): + def test_get_session_and_address_invalid_unix_socket(self, mock_conf, mock_stat): mock_conf.compressor_address = '/var/run/compressor.sock' mock_stat.return_value.st_mode = stat.S_IFREG - self.assertRaises(exception.CoriolisException, - data_transfer._get_session_and_address) + self.assertRaises( + exception.CoriolisException, data_transfer._get_session_and_address + ) @mock.patch.object(data_transfer.constants, 'COMPRESSION_FORMAT_GZIP') def test_compression_proxy_invalid_format(self, mock_compression_format): - self.assertRaises(exception.CoriolisException, - data_transfer.compression_proxy, - mock.sentinel.content, - mock_compression_format.return_value) + self.assertRaises( + exception.CoriolisException, + data_transfer.compression_proxy, + mock.sentinel.content, + mock_compression_format.return_value, + ) @mock.patch.object(data_transfer, '_COMPRESS_FUNC') def test_compression_proxy(self, mock_compress_func): result = data_transfer.compression_proxy(self.data_content, self.fmt) mock_compress_func[self.fmt].assert_called_once_with(self.data_content) - self.assertEqual( - result, (mock_compress_func[self.fmt].return_value, True)) + self.assertEqual(result, (mock_compress_func[self.fmt].return_value, True)) @mock.patch.object(data_transfer, '_get_session_and_address') @mock.patch.object(data_transfer, 'CONF') - def test_compression_proxy_with_compression(self, mock_conf, - mock_get_session_and_address): - mock_get_session_and_address.return_value = ( - self.mock_session, self.mock_url) + def test_compression_proxy_with_compression( + self, mock_conf, mock_get_session_and_address + ): + mock_get_session_and_address.return_value = (self.mock_session, self.mock_url) result = data_transfer.compression_proxy(self.data_content, self.fmt) self.mock_session.post.assert_called_once_with( - self.mock_url, data=self.data_content, + self.mock_url, + data=self.data_content, headers={'X-Compression-Format': self.fmt}, - timeout=mock_conf.default_requests_timeout) - self.assertEqual( - result, (self.mock_session.post.return_value.content, False)) + timeout=mock_conf.default_requests_timeout, + ) + self.assertEqual(result, (self.mock_session.post.return_value.content, False)) self.mock_session.close.assert_called_once() @mock.patch.object(data_transfer, '_get_session_and_address') @mock.patch.object(data_transfer, '_COMPRESS_FUNC') - def test_compression_proxy_with_exception(self, mock_compress_func, - mock_get_session_and_address): - mock_get_session_and_address.return_value = (self.mock_session, - self.mock_url) + def test_compression_proxy_with_exception( + self, mock_compress_func, mock_get_session_and_address + ): + mock_get_session_and_address.return_value = (self.mock_session, self.mock_url) self.mock_session.post.side_effect = Exception("mock_message") result = data_transfer.compression_proxy(self.data_content, self.fmt) - self.assertEqual( - result, (mock_compress_func[self.fmt].return_value, True)) + self.assertEqual(result, (mock_compress_func[self.fmt].return_value, True)) self.mock_session.close.assert_called_once() @mock.patch.object(data_transfer, 'compression_proxy') @mock.patch.object(data_transfer, 'struct') def test_encode_data(self, mock_struct, mock_compression_proxy): mock_compression_proxy.return_value = (self.data_content, True) - mock_struct.pack.side_effect = (lambda fmt, - *args: struct.pack(fmt, *args)) - - result = data_transfer.encode_data(self.msg_id, self.path, self.offset, - self.data_content, True) - expected_result = struct.pack('" % ( - self.target, self.timeout)) + self.assertEqual( + result, "" % (self.target, self.timeout) + ) @mock.patch.object(rpc, '_get_transport') - def test_transport_property_when_transport_is_None(self, - mock_get_transport): + def test_transport_property_when_transport_is_None(self, mock_get_transport): with mock.patch.object(rpc, '_TRANSPORT', None): result = self.client._transport @@ -151,7 +154,8 @@ def test_transport_property_when_transport_is_None(self, @mock.patch.object(rpc, '_get_transport') def test_transport_property_when_transport_conn_is_not_None( - self, mock_get_transport): + self, mock_get_transport + ): with mock.patch.object(rpc, '_TRANSPORT', None): self.client._transport_conn = mock.Mock() result = self.client._transport @@ -160,8 +164,7 @@ def test_transport_property_when_transport_conn_is_not_None( self.assertEqual(result, self.client._transport_conn) @mock.patch.object(rpc, '_TRANSPORT') - def test_transport_property_when_transport_is_not_None(self, - mock_transport): + def test_transport_property_when_transport_is_not_None(self, mock_transport): result = self.client._transport self.assertEqual(result, mock_transport) @@ -174,34 +177,37 @@ def test_rpc_client(self, mock_get_transport, mock_rpc_client): result = self.client._rpc_client() mock_rpc_client.assert_called_once_with( - self.client._transport, self.client._target, - serializer=self.client._serializer, timeout=self.client._timeout) + self.client._transport, + self.client._target, + serializer=self.client._serializer, + timeout=self.client._timeout, + ) self.assertEqual(result, mock_rpc_client.return_value) def test_call(self): with mock.patch('coriolis.rpc.BaseRPCClient._rpc_client') as rpc_mock: - result = self.client._call(mock.sentinel.ctxt, self.method, - **self.args) + result = self.client._call(mock.sentinel.ctxt, self.method, **self.args) rpc_mock.assert_called_once() rpc_mock.return_value.call.assert_called_once_with( - mock.sentinel.ctxt, self.method, **self.args) - self.assertEqual( - result, rpc_mock.return_value.call.return_value) + mock.sentinel.ctxt, self.method, **self.args + ) + self.assertEqual(result, rpc_mock.return_value.call.return_value) def test_call_on_host(self): with mock.patch('coriolis.rpc.BaseRPCClient._rpc_client') as rpc_mock: - result = self.client._call_on_host(self.host, mock.sentinel.ctxt, - self.method, **self.args) + result = self.client._call_on_host( + self.host, mock.sentinel.ctxt, self.method, **self.args + ) rpc_mock.assert_called_once() - rpc_mock.return_value.prepare.assert_called_once_with( - server=self.host) - rpc_mock.return_value.prepare.return_value.call.\ - assert_called_once_with(mock.sentinel.ctxt, self.method, - **self.args) - self.assertEqual(result, rpc_mock.return_value.prepare. - return_value.call.return_value) + rpc_mock.return_value.prepare.assert_called_once_with(server=self.host) + rpc_mock.return_value.prepare.return_value.call.assert_called_once_with( + mock.sentinel.ctxt, self.method, **self.args + ) + self.assertEqual( + result, rpc_mock.return_value.prepare.return_value.call.return_value + ) def test_cast(self): with mock.patch('coriolis.rpc.BaseRPCClient._rpc_client') as rpc_mock: @@ -209,16 +215,17 @@ def test_cast(self): rpc_mock.assert_called_once() rpc_mock.return_value.cast.assert_called_once_with( - mock.sentinel.ctxt, self.method, **self.args) + mock.sentinel.ctxt, self.method, **self.args + ) def test_cast_for_host(self): with mock.patch('coriolis.rpc.BaseRPCClient._rpc_client') as rpc_mock: - self.client._cast_for_host(self.host, mock.sentinel.ctxt, - self.method, **self.args) + self.client._cast_for_host( + self.host, mock.sentinel.ctxt, self.method, **self.args + ) rpc_mock.assert_called_once() - rpc_mock.return_value.prepare.assert_called_once_with( - server=self.host) - rpc_mock.return_value.prepare.return_value.cast.\ - assert_called_once_with(mock.sentinel.ctxt, self.method, - **self.args) + rpc_mock.return_value.prepare.assert_called_once_with(server=self.host) + rpc_mock.return_value.prepare.return_value.cast.assert_called_once_with( + mock.sentinel.ctxt, self.method, **self.args + ) diff --git a/coriolis/tests/test_schemas.py b/coriolis/tests/test_schemas.py index 3f61d6e7..466132a4 100644 --- a/coriolis/tests/test_schemas.py +++ b/coriolis/tests/test_schemas.py @@ -8,11 +8,9 @@ import jinja2 import jsonschema -from coriolis import exception -from coriolis import schemas +from coriolis import exception, schemas from coriolis.tests import test_base - RENDERED_TEMPLATE_SENTINEL = mock.sentinel.some_string_schema @@ -61,7 +59,8 @@ def test_get_schema(self, mock_loads, mock_loader, mock_environ): res = schemas.get_schema(test_package_name, test_schema_name) mock_loader.assert_called_once_with( - test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY) + test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY + ) mock_environ.assert_called_once_with(loader=test_loader) mock_env.get_template.assert_called_once_with(test_schema_name) mock_loads.assert_called_once_with(test_rendered_template) @@ -84,9 +83,12 @@ def test_get_schema_parent_package_exists(self, mock_loader, mock_environ): schemas.get_schema(test_package_name, test_schema_name) parent_package = "test.package" - mock_loader.assert_has_calls([ - mock.call(test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY), - mock.call(parent_package, schemas.DEFAULT_SCHEMAS_DIRECTORY)]) + mock_loader.assert_has_calls( + [ + mock.call(test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY), + mock.call(parent_package, schemas.DEFAULT_SCHEMAS_DIRECTORY), + ] + ) @mock.patch.object(jinja2, 'PackageLoader') def test_get_schema_parent_package_not_exists(self, mock_loader): @@ -96,11 +98,12 @@ def test_get_schema_parent_package_not_exists(self, mock_loader): mock_loader.side_effect = ValueError() self.assertRaises( - ValueError, schemas.get_schema, test_package_name, - test_schema_name) + ValueError, schemas.get_schema, test_package_name, test_schema_name + ) mock_loader.assert_called_once_with( - test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY) + test_package_name, schemas.DEFAULT_SCHEMAS_DIRECTORY + ) @mock.patch.object(jsonschema, 'validate') def test_validate_value(self, mock_validate): @@ -110,7 +113,8 @@ def test_validate_value(self, mock_validate): schemas.validate_value(test_value, test_schema) mock_validate.assert_called_once_with( - test_value, test_schema, format_checker=None) + test_value, test_schema, format_checker=None + ) @mock.patch.object(json, 'loads') @mock.patch.object(jsonschema, 'validate') @@ -125,29 +129,33 @@ def test_validate_string(self, mock_validate, mock_loads): mock_loads.assert_called_once_with(test_string) mock_validate.assert_called_once_with( - test_value, test_schema, format_checker=None) + test_value, test_schema, format_checker=None + ) @mock.patch.object(jsonschema, 'validate') def test_validate_value_raises_on_error(self, mock_validate): test_value = mock.sentinel.test_value test_schema = mock.sentinel.test_schema - mock_validate.side_effect = jsonschema.exceptions.ValidationError( - 'Test Error') + mock_validate.side_effect = jsonschema.exceptions.ValidationError('Test Error') self.assertRaises( - exception.SchemaValidationException, schemas.validate_value, - test_value, test_schema, raise_on_error=True) + exception.SchemaValidationException, + schemas.validate_value, + test_value, + test_schema, + raise_on_error=True, + ) @mock.patch.object(jsonschema, 'validate') def test_validate_value_no_raise_on_error(self, mock_validate): test_value = mock.sentinel.test_value test_schema = mock.sentinel.test_schema - mock_validate.side_effect = jsonschema.exceptions.ValidationError( - 'Test Error') + mock_validate.side_effect = jsonschema.exceptions.ValidationError('Test Error') with self.assertLogs('coriolis.schemas', level=logging.WARN): - result = schemas.validate_value(test_value, test_schema, - raise_on_error=False) + result = schemas.validate_value( + test_value, test_schema, raise_on_error=False + ) self.assertEqual(result, False) diff --git a/coriolis/tests/test_secrets.py b/coriolis/tests/test_secrets.py index 4d13ec58..253b8113 100644 --- a/coriolis/tests/test_secrets.py +++ b/coriolis/tests/test_secrets.py @@ -4,11 +4,10 @@ import json from unittest import mock -from barbicanclient import client as barbican_client import keystoneauth1 +from barbicanclient import client as barbican_client -from coriolis import keystone -from coriolis import secrets +from coriolis import keystone, secrets from coriolis.tests import test_base @@ -26,31 +25,32 @@ def test_get_barbican_secret_payload(self, mock_client, mock_session): mock_client.return_value = mock_barbican result = secrets._get_barbican_secret_payload( - mock.sentinel.ctxt, mock.sentinel.secret_ref) + mock.sentinel.ctxt, mock.sentinel.secret_ref + ) self.assertEqual(mock.sentinel.payload, result) mock_session.assert_called_once_with(mock.sentinel.ctxt) mock_client.assert_called_once_with(session=mock.sentinel.session) - mock_barbican.secrets.get.assert_called_once_with( - mock.sentinel.secret_ref) + mock_barbican.secrets.get.assert_called_once_with(mock.sentinel.secret_ref) @mock.patch.object(secrets, '_get_barbican_secret_payload') def test_get_secret_success(self, mock_get_payload): mock_get_payload.return_value = json.dumps({'key': 'value'}) - result = secrets.get_secret(mock.sentinel.ctxt, - mock.sentinel.secret_ref) + result = secrets.get_secret(mock.sentinel.ctxt, mock.sentinel.secret_ref) self.assertEqual({'key': 'value'}, result) mock_get_payload.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.secret_ref) + mock.sentinel.ctxt, mock.sentinel.secret_ref + ) @mock.patch.object(secrets, '_get_barbican_secret_payload') def test_get_secret_unauthorized(self, mock_get_payload): mock_ctxt = mock.MagicMock() mock_get_payload.side_effect = [ keystoneauth1.exceptions.http.Unauthorized, - json.dumps({'key': 'value'})] + json.dumps({'key': 'value'}), + ] with mock.patch('copy.deepcopy', side_effect=lambda func: func): result = secrets.get_secret(mock_ctxt, mock.sentinel.secret_ref) @@ -58,15 +58,16 @@ def test_get_secret_unauthorized(self, mock_get_payload): self.assertEqual({'key': 'value'}, result) self.assertEqual(2, mock_get_payload.call_count) self.assertEqual(mock_ctxt.trust_id, None) - mock_get_payload.assert_called_with( - mock_ctxt, mock.sentinel.secret_ref) + mock_get_payload.assert_called_with(mock_ctxt, mock.sentinel.secret_ref) @mock.patch.object(secrets, '_get_barbican_secret_payload') def test_get_secret_raises_value_error(self, mock_get_payload): mock_get_payload.side_effect = ValueError("Test exception") - self.assertRaises(ValueError, secrets.get_secret, mock.sentinel.ctxt, - mock.sentinel.secret_ref) + self.assertRaises( + ValueError, secrets.get_secret, mock.sentinel.ctxt, mock.sentinel.secret_ref + ) mock_get_payload.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.secret_ref) + mock.sentinel.ctxt, mock.sentinel.secret_ref + ) diff --git a/coriolis/tests/test_service.py b/coriolis/tests/test_service.py index 0c906c3b..75daf11e 100644 --- a/coriolis/tests/test_service.py +++ b/coriolis/tests/test_service.py @@ -21,22 +21,21 @@ class ServiceTestCase(test_base.CoriolisBaseTestCase): (['--worker-process-count', '15'], 15, [], False), (['--worker-process-count', '20'], 20, [], False), ([], None, [], False), - (['--worker-process-count', '5', '--unknown-arg'], 5, - ['--unknown-arg'], False), + (['--worker-process-count', '5', '--unknown-arg'], 5, ['--unknown-arg'], False), (['--worker-process-count', '-5'], None, None, True), (['--worker-process-count', '0'], None, None, True), - (['--worker-process-count', 'not-a-number'], None, None, True) + (['--worker-process-count', 'not-a-number'], None, None, True), ) @ddt.unpack - def test_get_worker_count_from_args(self, input_args, expected_count, - expected_unknown_args, - expect_exception): + def test_get_worker_count_from_args( + self, input_args, expected_count, expected_unknown_args, expect_exception + ): if expect_exception: - self.assertRaises(SystemExit, service.get_worker_count_from_args, - input_args) + self.assertRaises( + SystemExit, service.get_worker_count_from_args, input_args + ) else: - worker_count, unknown_args = service.get_worker_count_from_args( - input_args) + worker_count, unknown_args = service.get_worker_count_from_args(input_args) self.assertEqual(worker_count, expected_count) self.assertEqual(unknown_args, expected_unknown_args) @@ -45,22 +44,28 @@ def test_get_worker_count_from_args(self, input_args, expected_count, ({'lock_path': ""}, False, False, [], logging.WARN), ({'lock_path': "/path/to/locks"}, False, False, [], logging.WARN), ({'lock_path': "/path/to/locks"}, True, False, [], logging.WARN), - ({'lock_path': "/path/to/locks"}, True, True, ['file1', 'file2'], - logging.WARN), + ({'lock_path': "/path/to/locks"}, True, True, ['file1', 'file2'], logging.WARN), ({'lock_path': "/path/to/locks"}, True, True, [], logging.INFO), ) @ddt.unpack @mock.patch.object(os.path, 'exists') @mock.patch.object(os.path, 'isdir') @mock.patch.object(os, 'listdir') - def test_check_locks_dir_empty(self, oslo_concurrency, exists, isdir, - listdir, expected_log_level, mock_listdir, - mock_isdir, mock_exists): + def test_check_locks_dir_empty( + self, + oslo_concurrency, + exists, + isdir, + listdir, + expected_log_level, + mock_listdir, + mock_isdir, + mock_exists, + ): mock_exists.return_value = exists mock_isdir.return_value = isdir mock_listdir.return_value = listdir - with mock.patch.object(service.CONF, 'oslo_concurrency', - oslo_concurrency): + with mock.patch.object(service.CONF, 'oslo_concurrency', oslo_concurrency): with self.assertLogs('coriolis.service', level=expected_log_level): service.check_locks_dir_empty() @@ -79,8 +84,9 @@ def test_init(self, mock_server, mock_conf, mock_rpc, mock_load_app): mock_conf.api_migration_listen = mock.sentinel.listen mock_rpc.init.return_value = mock.sentinel.transport - wsgi_service = service.WSGIService(mock.sentinel.name, worker_count=10, - init_rpc=True) + wsgi_service = service.WSGIService( + mock.sentinel.name, worker_count=10, init_rpc=True + ) mock_rpc.init.assert_called_once_with() mock_load_app.assert_called_once_with(mock.sentinel.name) @@ -89,11 +95,11 @@ def test_init(self, mock_server, mock_conf, mock_rpc, mock_load_app): wsgi_app=mock_load_app.return_value, bind_addr=( mock_conf.api_migration_listen, - mock_conf.api_migration_listen_port), + mock_conf.api_migration_listen_port, + ), ) self.assertEqual(wsgi_service._host, mock_conf.api_migration_listen) - self.assertEqual(wsgi_service._port, - mock_conf.api_migration_listen_port) + self.assertEqual(wsgi_service._port, mock_conf.api_migration_listen_port) self.assertEqual(wsgi_service._workers, 10) self.assertEqual(wsgi_service._app, mock_load_app.return_value) self.assertEqual(wsgi_service._server, mock_server.return_value) @@ -103,8 +109,9 @@ def test_init(self, mock_server, mock_conf, mock_rpc, mock_load_app): @mock.patch('oslo_service.wsgi.Loader.load_app') @mock.patch('coriolis.rpc.messaging.get_transport') @mock.patch.object(service, 'CONF') - def test_init_windows(self, mock_conf, mock_get_transport, mock_load_app, - mock_server, mock_system): + def test_init_windows( + self, mock_conf, mock_get_transport, mock_load_app, mock_server, mock_system + ): mock_system.return_value = "Windows" mock_conf.api_migration_workers = None mock_load_app.return_value = mock.MagicMock() @@ -122,8 +129,14 @@ def test_init_windows(self, mock_conf, mock_get_transport, mock_load_app, @mock.patch('coriolis.rpc.messaging.get_transport') @mock.patch.object(service, 'CONF') def test_init_sets_workers_based_on_system_on_non_windows( - self, mock_conf, mock_get_transport, mock_get_worker_count, - mock_load_app, mock_server, mock_system): + self, + mock_conf, + mock_get_transport, + mock_get_worker_count, + mock_load_app, + mock_server, + mock_system, + ): mock_system.return_value = "Linux" mock_conf.api_migration_workers = None mock_load_app.return_value = mock.MagicMock() @@ -137,8 +150,7 @@ def test_init_sets_workers_based_on_system_on_non_windows( @mock.patch.object(service, 'CONF') @mock.patch('oslo_service.wsgi.Loader.load_app') @mock.patch('coriolis.rpc.messaging.get_transport') - def test_service_methods(self, mock_get_transport, - mock_load_app, mock_conf): + def test_service_methods(self, mock_get_transport, mock_load_app, mock_conf): mock_conf.api_migration_workers = 10 mock_load_app.return_value = mock.MagicMock() mock_get_transport.return_value = mock.MagicMock() @@ -147,8 +159,7 @@ def test_service_methods(self, mock_get_transport, result._server = mock.MagicMock() - self.assertEqual(result.get_workers_count(), - mock_conf.api_migration_workers) + self.assertEqual(result.get_workers_count(), mock_conf.api_migration_workers) result.start() result._server.serve.assert_called_once() @@ -178,27 +189,33 @@ def test_init(self, mock_get_hostname, mock_target, mock_conf, mock_rpc): mock_get_hostname.return_value = mock.sentinel.hostname messaging_service = service.MessagingService( - mock.sentinel.topic, mock.sentinel.endpoints, - mock.sentinel.version, worker_count=10, init_rpc=True) + mock.sentinel.topic, + mock.sentinel.endpoints, + mock.sentinel.version, + worker_count=10, + init_rpc=True, + ) mock_rpc.init.assert_called_once_with() mock_target.assert_called_once_with( topic=mock.sentinel.topic, server=mock.sentinel.hostname, - version=mock.sentinel.version) + version=mock.sentinel.version, + ) mock_rpc.get_server.assert_called_once_with( - mock_target.return_value, mock.sentinel.endpoints) + mock_target.return_value, mock.sentinel.endpoints + ) self.assertEqual(messaging_service._workers, 10) - self.assertEqual(messaging_service._server, - mock_rpc.get_server.return_value) + self.assertEqual(messaging_service._server, mock_rpc.get_server.return_value) @mock.patch('platform.system') @mock.patch.object(service, 'rpc') @mock.patch.object(service, 'CONF') @mock.patch.object(service.messaging, 'Target') @mock.patch.object(service.utils, 'get_hostname') - def test_init_windows(self, mock_get_hostname, mock_target, mock_conf, - mock_rpc, mock_system): + def test_init_windows( + self, mock_get_hostname, mock_target, mock_conf, mock_rpc, mock_system + ): mock_system.return_value = "Windows" mock_conf.api_migration_listen = mock.sentinel.listen mock_conf.api_migration_listen_port = mock.sentinel.listen_port @@ -208,27 +225,32 @@ def test_init_windows(self, mock_get_hostname, mock_target, mock_conf, mock_get_hostname.return_value = mock.sentinel.hostname messaging_service = service.MessagingService( - mock.sentinel.topic, mock.sentinel.endpoints, - mock.sentinel.version, worker_count=10, init_rpc=True) + mock.sentinel.topic, + mock.sentinel.endpoints, + mock.sentinel.version, + worker_count=10, + init_rpc=True, + ) mock_rpc.init.assert_called_once_with() mock_target.assert_called_once_with( topic=mock.sentinel.topic, server=mock.sentinel.hostname, - version=mock.sentinel.version) + version=mock.sentinel.version, + ) mock_rpc.get_server.assert_called_once_with( - mock_target.return_value, mock.sentinel.endpoints) + mock_target.return_value, mock.sentinel.endpoints + ) self.assertEqual(messaging_service._workers, 1) - self.assertEqual(messaging_service._server, - mock_rpc.get_server.return_value) + self.assertEqual(messaging_service._server, mock_rpc.get_server.return_value) @mock.patch.object(service, 'rpc') @mock.patch.object(service, 'CONF') @mock.patch.object(service.utils, 'get_hostname') @mock.patch.object(service.processutils, 'get_worker_count') - def test_init_with_none_worker_count(self, mock_get_worker_count, - mock_get_hostname, - mock_conf, mock_rpc): + def test_init_with_none_worker_count( + self, mock_get_worker_count, mock_get_hostname, mock_conf, mock_rpc + ): mock_conf.api_migration_listen = mock.sentinel.listen mock_conf.api_migration_listen_port = mock.sentinel.listen_port mock_conf.api_migration_listen = mock.sentinel.listen @@ -236,8 +258,12 @@ def test_init_with_none_worker_count(self, mock_get_worker_count, mock_get_hostname.return_value = mock.sentinel.hostname result = service.MessagingService( - mock.sentinel.topic, mock.sentinel.endpoints, - mock.sentinel.version, worker_count=None, init_rpc=True) + mock.sentinel.topic, + mock.sentinel.endpoints, + mock.sentinel.version, + worker_count=None, + init_rpc=True, + ) self.assertEqual(result._workers, mock_get_worker_count.return_value) @@ -245,8 +271,9 @@ def test_init_with_none_worker_count(self, mock_get_worker_count, @mock.patch.object(service, 'CONF') @mock.patch.object(service.utils, 'get_hostname') @mock.patch.object(service.processutils, 'get_worker_count') - def test_service_methods(self, mock_get_worker_count, mock_get_hostname, - mock_conf, mock_rpc): + def test_service_methods( + self, mock_get_worker_count, mock_get_hostname, mock_conf, mock_rpc + ): mock_conf.api_migration_listen = mock.sentinel.listen mock_conf.api_migration_listen_port = mock.sentinel.listen_port mock_conf.api_migration_workers = mock.sentinel.worker_count @@ -259,11 +286,14 @@ def test_service_methods(self, mock_get_worker_count, mock_get_hostname, mock_rpc.get_server.return_value = mock_server result = service.MessagingService( - mock.sentinel.topic, mock.sentinel.endpoints, - mock.sentinel.version, worker_count=None, init_rpc=True) + mock.sentinel.topic, + mock.sentinel.endpoints, + mock.sentinel.version, + worker_count=None, + init_rpc=True, + ) - self.assertEqual(result.get_workers_count(), - mock.sentinel.worker_count) + self.assertEqual(result.get_workers_count(), mock.sentinel.worker_count) result.start() mock_server.start.assert_called_once() diff --git a/coriolis/tests/test_utils.py b/coriolis/tests/test_utils.py index 2bfc8cbf..3344e3f7 100644 --- a/coriolis/tests/test_utils.py +++ b/coriolis/tests/test_utils.py @@ -7,17 +7,14 @@ import logging import os import socket -from unittest import mock import uuid +from unittest import mock import ddt from webob import exc -from coriolis import constants -from coriolis import exception -from coriolis.tests import test_base -from coriolis.tests import testutils -from coriolis import utils +from coriolis import constants, exception, utils +from coriolis.tests import test_base, testutils class CoriolisTestException(Exception): @@ -65,45 +62,55 @@ def test_get_single_result_single_element(self): def test_retry_on_error_no_exception(self): result = utils.retry_on_error( - max_attempts=5, sleep_seconds=0, - terminal_exceptions=[])(self.mock_func)( - mock.sentinel.arg1, kwarg1=mock.sentinel.kwarg1) + max_attempts=5, sleep_seconds=0, terminal_exceptions=[] + )(self.mock_func)(mock.sentinel.arg1, kwarg1=mock.sentinel.kwarg1) self.assertEqual(result, self.mock_func.return_value) self.assertEqual(self.mock_func.call_count, 1) self.mock_func.assert_called_with( - mock.sentinel.arg1, kwarg1=mock.sentinel.kwarg1) + mock.sentinel.arg1, kwarg1=mock.sentinel.kwarg1 + ) def test_retry_on_error_exception_keyboard_interrupt(self): self.mock_func.side_effect = KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, utils.retry_on_error( - max_attempts=5, sleep_seconds=0, - terminal_exceptions=[])(self.mock_func)) + self.assertRaises( + KeyboardInterrupt, + utils.retry_on_error( + max_attempts=5, sleep_seconds=0, terminal_exceptions=[] + )(self.mock_func), + ) self.assertEqual(self.mock_func.call_count, 1) def test_retry_on_error_terminal_exception(self): self.mock_func.side_effect = CoriolisTestException - self.assertRaises(CoriolisTestException, utils.retry_on_error( - max_attempts=5, sleep_seconds=0, - terminal_exceptions=[CoriolisTestException])(self.mock_func)) + self.assertRaises( + CoriolisTestException, + utils.retry_on_error( + max_attempts=5, + sleep_seconds=0, + terminal_exceptions=[CoriolisTestException], + )(self.mock_func), + ) self.assertEqual(self.mock_func.call_count, 1) def test_retry_on_error_exception_retry_max_attempts(self): self.mock_func.side_effect = CoriolisTestException - self.assertRaises(CoriolisTestException, utils.retry_on_error( - max_attempts=5, sleep_seconds=0, - terminal_exceptions=[])(self.mock_func)) + self.assertRaises( + CoriolisTestException, + utils.retry_on_error( + max_attempts=5, sleep_seconds=0, terminal_exceptions=[] + )(self.mock_func), + ) self.assertEqual(self.mock_func.call_count, 5) def test_get_udev_net_rules(self): - net_ifaces_info = {"eth0": "AA:BB:CC:DD:EE:FF", - "eth1": "FF:EE:DD:CC:BB:AA"} + net_ifaces_info = {"eth0": "AA:BB:CC:DD:EE:FF", "eth1": "FF:EE:DD:CC:BB:AA"} expected_result = ( 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' 'ATTR{address}=="aa:bb:cc:dd:ee:ff", ' @@ -123,8 +130,8 @@ def test_parse_os_release(self, mock_ssh_cmd): result = utils.parse_os_release(self.mock_ssh) mock_ssh_cmd.assert_called_once_with( - self.mock_ssh, - "[ -f '/etc/os-release' ] && cat /etc/os-release || true") + self.mock_ssh, "[ -f '/etc/os-release' ] && cat /etc/os-release || true" + ) self.assertEqual(result, ('ubuntu', '20.04')) @@ -135,8 +142,8 @@ def test_parse_os_release_no_equal(self, mock_ssh_cmd): result = utils.parse_os_release(self.mock_ssh) mock_ssh_cmd.assert_called_once_with( - self.mock_ssh, - "[ -f '/etc/os-release' ] && cat /etc/os-release || true") + self.mock_ssh, "[ -f '/etc/os-release' ] && cat /etc/os-release || true" + ) self.assertEqual(result, ("ubuntu", "20.04")) @@ -147,8 +154,8 @@ def test_parse_os_release_missing_fields(self, mock_ssh_cmd): result = utils.parse_os_release(self.mock_ssh) mock_ssh_cmd.assert_called_once_with( - self.mock_ssh, - "[ -f '/etc/os-release' ] && cat /etc/os-release || true") + self.mock_ssh, "[ -f '/etc/os-release' ] && cat /etc/os-release || true" + ) self.assertIsNone(result) @@ -158,8 +165,7 @@ def test_parse_lsb_release(self, mock_ssh_cmd): result = utils.parse_lsb_release(self.mock_ssh) - mock_ssh_cmd.assert_called_once_with( - self.mock_ssh, "lsb_release -a || true") + mock_ssh_cmd.assert_called_once_with(self.mock_ssh, "lsb_release -a || true") self.assertEqual(result, ("Ubuntu", "20.04")) @@ -169,9 +175,7 @@ def test_parse_lsb_release_missing_fields(self, mock_ssh_cmd): result = utils.parse_lsb_release(self.mock_ssh) - mock_ssh_cmd.assert_called_once_with( - self.mock_ssh, - "lsb_release -a || true") + mock_ssh_cmd.assert_called_once_with(self.mock_ssh, "lsb_release -a || true") self.assertIsNone(result) @@ -184,8 +188,9 @@ def test_get_linux_os_info(self, mock_parse_os_release): @mock.patch.object(utils, 'parse_os_release') @mock.patch.object(utils, 'parse_lsb_release') - def test_get_linux_os_info_lsb_release(self, mock_parse_lsb_release, - mock_parse_os_release): + def test_get_linux_os_info_lsb_release( + self, mock_parse_lsb_release, mock_parse_os_release + ): mock_parse_os_release.return_value = None mock_parse_lsb_release.return_value = ("ubuntu", "20.04") @@ -208,8 +213,7 @@ def test_test_ssh_path_exists(self): self.assertEqual(result, True) def test_test_ssh_path_not_exists(self): - self.mock_sftp.stat.side_effect = IOError(2, - "No such file or directory") + self.mock_sftp.stat.side_effect = IOError(2, "No such file or directory") self.mock_ssh.open_sftp.return_value = self.mock_sftp result = utils.test_ssh_path(self.mock_ssh, "/nonexistent/path") @@ -223,11 +227,11 @@ def test_test_ssh_path_raises(self): self.mock_sftp.stat.side_effect = IOError(1, "Some other error") self.mock_ssh.open_sftp.return_value = self.mock_sftp - original_test_ssh_path = testutils.get_wrapped_function( - utils.test_ssh_path) + original_test_ssh_path = testutils.get_wrapped_function(utils.test_ssh_path) - self.assertRaises(IOError, original_test_ssh_path, self.mock_ssh, - "/nonexistent/path") + self.assertRaises( + IOError, original_test_ssh_path, self.mock_ssh, "/nonexistent/path" + ) def test_read_ssh_file(self): self.mock_sftp.open.return_value = self.mock_file @@ -255,8 +259,9 @@ def test_write_winrm_file(self, mock_b64encode): self.mock_conn.test_path.return_value = True self.mock_conn.exec_ps_command.return_value = None - utils.write_winrm_file(self.mock_conn, "/test/file", "file content", - overwrite=True) + utils.write_winrm_file( + self.mock_conn, "/test/file", "file content", overwrite=True + ) self.mock_conn.test_path.assert_called_once_with("/test/file") self.assertEqual(self.mock_conn.exec_ps_command.call_count, 2) @@ -267,8 +272,9 @@ def test_write_winrm_file_file_does_not_exist(self, mock_b64encode): self.mock_conn.test_path.return_value = False self.mock_conn.exec_ps_command.return_value = None - utils.write_winrm_file(self.mock_conn, "/nonexistent/path", - "nonexistent-file", overwrite=True) + utils.write_winrm_file( + self.mock_conn, "/nonexistent/path", "nonexistent-file", overwrite=True + ) self.mock_conn.test_path.assert_called_once_with("/nonexistent/path") mock_b64encode.assert_called_once_with(b"nonexistent-file") @@ -277,9 +283,14 @@ def test_write_winrm_file_file_exists_overwrite_false(self): self.mock_conn.test_path.return_value = True self.mock_conn.exec_ps_command.return_value = None - self.assertRaises(exception.CoriolisException, utils.write_winrm_file, - self.mock_conn, "/test/file", "file content", - overwrite=False) + self.assertRaises( + exception.CoriolisException, + utils.write_winrm_file, + self.mock_conn, + "/test/file", + "file content", + overwrite=False, + ) self.mock_conn.test_path.assert_called_once_with("/test/file") @@ -288,8 +299,9 @@ def test_write_winrm_file_content_is_not_string(self, mock_b64encode): self.mock_conn.test_path.return_value = False self.mock_conn.exec_ps_command.return_value = None - utils.write_winrm_file(self.mock_conn, "/test/file", "file content", - overwrite=True) + utils.write_winrm_file( + self.mock_conn, "/test/file", "file content", overwrite=True + ) self.mock_conn.test_path.assert_called_once_with("/test/file") mock_b64encode.assert_called_once_with(b"file content") @@ -301,13 +313,16 @@ def test_write_winrm_file_long_content(self, mock_b64encode): mock_b64encode.return_value = b'encoded_content' long_content = "a" * 3000 - utils.write_winrm_file(self.mock_conn, "/test/file", long_content, - overwrite=True) + utils.write_winrm_file( + self.mock_conn, "/test/file", long_content, overwrite=True + ) self.mock_conn.test_path.assert_called_once_with("/test/file") self.assertEqual(self.mock_conn.exec_ps_command.call_count, 2) - expected_calls = [mock.call(long_content[:2048].encode()), - mock.call(long_content[2048:].encode())] + expected_calls = [ + mock.call(long_content[:2048].encode()), + mock.call(long_content[2048:].encode()), + ] mock_b64encode.assert_has_calls(expected_calls) def test_list_ssh_dir(self): @@ -323,138 +338,192 @@ def test_exec_ssh_cmd(self): self.mock_stdout.read.return_value = b'output\r\n' self.mock_stdout.channel.recv_exit_status.return_value = 0 self.mock_ssh.exec_command.return_value = ( - None, self.mock_stdout, self.mock_stdout) + None, + self.mock_stdout, + self.mock_stdout, + ) result = utils.exec_ssh_cmd(self.mock_ssh, "command") self.mock_ssh.exec_command.assert_called_once_with( - "command", environment=None, get_pty=False, timeout=None) + "command", environment=None, get_pty=False, timeout=None + ) self.assertEqual(result, 'output\n') def test_exec_ssh_cmd_timeout_with_timeout(self): self.mock_stdout.read.return_value = b'output\n' self.mock_stdout.channel.recv_exit_status.return_value = 0 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) result = utils.exec_ssh_cmd(self.mock_ssh, "command", timeout=10) self.mock_ssh.exec_command.assert_called_once_with( - "command", environment=None, get_pty=False, timeout=10.0) + "command", environment=None, get_pty=False, timeout=10.0 + ) - expected = self.mock_stdout.read.return_value.decode( - 'utf-8', errors='replace') + expected = self.mock_stdout.read.return_value.decode('utf-8', errors='replace') self.assertEqual(result, expected) def test_exec_ssh_cmd_getpeername_value_error(self): self.mock_stdout.read.return_value = b'output\n' self.mock_stdout.channel.recv_exit_status.return_value = 0 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) - self.mock_ssh.get_transport.return_value.sock.\ - getpeername.side_effect = ValueError + self.mock_ssh.get_transport.return_value.sock.getpeername.side_effect = ( + ValueError + ) with self.assertLogs('coriolis.utils', level=logging.WARN): output = utils.exec_ssh_cmd(self.mock_ssh, "command") self.mock_ssh.exec_command.assert_called_once_with( - "command", environment=None, get_pty=False, timeout=None) + "command", environment=None, get_pty=False, timeout=None + ) - expected = self.mock_stdout.read.return_value.decode( - 'utf-8', errors='replace') + expected = self.mock_stdout.read.return_value.decode('utf-8', errors='replace') self.assertEqual(output, expected) def test_exec_ssh_cmd_timeout(self): self.mock_stdout.read.side_effect = socket.timeout - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) - self.assertRaises(exception.MinionMachineCommandTimeout, - utils.exec_ssh_cmd, self.mock_ssh, "command") + self.assertRaises( + exception.MinionMachineCommandTimeout, + utils.exec_ssh_cmd, + self.mock_ssh, + "command", + ) self.mock_ssh.exec_command.assert_called_once_with( - "command", environment=None, get_pty=False, timeout=None) + "command", environment=None, get_pty=False, timeout=None + ) def test_exec_ssh_cmd_exception(self): self.mock_stdout.read.return_value = b'some error output' self.mock_stdout.channel.recv_exit_status.return_value = 1 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) - original_exec_ssh_cmd = testutils.get_wrapped_function( - utils.exec_ssh_cmd) + original_exec_ssh_cmd = testutils.get_wrapped_function(utils.exec_ssh_cmd) - self.assertRaises(exception.CoriolisException, original_exec_ssh_cmd, - self.mock_ssh, "command") + self.assertRaises( + exception.CoriolisException, original_exec_ssh_cmd, self.mock_ssh, "command" + ) self.mock_ssh.exec_command.assert_called_once_with( - "command", environment=None, get_pty=False, timeout=None) + "command", environment=None, get_pty=False, timeout=None + ) def test_exec_ssh_cmd_command_not_found_in_stdout(self): self.mock_stdout.read.return_value = b'sudo: foo: command not found' self.mock_stdout.channel.recv_exit_status.return_value = 1 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) - original_exec_ssh_cmd = testutils.get_wrapped_function( - utils.exec_ssh_cmd) + original_exec_ssh_cmd = testutils.get_wrapped_function(utils.exec_ssh_cmd) - self.assertRaises(exception.SSHCommandNotFoundException, - original_exec_ssh_cmd, self.mock_ssh, "command") + self.assertRaises( + exception.SSHCommandNotFoundException, + original_exec_ssh_cmd, + self.mock_ssh, + "command", + ) def test_exec_ssh_cmd_exit_code_127(self): self.mock_stdout.read.return_value = b'' self.mock_stdout.channel.recv_exit_status.return_value = 127 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) - original_exec_ssh_cmd = testutils.get_wrapped_function( - utils.exec_ssh_cmd) + original_exec_ssh_cmd = testutils.get_wrapped_function(utils.exec_ssh_cmd) - self.assertRaises(exception.SSHCommandNotFoundException, - original_exec_ssh_cmd, self.mock_ssh, "command") + self.assertRaises( + exception.SSHCommandNotFoundException, + original_exec_ssh_cmd, + self.mock_ssh, + "command", + ) def test_exec_ssh_cmd_chroot(self): self.mock_stdout.read.return_value = b'output\n' self.mock_stdout.channel.recv_exit_status.return_value = 0 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) result = utils.exec_ssh_cmd_chroot( - self.mock_ssh, "/chroot /bin/bash -c", "command") + self.mock_ssh, "/chroot /bin/bash -c", "command" + ) self.mock_ssh.exec_command.assert_called_once_with( "sudo -E chroot /chroot /bin/bash -c command", - environment=None, get_pty=False, timeout=None) + environment=None, + get_pty=False, + timeout=None, + ) - expected = self.mock_stdout.read.return_value.decode( - 'utf-8', errors='replace') + expected = self.mock_stdout.read.return_value.decode('utf-8', errors='replace') self.assertEqual(result, expected) def test_check_fs(self): - self.mock_stdout.read.return_value.replace.return_value = \ + self.mock_stdout.read.return_value.replace.return_value = ( self.mock_stdout.read.return_value + ) self.mock_stdout.channel.recv_exit_status.return_value = 0 - self.mock_ssh.exec_command.return_value = (None, self.mock_stdout, - self.mock_stdout) + self.mock_ssh.exec_command.return_value = ( + None, + self.mock_stdout, + self.mock_stdout, + ) utils.check_fs(self.mock_ssh, "ext4", "/dev/sda1") self.mock_ssh.exec_command.assert_called_once_with( - "sudo fsck -p -t ext4 /dev/sda1", environment=None, get_pty=True, - timeout=None) + "sudo fsck -p -t ext4 /dev/sda1", + environment=None, + get_pty=True, + timeout=None, + ) @mock.patch.object(utils, 'exec_ssh_cmd') def test_check_fs_exception(self, mock_exec_ssh_cmd): mock_exec_ssh_cmd.side_effect = exception.CoriolisException() - self.assertRaises(exception.CoriolisException, utils.check_fs, - self.mock_ssh, "ext4", "/dev/sda1") + self.assertRaises( + exception.CoriolisException, + utils.check_fs, + self.mock_ssh, + "ext4", + "/dev/sda1", + ) mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, "sudo fsck -p -t ext4 /dev/sda1", get_pty=True) + self.mock_ssh, "sudo fsck -p -t ext4 /dev/sda1", get_pty=True + ) @mock.patch.object(utils, 'exec_ssh_cmd') def test_run_xfs_repair(self, mock_exec_ssh_cmd): @@ -464,12 +533,9 @@ def test_run_xfs_repair(self, mock_exec_ssh_cmd): expected_calls = [ mock.call(self.mock_ssh, "mktemp -d"), - mock.call(self.mock_ssh, "sudo mount /dev/sda1 /tmp/tmp_dir", - get_pty=True), - mock.call(self.mock_ssh, "sudo umount /tmp/tmp_dir", - get_pty=True), - mock.call(self.mock_ssh, "sudo xfs_repair /dev/sda1", - get_pty=True), + mock.call(self.mock_ssh, "sudo mount /dev/sda1 /tmp/tmp_dir", get_pty=True), + mock.call(self.mock_ssh, "sudo umount /tmp/tmp_dir", get_pty=True), + mock.call(self.mock_ssh, "sudo xfs_repair /dev/sda1", get_pty=True), ] mock_exec_ssh_cmd.assert_has_calls(expected_calls) @@ -498,8 +564,7 @@ def test_check_port_open_exception(self, mock_socket): @mock.patch.object(utils, '_check_port_open') @mock.patch('time.sleep') - def test_wait_for_port_connectivity(self, mock_sleep, - mock_check_port_open): + def test_wait_for_port_connectivity(self, mock_sleep, mock_check_port_open): mock_check_port_open.return_value = True mock_sleep.return_value = None @@ -508,14 +573,19 @@ def test_wait_for_port_connectivity(self, mock_sleep, @mock.patch.object(utils, '_check_port_open') @mock.patch('time.sleep') - def test_wait_for_port_connectivity_exception(self, mock_sleep, - mock_check_port_open): + def test_wait_for_port_connectivity_exception( + self, mock_sleep, mock_check_port_open + ): mock_check_port_open.return_value = False mock_sleep.return_value = None - self.assertRaises(exception.CoriolisException, - utils.wait_for_port_connectivity, 'localhost', 8080, - max_wait=1) + self.assertRaises( + exception.CoriolisException, + utils.wait_for_port_connectivity, + 'localhost', + 8080, + max_wait=1, + ) mock_check_port_open.assert_called_with('localhost', 8080) @mock.patch('subprocess.Popen') @@ -535,8 +605,7 @@ def test_exec_process_exception(self, mock_popen): self.mock_process.communicate.return_value = (b'stdout', b'stderr') mock_popen.return_value = self.mock_process - self.assertRaises(exception.CoriolisException, utils.exec_process, - "command") + self.assertRaises(exception.CoriolisException, utils.exec_process, "command") mock_popen.assert_called_once_with("command", stdout=-1, stderr=-1) @@ -548,45 +617,74 @@ def test_get_disk_info(self, mock_exec_process): self.assertEqual(result, {'format': 'vhd'}) - mock_exec_process.assert_called_with([utils.CONF.qemu_img_path, - 'info', '--output=json', - 'disk_path']) + mock_exec_process.assert_called_with( + [utils.CONF.qemu_img_path, 'info', '--output=json', 'disk_path'] + ) @mock.patch.object(utils, 'exec_process') def test_convert_disk_format(self, mock_exec_process): mock_exec_process.return_value = None - utils.convert_disk_format('disk_path', 'target_disk_path', - constants.DISK_FORMAT_VHD, - preallocated=True) + utils.convert_disk_format( + 'disk_path', + 'target_disk_path', + constants.DISK_FORMAT_VHD, + preallocated=True, + ) - mock_exec_process.assert_called_with([utils.CONF.qemu_img_path, - 'convert', '-O', 'vpc', '-o', - 'subformat=fixed', - 'disk_path', - 'target_disk_path']) + mock_exec_process.assert_called_with( + [ + utils.CONF.qemu_img_path, + 'convert', + '-O', + 'vpc', + '-o', + 'subformat=fixed', + 'disk_path', + 'target_disk_path', + ] + ) def test_convert_disk_format_not_vhd(self): - self.assertRaises(NotImplementedError, utils.convert_disk_format, - 'disk_path', 'target_disk_path', 'not_vhd', - preallocated=True) + self.assertRaises( + NotImplementedError, + utils.convert_disk_format, + 'disk_path', + 'target_disk_path', + 'not_vhd', + preallocated=True, + ) @mock.patch.object(utils, 'exec_process') @mock.patch.object(utils, 'ignore_exceptions') - def test_convert_disk_format_exception(self, mock_ignore_exceptions, - mock_exec_process): + def test_convert_disk_format_exception( + self, mock_ignore_exceptions, mock_exec_process + ): mock_remove = mock.MagicMock() mock_exec_process.side_effect = exception.CoriolisException mock_ignore_exceptions.return_value = mock_remove - self.assertRaises(exception.CoriolisException, - utils.convert_disk_format, 'disk_path', - 'target_disk_path', constants.DISK_FORMAT_VHD, - preallocated=True) + self.assertRaises( + exception.CoriolisException, + utils.convert_disk_format, + 'disk_path', + 'target_disk_path', + constants.DISK_FORMAT_VHD, + preallocated=True, + ) mock_exec_process.assert_called_with( - [utils.CONF.qemu_img_path, 'convert', '-O', 'vpc', '-o', - 'subformat=fixed', 'disk_path', 'target_disk_path']) + [ + utils.CONF.qemu_img_path, + 'convert', + '-O', + 'vpc', + '-o', + 'subformat=fixed', + 'disk_path', + 'target_disk_path', + ] + ) mock_ignore_exceptions.assert_called_with(os.remove) mock_remove.assert_called_with('target_disk_path') @@ -617,19 +715,19 @@ class A: @mock.patch('socket.socket') @mock.patch('ssl.SSLContext') @mock.patch('OpenSSL.crypto') - def test_get_ssl_cert_thumbprint(self, mock_crypto, mock_ssl_context, - mock_socket): + def test_get_ssl_cert_thumbprint(self, mock_crypto, mock_ssl_context, mock_socket): mock_socket.return_value = mock.MagicMock() mock_ssl_context.return_value = mock.MagicMock() result = utils.get_ssl_cert_thumbprint( - mock_ssl_context.return_value, 'localhost', - digest_algorithm='sha1') + mock_ssl_context.return_value, 'localhost', digest_algorithm='sha1' + ) - self.assertEqual(result, mock_crypto.load_certificate.return_value. - digest.return_value.decode.return_value) - mock_crypto.load_certificate.return_value.digest.\ - assert_called_once_with('sha1') + self.assertEqual( + result, + mock_crypto.load_certificate.return_value.digest.return_value.decode.return_value, + ) + mock_crypto.load_certificate.return_value.digest.assert_called_once_with('sha1') @mock.patch('os.path') def test_get_resources_dir(self, mock_path): @@ -667,7 +765,8 @@ def test_deserialize_key(self, mock_rsa_key, mock_string_io): result = utils.deserialize_key('key', password='password') mock_rsa_key.from_private_key.assert_called_with( - mock_string_io.return_value, 'password') + mock_string_io.return_value, 'password' + ) self.assertEqual(result, mock_rsa_key.from_private_key.return_value) @mock.patch('coriolis.utils.jsonutils.dumps') @@ -703,14 +802,12 @@ def test_check_md5_with_matching_hashes(self): def test_check_md5_with_exception(self): data = b'test data' md5 = hashlib.md5(b'other data').hexdigest() - self.assertRaises(exception.CoriolisException, utils.check_md5, data, - md5) + self.assertRaises(exception.CoriolisException, utils.check_md5, data, md5) @mock.patch.object(utils, 'secrets') def test_get_secret_connection_info_with_secret_ref(self, mock_secrets): with self.assertLogs('coriolis.utils', level=logging.INFO): - result = utils.get_secret_connection_info('context', - {'secret_ref': 'ref'}) + result = utils.get_secret_connection_info('context', {'secret_ref': 'ref'}) self.assertEqual(result, mock_secrets.get_secret.return_value) @@ -743,8 +840,9 @@ def test_decode_base64_param(self, value, is_json, expected): ) @ddt.unpack def test_decode_base64_param_with_invalid_input(self, value, is_json): - self.assertRaises(exception.InvalidInput, utils.decode_base64_param, - value, is_json=is_json) + self.assertRaises( + exception.InvalidInput, utils.decode_base64_param, value, is_json=is_json + ) def test_quote_url(self): result = utils.quote_url('Hello world') @@ -768,8 +866,7 @@ def test_normalize_mac_address(self, input, expected): 123456789012, ) def test_normalize_mac_address_with_invalid_input(self, input): - self.assertRaises(ValueError, utils.normalize_mac_address, - input) + self.assertRaises(ValueError, utils.normalize_mac_address, input) def test_get_url_with_credentials(self): url = 'http://example.com' @@ -796,26 +893,26 @@ def test_get_url_with_credentials_existing_credentials(self): [ {'id': '1', 'name': 'Resource1'}, {'id': '2', 'name': 'Resource2'}, - {'id': '3', 'name': 'Resource1'} + {'id': '3', 'name': 'Resource1'}, ], - ['Resource2', '1', '3'] + ['Resource2', '1', '3'], ), ( [ {'id': '1', 'name': 'Resource1'}, {'id': '2', 'name': 'Resource2'}, - {'id': '3', 'name': 'Resource3'} + {'id': '3', 'name': 'Resource3'}, ], - ['Resource1', 'Resource2', 'Resource3'] + ['Resource1', 'Resource2', 'Resource3'], ), ( [ {'id': '1', 'name': 'Resource1'}, {'id': '2', 'name': 'Resource2'}, - {'id': '3'} + {'id': '3'}, ], - KeyError - ) + KeyError, + ), ) @ddt.unpack def test_get_unique_option_ids(self, resources, expected): @@ -829,11 +926,12 @@ def test_get_unique_option_ids_with_custom_keys(self): resources = [ {'custom_id': '1', 'custom_name': 'Resource1'}, {'custom_id': '2', 'custom_name': 'Resource2'}, - {'custom_id': '3', 'custom_name': 'Resource1'} + {'custom_id': '3', 'custom_name': 'Resource1'}, ] expected_result = ['Resource2', '1', '3'] result = utils.get_unique_option_ids( - resources, id_key='custom_id', name_key='custom_name') + resources, id_key='custom_id', name_key='custom_name' + ) self.assertEqual(sorted(result), sorted(expected_result)) def test_bad_request_on_error(self): @@ -853,40 +951,40 @@ def mock_func(): self.assertRaises(exc.HTTPBadRequest, mock_func) @ddt.data( - ({ - "key1": "value1", - "origin": {"connection_info": "sensitive_info"}, - "destination": {"connection_info": "sensitive_info"}, - "volumes_info": [ - { - "key2": "value2", - "replica_state": { - "key3": "value3", - "chunks": "sensitive_info" + ( + { + "key1": "value1", + "origin": {"connection_info": "sensitive_info"}, + "destination": {"connection_info": "sensitive_info"}, + "volumes_info": [ + { + "key2": "value2", + "replica_state": {"key3": "value3", "chunks": "sensitive_info"}, } - } - ] - }, { - "key1": "value1", - "origin": {"connection_info": {"got": "redacted"}}, - "destination": {"connection_info": {"got": "redacted"}}, - "volumes_info": [ - { - "key2": "value2", - "replica_state": { - "key3": "value3", - "chunks": [""] + ], + }, + { + "key1": "value1", + "origin": {"connection_info": {"got": "redacted"}}, + "destination": {"connection_info": {"got": "redacted"}}, + "volumes_info": [ + { + "key2": "value2", + "replica_state": {"key3": "value3", "chunks": [""]}, } - } - ] - }), - ({ - "key1": "value1", - "key2": "value2", - }, { - "key1": "value1", - "key2": "value2", - }), + ], + }, + ), + ( + { + "key1": "value1", + "key2": "value2", + }, + { + "key1": "value1", + "key2": "value2", + }, + ), ) @ddt.unpack def test_sanitize_task_info(self, task_info, expected): @@ -909,37 +1007,41 @@ def test_parse_ini_config(self): @mock.patch('coriolis.utils.test_ssh_path') @mock.patch('coriolis.utils.read_ssh_file') @mock.patch('coriolis.utils.parse_ini_config') - def test_read_ssh_ini_config_file_check_false(self, mock_parse_ini_config, - mock_read_ssh_file, - mock_test_ssh_path): + def test_read_ssh_ini_config_file_check_false( + self, mock_parse_ini_config, mock_read_ssh_file, mock_test_ssh_path + ): mock_test_ssh_path.return_value = True - result = utils.read_ssh_ini_config_file(self.mock_ssh, '/test/file', - check_exists=False) + result = utils.read_ssh_ini_config_file( + self.mock_ssh, '/test/file', check_exists=False + ) self.assertEqual(result, mock_parse_ini_config.return_value) mock_read_ssh_file.assert_called_once_with(self.mock_ssh, '/test/file') mock_parse_ini_config.assert_called_once_with( - mock_read_ssh_file.return_value.decode()) + mock_read_ssh_file.return_value.decode() + ) @mock.patch('coriolis.utils.test_ssh_path') def test_read_ssh_ini_config_file_check_true_path_not_exists( - self, mock_test_ssh_path): + self, mock_test_ssh_path + ): mock_test_ssh_path.return_value = False - result = utils.read_ssh_ini_config_file(self.mock_ssh, '/test/to/file', - check_exists=True) + result = utils.read_ssh_ini_config_file( + self.mock_ssh, '/test/to/file', check_exists=True + ) self.assertEqual(result, {}) - mock_test_ssh_path.assert_called_once_with(self.mock_ssh, - '/test/to/file') + mock_test_ssh_path.assert_called_once_with(self.mock_ssh, '/test/to/file') @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_write_systemd(self, mock_uuid, mock_test_ssh, - mock_write_ssh_file, mock_exec_ssh_cmd): + def test_write_systemd( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.side_effect = [True, False] mock_write_ssh_file.return_value = None @@ -947,131 +1049,167 @@ def test_write_systemd(self, mock_uuid, mock_test_ssh, utils._write_systemd(self.mock_ssh, 'cmdline', 'svc_name') mock_uuid.assert_called_once_with() - mock_test_ssh.assert_has_calls([ - mock.call(self.mock_ssh, '/lib/systemd/system'), - mock.call(self.mock_ssh, - '/lib/systemd/system/svc_name.service')]) - mock_write_ssh_file.assert_called_once_with(self.mock_ssh, - '/tmp/uuid.service', - mock.ANY) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.mock_ssh, 'sudo mv /tmp/uuid.service ' - '/lib/systemd/system/svc_name.service', get_pty=True), - mock.call(self.mock_ssh, 'sudo restorecon -v ' - '/lib/systemd/system/svc_name.service', get_pty=True), - mock.call(self.mock_ssh, 'sudo systemctl daemon-reload', - get_pty=True), - mock.call(self.mock_ssh, 'sudo systemctl start svc_name', - get_pty=True)]) + mock_test_ssh.assert_has_calls( + [ + mock.call(self.mock_ssh, '/lib/systemd/system'), + mock.call(self.mock_ssh, '/lib/systemd/system/svc_name.service'), + ] + ) + mock_write_ssh_file.assert_called_once_with( + self.mock_ssh, '/tmp/uuid.service', mock.ANY + ) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self.mock_ssh, + 'sudo mv /tmp/uuid.service /lib/systemd/system/svc_name.service', + get_pty=True, + ), + mock.call( + self.mock_ssh, + 'sudo restorecon -v /lib/systemd/system/svc_name.service', + get_pty=True, + ), + mock.call(self.mock_ssh, 'sudo systemctl daemon-reload', get_pty=True), + mock.call(self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True), + ] + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_write_systemd_usr_lib(self, mock_uuid, mock_test_ssh, - mock_write_ssh_file, mock_exec_ssh_cmd): + def test_write_systemd_usr_lib( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.side_effect = [False, False] mock_write_ssh_file.return_value = None utils._write_systemd(self.mock_ssh, 'cmdline', 'svc_name') - mock_test_ssh.assert_has_calls([ - mock.call(self.mock_ssh, '/lib/systemd/system'), - mock.call(self.mock_ssh, - '/usr/lib/systemd/system/svc_name.service')]) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.mock_ssh, 'sudo mv /tmp/uuid.service ' - '/usr/lib/systemd/system/svc_name.service', - get_pty=True)]) + mock_test_ssh.assert_has_calls( + [ + mock.call(self.mock_ssh, '/lib/systemd/system'), + mock.call(self.mock_ssh, '/usr/lib/systemd/system/svc_name.service'), + ] + ) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self.mock_ssh, + 'sudo mv /tmp/uuid.service ' + '/usr/lib/systemd/system/svc_name.service', + get_pty=True, + ) + ] + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_write_systemd_service_exists(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_write_systemd_service_exists(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.return_value = True utils._write_systemd(self.mock_ssh, 'cmdline', 'svc_name', start=True) - mock_test_ssh.assert_has_calls([ - mock.call(self.mock_ssh, '/lib/systemd/system'), - mock.call(self.mock_ssh, - '/lib/systemd/system/svc_name.service')]) + mock_test_ssh.assert_has_calls( + [ + mock.call(self.mock_ssh, '/lib/systemd/system'), + mock.call(self.mock_ssh, '/lib/systemd/system/svc_name.service'), + ] + ) mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True) + self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_write_systemd_service_selinux_exception(self, mock_uuid, - mock_test_ssh, - mock_write_ssh_file, - mock_exec_ssh_cmd): + def test_write_systemd_service_selinux_exception( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.side_effect = [True, False] mock_write_ssh_file.return_value = None mock_exec_ssh_cmd.side_effect = [ - None, exception.CoriolisException(), None, None] + None, + exception.CoriolisException(), + None, + None, + ] _write_systemd_undecorated = testutils.get_wrapped_function( - utils._write_systemd) + utils._write_systemd + ) with self.assertLogs('coriolis.utils', level=logging.WARN): - _write_systemd_undecorated(self.mock_ssh, '/test/file', 'svc_name', - start=True) + _write_systemd_undecorated( + self.mock_ssh, '/test/file', 'svc_name', start=True + ) mock_uuid.assert_called_once_with() - mock_test_ssh.assert_has_calls([ - mock.call(self.mock_ssh, '/lib/systemd/system'), - mock.call(self.mock_ssh, - '/lib/systemd/system/svc_name.service')]) - mock_write_ssh_file.assert_called_once_with(self.mock_ssh, - '/tmp/uuid.service', - mock.ANY) + mock_test_ssh.assert_has_calls( + [ + mock.call(self.mock_ssh, '/lib/systemd/system'), + mock.call(self.mock_ssh, '/lib/systemd/system/svc_name.service'), + ] + ) + mock_write_ssh_file.assert_called_once_with( + self.mock_ssh, '/tmp/uuid.service', mock.ANY + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_test_write_systemd_with_run_as(self, mock_uuid, mock_test_ssh, - mock_write_ssh_file, - mock_exec_ssh_cmd): + def test_test_write_systemd_with_run_as( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.side_effect = [True, False] - utils._write_systemd(self.mock_ssh, 'cmdline', 'svc_name', - run_as='test_user') + utils._write_systemd(self.mock_ssh, 'cmdline', 'svc_name', run_as='test_user') mock_uuid.assert_called_once_with() - mock_test_ssh.assert_has_calls([ - mock.call(self.mock_ssh, '/lib/systemd/system'), - mock.call(self.mock_ssh, - '/lib/systemd/system/svc_name.service')]) + mock_test_ssh.assert_has_calls( + [ + mock.call(self.mock_ssh, '/lib/systemd/system'), + mock.call(self.mock_ssh, '/lib/systemd/system/svc_name.service'), + ] + ) mock_write_ssh_file.assert_called_once_with( - self.mock_ssh, '/tmp/uuid.service', - utils.SYSTEMD_TEMPLATE % { - "cmdline": 'cmdline', - "username": 'test_user', - "svc_name": 'svc_name'}) - - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.mock_ssh, 'sudo mv /tmp/uuid.service ' - '/lib/systemd/system/svc_name.service', get_pty=True), - mock.call(self.mock_ssh, 'sudo restorecon -v ' - '/lib/systemd/system/svc_name.service', get_pty=True), - mock.call(self.mock_ssh, 'sudo systemctl daemon-reload', - get_pty=True), - mock.call(self.mock_ssh, 'sudo systemctl start svc_name', - get_pty=True)]) + self.mock_ssh, + '/tmp/uuid.service', + utils.SYSTEMD_TEMPLATE + % {"cmdline": 'cmdline', "username": 'test_user', "svc_name": 'svc_name'}, + ) + + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self.mock_ssh, + 'sudo mv /tmp/uuid.service /lib/systemd/system/svc_name.service', + get_pty=True, + ), + mock.call( + self.mock_ssh, + 'sudo restorecon -v /lib/systemd/system/svc_name.service', + get_pty=True, + ), + mock.call(self.mock_ssh, 'sudo systemctl daemon-reload', get_pty=True), + mock.call(self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True), + ] + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_write_upstart(self, mock_uuid, mock_test_ssh, - mock_write_ssh_file, mock_exec_ssh_cmd): + def test_write_upstart( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.return_value = False mock_write_ssh_file.return_value = None @@ -1079,15 +1217,20 @@ def test_write_upstart(self, mock_uuid, mock_test_ssh, utils._write_upstart(self.mock_ssh, 'cmdline', 'svc_name') mock_uuid.assert_called_once_with() - mock_test_ssh.assert_called_once_with( - self.mock_ssh, '/etc/init/svc_name.conf') - mock_write_ssh_file.assert_called_once_with(self.mock_ssh, - '/tmp/uuid.conf', - mock.ANY) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.mock_ssh, 'sudo mv /tmp/uuid.conf ' - '/etc/init/svc_name.conf', get_pty=True), - mock.call(self.mock_ssh, 'start svc_name')]) + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/etc/init/svc_name.conf') + mock_write_ssh_file.assert_called_once_with( + self.mock_ssh, '/tmp/uuid.conf', mock.ANY + ) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self.mock_ssh, + 'sudo mv /tmp/uuid.conf /etc/init/svc_name.conf', + get_pty=True, + ), + mock.call(self.mock_ssh, 'start svc_name'), + ] + ) @mock.patch('coriolis.utils.test_ssh_path') def test_write_upstart_service_exists(self, mock_test_ssh): @@ -1095,173 +1238,184 @@ def test_write_upstart_service_exists(self, mock_test_ssh): utils._write_upstart(self.mock_ssh, 'cmdline', 'svc_name') - mock_test_ssh.assert_called_once_with( - self.mock_ssh, '/etc/init/svc_name.conf') + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/etc/init/svc_name.conf') @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.write_ssh_file') @mock.patch('coriolis.utils.test_ssh_path') @mock.patch.object(uuid, 'uuid4') - def test_write_upstart_with_run_as(self, mock_uuid, mock_test_ssh, - mock_write_ssh_file, - mock_exec_ssh_cmd): + def test_write_upstart_with_run_as( + self, mock_uuid, mock_test_ssh, mock_write_ssh_file, mock_exec_ssh_cmd + ): mock_uuid.return_value = 'uuid' mock_test_ssh.return_value = False - utils._write_upstart(self.mock_ssh, 'cmdline', 'svc_name', - run_as='test-user') + utils._write_upstart(self.mock_ssh, 'cmdline', 'svc_name', run_as='test-user') mock_uuid.assert_called_once_with() - mock_test_ssh.assert_called_once_with( - self.mock_ssh, '/etc/init/svc_name.conf') + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/etc/init/svc_name.conf') mock_write_ssh_file.assert_called_once_with( - self.mock_ssh, '/tmp/uuid.conf', - utils.UPSTART_TEMPLATE % { - "cmdline": 'sudo -u test-user -- cmdline', - "svc_name": 'svc_name'}) + self.mock_ssh, + '/tmp/uuid.conf', + utils.UPSTART_TEMPLATE + % {"cmdline": 'sudo -u test-user -- cmdline', "svc_name": 'svc_name'}, + ) - mock_exec_ssh_cmd.assert_has_calls([ - mock.call(self.mock_ssh, 'sudo mv /tmp/uuid.conf ' - '/etc/init/svc_name.conf', get_pty=True), - mock.call(self.mock_ssh, 'start svc_name')]) + mock_exec_ssh_cmd.assert_has_calls( + [ + mock.call( + self.mock_ssh, + 'sudo mv /tmp/uuid.conf /etc/init/svc_name.conf', + get_pty=True, + ), + mock.call(self.mock_ssh, 'start svc_name'), + ] + ) @mock.patch('coriolis.utils._write_systemd') @mock.patch('coriolis.utils.test_ssh_path') def test_create_service_systemd(self, mock_test_ssh, mock_write_systemd): mock_test_ssh.return_value = True - utils.create_service(self.mock_ssh, 'cmdline', 'svc_name', - run_as='user', start=True) + utils.create_service( + self.mock_ssh, 'cmdline', 'svc_name', run_as='user', start=True + ) - mock_write_systemd.assert_called_once_with(self.mock_ssh, 'cmdline', - 'svc_name', run_as='user', - start=True) + mock_write_systemd.assert_called_once_with( + self.mock_ssh, 'cmdline', 'svc_name', run_as='user', start=True + ) @mock.patch('coriolis.utils._write_upstart') @mock.patch('coriolis.utils.test_ssh_path') def test_create_service_upstart(self, mock_test_ssh, mock_write_upstart): mock_test_ssh.side_effect = [False, False, True] - utils.create_service(self.mock_ssh, 'cmdline', 'svc_name', - run_as='user', start=True) + utils.create_service( + self.mock_ssh, 'cmdline', 'svc_name', run_as='user', start=True + ) - mock_write_upstart.assert_called_once_with(self.mock_ssh, 'cmdline', - 'svc_name', run_as='user', - start=True) + mock_write_upstart.assert_called_once_with( + self.mock_ssh, 'cmdline', 'svc_name', run_as='user', start=True + ) @mock.patch('coriolis.utils._write_systemd') @mock.patch('coriolis.utils.test_ssh_path') def test_create_service_exception(self, mock_test_ssh, mock_write_systemd): mock_test_ssh.return_value = False - create_svc_undecorated = testutils.get_wrapped_function( - utils.create_service) + create_svc_undecorated = testutils.get_wrapped_function(utils.create_service) - self.assertRaises(exception.CoriolisException, create_svc_undecorated, - self.mock_ssh, 'cmdline', 'svc_name', run_as='user', - start=True) + self.assertRaises( + exception.CoriolisException, + create_svc_undecorated, + self.mock_ssh, + 'cmdline', + 'svc_name', + run_as='user', + start=True, + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_restart_service_with_systemd(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_restart_service_with_systemd(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.return_value = True utils.restart_service(self.mock_ssh, 'svc_name') - mock_test_ssh.assert_called_once_with(self.mock_ssh, - '/lib/systemd/system') + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/lib/systemd/system') mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'sudo systemctl restart svc_name', get_pty=True) + self.mock_ssh, 'sudo systemctl restart svc_name', get_pty=True + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_restart_service_with_upstart(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_restart_service_with_upstart(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.side_effect = [False, False, True] utils.restart_service(self.mock_ssh, 'svc_name') - mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'restart svc_name') + mock_exec_ssh_cmd.assert_called_once_with(self.mock_ssh, 'restart svc_name') @mock.patch('coriolis.utils.test_ssh_path') def test_restart_service_exception(self, mock_test_ssh): mock_test_ssh.return_value = False - restart_svc_undecorated = testutils.get_wrapped_function( - utils.restart_service) + restart_svc_undecorated = testutils.get_wrapped_function(utils.restart_service) - self.assertRaises(exception.UnrecognizedWorkerInitSystem, - restart_svc_undecorated, self.mock_ssh, 'svc_name') + self.assertRaises( + exception.UnrecognizedWorkerInitSystem, + restart_svc_undecorated, + self.mock_ssh, + 'svc_name', + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_start_service_with_systemd(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_start_service_with_systemd(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.return_value = True utils.start_service(self.mock_ssh, 'svc_name') - mock_test_ssh.assert_called_once_with(self.mock_ssh, - '/lib/systemd/system') + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/lib/systemd/system') mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True) + self.mock_ssh, 'sudo systemctl start svc_name', get_pty=True + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_start_service_with_upstart(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_start_service_with_upstart(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.side_effect = [False, False, True] utils.start_service(self.mock_ssh, 'svc_name') - mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'start svc_name') + mock_exec_ssh_cmd.assert_called_once_with(self.mock_ssh, 'start svc_name') @mock.patch('coriolis.utils.test_ssh_path') def test_start_service_exception(self, mock_test_ssh): mock_test_ssh.return_value = False - start_svc_undecorated = testutils.get_wrapped_function( - utils.start_service) + start_svc_undecorated = testutils.get_wrapped_function(utils.start_service) - self.assertRaises(exception.UnrecognizedWorkerInitSystem, - start_svc_undecorated, self.mock_ssh, 'svc_name') + self.assertRaises( + exception.UnrecognizedWorkerInitSystem, + start_svc_undecorated, + self.mock_ssh, + 'svc_name', + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_stop_service_with_systemd(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_stop_service_with_systemd(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.return_value = True utils.stop_service(self.mock_ssh, 'svc_name') - mock_test_ssh.assert_called_once_with(self.mock_ssh, - '/lib/systemd/system') + mock_test_ssh.assert_called_once_with(self.mock_ssh, '/lib/systemd/system') mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'sudo systemctl stop svc_name', get_pty=True) + self.mock_ssh, 'sudo systemctl stop svc_name', get_pty=True + ) @mock.patch('coriolis.utils.exec_ssh_cmd') @mock.patch('coriolis.utils.test_ssh_path') - def test_stop_service_with_upstart(self, mock_test_ssh, - mock_exec_ssh_cmd): + def test_stop_service_with_upstart(self, mock_test_ssh, mock_exec_ssh_cmd): mock_test_ssh.side_effect = [False, False, True] utils.stop_service(self.mock_ssh, 'svc_name') - mock_exec_ssh_cmd.assert_called_once_with( - self.mock_ssh, 'stop svc_name') + mock_exec_ssh_cmd.assert_called_once_with(self.mock_ssh, 'stop svc_name') @mock.patch('coriolis.utils.test_ssh_path') def test_stop_service_exception(self, mock_test_ssh): mock_test_ssh.return_value = False - stop_svc_undecorated = testutils.get_wrapped_function( - utils.stop_service) + stop_svc_undecorated = testutils.get_wrapped_function(utils.stop_service) - self.assertRaises(exception.UnrecognizedWorkerInitSystem, - stop_svc_undecorated, self.mock_ssh, 'svc_name') + self.assertRaises( + exception.UnrecognizedWorkerInitSystem, + stop_svc_undecorated, + self.mock_ssh, + 'svc_name', + ) @ddt.ddt @@ -1275,8 +1429,7 @@ def setUp(self): def test__init__(self): result = utils.Grub2ConfigEditor(self.cfg) - self.assertEqual( - result._parsed, [{'type': 'raw', 'payload': self.cfg}]) + self.assertEqual(result._parsed, [{'type': 'raw', 'payload': self.cfg}]) def test_parse_cfg_comment_line(self): self.cfg = '# This is a comment' @@ -1286,52 +1439,80 @@ def test_parse_cfg_comment_line(self): def test_parse_cfg_option_line_with_quoted_value(self): self.cfg = 'option="value"' - expected_result = [{'type': 'option', 'payload': self.cfg, - 'quoted': True, 'option_name': 'option', - 'option_value': - [{'opt_type': 'single', 'opt_val': 'value'}]}] + expected_result = [ + { + 'type': 'option', + 'payload': self.cfg, + 'quoted': True, + 'option_name': 'option', + 'option_value': [{'opt_type': 'single', 'opt_val': 'value'}], + } + ] result = self.parser._parse_cfg(self.cfg) self.assertEqual(result, expected_result) def test_parse_cfg_option_line_without_value(self): self.cfg = 'option=' - expected_result = [{'type': 'option', 'payload': self.cfg, - 'quoted': False, 'option_name': 'option', - 'option_value': - [{'opt_type': 'single', 'opt_val': ''}]}] + expected_result = [ + { + 'type': 'option', + 'payload': self.cfg, + 'quoted': False, + 'option_name': 'option', + 'option_value': [{'opt_type': 'single', 'opt_val': ''}], + } + ] result = self.parser._parse_cfg(self.cfg) self.assertEqual(result, expected_result) def test_parse_cfg_option_line_with_value(self): self.cfg = 'option=value' - expected_result = [{'type': 'option', 'payload': self.cfg, - 'quoted': False, 'option_name': 'option', - 'option_value': - [{'opt_type': 'single', 'opt_val': 'value'}]}] + expected_result = [ + { + 'type': 'option', + 'payload': self.cfg, + 'quoted': False, + 'option_name': 'option', + 'option_value': [{'opt_type': 'single', 'opt_val': 'value'}], + } + ] result = self.parser._parse_cfg(self.cfg) self.assertEqual(result, expected_result) def test_parse_cfg_option_line_with_multiple_values(self): self.cfg = 'option=value1 value2' - expected_result = [{'type': 'option', 'payload': self.cfg, - 'quoted': False, 'option_name': 'option', - 'option_value': - [{'opt_type': 'single', 'opt_val': 'value1'}, - {'opt_type': 'single', 'opt_val': 'value2'}]}] + expected_result = [ + { + 'type': 'option', + 'payload': self.cfg, + 'quoted': False, + 'option_name': 'option', + 'option_value': [ + {'opt_type': 'single', 'opt_val': 'value1'}, + {'opt_type': 'single', 'opt_val': 'value2'}, + ], + } + ] result = self.parser._parse_cfg(self.cfg) self.assertEqual(result, expected_result) def test_parse_cfg_option_line_with_key_value(self): self.cfg = 'option=key=value' - expected_result = [{'type': 'option', 'payload': self.cfg, - 'quoted': False, 'option_name': 'option', - 'option_value': - [{'opt_type': 'key_val', 'opt_val': 'value', - 'opt_key': 'key'}]}] + expected_result = [ + { + 'type': 'option', + 'payload': self.cfg, + 'quoted': False, + 'option_name': 'option', + 'option_value': [ + {'opt_type': 'key_val', 'opt_val': 'value', 'opt_key': 'key'} + ], + } + ] result = self.parser._parse_cfg(self.cfg) self.assertEqual(result, expected_result) @@ -1353,107 +1534,170 @@ def test_validate_value(self, value, expected): self.parser._validate_value(value) def test_set_option_updates_existing_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": ["old_value"]}] - new_value = {"opt_type": "key_val", "opt_key": "key", "opt_val": - "new_value"} - expected_value = [{"option_name": "existing_option", - "option_value": [new_value]}] + self.parser._parsed = [ + {"option_name": "existing_option", "option_value": ["old_value"]} + ] + new_value = {"opt_type": "key_val", "opt_key": "key", "opt_val": "new_value"} + expected_value = [ + {"option_name": "existing_option", "option_value": [new_value]} + ] self.parser.set_option("existing_option", new_value) self.assertEqual(self.parser._parsed, expected_value) def test_set_option_adds_new_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": ["old_value"]}] - new_value = {"opt_type": "key_val", "opt_key": "key", - "opt_val": "new_value", "quoted": True, "type": "option"} - expected_value = [{"option_name": "existing_option", "option_value": - ["old_value"]}, {"option_name": "new_option", - "option_value": [new_value], - "quoted": True, "type": "option"}] + self.parser._parsed = [ + {"option_name": "existing_option", "option_value": ["old_value"]} + ] + new_value = { + "opt_type": "key_val", + "opt_key": "key", + "opt_val": "new_value", + "quoted": True, + "type": "option", + } + expected_value = [ + {"option_name": "existing_option", "option_value": ["old_value"]}, + { + "option_name": "new_option", + "option_value": [new_value], + "quoted": True, + "type": "option", + }, + ] self.parser.set_option("new_option", new_value) self.assertEqual(self.parser._parsed, expected_value) def test_append_to_option_updates_existing_key_val_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": [{"opt_type": "key_val", - "opt_key": "key", - "opt_val": "old_value"}]}] - new_value = {"opt_type": "key_val", "opt_key": "key", - "opt_val": "new_value"} - expected_value = [{"option_name": "existing_option", "option_value": - [new_value]}] + self.parser._parsed = [ + { + "option_name": "existing_option", + "option_value": [ + {"opt_type": "key_val", "opt_key": "key", "opt_val": "old_value"} + ], + } + ] + new_value = {"opt_type": "key_val", "opt_key": "key", "opt_val": "new_value"} + expected_value = [ + {"option_name": "existing_option", "option_value": [new_value]} + ] self.parser.append_to_option("existing_option", new_value) self.assertEqual(self.parser._parsed, expected_value) def test_append_to_option_ignores_existing_single_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": [{"opt_type": "single", - "opt_val": "old_value"}]}] + self.parser._parsed = [ + { + "option_name": "existing_option", + "option_value": [{"opt_type": "single", "opt_val": "old_value"}], + } + ] new_value = {"opt_type": "single", "opt_val": "old_value"} - expected_value = [{"option_name": "existing_option", - "option_value": [new_value]}] + expected_value = [ + {"option_name": "existing_option", "option_value": [new_value]} + ] self.parser.append_to_option("existing_option", new_value) self.assertEqual(self.parser._parsed, expected_value) def test_append_to_option_adds_new_single_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": [{"opt_type": "single", - "opt_val": "old_value"}]}] + self.parser._parsed = [ + { + "option_name": "existing_option", + "option_value": [{"opt_type": "single", "opt_val": "old_value"}], + } + ] new_value = {"opt_type": "single", "opt_val": "new_value"} - expected_value = [{ - "option_name": "existing_option", "option_value": - [{"opt_type": "single", "opt_val": "old_value"}, new_value]}] + expected_value = [ + { + "option_name": "existing_option", + "option_value": [ + {"opt_type": "single", "opt_val": "old_value"}, + new_value, + ], + } + ] self.parser.append_to_option("existing_option", new_value) self.assertEqual(self.parser._parsed, expected_value) def test_append_to_option_adds_new_option(self): - self.parser._parsed = [{"option_name": "existing_option", - "option_value": [{"opt_type": "single", - "opt_val": "old_value"}]}] - new_value = {"opt_type": "key_val", "opt_key": "key", "opt_val": - "new_value"} - expected_value = [{ - "option_name": "existing_option", "option_value": - [{"opt_type": "single", "opt_val": "old_value"}]}, - {"option_name": "new_option", "option_value": [new_value], - "quoted": True, "type": "option"}] + self.parser._parsed = [ + { + "option_name": "existing_option", + "option_value": [{"opt_type": "single", "opt_val": "old_value"}], + } + ] + new_value = {"opt_type": "key_val", "opt_key": "key", "opt_val": "new_value"} + expected_value = [ + { + "option_name": "existing_option", + "option_value": [{"opt_type": "single", "opt_val": "old_value"}], + }, + { + "option_name": "new_option", + "option_value": [new_value], + "quoted": True, + "type": "option", + }, + ] self.parser.append_to_option("new_option", new_value) self.assertEqual(self.parser._parsed, expected_value) @ddt.data( + ([{"type": "raw", "payload": "raw_data"}], "raw_data\n"), ( - [{"type": "raw", "payload": "raw_data"}], - "raw_data\n" + [ + { + "type": "option", + "option_name": "option1", + "option_value": [{"opt_type": "single", "opt_val": "value1"}], + "quoted": False, + } + ], + "option1=value1\n", ), ( - [{"type": "option", "option_name": "option1", "option_value": - [{"opt_type": "single", "opt_val": "value1"}], "quoted": False}], - "option1=value1\n" + [ + { + "type": "option", + "option_name": "option2", + "option_value": [ + {"opt_type": "key_val", "opt_key": "key2", "opt_val": "value2"} + ], + "quoted": True, + } + ], + "option2=\"key2=value2\"\n", ), ( - [{"type": "option", "option_name": "option2", "option_value": - [{"opt_type": "key_val", "opt_key": "key2", - "opt_val": "value2"}], - "quoted": True}], "option2=\"key2=value2\"\n" + [ + { + "type": "option", + "option_name": "option3", + "option_value": [], + "quoted": False, + } + ], + "option3=\n", ), ( - [{"type": "option", "option_name": "option3", "option_value": [], - "quoted": False}], "option3=\n" + [ + { + "type": "option", + "option_name": "option4", + "option_value": [ + {"opt_type": "single", "opt_val": "value4_1"}, + {"opt_type": "single", "opt_val": "value4_2"}, + ], + "quoted": False, + } + ], + "option4=\"value4_1 value4_2\"\n", ), - ( - [{"type": "option", "option_name": "option4", "option_value": - [{"opt_type": "single", "opt_val": "value4_1"}, - {"opt_type": "single", "opt_val": - "value4_2"}], "quoted": False}], - "option4=\"value4_1 value4_2\"\n" - )) + ) @ddt.unpack def test_dump(self, parsed, expected_output): self.parser._parsed = parsed diff --git a/coriolis/tests/test_wsman.py b/coriolis/tests/test_wsman.py index 7a9dacb7..3ad18b6a 100644 --- a/coriolis/tests/test_wsman.py +++ b/coriolis/tests/test_wsman.py @@ -6,9 +6,8 @@ import requests from winrm import protocol -from coriolis import exception +from coriolis import exception, wsman from coriolis.tests import test_base -from coriolis import wsman class WSManConnectionTestCase(test_base.CoriolisBaseTestCase): @@ -41,7 +40,7 @@ def test_connect(self, mock_protocol): username='username', password=None, cert_pem="test_cert", - cert_key_pem=None + cert_key_pem=None, ) @mock.patch.object(protocol, 'Protocol') @@ -53,33 +52,39 @@ def test_connect_no_auth(self, mock_protocol): username='username', password=None, cert_pem=None, - cert_key_pem=None + cert_key_pem=None, ) @mock.patch.object(wsman.WSManConnection, 'connect') @mock.patch('coriolis.utils.wait_for_port_connectivity') - def test_from_connection_info(self, mock_wait_for_port_connectivity, - mock_connect): + def test_from_connection_info(self, mock_wait_for_port_connectivity, mock_connect): connection_info = { "ip": "127.0.0.1", "username": "user", "password": "pass", } result = self.conn.from_connection_info(connection_info) - mock_wait_for_port_connectivity.assert_called_once_with( - "127.0.0.1", 5986) + mock_wait_for_port_connectivity.assert_called_once_with("127.0.0.1", 5986) mock_connect.assert_called_once_with( - url="https://127.0.0.1:5986/wsman", username="user", - password="pass", cert_pem=None, cert_key_pem=None) + url="https://127.0.0.1:5986/wsman", + username="user", + password="pass", + cert_pem=None, + cert_key_pem=None, + ) self.assertIsInstance(result, self.conn.__class__) def test_from_connection_info_missing_keys(self): - self.assertRaises(ValueError, self.conn.from_connection_info, - {"username": "user", "password": "pass"}) + self.assertRaises( + ValueError, + self.conn.from_connection_info, + {"username": "user", "password": "pass"}, + ) def test_from_connection_info_invalid_type(self): - self.assertRaises(ValueError, self.conn.from_connection_info, - 'invalid-connection-type') + self.assertRaises( + ValueError, self.conn.from_connection_info, 'invalid-connection-type' + ) def test_disconnect(self): self.conn.disconnect() @@ -87,69 +92,76 @@ def test_disconnect(self): def test_set_timeout(self): self.conn.set_timeout(self.conn._conn_timeout) - self.assertEqual(self.conn._protocol.transport.timeout, - self.conn._conn_timeout) - self.assertEqual(self.conn._protocol.timeout, - self.conn._conn_timeout) + self.assertEqual(self.conn._protocol.transport.timeout, self.conn._conn_timeout) + self.assertEqual(self.conn._protocol.timeout, self.conn._conn_timeout) def test__exec_command(self): - self.conn._protocol.get_command_output.return_value = ( - "std_out", "std_err", 0) - std_out, std_err, exit_code = self.conn._exec_command( - self.cmd, self.args) + self.conn._protocol.get_command_output.return_value = ("std_out", "std_err", 0) + std_out, std_err, exit_code = self.conn._exec_command(self.cmd, self.args) self.assertEqual(std_out, "std_out") self.assertEqual(std_err, "std_err") self.assertEqual(exit_code, 0) self.conn._protocol.open_shell.assert_called_once_with( - codepage=wsman.CODEPAGE_UTF8) + codepage=wsman.CODEPAGE_UTF8 + ) shell_id = self.conn._protocol.open_shell.return_value self.conn._protocol.run_command.assert_called_once_with( - shell_id, self.cmd, self.args) + shell_id, self.cmd, self.args + ) command_id = self.conn._protocol.run_command.return_value self.conn._protocol.get_command_output.assert_called_once_with( - shell_id, command_id) + shell_id, command_id + ) self.conn._protocol.cleanup_command.assert_called_once_with( - shell_id, command_id) + shell_id, command_id + ) self.conn._protocol.close_shell.assert_called_once_with(shell_id) def test__exec_command_exception(self): - self.conn._protocol.get_command_output.side_effect = requests.\ - exceptions.ReadTimeout - self.assertRaises(exception.OSMorphingWinRMOperationTimeout, - self.conn._exec_command, self.cmd, self.args) - self.conn._protocol.cleanup_command.assert_called_once_with( - mock.ANY, mock.ANY) + self.conn._protocol.get_command_output.side_effect = ( + requests.exceptions.ReadTimeout + ) + self.assertRaises( + exception.OSMorphingWinRMOperationTimeout, + self.conn._exec_command, + self.cmd, + self.args, + ) + self.conn._protocol.cleanup_command.assert_called_once_with(mock.ANY, mock.ANY) self.conn._protocol.close_shell.assert_called_once_with(mock.ANY) @mock.patch("time.sleep") def test__exec_command_invalid_credentials(self, mock_sleep): self.conn._protocol.open_shell.side_effect = ( - wsman.winrm_exceptions.InvalidCredentialsError) + wsman.winrm_exceptions.InvalidCredentialsError + ) - self.assertRaises(exception.NotAuthorized, self.conn._exec_command, - self.cmd, self.args) + self.assertRaises( + exception.NotAuthorized, self.conn._exec_command, self.cmd, self.args + ) self.conn._protocol.close_shell.assert_not_called() def test_exec_command(self): - self.conn._protocol.get_command_output.return_value = ( - "std_out", "std_err", 0) + self.conn._protocol.get_command_output.return_value = ("std_out", "std_err", 0) std_out = self.conn.exec_command(self.cmd, self.args) self.assertEqual(std_out, "std_out") def test_exec_command_exception(self): - self.conn._protocol.get_command_output.return_value = ( - "std_out", "std_err", 1) - self.assertRaises(exception.CoriolisException, self.conn.exec_command, - self.cmd, self.args) + self.conn._protocol.get_command_output.return_value = ("std_out", "std_err", 1) + self.assertRaises( + exception.CoriolisException, self.conn.exec_command, self.cmd, self.args + ) def test_exec_ps_command(self): self.conn.exec_command = mock.Mock() self.conn.exec_command.return_value = "std_out\n\n" result = self.conn.exec_ps_command(self.cmd) self.conn.exec_command.assert_called_once_with( - "powershell.exe", ["-EncodedCommand", 'dABlAHMAdABfAGMAbQBkAA=='], - timeout=None) + "powershell.exe", + ["-EncodedCommand", 'dABlAHMAdABfAGMAbQBkAA=='], + timeout=None, + ) self.assertEqual(result, "std_out") def test_test_path(self): @@ -157,7 +169,8 @@ def test_test_path(self): self.conn.exec_ps_command.return_value = "True" result = self.conn.test_path("test_path") self.conn.exec_ps_command.assert_called_once_with( - "Test-Path -Path \"test_path\"") + "Test-Path -Path \"test_path\"" + ) self.assertTrue(result) def test_download_file(self): @@ -168,8 +181,12 @@ def test_download_file(self): def test_download_file_exception(self): self.conn.exec_ps_command = mock.Mock() self.conn.exec_ps_command.side_effect = exception.CoriolisException - self.assertRaises(exception.CoriolisException, self.conn.download_file, - self.url, self.remote_path) + self.assertRaises( + exception.CoriolisException, + self.conn.download_file, + self.url, + self.remote_path, + ) self.conn.exec_ps_command.assert_called_once() def test_write_file(self): @@ -177,4 +194,6 @@ def test_write_file(self): self.conn.write_file(self.remote_path, b'file content') self.conn.exec_ps_command.assert_called_once_with( "[IO.File]::WriteAllBytes('%s', [Convert]::FromBase64String('%s'))" - % (self.remote_path, 'ZmlsZSBjb250ZW50'), ignore_stdout=True) + % (self.remote_path, 'ZmlsZSBjb250ZW50'), + ignore_stdout=True, + ) diff --git a/coriolis/tests/testutils.py b/coriolis/tests/testutils.py index c822e3a0..79116e90 100644 --- a/coriolis/tests/testutils.py +++ b/coriolis/tests/testutils.py @@ -54,9 +54,15 @@ def __init__(self, dictionary, skip_attrs=None): setattr(self, key, DictToObject(value, skip_attrs=skip_attrs)) elif isinstance(value, list): setattr( - self, key, - [DictToObject(item, skip_attrs=skip_attrs) if isinstance( - item, dict) else item for item in value]) + self, + key, + [ + DictToObject(item, skip_attrs=skip_attrs) + if isinstance(item, dict) + else item + for item in value + ], + ) else: setattr(self, key, value) diff --git a/coriolis/tests/transfer_cron/rpc/test_client.py b/coriolis/tests/transfer_cron/rpc/test_client.py index a6e9c084..f8038233 100644 --- a/coriolis/tests/transfer_cron/rpc/test_client.py +++ b/coriolis/tests/transfer_cron/rpc/test_client.py @@ -20,19 +20,20 @@ def test_register(self): self.client.register(self.ctxt, mock.sentinel.schedule) self.client._call.assert_called_once_with( - self.ctxt, 'register', schedule=mock.sentinel.schedule) + self.ctxt, 'register', schedule=mock.sentinel.schedule + ) def test_unregister(self): self.client._call = mock.Mock() self.client.unregister(self.ctxt, mock.sentinel.schedule) self.client._call.assert_called_once_with( - self.ctxt, 'unregister', schedule=mock.sentinel.schedule) + self.ctxt, 'unregister', schedule=mock.sentinel.schedule + ) def test_get_diagnostics(self): self.client._call = mock.Mock() result = self.client.get_diagnostics(self.ctxt) - self.client._call.assert_called_once_with( - self.ctxt, 'get_diagnostics') + self.client._call.assert_called_once_with(self.ctxt, 'get_diagnostics') self.assertEqual(result, self.client._call.return_value) diff --git a/coriolis/tests/transfer_cron/rpc/test_server.py b/coriolis/tests/transfer_cron/rpc/test_server.py index 0c00050a..e24868b9 100644 --- a/coriolis/tests/transfer_cron/rpc/test_server.py +++ b/coriolis/tests/transfer_cron/rpc/test_server.py @@ -1,14 +1,14 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. +import datetime import logging from unittest import mock -import datetime import ddt -from coriolis.conductor.rpc import client as rpc_client from coriolis import exception +from coriolis.conductor.rpc import client as rpc_client from coriolis.tests import test_base from coriolis.transfer_cron.rpc import server @@ -21,34 +21,45 @@ def test__trigger_transfer(self): mock_conductor_client.execute_transfer_tasks.return_value = { 'id': mock.sentinel.id, - 'action_id': mock.sentinel.action_id + 'action_id': mock.sentinel.action_id, } result = server._trigger_transfer( mock.sentinel.ctxt, mock_conductor_client, - mock.sentinel.transfer_id, False, False) + mock.sentinel.transfer_id, + False, + False, + ) mock_conductor_client.execute_transfer_tasks.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.transfer_id, - shutdown_instances=False, auto_deploy=False) + mock.sentinel.ctxt, + mock.sentinel.transfer_id, + shutdown_instances=False, + auto_deploy=False, + ) self.assertEqual( - result, 'Execution %s for Transfer %s' % ( - mock.sentinel.id, mock.sentinel.action_id)) + result, + 'Execution %s for Transfer %s' + % (mock.sentinel.id, mock.sentinel.action_id), + ) def test__trigger_transfer_invalid_transfer_state(self): mock_conductor_client = mock.MagicMock() mock_conductor_client.execute_transfer_tasks.side_effect = ( - exception.InvalidTransferState(reason='test_reason')) + exception.InvalidTransferState(reason='test_reason') + ) - with self.assertLogs('coriolis.transfer_cron.rpc.server', - level=logging.INFO): + with self.assertLogs('coriolis.transfer_cron.rpc.server', level=logging.INFO): server._trigger_transfer( mock.sentinel.ctxt, mock_conductor_client, - mock.sentinel.action_id, False, False) + mock.sentinel.action_id, + False, + False, + ) @ddt.ddt @@ -64,36 +75,40 @@ def setUp(self, _): { 'input': { 'expiration_date': '2024-03-28T22:00:00.000000+02:00', - 'schedule': '{"hour": 22, "minute": 0, "month": 7}' + 'schedule': '{"hour": 22, "minute": 0, "month": 7}', }, 'expected': { 'expiration_date': datetime.datetime(2024, 3, 28, 20, 0), - 'schedule': {"hour": 22, "minute": 0, "month": 7} - } + 'schedule': {"hour": 22, "minute": 0, "month": 7}, + }, }, ) def test__deserialize_schedule(self, data): result = self.server._deserialize_schedule(data['input']) self.assertEqual(result, data['expected']) - @mock.patch.object(server.TransferCronServerEndpoint, - '_deserialize_schedule') + @mock.patch.object(server.TransferCronServerEndpoint, '_deserialize_schedule') @mock.patch.object(server, '_trigger_transfer') @mock.patch.object(server.timeutils, 'utcnow') @mock.patch.object(server.context, 'get_admin_context') @mock.patch.object(server.cron, 'CronJob') @mock.patch.object(server.cron.Cron, 'register') - def test__register_schedule(self, mock_register, mock_cron_job, - mock_get_admin_context, mock_utcnow, - mock_trigger_transfer, - mock_deserialize_schedule): + def test__register_schedule( + self, + mock_register, + mock_cron_job, + mock_get_admin_context, + mock_utcnow, + mock_trigger_transfer, + mock_deserialize_schedule, + ): mock_get_admin_context.return_value = 'test_admin_context' mock_utcnow.return_value = datetime.datetime(2022, 1, 1) mock_deserialize_schedule.return_value = { 'id': 'test_id', 'enabled': True, 'expiration_date': datetime.datetime(2022, 12, 31), - 'schedule': 'test_schedule' + 'schedule': 'test_schedule', } test_schedule = { 'trust_id': 'test_schedule_trust_id', @@ -106,35 +121,42 @@ def test__register_schedule(self, mock_register, mock_cron_job, mock_deserialize_schedule.assert_called_once_with(test_schedule) mock_get_admin_context.assert_called_once_with( - trust_id='test_schedule_trust_id') + trust_id='test_schedule_trust_id' + ) mock_cron_job.assert_called_once_with( - 'test_id', 'Scheduled job for test_id', 'test_schedule', - True, datetime.datetime(2022, 12, 31), None, None, - mock_trigger_transfer, 'test_admin_context', - self.server._rpc_client, 'test_schedule_transfer_id', - False, False) + 'test_id', + 'Scheduled job for test_id', + 'test_schedule', + True, + datetime.datetime(2022, 12, 31), + None, + None, + mock_trigger_transfer, + 'test_admin_context', + self.server._rpc_client, + 'test_schedule_transfer_id', + False, + False, + ) mock_register.assert_called_once() - @mock.patch.object(server.TransferCronServerEndpoint, - '_deserialize_schedule') + @mock.patch.object(server.TransferCronServerEndpoint, '_deserialize_schedule') @mock.patch.object(server.timeutils, 'utcnow') - def test__register_schedule_expired(self, mock_utcnow, - mock_deserialize_schedule): + def test__register_schedule_expired(self, mock_utcnow, mock_deserialize_schedule): mock_utcnow.return_value = datetime.datetime(2022, 12, 31) mock_deserialize_schedule.return_value = { 'id': 'test_id', 'enabled': True, 'expiration_date': datetime.datetime(2022, 1, 1), - 'schedule': 'test_schedule' + 'schedule': 'test_schedule', } test_schedule = { 'trust_id': 'test_schedule_trust_id', 'transfer_id': 'test_schedule_transfer_id', - 'shutdown_instance': 'test_schedule_shutdown_instance' + 'shutdown_instance': 'test_schedule_shutdown_instance', } - with self.assertLogs('coriolis.transfer_cron.rpc.server', - level=logging.INFO): + with self.assertLogs('coriolis.transfer_cron.rpc.server', level=logging.INFO): self.server._register_schedule(test_schedule) mock_deserialize_schedule.assert_called_once_with(test_schedule) @@ -143,8 +165,13 @@ def test__register_schedule_expired(self, mock_utcnow, @mock.patch.object(server.TransferCronServerEndpoint, '_get_all_schedules') @mock.patch.object(server.TransferCronServerEndpoint, '_register_schedule') @mock.patch.object(server.cron.Cron, 'start') - def test__init_cron(self, mock_cron_start, mock_register_schedule, - mock_get_all_schedules, mock_utcnow): + def test__init_cron( + self, + mock_cron_start, + mock_register_schedule, + mock_get_all_schedules, + mock_utcnow, + ): mock_utcnow.return_value = datetime.datetime(2022, 1, 1) mock_get_all_schedules.return_value = [ {'id': 'schedule1'}, @@ -156,38 +183,43 @@ def test__init_cron(self, mock_cron_start, mock_register_schedule, mock_utcnow.assert_called_once() mock_get_all_schedules.assert_called_once() - mock_register_schedule.assert_has_calls([ - mock.call({'id': 'schedule1'}, date=mock_utcnow.return_value), - mock.call({'id': 'schedule2'}, date=mock_utcnow.return_value), - ]) + mock_register_schedule.assert_has_calls( + [ + mock.call({'id': 'schedule1'}, date=mock_utcnow.return_value), + mock.call({'id': 'schedule2'}, date=mock_utcnow.return_value), + ] + ) mock_cron_start.assert_called_once() @mock.patch.object(server.TransferCronServerEndpoint, '_get_all_schedules') @mock.patch.object(server.TransferCronServerEndpoint, '_register_schedule') - def test__init_cron_with_exception(self, mock_register_schedule, - mock_get_all_schedules): + def test__init_cron_with_exception( + self, mock_register_schedule, mock_get_all_schedules + ): mock_get_all_schedules.return_value = [ {'id': 'schedule1'}, {'id': 'schedule2'}, ] mock_register_schedule.side_effect = Exception('test_exception') - with self.assertLogs('coriolis.transfer_cron.rpc.server', - level=logging.ERROR): - self.server._init_cron() + with self.assertLogs('coriolis.transfer_cron.rpc.server', level=logging.ERROR): + self.server._init_cron() mock_get_all_schedules.assert_called_once() - mock_register_schedule.assert_has_calls([ - mock.call({'id': 'schedule1'}, date=mock.ANY), - mock.call({'id': 'schedule2'}, date=mock.ANY), - ]) + mock_register_schedule.assert_has_calls( + [ + mock.call({'id': 'schedule1'}, date=mock.ANY), + mock.call({'id': 'schedule2'}, date=mock.ANY), + ] + ) @mock.patch.object(rpc_client.ConductorClient, 'get_transfer_schedules') def test__get_all_schedules(self, mock_get_transfer_schedules): result = self.server._get_all_schedules() mock_get_transfer_schedules.assert_called_once_with( - self.server._admin_ctx, expired=False) + self.server._admin_ctx, expired=False + ) self.assertEqual(result, mock_get_transfer_schedules.return_value) @@ -201,7 +233,8 @@ def test_register(self, mock_utcnow, mock_register_schedule): mock_utcnow.assert_called_once() mock_register_schedule.assert_called_once_with( - schedule, date=mock_utcnow.return_value) + schedule, date=mock_utcnow.return_value + ) @mock.patch.object(server.cron.Cron, 'unregister') def test_unregister(self, mock_unregister): diff --git a/coriolis/tests/transfer_cron/test_api.py b/coriolis/tests/transfer_cron/test_api.py index 8334a6fc..f59fd5e6 100644 --- a/coriolis/tests/transfer_cron/test_api.py +++ b/coriolis/tests/transfer_cron/test_api.py @@ -27,44 +27,56 @@ def test_create(self): auto_deploy = mock.sentinel.auto_deploy result = self.api.create( - self.ctxt, self.transfer_id, schedule, enabled, exp_date, - shutdown_instance, auto_deploy) + self.ctxt, + self.transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ) self.rpc_client.create_transfer_schedule.assert_called_once_with( - self.ctxt, self.transfer_id, schedule, enabled, exp_date, - shutdown_instance, auto_deploy) - self.assertEqual(result, - self.rpc_client.create_transfer_schedule.return_value) + self.ctxt, + self.transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ) + self.assertEqual(result, self.rpc_client.create_transfer_schedule.return_value) def test_get_schedules(self): result = self.api.get_schedules(self.ctxt, self.transfer_id) self.rpc_client.get_transfer_schedules.assert_called_once_with( - self.ctxt, self.transfer_id, expired=True) - self.assertEqual(result, - self.rpc_client.get_transfer_schedules.return_value) + self.ctxt, self.transfer_id, expired=True + ) + self.assertEqual(result, self.rpc_client.get_transfer_schedules.return_value) def test_get_schedule(self): - result = self.api.get_schedule(self.ctxt, self.transfer_id, - self.schedule_id) + result = self.api.get_schedule(self.ctxt, self.transfer_id, self.schedule_id) self.rpc_client.get_transfer_schedule.assert_called_once_with( - self.ctxt, self.transfer_id, self.schedule_id, expired=True) - self.assertEqual(result, - self.rpc_client.get_transfer_schedule.return_value) + self.ctxt, self.transfer_id, self.schedule_id, expired=True + ) + self.assertEqual(result, self.rpc_client.get_transfer_schedule.return_value) def test_update(self): update_values = mock.sentinel.update_values - result = self.api.update(self.ctxt, self.transfer_id, self.schedule_id, - update_values) + result = self.api.update( + self.ctxt, self.transfer_id, self.schedule_id, update_values + ) self.rpc_client.update_transfer_schedule.assert_called_once_with( - self.ctxt, self.transfer_id, self.schedule_id, update_values) - self.assertEqual(result, - self.rpc_client.update_transfer_schedule.return_value) + self.ctxt, self.transfer_id, self.schedule_id, update_values + ) + self.assertEqual(result, self.rpc_client.update_transfer_schedule.return_value) def test_delete(self): self.api.delete(self.ctxt, self.transfer_id, self.schedule_id) self.rpc_client.delete_transfer_schedule.assert_called_once_with( - self.ctxt, self.transfer_id, self.schedule_id) + self.ctxt, self.transfer_id, self.schedule_id + ) diff --git a/coriolis/tests/transfer_tasks_executions/test_api.py b/coriolis/tests/transfer_tasks_executions/test_api.py index 36192446..e791d16f 100644 --- a/coriolis/tests/transfer_tasks_executions/test_api.py +++ b/coriolis/tests/transfer_tasks_executions/test_api.py @@ -23,33 +23,39 @@ def test_create(self): shutdown_instances = mock.sentinel.shutdown_instances auto_deploy = mock.sentinel.auto_deploy - result = self.api.create(self.ctxt, self.transfer_id, - shutdown_instances, auto_deploy) + result = self.api.create( + self.ctxt, self.transfer_id, shutdown_instances, auto_deploy + ) self.rpc_client.execute_transfer_tasks.assert_called_once_with( - self.ctxt, self.transfer_id, shutdown_instances, auto_deploy) - self.assertEqual(result, - self.rpc_client.execute_transfer_tasks.return_value) + self.ctxt, self.transfer_id, shutdown_instances, auto_deploy + ) + self.assertEqual(result, self.rpc_client.execute_transfer_tasks.return_value) def test_delete(self): self.api.delete(self.ctxt, self.transfer_id, self.execution_id) - (self.rpc_client.delete_transfer_tasks_execution - .assert_called_once_with( - self.ctxt, self.transfer_id, self.execution_id)) + ( + self.rpc_client.delete_transfer_tasks_execution.assert_called_once_with( + self.ctxt, self.transfer_id, self.execution_id + ) + ) def test_cancel(self): force = mock.sentinel.force self.api.cancel(self.ctxt, self.transfer_id, self.execution_id, force) - (self.rpc_client.cancel_transfer_tasks_execution - .assert_called_once_with( - self.ctxt, self.transfer_id, self.execution_id, force)) + ( + self.rpc_client.cancel_transfer_tasks_execution.assert_called_once_with( + self.ctxt, self.transfer_id, self.execution_id, force + ) + ) def test_get_executions(self): result = self.api.get_executions( - self.ctxt, self.transfer_id, + self.ctxt, + self.transfer_id, mock.sentinel.include_tasks, mock.sentinel.marker, mock.sentinel.limit, @@ -58,7 +64,8 @@ def test_get_executions(self): ) self.rpc_client.get_transfer_tasks_executions.assert_called_once_with( - self.ctxt, self.transfer_id, + self.ctxt, + self.transfer_id, mock.sentinel.include_tasks, mock.sentinel.marker, mock.sentinel.limit, @@ -66,13 +73,15 @@ def test_get_executions(self): mock.sentinel.sort_dirs, ) self.assertEqual( - result, self.rpc_client.get_transfer_tasks_executions.return_value) + result, self.rpc_client.get_transfer_tasks_executions.return_value + ) def test_get_execution(self): - result = self.api.get_execution(self.ctxt, self.transfer_id, - self.execution_id) + result = self.api.get_execution(self.ctxt, self.transfer_id, self.execution_id) self.rpc_client.get_transfer_tasks_execution.assert_called_once_with( - self.ctxt, self.transfer_id, self.execution_id) + self.ctxt, self.transfer_id, self.execution_id + ) self.assertEqual( - result, self.rpc_client.get_transfer_tasks_execution.return_value) + result, self.rpc_client.get_transfer_tasks_execution.return_value + ) diff --git a/coriolis/tests/transfers/test_api.py b/coriolis/tests/transfers/test_api.py index 9992e9eb..27617fe6 100644 --- a/coriolis/tests/transfers/test_api.py +++ b/coriolis/tests/transfers/test_api.py @@ -18,8 +18,8 @@ def setUp(self): self.ctxt = mock.sentinel.ctxt self.transfer_id = mock.sentinel.transfer_id self._mock_pagination_args = dict( - marker="mock_marker", limit=5, - sort_keys=["mock_column"], sort_dirs=["desc"]) + marker="mock_marker", limit=5, sort_keys=["mock_column"], sort_dirs=["desc"] + ) def test_create(self): origin_endpoint_id = mock.sentinel.origin_endpoint_id @@ -27,7 +27,8 @@ def test_create(self): origin_minion_pool_id = mock.sentinel.origin_minion_pool_id destination_minion_pool_id = mock.sentinel.destination_minion_pool_id instance_osmorphing_minion_pool_mappings = ( - mock.sentinel.instance_osmorphing_minion_pool_mappings) + mock.sentinel.instance_osmorphing_minion_pool_mappings + ) source_environment = mock.sentinel.source_environment destination_environment = mock.sentinel.destination_environment instances = mock.sentinel.instances @@ -35,46 +36,68 @@ def test_create(self): storage_mappings = mock.sentinel.storage_mappings result = self.api.create( - self.ctxt, mock.sentinel.transfer_scenario, - origin_endpoint_id, destination_endpoint_id, - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, source_environment, - destination_environment, instances, network_map, storage_mappings) + self.ctxt, + mock.sentinel.transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + ) self.rpc_client.create_instances_transfer.assert_called_once_with( - self.ctxt, mock.sentinel.transfer_scenario, - origin_endpoint_id, destination_endpoint_id, - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, source_environment, - destination_environment, instances, network_map, storage_mappings, - None, None, True, False) - self.assertEqual( - result, self.rpc_client.create_instances_transfer.return_value) + self.ctxt, + mock.sentinel.transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + None, + None, + True, + False, + ) + self.assertEqual(result, self.rpc_client.create_instances_transfer.return_value) def test_update(self): updated_properties = mock.sentinel.updated_properties - result = self.api.update(self.ctxt, self.transfer_id, - updated_properties) + result = self.api.update(self.ctxt, self.transfer_id, updated_properties) self.rpc_client.update_transfer.assert_called_once_with( - self.ctxt, self.transfer_id, updated_properties) - self.assertEqual(result, - self.rpc_client.update_transfer.return_value) + self.ctxt, self.transfer_id, updated_properties + ) + self.assertEqual(result, self.rpc_client.update_transfer.return_value) def test_delete(self): self.api.delete(self.ctxt, self.transfer_id) self.rpc_client.delete_transfer.assert_called_once_with( - self.ctxt, self.transfer_id) + self.ctxt, self.transfer_id + ) def test_get_transfers(self): result = self.api.get_transfers( - self.ctxt, include_tasks_executions=False, include_task_info=False, + self.ctxt, + include_tasks_executions=False, + include_task_info=False, **self._mock_pagination_args, ) self.rpc_client.get_transfers.assert_called_once_with( - self.ctxt, False, include_task_info=False, + self.ctxt, + False, + include_task_info=False, **self._mock_pagination_args, ) self.assertEqual(result, self.rpc_client.get_transfers.return_value) @@ -83,13 +106,14 @@ def test_get_transfer(self): result = self.api.get_transfer(self.ctxt, self.transfer_id) self.rpc_client.get_transfer.assert_called_once_with( - self.ctxt, self.transfer_id, include_task_info=False) + self.ctxt, self.transfer_id, include_task_info=False + ) self.assertEqual(result, self.rpc_client.get_transfer.return_value) def test_delete_disks(self): result = self.api.delete_disks(self.ctxt, self.transfer_id) self.rpc_client.delete_transfer_disks.assert_called_once_with( - self.ctxt, self.transfer_id) - self.assertEqual(result, - self.rpc_client.delete_transfer_disks.return_value) + self.ctxt, self.transfer_id + ) + self.assertEqual(result, self.rpc_client.delete_transfer_disks.return_value) diff --git a/coriolis/tests/worker/rpc/test_client.py b/coriolis/tests/worker/rpc/test_client.py index 3b8e53b7..864e22f5 100644 --- a/coriolis/tests/worker/rpc/test_client.py +++ b/coriolis/tests/worker/rpc/test_client.py @@ -38,7 +38,6 @@ class WorkerClientTestCase(test_base.CoriolisRPCClientTestCase): - def setUp(self): super(WorkerClientTestCase, self).setUp() self.client = client.WorkerClient() @@ -50,11 +49,10 @@ def test__init__default_args(self, mock_target, mock_conf): mock_conf.worker.worker_rpc_timeout = expected_timeout result = client.WorkerClient() mock_target.assert_called_once_with( - topic=constants.WORKER_MAIN_MESSAGING_TOPIC, - version=client.VERSION) + topic=constants.WORKER_MAIN_MESSAGING_TOPIC, version=client.VERSION + ) self.assertEqual(result._target, mock_target.return_value) - self.assertEqual( - result._timeout, expected_timeout) + self.assertEqual(result._timeout, expected_timeout) @mock.patch.object(oslo_messaging, 'Target') def test__init__custom_args(self, mock_target): @@ -64,34 +62,47 @@ def test__init__custom_args(self, mock_target): expected_topic = "topic.host" result = client.WorkerClient( - timeout=expected_timeout, host=host, base_worker_topic=topic) + timeout=expected_timeout, host=host, base_worker_topic=topic + ) mock_target.assert_called_once_with( - topic=expected_topic, version=client.VERSION) + topic=expected_topic, version=client.VERSION + ) self.assertEqual(result._target, mock_target.return_value) self.assertEqual(result._timeout, expected_timeout) def test_from_service_definition_invalid_topic(self): self.assertRaises( - ValueError, self.client.from_service_definition, - {'topic': 'invalid_service'}) + ValueError, + self.client.from_service_definition, + {'topic': 'invalid_service'}, + ) def test_from_service_definition(self): rpc_client = self.client.from_service_definition( - {'topic': constants.WORKER_MAIN_MESSAGING_TOPIC}) + {'topic': constants.WORKER_MAIN_MESSAGING_TOPIC} + ) self.assertIsInstance(rpc_client, client.WorkerClient) def test_begin_task(self): args = EXEC_TASK_ARGS custom_args = {"report_to_conductor": True} self._test( - self.client.begin_task, args, rpc_op='_cast', - server_fun_name='exec_task', custom_args=custom_args) + self.client.begin_task, + args, + rpc_op='_cast', + server_fun_name='exec_task', + custom_args=custom_args, + ) def test_run_task(self): args = EXEC_TASK_ARGS custom_args = {"report_to_conductor": False} - self._test(self.client.run_task, args, server_fun_name='exec_task', - custom_args=custom_args) + self._test( + self.client.run_task, + args, + server_fun_name='exec_task', + custom_args=custom_args, + ) def test_cancel_task(self): args = { @@ -107,7 +118,7 @@ def test_get_endpoint_instances(self): 'marker': None, 'limit': None, 'instance_name_pattern': None, - 'refresh': False + 'refresh': False, } self._test(self.client.get_endpoint_instances, args) @@ -119,8 +130,7 @@ def test_get_endpoint_instance(self): self._test(self.client.get_endpoint_instance, args) def test_get_endpoint_destination_options(self): - self._test( - self.client.get_endpoint_destination_options, ENDPOINT_OPT_ARGS) + self._test(self.client.get_endpoint_destination_options, ENDPOINT_OPT_ARGS) def test_get_endpoint_source_options(self): self._test(self.client.get_endpoint_source_options, ENDPOINT_OPT_ARGS) @@ -169,18 +179,23 @@ def test_get_service_status(self): self._test(self.client.get_service_status, {}) def test_get_endpoint_source_minion_pool_options(self): - self._test(self.client.get_endpoint_source_minion_pool_options, - ENDPOINT_OPT_ARGS) + self._test( + self.client.get_endpoint_source_minion_pool_options, ENDPOINT_OPT_ARGS + ) def test_get_endpoint_minion_pool_options(self): - self._test(self.client.get_endpoint_destination_minion_pool_options, - ENDPOINT_OPT_ARGS) + self._test( + self.client.get_endpoint_destination_minion_pool_options, ENDPOINT_OPT_ARGS + ) def test_validate_endpoint_source_minion_pool_options(self): - self._test(self.client.validate_endpoint_source_minion_pool_options, - POOL_VALIDATION_ARGS) + self._test( + self.client.validate_endpoint_source_minion_pool_options, + POOL_VALIDATION_ARGS, + ) def test_validate_endpoint_destination_minion_pool_options(self): self._test( self.client.validate_endpoint_destination_minion_pool_options, - POOL_VALIDATION_ARGS) + POOL_VALIDATION_ARGS, + ) diff --git a/coriolis/tests/worker/rpc/test_server.py b/coriolis/tests/worker/rpc/test_server.py index 5c13ceeb..355cc10d 100644 --- a/coriolis/tests/worker/rpc/test_server.py +++ b/coriolis/tests/worker/rpc/test_server.py @@ -8,21 +8,17 @@ from unittest import mock import ddt -from oslo_log import log as logging import psutil +from oslo_log import log as logging from six.moves import queue +from coriolis import constants, context, exception, schemas, utils from coriolis.conductor.rpc import client as conductor_client from coriolis.conductor.rpc import utils as cond_rpc_utils -from coriolis import constants -from coriolis import context -from coriolis import exception from coriolis.minion_manager.rpc import client as minion_client from coriolis.providers import factory as providers_factory -from coriolis import schemas from coriolis.tasks import factory as task_runners_factory from coriolis.tests import test_base -from coriolis import utils from coriolis.worker.rpc import server @@ -30,32 +26,33 @@ class WorkerServerEndpointTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Worker RPC server.""" - @mock.patch.object(server.WorkerServerEndpoint, - "_register_worker_service") + @mock.patch.object(server.WorkerServerEndpoint, "_register_worker_service") def setUp(self, _): # pylint: disable=arguments-differ super(WorkerServerEndpointTestCase, self).setUp() self.server = server.WorkerServerEndpoint() @mock.patch.object(minion_client, 'MinionManagerPoolRpcEventHandler') - def test__get_event_handler_for_task_type_minion( - self, mock_minion_event_handler): + def test__get_event_handler_for_task_type_minion(self, mock_minion_event_handler): result = server._get_event_handler_for_task_type( constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS, mock.sentinel.ctxt, - mock.sentinel.task_object_id) + mock.sentinel.task_object_id, + ) mock_minion_event_handler.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.task_object_id) + mock.sentinel.ctxt, mock.sentinel.task_object_id + ) self.assertEqual(result, mock_minion_event_handler.return_value) @mock.patch.object(conductor_client, 'ConductorTaskRpcEventHandler') - def test__get_event_handler_for_task_type( - self, mock_conductor_event_handler): + def test__get_event_handler_for_task_type(self, mock_conductor_event_handler): result = server._get_event_handler_for_task_type( constants.TASK_TYPE_REPLICATE_DISKS, mock.sentinel.ctxt, - mock.sentinel.task_object_id) + mock.sentinel.task_object_id, + ) mock_conductor_event_handler.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.task_object_id) + mock.sentinel.ctxt, mock.sentinel.task_object_id + ) self.assertEqual(result, mock_conductor_event_handler.return_value) @mock.patch.object(conductor_client, 'ConductorClient') @@ -76,22 +73,30 @@ def test__rpc_conductor_client_instantiated(self, mock_cond_client): @mock.patch.object(utils, 'get_binary_name') @mock.patch.object(utils, 'get_hostname') def test__register_worker_service( - self, mock_hostname, mock_binary, mock_context, - mock_get_service_status, mock_check_create_service, - mock_cond_client): + self, + mock_hostname, + mock_binary, + mock_context, + mock_get_service_status, + mock_check_create_service, + mock_cond_client, + ): result = self.server._register_worker_service() mock_hostname.assert_called_once() mock_binary.assert_called_once() mock_context.assert_called_once_with('coriolis', 'admin') - mock_get_service_status.assert_called_once_with( - mock_context.return_value) + mock_get_service_status.assert_called_once_with(mock_context.return_value) mock_check_create_service.assert_called_once_with( - mock_cond_client.return_value, mock_context.return_value, - mock_hostname.return_value, mock_binary.return_value, - constants.WORKER_MAIN_MESSAGING_TOPIC, enabled=True, + mock_cond_client.return_value, + mock_context.return_value, + mock_hostname.return_value, + mock_binary.return_value, + constants.WORKER_MAIN_MESSAGING_TOPIC, + enabled=True, providers=mock_get_service_status.return_value['providers'], - specs=mock_get_service_status.return_value['specs']) + specs=mock_get_service_status.return_value['specs'], + ) self.assertEqual(result, mock_check_create_service.return_value) self.assertEqual(result, self.server._service_registration) @@ -105,15 +110,15 @@ def test__check_remove_dir(self): def test__check_remove_dir_fails(self, mock_rmtree): tmp = tempfile.mkdtemp() mock_rmtree.side_effect = Exception('YOLO') - with self.assertLogs('coriolis.worker.rpc.server', - level=logging.ERROR): + with self.assertLogs('coriolis.worker.rpc.server', level=logging.ERROR): self.server._check_remove_dir(tmp) os.rmdir(tmp) @mock.patch.object(server.WorkerServerEndpoint, 'get_available_providers') @mock.patch.object(server.WorkerServerEndpoint, 'get_diagnostics') - def test_get_service_status(self, mock_get_diagnostics, - mock_get_available_providers): + def test_get_service_status( + self, mock_get_diagnostics, mock_get_available_providers + ): expected_result = { "host": mock_get_diagnostics.return_value['hostname'], "binary": mock_get_diagnostics.return_value['application'], @@ -122,15 +127,14 @@ def test_get_service_status(self, mock_get_diagnostics, "specs": mock_get_diagnostics.return_value, } result = self.server.get_service_status(mock.sentinel.ctxt) - mock_get_available_providers.assert_called_once_with( - mock.sentinel.ctxt) + mock_get_available_providers.assert_called_once_with(mock.sentinel.ctxt) mock_get_diagnostics.assert_called_once() self.assertEqual(result, expected_result) - @mock.patch.object(server.WorkerServerEndpoint, - "_wait_for_process") - @mock.patch.object(server.WorkerServerEndpoint, - "_start_process_with_custom_library_paths") + @mock.patch.object(server.WorkerServerEndpoint, "_wait_for_process") + @mock.patch.object( + server.WorkerServerEndpoint, "_start_process_with_custom_library_paths" + ) @mock.patch.object(server, "_task_process") @mock.patch('coriolis.utils.start_thread') @mock.patch.object(server.WorkerServerEndpoint, "_rpc_conductor_client") @@ -235,9 +239,7 @@ def call_exec_task_process(report_to_conductor=True): # if return value is string, raise TaskProcessException mock_wait_process.return_value = "Test string" - self.assertRaises( - exception.TaskProcessException, call_exec_task_process - ) + self.assertRaises(exception.TaskProcessException, call_exec_task_process) @mock.patch.object(psutil, "Process") def test_cancel_task(self, mock_process): @@ -249,9 +251,7 @@ def test_cancel_task(self, mock_process): ) # Cancel task should be called with send_signal when not forced - mock_process.return_value.send_signal.assert_called_once_with( - signal.SIGINT - ) + mock_process.return_value.send_signal.assert_called_once_with(signal.SIGINT) mock_process.return_value.kill.assert_not_called() # Cancel task should be called with kill when forced @@ -281,9 +281,7 @@ def test_cancel_task(self, mock_process): # If process is not found it should just confirm task is cancelled mock_process.reset_mock() - mock_process.side_effect = psutil.NoSuchProcess( - mock.sentinel.proccess_id - ) + mock_process.side_effect = psutil.NoSuchProcess(mock.sentinel.proccess_id) with mock.patch.object( server.WorkerServerEndpoint, "_rpc_conductor_client" ) as mock_client: @@ -299,14 +297,14 @@ def test_cancel_task(self, mock_process): def test__handle_mp_log_events(self, mock_get_logger): mock_mp_log_q = mock.MagicMock() mock_p = mock.MagicMock() - mock_mp_log_q.get.side_effect = [ - mock.sentinel.record, queue.Empty, None] + mock_mp_log_q.get.side_effect = [mock.sentinel.record, queue.Empty, None] mock_p.is_alive.return_value = True result = self.server._handle_mp_log_events(mock_p, mock_mp_log_q) mock_get_logger.assert_called_once_with(mock.sentinel.record.name) mock_get_logger.return_value.logger.handle.assert_called_once_with( - mock.sentinel.record) + mock.sentinel.record + ) self.assertIsNone(result) def test__handle_mp_log_events_dead_process(self): @@ -328,17 +326,18 @@ def test__get_custom_ld_path(self, config): if exception_expected: self.assertRaises( - TypeError, self.server._get_custom_ld_path, - original_ld_path, extra_library_paths) + TypeError, + self.server._get_custom_ld_path, + original_ld_path, + extra_library_paths, + ) return - result = self.server._get_custom_ld_path( - original_ld_path, extra_library_paths) + result = self.server._get_custom_ld_path(original_ld_path, extra_library_paths) self.assertEqual(result, expected_result) @mock.patch.object(server.WorkerServerEndpoint, '_get_custom_ld_path') - def test__start_process_with_custom_library_paths( - self, mock_get_custom_ld_path): + def test__start_process_with_custom_library_paths(self, mock_get_custom_ld_path): original_ld_path = os.environ.get('LD_LIBRARY_PATH', '') # NOTE(dvincze): Return value is required to be string here, as this # value will be assigned to environment variable LD_LIBRARY_PATH @@ -346,44 +345,53 @@ def test__start_process_with_custom_library_paths( process = mock.MagicMock() self.server._start_process_with_custom_library_paths( - process, mock.sentinel.extra_library_paths) + process, mock.sentinel.extra_library_paths + ) process.start.assert_called_once() mock_get_custom_ld_path.assert_called_once_with( - original_ld_path, mock.sentinel.extra_library_paths) + original_ld_path, mock.sentinel.extra_library_paths + ) self.assertEqual(original_ld_path, os.environ['LD_LIBRARY_PATH']) @mock.patch.object(server.WorkerServerEndpoint, '_get_custom_ld_path') - def test__start_process_with_custom_library_paths_raises( - self, mock_custom_ld_path): + def test__start_process_with_custom_library_paths_raises(self, mock_custom_ld_path): original_ld_path = os.environ.get('LD_LIBRARY_PATH', '') mock_custom_ld_path.side_effect = TypeError() process = mock.MagicMock() with self.assertLogs('coriolis.worker.rpc.server', logging.WARNING): self.server._start_process_with_custom_library_paths( - process, mock.sentinel.extra_library_paths) + process, mock.sentinel.extra_library_paths + ) process.assert_not_called() self.assertEqual(original_ld_path, os.environ['LD_LIBRARY_PATH']) @mock.patch.object(task_runners_factory, 'get_task_runner_class') @mock.patch.object(server, '_get_event_handler_for_task_type') def test__get_extra_library_paths_for_providers( - self, mock_get_event_handler, mock_get_task_runner): + self, mock_get_event_handler, mock_get_task_runner + ): result = self.server._get_extra_library_paths_for_providers( - mock.sentinel.ctxt, mock.sentinel.task_id, mock.sentinel.task_type, - mock.sentinel.origin, mock.sentinel.destination) - mock_get_event_handler.assert_called_once_with(mock.sentinel.task_type, - mock.sentinel.ctxt, - mock.sentinel.task_id) + mock.sentinel.ctxt, + mock.sentinel.task_id, + mock.sentinel.task_type, + mock.sentinel.origin, + mock.sentinel.destination, + ) + mock_get_event_handler.assert_called_once_with( + mock.sentinel.task_type, mock.sentinel.ctxt, mock.sentinel.task_id + ) mock_get_task_runner.assert_called_once_with(mock.sentinel.task_type) - mock_task_runner = ( - mock_get_task_runner.return_value.return_value) + mock_task_runner = mock_get_task_runner.return_value.return_value mock_task_runner.get_shared_libs_for_providers.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.origin, - mock.sentinel.destination, mock_get_event_handler.return_value) + mock.sentinel.ctxt, + mock.sentinel.origin, + mock.sentinel.destination, + mock_get_event_handler.return_value, + ) self.assertEqual( - result, - mock_task_runner.get_shared_libs_for_providers.return_value) + result, mock_task_runner.get_shared_libs_for_providers.return_value + ) def test__wait_for_process(self): p = mock.MagicMock() @@ -487,9 +495,7 @@ def call_exec_task(report_to_conductor=True): # TaskProcessCanceledException handling when reporting to conductor mock_exec.reset_mock() mock_client.reset_mock() - mock_exec.side_effect = exception.TaskProcessCanceledException( - "mock_message" - ) + mock_exec.side_effect = exception.TaskProcessCanceledException("mock_message") call_exec_task() mock_client.confirm_task_cancellation.assert_called_once_with( mock.sentinel.context, mock.sentinel.task_id, "mock_message" @@ -533,8 +539,10 @@ def call_exec_task(report_to_conductor=True): mock_client.reset_mock() mock_exec.reset_mock() mock_exec.side_effect = Exception("mock_message") - self.assertRaises(Exception, # noqa: H202 - lambda: call_exec_task(False)) + self.assertRaises( + Exception, # noqa: H202 + lambda: call_exec_task(False), + ) @mock.patch.object(schemas, "validate_value") @mock.patch.object(utils, "get_secret_connection_info") @@ -665,30 +673,25 @@ def call_get_endpoint_destination_options(): mock_get_secret.assert_called_once_with( mock.sentinel.context, mock.sentinel.connection_info ) - mock_get_provider.return_value\ - .get_target_environment_options.assert_called_once_with( - mock.sentinel.context, - mock_get_secret.return_value, - env=mock.sentinel.environment, - option_names=mock.sentinel.option_names, - ) + mock_get_provider.return_value.get_target_environment_options.assert_called_once_with( + mock.sentinel.context, + mock_get_secret.return_value, + env=mock.sentinel.environment, + option_names=mock.sentinel.option_names, + ) mock_validate.assert_called_once_with( - mock_get_provider.return_value - .get_target_environment_options.return_value, + mock_get_provider.return_value.get_target_environment_options.return_value, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA, ) self.assertEqual( options, - mock_get_provider.return_value - .get_target_environment_options.return_value, + mock_get_provider.return_value.get_target_environment_options.return_value, ) # if the provider is not found, raise InvalidInput mock_get_provider.return_value = None - self.assertRaises( - exception.InvalidInput, call_get_endpoint_destination_options - ) + self.assertRaises(exception.InvalidInput, call_get_endpoint_destination_options) @mock.patch.object(schemas, "validate_value") @mock.patch.object(utils, "get_secret_connection_info") @@ -716,23 +719,20 @@ def call_get_endpoint_source_minion_pool_options(): mock_get_secret.assert_called_once_with( mock.sentinel.context, mock.sentinel.connection_info ) - mock_get_provider.return_value\ - .get_minion_pool_options.assert_called_once_with( - mock.sentinel.context, - mock_get_secret.return_value, - env=mock.sentinel.environment, - option_names=mock.sentinel.option_names, - ) + mock_get_provider.return_value.get_minion_pool_options.assert_called_once_with( + mock.sentinel.context, + mock_get_secret.return_value, + env=mock.sentinel.environment, + option_names=mock.sentinel.option_names, + ) mock_validate.assert_called_once_with( - mock_get_provider.return_value - .get_minion_pool_options.return_value, + mock_get_provider.return_value.get_minion_pool_options.return_value, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA, ) self.assertEqual( options, - mock_get_provider.return_value - .get_minion_pool_options.return_value, + mock_get_provider.return_value.get_minion_pool_options.return_value, ) # if the provider is not found, raise InvalidInput @@ -768,23 +768,20 @@ def call_get_endpoint_destination_minion_pool_options(): mock_get_secret.assert_called_once_with( mock.sentinel.context, mock.sentinel.connection_info ) - mock_get_provider.return_value\ - .get_minion_pool_options.assert_called_once_with( - mock.sentinel.context, - mock_get_secret.return_value, - env=mock.sentinel.environment, - option_names=mock.sentinel.option_names, - ) + mock_get_provider.return_value.get_minion_pool_options.assert_called_once_with( + mock.sentinel.context, + mock_get_secret.return_value, + env=mock.sentinel.environment, + option_names=mock.sentinel.option_names, + ) mock_validate.assert_called_once_with( - mock_get_provider.return_value - .get_minion_pool_options.return_value, + mock_get_provider.return_value.get_minion_pool_options.return_value, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA, ) self.assertEqual( options, - mock_get_provider.return_value - .get_minion_pool_options.return_value, + mock_get_provider.return_value.get_minion_pool_options.return_value, ) # if the provider is not found, raise InvalidInput @@ -820,30 +817,25 @@ def call_get_endpoint_source_options(): mock_get_secret.assert_called_once_with( mock.sentinel.context, mock.sentinel.connection_info ) - mock_get_provider.return_value\ - .get_source_environment_options.assert_called_once_with( - mock.sentinel.context, - mock_get_secret.return_value, - env=mock.sentinel.environment, - option_names=mock.sentinel.option_names, - ) + mock_get_provider.return_value.get_source_environment_options.assert_called_once_with( + mock.sentinel.context, + mock_get_secret.return_value, + env=mock.sentinel.environment, + option_names=mock.sentinel.option_names, + ) mock_validate.assert_called_once_with( - mock_get_provider.return_value - .get_source_environment_options.return_value, + mock_get_provider.return_value.get_source_environment_options.return_value, schemas.CORIOLIS_SOURCE_ENVIRONMENT_OPTIONS_SCHEMA, ) self.assertEqual( options, - mock_get_provider.return_value - .get_source_environment_options.return_value, + mock_get_provider.return_value.get_source_environment_options.return_value, ) # if the provider is not found, raise InvalidInput mock_get_provider.return_value = None - self.assertRaises( - exception.InvalidInput, call_get_endpoint_source_options - ) + self.assertRaises(exception.InvalidInput, call_get_endpoint_source_options) @mock.patch.object(schemas, "validate_value") @mock.patch.object(utils, "get_secret_connection_info") @@ -944,9 +936,7 @@ def test_get_endpoint_storage( @mock.patch.object(utils, "get_secret_connection_info") @mock.patch.object(providers_factory, "get_provider") - def test_get_endpoint_inventory_csv( - self, mock_get_provider, mock_get_secret - ): + def test_get_endpoint_inventory_csv(self, mock_get_provider, mock_get_secret): result = self.server.get_endpoint_inventory_csv( mock.sentinel.context, mock.sentinel.platform_name, @@ -962,16 +952,14 @@ def test_get_endpoint_inventory_csv( mock_get_secret.assert_called_once_with( mock.sentinel.context, mock.sentinel.connection_info ) - mock_get_provider.return_value.export_instance_inventory\ - .assert_called_once_with( - mock.sentinel.context, - mock_get_secret.return_value, - mock.sentinel.source_environment, - ) + mock_get_provider.return_value.export_instance_inventory.assert_called_once_with( + mock.sentinel.context, + mock_get_secret.return_value, + mock.sentinel.source_environment, + ) self.assertEqual( result, - mock_get_provider.return_value.export_instance_inventory - .return_value + mock_get_provider.return_value.export_instance_inventory.return_value, ) @mock.patch.object(utils, "get_secret_connection_info") @@ -1015,15 +1003,13 @@ def test_validate_endpoint_source_environment( ) mock_validate.assert_called_once_with( mock.sentinel.source_environment, - mock_get_provider.return_value - .get_source_environment_schema.return_value, + mock_get_provider.return_value.get_source_environment_schema.return_value, ) self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException( - "test") + mock_validate.side_effect = exception.SchemaValidationException("test") result = self.server.validate_endpoint_source_environment( mock.sentinel.context, mock.sentinel.source_platform_name, @@ -1050,15 +1036,13 @@ def test_validate_endpoint_target_environment( ) mock_validate.assert_called_once_with( mock.sentinel.target_environment, - mock_get_provider.return_value - .get_target_environment_schema.return_value, + mock_get_provider.return_value.get_target_environment_schema.return_value, ) self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException( - "test") + mock_validate.side_effect = exception.SchemaValidationException("test") result = self.server.validate_endpoint_target_environment( mock.sentinel.context, mock.sentinel.target_platform_name, @@ -1085,15 +1069,13 @@ def test_validate_endpoint_source_minion_pool_options( ) mock_validate.assert_called_once_with( mock.sentinel.pool_environment, - mock_get_provider.return_value - .get_minion_pool_environment_schema.return_value, + mock_get_provider.return_value.get_minion_pool_environment_schema.return_value, ) self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException( - "test") + mock_validate.side_effect = exception.SchemaValidationException("test") result = self.server.validate_endpoint_source_minion_pool_options( mock.sentinel.context, mock.sentinel.source_platform_name, @@ -1120,15 +1102,13 @@ def test_validate_endpoint_destination_minion_pool_options( ) mock_validate.assert_called_once_with( mock.sentinel.pool_environment, - mock_get_provider.return_value - .get_minion_pool_environment_schema.return_value, + mock_get_provider.return_value.get_minion_pool_environment_schema.return_value, ) self.assertEqual(result, (True, None)) # handle SchemaValidationException - mock_validate.side_effect = exception.SchemaValidationException( - "test") + mock_validate.side_effect = exception.SchemaValidationException("test") result = self.server.validate_endpoint_destination_minion_pool_options( mock.sentinel.context, mock.sentinel.destination_platform_name, @@ -1160,13 +1140,11 @@ def call_validate_endpoint_connection(): ) mock_validate.assert_called_once_with( mock_get_secret.return_value, - mock_get_provider.return_value - .get_connection_info_schema.return_value, + mock_get_provider.return_value.get_connection_info_schema.return_value, + ) + mock_get_provider.return_value.validate_connection.assert_called_once_with( + mock.sentinel.context, mock_get_secret.return_value ) - mock_get_provider.return_value\ - .validate_connection.assert_called_once_with( - mock.sentinel.context, mock_get_secret.return_value - ) self.assertEqual(result, (True, None)) @@ -1188,18 +1166,12 @@ def call_validate_endpoint_connection(): @mock.patch.object(providers_factory, "get_provider") @ddt.data( - ( - constants.PROVIDER_TYPE_ENDPOINT, - "connection_info_schema" - ), + (constants.PROVIDER_TYPE_ENDPOINT, "connection_info_schema"), ( constants.PROVIDER_TYPE_TRANSFER_IMPORT, "destination_environment_schema", ), - ( - constants.PROVIDER_TYPE_TRANSFER_EXPORT, - "source_environment_schema" - ), + (constants.PROVIDER_TYPE_TRANSFER_EXPORT, "source_environment_schema"), ( constants.PROVIDER_TYPE_SOURCE_MINION_POOL, "source_minion_pool_environment_schema", @@ -1231,9 +1203,15 @@ def test_get_diagnostics(self, mock_get_diagnostics_info): @mock.patch('coriolis.utils.setup_logging') @mock.patch('oslo_log.log.getLogger') @mock.patch('logging.handlers.QueueHandler') - def test__setup_task_process(self, mock_queue_handler, mock_get_logger, - mock_setup_logging, mock_conf, mock_argv, - mock_get_worker_count_from_args): + def test__setup_task_process( + self, + mock_queue_handler, + mock_get_logger, + mock_setup_logging, + mock_conf, + mock_argv, + mock_get_worker_count_from_args, + ): mock_get_worker_count_from_args.return_value = (None, "args") mock_logger = mock_get_logger.return_value.logger mock_logger.handlers = [mock.sentinel.handler] @@ -1242,43 +1220,56 @@ def test__setup_task_process(self, mock_queue_handler, mock_get_logger, mock_get_worker_count_from_args.assert_called_once_with(mock_argv) mock_conf.assert_called_once_with( mock_get_worker_count_from_args.return_value[1][1:], - project='coriolis', version='1.0.0') + project='coriolis', + version='1.0.0', + ) mock_setup_logging.assert_called_once_with() mock_get_logger.assert_called_once_with(None) - mock_logger.removeHandler.assert_called_once_with( - mock.sentinel.handler) - mock_queue_handler.assert_called_once_with( - mock.sentinel.mp_log_q) - mock_logger.addHandler.assert_called_once_with( - mock_queue_handler.return_value) + mock_logger.removeHandler.assert_called_once_with(mock.sentinel.handler) + mock_queue_handler.assert_called_once_with(mock.sentinel.mp_log_q) + mock_logger.addHandler.assert_called_once_with(mock_queue_handler.return_value) @mock.patch.object(server, '_setup_task_process') @mock.patch.object(task_runners_factory, 'get_task_runner_class') @mock.patch.object(server, '_get_event_handler_for_task_type') @mock.patch('coriolis.utils.is_serializable') - def test__task_process(self, mock_is_serializable, - mock_get_event_handler, mock_get_task_runner_class, - mock_setup_task_process): + def test__task_process( + self, + mock_is_serializable, + mock_get_event_handler, + mock_get_task_runner_class, + mock_setup_task_process, + ): mp_q = mock.MagicMock() mp_log_q = mock.MagicMock() task_info = {} mock_task_runner = mock_get_task_runner_class.return_value.return_value mock_task_result = mock_task_runner.run.return_value - server._task_process(mock.sentinel.ctxt, mock.sentinel.task_id, - mock.sentinel.task_type, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.instance, - task_info, mp_q, mp_log_q) + server._task_process( + mock.sentinel.ctxt, + mock.sentinel.task_id, + mock.sentinel.task_type, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.instance, + task_info, + mp_q, + mp_log_q, + ) mock_setup_task_process.assert_called_once_with(mp_log_q) - mock_get_task_runner_class.assert_called_once_with( - mock.sentinel.task_type) - mock_get_event_handler.assert_called_once_with(mock.sentinel.task_type, - mock.sentinel.ctxt, - mock.sentinel.task_id) + mock_get_task_runner_class.assert_called_once_with(mock.sentinel.task_type) + mock_get_event_handler.assert_called_once_with( + mock.sentinel.task_type, mock.sentinel.ctxt, mock.sentinel.task_id + ) mock_task_runner.run.assert_called_once_with( - mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, - mock.sentinel.destination, task_info, - mock_get_event_handler.return_value) + mock.sentinel.ctxt, + mock.sentinel.instance, + mock.sentinel.origin, + mock.sentinel.destination, + task_info, + mock_get_event_handler.return_value, + ) mock_is_serializable.assert_called_once_with(mock_task_result) mp_q.put.assert_called_once_with(mock_task_result) mp_log_q.put.assert_called_once_with(None) @@ -1289,9 +1280,16 @@ def test__task_process_raise(self, mock_setup_task_process): mp_q = mock.MagicMock() mp_log_q = mock.MagicMock() - server._task_process(mock.sentinel.ctxt, mock.sentinel.task_id, - mock.sentinel.task_type, mock.sentinel.origin, - mock.sentinel.destination, mock.sentinel.instance, - mock.sentinel.task_info, mp_q, mp_log_q) + server._task_process( + mock.sentinel.ctxt, + mock.sentinel.task_id, + mock.sentinel.task_type, + mock.sentinel.origin, + mock.sentinel.destination, + mock.sentinel.instance, + mock.sentinel.task_info, + mp_q, + mp_log_q, + ) mp_q.put.assert_called_once_with("YOLO") mp_log_q.put.assert_called_once_with(None) diff --git a/coriolis/transfer_cron/api.py b/coriolis/transfer_cron/api.py index 288b9d84..8db9a4cc 100644 --- a/coriolis/transfer_cron/api.py +++ b/coriolis/transfer_cron/api.py @@ -8,24 +8,40 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create(self, ctxt, transfer_id, schedule, enabled, - exp_date, shutdown_instance, auto_deploy): + def create( + self, + ctxt, + transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ): return self._rpc_client.create_transfer_schedule( - ctxt, transfer_id, schedule, enabled, exp_date, - shutdown_instance, auto_deploy) + ctxt, + transfer_id, + schedule, + enabled, + exp_date, + shutdown_instance, + auto_deploy, + ) def get_schedules(self, ctxt, transfer_id, expired=True): return self._rpc_client.get_transfer_schedules( - ctxt, transfer_id, expired=expired) + ctxt, transfer_id, expired=expired + ) def get_schedule(self, ctxt, transfer_id, schedule_id, expired=True): return self._rpc_client.get_transfer_schedule( - ctxt, transfer_id, schedule_id, expired=expired) + ctxt, transfer_id, schedule_id, expired=expired + ) def update(self, ctxt, transfer_id, schedule_id, update_values): return self._rpc_client.update_transfer_schedule( - ctxt, transfer_id, schedule_id, update_values) + ctxt, transfer_id, schedule_id, update_values + ) def delete(self, ctxt, transfer_id, schedule_id): - self._rpc_client.delete_transfer_schedule( - ctxt, transfer_id, schedule_id) + self._rpc_client.delete_transfer_schedule(ctxt, transfer_id, schedule_id) diff --git a/coriolis/transfer_cron/rpc/client.py b/coriolis/transfer_cron/rpc/client.py index 1483e239..4e49daba 100644 --- a/coriolis/transfer_cron/rpc/client.py +++ b/coriolis/transfer_cron/rpc/client.py @@ -3,16 +3,14 @@ import oslo_messaging as messaging -from coriolis import constants -from coriolis import rpc +from coriolis import constants, rpc VERSION = "1.0" class TransferCronClient(rpc.BaseRPCClient): def __init__(self, topic=constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC): - target = messaging.Target( - topic=topic, version=VERSION) + target = messaging.Target(topic=topic, version=VERSION) super(TransferCronClient, self).__init__(target) def register(self, ctxt, schedule): diff --git a/coriolis/transfer_cron/rpc/server.py b/coriolis/transfer_cron/rpc/server.py index 1b0b2cb2..9a3ff534 100644 --- a/coriolis/transfer_cron/rpc/server.py +++ b/coriolis/transfer_cron/rpc/server.py @@ -6,33 +6,35 @@ from oslo_log import log as logging from oslo_utils import timeutils +from coriolis import context, exception, utils from coriolis.conductor.rpc import client as rpc_client -from coriolis import context from coriolis.cron import cron -from coriolis import exception -from coriolis import utils LOG = logging.getLogger(__name__) VERSION = "1.0" -def _trigger_transfer(ctxt, conductor_client, transfer_id, shutdown_instance, - auto_deploy): +def _trigger_transfer( + ctxt, conductor_client, transfer_id, shutdown_instance, auto_deploy +): try: execution = conductor_client.execute_transfer_tasks( - ctxt, transfer_id, shutdown_instances=shutdown_instance, - auto_deploy=auto_deploy) + ctxt, + transfer_id, + shutdown_instances=shutdown_instance, + auto_deploy=auto_deploy, + ) result_msg = 'Execution %s for Transfer %s' % ( - execution.get('id'), execution.get('action_id')) + execution.get('id'), + execution.get('action_id'), + ) return result_msg - except (exception.InvalidTransferState, - exception.InvalidActionTasksExecutionState): + except (exception.InvalidTransferState, exception.InvalidActionTasksExecutionState): LOG.info("A replica or migration already running") class TransferCronServerEndpoint(object): - def __init__(self): self._rpc_client = rpc_client.ConductorClient() # Setup cron loop @@ -44,7 +46,8 @@ def _deserialize_schedule(self, sched): expires = sched.get("expiration_date") if expires: sched["expiration_date"] = timeutils.normalize_time( - timeutils.parse_isotime(expires)) + timeutils.parse_isotime(expires) + ) tmp = sched["schedule"] if type(tmp) is str: sched["schedule"] = json.loads(tmp) @@ -57,15 +60,23 @@ def _register_schedule(self, schedule, date=None): if expires and expires <= date: LOG.info("Not registering expired schedule: %s" % sched["id"]) return - trust_ctxt = context.get_admin_context( - trust_id=schedule["trust_id"]) + trust_ctxt = context.get_admin_context(trust_id=schedule["trust_id"]) description = "Scheduled job for %s" % sched["id"] job = cron.CronJob( - sched["id"], description, sched["schedule"], - sched["enabled"], sched["expiration_date"], - None, None, _trigger_transfer, trust_ctxt, - self._rpc_client, schedule["transfer_id"], - schedule["shutdown_instance"], schedule["auto_deploy"]) + sched["id"], + description, + sched["schedule"], + sched["enabled"], + sched["expiration_date"], + None, + None, + _trigger_transfer, + trust_ctxt, + self._rpc_client, + schedule["transfer_id"], + schedule["shutdown_instance"], + schedule["auto_deploy"], + ) self._cron.register(job) def _init_cron(self): @@ -84,13 +95,15 @@ def _init_cron(self): def _get_all_schedules(self): schedules = self._rpc_client.get_transfer_schedules( - self._admin_ctx, expired=False) + self._admin_ctx, expired=False + ) return schedules def register(self, ctxt, schedule): now = timeutils.utcnow() - LOG.debug("Registering new schedule %s: %r" % ( - schedule["id"], schedule["schedule"])) + LOG.debug( + "Registering new schedule %s: %r" % (schedule["id"], schedule["schedule"]) + ) self._register_schedule(schedule, date=now) def unregister(self, ctxt, schedule): diff --git a/coriolis/transfer_tasks_executions/api.py b/coriolis/transfer_tasks_executions/api.py index 7158f93e..90f54fcf 100644 --- a/coriolis/transfer_tasks_executions/api.py +++ b/coriolis/transfer_tasks_executions/api.py @@ -10,23 +10,34 @@ def __init__(self): def create(self, ctxt, transfer_id, shutdown_instances, auto_deploy): return self._rpc_client.execute_transfer_tasks( - ctxt, transfer_id, shutdown_instances, auto_deploy) + ctxt, transfer_id, shutdown_instances, auto_deploy + ) def delete(self, ctxt, transfer_id, execution_id): self._rpc_client.delete_transfer_tasks_execution( - ctxt, transfer_id, execution_id) + ctxt, transfer_id, execution_id + ) def cancel(self, ctxt, transfer_id, execution_id, force): self._rpc_client.cancel_transfer_tasks_execution( - ctxt, transfer_id, execution_id, force) - - def get_executions(self, ctxt, transfer_id, include_tasks=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): + ctxt, transfer_id, execution_id, force + ) + + def get_executions( + self, + ctxt, + transfer_id, + include_tasks=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return self._rpc_client.get_transfer_tasks_executions( - ctxt, transfer_id, include_tasks, marker, limit, - sort_keys, sort_dirs) + ctxt, transfer_id, include_tasks, marker, limit, sort_keys, sort_dirs + ) def get_execution(self, ctxt, transfer_id, execution_id): return self._rpc_client.get_transfer_tasks_execution( - ctxt, transfer_id, execution_id) + ctxt, transfer_id, execution_id + ) diff --git a/coriolis/transfers/api.py b/coriolis/transfers/api.py index 31ff9623..e9452e8e 100644 --- a/coriolis/transfers/api.py +++ b/coriolis/transfers/api.py @@ -8,42 +8,74 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create(self, ctxt, transfer_scenario, - origin_endpoint_id, destination_endpoint_id, - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, destination_environment, instances, - network_map, storage_mappings, notes=None, user_scripts=None, - clone_disks=True, skip_os_morphing=False): + def create( + self, + ctxt, + transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, + instance_osmorphing_minion_pool_mappings, + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes=None, + user_scripts=None, + clone_disks=True, + skip_os_morphing=False, + ): return self._rpc_client.create_instances_transfer( - ctxt, transfer_scenario, - origin_endpoint_id, destination_endpoint_id, - origin_minion_pool_id, destination_minion_pool_id, + ctxt, + transfer_scenario, + origin_endpoint_id, + destination_endpoint_id, + origin_minion_pool_id, + destination_minion_pool_id, instance_osmorphing_minion_pool_mappings, - source_environment, destination_environment, instances, - network_map, storage_mappings, notes, user_scripts, clone_disks, - skip_os_morphing) + source_environment, + destination_environment, + instances, + network_map, + storage_mappings, + notes, + user_scripts, + clone_disks, + skip_os_morphing, + ) def update(self, ctxt, transfer_id, updated_properties): - return self._rpc_client.update_transfer( - ctxt, transfer_id, updated_properties) + return self._rpc_client.update_transfer(ctxt, transfer_id, updated_properties) def delete(self, ctxt, transfer_id): self._rpc_client.delete_transfer(ctxt, transfer_id) - def get_transfers(self, ctxt, include_tasks_executions=False, - include_task_info=False, - marker=None, limit=None, - sort_keys=None, sort_dirs=None): + def get_transfers( + self, + ctxt, + include_tasks_executions=False, + include_task_info=False, + marker=None, + limit=None, + sort_keys=None, + sort_dirs=None, + ): return self._rpc_client.get_transfers( - ctxt, include_tasks_executions, + ctxt, + include_tasks_executions, include_task_info=include_task_info, - marker=marker, limit=limit, - sort_keys=sort_keys, sort_dirs=sort_dirs) + marker=marker, + limit=limit, + sort_keys=sort_keys, + sort_dirs=sort_dirs, + ) def get_transfer(self, ctxt, transfer_id, include_task_info=False): return self._rpc_client.get_transfer( - ctxt, transfer_id, include_task_info=include_task_info) + ctxt, transfer_id, include_task_info=include_task_info + ) def delete_disks(self, ctxt, transfer_id): return self._rpc_client.delete_transfer_disks(ctxt, transfer_id) diff --git a/coriolis/utils.py b/coriolis/utils.py index 0655e05a..26e42b76 100644 --- a/coriolis/utils.py +++ b/coriolis/utils.py @@ -9,7 +9,6 @@ import hashlib import io import json -import OpenSSL import os import pickle import re @@ -21,15 +20,15 @@ import time import traceback import uuid - from io import StringIO +import netifaces +import OpenSSL +import paramiko from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils -import netifaces -import paramiko # NOTE(gsamfira): I am aware that this is not ideal, but pip # developers have decided to move all logic inside an _internal # package, and I really don't want to do an exec call to pip @@ -39,15 +38,12 @@ from six.moves.urllib import parse from webob import exc -from coriolis import constants -from coriolis import exception -from coriolis import secrets - +from coriolis import constants, exception, secrets opts = [ - cfg.StrOpt('qemu_img_path', - default='qemu-img', - help='The path of the qemu-img tool.'), + cfg.StrOpt( + 'qemu_img_path', default='qemu-img', help='The path of the qemu-img tool.' + ), ] CONF = cfg.CONF @@ -151,6 +147,7 @@ def _ignore_exceptions(*args, **kwargs): return func(*args, **kwargs) except Exception: LOG.warn("Ignoring exception:\n%s", get_exception_details()) + return _ignore_exceptions @@ -167,8 +164,7 @@ def get_single_result(lis): return lis[0] -def retry_on_error(max_attempts=5, sleep_seconds=1, - terminal_exceptions=[]): +def retry_on_error(max_attempts=5, sleep_seconds=1, terminal_exceptions=[]): def _retry_on_error(func): @functools.wraps(func) def _exec_retry(*args, **kwargs): @@ -181,35 +177,41 @@ def _exec_retry(*args, **kwargs): LOG.exception(ex) raise except Exception as ex: - if any([isinstance(ex, tex) - for tex in terminal_exceptions]): + if any([isinstance(ex, tex) for tex in terminal_exceptions]): raise i += 1 if i < max_attempts: LOG.warn( "Exception occurred, retrying (%d/%d):\n%s", - i, max_attempts, get_exception_details()) + i, + max_attempts, + get_exception_details(), + ) time.sleep(sleep_seconds) else: raise + return _exec_retry + return _retry_on_error def get_udev_net_rules(net_ifaces_info): content = "" for name, mac_address in net_ifaces_info.items(): - content += ('SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' - 'ATTR{address}=="%(mac_address)s", NAME="%(name)s"\n' % - {"name": name, "mac_address": mac_address.lower()}) + content += ( + 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ' + 'ATTR{address}=="%(mac_address)s", NAME="%(name)s"\n' + % {"name": name, "mac_address": mac_address.lower()} + ) return content def parse_os_release(ssh): os_release_info = exec_ssh_cmd( - ssh, - "[ -f '/etc/os-release' ] && cat /etc/os-release || true") + ssh, "[ -f '/etc/os-release' ] && cat /etc/os-release || true" + ) info = {} for line in os_release_info.splitlines(): if "=" not in line: @@ -275,26 +277,25 @@ def write_winrm_file(conn, remote_path, content, overwrite=True): """ if conn.test_path(remote_path): if overwrite: - conn.exec_ps_command( - 'Remove-Item -Force "%s"' % remote_path) + conn.exec_ps_command('Remove-Item -Force "%s"' % remote_path) else: - raise exception.CoriolisException( - "File %s already exists" % remote_path) + raise exception.CoriolisException("File %s already exists" % remote_path) idx = 0 while True: - data = content[idx:idx + 2048] + data = content[idx : idx + 2048] if not data: break if type(data) is str: data = data.encode() asb64 = base64.b64encode(data).decode() - cmd = ("$ErrorActionPreference = 'Stop';" - "$x = New-Object System.IO.FileStream(\"%s\", " - "[System.IO.FileMode]::Append); $bytes = " - "[Convert]::FromBase64String('%s'); $x.Write($bytes, " - "0, $bytes.Length); $x.Close()") % ( - remote_path, asb64) + cmd = ( + "$ErrorActionPreference = 'Stop';" + "$x = New-Object System.IO.FileStream(\"%s\", " + "[System.IO.FileMode]::Append); $bytes = " + "[Convert]::FromBase64String('%s'); $x.Write($bytes, " + "0, $bytes.Length); $x.Close()" + ) % (remote_path, asb64) conn.exec_ps_command(cmd) idx += 2048 @@ -308,15 +309,19 @@ def list_ssh_dir(ssh, remote_path): return sftp.listdir(remote_path) except Exception: LOG.warning( - "SFTP listdir failed, falling back to shell command. " - "Error: %s", get_exception_details()) + "SFTP listdir failed, falling back to shell command. Error: %s", + get_exception_details(), + ) output = exec_ssh_cmd(ssh, "sudo ls -1 %s" % remote_path, get_pty=True) return [f for f in output.splitlines() if f.strip()] -@retry_on_error(terminal_exceptions=[ - exception.MinionMachineCommandTimeout, - exception.SSHCommandNotFoundException]) +@retry_on_error( + terminal_exceptions=[ + exception.MinionMachineCommandTimeout, + exception.SSHCommandNotFoundException, + ] +) def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): remote_str = "" if timeout is not None: @@ -326,13 +331,18 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): except (ValueError, AttributeError, TypeError): LOG.warn( "Failed to determine connection string for SSH connection: %s", - get_exception_details()) + get_exception_details(), + ) LOG.debug( - "Executing the following SSH command on '%s' with " - "environment %s: '%s'", remote_str, environment, cmd) + "Executing the following SSH command on '%s' with environment %s: '%s'", + remote_str, + environment, + cmd, + ) _, stdout, stderr = ssh.exec_command( - cmd, environment=environment, get_pty=get_pty, timeout=timeout) + cmd, environment=environment, get_pty=get_pty, timeout=timeout + ) try: std_out = stdout.read() std_err = stderr.read() @@ -344,11 +354,14 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): stderr_str = std_err.decode(errors='ignore') msg = ( "Command \"%s\" failed on host '%s' with exit code: %s\n" - "stdout: %s\nstd_err: %s" % - (cmd, remote_str, exit_code, stdout_str, stderr_str)) - if (exit_code == 127 or - "command not found" in stdout_str or - "command not found" in stderr_str): + "stdout: %s\nstd_err: %s" + % (cmd, remote_str, exit_code, stdout_str, stderr_str) + ) + if ( + exit_code == 127 + or "command not found" in stdout_str + or "command not found" in stderr_str + ): raise exception.SSHCommandNotFoundException(msg) raise exception.CoriolisException(msg) # Most of the commands will use pseudo-terminal which unfortunately will @@ -360,40 +373,43 @@ def exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): return std_out.decode('utf-8', errors='replace') -def exec_ssh_cmd_chroot(ssh, chroot_dir, cmd, environment=None, get_pty=False, - timeout=None): - return exec_ssh_cmd(ssh, "sudo -E chroot %s %s" % (chroot_dir, cmd), - environment=environment, get_pty=get_pty, - timeout=timeout) +def exec_ssh_cmd_chroot( + ssh, chroot_dir, cmd, environment=None, get_pty=False, timeout=None +): + return exec_ssh_cmd( + ssh, + "sudo -E chroot %s %s" % (chroot_dir, cmd), + environment=environment, + get_pty=get_pty, + timeout=timeout, + ) def check_fs(ssh, fs_type, dev_path): try: out = exec_ssh_cmd( - ssh, "sudo fsck -p -t %s %s" % (fs_type, dev_path), - get_pty=True) + ssh, "sudo fsck -p -t %s %s" % (fs_type, dev_path), get_pty=True + ) LOG.debug("File system checked:\n%s", out) except Exception: - LOG.warn("Checking file system returned an error:\n%s" % ( - get_exception_details())) + LOG.warn( + "Checking file system returned an error:\n%s" % (get_exception_details()) + ) raise def run_xfs_repair(ssh, dev_path): try: - tmp_dir = exec_ssh_cmd( - ssh, "mktemp -d").rstrip("\n") + tmp_dir = exec_ssh_cmd(ssh, "mktemp -d").rstrip("\n") LOG.debug("mounting %s on %s" % (dev_path, tmp_dir)) mount_out = exec_ssh_cmd( - ssh, "sudo mount %s %s" % (dev_path, tmp_dir), - get_pty=True) + ssh, "sudo mount %s %s" % (dev_path, tmp_dir), get_pty=True + ) LOG.debug("mount returned: %s" % mount_out) LOG.debug("Umounting %s" % tmp_dir) - umount_out = exec_ssh_cmd( - ssh, "sudo umount %s" % tmp_dir, get_pty=True) + umount_out = exec_ssh_cmd(ssh, "sudo umount %s" % tmp_dir, get_pty=True) LOG.debug("umounting returned: %s" % umount_out) - out = exec_ssh_cmd( - ssh, "sudo xfs_repair %s" % dev_path, get_pty=True) + out = exec_ssh_cmd(ssh, "sudo xfs_repair %s" % dev_path, get_pty=True) LOG.debug("File system repaired:\n%s", out) except Exception as ex: LOG.warn("xfs_repair returned an error:\n%s", str(ex)) @@ -417,8 +433,7 @@ def wait_for_port_connectivity(address, port, max_wait=300): time.sleep(1) i += 1 if i == max_wait: - raise exception.CoriolisException("Connection failed on port %s" % - port) + raise exception.CoriolisException("Connection failed on port %s" % port) def exec_process(args): @@ -427,13 +442,13 @@ def exec_process(args): if p.returncode: raise exception.CoriolisException( "Command \"%s\" failed with exit code: %s\nstdout: %s\nstd_err: %s" - % (args, p.returncode, std_out, std_err)) + % (args, p.returncode, std_out, std_err) + ) return std_out def get_disk_info(disk_path): - out = exec_process([CONF.qemu_img_path, 'info', '--output=json', - disk_path]) + out = exec_process([CONF.qemu_img_path, 'info', '--output=json', disk_path]) disk_info = json.loads(out.decode()) if disk_info["format"] == "vpc": @@ -441,23 +456,25 @@ def get_disk_info(disk_path): return disk_info -def convert_disk_format(disk_path, target_disk_path, target_format, - preallocated=False): +def convert_disk_format(disk_path, target_disk_path, target_format, preallocated=False): allocation_args = [] if preallocated: if target_format != constants.DISK_FORMAT_VHD: raise NotImplementedError( - "Preallocation is supported only for the VHD format.") + "Preallocation is supported only for the VHD format." + ) allocation_args = ['-o', 'subformat=fixed'] if target_format == constants.DISK_FORMAT_VHD: target_format = "vpc" - args = ([CONF.qemu_img_path, 'convert', '-O', target_format] + - allocation_args + - [disk_path, target_disk_path]) + args = ( + [CONF.qemu_img_path, 'convert', '-O', target_format] + + allocation_args + + [disk_path, target_disk_path] + ) try: exec_process(args) @@ -504,8 +521,7 @@ def get_ssl_cert_thumbprint(context, host, port=443, digest_algorithm="sha1"): def get_resources_dir(): - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), "resources") + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources") def get_resources_bin_dir(): @@ -523,8 +539,15 @@ def deserialize_key(key_bytes, password=None): return paramiko.RSAKey.from_private_key(key_io, password) -def connect_ssh(hostname, port, username, pkey=None, password=None, - connect_timeout=None, banner_timeout=None): +def connect_ssh( + hostname, + port, + username, + pkey=None, + password=None, + connect_timeout=None, + banner_timeout=None, +): """Open and return a connected paramiko SSHClient. :param pkey: a paramiko.PKey instance or None. @@ -536,8 +559,8 @@ def connect_ssh(hostname, port, username, pkey=None, password=None, ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) kwargs = dict( - hostname=hostname, port=port, username=username, - pkey=pkey, password=password) + hostname=hostname, port=port, username=username, pkey=pkey, password=password + ) if connect_timeout is not None: kwargs["timeout"] = connect_timeout @@ -548,7 +571,8 @@ def connect_ssh(hostname, port, username, pkey=None, password=None, ssh.connect(**kwargs) except paramiko.ssh_exception.SSHException as ex: raise exception.CoriolisException( - "Failed to setup SSH client: %s" % str(ex)) from ex + "Failed to setup SSH client: %s" % str(ex) + ) from ex except (Exception, KeyboardInterrupt): ssh.close() raise @@ -562,11 +586,17 @@ def is_serializable(obj): def to_dict(obj, max_depth=10): # jsonutils.dumps() has a max_depth of 3 by default - def _to_primitive(value, convert_instances=False, - convert_datetime=True, level=0, - max_depth=max_depth): + def _to_primitive( + value, + convert_instances=False, + convert_datetime=True, + level=0, + max_depth=max_depth, + ): return jsonutils.to_primitive( - value, convert_instances, convert_datetime, level, max_depth) + value, convert_instances, convert_datetime, level, max_depth + ) + return jsonutils.loads(jsonutils.dumps(obj, default=_to_primitive)) @@ -615,12 +645,13 @@ def quote_url(text): def normalize_mac_address(original_mac_address): - """ Normalizez capitalized MAC addresses with or without '-' or ':' - as separators into all-lower-case ':'-separated form. """ + """Normalizez capitalized MAC addresses with or without '-' or ':' + as separators into all-lower-case ':'-separated form.""" if not isinstance(original_mac_address, str): raise ValueError( - "MAC address must be a str, got type '%s': %s" % ( - type(original_mac_address), original_mac_address)) + "MAC address must be a str, got type '%s': %s" + % (type(original_mac_address), original_mac_address) + ) res = "" mac_address = original_mac_address.strip().lower().replace('-', ':') @@ -628,28 +659,25 @@ def normalize_mac_address(original_mac_address): res = mac_address elif re.match(UNSPACED_MAC_ADDRESS_REGEX, mac_address): for i in range(0, len(mac_address), 2): - res = "%s:%s" % (res, mac_address[i:i + 2]) + res = "%s:%s" % (res, mac_address[i : i + 2]) res = res.strip(':') if not re.match(SPACED_MAC_ADDRESS_REGEX, res): raise ValueError( "Failed to normalize MAC address '%s': ended up " - "with: '%s'" % (original_mac_address, res)) + "with: '%s'" % (original_mac_address, res) + ) else: - raise ValueError( - "Improperly formatted MAC address: %s" % original_mac_address) + raise ValueError("Improperly formatted MAC address: %s" % original_mac_address) - LOG.debug( - "Normalized MAC address '%s' to '%s'", - original_mac_address, res) + LOG.debug("Normalized MAC address '%s' to '%s'", original_mac_address, res) return res def get_url_with_credentials(url, username, password): parts = parse.urlsplit(url) # Remove previous credentials if set - netloc = parts.netloc[parts.netloc.find('@') + 1:] - netloc = "%s:%s@%s" % ( - quote_url(username), quote_url(password or ''), netloc) + netloc = parts.netloc[parts.netloc.find('@') + 1 :] + netloc = "%s:%s@%s" % (quote_url(username), quote_url(password or ''), netloc) parts = parts._replace(netloc=netloc) return parse.urlunsplit(parts) @@ -665,7 +693,8 @@ def get_unique_option_ids(resources, id_key="id", name_key="name"): if not all([name_key in d and id_key in d for d in resources]): raise KeyError( "Some resources are missing the name key '%s' " - "or ID key '%s': %s" % (name_key, id_key, resources)) + "or ID key '%s': %s" % (name_key, id_key, resources) + ) name_mappings = {} for resource in resources: @@ -691,15 +720,16 @@ def _bad_request_on_error(func): def wrapper(*args, **kwargs): (is_valid, message) = func(*args, **kwargs) if not is_valid: - raise exc.HTTPBadRequest( - explanation=(error_message % message)) + raise exc.HTTPBadRequest(explanation=(error_message % message)) return (is_valid, message) + return wrapper + return _bad_request_on_error def sanitize_task_info(task_info): - """ Returns a copy of the given task info with any chunking + """Returns a copy of the given task info with any chunking info for volumes and sensitive credentials removed. """ new = {} @@ -727,19 +757,18 @@ def sanitize_task_info(task_info): vol_cpy['replica_state'] = {} for statekey in vol['replica_state']: if statekey != "chunks": - vol_cpy['replica_state'][statekey] = ( - copy.deepcopy( - vol['replica_state'][statekey])) + vol_cpy['replica_state'][statekey] = copy.deepcopy( + vol['replica_state'][statekey] + ) else: - vol_cpy['replica_state']["chunks"] = ( - [""]) + vol_cpy['replica_state']["chunks"] = [""] new['volumes_info'].append(vol_cpy) return new def parse_ini_config(file_contents): - """ Parses the contents of the given .ini config file and + """Parses the contents of the given .ini config file and returns a dict with the options/values within it. """ config = {} @@ -753,7 +782,7 @@ def parse_ini_config(file_contents): def read_ssh_ini_config_file(ssh, path, check_exists=False): - """ Reads and parses the contents of an .ini file at the given path. """ + """Reads and parses the contents of an .ini file at the given path.""" if not check_exists or test_ssh_path(ssh, path): content = read_ssh_file(ssh, path).decode() return parse_ini_config(content) @@ -774,18 +803,13 @@ def _write_systemd(ssh, cmdline, svcname, run_as=None, start=True): if test_ssh_path(ssh, serviceFilePath): if start: - exec_ssh_cmd( - ssh, "sudo systemctl start %s" % svcname, get_pty=True) + exec_ssh_cmd(ssh, "sudo systemctl start %s" % svcname, get_pty=True) return def _reload_and_start(start=True): - exec_ssh_cmd( - ssh, "sudo systemctl daemon-reload", - get_pty=True) + exec_ssh_cmd(ssh, "sudo systemctl daemon-reload", get_pty=True) if start: - exec_ssh_cmd( - ssh, "sudo systemctl start %s" % svcname, - get_pty=True) + exec_ssh_cmd(ssh, "sudo systemctl start %s" % svcname, get_pty=True) def _correct_selinux_label(): cmd = "sudo restorecon -v %s" % serviceFilePath @@ -794,7 +818,10 @@ def _correct_selinux_label(): except exception.CoriolisException: LOG.warn( "Could not relabel service '%s'. SELinux might not be " - "installed. Error was: %s", svcname, get_exception_details()) + "installed. Error was: %s", + svcname, + get_exception_details(), + ) systemd_args = { "cmdline": cmdline, @@ -807,12 +834,10 @@ def _correct_selinux_label(): systemdService = SYSTEMD_TEMPLATE % systemd_args name = str(uuid.uuid4()) - write_ssh_file( - ssh, '/tmp/%s.service' % name, systemdService) + write_ssh_file(ssh, '/tmp/%s.service' % name, systemdService) exec_ssh_cmd( - ssh, - "sudo mv /tmp/%s.service %s" % (name, serviceFilePath), - get_pty=True) + ssh, "sudo mv /tmp/%s.service %s" % (name, serviceFilePath), get_pty=True + ) _correct_selinux_label() _reload_and_start(start=start) @@ -831,20 +856,17 @@ def _write_upstart(ssh, cmdline, svcname, run_as=None, start=True): "svc_name": svcname, } name = str(uuid.uuid4()) - write_ssh_file( - ssh, '/tmp/%s.conf' % name, upstartService) - exec_ssh_cmd( - ssh, - "sudo mv /tmp/%s.conf %s" % (name, serviceFilePath), - get_pty=True) + write_ssh_file(ssh, '/tmp/%s.conf' % name, upstartService) + exec_ssh_cmd(ssh, "sudo mv /tmp/%s.conf %s" % (name, serviceFilePath), get_pty=True) if start: exec_ssh_cmd(ssh, "start %s" % svcname) def _has_systemd(ssh): """Check if the remote system uses systemd as its init system.""" - return (test_ssh_path(ssh, "/lib/systemd/system") or - test_ssh_path(ssh, "/usr/lib/systemd/system")) + return test_ssh_path(ssh, "/lib/systemd/system") or test_ssh_path( + ssh, "/usr/lib/systemd/system" + ) @retry_on_error() @@ -859,8 +881,7 @@ def create_service(ssh, cmdline, svcname, run_as=None, start=True): elif test_ssh_path(ssh, "/etc/init"): _write_upstart(ssh, cmdline, svcname, run_as=run_as, start=start) else: - raise exception.CoriolisException( - "could not determine init system") + raise exception.CoriolisException("could not determine init system") @retry_on_error() @@ -911,21 +932,11 @@ def _parse_cfg(self, cfg): ret = [] for line in cfg.splitlines(): if line.startswith("#") or len(line.strip()) == 0: - ret.append( - { - "type": "raw", - "payload": line - } - ) + ret.append({"type": "raw", "payload": line}) continue vals = line.split("=", 1) if len(vals) != 2: - ret.append( - { - "type": "raw", - "payload": line - } - ) + ret.append({"type": "raw", "payload": line}) continue quoted = False @@ -946,7 +957,7 @@ def _parse_cfg(self, cfg): "opt_type": "single", "opt_val": vals[1], }, - ] + ], } ) continue @@ -988,35 +999,33 @@ def _validate_value(self, value): raise ValueError("invalid value type %s" % opt_type) if opt_type == "key_val": if "opt_val" not in value or "opt_key" not in value: - raise ValueError( - "key_val option type requires " - "opt_key key and opt_val") + raise ValueError("key_val option type requires opt_key key and opt_val") elif opt_type == "single": if "opt_val" not in value: - raise ValueError( - "single option type requires opt_val") + raise ValueError("single option type requires opt_val") else: raise ValueError("unknown option type: %s" % opt_type) def set_option(self, option, value): - """Replaces the value of an option completely - """ + """Replaces the value of an option completely""" self._validate_value(value) opt_found = False for opt in self._parsed: if opt.get("option_name") == option: opt_found = True - opt["option_value"] = [value, ] + opt["option_value"] = [ + value, + ] break if not opt_found: - self._parsed.append({ - "type": "option", - "quoted": True, - "option_name": option, - "option_value": [ - value - ], - }) + self._parsed.append( + { + "type": "option", + "quoted": True, + "option_name": option, + "option_value": [value], + } + ) def append_to_option(self, option, value): """Appends a value to the specified option. If we're passing @@ -1032,27 +1041,25 @@ def append_to_option(self, option, value): opt_found = True found = False for val in opt["option_value"]: - if (val["opt_type"] == "key_val" and - value["opt_type"] == "key_val"): + if val["opt_type"] == "key_val" and value["opt_type"] == "key_val": if str(val["opt_key"]) == str(value["opt_key"]): val["opt_val"] = value["opt_val"] found = True - elif (val["opt_type"] == "single" and - value["opt_type"] == "single"): + elif val["opt_type"] == "single" and value["opt_type"] == "single": if str(val["opt_val"]) == str(value["opt_val"]): found = True if not found: opt["option_value"].append(value) break if not opt_found: - self._parsed.append({ - "type": "option", - "quoted": True, - "option_name": option, - "option_value": [ - value - ], - }) + self._parsed.append( + { + "type": "option", + "quoted": True, + "option_name": option, + "option_value": [value], + } + ) def dump(self): """dumps the contents of the file""" diff --git a/coriolis/worker/rpc/client.py b/coriolis/worker/rpc/client.py index 7625d2dd..db9ba3a7 100644 --- a/coriolis/worker/rpc/client.py +++ b/coriolis/worker/rpc/client.py @@ -1,11 +1,10 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -from oslo_config import cfg import oslo_messaging as messaging +from oslo_config import cfg -from coriolis import constants -from coriolis import rpc +from coriolis import constants, rpc VERSION = "1.0" @@ -13,7 +12,9 @@ worker_opts = [ cfg.IntOpt( "worker_rpc_timeout", - help="Number of seconds until RPC calls to the worker timeout.")] + help="Number of seconds until RPC calls to the worker timeout.", + ) +] CONF = cfg.CONF CONF.register_opts(worker_opts, 'worker') @@ -21,148 +22,191 @@ class WorkerClient(rpc.BaseRPCClient): def __init__( - self, timeout=None, host=None, - base_worker_topic=constants.WORKER_MAIN_MESSAGING_TOPIC): + self, + timeout=None, + host=None, + base_worker_topic=constants.WORKER_MAIN_MESSAGING_TOPIC, + ): topic = base_worker_topic if host is not None: - topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ({ - "main_topic": base_worker_topic, - "host": host}) + topic = constants.SERVICE_MESSAGING_TOPIC_FORMAT % ( + {"main_topic": base_worker_topic, "host": host} + ) target = messaging.Target(topic=topic, version=VERSION) if timeout is None: timeout = CONF.worker.worker_rpc_timeout - super(WorkerClient, self).__init__( - target, timeout=timeout) + super(WorkerClient, self).__init__(target, timeout=timeout) @classmethod - def from_service_definition( - cls, service, timeout=None, topic_override=None): + def from_service_definition(cls, service, timeout=None, topic_override=None): if service.get('topic') != constants.WORKER_MAIN_MESSAGING_TOPIC: raise ValueError( "Unknown topic '%s' for worker service client. Only " - "acceptable value is '%s': %s" % ( - service.get('topic'), - constants.WORKER_MAIN_MESSAGING_TOPIC, - service)) + "acceptable value is '%s': %s" + % (service.get('topic'), constants.WORKER_MAIN_MESSAGING_TOPIC, service) + ) topic = constants.WORKER_MAIN_MESSAGING_TOPIC if topic_override: topic = topic_override - return cls( - timeout=timeout, base_worker_topic=topic, - host=service.get('host')) + return cls(timeout=timeout, base_worker_topic=topic, host=service.get('host')) - def begin_task(self, ctxt, task_id, task_type, origin, destination, - instance, task_info): + def begin_task( + self, ctxt, task_id, task_type, origin, destination, instance, task_info + ): self._cast( - ctxt, 'exec_task', task_id=task_id, task_type=task_type, - origin=origin, destination=destination, instance=instance, - task_info=task_info, report_to_conductor=True) + ctxt, + 'exec_task', + task_id=task_id, + task_type=task_type, + origin=origin, + destination=destination, + instance=instance, + task_info=task_info, + report_to_conductor=True, + ) - def run_task(self, ctxt, task_id, task_type, origin, destination, - instance, task_info): + def run_task( + self, ctxt, task_id, task_type, origin, destination, instance, task_info + ): return self._call( - ctxt, 'exec_task', task_id=task_id, task_type=task_type, - origin=origin, destination=destination, instance=instance, - task_info=task_info, report_to_conductor=False) + ctxt, + 'exec_task', + task_id=task_id, + task_type=task_type, + origin=origin, + destination=destination, + instance=instance, + task_info=task_info, + report_to_conductor=False, + ) def cancel_task(self, ctxt, task_id, process_id, force): return self._call( - ctxt, 'cancel_task', - task_id=task_id, process_id=process_id, force=force) + ctxt, 'cancel_task', task_id=task_id, process_id=process_id, force=force + ) - def get_endpoint_instances(self, ctxt, platform_name, connection_info, - source_environment, marker=None, limit=None, - instance_name_pattern=None, refresh=False): + def get_endpoint_instances( + self, + ctxt, + platform_name, + connection_info, + source_environment, + marker=None, + limit=None, + instance_name_pattern=None, + refresh=False, + ): return self._call( - ctxt, 'get_endpoint_instances', + ctxt, + 'get_endpoint_instances', platform_name=platform_name, connection_info=connection_info, source_environment=source_environment, marker=marker, limit=limit, instance_name_pattern=instance_name_pattern, - refresh=refresh) + refresh=refresh, + ) - def get_endpoint_instance(self, ctxt, platform_name, connection_info, - source_environment, instance_name): + def get_endpoint_instance( + self, ctxt, platform_name, connection_info, source_environment, instance_name + ): return self._call( - ctxt, 'get_endpoint_instance', + ctxt, + 'get_endpoint_instance', platform_name=platform_name, connection_info=connection_info, source_environment=source_environment, - instance_name=instance_name) + instance_name=instance_name, + ) def get_endpoint_destination_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): return self._call( - ctxt, 'get_endpoint_destination_options', + ctxt, + 'get_endpoint_destination_options', platform_name=platform_name, connection_info=connection_info, env=env, - option_names=option_names) + option_names=option_names, + ) def get_endpoint_source_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): return self._call( - ctxt, 'get_endpoint_source_options', + ctxt, + 'get_endpoint_source_options', platform_name=platform_name, connection_info=connection_info, env=env, - option_names=option_names) + option_names=option_names, + ) - def get_endpoint_networks( - self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): return self._call( - ctxt, 'get_endpoint_networks', + ctxt, + 'get_endpoint_networks', platform_name=platform_name, connection_info=connection_info, - env=env) + env=env, + ) - def validate_endpoint_connection(self, ctxt, platform_name, - connection_info): + def validate_endpoint_connection(self, ctxt, platform_name, connection_info): return self._call( - ctxt, 'validate_endpoint_connection', + ctxt, + 'validate_endpoint_connection', platform_name=platform_name, - connection_info=connection_info) + connection_info=connection_info, + ) - def validate_endpoint_target_environment( - self, ctxt, platform_name, target_env): + def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): return self._call( - ctxt, 'validate_endpoint_target_environment', + ctxt, + 'validate_endpoint_target_environment', platform_name=platform_name, - target_env=target_env) + target_env=target_env, + ) - def validate_endpoint_source_environment( - self, ctxt, platform_name, source_env): + def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): return self._call( - ctxt, 'validate_endpoint_source_environment', + ctxt, + 'validate_endpoint_source_environment', platform_name=platform_name, - source_env=source_env) + source_env=source_env, + ) def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): return self._call( - ctxt, 'get_endpoint_storage', + ctxt, + 'get_endpoint_storage', platform_name=platform_name, connection_info=connection_info, - env=env) + env=env, + ) def get_endpoint_inventory_csv( - self, ctxt, platform_name, connection_info, source_environment): + self, ctxt, platform_name, connection_info, source_environment + ): return self._call( - ctxt, 'get_endpoint_inventory_csv', + ctxt, + 'get_endpoint_inventory_csv', platform_name=platform_name, connection_info=connection_info, - source_environment=source_environment) + source_environment=source_environment, + ) def get_available_providers(self, ctxt): - return self._call( - ctxt, 'get_available_providers') + return self._call(ctxt, 'get_available_providers') def get_provider_schemas(self, ctxt, platform_name, provider_type): return self._call( - ctxt, 'get_provider_schemas', + ctxt, + 'get_provider_schemas', platform_name=platform_name, - provider_type=provider_type) + provider_type=provider_type, + ) def get_diagnostics(self, ctxt): return self._call(ctxt, 'get_diagnostics') @@ -171,33 +215,54 @@ def get_service_status(self, ctxt): return self._call(ctxt, 'get_service_status') def get_endpoint_source_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): return self._call( - ctxt, 'get_endpoint_source_minion_pool_options', - platform_name=platform_name, connection_info=connection_info, - env=env, option_names=option_names) + ctxt, + 'get_endpoint_source_minion_pool_options', + platform_name=platform_name, + connection_info=connection_info, + env=env, + option_names=option_names, + ) def get_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): return self._call( - ctxt, 'get_endpoint_destination_minion_pool_options', - platform_name=platform_name, connection_info=connection_info, - env=env, option_names=option_names) + ctxt, + 'get_endpoint_destination_minion_pool_options', + platform_name=platform_name, + connection_info=connection_info, + env=env, + option_names=option_names, + ) def validate_endpoint_source_minion_pool_options( - self, ctxt, platform_name, pool_environment): + self, ctxt, platform_name, pool_environment + ): return self._call( - ctxt, 'validate_endpoint_source_minion_pool_options', - platform_name=platform_name, pool_environment=pool_environment) + ctxt, + 'validate_endpoint_source_minion_pool_options', + platform_name=platform_name, + pool_environment=pool_environment, + ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, pool_environment): + self, ctxt, platform_name, pool_environment + ): return self._call( - ctxt, 'validate_endpoint_destination_minion_pool_options', - platform_name=platform_name, pool_environment=pool_environment) + ctxt, + 'validate_endpoint_destination_minion_pool_options', + platform_name=platform_name, + pool_environment=pool_environment, + ) - def execute_auto_deployment( - self, ctxt, transfer_id, deployer_execution, **kwargs): + def execute_auto_deployment(self, ctxt, transfer_id, deployer_execution, **kwargs): self._cast( - ctxt, 'execute_auto_deployment', transfer_id=transfer_id, - deployer_execution=deployer_execution, **kwargs) + ctxt, + 'execute_auto_deployment', + transfer_id=transfer_id, + deployer_execution=deployer_execution, + **kwargs, + ) diff --git a/coriolis/worker/rpc/server.py b/coriolis/worker/rpc/server.py index cae63b6f..e0676d04 100644 --- a/coriolis/worker/rpc/server.py +++ b/coriolis/worker/rpc/server.py @@ -1,7 +1,6 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. -from logging import handlers import multiprocessing import os import shutil @@ -9,23 +8,19 @@ import sys import threading import time +from logging import handlers +import psutil from oslo_config import cfg from oslo_log import log as logging -import psutil from six.moves import queue +from coriolis import constants, context, exception, schemas, service, utils from coriolis.conductor.rpc import client as rpc_conductor_client from coriolis.conductor.rpc import utils as conductor_rpc_utils -from coriolis import constants -from coriolis import context -from coriolis import exception from coriolis.minion_manager.rpc import client as rpc_minion_manager_client from coriolis.providers import factory as providers_factory -from coriolis import schemas -from coriolis import service from coriolis.tasks import factory as task_runners_factory -from coriolis import utils CONF = cfg.CONF CONF.register_opts([], 'worker') @@ -40,9 +35,9 @@ def _get_event_handler_for_task_type(task_type, ctxt, task_object_id): if task_type in constants.MINION_POOL_OPERATIONS_TASKS: return rpc_minion_manager_client.MinionManagerPoolRpcEventHandler( - ctxt, task_object_id) - return rpc_conductor_client.ConductorTaskRpcEventHandler( - ctxt, task_object_id) + ctxt, task_object_id + ) + return rpc_conductor_client.ConductorTaskRpcEventHandler(ctxt, task_object_id) class WorkerServerEndpoint(object): @@ -60,8 +55,7 @@ def _rpc_conductor_client(self): # process" as well as forking child processes, it is safest to # re-instantiate the client every time: if self._rpc_conductor_client_instance is None: - self._rpc_conductor_client_instance = ( - rpc_conductor_client.ConductorClient()) + self._rpc_conductor_client_instance = rpc_conductor_client.ConductorClient() return self._rpc_conductor_client_instance def _register_worker_service(self): @@ -71,18 +65,29 @@ def _register_worker_service(self): # TODO(aznashwan): we should ideally have a dedicated # user/pass/tenant just for service registration. # Either way, these values are not used and thus redundant. - "coriolis", "admin") + "coriolis", + "admin", + ) status = self.get_service_status(dummy_context) service_registration = ( conductor_rpc_utils.check_create_registration_for_service( # NOTE: considering this only runs once on startup, we # instantiate a fresh conductor client instance for it: - rpc_conductor_client.ConductorClient(), dummy_context, host, - binary, constants.WORKER_MAIN_MESSAGING_TOPIC, enabled=True, - providers=status['providers'], specs=status['specs'])) + rpc_conductor_client.ConductorClient(), + dummy_context, + host, + binary, + constants.WORKER_MAIN_MESSAGING_TOPIC, + enabled=True, + providers=status['providers'], + specs=status['specs'], + ) + ) LOG.info( "Worker service is successfully registered with the following " - "parameters: %s", service_registration) + "parameters: %s", + service_registration, + ) self._service_registration = service_registration return service_registration @@ -101,18 +106,19 @@ def get_service_status(self, ctxt): "binary": diagnostics["application"], "topic": constants.WORKER_MAIN_MESSAGING_TOPIC, "providers": self.get_available_providers(ctxt), - "specs": diagnostics + "specs": diagnostics, } return status def cancel_task(self, ctxt, task_id, process_id, force): LOG.debug( - "Received request to cancel task '%s' (process %s)", - task_id, process_id) + "Received request to cancel task '%s' (process %s)", task_id, process_id + ) if not force and os.name == "nt": - LOG.warn("Windows does not support SIGINT, performing a " - "forced task termination") + LOG.warn( + "Windows does not support SIGINT, performing a forced task termination" + ) force = True try: @@ -128,11 +134,10 @@ def cancel_task(self, ctxt, task_id, process_id, force): msg = ( "Unable to find process '%s' for task '%s' for cancellation. " "Presuming it was already canceled or had " - "completed/error'd." % ( - process_id, task_id)) + "completed/error'd." % (process_id, task_id) + ) LOG.error(msg) - self._rpc_conductor_client.confirm_task_cancellation( - ctxt, task_id, msg) + self._rpc_conductor_client.confirm_task_cancellation(ctxt, task_id, msg) def _handle_mp_log_events(self, p, mp_log_q): while True: @@ -156,9 +161,8 @@ def _get_custom_ld_path(self, original_ld_path, extra_library_paths): else: return "%s:%s" % (original_ld_path, extra_libdirs) - def _start_process_with_custom_library_paths( - self, process, extra_library_paths): - """ Given a process instance, this method will add any shared libs + def _start_process_with_custom_library_paths(self, process, extra_library_paths): + """Given a process instance, this method will add any shared libs needed by the origin/destination provider plugins to the 'LD_LIBRARY_PATH' env variable and start the process. This method will always restore the 'LD_LIBRARY_PATH' to the @@ -174,30 +178,34 @@ def _start_process_with_custom_library_paths( original_ld_path = os.environ.get('LD_LIBRARY_PATH', "") LOG.debug( "Starting new worker process with extra libraries: '%s'", - extra_library_paths) + extra_library_paths, + ) try: os.environ['LD_LIBRARY_PATH'] = self._get_custom_ld_path( - original_ld_path, extra_library_paths) + original_ld_path, extra_library_paths + ) process.start() except TypeError: LOG.warning( "Failed to set extra library paths: %s. Error was: %s", - extra_library_paths, utils.get_exception_details()) + extra_library_paths, + utils.get_exception_details(), + ) finally: os.environ['LD_LIBRARY_PATH'] = original_ld_path def _get_extra_library_paths_for_providers( - self, ctxt, task_id, task_type, origin, destination): - """ Returns a list of strings with paths on the worker with shared + self, ctxt, task_id, task_type, origin, destination + ): + """Returns a list of strings with paths on the worker with shared libraries needed by the source/destination providers. """ - event_handler = _get_event_handler_for_task_type( - task_type, ctxt, task_id) - task_runner = task_runners_factory.get_task_runner_class( - task_type)() + event_handler = _get_event_handler_for_task_type(task_type, ctxt, task_id) + task_runner = task_runners_factory.get_task_runner_class(task_type)() return task_runner.get_shared_libs_for_providers( - ctxt, origin, destination, event_handler) + ctxt, origin, destination, event_handler + ) def _wait_for_process(self, p, mp_q): result = None @@ -215,57 +223,78 @@ def _wait_for_process(self, p, mp_q): except BaseException: pass break - time.sleep(.2) + time.sleep(0.2) return result def _exec_task_process( - self, ctxt, task_id, task_type, origin, destination, instance, - task_info, report_to_conductor=True): + self, + ctxt, + task_id, + task_type, + origin, + destination, + instance, + task_info, + report_to_conductor=True, + ): mp_ctx = multiprocessing.get_context('spawn') mp_q = mp_ctx.Queue() mp_log_q = mp_ctx.Queue() p = mp_ctx.Process( target=_task_process, - args=(ctxt, task_id, task_type, origin, destination, instance, - task_info, mp_q, mp_log_q)) + args=( + ctxt, + task_id, + task_type, + origin, + destination, + instance, + task_info, + mp_q, + mp_log_q, + ), + ) extra_library_paths = self._get_extra_library_paths_for_providers( - ctxt, task_id, task_type, origin, destination) + ctxt, task_id, task_type, origin, destination + ) try: if report_to_conductor: LOG.debug( - "Attempting to set task host on Conductor for task '%s'.", - task_id) - self._rpc_conductor_client.set_task_host( - ctxt, task_id, self._server) - LOG.debug( - "Attempting to start process for task with ID '%s'", task_id) - self._start_process_with_custom_library_paths( - p, extra_library_paths) + "Attempting to set task host on Conductor for task '%s'.", task_id + ) + self._rpc_conductor_client.set_task_host(ctxt, task_id, self._server) + LOG.debug("Attempting to start process for task with ID '%s'", task_id) + self._start_process_with_custom_library_paths(p, extra_library_paths) LOG.info("Task process started: %s", task_id) if report_to_conductor: LOG.debug( - "Attempting to set task process on Conductor " - "for task '%s'.", - task_id) - self._rpc_conductor_client.set_task_process( - ctxt, task_id, p.pid) + "Attempting to set task process on Conductor for task '%s'.", + task_id, + ) + self._rpc_conductor_client.set_task_process(ctxt, task_id, p.pid) LOG.debug( "Successfully started and reported task process for task " - "with ID '%s' (PID %d)", task_id, p.pid) + "with ID '%s' (PID %d)", + task_id, + p.pid, + ) except (Exception, KeyboardInterrupt) as ex: LOG.debug( - "Exception occurred whilst setting host for task '%s'. Error " - "was: %s", task_id, utils.get_exception_details()) + "Exception occurred whilst setting host for task '%s'. Error was: %s", + task_id, + utils.get_exception_details(), + ) # NOTE: because the task error classes are wrapped, # it's easiest to just check that the messages align: - cancelling_msg = ( - exception.TASK_ALREADY_CANCELLING_EXCEPTION_FMT % { - "task_id": task_id}) + cancelling_msg = exception.TASK_ALREADY_CANCELLING_EXCEPTION_FMT % { + "task_id": task_id + } if cancelling_msg in str(ex): raise exception.TaskIsCancelling( - "Task '%s' was already in cancelling status." % task_id) + "Task '%s' was already in cancelling status." % task_id + ) raise # TODO(lpetrut): one logger thread per subprocess may be excessive when @@ -276,9 +305,7 @@ def _exec_task_process( # Note that asyncio coroutines can't directly consume multiprocessing # queues, we'd probably need pipes instead. There's also the option of # using select/poll/epoll directly. - utils.start_thread( - target=self._handle_mp_log_events, - args=(p, mp_log_q)) + utils.start_thread(target=self._handle_mp_log_events, args=(p, mp_log_q)) result = self._wait_for_process(p, mp_q) p.join() @@ -287,42 +314,66 @@ def _exec_task_process( LOG.debug( "No result from process (%s) running task '%s'. " "Presuming task was cancelled.", - p.pid, task_id) - raise exception.TaskProcessCanceledException( - "Task was canceled.") + p.pid, + task_id, + ) + raise exception.TaskProcessCanceledException("Task was canceled.") if isinstance(result, str): LOG.debug( - "Error message while running task '%s' on process " - "with PID '%s': %s", task_id, p.pid, result) + "Error message while running task '%s' on process with PID '%s': %s", + task_id, + p.pid, + result, + ) raise exception.TaskProcessException(result) return result - def exec_task(self, ctxt, task_id, task_type, origin, destination, - instance, task_info, report_to_conductor=True): + def exec_task( + self, + ctxt, + task_id, + task_type, + origin, + destination, + instance, + task_info, + report_to_conductor=True, + ): try: task_result = self._exec_task_process( - ctxt, task_id, task_type, origin, destination, - instance, task_info, report_to_conductor=report_to_conductor) + ctxt, + task_id, + task_type, + origin, + destination, + instance, + task_info, + report_to_conductor=report_to_conductor, + ) LOG.info( "Output of completed %s task with ID %s: %s", - task_type, task_id, - utils.sanitize_task_info(task_result)) + task_type, + task_id, + utils.sanitize_task_info(task_result), + ) if not report_to_conductor: return task_result - self._rpc_conductor_client.task_completed( - ctxt, task_id, task_result) + self._rpc_conductor_client.task_completed(ctxt, task_id, task_result) except exception.TaskProcessCanceledException as ex: if report_to_conductor: LOG.debug( "Task with ID '%s' appears to have been cancelled. " "Confirming cancellation to Conductor now. Error was: %s", - task_id, utils.get_exception_details()) + task_id, + utils.get_exception_details(), + ) LOG.exception(ex) self._rpc_conductor_client.confirm_task_cancellation( - ctxt, task_id, str(ex)) + ctxt, task_id, str(ex) + ) else: raise except exception.NoSuitableWorkerServiceError: @@ -330,7 +381,9 @@ def exec_task(self, ctxt, task_id, task_type, origin, destination, LOG.warn( "A conductor-side scheduling error has occurred following " "the completion of task '%s'. Ignoring. Error was: %s", - task_id, utils.get_exception_details()) + task_id, + utils.get_exception_details(), + ) else: raise except Exception as ex: @@ -338,196 +391,226 @@ def exec_task(self, ctxt, task_id, task_type, origin, destination, LOG.debug( "Task with ID '%s' has error'd out. Reporting error to " "Conductor now. Error was: %s", - task_id, utils.get_exception_details()) + task_id, + utils.get_exception_details(), + ) LOG.exception(ex) - self._rpc_conductor_client.set_task_error( - ctxt, task_id, str(ex)) + self._rpc_conductor_client.set_task_error(ctxt, task_id, str(ex)) else: raise - def get_endpoint_instances(self, ctxt, platform_name, connection_info, - source_environment, marker, limit, - instance_name_pattern, refresh=False): + def get_endpoint_instances( + self, + ctxt, + platform_name, + connection_info, + source_environment, + marker, + limit, + instance_name_pattern, + refresh=False, + ): export_provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) instances_info = export_provider.get_instances( - ctxt, secret_connection_info, source_environment, - last_seen_id=marker, limit=limit, - instance_name_pattern=instance_name_pattern, refresh=refresh) + ctxt, + secret_connection_info, + source_environment, + last_seen_id=marker, + limit=limit, + instance_name_pattern=instance_name_pattern, + refresh=refresh, + ) for instance_info in instances_info: schemas.validate_value( - instance_info, schemas.CORIOLIS_VM_INSTANCE_INFO_SCHEMA) + instance_info, schemas.CORIOLIS_VM_INSTANCE_INFO_SCHEMA + ) return instances_info - def get_endpoint_instance(self, ctxt, platform_name, connection_info, - source_environment, instance_name): + def get_endpoint_instance( + self, ctxt, platform_name, connection_info, source_environment, instance_name + ): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) instance_info = provider.get_instance( - ctxt, secret_connection_info, source_environment, - instance_name) + ctxt, secret_connection_info, source_environment, instance_name + ) - schemas.validate_value( - instance_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) + schemas.validate_value(instance_info, schemas.CORIOLIS_VM_EXPORT_INFO_SCHEMA) return instance_info def get_endpoint_destination_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS, - None, raise_if_not_found=False) + None, + raise_if_not_found=False, + ) if not provider: raise exception.InvalidInput( "Provider plugin for platform '%s' does not support listing " - "destination environment options." % platform_name) + "destination environment options." % platform_name + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) options = provider.get_target_environment_options( - ctxt, secret_connection_info, env=env, option_names=option_names) + ctxt, secret_connection_info, env=env, option_names=option_names + ) schemas.validate_value( - options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA) + options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA + ) return options def get_endpoint_source_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_SOURCE_MINION_POOL, - None, raise_if_not_found=False) + None, + raise_if_not_found=False, + ) if not provider: raise exception.InvalidInput( "Provider plugin for platform '%s' does not support source " - "minion pool creation or management." % platform_name) + "minion pool creation or management." % platform_name + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) options = provider.get_minion_pool_options( - ctxt, secret_connection_info, env=env, - option_names=option_names) + ctxt, secret_connection_info, env=env, option_names=option_names + ) # NOTE: the structure of option values is the same for minion pools: schemas.validate_value( - options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA) + options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA + ) return options def get_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, - None, raise_if_not_found=False) + None, + raise_if_not_found=False, + ) if not provider: raise exception.InvalidInput( "Provider plugin for platform '%s' does not support " - "destination minion pool creation or management." % ( - platform_name)) + "destination minion pool creation or management." % (platform_name) + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) options = provider.get_minion_pool_options( - ctxt, secret_connection_info, env=env, - option_names=option_names) + ctxt, secret_connection_info, env=env, option_names=option_names + ) # NOTE: the structure of option values is the same for minion pools: schemas.validate_value( - options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA) + options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA + ) return options def get_endpoint_source_options( - self, ctxt, platform_name, connection_info, env, option_names): + self, ctxt, platform_name, connection_info, env, option_names + ): provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_SOURCE_ENDPOINT_OPTIONS, - None, raise_if_not_found=False) + None, + raise_if_not_found=False, + ) if not provider: raise exception.InvalidInput( "Provider plugin for platform '%s' does not support listing " - "source environment options." % platform_name) + "source environment options." % platform_name + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) options = provider.get_source_environment_options( - ctxt, secret_connection_info, env=env, option_names=option_names) + ctxt, secret_connection_info, env=env, option_names=option_names + ) schemas.validate_value( - options, schemas.CORIOLIS_SOURCE_ENVIRONMENT_OPTIONS_SCHEMA) + options, schemas.CORIOLIS_SOURCE_ENVIRONMENT_OPTIONS_SCHEMA + ) return options - def get_endpoint_networks( - self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): env = env or {} provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_ENDPOINT_NETWORKS, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT_NETWORKS, None + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) - networks_info = provider.get_networks( - ctxt, secret_connection_info, env) + networks_info = provider.get_networks(ctxt, secret_connection_info, env) for network_info in networks_info: - schemas.validate_value( - network_info, schemas.CORIOLIS_VM_NETWORK_SCHEMA) + schemas.validate_value(network_info, schemas.CORIOLIS_VM_NETWORK_SCHEMA) return networks_info def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_ENDPOINT_STORAGE, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT_STORAGE, None + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) - storage = provider.get_storage( - ctxt, secret_connection_info, env) + storage = provider.get_storage(ctxt, secret_connection_info, env) schemas.validate_value(storage, schemas.CORIOLIS_VM_STORAGE_SCHEMA) return storage def get_endpoint_inventory_csv( - self, ctxt, platform_name, connection_info, source_environment): + self, ctxt, platform_name, connection_info, source_environment + ): provider = providers_factory.get_provider( - platform_name, - constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT, None + ) if not provider: raise exception.InvalidInput( "Provider plugin for platform '%s' does not support " - "VM inventory CSV export." % platform_name) + "VM inventory CSV export." % platform_name + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) return provider.export_instance_inventory( - ctxt, secret_connection_info, source_environment) + ctxt, secret_connection_info, source_environment + ) def get_available_providers(self, ctxt): return providers_factory.get_available_providers() - def validate_endpoint_target_environment( - self, ctxt, platform_name, target_env): + def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_OS_MORPHING, None) + platform_name, constants.PROVIDER_TYPE_OS_MORPHING, None + ) target_env_schema = provider.get_target_environment_schema() is_valid = True @@ -541,10 +624,10 @@ def validate_endpoint_target_environment( return (is_valid, message) - def validate_endpoint_source_environment( - self, ctxt, platform_name, source_env): + def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_TRANSFER_EXPORT, None) + platform_name, constants.PROVIDER_TYPE_TRANSFER_EXPORT, None + ) source_env_schema = provider.get_source_environment_schema() is_valid = True @@ -559,9 +642,11 @@ def validate_endpoint_source_environment( return (is_valid, message) def validate_endpoint_source_minion_pool_options( - self, ctxt, platform_name, pool_environment): + self, ctxt, platform_name, pool_environment + ): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_SOURCE_MINION_POOL, None) + platform_name, constants.PROVIDER_TYPE_SOURCE_MINION_POOL, None + ) pool_options_schema = provider.get_minion_pool_environment_schema() is_valid = True @@ -576,10 +661,11 @@ def validate_endpoint_source_minion_pool_options( return (is_valid, message) def validate_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, pool_environment): + self, ctxt, platform_name, pool_environment + ): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, - None) + platform_name, constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, None + ) pool_options_schema = provider.get_minion_pool_environment_schema() is_valid = True @@ -593,19 +679,19 @@ def validate_endpoint_destination_minion_pool_options( return (is_valid, message) - def validate_endpoint_connection(self, ctxt, platform_name, - connection_info): + def validate_endpoint_connection(self, ctxt, platform_name, connection_info): provider = providers_factory.get_provider( - platform_name, constants.PROVIDER_TYPE_ENDPOINT, None) + platform_name, constants.PROVIDER_TYPE_ENDPOINT, None + ) - secret_connection_info = utils.get_secret_connection_info( - ctxt, connection_info) + secret_connection_info = utils.get_secret_connection_info(ctxt, connection_info) is_valid = True message = None try: schemas.validate_value( - secret_connection_info, provider.get_connection_info_schema()) + secret_connection_info, provider.get_connection_info_schema() + ) provider.validate_connection(ctxt, secret_connection_info) except exception.SchemaValidationException as ex: LOG.debug("Connection info schema validation failed: %s", ex) @@ -614,8 +700,8 @@ def validate_endpoint_connection(self, ctxt, platform_name, "Schema validation for the provided connection parameters has " "failed. Please ensure that you have included all the " "necessary connection parameters and they are all properly " - "formatted for the '%s' Coriolis plugin in use." % - (platform_name)) + "formatted for the '%s' Coriolis plugin in use." % (platform_name) + ) except exception.ConnectionValidationException as ex: LOG.warn(utils.get_exception_details()) is_valid = False @@ -623,15 +709,14 @@ def validate_endpoint_connection(self, ctxt, platform_name, except BaseException as ex: LOG.warn(utils.get_exception_details()) is_valid = False - message = ( - "An unexpected connection validation exception " - "ocurred: %s" % str(ex)) + message = "An unexpected connection validation exception ocurred: %s" % str( + ex + ) return (is_valid, message) def get_provider_schemas(self, ctxt, platform_name, provider_type): - provider = providers_factory.get_provider( - platform_name, provider_type, None) + provider = providers_factory.get_provider(platform_name, provider_type, None) schemas = {} @@ -674,27 +759,32 @@ def _setup_task_process(mp_log_q): log_root.addHandler(handlers.QueueHandler(mp_log_q)) -def _task_process(ctxt, task_id, task_type, origin, destination, instance, - task_info, mp_q, mp_log_q): +def _task_process( + ctxt, task_id, task_type, origin, destination, instance, task_info, mp_q, mp_log_q +): try: _setup_task_process(mp_log_q) - task_runner = task_runners_factory.get_task_runner_class( - task_type)() - event_handler = _get_event_handler_for_task_type( - task_type, ctxt, task_id) + task_runner = task_runners_factory.get_task_runner_class(task_type)() + event_handler = _get_event_handler_for_task_type(task_type, ctxt, task_id) - LOG.debug("Executing task: %(task_id)s, type: %(task_type)s, " - "origin: %(origin)s, destination: %(destination)s, " - "instance: %(instance)s, task_info: %(task_info)s", - {"task_id": task_id, "task_type": task_type, - "origin": origin, "destination": destination, - "instance": instance, - "task_info": utils.sanitize_task_info( - task_info)}) + LOG.debug( + "Executing task: %(task_id)s, type: %(task_type)s, " + "origin: %(origin)s, destination: %(destination)s, " + "instance: %(instance)s, task_info: %(task_info)s", + { + "task_id": task_id, + "task_type": task_type, + "origin": origin, + "destination": destination, + "instance": instance, + "task_info": utils.sanitize_task_info(task_info), + }, + ) task_result = task_runner.run( - ctxt, instance, origin, destination, task_info, event_handler) + ctxt, instance, origin, destination, task_info, event_handler + ) # mq_p.put() doesn't raise if new_task_info is not serializable utils.is_serializable(task_result) mp_q.put(task_result) diff --git a/coriolis/wsman.py b/coriolis/wsman.py index adb69cc2..ff6ded03 100644 --- a/coriolis/wsman.py +++ b/coriolis/wsman.py @@ -2,15 +2,13 @@ # All Rights Reserved. import base64 -import requests +import requests from oslo_log import log as logging - from winrm import exceptions as winrm_exceptions from winrm import protocol -from coriolis import exception -from coriolis import utils +from coriolis import exception, utils AUTH_BASIC = "basic" AUTH_KERBEROS = "kerberos" @@ -30,17 +28,20 @@ def __init__(self, timeout=None): EOL = "\r\n" @utils.retry_on_error() - def connect(self, url, username, auth=None, password=None, - cert_pem=None, cert_key_pem=None): + def connect( + self, url, username, auth=None, password=None, cert_pem=None, cert_key_pem=None + ): if not auth: if cert_pem: auth = AUTH_CERTIFICATE else: auth = AUTH_BASIC - auth_transport_map = {AUTH_BASIC: 'plaintext', - AUTH_KERBEROS: 'kerberos', - AUTH_CERTIFICATE: 'ssl'} + auth_transport_map = { + AUTH_BASIC: 'plaintext', + AUTH_KERBEROS: 'kerberos', + AUTH_CERTIFICATE: 'ssl', + } self._protocol = protocol.Protocol( endpoint=url, @@ -48,23 +49,25 @@ def connect(self, url, username, auth=None, password=None, username=username, password=password, cert_pem=cert_pem, - cert_key_pem=cert_key_pem) + cert_key_pem=cert_key_pem, + ) @classmethod def from_connection_info(cls, connection_info, timeout=DEFAULT_TIMEOUT): - """ Returns a wsman.WSManConnection obj for the provided conn info. """ + """Returns a wsman.WSManConnection obj for the provided conn info.""" if not isinstance(connection_info, dict): raise ValueError( - "WSMan connection must be a dict. Got type '%s', value: %s" % - (type(connection_info), - connection_info)) + "WSMan connection must be a dict. Got type '%s', value: %s" + % (type(connection_info), connection_info) + ) required_keys = ["ip", "username", "password"] missing = [key for key in required_keys if key not in connection_info] if missing: raise ValueError( "The following keys were missing from WSMan connection " - "info %s. Got: %s" % (missing, connection_info)) + "info %s. Got: %s" % (missing, connection_info) + ) host = connection_info["ip"] port = connection_info.get("port", 5986) @@ -76,13 +79,20 @@ def from_connection_info(cls, connection_info, timeout=DEFAULT_TIMEOUT): LOG.info("Connection info: %s", str(connection_info)) - LOG.info("Waiting for connectivity on host: %(host)s:%(port)s", - {"host": host, "port": port}) + LOG.info( + "Waiting for connectivity on host: %(host)s:%(port)s", + {"host": host, "port": port}, + ) utils.wait_for_port_connectivity(host, port) conn = cls(timeout) - conn.connect(url=url, username=username, password=password, - cert_pem=cert_pem, cert_key_pem=cert_key_pem) + conn.connect( + url=url, + username=username, + password=password, + cert_pem=cert_pem, + cert_key_pem=cert_key_pem, + ) return conn @@ -95,8 +105,11 @@ def set_timeout(self, timeout): self._protocol.transport.timeout = timeout @utils.retry_on_error( - terminal_exceptions=[winrm_exceptions.InvalidCredentialsError, - exception.OSMorphingWinRMOperationTimeout]) + terminal_exceptions=[ + winrm_exceptions.InvalidCredentialsError, + exception.OSMorphingWinRMOperationTimeout, + ] + ) def _exec_command(self, cmd, args=[], timeout=None): timeout = int(timeout or self._conn_timeout) self.set_timeout(timeout) @@ -105,13 +118,13 @@ def _exec_command(self, cmd, args=[], timeout=None): shell_id = self._protocol.open_shell(codepage=CODEPAGE_UTF8) command_id = self._protocol.run_command(shell_id, cmd, args) try: - (std_out, - std_err, - exit_code) = self._protocol.get_command_output( - shell_id, command_id) + (std_out, std_err, exit_code) = self._protocol.get_command_output( + shell_id, command_id + ) except requests.exceptions.ReadTimeout: raise exception.OSMorphingWinRMOperationTimeout( - cmd=("%s %s" % (cmd, " ".join(args))), timeout=timeout) + cmd=("%s %s" % (cmd, " ".join(args))), timeout=timeout + ) finally: self._protocol.cleanup_command(shell_id, command_id) @@ -119,29 +132,30 @@ def _exec_command(self, cmd, args=[], timeout=None): except winrm_exceptions.InvalidCredentialsError as ex: raise exception.NotAuthorized( message="The WinRM connection credentials are invalid. " - "If you are using a template with a default " - "pre-baked username/password, please ensure " - "that you have passed the credentials to the " - "destination Coriolis plugin you have selected," - " either via the Target Environment parameters " - "set when creating the Migration/Replica, or " - "by setting it in the destination plugin's " - "dedicated section of the coriolis.conf " - "static configuration file.") from ex + "If you are using a template with a default " + "pre-baked username/password, please ensure " + "that you have passed the credentials to the " + "destination Coriolis plugin you have selected," + " either via the Target Environment parameters " + "set when creating the Migration/Replica, or " + "by setting it in the destination plugin's " + "dedicated section of the coriolis.conf " + "static configuration file." + ) from ex finally: if shell_id: self._protocol.close_shell(shell_id) def exec_command(self, cmd, args=[], timeout=None): LOG.debug("Executing WSMAN command: %s", str([cmd] + args)) - std_out, std_err, exit_code = self._exec_command( - cmd, args, timeout=timeout) + std_out, std_err, exit_code = self._exec_command(cmd, args, timeout=timeout) if exit_code: raise exception.CoriolisException( "Command \"%s\" failed with exit code: %s\n" - "stdout: %s\nstd_err: %s" % - (str([cmd] + args), exit_code, std_out, std_err)) + "stdout: %s\nstd_err: %s" + % (str([cmd] + args), exit_code, std_out, std_err) + ) return std_out @@ -149,16 +163,18 @@ def exec_ps_command(self, cmd, ignore_stdout=False, timeout=None): LOG.debug("Executing PS command: %s", cmd) base64_cmd = base64.b64encode(cmd.encode('utf-16le')).decode() return self.exec_command( - "powershell.exe", ["-EncodedCommand", base64_cmd], - timeout=timeout)[:-2] + "powershell.exe", ["-EncodedCommand", base64_cmd], timeout=timeout + )[:-2] def test_path(self, remote_path): ret_val = self.exec_ps_command("Test-Path -Path \"%s\"" % remote_path) return ret_val == "True" def download_file(self, url, remote_path): - LOG.debug("Downloading: \"%(url)s\" to \"%(path)s\"", - {"url": url, "path": remote_path}) + LOG.debug( + "Downloading: \"%(url)s\" to \"%(path)s\"", + {"url": url, "path": remote_path}, + ) try: # Nano Server does not have Invoke-WebRequest and additionally # this is also faster @@ -172,16 +188,18 @@ def download_file(self, url, remote_path): "GetStreamAsync('%(url)s').Result.CopyTo(" "(New-Object IO.FileStream '%(outfile)s', Create, Write, " "None), 1MB)" % {"url": url, "outfile": remote_path}, - ignore_stdout=True) + ignore_stdout=True, + ) except exception.CoriolisException as ex: LOG.trace(utils.get_exception_details()) raise exception.CoriolisException( "Failed to download file from URL: %s to path: %s. Please " - "check logs for more details." % ( - url, remote_path)) from ex + "check logs for more details." % (url, remote_path) + ) from ex def write_file(self, remote_path, content): self.exec_ps_command( "[IO.File]::WriteAllBytes('%s', [Convert]::FromBase64String('%s'))" % (remote_path, base64.b64encode(content).decode()), - ignore_stdout=True) + ignore_stdout=True, + ) diff --git a/setup.py b/setup.py index 1fac05ce..087cbfcb 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ import os import platform -import setuptools import subprocess +import setuptools + def _compile_and_install(): if platform.system() != "Linux": @@ -10,11 +11,8 @@ def _compile_and_install(): dirname = os.path.dirname(__file__) resources = os.path.join(dirname, "coriolis", "resources") if os.path.isdir(resources): - subprocess.check_call( - ["make"], cwd=resources, shell=True) + subprocess.check_call(["make"], cwd=resources, shell=True) _compile_and_install() -setuptools.setup( - setup_requires=['pbr>=1.8'], - pbr=True) +setuptools.setup(setup_requires=['pbr>=1.8'], pbr=True)