Merge "Improve unit test coverage for dcmanager's orchestrator/states/kube"

distributedcloud/dcmanager/tests/unit/orchestrator/states/kube/test_creating_vim_kube_upgrade_strategy.py

@@ -8,9 +8,9 @@ import mock
 
 from dccommon.drivers.openstack import vim
 from dcmanager.common import consts
-
 from dcmanager.tests.unit.common import fake_strategy
 from dcmanager.tests.unit.fakes import FakeVimStrategy
+from dcmanager.tests.unit.orchestrator.states.fakes import FakeKubeUpgrade
 from dcmanager.tests.unit.orchestrator.states.fakes import FakeKubeVersion
 from dcmanager.tests.unit.orchestrator.states.fakes \
     import PREVIOUS_KUBE_VERSION
@@ -53,16 +53,37 @@ KUBE_VERSION_LIST_SC_2 = [
                     state='active')
 ]
 
+KUBE_UPGRADE_LIST = [
+    FakeKubeUpgrade(
+        obj_id=1,
+        to_version='v1.2.5',
+        from_version='v1.2.4',
+        state='active'
+    )
+]
 
-class TestCreatingVIMKubeUpgradeStrategyStage(CreatingVIMStrategyStageMixin,
+KUBE_VERSION_LIST_WITHOUT_ACTIVE = [
-                                              TestKubeUpgradeState):
+    FakeKubeVersion(
+        obj_id=1,
+        version='v1.2.3',
+        target=True,
+        state='available'
+    )
+]
+
+
+class TestCreatingVIMKubeUpgradeStrategyStage(
+    CreatingVIMStrategyStageMixin, TestKubeUpgradeState
+):
     """Test a vim kube upgrade strategy during kube orchestration"""
 
     def setUp(self):
-        super(TestCreatingVIMKubeUpgradeStrategyStage, self).setUp()
+        super().setUp()
+
         self.set_state(
             consts.STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY,
-            consts.STRATEGY_STATE_KUBE_APPLYING_VIM_KUBE_UPGRADE_STRATEGY)
+            consts.STRATEGY_STATE_KUBE_APPLYING_VIM_KUBE_UPGRADE_STRATEGY
+        )
 
         # creating the vim strategy checks if an existing upgrade exists
         self.sysinv_client.get_kube_upgrades = mock.MagicMock()
@@ -81,8 +102,10 @@ class TestCreatingVIMKubeUpgradeStrategyStage(CreatingVIMStrategyStageMixin,
                             state='available'),
         ]
 
-    def test_creating_vim_strategy_success_highest_kube_version(self):
+    def mock_and_assert_step_update(
-        """Test creating a VIM strategy when selecting the highest kube version"""
+        self, is_upgrade=False, kube_version=None, kube_version_list=None
+    ):
+        """Encapsulates the required arrangements to run the tests"""
 
         # first api query is before the create
         # remaining api query results are waiting for the strategy to build
@@ -91,14 +114,22 @@ class TestCreatingVIMKubeUpgradeStrategyStage(CreatingVIMStrategyStageMixin,
             STRATEGY_BUILDING,
             STRATEGY_DONE_BUILDING,
         ]
-
         self.sysinv_client.get_kube_versions.return_value = KUBE_VERSION_LIST
 
-        extra_args = {"to-version": 'v1.2.5'}
+        if is_upgrade:
-        self.strategy = fake_strategy.create_fake_strategy(
+            self.sysinv_client.get_kube_upgrades.return_value = kube_version_list
-            self.ctx,
+            kube_version = kube_version_list[0].to_version
-            self.DEFAULT_STRATEGY_TYPE,
+
-            extra_args=extra_args)
+        if kube_version:
+            extra_args = {"to-version": kube_version}
+            self.strategy = fake_strategy.create_fake_strategy(
+                self.ctx,
+                self.DEFAULT_STRATEGY_TYPE,
+                extra_args=extra_args)
+        else:
+            kube_version = kube_version_list[0].version
+            self.sysinv_client.get_kube_versions.side_effect = \
+                [kube_version_list, KUBE_VERSION_LIST]
 
         # API calls acts as expected
         self.vim_client.create_strategy.return_value = STRATEGY_BUILDING
@@ -106,115 +137,55 @@ class TestCreatingVIMKubeUpgradeStrategyStage(CreatingVIMStrategyStageMixin,
         # invoke the strategy state operation on the orch thread
         self.worker.perform_state_action(self.strategy_step)
 
-        self.vim_client.create_strategy.assert_called_with('kube-upgrade',
+        self.vim_client.create_strategy.assert_called_with(
-                                                           'parallel',
+            'kube-upgrade', 'parallel', 'parallel', 10,
-                                                           'parallel',
+            'migrate', 'relaxed', to_version=kube_version
-                                                           10,
+        )
-                                                           'migrate',
-                                                           'relaxed',
-                                                           to_version='v1.2.5')
 
         # Successful promotion to next state
-        self.assert_step_updated(self.strategy_step.subcloud_id,
+        self.assert_step_updated(
-                                 self.on_success_state)
+            self.strategy_step.subcloud_id, self.on_success_state
+        )
 
-    def test_creating_vim_strategy_success_lowest_kube_version(self):
+    def test_strategy_succeeds_with_highest_kube_version(self):
-        """Test creating a VIM strategy when selecting the lowest kube version"""
+        """Test strategy succeeds when selecting the highest kube version"""
 
-        # first api query is before the create
+        self.mock_and_assert_step_update(kube_version='v1.2.5')
-        # remaining api query results are waiting for the strategy to build
-        self.vim_client.get_strategy.side_effect = [
-            None,
-            STRATEGY_BUILDING,
-            STRATEGY_DONE_BUILDING,
-        ]
 
-        self.sysinv_client.get_kube_versions.return_value = KUBE_VERSION_LIST
+    def test_strategy_succeeds_with_lowest_kube_version(self):
+        """Test strategy succeeds when selecting the lowest kube version"""
 
-        extra_args = {"to-version": 'v1.2.4'}
+        self.mock_and_assert_step_update(kube_version='v1.2.4')
-        self.strategy = fake_strategy.create_fake_strategy(
-            self.ctx,
-            self.DEFAULT_STRATEGY_TYPE,
-            extra_args=extra_args)
 
-        # API calls acts as expected
+    def test_strategy_succeeds_without_kube_version_selected(self):
-        self.vim_client.create_strategy.return_value = STRATEGY_BUILDING
+        """Test strategy succeeds without a selected kube_version"""
+
+        self.mock_and_assert_step_update(kube_version_list=KUBE_VERSION_LIST_SC)
+
+    def test_strategy_succeeds_when_sc_has_middle_version_active(self):
+        """Test strategy succeeds when sc has the middle version active"""
+
+        self.mock_and_assert_step_update(kube_version_list=KUBE_VERSION_LIST_SC_2)
+
+    def test_strategy_succeeds_with_subcloud_kube_upgrade(self):
+        """Test strategy succeeds when there are subcloud kube upgrades"""
+
+        self.mock_and_assert_step_update(
+            is_upgrade=True, kube_version_list=KUBE_UPGRADE_LIST
+        )
+
+    def test_strategy_fails_without_active_version_to_upgrade(self):
+        """Test upgrade fails without an active version to upgrade"""
+
+        fake_strategy.create_fake_strategy(
+            self.ctx, self.DEFAULT_STRATEGY_TYPE
+        )
+
+        self.sysinv_client.get_kube_versions.return_value = \
+            KUBE_VERSION_LIST_WITHOUT_ACTIVE
 
-        # invoke the strategy state operation on the orch thread
         self.worker.perform_state_action(self.strategy_step)
 
-        self.vim_client.create_strategy.assert_called_with('kube-upgrade',
+        self.assert_step_updated(
-                                                           'parallel',
+            self.strategy_step.subcloud_id, consts.STRATEGY_STATE_FAILED
-                                                           'parallel',
+        )
-                                                           10,
-                                                           'migrate',
-                                                           'relaxed',
-                                                           to_version='v1.2.4')
-
-        # Successful promotion to next state
-        self.assert_step_updated(self.strategy_step.subcloud_id,
-                                 self.on_success_state)
-
-    def test_creating_vim_strategy_success_no_kube_version_selected(self):
-        """Test creating a VIM strategy when --to-version is not provided"""
-
-        # first api query is before the create
-        # remaining api query results are waiting for the strategy to build
-        self.vim_client.get_strategy.side_effect = [
-            None,
-            STRATEGY_BUILDING,
-            STRATEGY_DONE_BUILDING,
-        ]
-
-        self.sysinv_client.get_kube_versions.side_effect = [KUBE_VERSION_LIST_SC,
-                                                            KUBE_VERSION_LIST]
-
-        # API calls acts as expectedlowest
-        self.vim_client.create_strategy.return_value = STRATEGY_BUILDING
-
-        # invoke the strategy state operation on the orch thread
-        self.worker.perform_state_action(self.strategy_step)
-
-        self.vim_client.create_strategy.assert_called_with('kube-upgrade',
-                                                           'parallel',
-                                                           'parallel',
-                                                           10,
-                                                           'migrate',
-                                                           'relaxed',
-                                                           to_version='v1.2.5')
-
-        # Successful promotion to next state
-        self.assert_step_updated(self.strategy_step.subcloud_id,
-                                 self.on_success_state)
-
-    def test_creating_vim_strategy_when_SC_has_middle_version_active(self):
-        """Test creating a VIM strategy when --to-version is not provided"""
-
-        # first api query is before the create
-        # remaining api query results are waiting for the strategy to build
-        self.vim_client.get_strategy.side_effect = [
-            None,
-            STRATEGY_BUILDING,
-            STRATEGY_DONE_BUILDING,
-        ]
-
-        self.sysinv_client.get_kube_versions.side_effect = [KUBE_VERSION_LIST_SC_2,
-                                                            KUBE_VERSION_LIST]
-
-        # API calls acts as expectedlowest
-        self.vim_client.create_strategy.return_value = STRATEGY_BUILDING
-
-        # invoke the strategy state operation on the orch thread
-        self.worker.perform_state_action(self.strategy_step)
-
-        self.vim_client.create_strategy.assert_called_with('kube-upgrade',
-                                                           'parallel',
-                                                           'parallel',
-                                                           10,
-                                                           'migrate',
-                                                           'relaxed',
-                                                           to_version='v1.2.4')
-
-        # Successful promotion to next state
-        self.assert_step_updated(self.strategy_step.subcloud_id,
-                                 self.on_success_state)

distributedcloud/dcmanager/tests/unit/orchestrator/states/kube/test_pre_check.py

@@ -13,7 +13,6 @@ from dcmanager.common.consts \
     import STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
 from dcmanager.common.consts import STRATEGY_STATE_KUBE_UPGRADE_PRE_CHECK
 from dcmanager.db.sqlalchemy import api as db_api
-
 from dcmanager.tests.unit.common import fake_strategy
 from dcmanager.tests.unit.orchestrator.states.fakes import FakeAlarm
 from dcmanager.tests.unit.orchestrator.states.fakes import FakeKubeUpgrade
@@ -60,34 +59,31 @@ KUBERNETES_UPGRADE_HEALTH_RESPONSE_NON_MGMT_AFFECTING_ALARM = \
     "All kubernetes control plane pods are ready: [OK]\n" \
     "All kubernetes applications are in a valid state: [OK]"
 
+KUBERNETES_UPGRADE_HEALTH_RESPONSE_MGMT_AFFECTING_AND_KUBERNETES_ALARM = \
+    "System Health:\n" \
+    "All hosts are provisioned: [OK]\n" \
+    "All hosts are unlocked/enabled: [OK]\n" \
+    "All hosts have current configurations: [OK]\n" \
+    "All hosts are patch current: [OK]\n" \
+    "No alarms: [Fail]\n" \
+    "[2] alarms found, [2] of which are management affecting\n" \
+    "All kubernetes nodes are ready: [Fail]\n" \
+    "All kubernetes control plane pods are ready: [OK]\n" \
+    "All kubernetes applications are in a valid state: [OK]"
+
 MEMORY_THRESHOLD_ALARM = FakeAlarm('100.101', 'True')
 KUBERNETES_UPGRADE_ALARM = FakeAlarm('900.007', 'True')
 CONFIG_OUT_OF_DATE_ALARM = FakeAlarm('250.001', 'False')
 
 KUBE_VERSION_LIST = [
-    FakeKubeVersion(obj_id=1,
+    FakeKubeVersion(obj_id=1, version='v1.2.3', target=True, state='active'),
-                    version='v1.2.3',
+    FakeKubeVersion(obj_id=2, version='v1.2.5', target=False, state='available'),
-                    target=True,
-                    state='active'),
-    FakeKubeVersion(obj_id=2,
-                    version='v1.2.5',
-                    target=False,
-                    state='available'),
 ]
 
 KUBE_VERSION_LIST_2 = [
-    FakeKubeVersion(obj_id=1,
+    FakeKubeVersion(obj_id=1, version='v1.2.3', target=True, state='active'),
-                    version='v1.2.3',
+    FakeKubeVersion(obj_id=2, version='v1.2.5', target=False, state='available'),
-                    target=True,
+    FakeKubeVersion(obj_id=3, version='v1.2.6', target=False, state='available'),
-                    state='active'),
-    FakeKubeVersion(obj_id=2,
-                    version='v1.2.5',
-                    target=False,
-                    state='available'),
-    FakeKubeVersion(obj_id=3,
-                    version='v1.2.6',
-                    target=False,
-                    state='available'),
 ]
 
 
@@ -123,6 +119,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         When a kube upgrade exists in the subcloud, do not skip, go to the
         next step, which is 'create the vim kube upgrade strategy'
         """
+
         next_state = STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -146,6 +143,31 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         # Verify the transition to the  expected next state
         self.assert_step_updated(self.strategy_step.subcloud_id, next_state)
 
+    def test_pre_check_succeeds_with_strategy_without_extra_args(self):
+        """Test pre-check succeeds with strategy without extra args"""
+
+        self.strategy = fake_strategy.create_fake_strategy(
+            self.ctx, self.DEFAULT_STRATEGY_TYPE
+        )
+        next_state = STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
+
+        db_api.subcloud_update(
+            self.ctx, self.subcloud.id, deploy_status=DEPLOY_STATE_DONE
+        )
+        self.sysinv_client.get_kube_upgrades.return_value = [FakeKubeUpgrade()]
+
+        self.sysinv_client.get_kube_versions.return_value = [
+            FakeKubeVersion(
+                obj_id=1, version=UPGRADED_KUBE_VERSION, target=True, state='active'
+            )
+        ]
+
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.sysinv_client.get_kube_versions.assert_called_once()
+
+        self.assert_step_updated(self.strategy_step.subcloud_id, next_state)
+
     def test_pre_check_subcloud_failed_health_check_with_management_alarms(self):
         """Test pre check step where subcloud has management affecting alarms"""
 
@@ -169,8 +191,31 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         self.assert_step_updated(self.strategy_step.subcloud_id,
                                  STRATEGY_STATE_FAILED)
 
+    def test_pre_check_subcloud_failed_health_check_with_mgmt_and_kubernetes_alarm(
+        self
+    ):
+        """Test pre check step where subcloud has management and kubernetes
+
+        affecting alarms
+        """
+
+        db_api.subcloud_update(
+            self.ctx, self.subcloud.id, deploy_status=DEPLOY_STATE_DONE
+        )
+
+        self.sysinv_client.get_kube_upgrade_health.return_value = \
+            KUBERNETES_UPGRADE_HEALTH_RESPONSE_MGMT_AFFECTING_AND_KUBERNETES_ALARM
+
+        self.worker.perform_state_action(self.strategy_step)
+
+        self.sysinv_client.get_kube_upgrade_health.assert_called_once()
+        self.assert_step_updated(
+            self.strategy_step.subcloud_id, STRATEGY_STATE_FAILED
+        )
+
     def test_pre_check_subcloud_failed_health_check_with_allowed_management_alarms(
-            self):
+        self
+    ):
         """Test pre check step where subcloud has management affecting alarms"""
 
         db_api.subcloud_update(self.ctx,
@@ -225,6 +270,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         The target version is derived from the system controller.  Inability
         to query that version should fail orchestration.
         """
+
         next_state = STRATEGY_STATE_FAILED
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -265,6 +311,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         Expectation:
         - Skip orchestration,  jump to 'complete' for this state.
         """
+
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
                                self.subcloud.id,
@@ -312,6 +359,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         When a kube upgrade exists in the subcloud, it is skipped if to-version
         if less than its version.  This test should not skip the subcloud.
         """
+
         next_state = STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -355,6 +403,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         For this test, the subcloud version is higher than the target, so
         it should not be resumed and the skip should occur.
         """
+
         next_state = STRATEGY_STATE_COMPLETE
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -409,6 +458,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
         is the same or higher.  The to-version can be a partial version.
         Test supports partial values for target_version and subcloud_version
         """
+
         next_state = STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -460,6 +510,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
 
     def test_pre_check_skip_when_target_version_is_greater_than_to_version(self):
         """Test creating pre check when target version is greater than to_version."""
+
         next_state = STRATEGY_STATE_COMPLETE
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -484,6 +535,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
 
     def test_pre_check_skip_when_there_is_no_version_available(self):
         """Test creating pre check when there is no version available."""
+
         next_state = STRATEGY_STATE_COMPLETE
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,
@@ -508,6 +560,7 @@ class TestKubeUpgradePreCheckStage(TestKubeUpgradeState):
 
     def test_pre_check_skip_when_there_are_multiple_available_versions(self):
         """Test creating pre check when there are multiple_available_versions."""
+
         next_state = STRATEGY_STATE_KUBE_CREATING_VIM_KUBE_UPGRADE_STRATEGY
         # Update the subcloud to have deploy state as "complete"
         db_api.subcloud_update(self.ctx,

distributedcloud/dcmanager/tests/unit/orchestrator/states/kube_rootca/test_upload_cert.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-
 import mock
 
 from dcmanager.common.consts import \