270 lines
12 KiB
Python
270 lines
12 KiB
Python
#
|
|
# Copyright (c) 2024 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import copy
|
|
import json
|
|
import uuid
|
|
|
|
import mock
|
|
|
|
from dccommon import consts as dccommon_consts
|
|
from dcmanager.common import consts
|
|
from dcmanager.common import utils
|
|
from dcmanager.db.sqlalchemy import api as db_api
|
|
from dcmanager.manager import peer_group_audit_manager
|
|
from dcmanager.manager.system_peer_manager import SystemPeerManager
|
|
from dcmanager.tests.base import DCManagerTestCase
|
|
from dcmanager.tests.unit.manager import test_peer_monitor_manager as tpm
|
|
from dcmanager.tests.unit.manager import test_system_peer_manager as tsm
|
|
|
|
# FAKE SUBCLOUD PEER GROUP DATA (SITE1)
|
|
FAKE_SITE1_PEER_GROUP_ID = 9
|
|
FAKE_SITE1_PEER_GROUP_NAME = 'PeerGroup2'
|
|
FAKE_SITE1_PEER_GROUP_SYSTEM_LEADER_ID = tpm.FAKE_SYSTEM_PEER_UUID # SITE1 UUID
|
|
FAKE_SITE1_PEER_GROUP_SYSTEM_LEADER_NAME = tpm.FAKE_SYSTEM_PEER_NAME # SITE1 NAME
|
|
FAKE_SITE1_PEER_GROUP_MAX_SUBCLOUDS_REHOMING = 20
|
|
FAKE_SITE1_PEER_GROUP_PRIORITY = 1
|
|
FAKE_SITE1_PEER_GROUP_STATE = 'enabled'
|
|
FAKE_SITE1_PEER_GROUP_MIGRATION_STATUS = consts.PEER_GROUP_MIGRATION_COMPLETE
|
|
FAKE_SITE1_PEER_GROUP_DATA = {
|
|
"peer_group_name": FAKE_SITE1_PEER_GROUP_NAME,
|
|
"system_leader_id": FAKE_SITE1_PEER_GROUP_SYSTEM_LEADER_ID,
|
|
"system_leader_name": FAKE_SITE1_PEER_GROUP_SYSTEM_LEADER_NAME,
|
|
"group_priority": FAKE_SITE1_PEER_GROUP_PRIORITY,
|
|
"group_state": FAKE_SITE1_PEER_GROUP_STATE,
|
|
"max_subcloud_rehoming": FAKE_SITE1_PEER_GROUP_MAX_SUBCLOUDS_REHOMING,
|
|
"migration_status": FAKE_SITE1_PEER_GROUP_MIGRATION_STATUS
|
|
}
|
|
|
|
# FAKE SUBCLOUD
|
|
FAKE_SUBCLOUD1_REGION_NAME = str(uuid.uuid4())
|
|
FAKE_SUBCLOUD1_NAME = 'subcloud1'
|
|
FAKE_SUBCLOUD2_REGION_NAME = str(uuid.uuid4())
|
|
FAKE_SUBCLOUD2_NAME = 'subcloud2'
|
|
FAKE_SUBCLOUD3_REGION_NAME = str(uuid.uuid4())
|
|
FAKE_SUBCLOUD3_NAME = 'subcloud3'
|
|
|
|
# FAKE SUBCLOUD REHOME DATA
|
|
FAKE_REHOME_DATA1 = {
|
|
"saved_payload": {
|
|
"bootstrap-address": "192.168.10.11",
|
|
"systemcontroller_gateway_address": "192.168.204.101"
|
|
}
|
|
}
|
|
FAKE_REHOME_DATA2 = {
|
|
"saved_payload": {
|
|
"bootstrap-address": "192.168.10.12",
|
|
"systemcontroller_gateway_address": "192.168.204.101"
|
|
}
|
|
}
|
|
FAKE_REHOME_DATA3 = {
|
|
"saved_payload": {
|
|
"bootstrap-address": "192.168.10.13",
|
|
"systemcontroller_gateway_address": "192.168.204.101"
|
|
}
|
|
}
|
|
|
|
# FAKE SUBCLOUD DATA (SITE1)
|
|
FAKE_SITE1_SUBCLOUD1_ID = 11
|
|
FAKE_SITE1_SUBCLOUD1_REGION_NAME = FAKE_SUBCLOUD1_REGION_NAME
|
|
FAKE_SITE1_SUBCLOUD1_DEPLOY_STATUS = consts.DEPLOY_STATE_DONE
|
|
FAKE_SITE1_SUBCLOUD1_MANAGEMENT_STATE = dccommon_consts.MANAGEMENT_MANAGED
|
|
FAKE_SITE1_SUBCLOUD1_PEER_GROUP_ID = FAKE_SITE1_PEER_GROUP_ID
|
|
FAKE_SITE1_SUBCLOUD1_DATA = {
|
|
"id": FAKE_SITE1_SUBCLOUD1_ID,
|
|
"name": FAKE_SUBCLOUD1_NAME,
|
|
"region-name": FAKE_SITE1_SUBCLOUD1_REGION_NAME,
|
|
"deploy-status": FAKE_SITE1_SUBCLOUD1_DEPLOY_STATUS,
|
|
"management-state": FAKE_SITE1_SUBCLOUD1_MANAGEMENT_STATE,
|
|
"peer_group_id": FAKE_SITE1_SUBCLOUD1_PEER_GROUP_ID,
|
|
"rehome_data": json.dumps(FAKE_REHOME_DATA1)
|
|
}
|
|
FAKE_SITE1_SUBCLOUD2_ID = 12
|
|
FAKE_SITE1_SUBCLOUD2_REGION_NAME = FAKE_SUBCLOUD2_REGION_NAME
|
|
FAKE_SITE1_SUBCLOUD2_DEPLOY_STATUS = consts.DEPLOY_STATE_DONE
|
|
FAKE_SITE1_SUBCLOUD2_MANAGEMENT_STATE = dccommon_consts.MANAGEMENT_MANAGED
|
|
FAKE_SITE1_SUBCLOUD2_PEER_GROUP_ID = FAKE_SITE1_PEER_GROUP_ID
|
|
FAKE_SITE1_SUBCLOUD2_DATA = {
|
|
"id": FAKE_SITE1_SUBCLOUD2_ID,
|
|
"name": FAKE_SUBCLOUD2_NAME,
|
|
"region-name": FAKE_SITE1_SUBCLOUD2_REGION_NAME,
|
|
"deploy-status": FAKE_SITE1_SUBCLOUD2_DEPLOY_STATUS,
|
|
"management-state": FAKE_SITE1_SUBCLOUD2_MANAGEMENT_STATE,
|
|
"peer_group_id": FAKE_SITE1_SUBCLOUD2_PEER_GROUP_ID,
|
|
# To test syncing rehome_data from site1(remote) to site0(local),
|
|
# we set the rehome_data to data3 instead of data2 for remote subcloud2
|
|
"rehome_data": json.dumps(FAKE_REHOME_DATA3)
|
|
}
|
|
|
|
|
|
class TestPeerGroupAudit(DCManagerTestCase):
|
|
def setUp(self):
|
|
super(TestPeerGroupAudit, self).setUp()
|
|
|
|
self.peer = tpm.TestPeerMonitor.create_system_peer_static(
|
|
self.ctx,
|
|
peer_name='SystemPeer1')
|
|
self.peer_group = tsm.TestSystemPeerManager. \
|
|
create_subcloud_peer_group_static(
|
|
self.ctx,
|
|
peer_group_name='SubcloudPeerGroup1')
|
|
# Create local dc subcloud1 mock data in database
|
|
self.subcloud1 = tsm.TestSystemPeerManager.create_subcloud_with_pg_static(
|
|
self.ctx,
|
|
peer_group_id=self.peer_group.id,
|
|
rehome_data=json.dumps(FAKE_REHOME_DATA1),
|
|
name=FAKE_SUBCLOUD1_NAME,
|
|
region_name=FAKE_SUBCLOUD1_REGION_NAME,
|
|
deploy_status=consts.DEPLOY_STATE_REHOME_PENDING)
|
|
# Create local dc subcloud2 mock data in database
|
|
self.subcloud2 = tsm.TestSystemPeerManager.create_subcloud_with_pg_static(
|
|
self.ctx,
|
|
peer_group_id=self.peer_group.id,
|
|
rehome_data=json.dumps(FAKE_REHOME_DATA2),
|
|
name=FAKE_SUBCLOUD2_NAME,
|
|
region_name=FAKE_SUBCLOUD2_REGION_NAME,
|
|
deploy_status=consts.DEPLOY_STATE_REHOME_PENDING)
|
|
# Create local dc subcloud3 mock data in database
|
|
self.subcloud3 = tsm.TestSystemPeerManager.create_subcloud_with_pg_static(
|
|
self.ctx,
|
|
peer_group_id=self.peer_group.id,
|
|
rehome_data=json.dumps(FAKE_REHOME_DATA3),
|
|
name=FAKE_SUBCLOUD3_NAME,
|
|
region_name=FAKE_SUBCLOUD3_REGION_NAME,
|
|
deploy_status=consts.DEPLOY_STATE_REHOME_PENDING)
|
|
# Remote subclouds
|
|
self.peer_subcloud1 = copy.deepcopy(FAKE_SITE1_SUBCLOUD1_DATA)
|
|
self.peer_subcloud2 = copy.deepcopy(FAKE_SITE1_SUBCLOUD2_DATA)
|
|
# Remote peer group
|
|
self.remote_peer_group = FAKE_SITE1_PEER_GROUP_DATA
|
|
|
|
# Initialize mock objects
|
|
self.mock_update_sync_status = \
|
|
mock.patch.object(SystemPeerManager, 'update_sync_status').start()
|
|
self.mock_get_peer_dc_client = \
|
|
mock.patch.object(SystemPeerManager, 'get_peer_dc_client').start()
|
|
mock_get_local_system = mock.patch.object(utils, 'get_local_system').start()
|
|
|
|
# Cleanup mock objects after test finishes
|
|
self.addCleanup(self.mock_update_sync_status.stop())
|
|
self.addCleanup(self.mock_get_peer_dc_client.stop())
|
|
self.addCleanup(mock_get_local_system.stop())
|
|
|
|
def run_audit(self):
|
|
self.mock_dc_client = mock.MagicMock()
|
|
self.mock_subcloud_manager = mock.MagicMock()
|
|
self.mock_get_peer_dc_client.return_value = self.mock_dc_client()
|
|
self.mock_dc_client().get_subcloud_list_by_peer_group.return_value = [
|
|
self.peer_subcloud1, self.peer_subcloud2]
|
|
self.mock_dc_client().get_system_peer.return_value = mock.MagicMock()
|
|
self.mock_dc_client().get_peer_group_association_with_peer_id_and_pg_id. \
|
|
return_value = {
|
|
"sync-status": consts.ASSOCIATION_SYNC_STATUS_OUT_OF_SYNC}
|
|
pm = peer_group_audit_manager.PeerGroupAuditManager(
|
|
self.mock_subcloud_manager, FAKE_SITE1_PEER_GROUP_ID)
|
|
pm._set_local_subcloud_to_secondary = mock.MagicMock(
|
|
wraps=pm._set_local_subcloud_to_secondary)
|
|
pm.audit(self.peer, self.remote_peer_group, self.peer_group)
|
|
return pm
|
|
|
|
def set_subcloud_rehome_failed(self, subcloud):
|
|
subcloud["deploy-status"] = consts.DEPLOY_STATE_REHOME_FAILED
|
|
subcloud["management-state"] = dccommon_consts.MANAGEMENT_UNMANAGED
|
|
|
|
def test_audit_migration_complete_with_all_success(self):
|
|
pm = self.run_audit()
|
|
|
|
# Verify all of three local subclouds are set as secondary,
|
|
# even including subcloud3, which is deleted afterward
|
|
self.assertEqual(3, pm._set_local_subcloud_to_secondary.call_count)
|
|
# Verify that the rehome_data of the local site subcloud2 is updated
|
|
# from data2 to data3, syncing from the remote site subcloud2
|
|
self.assertEqual(
|
|
json.dumps(FAKE_REHOME_DATA3),
|
|
db_api.subcloud_get(self.ctx, self.subcloud2.id).rehome_data
|
|
)
|
|
# Verify that the subcloud3 is deleted because it doesn't
|
|
# exist in the peer site
|
|
self.mock_subcloud_manager.delete_subcloud.assert_called_with(
|
|
pm.context, self.subcloud3.id)
|
|
# Verify that the system leader id is updated to the peer site uuid
|
|
self.assertEqual(
|
|
tpm.FAKE_SITE1_SYSTEM_UUID,
|
|
db_api.subcloud_peer_group_get(self.ctx, self.peer_group.id)
|
|
.system_leader_id
|
|
)
|
|
# Verify that the migration status of the remote peer group is updated
|
|
# to None since the migration completed
|
|
self.mock_dc_client().update_subcloud_peer_group.assert_called_with(
|
|
self.remote_peer_group.get("peer_group_name"), migration_status=None)
|
|
# Verify that the PGA sync status is updated to in-sync
|
|
self.mock_update_sync_status.assert_called_with(
|
|
pm.context, self.peer, consts.ASSOCIATION_SYNC_STATUS_IN_SYNC,
|
|
self.peer_group, self.remote_peer_group)
|
|
|
|
def test_audit_migration_complete_with_partial_failure(self):
|
|
# Remove local subcloud3
|
|
db_api.subcloud_destroy(self.ctx, self.subcloud3.id)
|
|
# Remote subclouds: subcloud1 success and subcloud2 failed
|
|
self.set_subcloud_rehome_failed(self.peer_subcloud2)
|
|
|
|
pm = self.run_audit()
|
|
|
|
# Verify that only subcloud1, the successful one, is set as secondary
|
|
self.assertEqual(1, pm._set_local_subcloud_to_secondary.call_count)
|
|
self.mock_subcloud_manager.delete_subcloud.assert_not_called()
|
|
# Verify that the local subcloud2 is also set to rehome-failed
|
|
self.assertEqual(
|
|
consts.DEPLOY_STATE_REHOME_FAILED,
|
|
db_api.subcloud_get(self.ctx, self.subcloud2.id).deploy_status
|
|
)
|
|
# Verify that the system leader id is updated to the peer site uuid
|
|
self.assertEqual(
|
|
tpm.FAKE_SITE1_SYSTEM_UUID,
|
|
db_api.subcloud_peer_group_get(self.ctx, self.peer_group.id)
|
|
.system_leader_id
|
|
)
|
|
# Verify that the migration status of the remote peer group is updated
|
|
# to None since the migration completed
|
|
self.mock_dc_client().update_subcloud_peer_group.assert_called_with(
|
|
self.remote_peer_group.get('peer_group_name'), migration_status=None)
|
|
# Verify that the PGA sync status remains out-of-sync due to rehome failure
|
|
self.mock_update_sync_status.assert_not_called()
|
|
|
|
def test_audit_migration_complete_with_all_failed(self):
|
|
# Remove local subcloud3
|
|
db_api.subcloud_destroy(self.ctx, self.subcloud3.id)
|
|
# Remote subclouds: both failed
|
|
self.set_subcloud_rehome_failed(self.peer_subcloud1)
|
|
self.set_subcloud_rehome_failed(self.peer_subcloud2)
|
|
|
|
pm = self.run_audit()
|
|
|
|
# Verify that none of the subclouds are set as secondary,
|
|
# as all of them are rehome-failed.
|
|
pm._set_local_subcloud_to_secondary.assert_not_called()
|
|
self.mock_subcloud_manager.delete_subcloud.assert_not_called()
|
|
# Verify that the local subclouds are also set to rehome-failed
|
|
self.assertEqual(
|
|
consts.DEPLOY_STATE_REHOME_FAILED,
|
|
db_api.subcloud_get(self.ctx, self.subcloud1.id).deploy_status
|
|
)
|
|
self.assertEqual(
|
|
consts.DEPLOY_STATE_REHOME_FAILED,
|
|
db_api.subcloud_get(self.ctx, self.subcloud2.id).deploy_status
|
|
)
|
|
# Verify that the system leader id is updated to the peer site uuid
|
|
self.assertEqual(
|
|
tpm.FAKE_SITE1_SYSTEM_UUID,
|
|
db_api.subcloud_peer_group_get(self.ctx, self.peer_group.id)
|
|
.system_leader_id
|
|
)
|
|
# Verify that the migration status of the remote peer group is updated
|
|
# to None since the migration completed
|
|
self.mock_dc_client().update_subcloud_peer_group.assert_called_with(
|
|
self.remote_peer_group.get('peer_group_name'), migration_status=None)
|
|
# Verify that the PGA sync status remains out-of-sync due to rehome failure
|
|
self.mock_update_sync_status.assert_not_called()
|