starlingx
/
nfv
Code Issues Proposed changes

sw-deploy-strategy backend overhaul 

Roughed in sw-deploy by replacing sw-upgrade and adding things as
needed.  The intention here being is to create the basic stage/step flow
for followup reviews.

There are numerous changes still required to complete the strategy.

TODO in followups:
* SWACT/unlock retries
* Fix any broken/new steps (precheck/deploy)
* Update USM API calls after they are changed
* Maintain state using USM api responses
* Major release support, including SWACT to controller-0 if required
* strategy reentrancy

TEST PLAN
PASSING: Unit tests
NOT PASSING: sw-deploy-strategy on AIO-SX
NOT PASSING: sw-deploy-strategy on system controllers (AIO-DX)

Change-Id: If1eb5b45089f4a67d6d88093d0e215e510fd8c55
Signed-off-by: Joshua Kraitberg <joshua.kraitberg@windriver.com>
This commit is contained in:
Joshua Kraitberg 2024-03-19 11:20:32 -04:00
parent 3399c7fc54
commit 89ad86ec00
24 changed files with 1567 additions and 674 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nfv/nfv-client/nfv_client/openstack/sw_update.py
View File

@ -299,13 +299,9 @@ def create_strategy(token_id,
elif sw_update.STRATEGY_NAME_SYSTEM_CONFIG_UPDATE == strategy_name:
api_cmd_payload['controller-apply-type'] = controller_apply_type
api_cmd_payload['default-instance-action'] = default_instance_action
# TODO(jkraitbe): Backend for sw-deploy will continue as old sw-upgrade for now
elif sw_update.STRATEGY_NAME_SW_UPGRADE == strategy_name:
# for upgrade: default-instance-action is hardcoded to MIGRATE
if 'start_upgrade' in kwargs and kwargs['start_upgrade']:
api_cmd_payload['start-upgrade'] = True
if 'complete_upgrade' in kwargs and kwargs['complete_upgrade']:
api_cmd_payload['complete-upgrade'] = True
if 'release' in kwargs and kwargs['release']:
api_cmd_payload['release'] = kwargs['release']
api_cmd_payload['storage-apply-type'] = storage_apply_type
api_cmd_payload['worker-apply-type'] = worker_apply_type
if max_parallel_worker_hosts is not None:

13
nfv/nfv-client/nfv_client/shell.py
View File

@ -51,9 +51,7 @@ def get_extra_create_args(cmd_area, args):
# no additional kwargs for patch
return {}
elif sw_update.CMD_NAME_SW_DEPLOY == cmd_area:
# TODO(jkraitbe): Args will be updated to use new release parameter
# upgrade supports: complete_upgrade
return {'complete_upgrade': args.complete_upgrade}
return {'release': args.release}
elif sw_update.CMD_NAME_FW_UPDATE == cmd_area:
# no additional kwargs for firmware update
return {}
@ -465,14 +463,7 @@ def setup_sw_deploy_parser(commands):
# add sw-deploy specific arguments to the create command
# The get_extra_create_args method is updated to align with these
# Disable support for --start-upgrade as it was not completed
# create_strategy_cmd.add_argument('--start-upgrade',
# action='store_true',
# help=argparse.SUPPRESS)
# TODO(jkraitbe): Args will be updated to use new release parameter
create_strategy_cmd.add_argument('--complete-upgrade',
action='store_true',
create_strategy_cmd.add_argument('--release',
help=argparse.SUPPRESS)
# define the delete command

1
nfv/nfv-client/nfv_client/sw_update/_sw_update.py
View File

@ -330,6 +330,7 @@ def show_strategy(os_auth_uri, os_project_name, os_project_domain_name,
"""
Software Update - Show Strategy
"""
token = openstack.get_token(os_auth_uri, os_project_name,
os_project_domain_name, os_username, os_password,
os_user_domain_name)

152
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_infrastructure_api.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -23,6 +23,7 @@ from nfv_plugins.nfvi_plugins.openstack import mtc
from nfv_plugins.nfvi_plugins.openstack import openstack
from nfv_plugins.nfvi_plugins.openstack import rest_api
from nfv_plugins.nfvi_plugins.openstack import sysinv
from nfv_plugins.nfvi_plugins.openstack import usm
from nfv_plugins.nfvi_plugins.openstack.objects import OPENSTACK_SERVICE
@ -2202,9 +2203,9 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def get_upgrade(self, future, callback):
def get_upgrade(self, future, release, callback):
"""
Get information about the upgrade from the plugin
Get information about the sw-deploy from the plugin
"""
response = dict()
response['completed'] = False
@ -2225,26 +2226,30 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(sysinv.get_upgrade, self._platform_token)
future.work(usm.sw_deploy_get_release, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv get-upgrade did not complete.")
DLOG.error("USM sw-deploy get release did not complete.")
return
upgrade_data_list = future.result.data
if 1 < len(upgrade_data_list):
DLOG.critical("Too many upgrades retrieved, num_upgrades=%i"
% len(upgrade_data_list))
release_data = future.result.data
release_info = release_data["metadata"][release]
upgrade_obj = None
future.work(usm.sw_deploy_host_list, self._platform_token)
future.result = (yield)
for upgrade_data in upgrade_data_list['upgrades']:
upgrade_obj = nfvi.objects.v1.Upgrade(
upgrade_data['state'],
upgrade_data['from_release'],
upgrade_data['to_release'])
break
if not future.result.is_complete():
DLOG.error("USM sw-deploy host list did not complete.")
return
hosts_states_data = future.result.data
upgrade_obj = nfvi.objects.v1.Upgrade(
release,
release_info["state"],
release_info["reboot_required"] == "Y",
hosts_states_data["data"])
response['result-data'] = upgrade_obj
response['completed'] = True
@ -2267,15 +2272,16 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
callback.send(response)
callback.close()
def upgrade_start(self, future, callback):
def upgrade_precheck(self, future, release, callback):
"""
Start an upgrade
Precheck a USM stoftware deploy
"""
response = dict()
response['completed'] = False
response['reason'] = ''
try:
upgrade_data = future.result.data
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
@ -2290,18 +2296,76 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(sysinv.upgrade_start, self._platform_token)
future.work(usm.sw_deploy_precheck, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv upgrade-start did not complete.")
DLOG.error("USM sw-deploy precheck did not complete.")
return
upgrade_data = future.result.data
upgrade_obj = nfvi.objects.v1.Upgrade(
self._release,
None,
None,
None)
response['result-data'] = upgrade_obj
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
if self._platform_token is not None:
self._platform_token.set_expired()
else:
DLOG.exception("Caught exception while trying to precheck "
"USM sw-deploy, error=%s." % e)
except Exception as e:
DLOG.exception("Caught exception while trying to precheck USM sw-deploy, "
"error=%s." % e)
finally:
callback.send(response)
callback.close()
def upgrade_start(self, future, release, callback):
"""
Start a USM stoftware deploy
"""
response = dict()
response['completed'] = False
response['reason'] = ''
try:
upgrade_data = future.result.data
future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
if self._platform_token is None or \
self._platform_token.is_expired():
future.work(openstack.get_token, self._platform_directory)
future.result = (yield)
if not future.result.is_complete() or \
future.result.data is None:
DLOG.error("OpenStack get-token did not complete.")
return
self._platform_token = future.result.data
future.work(usm.sw_deploy_start, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("USM sw-deploy start did not complete.")
return
upgrade_obj = nfvi.objects.v1.Upgrade(
self._release,
upgrade_data['state'],
upgrade_data['from_release'],
upgrade_data['to_release'])
None,
None)
response['result-data'] = upgrade_obj
response['completed'] = True
@ -2314,19 +2378,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
else:
DLOG.exception("Caught exception while trying to start "
"upgrade, error=%s." % e)
"USM sw-deploy, error=%s." % e)
except Exception as e:
DLOG.exception("Caught exception while trying to start upgrade, "
DLOG.exception("Caught exception while trying to start USM sw-deploy, "
"error=%s." % e)
finally:
callback.send(response)
callback.close()
def upgrade_activate(self, future, callback):
def upgrade_activate(self, future, release, callback):
"""
Activate an upgrade
Activate a USM software deployement
"""
response = dict()
response['completed'] = False
@ -2347,18 +2411,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(sysinv.upgrade_activate, self._platform_token)
future.work(usm.sw_deploy_activate, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv upgrade-activate did not complete.")
DLOG.error("USM sw-deploy activate did not complete.")
return
upgrade_data = future.result.data
upgrade_obj = nfvi.objects.v1.Upgrade(
self._release,
upgrade_data['state'],
upgrade_data['from_release'],
upgrade_data['to_release'])
None,
None)
response['result-data'] = upgrade_obj
response['completed'] = True
@ -2371,19 +2436,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
else:
DLOG.exception("Caught exception while trying to activate "
"upgrade, error=%s." % e)
"USM sw-deploy, error=%s." % e)
except Exception as e:
DLOG.exception("Caught exception while trying to activate upgrade, "
DLOG.exception("Caught exception while trying to activate USM sw-deploy, "
"error=%s." % e)
finally:
callback.send(response)
callback.close()
def upgrade_complete(self, future, callback):
def upgrade_complete(self, future, release, callback):
"""
Complete an upgrade
Complete a USM software deployement
"""
response = dict()
response['completed'] = False
@ -2404,18 +2469,19 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(sysinv.upgrade_complete, self._platform_token)
future.work(usm.sw_deploy_complete, self._platform_token, release)
future.result = (yield)
if not future.result.is_complete():
DLOG.error("SysInv upgrade-complete did not complete.")
DLOG.error("USM sw-deploy complete did not complete.")
return
upgrade_data = future.result.data
upgrade_obj = nfvi.objects.v1.Upgrade(
self._release,
upgrade_data['state'],
upgrade_data['from_release'],
upgrade_data['to_release'])
None,
None)
response['result-data'] = upgrade_obj
response['completed'] = True
@ -2428,10 +2494,10 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
else:
DLOG.exception("Caught exception while trying to complete "
"upgrade, error=%s." % e)
"USM sw-deploy, error=%s." % e)
except Exception as e:
DLOG.exception("Caught exception while trying to complete upgrade, "
DLOG.exception("Caught exception while trying to complete USM sw-deploy, "
"error=%s." % e)
finally:
@ -3594,7 +3660,7 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
self._platform_token = future.result.data
future.work(sysinv.upgrade_host, self._platform_token, host_uuid)
future.work(usm.sw_deploy_execute, self._platform_token, host_name)
future.result = (yield)
if not future.result.is_complete():
@ -4379,4 +4445,4 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
Finalize the plugin
"""
if self._host_listener is not None:
self._host_listener.shutdown()
self._host_listener.shutdown()

3
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2018 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -40,6 +40,7 @@ class PlatformServices(Constants):
SYSINV = Constant('sysinv')
PATCHING = Constant('patching')
FM = Constant('fm')
USM = Constant('usm')
# Platform Services Constant

138
nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/usm.py Executable file
View File

@ -0,0 +1,138 @@
#
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import json
import os
from nfv_common import debug
from nfv_plugins.nfvi_plugins.openstack.objects import PLATFORM_SERVICE
from nfv_plugins.nfvi_plugins.openstack.rest_api import rest_api_request
from nfv_plugins.nfvi_plugins import config
REST_API_REQUEST_TIMEOUT = 60
DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.openstack.usm')
def _usm_api_cmd(token, endpoint):
base_url = token.get_service_url(PLATFORM_SERVICE.USM)
if base_url is None:
raise ValueError("PlatformService USM URL is invalid")
url = os.path.join(base_url, "v1/software", endpoint)
return url
def _api_cmd_headers():
api_cmd_headers = dict()
api_cmd_headers['Content-Type'] = "application/json"
api_cmd_headers['User-Agent'] = "vim/1.0"
return api_cmd_headers
def _api_get(token, url):
"""
Perform a generic GET for a particular API endpoint
"""
response = rest_api_request(token,
"GET",
url,
timeout_in_secs=REST_API_REQUEST_TIMEOUT)
return response
def _api_post(token, url, payload, headers=_api_cmd_headers()):
"""
Generic POST to an endpoint with a payload
"""
response = rest_api_request(token,
"POST",
url,
headers,
json.dumps(payload),
timeout_in_secs=REST_API_REQUEST_TIMEOUT)
return response
def sw_deploy_get_release(token, release):
"""
Query USM for information about a specific upgrade
"""
uri = f"show/{release}"
url = _usm_api_cmd(token, uri)
response = _api_get(token, url)
return response
def sw_deploy_host_list(token):
"""
Query USM for information about a hosts during a deployment
"""
# TODO(jkraitbe): This API will change in the future
uri = "host_list"
url = _usm_api_cmd(token, uri)
response = _api_get(token, url)
return response
def sw_deploy_precheck(token, release, force=False):
"""
Ask USM to precheck before a deployment
"""
uri = f"deploy_precheck/{release}/force" if force else f"deploy_precheck/{release}"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url,{})
return response
def sw_deploy_start(token, release, force=False):
"""
Ask USM to start a deployment
"""
uri = f"deploy_start/{release}/force" if force else f"deploy_start/{release}"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url,{})
return response
def sw_deploy_execute(token, host_name):
"""
Ask USM to execute a deployment on a host
"""
uri = f"deploy_host/{host_name}"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url,{})
return response
def sw_deploy_activate(token, release):
"""
Ask USM activate a deployment
"""
uri = f"deploy_activate/{release}"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url,{})
return response
def sw_deploy_activate(token, release):
"""
Ask USM complete a deployment
"""
uri = f"deploy_complete/{release}"
url = _usm_api_cmd(token, uri)
response = _api_post(token, url,{})
return response

0
nfv/nfv-tests/nfv_unit_tests/__init__.py Normal file
View File

36
nfv/nfv-tests/nfv_unit_tests/tests/sw_update_testcase.py
View File

@ -1,11 +1,12 @@
#
# Copyright (c) 2020-2023 Wind River Systems, Inc.
# Copyright (c) 2020-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import fixtures
import json
import pprint
import uuid
@ -27,8 +28,9 @@ from nfv_unit_tests.tests import utils
# change the following 2 values to assist with
# unit test comparison between json structures
DEBUG_PRINTING = False
DEBUG_DEPTH = 3
DEBUG_PRINTING = True
DEBUG_DEPTH = 7
DEBUG_WITH_JSON = True
def validate_strategy_persists(strategy):
@ -44,9 +46,15 @@ def validate_strategy_persists(strategy):
if DEBUG_PRINTING:
if strategy.as_dict() != new_strategy.as_dict():
print("==================== Strategy ====================")
pprint.pprint(strategy.as_dict(), depth=DEBUG_DEPTH)
if DEBUG_WITH_JSON:
print(json.dumps(strategy.as_dict(), indent=2))
else:
pprint.pprint(strategy.as_dict(), depth=DEBUG_DEPTH)
print("============== Converted Strategy ================")
pprint.pprint(new_strategy.as_dict(), depth=DEBUG_DEPTH)
if DEBUG_WITH_JSON:
print(json.dumps(new_strategy.as_dict(), indent=2))
else:
pprint.pprint(new_strategy.as_dict(), depth=DEBUG_DEPTH)
assert strategy.as_dict() == new_strategy.as_dict(), \
"Strategy changed when converting to/from dict"
@ -59,9 +67,15 @@ def validate_phase(phase, expected_results):
"""
if DEBUG_PRINTING:
print("====================== Phase Results ========================")
pprint.pprint(phase, depth=DEBUG_DEPTH)
if DEBUG_WITH_JSON:
print(json.dumps(phase, indent=2))
else:
pprint.pprint(phase, depth=DEBUG_DEPTH)
print("===================== Expected Results ======================")
pprint.pprint(expected_results, depth=DEBUG_DEPTH)
if DEBUG_WITH_JSON:
print(json.dumps(expected_results, indent=2))
else:
pprint.pprint(expected_results, depth=DEBUG_DEPTH)
for key in expected_results:
if key == 'stages':
@ -75,10 +89,14 @@ def validate_phase(phase, expected_results):
apply_step = apply_stage[stages_key][step_number]
for step_key in step:
assert apply_step[step_key] == step[step_key], \
"for [%s][%d][%s][%d][%s] found: %s but expected: %s" % \
"for [%s][%d][%s][%d][%s] found: %s but expected: %s\n" \
"\n===== Found Step =====\n%s\n" \
"\n===== Expected Step =====\n%s" % \
(key, stage_number, stages_key,
step_number, step_key,
apply_step[step_key], step[step_key])
apply_step[step_key], step[step_key],
json.dumps(apply_step, indent=2),
json.dumps(step, indent=2))
step_number += 1
else:
assert apply_stage[stages_key] == stage[stages_key], \

791
nfv/nfv-tests/nfv_unit_tests/tests/test_sw_deploy_strategy.py Executable file
View File

@ -0,0 +1,791 @@
#
# Copyright (c) 2016-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import testtools
from unittest import mock
import uuid
from nfv_common import strategy as common_strategy
from nfv_vim import nfvi
from nfv_vim.objects import HOST_NAME
from nfv_vim.objects import HOST_PERSONALITY
from nfv_vim.objects import SW_UPDATE_ALARM_RESTRICTION
from nfv_vim.objects import SW_UPDATE_APPLY_TYPE
from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION
from nfv_vim.objects import SwUpgrade
from nfv_vim.strategy._strategy import strategy_rebuild_from_dict
from nfv_vim.strategy._strategy import SwUpgradeStrategy
from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE
from nfv_unit_tests.tests import sw_update_testcase
# utility method for the formatting of unlock-hosts stage as dict
# workers default to 5 retries with 120 second delay between attempts
# std controllers and storage have 0 retries
def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120):
return {
'name': 'unlock-hosts',
'entity_names': host_names,
'retry_count': retry_count,
'retry_delay': retry_delay,
'timeout': 1800,
}
@mock.patch('nfv_vim.event_log._instance._event_issue',
sw_update_testcase.fake_event_issue)
@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save',
sw_update_testcase.fake_save)
@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer',
sw_update_testcase.fake_timer)
@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled',
sw_update_testcase.fake_nfvi_compute_plugin_disabled)
class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
def create_sw_deploy_strategy(self,
controller_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE,
storage_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE,
worker_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE,
max_parallel_worker_hosts=10,
alarm_restrictions=SW_UPDATE_ALARM_RESTRICTION.STRICT,
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
release="123.1",
nfvi_upgrade=None,
single_controller=False
):
"""
Create a software update strategy
"""
strategy = SwUpgradeStrategy(
uuid=str(uuid.uuid4()),
controller_apply_type=controller_apply_type,
storage_apply_type=storage_apply_type,
worker_apply_type=worker_apply_type,
max_parallel_worker_hosts=max_parallel_worker_hosts,
default_instance_action=default_instance_action,
alarm_restrictions=alarm_restrictions,
release=release,
ignore_alarms=[],
single_controller=single_controller,
)
strategy.nfvi_upgrade = nfvi_upgrade
return strategy
def _gen_aiosx_hosts_and_strategy(
self,
openstack=True,
# aio-sx must be stop_start
default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START,
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
instances=[],
**kwargs,
):
self.create_host('controller-0', aio=True, openstack_installed=openstack)
controller_hosts = []
for host in list(self._host_table.values()):
if HOST_PERSONALITY.WORKER in host.personality:
controller_hosts.append(host)
for args in instances:
self.create_instance(*args)
strategy = self.create_sw_deploy_strategy(
single_controller=True,
default_instance_action=default_instance_action,
worker_apply_type=worker_apply_type,
**kwargs,
)
return controller_hosts, strategy
def _gen_aiodx_hosts_and_strategy(
self,
openstack=True,
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
instances=[],
**kwargs,
):
self.create_host('controller-0', aio=True, openstack_installed=openstack)
self.create_host('controller-1', aio=True, openstack_installed=openstack)
controller_hosts = []
for host in list(self._host_table.values()):
if HOST_PERSONALITY.WORKER in host.personality:
controller_hosts.append(host)
for args in instances:
self.create_instance(*args)
strategy = self.create_sw_deploy_strategy(
default_instance_action=default_instance_action,
worker_apply_type=worker_apply_type,
**kwargs,
)
return controller_hosts, strategy
# ~~~ SW-DEPLOY Start ~~~
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_0)
def test_sw_deploy_strategy_start_on_controller_0_sx(self):
"""
Test the sw_upgrade strategy start stage controller-0
- sx
Verify:
- pass
"""
_, strategy = self._gen_aiosx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_start_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 4,
'steps': [
{'name': 'query-alarms'},
{'name': 'sw-deploy-precheck',
'release': '888.8'},
{'name': 'start-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
]
}
]
}
# sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_0)
def test_sw_deploy_strategy_start_on_controller_0_dx(self):
"""
Test the sw_upgrade strategy start stages on controller-0:
- dx
Verify:
- pass
"""
_, strategy = self._gen_aiodx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_start_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 6,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'sw-deploy-precheck',
'release': '888.8'},
{'name': 'start-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
{'name': 'swact-hosts',
'entity_names': ['controller-0']},
]
}
]
}
# sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_1)
def test_sw_deploy_strategy_start_on_controller_1_dx(self):
"""
Test the sw_upgrade strategy start stages on controller-1:
- dx
Verify:
- pass
"""
_, strategy = self._gen_aiodx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_start_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-start',
'total_steps': 6,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'sw-deploy-precheck',
'release': '888.8'},
{'name': 'start-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
{'name': 'swact-hosts',
'entity_names': ['controller-0']},
]
}
]
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
# ~~~ SW-DEPLOY Complete ~~~
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_0)
def test_sw_deploy_strategy_complete_on_controller_0_sx(self):
"""
Test the sw_upgrade strategy complete stage controller-0
- sx
Verify:
- pass
"""
_, strategy = self._gen_aiosx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_complete_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-complete',
'total_steps': 4,
'steps': [
{'name': 'query-alarms'},
{'name': 'activate-upgrade',
'release': '888.8'},
{'name': 'complete-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
]
}
]
}
# sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_0)
def test_sw_deploy_strategy_complete_on_controller_0_dx(self):
"""
Test the sw_upgrade strategy complete stages on controller-0:
- dx
Verify:
- pass
"""
_, strategy = self._gen_aiodx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_complete_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-complete',
'total_steps': 5,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'activate-upgrade',
'release': '888.8'},
{'name': 'complete-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
]
}
]
}
# sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_1)
def test_sw_deploy_strategy_complete_on_controller_1_dx(self):
"""
Test the sw_upgrade strategy complete stages on controller-1:
- dx
Verify:
- pass
"""
_, strategy = self._gen_aiodx_hosts_and_strategy(
release='888.8',
)
strategy._add_upgrade_complete_stage()
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-complete',
'total_steps': 5,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
'entity_names': ['controller-1']},
{'name': 'activate-upgrade',
'release': '888.8'},
{'name': 'complete-upgrade',
'release': '888.8'},
{'name': 'system-stabilize',
'timeout': 60},
]
}
]
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
# ~~~ AIO-SX NRR ~~~
def test_sw_deploy_strategy_aiosx_controllers_serial_nrr(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- serial apply
- no reboot required
- stop_start instances
- no instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy()
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=False)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 3,
'steps': [
{'name': 'query-alarms',},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 30,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- no reboot required
- stop_start instances
- no instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=False)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 3,
'steps': [
{'name': 'query-alarms',},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 30,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_no_openstack(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- no reboot required
- stop_start instances
- no instances
- no openstack
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
openstack=False,
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=False)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 3,
'steps': [
{'name': 'query-alarms',},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 30,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_instances_migrate(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- no reboot required
- migrate instances
- instances
Verify:
- Fail
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
instances=[('small', 'test_instance_0', 'controller-0')],
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=False)
assert success is False
assert reason == 'cannot migrate instances in a single controller configuration'
sw_update_testcase.validate_strategy_persists(strategy)
def test_sw_deploy_strategy_aiosx_controllers_parallel_nrr_instances_stop_start(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- no reboot required
- stop_start instances
- instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
instances=[('small', 'test_instance_0', 'controller-0')],
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=False)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 3,
'steps': [
{'name': 'query-alarms',},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 30,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
# ~~~ AIO-SX RR ~~~
def test_sw_deploy_strategy_aiosx_controllers_serial_rr(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- serial apply
- reboot required
- stop_start instances
- no instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy()
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'steps': [
{'name': 'query-alarms',},
{'name': 'lock-hosts', 'entity_names': ['controller-0'],},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
{'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120,},
{'name': 'wait-alarms-clear', 'timeout': 1800,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_rr(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- reboot required
- stop_start instances
- no instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'steps': [
{'name': 'query-alarms',},
{'name': 'lock-hosts', 'entity_names': ['controller-0'],},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
{'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120,},
{'name': 'wait-alarms-clear', 'timeout': 1800,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_no_openstack(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- reboot required
- stop_start instances
- no instances
- no openstack
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
openstack=False,
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'steps': [
{'name': 'query-alarms',},
{'name': 'lock-hosts', 'entity_names': ['controller-0'],},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
{'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120,},
{'name': 'wait-alarms-clear', 'timeout': 1800,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_instances_migrate(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- reboot required
- migrate instances
- instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
instances=[('small', 'test_instance_0', 'controller-0')],
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is False
assert reason == 'cannot migrate instances in a single controller configuration'
sw_update_testcase.validate_strategy_persists(strategy)
def test_sw_deploy_strategy_aiosx_controllers_parallel_rr_instances_stop_start(self):
"""
Test the sw_deploy strategy add controller strategy stages:
- aio-sx host
- parallel apply
- reboot required
- stop_start instances
- instances
Verify:
- Pass
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.PARALLEL,
instances=[('small', 'test_instance_0', 'controller-0')],
)
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 8,
'steps': [
{'name': 'query-alarms',},
{'name': 'stop-instances', 'entity_names': ['test_instance_0'],},
{'name': 'lock-hosts', 'entity_names': ['controller-0'],},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
{'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120,},
{'name': 'start-instances', 'entity_names': ['test_instance_0'],},
{'name': 'wait-alarms-clear', 'timeout': 1800,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)

234
nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py
View File

@ -14,6 +14,7 @@ from nfv_vim.objects import HOST_NAME
from nfv_vim.objects import HOST_PERSONALITY
from nfv_vim.objects import SW_UPDATE_ALARM_RESTRICTION
from nfv_vim.objects import SW_UPDATE_APPLY_TYPE
from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION
from nfv_vim.objects import SwUpgrade
from nfv_vim.strategy._strategy import strategy_rebuild_from_dict
from nfv_vim.strategy._strategy import SwUpgradeStrategy
@ -23,6 +24,7 @@ from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE
from nfv_unit_tests.tests import sw_update_testcase
# TODO(jkraitbe): Update this when retry count is decicded.
# utility method for the formatting of unlock-hosts stage as dict
# workers default to 5 retries with 120 second delay between attempts
# std controllers and storage have 0 retries
@ -30,9 +32,9 @@ def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120):
return {
'name': 'unlock-hosts',
'entity_names': host_names,
'retry_count': retry_count,
'retry_delay': retry_delay,
'timeout': 1800,
# 'retry_count': retry_count,
# 'retry_delay': retry_delay,
# 'timeout': 1800,
}
@ -51,8 +53,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
worker_apply_type=SW_UPDATE_APPLY_TYPE.IGNORE,
max_parallel_worker_hosts=10,
alarm_restrictions=SW_UPDATE_ALARM_RESTRICTION.STRICT,
start_upgrade=False,
complete_upgrade=False,
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
release="123.1",
nfvi_upgrade=None,
single_controller=False
):
@ -64,9 +66,9 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
storage_apply_type=storage_apply_type,
worker_apply_type=worker_apply_type,
max_parallel_worker_hosts=max_parallel_worker_hosts,
default_instance_action=default_instance_action,
alarm_restrictions=alarm_restrictions,
start_upgrade=start_upgrade,
complete_upgrade=complete_upgrade,
release=release,
ignore_alarms=[],
single_controller=single_controller,
)
@ -562,13 +564,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 13,
'stages': [
{'name': 'sw-upgrade-worker-hosts',
'total_steps': 5,
'total_steps': 8,
'steps': [
{'name': 'query-alarms'},
{'name': 'disable-host-services'},
{'name': 'migrate-instances-from-host',
'host_names': stage_hosts[0],
'entity_names': []},
{'name': 'lock-hosts',
'entity_names': stage_hosts[0]},
{'name': 'upgrade-hosts',
'entity_names': stage_hosts[0]},
{'name': 'system-stabilize',
'timeout': 15},
_unlock_hosts_stage_as_dict(stage_hosts[0]),
{'name': 'wait-alarms-clear',
'timeout': 600}
@ -580,16 +588,19 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
for x in range(1, len(stage_hosts)):
expected_results['stages'].append(
{'name': 'sw-upgrade-worker-hosts',
'total_steps': 7,
'total_steps': 8,
'steps': [
{'name': 'query-alarms'},
{'name': 'disable-host-services'},
{'name': 'migrate-instances',
{'name': 'migrate-instances-from-host',
'host_names': stage_hosts[x],
'entity_names': stage_instances[x]},
{'name': 'lock-hosts',
'entity_names': stage_hosts[x]},
{'name': 'upgrade-hosts',
'entity_names': stage_hosts[x]},
{'name': 'system-stabilize',
'timeout': 15},
_unlock_hosts_stage_as_dict(stage_hosts[x]),
{'name': 'wait-alarms-clear',
'timeout': 600}
@ -927,42 +938,45 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 3,
'total_stages': 2,
'stages': [
{'name': 'sw-upgrade-storage-hosts',
'total_steps': 5,
'total_steps': 6,
'steps': [
{'name': 'query-alarms'},
{'name': 'lock-hosts',
'entity_names': ['storage-0']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-0']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['storage-0']),
{'name': 'wait-data-sync',
'timeout': 7200}
]
},
{'name': 'sw-upgrade-storage-hosts',
'total_steps': 5,
'total_steps': 6,
'steps': [
{'name': 'query-alarms'},
{'name': 'lock-hosts',
'entity_names': ['storage-1', 'storage-2']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-1', 'storage-2']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['storage-1', 'storage-2']),
{'name': 'wait-data-sync',
'timeout': 7200}
]
},
{'name': 'sw-upgrade-storage-hosts',
'total_steps': 5,
'total_steps': 6,
'steps': [
{'name': 'query-alarms'},
{'name': 'lock-hosts',
'entity_names': ['storage-3']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-3']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['storage-3']),
{'name': 'wait-data-sync',
'timeout': 7200}
@ -1075,7 +1089,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_1)
sw_update_testcase.fake_host_name_controller_0)
def test_sw_upgrade_strategy_controller_stages_serial(self):
"""
Test the sw_upgrade strategy add controller strategy stages:
@ -1103,13 +1117,14 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 1,
'stages': [
{'name': 'sw-upgrade-controllers',
'total_steps': 5,
'total_steps': 7,
'steps': [
{'name': 'query-alarms'},
{'name': 'lock-hosts',
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['controller-0']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'],
@ -1150,13 +1165,14 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 2,
'stages': [
{'name': 'sw-upgrade-controllers',
'total_steps': 5,
'total_steps': 7,
'steps': [
{'name': 'query-alarms'},
{'name': 'lock-hosts',
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['controller-1']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'],
@ -1173,6 +1189,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
_unlock_hosts_stage_as_dict(['controller-0']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006', '100.119', '900.701'],
@ -1235,7 +1252,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'total_stages': 2,
'stages': [
{'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'total_steps': 7,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
@ -1251,7 +1268,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
]
},
{'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'total_steps': 7,
'steps': [
{'name': 'query-alarms'},
{'name': 'swact-hosts',
@ -1272,36 +1289,95 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_1)
def test_sw_upgrade_strategy_aiosx_stages_serial(self):
"""
Test the sw_upgrade strategy add controller strategy stages:
- aio-sx hosts
- serial apply
Verify:
- failure
"""
self.create_host('controller-0', aio=True)
def _gen_aiosx_hosts_and_strategy(
self,
openstack=True,
default_instance_action=SW_UPDATE_INSTANCE_ACTION.STOP_START,
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
**kwargs,
):
self.create_host('controller-0', aio=True, openstack_installed=openstack)
controller_hosts = []
for host in list(self._host_table.values()):
if (HOST_PERSONALITY.CONTROLLER in host.personality and
HOST_NAME.CONTROLLER_0 == host.name):
if HOST_PERSONALITY.WORKER in host.personality:
controller_hosts.append(host)
strategy = self.create_sw_upgrade_strategy(single_controller=True)
strategy = self.create_sw_upgrade_strategy(
single_controller=True,
default_instance_action=default_instance_action,
worker_apply_type=worker_apply_type,
**kwargs,
)
success, reason = strategy._add_controller_strategy_stages(
controllers=controller_hosts,
return controller_hosts, strategy
def _gen_aiodx_hosts_and_strategy(
self,
openstack=True,
default_instance_action=SW_UPDATE_INSTANCE_ACTION.MIGRATE,
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
**kwargs,
):
self.create_host('controller-0', aio=True, openstack_installed=openstack)
self.create_host('controller-1', aio=True, openstack_installed=openstack)
controller_hosts = []
for host in list(self._host_table.values()):
if HOST_PERSONALITY.WORKER in host.personality:
controller_hosts.append(host)
strategy = self.create_sw_upgrade_strategy(
default_instance_action=default_instance_action,
worker_apply_type=worker_apply_type,
**kwargs,
)
return controller_hosts, strategy
def test_sw_upgrade_strategy_aiosx_controllers_serial_rr(self):
"""
Test the sw_upgrade strategy add controller strategy stages:
- aio-sx host
- serial apply
- reboot required
- stop_start instances
- no instances
Verify:
- failure
"""
controller_hosts, strategy = self._gen_aiosx_hosts_and_strategy()
# self.create_instance('small',
# "test_instance_0",
# 'controller-0')
success, reason = strategy._add_worker_strategy_stages(
worker_hosts=controller_hosts,
reboot=True)
assert success is False
assert reason == "not enough controllers to apply software upgrades"
assert success is True, reason
apply_phase = strategy.apply_phase.as_dict()
expected_results = {
'total_stages': 0,
'total_stages': 1,
'stages': [
{
'name': 'sw-upgrade-worker-hosts',
'total_steps': 6,
'steps': [
{'name': 'query-alarms',},
{'name': 'lock-hosts', 'entity_names': ['controller-0'],},
{'name': 'upgrade-hosts', 'entity_names': ['controller-0']},
{'name': 'system-stabilize', 'timeout': 15,},
{'name': 'unlock-hosts', 'entity_names': ['controller-0'], 'retry_count': 0, 'retry_delay': 120,},
{'name': 'wait-alarms-clear', 'timeout': 1800,},
]
},
],
}
sw_update_testcase.validate_strategy_persists(strategy)
sw_update_testcase.validate_phase(apply_phase, expected_results)
@ -1477,9 +1553,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
'13.01',
UPGRADE_STATE.UPGRADING_CONTROLLERS,
'12.01',
'13.01')
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1581,9 +1658,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
UPGRADE_STATE.DATA_MIGRATION_COMPLETE,
'12.01',
'13.01')
'13.01',
UPGRADE_STATE.STARTED,
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1594,49 +1672,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
expected_results = {
'total_stages': 0,
'result': 'failed',
'result_reason': 'invalid upgrade state for orchestration: data-migration-complete'
}
sw_update_testcase.validate_phase(build_phase, expected_results)
@mock.patch('nfv_vim.strategy._strategy.get_local_host_name',
sw_update_testcase.fake_host_name_controller_1)
def test_sw_upgrade_strategy_build_complete_no_upgrade_required(self):
"""
Test the sw_upgrade strategy build_complete:
- no upgrade required
Verify:
- build fails
"""
self.create_host('controller-0')
self.create_host('controller-1')
self.create_host('compute-0')
self.create_host('compute-1')
self.create_host('compute-2')
self.create_host('compute-3')
self.create_instance('small',
"test_instance_0",
'compute-0')
self.create_instance('small',
"test_instance_1",
'compute-1')
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL
)
fake_upgrade_obj = SwUpgrade()
strategy.sw_update_obj = fake_upgrade_obj
strategy.build_complete(common_strategy.STRATEGY_RESULT.SUCCESS, "")
build_phase = strategy.build_phase.as_dict()
expected_results = {
'total_stages': 0,
'result': 'failed',
'result_reason': 'no upgrade in progress'
'result': 'initial',
}
sw_update_testcase.validate_phase(build_phase, expected_results)
@ -1667,9 +1703,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
UPGRADE_STATE.DATA_MIGRATION_COMPLETE,
'12.01',
'13.01')
'13.01',
UPGRADE_STATE.STARTED,
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1680,8 +1717,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
expected_results = {
'total_stages': 0,
'result': 'failed',
'result_reason': 'invalid upgrade state for orchestration: data-migration-complete'
'result': 'initial',
}
sw_update_testcase.validate_phase(build_phase, expected_results)
@ -1712,9 +1748,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
'13.01',
UPGRADE_STATE.UPGRADING_CONTROLLERS,
'12.01',
'13.01')
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1759,9 +1796,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
'13.01',
UPGRADE_STATE.UPGRADING_CONTROLLERS,
'12.01',
'13.01')
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1806,9 +1844,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
strategy = self.create_sw_upgrade_strategy(
worker_apply_type=SW_UPDATE_APPLY_TYPE.SERIAL,
nfvi_upgrade=nfvi.objects.v1.Upgrade(
'13.01',
UPGRADE_STATE.UPGRADING_CONTROLLERS,
'12.01',
'13.01')
True,
None)
)
fake_upgrade_obj = SwUpgrade()
@ -1872,3 +1911,4 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
self.assertEqual(
120,
strategy_dict['apply_phase']['stages'][0]['steps'][3]['retry_delay'])

13
nfv/nfv-vim/nfv_vim/api/controllers/v1/orchestration/sw_update/_sw_update_strategy.py
View File

@ -180,11 +180,8 @@ class SwUpgradeStrategyCreateData(wsme_types.Base):
name='worker-apply-type')
max_parallel_worker_hosts = wsme_types.wsattr(
int, mandatory=False, name='max-parallel-worker-hosts')
# Disable support for start-upgrade as it was not completed
# start_upgrade = wsme_types.wsattr(
# bool, mandatory=False, default=False, name='start-upgrade')
complete_upgrade = wsme_types.wsattr(
bool, mandatory=False, default=False, name='complete-upgrade')
release = wsme_types.wsattr(
str, mandatory=True, default=None, name='release')
alarm_restrictions = wsme_types.wsattr(
SwUpdateAlarmRestrictionTypes, mandatory=False,
default=SW_UPDATE_ALARM_RESTRICTION_TYPES.STRICT,
@ -686,11 +683,9 @@ class SwUpgradeStrategyAPI(SwUpdateStrategyAPI):
"Invalid value for max-parallel-worker-hosts")
rpc_request.max_parallel_worker_hosts = \
request_data.max_parallel_worker_hosts
rpc_request.default_instance_action = SW_UPDATE_INSTANCE_ACTION.MIGRATE
rpc_request.default_instance_action = SW_UPDATE_INSTANCE_ACTION.STOP_START
rpc_request.alarm_restrictions = request_data.alarm_restrictions
# rpc_request.start_upgrade = request_data.start_upgrade
rpc_request.start_upgrade = False
rpc_request.complete_upgrade = request_data.complete_upgrade
rpc_request.release = request_data.release
vim_connection = pecan.request.vim.open_connection()
vim_connection.send(rpc_request.serialize())
msg = vim_connection.receive(timeout_in_secs=30)

3
nfv/nfv-vim/nfv_vim/api/openstack/_objects.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2019, 2021-2022 Wind River Systems, Inc.
# Copyright (c) 2015-2019, 2021-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -40,6 +40,7 @@ class PlatformServices(Constants):
SYSINV = Constant('sysinv')
PATCHING = Constant('patching')
FM = Constant('fm')
USM = Constant('usm')
# Platform Services Constant

3
nfv/nfv-vim/nfv_vim/debug.ini
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -193,6 +193,7 @@ nfv_plugins.nfvi_plugins.openstack.fm: debug.level.info
nfv_plugins.nfvi_plugins.openstack.patching: debug.level.info
nfv_plugins.nfvi_plugins.openstack.keystone: debug.level.info
nfv_plugins.nfvi_plugins.openstack.sysinv: debug.level.info
nfv_plugins.nfvi_plugins.openstack.usm: debug.level.info
nfv_plugins.nfvi_plugins.openstack.mtc: debug.level.info
nfv_plugins.nfvi_plugins.openstack.glance: debug.level.info
nfv_plugins.nfvi_plugins.openstack.cinder: debug.level.info

16
nfv/nfv-vim/nfv_vim/directors/_sw_mgmt_director.py
View File

@ -76,8 +76,7 @@ class SwMgmtDirector(object):
def create_sw_upgrade_strategy(self, storage_apply_type, worker_apply_type,
max_parallel_worker_hosts,
alarm_restrictions, start_upgrade,
complete_upgrade, callback):
alarm_restrictions, release, callback):
"""
Create Software Upgrade Strategy
"""
@ -90,13 +89,16 @@ class SwMgmtDirector(object):
reason = "strategy already exists of type:%s" % self._sw_update._sw_update_type
return None, reason
# TODO(jkraitbe): Fix controller_apply_type
from nfv_vim.objects import SW_UPDATE_APPLY_TYPE
from nfv_vim.objects import SW_UPDATE_INSTANCE_ACTION
self._sw_update = objects.SwUpgrade()
success, reason = self._sw_update.strategy_build(
strategy_uuid, storage_apply_type,
worker_apply_type, max_parallel_worker_hosts,
alarm_restrictions, start_upgrade,
complete_upgrade, self._ignore_alarms,
self._single_controller)
strategy_uuid, SW_UPDATE_APPLY_TYPE.SERIAL, storage_apply_type,
worker_apply_type, max_parallel_worker_hosts, SW_UPDATE_INSTANCE_ACTION.STOP_START,
alarm_restrictions, release,
self._ignore_alarms, self._single_controller)
schedule.schedule_function_call(callback, success, reason,
self._sw_update.strategy)

9
nfv/nfv-vim/nfv_vim/events/_vim_sw_update_api_events.py
View File

@ -99,13 +99,12 @@ def vim_sw_update_api_create_strategy(connection, msg):
default_instance_action,
alarm_restrictions, _vim_sw_update_api_create_strategy_callback)
elif 'sw-upgrade' == msg.sw_update_type:
start_upgrade = msg.start_upgrade
complete_upgrade = msg.complete_upgrade
release = msg.release
# start_upgrade = msg.start_upgrade
# complete_upgrade = msg.complete_upgrade
uuid, reason = sw_mgmt_director.create_sw_upgrade_strategy(
storage_apply_type, worker_apply_type, max_parallel_worker_hosts,
alarm_restrictions,
start_upgrade, complete_upgrade,
_vim_sw_update_api_create_strategy_callback)
alarm_restrictions, release, _vim_sw_update_api_create_strategy_callback)
elif 'fw-update' == msg.sw_update_type:
uuid, reason = sw_mgmt_director.create_fw_update_strategy(
controller_apply_type, storage_apply_type,

3
nfv/nfv-vim/nfv_vim/nfvi/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -150,6 +150,7 @@ from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_host_update_c
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_host_upgrade_callback # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_register_sw_update_get_callback # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_swact_from_host # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_sw_deploy_precheck # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_unlock_host # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_upgrade_activate # noqa: F401
from nfv_vim.nfvi._nfvi_infrastructure_module import nfvi_upgrade_complete # noqa: F401

37
nfv/nfv-vim/nfv_vim/nfvi/_nfvi_infrastructure_module.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -367,38 +367,51 @@ def nfvi_get_kube_version_list(callback):
return cmd_id
def nfvi_get_upgrade(callback):
def nfvi_get_upgrade(release, callback):
"""
Get upgrade
Get Software deploy
"""
cmd_id = _infrastructure_plugin.invoke_plugin('get_upgrade',
release,
callback=callback)
return cmd_id
def nfvi_upgrade_start(callback):
def nfvi_sw_deploy_precheck(release, callback):
"""
Upgrade start
"""
cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_start',
cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_precheck',
release,
callback=callback)
return cmd_id
def nfvi_upgrade_activate(callback):
def nfvi_upgrade_start(release, callback):
"""
Upgrade activate
Software deploy start
"""
cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_activate',
cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_start',
release,
callback=callback)
return cmd_id
def nfvi_upgrade_complete(callback):
def nfvi_upgrade_activate(release, callback):
"""
Upgrade complete
Software deploy activate
"""
cmd_id = _infrastructure_plugin.invoke_plugin('upgrade_complete',
cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_activate',
release,
callback=callback)
return cmd_id
def nfvi_upgrade_complete(release, callback):
"""
Software deploy complete
"""
cmd_id = _infrastructure_plugin.invoke_plugin('sw_deploy_complete',
release,
callback=callback)
return cmd_id

18
nfv/nfv-vim/nfv_vim/nfvi/objects/v1/_upgrade.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2016 Wind River Systems, Inc.
# Copyright (c) 2016-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -18,11 +18,10 @@ class UpgradeState(Constants):
Upgrade State Constants
"""
UNKNOWN = Constant('unknown')
PRECHECKING = Constant('prechecking')
PRECHECK_PASSED = Constant('precheck-passed')
STARTING = Constant('starting')
STARTED = Constant('started')
DATA_MIGRATION = Constant('data-migration')
DATA_MIGRATION_COMPLETE = Constant('data-migration-complete')
DATA_MIGRATION_FAILED = Constant('data-migration-failed')
UPGRADING_CONTROLLERS = Constant('upgrading-controllers')
UPGRADING_HOSTS = Constant('upgrading-hosts')
ACTIVATION_REQUESTED = Constant('activation-requested')
@ -32,7 +31,7 @@ class UpgradeState(Constants):
COMPLETED = Constant('completed')
ABORTING = Constant('aborting')
ABORT_COMPLETING = Constant('abort-completing')
ABORTING_ROLLBACK = Constant('aborting-reinstall')
ABORTING_ROLLBACK = Constant('aborting-redeploy')
# Upgrade Constant Instantiation
@ -43,8 +42,9 @@ class Upgrade(ObjectData):
"""
NFVI Upgrade Object
"""
def __init__(self, state, from_release, to_release):
def __init__(self, release, state, reboot_required, hosts_states):
super(Upgrade, self).__init__('1.0.0')
self.update(dict(state=state,
from_release=from_release,
to_release=to_release))
self.update(dict(release=release,
state=state,
reboot_required=reboot_required,
hosts_states=hosts_states))

23
nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2016 Wind River Systems, Inc.
# Copyright (c) 2016-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -29,10 +29,10 @@ class SwUpgrade(SwUpdate):
sw_update_uuid=sw_update_uuid,
strategy_data=strategy_data)
def strategy_build(self, strategy_uuid, storage_apply_type,
worker_apply_type, max_parallel_worker_hosts,
alarm_restrictions, start_upgrade,
complete_upgrade, ignore_alarms, single_controller):
def strategy_build(self, strategy_uuid, controller_apply_type, storage_apply_type,
worker_apply_type, default_instance_action, max_parallel_worker_hosts,
alarm_restrictions, release,
ignore_alarms, single_controller):
"""
Create a software upgrade strategy
"""
@ -43,10 +43,17 @@ class SwUpgrade(SwUpdate):
return False, reason
self._strategy = strategy.SwUpgradeStrategy(
strategy_uuid, storage_apply_type, worker_apply_type,
strategy_uuid,
controller_apply_type,
storage_apply_type,
worker_apply_type,
default_instance_action,
max_parallel_worker_hosts,
alarm_restrictions, start_upgrade, complete_upgrade,
ignore_alarms, single_controller)
alarm_restrictions,
release,
ignore_alarms,
single_controller,
)
self._strategy.sw_update_obj = self
self._strategy.build()

15
nfv/nfv-vim/nfv_vim/rpc/_rpc_message_sw_update.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -59,10 +59,9 @@ class APIRequestCreateSwUpdateStrategy(RPCMessage):
class APIRequestCreateSwUpgradeStrategy(APIRequestCreateSwUpdateStrategy):
"""
RPC API Request Message - Create Software Upgrade Strategy
RPC API Request Message - Create Software Deploy Strategy
"""
start_upgrade = None
complete_upgrade = None
release = None
def __init__(self, msg_version=RPC_MSG_VERSION.VERSION_1_0,
msg_type=RPC_MSG_TYPE.CREATE_SW_UPGRADE_STRATEGY_REQUEST,
@ -72,16 +71,14 @@ class APIRequestCreateSwUpgradeStrategy(APIRequestCreateSwUpdateStrategy):
def serialize_payload(self, msg):
super(APIRequestCreateSwUpgradeStrategy, self).serialize_payload(msg)
msg['start_upgrade'] = self.start_upgrade
msg['complete_upgrade'] = self.complete_upgrade
msg['release'] = self.release
def deserialize_payload(self, msg):
super(APIRequestCreateSwUpgradeStrategy, self).deserialize_payload(msg)
self.start_upgrade = msg.get('start_upgrade', None)
self.complete_upgrade = msg.get('complete_upgrade', None)
self.release = msg.get('release', None)
def __str__(self):
return "create-sw-upgrade-strategy request: %s" % self.deserialize_payload
return "create-sw-deploy-strategy request: %s" % self.deserialize_payload
class APIRequestCreateKubeRootcaUpdateStrategy(APIRequestCreateSwUpdateStrategy):

3
nfv/nfv-vim/nfv_vim/strategy/__init__.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2015-2023 Wind River Systems, Inc.
# Copyright (c) 2015-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -56,6 +56,7 @@ from nfv_vim.strategy._strategy_steps import StartInstancesStep # noqa: F401
from nfv_vim.strategy._strategy_steps import StopInstancesStep # noqa: F401
from nfv_vim.strategy._strategy_steps import STRATEGY_STEP_NAME # noqa: F401
from nfv_vim.strategy._strategy_steps import SwactHostsStep # noqa: F401
from nfv_vim.strategy._strategy_steps import SwDeployPrecheckStep # noqa: F401
from nfv_vim.strategy._strategy_steps import SwPatchHostsStep # noqa: F401
from nfv_vim.strategy._strategy_steps import SystemConfigUpdateHostsStep # noqa: F401
from nfv_vim.strategy._strategy_steps import SystemStabilizeStep # noqa: F401

559
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
View File

@ -1054,6 +1054,16 @@ class PatchControllerHostsMixin(UpdateControllerHostsMixin):
strategy.SwPatchHostsStep)
class SwDeployControllerHostsMixin(UpdateControllerHostsMixin):
def _add_controller_strategy_stages(self, controllers, reboot):
from nfv_vim import strategy
return self._add_update_controller_strategy_stages(
controllers,
reboot,
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS,
strategy.UpgradeHostsStep)
class UpdateSystemConfigControllerHostsMixin(UpdateControllerHostsMixin):
def _add_system_config_controller_strategy_stages(self, controllers):
"""
@ -1141,6 +1151,19 @@ class PatchStorageHostsMixin(UpdateStorageHostsMixin):
strategy.SwPatchHostsStep)
class SwDeployStorageHostsMixin(UpdateStorageHostsMixin):
def _add_storage_strategy_stages(self, storage_hosts, reboot):
"""
Add storage software patch stages to a strategy
"""
from nfv_vim import strategy
return self._add_update_storage_strategy_stages(
storage_hosts,
reboot,
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_STORAGE_HOSTS,
strategy.UpgradeHostsStep)
class UpdateSystemConfigStorageHostsMixin(UpdateStorageHostsMixin):
def _add_system_config_storage_strategy_stages(self, storage_hosts):
"""
@ -1238,6 +1261,8 @@ class UpdateWorkerHostsMixin(object):
# Migrate or stop instances as necessary
if SW_UPDATE_INSTANCE_ACTION.MIGRATE == self._default_instance_action:
# TODO(jkraitbe): Should probably be:
# if len(openstack_hosts) and len(instances)
if len(openstack_hosts):
if SW_UPDATE_APPLY_TYPE.PARALLEL == self._worker_apply_type:
# Disable host services before migrating to ensure
@ -1327,6 +1352,15 @@ class PatchWorkerHostsMixin(UpdateWorkerHostsMixin):
strategy.SwPatchHostsStep)
class SwDeployWorkerHostsMixin(UpdateWorkerHostsMixin):
def _add_worker_strategy_stages(self, worker_hosts, reboot):
from nfv_vim import strategy
return self._add_update_worker_strategy_stages(
worker_hosts,
reboot,
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS,
strategy.UpgradeHostsStep)
class UpgradeKubeletWorkerHostsMixin(UpdateWorkerHostsMixin):
def _add_kubelet_worker_strategy_stages(self, worker_hosts, to_version, reboot, stage_name):
from nfv_vim import strategy
@ -1703,40 +1737,34 @@ class SwPatchStrategy(SwUpdateStrategy,
# The Software Upgrade Strategy
#
###################################################################
class SwUpgradeStrategy(SwUpdateStrategy):
class SwUpgradeStrategy(
SwUpdateStrategy,
SwDeployControllerHostsMixin,
SwDeployStorageHostsMixin,
SwDeployWorkerHostsMixin,
):
"""
Software Upgrade - Strategy
"""
def __init__(self, uuid, storage_apply_type, worker_apply_type,
max_parallel_worker_hosts,
alarm_restrictions, start_upgrade, complete_upgrade,
def __init__(self, uuid,
controller_apply_type, storage_apply_type, worker_apply_type,
max_parallel_worker_hosts, default_instance_action,
alarm_restrictions, release,
ignore_alarms, single_controller):
super(SwUpgradeStrategy, self).__init__(
uuid,
STRATEGY_NAME.SW_UPGRADE,
SW_UPDATE_APPLY_TYPE.SERIAL,
controller_apply_type,
storage_apply_type,
SW_UPDATE_APPLY_TYPE.IGNORE,
worker_apply_type,
max_parallel_worker_hosts,
SW_UPDATE_INSTANCE_ACTION.MIGRATE,
default_instance_action,
alarm_restrictions,
ignore_alarms)
# Note: The support for start_upgrade was implemented and (mostly)
# tested, but there is a problem. When the sw-upgrade-start stage
# runs, it will start the upgrade, upgrade controller-1 and swact to
# it. However, when controller-1 becomes active, it will be using the
# snapshot of the VIM database that was created when the upgrade was
# started, so the strategy object created from the database will be
# long out of date (it thinks the upgrade start step is still in
# progress) and the strategy apply will fail. Fixing this would be
# complex, so we will not support the start_upgrade option for now,
# which would only have been for lab use.
if start_upgrade:
raise Exception("No support for start_upgrade")
self._start_upgrade = start_upgrade
self._complete_upgrade = complete_upgrade
self._release = release
# The following alarms will not prevent a software upgrade operation
IGNORE_ALARMS = ['900.005', # Upgrade in progress
'900.201', # Software upgrade auto apply in progress
@ -1768,37 +1796,50 @@ class SwUpgradeStrategy(SwUpdateStrategy):
"""
from nfv_vim import strategy
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
stage.add_step(strategy.QueryAlarmsStep(
ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.QueryUpgradeStep())
stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_QUERY)
stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.QueryUpgradeStep(release=self._release))
self.build_phase.add_stage(stage)
super(SwUpgradeStrategy, self).build()
def _swact_fix(self, stage, controller_name):
"""Add a SWACT to a stage on DX systems
Currently, certains steps during sw-deploy must be done on a specific controller.
Here we insert arbitrary SWACTs to meet those requirements.
"""
if self._single_controller:
return
from nfv_vim import strategy
from nfv_vim import tables
host_table = tables.tables_get_host_table()
for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER):
if controller_name == host.name:
stage.add_step(strategy.SwactHostsStep([host]))
break
def _add_upgrade_start_stage(self):
"""
Add upgrade start strategy stage
"""
from nfv_vim import strategy
from nfv_vim import tables
host_table = tables.tables_get_host_table()
controller_1_host = None
for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER):
if HOST_NAME.CONTROLLER_1 == host.name:
controller_1_host = host
break
host_list = [controller_1_host]
stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START)
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_START)
# Do not ignore any alarms when starting an upgrade
stage.add_step(strategy.QueryAlarmsStep(True))
# Upgrade start can only be done from controller-0
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.UpgradeStartStep())
stage.add_step(strategy.QueryAlarmsStep(True, ignore_alarms=self._ignore_alarms))
# sw-deploy start must be done on controller-0
self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
stage.add_step(strategy.SwDeployPrecheckStep(release=self._release))
stage.add_step(strategy.UpgradeStartStep(release=self._release))
stage.add_step(strategy.SystemStabilizeStep())
# sw-deploy host must first be on controller-1
self._swact_fix(stage, HOST_NAME.CONTROLLER_0)
self.apply_phase.add_stage(stage)
def _add_upgrade_complete_stage(self):
@ -1806,280 +1847,16 @@ class SwUpgradeStrategy(SwUpdateStrategy):
Add upgrade complete strategy stage
"""
from nfv_vim import strategy
from nfv_vim import tables
host_table = tables.tables_get_host_table()
controller_1_host = None
for host in host_table.get_by_personality(HOST_PERSONALITY.CONTROLLER):
if HOST_NAME.CONTROLLER_1 == host.name:
controller_1_host = host
break
host_list = [controller_1_host]
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
# Upgrade complete can only be done from controller-0
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.UpgradeActivateStep())
stage.add_step(strategy.UpgradeCompleteStep())
stage = strategy.StrategyStage(strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_COMPLETE)
stage.add_step(strategy.QueryAlarmsStep(ignore_alarms=self._ignore_alarms))
# sw-deploy activate/complete should be on controller-0
self._swact_fix(stage, HOST_NAME.CONTROLLER_1)
stage.add_step(strategy.UpgradeActivateStep(release=self._release))
stage.add_step(strategy.UpgradeCompleteStep(release=self._release))
stage.add_step(strategy.SystemStabilizeStep())
self.apply_phase.add_stage(stage)
def _add_controller_strategy_stages(self, controllers, reboot):
"""
Add controller software upgrade strategy stages
"""
from nfv_vim import strategy
from nfv_vim import tables
host_table = tables.tables_get_host_table()
if 2 > host_table.total_by_personality(HOST_PERSONALITY.CONTROLLER):
DLOG.warn("Not enough controllers to apply software upgrades.")
reason = 'not enough controllers to apply software upgrades'
return False, reason
controller_0_host = None
controller_1_host = None
for host in controllers:
if HOST_PERSONALITY.WORKER in host.personality:
# Do nothing for AIO hosts. We let the worker code handle everything.
# This is done to handle the case where stx-openstack is
# installed and there could be instances running on the
# AIO-DX controllers which need to be migrated.
if self._single_controller:
DLOG.warn("Cannot apply software upgrades to AIO-SX deployment.")
reason = 'cannot apply software upgrades to AIO-SX deployment'
return False, reason
else:
return True, ''
elif HOST_NAME.CONTROLLER_1 == host.name:
controller_1_host = host
elif HOST_NAME.CONTROLLER_0 == host.name:
controller_0_host = host
if controller_1_host is not None:
host_list = [controller_1_host]
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1946255
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
timeout_in_secs=4 * 60 * 60,
ignore_alarms=self._ignore_alarms))
self.apply_phase.add_stage(stage)
if controller_0_host is not None:
host_list = [controller_0_host]
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_CONTROLLERS)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
if controller_1_host is not None:
# Only swact to controller-1 if it was upgraded. If we are only
# upgrading controller-0, then controller-1 needs to be
# active already.
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1946255
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
timeout_in_secs=4 * 60 * 60,
ignore_alarms=self._ignore_alarms))
self.apply_phase.add_stage(stage)
return True, ''
def _add_storage_strategy_stages(self, storage_hosts, reboot):
"""
Add storage software upgrade strategy stages
"""
from nfv_vim import strategy
storage_0_host_list = list()
storage_0_host_lists = list()
other_storage_host_list = list()
for host in storage_hosts:
if HOST_NAME.STORAGE_0 == host.name:
storage_0_host_list.append(host)
else:
other_storage_host_list.append(host)
if len(storage_0_host_list) == 1:
storage_0_host_lists, reason = self._create_storage_host_lists(
storage_0_host_list)
if storage_0_host_lists is None:
return False, reason
other_storage_host_lists, reason = self._create_storage_host_lists(
other_storage_host_list)
if other_storage_host_lists is None:
return False, reason
# Upgrade storage-0 first and on its own since it has a ceph monitor
if len(storage_0_host_lists) == 1:
combined_host_lists = storage_0_host_lists + other_storage_host_lists
else:
combined_host_lists = other_storage_host_lists
for host_list in combined_host_lists:
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_STORAGE_HOSTS)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1946255
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
# After storage node(s) are unlocked, we need extra time to
# allow the OSDs to go back in sync and the storage related
# alarms to clear. We no longer wipe the OSD disks when upgrading
# a storage node, so they should only be syncing data that changed
# while they were being upgraded.
stage.add_step(strategy.WaitDataSyncStep(
timeout_in_secs=2 * 60 * 60,
ignore_alarms=self._ignore_alarms))
self.apply_phase.add_stage(stage)
return True, ''
def _add_worker_strategy_stages(self, worker_hosts, reboot):
"""
Add worker software upgrade strategy stages
"""
from nfv_vim import strategy
from nfv_vim import tables
host_lists, reason = self._create_worker_host_lists(worker_hosts, reboot)
if host_lists is None:
return False, reason
instance_table = tables.tables_get_instance_table()
for host_list in host_lists:
instance_list = list()
for host in host_list:
for instance in instance_table.on_host(host.name):
if not instance.is_locked():
instance_list.append(instance)
else:
DLOG.warn("Instance %s must not be shut down" %
instance.name)
reason = ('instance %s must not be shut down' %
instance.name)
return False, reason
# Computes with no instances
if 0 == len(instance_list):
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1914836
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
# AIO Controller hosts will undergo WaitDataSyncStep step
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
timeout_in_secs=4 * 60 * 60,
ignore_alarms=self._ignore_alarms))
else:
# Worker hosts will undergo:
# 1) WaitAlarmsClear step if openstack is installed.
# 2) SystemStabilizeStep step if openstack is not installed.
if any([host.openstack_control or host.openstack_compute
for host in host_list]):
# Hosts with openstack that just need to wait for services to start up:
stage.add_step(strategy.WaitAlarmsClearStep(
timeout_in_secs=10 * 60,
ignore_alarms=self._ignore_alarms))
else:
stage.add_step(strategy.SystemStabilizeStep())
self.apply_phase.add_stage(stage)
continue
# Computes with instances
stage = strategy.StrategyStage(
strategy.STRATEGY_STAGE_NAME.SW_UPGRADE_WORKER_HOSTS)
stage.add_step(strategy.QueryAlarmsStep(
True, ignore_alarms=self._ignore_alarms))
if SW_UPDATE_APPLY_TYPE.PARALLEL == self._worker_apply_type:
# Disable host services before migrating to ensure
# instances do not migrate to worker hosts in the
# same set of hosts.
if host_list[0].host_service_configured(
HOST_SERVICES.COMPUTE):
stage.add_step(strategy.DisableHostServicesStep(
host_list, HOST_SERVICES.COMPUTE))
# TODO(ksmith)
# When support is added for orchestration on
# non-OpenStack worker nodes, support for disabling
# kubernetes services will have to be added.
stage.add_step(strategy.MigrateInstancesStep(instance_list))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1914836
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
# AIO Controller hosts will undergo WaitDataSyncStep step
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
timeout_in_secs=4 * 60 * 60,
ignore_alarms=self._ignore_alarms))
else:
# Worker hosts will undergo:
# 1) WaitAlarmsClear step if openstack is installed.
# 2) SystemStabilizeStep step if openstack is not installed.
if any([host.openstack_control or host.openstack_compute
for host in host_list]):
# Hosts with openstack that just need to wait for
# services to start up:
stage.add_step(strategy.WaitAlarmsClearStep(
timeout_in_secs=10 * 60,
ignore_alarms=self._ignore_alarms))
else:
stage.add_step(strategy.SystemStabilizeStep())
self.apply_phase.add_stage(stage)
return True, ''
def build_complete(self, result, result_reason):
"""
Strategy Build Complete
@ -2096,86 +1873,24 @@ class SwUpgradeStrategy(SwUpdateStrategy):
if result in [strategy.STRATEGY_RESULT.SUCCESS,
strategy.STRATEGY_RESULT.DEGRADED]:
# Check whether the upgrade is in a valid state for orchestration
if self.nfvi_upgrade is None:
if not self._start_upgrade:
DLOG.warn("No upgrade in progress.")
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = 'no upgrade in progress'
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
self.save()
return
else:
if self._start_upgrade:
valid_states = [UPGRADE_STATE.STARTED,
UPGRADE_STATE.DATA_MIGRATION_COMPLETE,
UPGRADE_STATE.UPGRADING_CONTROLLERS,
UPGRADE_STATE.UPGRADING_HOSTS]
else:
valid_states = [UPGRADE_STATE.UPGRADING_CONTROLLERS,
UPGRADE_STATE.UPGRADING_HOSTS]
if self.nfvi_upgrade.state not in valid_states:
DLOG.warn("Invalid upgrade state for orchestration: %s." %
self.nfvi_upgrade.state)
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = (
'invalid upgrade state for orchestration: %s' %
self.nfvi_upgrade.state)
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
self.save()
return
# If controller-1 has been upgraded and we have yet to upgrade
# controller-0, then controller-1 must be active.
if UPGRADE_STATE.UPGRADING_CONTROLLERS == self.nfvi_upgrade.state:
if HOST_NAME.CONTROLLER_1 != get_local_host_name():
DLOG.warn(
"Controller-1 must be active for orchestration to "
"upgrade controller-0.")
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = \
strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = (
'controller-1 must be active for orchestration to '
'upgrade controller-0')
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
self.save()
return
host_table = tables.tables_get_host_table()
if self._nfvi_alarms:
DLOG.warn(
"Active alarms found, can't apply software upgrade.")
alarm_id_list = ""
for alarm_data in self._nfvi_alarms:
if alarm_id_list:
alarm_id_list += ', '
alarm_id_list += alarm_data['alarm_id']
DLOG.warn("... active alarms: %s" % alarm_id_list)
DLOG.warn("Active alarms found, can't apply sw-deployment.")
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = 'active alarms present ; '
self.build_phase.result_reason += alarm_id_list
self.build_phase.result_reason = 'active alarms present'
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
self.save()
return
host_table = tables.tables_get_host_table()
for host in list(host_table.values()):
# Only allow upgrade orchestration when all hosts are
# available. It is not safe to automate upgrade application
# when we do not have full redundancy.
if not (host.is_unlocked() and host.is_enabled() and
host.is_available()):
# All hosts must be unlock/enabled/online
if not (host.is_unlocked() and host.is_enabled() and host.is_available()):
DLOG.warn(
"All %s hosts must be unlocked-enabled-available, "
"can't apply software upgrades." % host.personality)
"All hosts must be unlocked-enabled-available, "
"can't apply sw-deployment: %s" % host.name)
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = \
strategy.STRATEGY_PHASE_RESULT.FAILED
@ -2187,55 +1902,51 @@ class SwUpgradeStrategy(SwUpdateStrategy):
self.save()
return
controller_hosts = list()
reboot_required = self.nfvi_upgrade.reboot_required
controller_strategy = self._add_controller_strategy_stages
controllers_hosts = list()
storage_hosts = list()
worker_hosts = list()
if self.nfvi_upgrade is None:
# Start upgrade
self._add_upgrade_start_stage()
# All hosts will be upgraded
for host in list(host_table.values()):
if HOST_PERSONALITY.CONTROLLER in host.personality:
controller_hosts.append(host)
elif HOST_PERSONALITY.STORAGE in host.personality:
storage_hosts.append(host)
# TODO(jkraitbe): Exclude hosts that are already deployed.
# The hosts states are found in self.nfvi_upgrade.hosts_states.
# None means deployment hasn't started.
self._add_upgrade_start_stage()
for host in host_table.values():
if HOST_PERSONALITY.CONTROLLER in host.personality:
controllers_hosts.append(host)
if HOST_PERSONALITY.WORKER in host.personality:
worker_hosts.append(host)
else:
# Only hosts not yet upgraded will be upgraded
to_load = self.nfvi_upgrade.to_release
for host in list(host_table.values()):
if host.software_load == to_load:
# No need to upgrade this host
continue
# We need to use this strategy on AIO type
controller_strategy = self._add_worker_strategy_stages
if HOST_PERSONALITY.CONTROLLER in host.personality:
controller_hosts.append(host)
elif HOST_PERSONALITY.STORAGE in host.personality:
storage_hosts.append(host)
elif HOST_PERSONALITY.STORAGE in host.personality:
storage_hosts.append(host)
elif HOST_PERSONALITY.WORKER in host.personality:
worker_hosts.append(host)
if HOST_PERSONALITY.WORKER in host.personality:
worker_hosts.append(host)
else:
DLOG.error(f"Unsupported personality for host {host.name}.")
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = \
strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = \
'Unsupported personality for host'
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
self.save()
return
STRATEGY_CREATION_COMMANDS = [
(self._add_controller_strategy_stages,
controller_hosts, True),
(self._add_storage_strategy_stages,
storage_hosts, True),
(self._add_worker_strategy_stages,
worker_hosts, True)
strategy_pairs = [
(controller_strategy, controllers_hosts),
(self._add_storage_strategy_stages, storage_hosts),
(self._add_worker_strategy_stages, worker_hosts)
]
for add_strategy_stages_function, host_list, reboot in \
STRATEGY_CREATION_COMMANDS:
for stage_func, host_list in strategy_pairs:
if host_list:
success, reason = add_strategy_stages_function(
host_list, reboot)
success, reason = stage_func(host_list, reboot_required)
if not success:
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = \
@ -2246,14 +1957,13 @@ class SwUpgradeStrategy(SwUpdateStrategy):
self.save()
return
if self._complete_upgrade:
self._add_upgrade_complete_stage()
self._add_upgrade_complete_stage()
if 0 == len(self.apply_phase.stages):
DLOG.warn("No software upgrades need to be applied.")
DLOG.warn("No sw-deployments need to be applied.")
self._state = strategy.STRATEGY_STATE.BUILD_FAILED
self.build_phase.result = strategy.STRATEGY_PHASE_RESULT.FAILED
self.build_phase.result_reason = ('no software upgrades need to be '
self.build_phase.result_reason = ('no sw-deployments patches need to be '
'applied')
self.sw_update_obj.strategy_build_complete(
False, self.build_phase.result_reason)
@ -2275,14 +1985,14 @@ class SwUpgradeStrategy(SwUpdateStrategy):
super(SwUpgradeStrategy, self).from_dict(data, build_phase, apply_phase,
abort_phase)
self._single_controller = data['single_controller']
self._start_upgrade = data['start_upgrade']
self._complete_upgrade = data['complete_upgrade']
self._release = data['release']
nfvi_upgrade_data = data['nfvi_upgrade_data']
if nfvi_upgrade_data:
self._nfvi_upgrade = nfvi.objects.v1.Upgrade(
nfvi_upgrade_data['release'],
nfvi_upgrade_data['state'],
nfvi_upgrade_data['from_release'],
nfvi_upgrade_data['to_release'])
nfvi_upgrade_data['reboot_required'],
nfvi_upgrade_data['hosts_states'])
else:
self._nfvi_upgrade = None
@ -2294,8 +2004,7 @@ class SwUpgradeStrategy(SwUpdateStrategy):
"""
data = super(SwUpgradeStrategy, self).as_dict()
data['single_controller'] = self._single_controller
data['start_upgrade'] = self._start_upgrade
data['complete_upgrade'] = self._complete_upgrade
data['release'] = self._release
if self._nfvi_upgrade:
nfvi_upgrade_data = self._nfvi_upgrade.as_dict()
else:

151
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
View File

@ -36,6 +36,7 @@ class StrategyStepNames(Constants):
UNLOCK_HOSTS = Constant('unlock-hosts')
REBOOT_HOSTS = Constant('reboot-hosts')
UPGRADE_HOSTS = Constant('upgrade-hosts')
SW_DEPLOY_PRECHECK = Constant('sw-deploy-precheck')
START_UPGRADE = Constant('start-upgrade')
ACTIVATE_UPGRADE = Constant('activate-upgrade')
COMPLETE_UPGRADE = Constant('complete-upgrade')
@ -934,6 +935,8 @@ class UpgradeHostsStep(strategy.StrategyStep):
"""
Returns the number of hosts that are upgraded
"""
# TODO(jkraitbe): Use deploy/host_list instead
total_hosts_upgraded = 0
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
@ -942,8 +945,8 @@ class UpgradeHostsStep(strategy.StrategyStep):
return -1
if (host.is_online() and
host.target_load == self.strategy.nfvi_upgrade.to_release and
host.software_load == self.strategy.nfvi_upgrade.to_release):
host.target_load == self.strategy.nfvi_upgrade.release and
host.software_load == self.strategy.nfvi_upgrade.release):
total_hosts_upgraded += 1
return total_hosts_upgraded
@ -1029,17 +1032,124 @@ class UpgradeHostsStep(strategy.StrategyStep):
data['entity_uuids'] = self._host_uuids
return data
class SwDeployPrecheckStep(strategy.StrategyStep):
"""
Software Deploy Precheck - Strategy Step
"""
def __init__(self, release):
super(SwDeployPrecheckStep, self).__init__(
STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK, timeout_in_secs=600)
self._wait_time = 0
self._query_inprogress = False
self._release = release
@coroutine
def _sw_deploy_precheck_callback(self):
"""
Software deploy precheck callback
"""
response = (yield)
DLOG.debug("sw-deploy precheck callback response=%s." % response)
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
@coroutine
def _get_upgrade_callback(self):
"""
Get sw-deploy precheck callback
"""
from nfv_vim import nfvi
response = (yield)
DLOG.debug("Get-sw-deploy precheck callback response=%s." % response)
self._query_inprogress = False
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_upgrade = response['result-data']
if self.strategy.nfvi_upgrade.state != \
nfvi.objects.v1.UPGRADE_STATE.STARTED:
# Keep waiting for sw-deploy to start
pass
else:
# Sw_deploy has started
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "")
def apply(self):
"""
Software deploy precheck
"""
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_sw_deploy_precheck(self._release, self._sw_deploy_precheck_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
"""
Handle Host events
"""
from nfv_vim import nfvi
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event == STRATEGY_EVENT.HOST_AUDIT:
if 0 == self._wait_time:
self._wait_time = timers.get_monotonic_timestamp_in_ms()
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) // 1000
# Wait at least 60 seconds before checking sw_deploy for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback())
return True
return False
def from_dict(self, data):
"""
Returns the sw-deploy precheck step object initialized using the given
dictionary
"""
super(SwDeployPrecheckStep, self).from_dict(data)
self._release = data["release"]
return self
def as_dict(self):
"""
Represent the sw-deploy precheck step as a dictionary
"""
data = super(SwDeployPrecheckStep, self).as_dict()
data['release'] = self._release
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class UpgradeStartStep(strategy.StrategyStep):
"""
Upgrade Start - Strategy Step
"""
def __init__(self):
def __init__(self, release):
super(UpgradeStartStep, self).__init__(
STRATEGY_STEP_NAME.START_UPGRADE, timeout_in_secs=600)
self._wait_time = 0
self._query_inprogress = False
self._release = release
@coroutine
def _start_upgrade_callback(self):
@ -1091,7 +1201,7 @@ class UpgradeStartStep(strategy.StrategyStep):
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_start(self._start_upgrade_callback())
nfvi.nfvi_upgrade_start(self._release, self._start_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
@ -1111,7 +1221,7 @@ class UpgradeStartStep(strategy.StrategyStep):
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback())
return True
return False
@ -1124,6 +1234,7 @@ class UpgradeStartStep(strategy.StrategyStep):
super(UpgradeStartStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
self._release = data["release"]
return self
def as_dict(self):
@ -1134,6 +1245,7 @@ class UpgradeStartStep(strategy.StrategyStep):
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['release'] = self._release
return data
@ -1142,12 +1254,13 @@ class UpgradeActivateStep(strategy.StrategyStep):
Upgrade Activate - Strategy Step
"""
def __init__(self):
def __init__(self, release):
super(UpgradeActivateStep, self).__init__(
STRATEGY_STEP_NAME.ACTIVATE_UPGRADE, timeout_in_secs=900)
self._wait_time = 0
self._query_inprogress = False
self._release = release
@coroutine
def _activate_upgrade_callback(self):
@ -1199,7 +1312,7 @@ class UpgradeActivateStep(strategy.StrategyStep):
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_activate(self._activate_upgrade_callback())
nfvi.nfvi_upgrade_activate(self._release, self._activate_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
@ -1219,7 +1332,7 @@ class UpgradeActivateStep(strategy.StrategyStep):
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback())
return True
return False
@ -1232,6 +1345,7 @@ class UpgradeActivateStep(strategy.StrategyStep):
super(UpgradeActivateStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
self._release = data["release"]
return self
def as_dict(self):
@ -1242,6 +1356,7 @@ class UpgradeActivateStep(strategy.StrategyStep):
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['release'] = self._release
return data
@ -1250,12 +1365,13 @@ class UpgradeCompleteStep(strategy.StrategyStep):
Upgrade Complete - Strategy Step
"""
def __init__(self):
def __init__(self, release):
super(UpgradeCompleteStep, self).__init__(
STRATEGY_STEP_NAME.COMPLETE_UPGRADE, timeout_in_secs=300)
self._wait_time = 0
self._query_inprogress = False
self._release = release
@coroutine
def _complete_upgrade_callback(self):
@ -1304,7 +1420,7 @@ class UpgradeCompleteStep(strategy.StrategyStep):
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_upgrade_complete(self._complete_upgrade_callback())
nfvi.nfvi_upgrade_complete(self._release, self._complete_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def handle_event(self, event, event_data=None):
@ -1324,7 +1440,7 @@ class UpgradeCompleteStep(strategy.StrategyStep):
# Wait at least 60 seconds before checking upgrade for first time
if 60 <= secs_expired and not self._query_inprogress:
self._query_inprogress = True
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback())
return True
return False
@ -1337,6 +1453,7 @@ class UpgradeCompleteStep(strategy.StrategyStep):
super(UpgradeCompleteStep, self).from_dict(data)
self._wait_time = 0
self._query_inprogress = False
self._release = data["release"]
return self
def as_dict(self):
@ -1347,6 +1464,7 @@ class UpgradeCompleteStep(strategy.StrategyStep):
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['release'] = self._release
return data
@ -2755,15 +2873,18 @@ class QueryUpgradeStep(strategy.StrategyStep):
"""
Query Upgrade - Strategy Step
"""
def __init__(self):
def __init__(self, release):
super(QueryUpgradeStep, self).__init__(
STRATEGY_STEP_NAME.QUERY_UPGRADE, timeout_in_secs=60)
self._release = release
@coroutine
def _get_upgrade_callback(self):
"""
Get Upgrade Callback
"""
response = (yield)
DLOG.debug("Query-Upgrade callback response=%s." % response)
@ -2784,7 +2905,7 @@ class QueryUpgradeStep(strategy.StrategyStep):
from nfv_vim import nfvi
DLOG.info("Step (%s) apply." % self._name)
nfvi.nfvi_get_upgrade(self._get_upgrade_callback())
nfvi.nfvi_get_upgrade(self._release, self._get_upgrade_callback())
return strategy.STRATEGY_STEP_RESULT.WAIT, ""
def as_dict(self):
@ -2795,6 +2916,7 @@ class QueryUpgradeStep(strategy.StrategyStep):
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
data['release'] = None
return data
@ -4965,6 +5087,9 @@ def strategy_step_rebuild_from_dict(data):
elif STRATEGY_STEP_NAME.UPGRADE_HOSTS == data['name']:
step_obj = object.__new__(UpgradeHostsStep)
elif STRATEGY_STEP_NAME.SW_DEPLOY_PRECHECK == data['name']:
step_obj = object.__new__(SwDeployPrecheckStep)
elif STRATEGY_STEP_NAME.START_UPGRADE == data['name']:
step_obj = object.__new__(UpgradeStartStep)

12
nfv/tox.ini
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2018-2023 Wind River Systems, Inc.
# Copyright (c) 2018-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -38,11 +38,11 @@ nfv_plugins_src_dir = {[nfv]nfv_plugins_dir}/nfv_plugins
nfv_vim_src_dir = {[nfv]nfv_vim_dir}/nfv_vim
nfv_test_src_dir = {[nfv]nfv_test_dir}
deps = {[nfv]nfv_client_dir}
{[nfv]nfv_common_dir}
{[nfv]nfv_plugins_dir}
{[nfv]nfv_vim_dir}
{[nfv]stx_fault_dir}/fm-api/source
deps = -e{[nfv]nfv_client_dir}
-e{[nfv]nfv_common_dir}
-e{[nfv]nfv_plugins_dir}
-e{[nfv]nfv_vim_dir}
-e{[nfv]stx_fault_dir}/fm-api/source
iso8601
keyring
kombu