133 lines
5.6 KiB
Python
133 lines
5.6 KiB
Python
#
|
|
# Copyright (c) 2020-2022 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import retrying
|
|
import time
|
|
|
|
from dcmanager.common import consts
|
|
from dcmanager.common.exceptions import StrategyStoppedException
|
|
from dcmanager.db import api as db_api
|
|
from dcmanager.orchestrator.states.base import BaseState
|
|
|
|
|
|
# Max time: 10 minutes = 60 queries x 10 seconds between each query
|
|
DEFAULT_MAX_QUERIES = 60
|
|
DEFAULT_SLEEP_DURATION = 10
|
|
|
|
|
|
class CompletingUpgradeState(BaseState):
|
|
"""Upgrade state actions for completing an upgrade"""
|
|
|
|
def __init__(self, region_name):
|
|
super(CompletingUpgradeState, self).__init__(
|
|
next_state=consts.STRATEGY_STATE_DELETING_LOAD, region_name=region_name)
|
|
# max time to wait (in seconds) is: sleep_duration * max_queries
|
|
self.sleep_duration = DEFAULT_SLEEP_DURATION
|
|
self.max_queries = DEFAULT_MAX_QUERIES
|
|
|
|
@retrying.retry(stop_max_attempt_number=consts.PLATFORM_RETRY_MAX_ATTEMPTS,
|
|
wait_fixed=consts.PLATFORM_RETRY_SLEEP_MILLIS)
|
|
def _get_software_version(self, strategy_step):
|
|
"""Internal utility method to query software version from a subcloud
|
|
|
|
This method is 'retry' wrapped to attempt multiple times with a
|
|
small wait period between attempts if any exception is raised
|
|
"""
|
|
region = self.get_region_name(strategy_step)
|
|
return self.get_sysinv_client(region).get_system().software_version
|
|
|
|
@retrying.retry(stop_max_attempt_number=consts.PLATFORM_RETRY_MAX_ATTEMPTS,
|
|
wait_fixed=consts.PLATFORM_RETRY_SLEEP_MILLIS)
|
|
def _get_upgrades(self, strategy_step):
|
|
"""Internal utility method to query a subcloud for its upgrades
|
|
|
|
This method is 'retry' wrapped to attempt multiple times with a
|
|
small wait period between attempts if any exception is raised
|
|
"""
|
|
region = self.get_region_name(strategy_step)
|
|
return self.get_sysinv_client(region).get_upgrades()
|
|
|
|
@retrying.retry(stop_max_attempt_number=consts.PLATFORM_RETRY_MAX_ATTEMPTS,
|
|
wait_fixed=consts.PLATFORM_RETRY_SLEEP_MILLIS)
|
|
def _upgrade_complete(self, strategy_step):
|
|
"""Internal utility method to complete an upgrade in a subcloud
|
|
|
|
This method is 'retry' wrapped to attempt multiple times with a
|
|
small wait period between attempts if any exception is raised
|
|
|
|
returns None
|
|
"""
|
|
region = self.get_region_name(strategy_step)
|
|
return self.get_sysinv_client(region).upgrade_complete()
|
|
|
|
def finalize_upgrade(self, strategy_step):
|
|
software_version = self._get_software_version(strategy_step)
|
|
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
deploy_status=consts.DEPLOY_STATE_DONE,
|
|
software_version=software_version)
|
|
return self.next_state
|
|
|
|
# todo(abailey): determine if service restarts can be made predictable
|
|
# todo(abailey): other states should have similar retry decorators and
|
|
# this may also be reasonable to add within the client API calls.
|
|
def perform_state_action(self, strategy_step):
|
|
"""Complete an upgrade on a subcloud
|
|
|
|
We should never cache the client. re-query it.
|
|
Returns the next state in the state machine on success.
|
|
Any exceptions raised by this method set the strategy to FAILED.
|
|
|
|
This state runs during a time when manifests are applying and services
|
|
are restarting, and therefore any API call in this method can randomly
|
|
fail. To accomodate this, every call is wrapped with retries.
|
|
"""
|
|
|
|
# upgrade-complete causes the upgrade to be deleted.
|
|
# if no upgrade exists, there is no need to call it.
|
|
# The API should always return a list
|
|
upgrades = self._get_upgrades(strategy_step)
|
|
if len(upgrades) == 0:
|
|
self.info_log(strategy_step,
|
|
"No upgrades exist. Nothing needs completing")
|
|
return self.finalize_upgrade(strategy_step)
|
|
# invoke the API 'upgrade-complete'
|
|
# This is a partially blocking call that raises exception on failure.
|
|
# We will re-attempt even if that failure is encountered
|
|
try:
|
|
message = self._upgrade_complete(strategy_step)
|
|
except Exception as e:
|
|
msg = ("Failed to complete upgrade. %s" %
|
|
str(e))
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
|
raise
|
|
# 'completion' deletes the upgrade. Need to loop until it is deleted
|
|
counter = 0
|
|
while True:
|
|
# If event handler stop has been triggered, fail the state
|
|
if self.stopped():
|
|
raise StrategyStoppedException()
|
|
|
|
upgrades = self._get_upgrades(strategy_step)
|
|
if len(upgrades) == 0:
|
|
self.info_log(strategy_step, "Upgrade completed.")
|
|
break
|
|
counter += 1
|
|
if counter >= self.max_queries:
|
|
msg = ("Timeout waiting for completion to complete: %s:" %
|
|
message)
|
|
db_api.subcloud_update(
|
|
self.context, strategy_step.subcloud_id,
|
|
error_description=msg[0:consts.ERROR_DESCRIPTION_LENGTH])
|
|
raise Exception(msg)
|
|
time.sleep(self.sleep_duration)
|
|
|
|
# When we return from this method without throwing an exception, the
|
|
# state machine can proceed to the next state
|
|
return self.finalize_upgrade(strategy_step)
|