From d89fa1d67cf60e9c1ffa53a449d259236bcb3a28 Mon Sep 17 00:00:00 2001 From: junfeng-li Date: Tue, 2 Apr 2024 14:33:38 +0000 Subject: [PATCH] create proxy API for sysinv to access USM This commit is to replace the direct request to db API to get the upgrade state with a new proxy API. The proxy API will firstly direct the request to USM REST API if the USM endpoint is available. If not, the request will be directed to legacy db API. Test Plan: PASS: run the upgrade with USM available PASS: run the upgrade with legacy upgrade method Task: 49798 Story: 2010676 Change-Id: If64c5fd6585ce7a96bee84393205194bd2fd92a4 Signed-off-by: junfeng-li --- .../sysinv/sysinv/api/controllers/v1/host.py | 25 +++--- .../api/controllers/v1/kube_rootca_update.py | 3 +- .../sysinv/api/controllers/v1/kube_upgrade.py | 3 +- .../sysinv/sysinv/api/controllers/v1/load.py | 1 + .../sysinv/sysinv/api/controllers/v1/utils.py | 3 +- .../sysinv/sysinv/sysinv/common/rest_api.py | 20 ++++- .../sysinv/sysinv/common/usm_service.py | 79 +++++++++++++++++++ sysinv/sysinv/sysinv/sysinv/common/utils.py | 3 +- sysinv/sysinv/sysinv/sysinv/conductor/ceph.py | 3 +- .../sysinv/sysinv/sysinv/conductor/manager.py | 31 +++++--- sysinv/sysinv/sysinv/sysinv/puppet/puppet.py | 3 +- .../sysinv/tests/common/test_usm_service.py | 45 +++++++++++ 12 files changed, 190 insertions(+), 29 deletions(-) create mode 100644 sysinv/sysinv/sysinv/sysinv/common/usm_service.py create mode 100644 sysinv/sysinv/sysinv/sysinv/tests/common/test_usm_service.py diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py index 83efc1073e..30df8d6f3c 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py @@ -94,6 +94,7 @@ from sysinv.common import constants from sysinv.common import device from sysinv.common import exception from sysinv.common import kubernetes +from sysinv.common import usm_service as usm_service from sysinv.common import utils as cutils from sysinv.common.storage_backend_conf import StorageBackendConfig from sysinv.common import health @@ -2566,7 +2567,7 @@ class HostController(rest.RestController): upgrade = None try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: return True @@ -2822,7 +2823,7 @@ class HostController(rest.RestController): return try: - pecan.request.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: return @@ -3799,7 +3800,7 @@ class HostController(rest.RestController): try: # Check if there's an upgrade in progress - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) if upgrade.state == constants.UPGRADE_UPGRADING_CONTROLLERS: host_upgrade = objects.host_upgrade.get_by_host_id( pecan.request.context, ihost['id']) @@ -3815,7 +3816,7 @@ class HostController(rest.RestController): # Don't allow unlock of controller-1 if it is being upgraded try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: # No upgrade in progress return @@ -3978,8 +3979,7 @@ class HostController(rest.RestController): # Determine required platform reserved memory for this numa node low_core = cutils.is_low_core_system(ihost, pecan.request.dbapi) - reserved = cutils. \ - get_required_platform_reserved_memory( + reserved = cutils.get_required_platform_reserved_memory( pecan.request.dbapi, ihost, node['numa_node'], low_core) # Determine configured memory for this numa node @@ -5986,7 +5986,7 @@ class HostController(rest.RestController): def _check_lock_controller_during_upgrade(hostname): # Check to ensure in valid upgrade state for host-lock try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: # No upgrade in progress return @@ -6317,12 +6317,14 @@ class HostController(rest.RestController): # First check if we are in an upgrade try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: # No upgrade in progress so nothing to check return # Get the load running on the destination controller + # TODO(bqian) below should call USM for host upgrade for USM major release + # deploy host_upgrade = objects.host_upgrade.get_by_host_id( pecan.request.context, to_host['id']) to_host_load_id = host_upgrade.software_load @@ -6510,8 +6512,7 @@ class HostController(rest.RestController): if ihost_ctr.config_target and\ ihost_ctr.config_target != ihost_ctr.config_applied: try: - upgrade = \ - pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: upgrade = None if upgrade and upgrade.state == \ @@ -6632,7 +6633,7 @@ class HostController(rest.RestController): if not force: # Check if there is upgrade in progress try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) if upgrade.state in [constants.UPGRADE_ABORTING_ROLLBACK]: LOG.info("%s not in a force lock and in an upgrade abort, " "do not check Ceph status" @@ -6687,7 +6688,7 @@ class HostController(rest.RestController): constants.WORKER in subfunctions_set): upgrade = None try: - upgrade = pecan.request.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(pecan.request.dbapi) upgrade_state = upgrade.state except exception.NotFound: upgrade_state = None diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py index 664b5906a6..8041ad5959 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_rootca_update.py @@ -28,6 +28,7 @@ from sysinv.common import constants from sysinv.common import dc_api from sysinv.common import exception from sysinv.common import kubernetes +from sysinv.common import usm_service as usm_service from sysinv.common import utils as cutils from sysinv._i18n import _ from wsme import types as wtypes @@ -492,7 +493,7 @@ class KubeRootCAUpdateController(rest.RestController): # There must not be a platform upgrade in progress try: - pecan.request.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: pass else: diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py index be0f4d816f..172512b112 100755 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_upgrade.py @@ -27,6 +27,7 @@ from sysinv.common import constants from sysinv.common import dc_api from sysinv.common import exception from sysinv.common import kubernetes +from sysinv.common import usm_service as usm_service from sysinv.common import utils as cutils from sysinv import objects @@ -179,7 +180,7 @@ class KubeUpgradeController(rest.RestController): # There must not be a platform upgrade in progress try: - pecan.request.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: pass else: diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/load.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/load.py index 776a89609a..472a485844 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/load.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/load.py @@ -563,6 +563,7 @@ class LoadController(rest.RestController): # make sure the load isn't in use by an upgrade try: + # NOTE(bqian) load relates only to the legacy upgrade upgrade = pecan.request.dbapi.software_upgrade_get_one() except exception.NotFound: pass diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py index 1e70da86d2..f2d30a5ffc 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py @@ -40,6 +40,7 @@ from sysinv.common import ceph from sysinv.common import constants from sysinv.common import exception from sysinv.common import health +from sysinv.common import usm_service as usm_service from sysinv.helm import common as helm_common @@ -508,7 +509,7 @@ def check_disallow_during_upgrades(): # There must not already be a platform upgrade in progress try: - pecan.request.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(pecan.request.dbapi) except exception.NotFound: pass else: diff --git a/sysinv/sysinv/sysinv/sysinv/common/rest_api.py b/sysinv/sysinv/sysinv/sysinv/common/rest_api.py index 711597fb58..8d45990f4a 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/rest_api.py +++ b/sysinv/sysinv/sysinv/sysinv/common/rest_api.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # import json +import os import signal import six @@ -17,7 +18,6 @@ from oslo_log import log from oslo_utils import encodeutils from sysinv.common import configp from sysinv.common import exception as si_exception -from sysinv.common import utils as cutils from sysinv.openstack.common.keystone_objects import Token from sysinv.common.exception import OpenStackException @@ -133,7 +133,7 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None, if api_cmd_payload is not None: request_info.data = encodeutils.safe_encode(api_cmd_payload) - ca_file = cutils.get_system_ca_file() + ca_file = get_system_ca_file() ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=ca_file) request = urlopen(request_info, timeout=timeout, context=ssl_context) @@ -170,3 +170,19 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None, finally: signal.alarm(0) return response + + +def get_system_ca_file(): + """Return path to system default CA file.""" + # Duplicate of sysinv.common.utils.get_system_ca_file() to + # avoid circular import + # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora, + # Suse, FreeBSD/OpenBSD + ca_path = ['/etc/ssl/certs/ca-certificates.crt', + '/etc/pki/tls/certs/ca-bundle.crt', + '/etc/ssl/ca-bundle.pem', + '/etc/ssl/cert.pem'] + for ca in ca_path: + if os.path.exists(ca): + return ca + return None diff --git a/sysinv/sysinv/sysinv/sysinv/common/usm_service.py b/sysinv/sysinv/sysinv/sysinv/common/usm_service.py new file mode 100644 index 0000000000..834a97af38 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/common/usm_service.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# USM Unified Software Management Handling + +from oslo_log import log + +from sysinv.common.rest_api import get_token +from sysinv.common.rest_api import rest_api_request + + +LOG = log.getLogger(__name__) + + +# TODO (bqian) for compatibility, create a software upgrade +# entity. +# This is temporary to bridge between legacy upgrade and USM +# major release deploy and should be removed once the transion +# completes. +class UsmUpgrade(object): + def __init__(self, state, from_load, to_load): + self.state = None + self.from_load = None + self.to_load = None + + def __eq__(self, other): + return self.state == other.state and \ + self.from_load == other.from_load and \ + self.to_load == other.to_load + + def __ne__(self, other): + return not (self == other) + + +def get_software_upgrade(token, region_name, timeout=30): + + if not token: + token = get_token(region_name) + + endpoint = token.get_service_url("usm", "usm") + + if not endpoint: + return None + + endpoint += "/v1/deploy/software_upgrade" + + response = rest_api_request(token, "GET", endpoint, timeout=timeout) + return response + + +def get_platform_upgrade(dbapi): + """ + Get upgrade object from either sysinv db or USM service. + Upgrade object is from USM service if the service is present, + if not, the object is from sysinv db. + """ + + upgrade = None + system = dbapi.isystem_get_one() + region_name = system.region_name + + try: + response = get_software_upgrade(None, region_name) + if response: + upgrade = UsmUpgrade(state=response["state"], + from_load=response["from_release"], + to_load=response["to_release"]) + except Exception: + # it is ok, legacy upgrade does not have usm service available + pass + + if upgrade is None: + upgrade = dbapi.software_upgrade_get_one() + + return upgrade diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index d353dfa32c..97f8b1d2e2 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -88,6 +88,7 @@ from sysinv.common import exception from sysinv.common import constants from sysinv.helm import common as helm_common from sysinv.common import kubernetes +from sysinv.common import usm_service as usm_service try: @@ -1873,7 +1874,7 @@ def is_upgrade_in_progress(dbapi): """ try: - upgrade = dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(dbapi) LOG.debug("Platform Upgrade in Progress: state=%s" % upgrade.state) return True, upgrade except exception.NotFound: diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/ceph.py b/sysinv/sysinv/sysinv/sysinv/conductor/ceph.py index 12bd52d2ce..a814c8d3e5 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/ceph.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/ceph.py @@ -27,6 +27,7 @@ from oslo_utils import uuidutils from sysinv._i18n import _ from sysinv.common import constants from sysinv.common import exception +from sysinv.common import usm_service as usm_service from sysinv.common import utils as cutils from sysinv.common.storage_backend_conf import StorageBackendConfig @@ -1072,7 +1073,7 @@ class CephOperator(object): # Get upgrade status upgrade = None try: - upgrade = self._db_api.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(self._db_api) except exception.NotFound: LOG.info("No upgrade in progress. Skipping quota " "upgrade checks.") diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 2f5622637f..cd4ab6f680 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -113,6 +113,7 @@ from sysinv.common import kubernetes from sysinv.common import openstack_config_endpoints from sysinv.common import retrying from sysinv.common import service +from sysinv.common import usm_service as usm_service from sysinv.common import utils as cutils from sysinv.common.inotify import flags from sysinv.common.inotify import INotify @@ -739,6 +740,7 @@ class ConductorManager(service.PeriodicService): def _upgrade_init_actions(self): """ Perform any upgrade related startup actions""" try: + # NOTE(bqian) this is legacy upgrade only code upgrade = self.dbapi.software_upgrade_get_one() except exception.NotFound: # Not upgrading. No need to update status @@ -4830,12 +4832,15 @@ class ConductorManager(service.PeriodicService): :return: """ try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: # Not upgrading. We assume the host versions match # If they somehow don't match we've got bigger problems return True + # TODO(bqian) this is to be replaced with host.sw_version after + # https://review.opendev.org/c/starlingx/config/+/915376 + # in a USM upgrade scenario. host_obj = self.dbapi.ihost_get(host_uuid) host_version = host_obj.software_load @@ -5081,7 +5086,7 @@ class ConductorManager(service.PeriodicService): return try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: # No upgrade in progress pass @@ -5518,7 +5523,7 @@ class ConductorManager(service.PeriodicService): upgrade_in_progress = False try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) upgrade_in_progress = True except exception.NotFound: # No upgrade in progress @@ -5796,7 +5801,7 @@ class ConductorManager(service.PeriodicService): return try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: # No upgrade in progress pass @@ -6988,6 +6993,7 @@ class ConductorManager(service.PeriodicService): @periodic_task.periodic_task(spacing=CONF.conductor_periodic_task_intervals.upgrade_status) def _audit_upgrade_status(self, context): """Audit upgrade related status""" + # NOTE(bqian) legacy upgrade only code try: upgrade = self.dbapi.software_upgrade_get_one() except exception.NotFound: @@ -10627,7 +10633,7 @@ class ConductorManager(service.PeriodicService): Raise an exception if one is found. """ try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: pass else: @@ -11378,7 +11384,9 @@ class ConductorManager(service.PeriodicService): Callback for Sysinv Agent on upgrade manifest failure """ try: - upgrade = self.dbapi.software_upgrade_get_one() + # TODO (bqian) change below report to USM if USM major release + # deploy activate failed + upgrade = usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: LOG.error("Upgrade record not found during config failure") return @@ -13430,7 +13438,7 @@ class ConductorManager(service.PeriodicService): host_uuids = config_dict.get('host_uuids') try: - self.dbapi.software_upgrade_get_one() + usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: # No upgrade in progress pass @@ -14279,7 +14287,7 @@ class ConductorManager(service.PeriodicService): # Check if there is an upgrade in progress try: - upgrade = self.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(self.dbapi) except exception.NotFound: # No upgrade in progress pass @@ -14353,6 +14361,8 @@ class ConductorManager(service.PeriodicService): if tsc.system_mode == constants.SYSTEM_MODE_SIMPLEX: LOG.info("Generating agent request to create simplex upgrade " "data") + # NOTE(bqian) this is legacy upgrade only code, so only fetch upgrade + # entity from sysinv db software_upgrade = self.dbapi.software_upgrade_get_one() rpcapi = agent_rpcapi.AgentAPI() # In cases where there is no backup in progress alarm but the flag exists, @@ -14678,6 +14688,7 @@ class ConductorManager(service.PeriodicService): :param success: If the create_simplex_backup call completed """ try: + # NOTE(bqian) legacy upgrade only code upgrade = self.dbapi.software_upgrade_get_one() except exception.NotFound: LOG.error("Software upgrade record not found") @@ -14984,7 +14995,9 @@ class ConductorManager(service.PeriodicService): 'to_version': None, 'state': None} try: - row = self.dbapi.software_upgrade_get_one() + # this is checked by ceph-manager, so report both legacy upgrade or + # USM major release deploy + row = usm_service.get_platform_upgrade(self.dbapi) upgrade['from_version'] = row.from_release upgrade['to_version'] = row.to_release upgrade['state'] = row.state diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py index e12a9f9e91..83f54c0c78 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py @@ -21,6 +21,7 @@ from sysinv.common import constants from oslo_log import log as logging from sysinv.puppet import common +from sysinv.common import usm_service as usm_service from sysinv.common import utils @@ -193,7 +194,7 @@ class PuppetOperator(object): host.hostname == constants.CONTROLLER_0_HOSTNAME and not os.path.exists(hiera_file)): try: - upgrade = self.dbapi.software_upgrade_get_one() + upgrade = usm_service.get_platform_upgrade(self.dbapi) if (upgrade.state == constants.UPGRADE_ABORTING_ROLLBACK): LOG.info("controller-0 downgrade for a version using .yaml") return True diff --git a/sysinv/sysinv/sysinv/sysinv/tests/common/test_usm_service.py b/sysinv/sysinv/sysinv/sysinv/tests/common/test_usm_service.py new file mode 100644 index 0000000000..f885809f63 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/tests/common/test_usm_service.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from unittest import TestCase +from unittest import mock +from sysinv.common.usm_service import get_platform_upgrade +from sysinv.common.usm_service import UsmUpgrade + + +class TestUSMService(TestCase): + @mock.patch('sysinv.common.usm_service.get_software_upgrade') + def test_get_platform_upgrade_with_usm_service(self, mock_get_software_upgrade): + usm_deploy = { + "from_release": "1.0", + "to_release": "2.0", + "state": "in_progress" + } + expected_response = UsmUpgrade( + "in_progress", + "1.0", + "2.0") + mock_get_software_upgrade.return_value = usm_deploy + mock_dbapi = mock.Mock() + mock_dbapi.software_upgrade_get_one.return_value = None + + result = get_platform_upgrade(mock_dbapi) + + self.assertEqual(result, expected_response) + + def test_get_platform_upgrade_without_usm_service(self): + mock_dbapi_response = { + "from_release": "1.0", + "to_release": "2.0", + "state": "in_progress" + } + + mock_dbapi = mock.Mock() + mock_dbapi.software_upgrade_get_one.return_value = mock_dbapi_response + + result = get_platform_upgrade(mock_dbapi) + + self.assertEqual(result, mock_dbapi_response)