From 4b876fa82c92deba549f8d1f18b4ba5f235188da Mon Sep 17 00:00:00 2001 From: Li Zhu Date: Thu, 5 Oct 2023 16:04:43 -0400 Subject: [PATCH] Decontainerizing rvmc.py and Modular Integration in DC Repo Make the following changes for decontainerizing rvmc.py: 1. Copy rvmc.py from the Metal repo to the DC repo. Convert it into a common module that can be imported either from an independent script (rvmc_install.py), which is invoked from the Ansible installation playbook, or from other DC Python modules, such as the subcloud_install module. This module is used to shut down the host when subcloud deployment is aborted. 2. Create a Redfish Debian package called python3-redfish to install the Redfish Python library for the rvmc module. 3. Install rvmc_install.py to /usr/local/bin using the distributedcloud-dccommon package. Make the following changes in the rvmc module/script: 1. Create LoggingUtil and ExitHandler classes to differentiate the way of logging and exit handling when called from the module or an independent script. 2. Implement HTTP request logging when the debug log level is set to 0. 3. Introduce a timeout mechanism to restrict the total execution time of the script during installation. 4. Add a signal handler to ensure the script exits properly when aborting a subcloud installation. Implement process ID tracking to allow only one RVMC process to run at a time for a subcloud installation, regardless of the termination status of previous installations. 5. Incorporate a power-off function to be called from the DC subcloud_install module, which is used in "subcloud deploy abort." 6. Include "subcloud_name" and "config_file" parameters/arguments to enhance functionality and flexibility. Test Plan: PASS: Verify successful subcloud installation. PASS: Verify successful "subcloud deploy abort/resume". PASS: Verify the playbook execution is halted, and the subcloud is shut down when an abort is triggered during subcloud installation PASS: Verify the successful generation of the Redfish Debian package using the 'build-pkgs' command. PASS: Verify that the image is successfully generated, along with the required Redfish package and its dependencies, using the 'build-image' command. PASS: Verify that 'rvmc_install.py' is located in '/usr/local/bin' after the successful installation of the system controller with the new image. PASS: Verify HTTP request and response logging are generated regardless of debug log level for failure conditions. PASS: Verify the retry attempts to be applied only when the response status code is 5XX. PASS: Verify that the subcloud installation command fails with the expected error message when the value of 'rvmc_debug_level' is set outside of the range [0..4]. PASS: Verify the success of the subcloud installation command by setting the 'rvmc_debug_level' value within the range [0..4]. PASS: Verify that the RVMC script exits when the timeout is reached. PASS: Verify that only one RVMC process is running at a time for a subcloud installation, regardless of whether the previous subcloud installation terminated properly or not. Story: 2010144 Task: 48897 Depends-On: https://review.opendev.org/c/starlingx/tools/+/900206 Change-Id: Ic35da91a35594ede668cef6875c752b93a19d6d5 Signed-off-by: lzhu1 --- debian_iso_image.inc | 2 +- debian_pkg_dirs | 1 + distributedcloud/dccommon/exceptions.py | 13 +- distributedcloud/dccommon/rvmc.py | 1692 +++++++++++++++++ distributedcloud/dccommon/subcloud_install.py | 146 +- distributedcloud/dccommon/utils.py | 17 + distributedcloud/dcmanager/common/consts.py | 4 + .../common/phased_subcloud_deploy.py | 10 + .../dcmanager/manager/subcloud_manager.py | 29 +- .../states/upgrade/upgrading_simplex.py | 7 +- .../unit/manager/test_subcloud_manager.py | 6 +- .../distributedcloud-dccommon.install | 1 + distributedcloud/debian/deb_folder/rules | 7 + distributedcloud/requirements.txt | 3 +- distributedcloud/scripts/rvmc_install.py | 311 +++ distributedcloud/test-requirements.txt | 1 + .../debian/deb_folder/changelog | 5 + .../python3-redfish/debian/deb_folder/control | 18 + .../debian/deb_folder/copyright | 48 + .../python3-redfish/debian/deb_folder/rules | 17 + python/python3-redfish/debian/meta_data.yaml | 14 + tox.ini | 11 +- 22 files changed, 2190 insertions(+), 173 deletions(-) create mode 100755 distributedcloud/dccommon/rvmc.py create mode 100644 distributedcloud/scripts/rvmc_install.py create mode 100644 python/python3-redfish/debian/deb_folder/changelog create mode 100644 python/python3-redfish/debian/deb_folder/control create mode 100644 python/python3-redfish/debian/deb_folder/copyright create mode 100755 python/python3-redfish/debian/deb_folder/rules create mode 100644 python/python3-redfish/debian/meta_data.yaml diff --git a/debian_iso_image.inc b/debian_iso_image.inc index 1e863c293..5f196de81 100644 --- a/debian_iso_image.inc +++ b/debian_iso_image.inc @@ -6,4 +6,4 @@ distributedcloud-dcdbsync distributedcloud-dcmanager distributedcloud-dcorch - +python3-redfish diff --git a/debian_pkg_dirs b/debian_pkg_dirs index 3211ad056..77600387b 100644 --- a/debian_pkg_dirs +++ b/debian_pkg_dirs @@ -1 +1,2 @@ distributedcloud +python/python3-redfish diff --git a/distributedcloud/dccommon/exceptions.py b/distributedcloud/dccommon/exceptions.py index b4670b111..40863f616 100644 --- a/distributedcloud/dccommon/exceptions.py +++ b/distributedcloud/dccommon/exceptions.py @@ -121,10 +121,6 @@ class ImageNotInLocalRegistry(NotFound): "system registry-image-tags %(image_name)s") -class SubcloudShutdownError(PlaybookExecutionFailed): - message = _("Subcloud %(subcloud_name)s could not be shut down.") - - class ApiException(DCCommonException): message = _("%(endpoint)s failed with status code: %(rc)d") @@ -140,3 +136,12 @@ class SubcloudPeerGroupNotFound(NotFound): class SubcloudPeerGroupDeleteFailedAssociated(DCCommonException): message = _("Subcloud Peer Group %(peer_group_ref)s delete failed " "cause it is associated with a system peer.") + + +class RvmcException(Exception): + def __init__(self, message=None): + super(RvmcException, self).__init__(message) + + +class RvmcExit(DCCommonException): + message = _("Rvmc failed with status code: %(rc)d") diff --git a/distributedcloud/dccommon/rvmc.py b/distributedcloud/dccommon/rvmc.py new file mode 100755 index 000000000..6fe826719 --- /dev/null +++ b/distributedcloud/dccommon/rvmc.py @@ -0,0 +1,1692 @@ +# Copyright (c) 2019-2023 Wind River Systems, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Redfish Virtual Media Controller +# +# Note that any changes in error messages need to be reflected in +# the installation Ansible playbook. + +import base64 +import datetime +import json +import os +import socket +import sys +import time +import yaml + +# Import Redfish Python Library +# Module: https://pypi.org/project/redfish/ +import redfish +from redfish.rest.v1 import InvalidCredentialsError + +from dccommon import exceptions + + +# Constants +# --------- +POWER_ON = 'On' +POWER_OFF = "Off" + +REDFISH_ROOT_PATH = '/redfish/v1' +SUPPORTED_VIRTUAL_MEDIA_DEVICES = ['CD', 'DVD'] # Maybe add USB to list + +# headers for each request type +HDR_CONTENT_TYPE = "'Content-Type': 'application/json'" +HDR_ACCEPT = "'Accept': 'application/json'" + +# they all happen to be the same right now +GET_HEADERS = {HDR_CONTENT_TYPE, HDR_ACCEPT} +POST_HEADERS = {HDR_CONTENT_TYPE, HDR_ACCEPT} +PATCH_HEADERS = {HDR_CONTENT_TYPE, HDR_ACCEPT} + +# HTTP request types ; only 3 are required by this tool +POST = 'POST' +GET = 'GET' +PATCH = 'PATCH' + +# max number of polling retries while waiting for some long task to complete +MAX_POLL_COUNT = 200 +# some servers timeout on inter comm gaps longer than 10 secs +RETRY_DELAY_SECS = 10 +# 2 second delay constant +DELAY_2_SECS = 2 + +# max number of establishing BMC connection attempts +MAX_CONNECTION_ATTEMPTS = 3 +# interval in seconds between BMC connection attempts +CONNECTION_RETRY_INTERVAL = 15 + +# max number of session creation attempts +MAX_SESSION_CREATION_ATTEMPTS = 3 +# interval in seconds between session creation attempts +SESSION_CREATION_RETRY_INTERVAL = 15 + +# max number of retries for http transient error (e.g. response status: 500) +MAX_HTTP_TRANSIENT_ERROR_RETRIES = 5 +# interval in seconds between http request retries +HTTP_REQUEST_RETRY_INTERVAL = 10 + + +class LoggingUtil(object): + """The logging utility class. + + If no logger is given, the messages will be written to the standard + output stream. + """ + + def __init__(self, logger=None, subcloud_name='', debug_level=0): + """Logger object constructor. + + :param logger: the logger of the class which is calling rvmc module + :type logger: logging.Logger. + :param subcloud_name: the subcloud name to be appended to the logging + messages + :type subcloud_name: str. + :param debug_level: the debug level + :type debug_level: int. + """ + + self.logger = logger + self.subcloud_name = subcloud_name + self.debug_level = debug_level + + def t(self): + """Return current time for log functions.""" + + return datetime.datetime.now().replace(microsecond=0) + + def ilog(self, string): + """Info Log Utility""" + + if self.logger: + self.logger.info( + self.subcloud_name + ': ' + string if self.subcloud_name + else string) + else: + sys.stdout.write("\n%s Info : %s" % (self.t(), string)) + + def wlog(self, string): + """Warning Log Utility""" + + if self.logger: + self.logger.warn( + self.subcloud_name + ': ' + string if self.subcloud_name + else string) + else: + sys.stdout.write("\n%s Warn : %s" % (self.t(), string)) + + def elog(self, string): + """Error Log Utility""" + + if self.logger: + self.logger.error( + self.subcloud_name + ': ' + string if self.subcloud_name + else string) + else: + sys.stdout.write("\n%s Error : %s" % (self.t(), string)) + + def alog(self, string): + """Action Log Utility""" + + if self.logger: + self.logger.info( + self.subcloud_name + ': ' + string if self.subcloud_name + else string) + else: + sys.stdout.write("\n%s Action: %s" % (self.t(), string)) + + def dlog1(self, string, level=1): + """Debug Log - Level""" + + if self.debug_level and level <= self.debug_level: + if self.logger: + self.logger.debug( + self.subcloud_name + ': ' + string if self.subcloud_name + else string) + else: + sys.stdout.write("\n%s Debug%d: %s" % + (self.t(), level, string)) + + def dlog2(self, string): + """Debug Log - Level 2""" + + self.dlog1(string, 2) + + def dlog3(self, string): + """Debug Log - Level 3""" + + self.dlog1(string, 3) + + def dlog4(self, string): + """Debug Log - Level 4""" + + self.dlog1(string, 4) + + def slog(self, stage): + """Execution Stage Log""" + + if self.logger: + self.logger.info( + self.subcloud_name + ': ' + stage if self.subcloud_name + else stage) + else: + sys.stdout.write("\n%s Stage : %s" % (self.t(), stage)) + + +class ExitHandler(object): + """A utility class for handling different exit scenarios in a process. + + Provides methods to manage the process exit in various situations. + """ + + def exit(self, code): + """Early fault handling + + :param code: the exit status code + :type code: int. + """ + if code != 0: + raise exceptions.RvmcExit(rc=code) + + +def is_ipv6_address(address, logging_util): + """Check IPv6 Address. + + :param address: the ip address to compare user name. + :type address: str. + :param logging_util: the logging utility. + :type logging_util: LoggingUtil + :returns bool: True if address is an IPv6 address else False + """ + try: + socket.inet_pton(socket.AF_INET6, address) + logging_util.dlog3("Address : %s is IPv6" % address) + except socket.error: + logging_util.dlog3("Address : %s is IPv4" % address) + return False + return True + + +def supported_device(devices): + """Supported Device + + :param devices: list of devices + :type : list + :returns True if a device in devices list is in the + SUPPORTED_VIRTUAL_MEDIA_DEVICES list. + Otherwise, False is returned. + """ + for device in devices: + if device in SUPPORTED_VIRTUAL_MEDIA_DEVICES: + return True + return False + + +def parse_target(target_name, target_dict, config_file, logging_util, + exit_handler): + """Parse key value pairs in the target config file. + + :param target_name: the subcloud name + :type target_name: str. + :param target_dict: dictionary of key value config file pairs + :type target_dict: dictionary + :param logging_util: the RVMC config file. + :type logging_util: str + :param logging_util: the logging utility. + :type logging_util: LoggingUtil + :param exit_handler: the exit handler. + :type exit_handler: ExitHandler + :returns the control object; None if any error + """ + logging_util.dlog3("Parse Target: %s:%s" % (target_name, target_dict)) + + pw = target_dict.get('bmc_password') + if pw is None: + logging_util.elog("Failed get bmc password from config file") + return None + + try: + pw_dec = base64.b64decode(pw).decode('utf-8') + except Exception as ex: + logging_util.elog( + "Failed to decode bmc password found in config file (%s)" % ex) + logging_util.alog( + "Verify config file's bmc password is base64 encoded") + return None + + address = target_dict.get('bmc_address') + if address is None: + logging_util.elog("Failed to decode bmc password found in %s" % + config_file) + logging_util.alog( + "Verify config file's bmc password is base64 encoded") + return None + + #################################################################### + # + # Add url encoding [] for ipv6 addresses only. + # + # Note: The imported redfish library produces a python exception + # on the session close if the ipv4 address has [] around it. + # + # I debugged the issue and know what it is and how to fix it + # but requires an upstream change that is not worth doing. + # + # URL Encoding for IPv6 only is an easy solution. + # + ###################################################################### + if is_ipv6_address(address, logging_util) is True: + bmc_ipv6 = True + address = '[' + address + ']' + else: + bmc_ipv6 = False + + # Create control object + try: + vmc_obj = VmcObject(target_name, + address, + target_dict.get('bmc_username'), + pw, + str(pw_dec), + target_dict.get('image'), + logging_util, + exit_handler) + if vmc_obj: + vmc_obj.ipv6 = bmc_ipv6 + return vmc_obj + else: + logging_util.elog( + "Unable to create control object for target:%s ; " + "skipping ..." % target_dict) + + except Exception as ex: + logging_util.elog( + "Unable to parse configuration '%s' (%s) in %s config file." % + (target_dict, ex, target_name)) + logging_util.alog( + "Check presence and spelling of configuration members in " + "%s config file." % target_name) + return None + + +def parse_config_file(target_name, config_file, logging_util, exit_handler): + """Parse BMC target info from config file. + + Create target object through parse_target. + + :param target_name: the subcloud_name + :type target_name: str + :param config_file: the RVMC config file + :type config_file: str + :param logging_util: the logging utility. + :type logging_util: LoggingUtil + :param exit_handler: the exit handler. + :type exit_handler: ExitHandler + :returns binary data with configuration loaded from the config file + and the target object + """ + # Find, Open and Read callers config file + cfg = None + if not os.path.exists(config_file): + logging_util.elog("Unable to find specified config file: %s" % + config_file) + logging_util.alog("Check config file spelling and presence\n\n") + exit_handler.exit(1) + try: + with open(config_file, 'r') as yaml_config: + logging_util.dlog1("Config File : %s" % config_file) + cfg = yaml.safe_load(yaml_config) + logging_util.dlog3("Config Data : %s" % cfg) + except Exception as ex: + logging_util.elog("Unable to open specified config file: %s (%s)" % + (config_file, ex)) + logging_util.alog("Check config file access and permissions.\n\n") + exit_handler.exit(1) + + # Parse the config file + target_object = parse_target( + target_name, cfg, config_file, logging_util, exit_handler) + + return cfg, target_object + + +class VmcObject(object): + """Virtual Media Controller Class Object. One for each BMC.""" + + def __init__(self, + hostname, + address, + username, + password, + password_decoded, + image, + logging_util, + exit_handler): + + self.target = hostname + self.uri = "https://" + address + self.url = REDFISH_ROOT_PATH + self.un = username.rstrip() + self.ip = address.rstrip() + self.pw_encoded = password.rstrip() + self.pw = password_decoded + self.img = image.rstrip() + self.logging_util = logging_util + self.exit_handler = exit_handler + self.ipv6 = False + self.redfish_obj = None # redfish client connection object + self.session = False # True when session for this BMC is created + + self.response = None # holds response from last http request + self.response_json = None # json formatted version of above response + self.response_dict = None # dictionary version of aboe response + + # redfish root query response + self.root_query_info = None # json version of the full root query + + # Managers Info + self.managers_group_url = None + self.manager_members_list = [] + + # Virtual Media Info + self.vm_url = None + self.vm_eject_url = None + self.vm_group_url = None + self.vm_group = None + self.vm_label = None + self.vm_version = None + self.vm_actions = {} + self.vm_members_array = [] + self.vm_media_types = [] + + # systems info + self.systems_group_url = None + self.systems_member_url = None + self.systems_members_list = [] + self.systems_members = 0 + self.power_state = None + + # boot control info + self.boot_control_dict = {} + + # systems reset info + self.reset_command_url = None + self.reset_action_dict = {} + + # parsed target object info + if self.target is not None: + self.logging_util.dlog1("Target : %s" % self.target) + self.logging_util.dlog1("BMC IP : %s" % self.ip) + self.logging_util.dlog1("Username : %s" % self.un) + self.logging_util.dlog1("Password : %s" % self.pw_encoded) + self.logging_util.dlog1("Image : %s" % self.img) + + def make_request(self, operation=None, path=None, payload=None, retry=-1): + """Issue a Redfish http request + + Check response, + Convert response to dictionary format + Convert response to json format + + :param operation: HTTP GET, POST or PATCH operation + :type operation: str. + :param path: url to perform request to + :type path: str + :param payload: POST or PATCH payload data + :type payload: dictionary + :param retry: The number of retries. The default value -1 means + disabling retry. If the number is in + [0 .. MAX_HTTP_TRANSIENT_ERROR_RETRIES), the retry will be executed + at most (MAX_HTTP_TRANSIENT_ERROR_RETRIES - retry) time(s). + :type retry: int + :returns True if request succeeded (200,202(accepted),204(no content) + """ + self.response = None + if path is not None: + url = path + else: + url = self.url + + before_request_time = datetime.datetime.now().replace(microsecond=0) + request_log = "Request : %s %s" % (operation, url) + try: + if operation == GET: + request_log += "\nHeaders : %s : %s" % \ + (operation, GET_HEADERS) + self.response = self.redfish_obj.get(url, headers=GET_HEADERS) + + elif operation == POST: + request_log += "\nHeaders : %s : %s" % \ + (operation, POST_HEADERS) + request_log += "\nPayload : %s" % payload + self.response = self.redfish_obj.post(url, + body=payload, + headers=POST_HEADERS) + elif operation == PATCH: + request_log += "\nHeaders : %s : %s" % \ + (operation, PATCH_HEADERS) + request_log += "\nPayload : %s" % payload + self.response = self.redfish_obj.patch(url, + body=payload, + headers=PATCH_HEADERS) + else: + self.logging_util.dlog3(request_log) + self.logging_util.elog("Unsupported operation: %s" % operation) + return False + + self.logging_util.dlog3(request_log) + + except Exception as ex: + self.logging_util.elog("Failed operation on '%s' (%s)" % (url, ex)) + + if self.response is not None: + after_request_time = datetime.datetime.now().replace(microsecond=0) + delta = after_request_time - before_request_time + # if we got a response, check its status + if self.check_ok_status(url, operation, delta.seconds) is False: + self.logging_util.elog("Got an error response for: \n%s" % + request_log) + if retry < 0 or retry >= MAX_HTTP_TRANSIENT_ERROR_RETRIES: + self._exit(1) + elif self.response.status < 500: + self.logging_util.ilog( + "Stop retrying for the non-transient error (%s)." % + self.response.status) + self._exit(1) + else: + retry += 1 + self.logging_util.ilog( + "Make request: retry (%i of %i) in %i secs." % + (retry, MAX_HTTP_TRANSIENT_ERROR_RETRIES, + HTTP_REQUEST_RETRY_INTERVAL)) + time.sleep(HTTP_REQUEST_RETRY_INTERVAL) + return self.make_request(operation=operation, + path=path, + payload=payload, + retry=retry) + + # handle 204 success with no content ; clear last response + if self.response.status == 204: + self.response = "" + return True + try: + if self.resp_dict() is True: + if self.format() is True: + self.logging_util.dlog4( + "Response:\n%s\n" % self.response_json) + return True + else: + self.logging_util.elog( + "Failed to parse BMC %s response '%s'" % + (operation, url)) + + except Exception as ex: + self.logging_util.elog( + "Failed to parse BMC %s response '%s' (%s)" % + (operation, url, ex)) + else: + self.logging_util.elog("No response from %s:%s" % (operation, url)) + return False + + def resp_dict(self): + """Create Response Dictionary""" + + if self.response.read: + self.response_dict = None + try: + self.response_dict = json.loads(self.response.read) + return True + except Exception as ex: + self.logging_util.elog( + "Got exception key valuing response ; (%s)" % ex) + self.logging_util.elog("Response: %s" % self.response.read) + else: + self.logging_util.elog("No response from last command") + return False + + def format(self): + """Format Response as Json""" + + self.response_json = None + try: + if self.resp_dict() is True: + self.response_json = json.dumps(self.response_dict, + indent=4, + sort_keys=True) + return True + else: + return False + + except Exception as ex: + self.logging_util.elog( + "Got exception formatting response ; (%s)\n" % ex) + return False + + def get_key_value(self, key1, key2=None): + """Retrieve a value by providing key(s) + + Get key1 value if no key2 is specified. + Get key2 value from key1 value if key2 is specified. + + :param : key1 value is returned if no key2 is provided. + :type : str. + :param : key2 value is optional but if provided its value is returned + :type : str + :returns key1 value or key2 value if key2 is specified + """ + value1 = self.response_dict.get(key1) + if key2 is None: + return value1 + return value1.get(key2) + + def check_ok_status(self, function, operation, seconds): + """Status + + :param function: description of operation + :type : str + :param operation: http GET, POST or PATCH + :type : str + :returns True if response status is OK. Otherwise False. + """ + # Accept applicable 400 series error from an Eject Request POST. + # This error is dealt with by the eject handler. + if self.response.status in [400, 403, 404] and \ + function == self.vm_eject_url and \ + operation == POST: + return True + + if self.response.status not in [200, 202, 204]: + try: + self.logging_util.elog( + "HTTP Status : %d ; %s %s failed after %i seconds\n%s\n" % + (self.response.status, + operation, function, seconds, + json.dumps(self.response.dict, + indent=4, sort_keys=True))) + return False + except Exception as ex: + self.logging_util.elog("check status exception ; %s" % ex) + + self.logging_util.dlog2( + "HTTP Status : %s %s Ok (%d) (took %i seconds)" % + (operation, function, self.response.status, seconds)) + return True + + def _exit(self, code): + """Exit the tool but not before closing an open Redfish + + client connection. + + :param code: the exit code + :type code: int + """ + if self.redfish_obj is not None and self.session is True: + try: + self.redfish_obj.logout() + self.redfish_obj = None + self.session = False + self.logging_util.dlog1("Session : Closed") + + except Exception as ex: + self.logging_util.elog("Session close failed ; %s" % ex) + self.logging_util.alog( + "Check BMC username and password in config file") + + if code: + sys.stdout.write("\n-------------------------------------------\n") + + # If exit with reason code then print that reason code and dump + # the redfish query data that was learned up to that point + self.logging_util.elog("Code : %s" % code) + + # Other info + self.logging_util.ilog("IPv6 : %s" % self.ipv6) + + # Root Query Info + self.logging_util.ilog("Root Query: %s" % self.root_query_info) + + # Managers Info + self.logging_util.ilog("Manager URL: %s" % self.managers_group_url) + self.logging_util.ilog("Manager Members List: %s" % + self.manager_members_list) + + # Systems Info + self.logging_util.ilog("Systems Group URL: %s" % + self.systems_group_url) + self.logging_util.ilog("Systems Member URL: %s" % + self.systems_member_url) + self.logging_util.ilog("Systems Members: %d" % + self.systems_members) + self.logging_util.ilog("Systems Members List: %s" % + self.systems_members_list) + + self.logging_util.ilog("Power State: %s" % self.power_state) + self.logging_util.ilog("Reset Actions: %s" % + self.reset_action_dict) + self.logging_util.ilog("Reset Command URL: %s" % + self.reset_command_url) + self.logging_util.ilog("Boot Control Dict: %s" % + self.boot_control_dict) + + self.logging_util.ilog("VM Members Array: %s" % + self.vm_members_array) + self.logging_util.ilog("VM Group URL: %s" % self.vm_group_url) + self.logging_util.ilog("VM Group: %s" % self.vm_group) + self.logging_util.ilog("VM URL: %s" % self.vm_url) + self.logging_util.ilog("VM Label: %s" % self.vm_label) + self.logging_util.ilog("VM Version: %s" % self.vm_version) + self.logging_util.ilog("VM Actions: %s" % self.vm_actions) + self.logging_util.ilog("VM Media Types: %s" % self.vm_media_types) + + self.logging_util.ilog("Last Response raw: %s" % self.response) + self.logging_util.ilog("Last Response json: %s" % + self.response_json) + + self.exit_handler.exit(code) + + ########################################################################### + # + # P R I V A T E S T A G E M E M B E R F U N C T I O N S + # + ########################################################################### + + ########################################################################### + # Redfish Client Connect + ########################################################################### + def _redfish_client_connect(self): + """Connect to target Redfish service.""" + + stage = 'Redfish Client Connection' + self.logging_util.slog(stage) + + # Verify ping response + ping_ok = False + ping_count = 0 + MAX_PING_COUNT = 10 + while ping_count < MAX_PING_COUNT and ping_ok is False: + if self.ipv6 is True: + response = os.system("ping -6 -c 1 " + + self.ip[1:-1] + " > /dev/null 2>&1") + else: + response = os.system("ping -c 1 " + + self.ip + " > /dev/null 2>&1") + + if response == 0: + ping_ok = True + else: + ping_count = ping_count + 1 + self.logging_util.ilog("BMC Ping : retry (%i of %i)" % + (ping_count, MAX_PING_COUNT)) + time.sleep(2) + + if ping_ok is False: + self.logging_util.elog("Unable to ping '%s' (%i)" % + (self.ip, ping_count)) + self.logging_util.alog("Check BMC ip address is pingable") + self._exit(1) + else: + self.logging_util.ilog("BMC Ping Ok : %s (%i)" % + (self.ip, ping_count)) + + # try to connect + fail_counter = 0 + err_msg = "Unable to establish %s to BMC at %s." % (stage, self.uri) + while fail_counter < MAX_CONNECTION_ATTEMPTS: + ex_log = "" + try: + # One time Redfish Client Object Create + self.redfish_obj = \ + redfish.redfish_client(base_url=self.uri, + username=self.un, + password=self.pw, + default_prefix=REDFISH_ROOT_PATH) + if self.redfish_obj is None: + fail_counter += 1 + else: + return + except Exception as ex: + fail_counter += 1 + ex_log = " (%s)" % str(ex) + + if fail_counter < MAX_CONNECTION_ATTEMPTS: + self.logging_util.wlog(err_msg + " Retry (%i/%i) in %i secs." % + (fail_counter, + MAX_CONNECTION_ATTEMPTS - 1, + CONNECTION_RETRY_INTERVAL) + ex_log) + time.sleep(CONNECTION_RETRY_INTERVAL) + + self.logging_util.elog(err_msg) + self.logging_util.alog( + "Check BMC ip address is pingable and supports Redfish") + self._exit(1) + + ########################################################################### + # Redfish Root Query + ########################################################################### + def _redfish_root_query(self): + """Redfish Root Query""" + + stage = 'Root Query' + self.logging_util.slog(stage) + + if self.make_request(operation=GET, path=None) is False: + self.logging_util.elog("Failed %s GET request") + self._exit(1) + + if self.response_json: + self.root_query_info = self.response_json + + # extract the systems get url needed to learn reset + # actions for the eventual reset. + # + # "Systems": { "@odata.id": "/redfish/v1/Systems/" }, + # + # See Reset section below ; following iso insertion where + # systems_group_url is used. + self.systems_group_url = self.get_key_value('Systems', '@odata.id') + + ########################################################################### + # Create Redfish Communication Session + ########################################################################### + def _redfish_create_session(self): + """Create Redfish Communication Session""" + + stage = 'Create Communication Session' + self.logging_util.slog(stage) + + fail_counter = 0 + while fail_counter < MAX_SESSION_CREATION_ATTEMPTS: + try: + self.redfish_obj.login(auth="session") + self.logging_util.dlog1("Session : Open") + self.session = True + return + except InvalidCredentialsError: + self.logging_util.elog( + "Failed to Create session due to invalid credentials.") + self.logging_util.alog( + "Check BMC username and password in config file") + self._exit(2) + except Exception as ex: + err_msg = "Failed to Create session ; %s." % str(ex) + fail_counter += 1 + if fail_counter >= MAX_SESSION_CREATION_ATTEMPTS: + self.logging_util.elog(err_msg) + self._exit(1) + self.logging_util.wlog(err_msg + " Retry (%i/%i) in %i secs." % + (fail_counter, + MAX_SESSION_CREATION_ATTEMPTS - 1, + CONNECTION_RETRY_INTERVAL)) + time.sleep(SESSION_CREATION_RETRY_INTERVAL) + + ########################################################################### + # Query Redfish Managers + ########################################################################### + def _redfish_get_managers(self): + """Query Redfish Managers""" + + stage = 'Get Managers' + self.logging_util.slog(stage) + + # Virtual Media support is located through the + # Managers link of the root query response. + # + # This section learns that Managers URL Link from the + # Root Query Result: + # + # Expecting something like this ... + # + # { + # ... + # "Managers": + # { + # "@odata.id": "/redfish/v1/Managers/" + # }, + # ... + # } + + # Get Managers Link from the last Get response currently + # in self.response_json + self.managers_group_url = self.get_key_value('Managers', '@odata.id') + if self.managers_group_url is None: + self.logging_util.elog("Failed to learn BMC RedFish Managers link") + self._exit(1) + + # Managers Query (/redfish/v1/Managers/) + if self.make_request(operation=GET, + path=self.managers_group_url) is False: + self.logging_util.elog("Failed GET Managers from %s" % + self.managers_group_url) + self._exit(1) + + # Look for the Managers 'Members' URL Link list from the Managers Query + # + # Expect something like this ... + # + # { + # ... + # "Members": + # [ + # { "@odata.id": "/redfish/v1/Managers/1/" } + # ], + # ... + # } + # Support multiple Managers in the list + + self.manager_members_list = self.get_key_value('Members') + + ###################################################################### + # Get Systems Members + ###################################################################### + def _redfish_get_systems_members(self): + """Get Systems Members""" + + stage = 'Get Systems' + self.logging_util.slog(stage) + + # Query Systems Group URL for list of Systems Members + if self.make_request(operation=GET, + path=self.systems_group_url) is False: + self.logging_util.elog("Unable to %s Members from %s" % + (stage, self.systems_group_url)) + self._exit(1) + + self.systems_members_list = self.get_key_value('Members') + self.logging_util.dlog3("Systems Members List: %s" % + self.systems_members_list) + if self.systems_members_list is None: + self.logging_util.elog("Systems Members URL GET Response\n%s" % + self.response_json) + self._exit(1) + + self.systems_members = len(self.systems_members_list) + if self.systems_members == 0: + self.logging_util.elog( + "BMC not publishing any System Members:\n%s" % + self.response_json) + self._exit(1) + + ###################################################################### + # Power On or Off Host + ###################################################################### + def _redfish_powerctl_host(self, state, verify=True, request_command=None): + """Power On or Off the Host + + :param state: The power state (On/Off). + :type state: str. + :param verify: True if verification of power state is required. + :type verify: bool. + :param request_command: Specify a dedicated type of power-off. + :type request_command: str. + """ + stage = 'Power ' + state + ' Host' + self.logging_util.slog(stage) + + if self.power_state == state: + # already in required state + return + + # Walk the Systems Members list looking for Action support. + # + # "Members": [ { "@odata.id": "/redfish/v1/Systems/1/" } ], + # + # Loop over Systems Members List looking for Reset Actions Dictionary + info = 'Redfish Systems Actions Member' + self.systems_member_url = None + for member in range(self.systems_members): + systems_member = self.systems_members_list[member] + if systems_member: + self.systems_member_url = systems_member.get('@odata.id') + if self.systems_member_url is None: + self.logging_util.elog("Unable to get %s URL:\n%s\n" % + (info, self.response_json)) + self._exit(1) + + if self.make_request(operation=GET, + path=self.systems_member_url, + retry=0) is False: + self.logging_util.elog("Unable to get %s from %s" % + (info, self.systems_member_url)) + self._exit(1) + + # Look for Reset Actions Dictionary + self.reset_action_dict = \ + self.get_key_value('Actions', '#ComputerSystem.Reset') + if self.reset_action_dict is None: + # try other URL + self.logging_util.dlog2( + "No #ComputerSystem.Reset actions from %s. Try other URL." + % self.systems_member_url) + self.systems_member_url = None + continue + else: + # Got the Reset Actions Dictionary + + # get powerState + self.power_state = self.get_key_value('PowerState') + + # Ensure we don't issue current state command + if state in [POWER_OFF, POWER_ON]: + # This is a Power ON or Off command + if self.power_state == state: + self.logging_util.dlog2("Power already %s" % state) + # ... AND we are already in that state then + # we are done. Issuing a power command while + # in the same state will error out. + # So don't do it. + return + break + + info = 'Systems Reset Action Dictionary' + if self.reset_action_dict is None: + self.logging_util.elog("BMC not publishing %s:\n%s\n" % + (info, self.response_json)) + self._exit(1) + + ############################################################## + # Reset Actions Dictionary. This is what we are looking for # + ############################################################## + # + # Look for Reset Actions label + # + # "Actions": + # { + # "#ComputerSystem.Reset": + # { + # "ResetType@Redfish.AllowableValues": [ + # "On", + # "ForceOff", + # "ForceRestart", + # "Nmi", + # "PushPowerButton" + # ], + # "target":"/redfish/v1/Systems/1/Actions/ComputerSystem.Reset/" + # } + # } + # + # Need to get 2 pieces of information out of the Actions output + # + # 1. the Redfish Systems Reset Action Target + # 2. the Redfish Systems Reset Action List + # + ############################################################### + + info = 'Systems Reset Action Target' + self.reset_command_url = self.reset_action_dict.get('target') + if self.reset_command_url is None: + self.logging_util.elog( + "Unable to get Reset Command URL (members:%d)\n%s" % + (self.systems_members, self.reset_action_dict)) + self._exit(1) + + # With the reset target url in hand, all that is needed now + # is the reset command this target supports + # + # The reset command list looks like this. + # + # "ResetType@Redfish.AllowableValues": [ + # "On", + # "ForceOff", + # "ForceRestart", + # "Nmi", + # "PushPowerButton" + # ], + # + # Some targets support GracefulRestart and/or ForceRestart + + info = 'Allowable Reset Actions' + reset_command_list = \ + self.reset_action_dict.get('ResetType@Redfish.AllowableValues') + if reset_command_list is None: + self.logging_util.elog("BMC is not publishing any %s" % info) + self._exit(1) + + self.logging_util.ilog("ResetActions: %s" % reset_command_list) + + if request_command: + acceptable_commands = [request_command] + else: + # load the appropriate acceptable command list + if state == POWER_OFF: + acceptable_commands = ['ForceOff', 'GracefulShutdown'] + elif state == POWER_ON: + acceptable_commands = ['ForceOn', 'On'] + else: + acceptable_commands = ['ForceRestart', 'GracefulRestart'] + + # Look for the best command for the power state requested. + command = None + for acceptable_command in acceptable_commands: + for reset_command in reset_command_list: + if reset_command == acceptable_command: + self.logging_util.ilog( + "Selected Command: %s" % reset_command) + command = reset_command + break + else: + continue + break + + if command is None: + self.logging_util.elog( + "Failed to find acceptable Power %s command in:\n%s" % + (state, reset_command_list)) + self._exit(1) + + # All that is left to do is POST the reset command + # to the reset_command_url. + payload = {'ResetType': command} + if self.make_request(operation=POST, + payload=payload, + path=self.reset_command_url) is False: + self.logging_util.elog("Failed to Power %s Host" % state) + self._exit(1) + + if (state not in [POWER_OFF, POWER_ON]) or (not verify): + # no need to refresh power state if + # this was not a power command; + # no need to verify the power state + return + + # Set the timeout in seconds (14 minutes = 840 seconds) + timeout = int(os.environ.get('RVMC_POWER_ACTION_TIMEOUT', 840)) + self.logging_util.ilog("%s timeout is %d seconds" % (stage, timeout)) + + # Get the start time + start_time = time.time() + + # init wait duration + duration = 0 + + # poll for requested power state. + while time.time() - start_time < timeout and self.power_state != state: + time.sleep(10) + + # update wait duration + duration = int(time.time() - start_time) + + # get systems info + if self.make_request(operation=GET, + path=self.systems_member_url) is False: + self.logging_util.elog( + "Failed to Get System State (after %d secs)" % duration) + else: + # get powerState + self.power_state = self.get_key_value('PowerState') + if self.power_state != state: + self.logging_util.dlog1( + "Waiting for Power %s (currently %s) (%d secs)" % + (state, self.power_state, duration)) + + if self.power_state != state: + self.logging_util.elog( + "Failed to Set System Power State to %s after %d secs (%s)" % + (state, duration, self.systems_member_url)) + self._exit(1) + else: + self.logging_util.ilog("%s verified (after %d seconds)" % + (stage, duration)) + + ###################################################################### + # Get CD/DVD Virtual Media URL + ###################################################################### + def _redfish_get_vm_url(self): + """Get CD/DVD Virtual Media URL from one of the Manager Members list""" + + stage = 'Get CD/DVD Virtual Media' + self.logging_util.slog(stage) + + if self.manager_members_list is None: + self.logging_util.elog("Unable to index Managers Members from %s" % + self.managers_group_url) + self._exit(1) + + members = len(self.manager_members_list) + if members == 0: + self.logging_util.elog( + "BMC is not publishing any redfish Manager Members") + self._exit(1) + + # Issue a Get from each 'Manager Member URL Link looking + # for supported virtual devices. + for member in range(members): + member_url = None + this_member = self.manager_members_list[member] + if this_member: + member_url = this_member.get('@odata.id') + if member_url is None: + continue + if self.make_request(operation=GET, path=member_url) is False: + self.logging_util.elog( + "Unable to get Manager Member from %s" % member_url) + self._exit(1) + + ######################################################## + # Query Virtual Media # + ######################################################## + # Look for Virtual Media Support by this Manager Member + # + # Expect something like this ... + # + # { + # ... + # "VirtualMedia": + # { + # "@odata.id": "/redfish/v1/Managers/1/VirtualMedia/" + # } + # ... + # } + self.vm_group_url = None + self.vm_group = self.get_key_value('VirtualMedia') + if self.vm_group is None: + if (member + 1) == members: + self.logging_util.elog( + "Virtual Media not supported by target BMC") + self._exit(1) + else: + self.logging_util.dlog3( + "Virtual Media not supported by member %d" % member) + continue + else: + try: + self.vm_group_url = self.vm_group.get('@odata.id') + except Exception: + self.logging_util.elog( + "Unable to get Virtual Media Group from %s" % + self.vm_group_url) + self._exit(1) + + # Query this member's Virtual Media Service Group + if self.make_request( + operation=GET, path=self.vm_group_url) is False: + self.logging_util.elog( + "Failed to GET Virtual Media Service group from %s" % + self.vm_group_url) + continue + + # Look for Virtual Media Device URL Links + # + # Expect something like this ... + # + # { + # ... + # "Members": + # [ + # { "@odata.id": "/redfish/v1/Managers/1/VirtualMedia/1/" }, + # { "@odata.id": "/redfish/v1/Managers/1/VirtualMedia/2/" } + # ], + # ... + # } + self.vm_members_array = [] + try: + self.vm_members_array = self.get_key_value('Members') + vm_members = len(self.vm_members_array) + except Exception: + vm_members = 0 + + if vm_members == 0: + self.logging_util.elog("No Virtual Media members found at %s" % + self.vm_group_url) + self._exit(1) + + # Loop over each member's URL looking for the CD or DVD device + # Consider trying the USB device as well if BMC supports that. + for vm_member in range(vm_members): + + # Look for Virtual Media Device URL + this_member = self.vm_members_array[vm_member] + if this_member: + self.vm_url = this_member.get('@odata.id') + + if self.make_request(operation=GET, path=self.vm_url) is False: + self.logging_util.elog( + "Failed to GET Virtual Media Service group from %s" % + self.vm_group_url) + continue + + # Query Virtual Media Device Type looking for supported device + self.vm_media_types = self.get_key_value('MediaTypes') + if self.vm_media_types is None: + self.logging_util.dlog3( + "No Virtual MediaTypes found at %s ; " + "trying other members" % self.vm_url) + break + + self.logging_util.dlog4("Virtual Media Service:\n%s" % + self.response_json) + + if supported_device(self.vm_media_types) is True: + self.logging_util.dlog3( + "Supported Virtual Media found at %s ; %s" % + (self.vm_url, self.vm_media_types)) + break + else: + self.logging_util.dlog3( + "Virtual Media %s does not support CD/DVD ; " + "trying other members" % self.vm_url) + self.vm_url = None + + if self.vm_url is None: + self.logging_util.elog( + "Failed to find CD or DVD Virtual media type") + self._exit(1) + + ###################################################################### + # Load Selected Virtual Media Version and Actions + ###################################################################### + def _redfish_load_vm_actions(self): + """Load Selected Virtual Media Version and Actions""" + + stage = 'Load Selected Virtual Media Version and Actions' + self.logging_util.slog(stage) + + if self.vm_url is None: + self.logging_util.elog( + "Failed to find CD or DVD Virtual media type") + self._exit(1) + + # Extract Virtual Media Version and Insert/Eject Actions + # + # Looks something like this. First half of odata.type is the VM label + # + # { + # ... + # "@odata.type": "#VirtualMedia.v1_2_0.VirtualMedia", + # "Actions": { + # "#VirtualMedia.EjectMedia": + # { + # "target" : + # ".../Managers/1/VirtualMedia/2/Actions/VirtualMedia.EjectMedia/" + # }, + # "#VirtualMedia.InsertMedia": + # { + # "target": + # ".../Managers/1/VirtualMedia/2/Actions/VirtualMedia.InsertMedia/" + # } + # ... + # }, + vm_data_type = self.get_key_value('@odata.type') + if vm_data_type: + self.vm_label = vm_data_type.split('.')[0] + self.vm_version = vm_data_type.split('.')[1] + self.vm_actions = self.get_key_value('Actions') + self.logging_util.dlog1("VM Version : %s" % self.vm_version) + self.logging_util.dlog1("VM Label : %s" % self.vm_label) + self.logging_util.dlog3("VM Actions :\n%s\n" % self.vm_actions) + + ###################################################################### + # Power Off Host + ###################################################################### + def _redfish_poweroff_host(self, verify=True, request_command=None): + """Power Off the Host + + :param verify: True if verification of power state is required. + :type verify: bool. + :param request_command: Specify a dedicated type of power-off. + :type request_command: str. + """ + self._redfish_powerctl_host(POWER_OFF, verify, request_command) + + ###################################################################### + # Eject Current Image + ###################################################################### + def _redfish_eject_image(self): + """Eject Current Image""" + + stage = 'Eject Current Image' + self.logging_util.slog(stage) + + if self.make_request(operation=GET, path=self.vm_url) is False: + self.logging_util.elog( + "Virtual media status query failed (%s)" % self.vm_url) + self._exit(1) + + if self.get_key_value('Inserted') is False: + self.logging_util.ilog( + "Skip ejection of the image because no image is inserted.") + return + + # Ensure there is no image inserted and handle the case where + # one might be in the process of loading. + MAX_EJECT_RETRY_COUNT = 10 + eject_retry_count = 0 + ejecting = True + eject_media_label = '#VirtualMedia.EjectMedia' + while eject_retry_count < MAX_EJECT_RETRY_COUNT and ejecting: + eject_retry_count = eject_retry_count + 1 + vm_eject = self.vm_actions.get(eject_media_label) + if not vm_eject: + self.logging_util.elog( + "Failed to get eject target (%s)" % eject_media_label) + self._exit(1) + + self.vm_eject_url = vm_eject.get('target') + if self.vm_eject_url: + if self.get_key_value('Image'): + self.logging_util.ilog("Eject Request Image %s" % + (self.get_key_value('Image'))) + else: + self.logging_util.dlog1("Eject Request") + + if self.make_request(operation=POST, + payload={}, + path=self.vm_eject_url) is False: + self.logging_util.elog( + "Eject request failed (%s)" % self.vm_eject_url) + # accept this and continue to poll + + time.sleep(DELAY_2_SECS) + poll_count = 0 + while poll_count < MAX_POLL_COUNT and ejecting: + # verify the image is not in inserted + poll_count = poll_count + 1 + if self.make_request(operation=GET, + path=self.vm_url) is True: + if self.get_key_value('Inserted') is False: + self.logging_util.ilog("Ejected") + ejecting = False + elif self.get_key_value('Image'): + # if image is present then its ready to + # retry the eject, break out of poll loop + self.logging_util.dlog1( + "Image Present ; %s" % + self.get_key_value('Image')) + break + else: + self.logging_util.dlog1( + "Eject Wait ; Image: %s" % + self.get_key_value('Image')) + time.sleep(RETRY_DELAY_SECS) + else: + self.logging_util.elog( + "Failed to query vm state (%s)" % self.vm_url) + self._exit(1) + + if ejecting is True: + self.logging_util.elog("%s wait timeout" % stage) + self._exit(1) + + ###################################################################### + # Insert Image into Virtual Media CD/DVD + ###################################################################### + def _redfish_insert_image(self): + """Insert Image into Virtual Media CD/DVD""" + + stage = 'Insert Image into Virtual Media CD/DVD' + self.logging_util.slog(stage) + + vm_insert_url = None + vm_insert_act = self.vm_actions.get('#VirtualMedia.InsertMedia') + if vm_insert_act: + vm_insert_url = vm_insert_act.get('target') + + if vm_insert_url is None: + self.logging_util.elog( + "Unable to get Virtual Media Insertion URL\n%s\n" % + self.response_json) + self._exit(1) + + payload = {'Image': self.img, + 'Inserted': True, + 'WriteProtected': True} + if self.make_request(operation=POST, + payload=payload, + path=vm_insert_url) is False: + self.logging_util.elog("Failed to Insert Media") + self._exit(1) + + # Handle case where the BMC loads the iso image during the insertion. + # In that case the 'Inserted' is True but the Image is not immediately + # mounted. + poll_count = 0 + ImageInserting = True + while poll_count < MAX_POLL_COUNT and ImageInserting: + if self.make_request(operation=GET, path=self.vm_url) is False: + self.logging_util.elog( + "Unable to verify Image insertion (%s)" % self.vm_url) + self._exit(1) + + if self.get_key_value('Image') == self.img: + self.logging_util.ilog("Image Insertion (took %i seconds)" % + (poll_count * RETRY_DELAY_SECS)) + ImageInserting = False + else: + time.sleep(RETRY_DELAY_SECS) + poll_count = poll_count + 1 + self.logging_util.dlog1( + "Image Insertion Wait ; %3d secs (%3d of %3d)" % + (poll_count * RETRY_DELAY_SECS, + poll_count, + MAX_POLL_COUNT)) + + if ImageInserting is True: + self.logging_util.elog("Image insertion timeout") + self._exit(1) + else: + self.logging_util.ilog("%s verified (took %i seconds)" % + (stage, poll_count * RETRY_DELAY_SECS)) + + if self.get_key_value('Image') != self.img: + self.logging_util.elog("Insertion verification failed.") + self.logging_util.ilog("Expected Image: %s" % self.img) + self.logging_util.ilog( + "Detected Image: %s" % self.get_key_value('Image')) + self._exit(1) + + # Verify Insertion + # + # Looking for the following values + # + self.logging_util.dlog3( + "Image URI : %s" % self.get_key_value('Image')) + self.logging_util.dlog3( + "ImageName : %s" % self.get_key_value('ImageName')) + self.logging_util.dlog3( + "Inserted : %s" % self.get_key_value('Inserted')) + self.logging_util.dlog3( + "Protected : %s" % self.get_key_value('WriteProtected')) + + ###################################################################### + # Set Next Boot Override to CD/DVD + ###################################################################### + def _redfish_set_boot_override(self): + """Set Next Boot Override to CD/DVD""" + + stage = 'Set Next Boot Override to CD/DVD' + self.logging_util.slog(stage) + + # Walk the Systems Members list looking for Boot support. + # + # "Members": [ { "@odata.id": "/redfish/v1/Systems/1/" } ], + # + # Loop over Systems Members List looking for Boot Dictionary + info = 'Systems Boot Member' + for member in range(self.systems_members): + + self.systems_member_url = None + systems_member = self.systems_members_list[member] + if systems_member: + self.systems_member_url = systems_member.get('@odata.id') + if self.systems_member_url is None: + self.logging_util.elog("Unable to get %s from %s" % + (info, self.systems_members_list)) + self._exit(1) + + if self.make_request(operation=GET, + path=self.systems_member_url) is False: + self.logging_util.elog("Unable to get %s from %s" % + (info, self.systems_member_url)) + self._exit(1) + + # Look for Reset Actions Dictionary + self.boot_control_dict = self.get_key_value('Boot') + if self.boot_control_dict is None: + continue + + if self.boot_control_dict is None: + self.logging_util.elog( + "Unable to get %s from %s" % (info, self.systems_member_url)) + self._exit(1) + else: + allowable_label = 'BootSourceOverrideMode@Redfish.AllowableValues' + mode_list = self.get_key_value('Boot', allowable_label) + if mode_list is None: + payload = {"Boot": {"BootSourceOverrideEnabled": "Once", + "BootSourceOverrideTarget": "Cd"}} + else: + self.logging_util.dlog1("Boot Override Modes: %s" % mode_list) + + # Prioritize UEFI over Legacy + if "UEFI" in mode_list: + payload = {"Boot": {"BootSourceOverrideEnabled": "Once", + "BootSourceOverrideMode": "UEFI", + "BootSourceOverrideTarget": "Cd"}} + elif "Legacy" in mode_list: + payload = {"Boot": {"BootSourceOverrideEnabled": "Once", + "BootSourceOverrideMode": "Legacy", + "BootSourceOverrideTarget": "Cd"}} + else: + self.logging_util.elog( + "BootSourceOverrideModes %s not supported" % mode_list) + self._exit(0) + + self.logging_util.dlog2("Boot Override Payload: %s" % payload) + + if self.make_request(operation=PATCH, + path=self.systems_member_url, + payload=payload) is False: + self.logging_util.elog( + "Unable to Set Boot Override (%s)" % self.vm_url) + self._exit(1) + + if self.make_request(operation=GET, + path=self.systems_member_url) is False: + self.logging_util.elog( + "Unable to verify Set Boot Override (%s)" % self.vm_url) + self._exit(1) + else: + enabled = self.get_key_value('Boot', 'BootSourceOverrideEnabled') + device = self.get_key_value('Boot', 'BootSourceOverrideTarget') + mode = self.get_key_value('Boot', 'BootSourceOverrideMode') + if enabled == "Once" and \ + supported_device(self.vm_media_types) is True: + self.logging_util.ilog("%s verified [%s:%s:%s]" % + (stage, enabled, device, mode)) + else: + self.logging_util.elog( + "Unable to verify Set Boot Override [%s:%s:%s]" % + (enabled, device, mode)) + self._exit(1) + + ###################################################################### + # Power On Host + ###################################################################### + def _redfish_poweron_host(self): + """Power On or Off the Host""" + + self._redfish_powerctl_host(POWER_ON) + + def execute(self): + """The main controller function that executes the iso insertion + + algorithm for the specified target object (self) + """ + self._redfish_client_connect() + self._redfish_root_query() + self._redfish_create_session() + self._redfish_get_managers() + self._redfish_get_systems_members() + self._redfish_get_vm_url() + self._redfish_load_vm_actions() + self._redfish_eject_image() + self._redfish_poweroff_host() + self._redfish_insert_image() + self._redfish_set_boot_override() + self._redfish_poweron_host() + + self.logging_util.ilog("Done") + + if self.redfish_obj is not None and self.session is True: + self.redfish_obj.logout() + self.logging_util.dlog1("Session : Closed") + + def poweroff_only(self, verify=False, request_command='ForceOff'): + """Power-off only without any iso related operations. + + :param verify: True if verification of power state is required. + :type verify: bool. + :param request_command: Specify a dedicated type of power-off. + :type request_command: str. + """ + self._redfish_client_connect() + self._redfish_root_query() + self._redfish_create_session() + self._redfish_get_managers() + self._redfish_get_systems_members() + self._redfish_poweroff_host(verify, request_command) + + vstr = 'with' if verify else 'without' + self.logging_util.ilog("%s request was sent out %s verification." % + (request_command, vstr)) + + if self.redfish_obj is not None and self.session is True: + self.redfish_obj.logout() + self.logging_util.dlog1("Session : Closed") + + +############################################################################## +# Methods to be called from rvmc module +############################################################################## +def power_off(subcloud_name, config_file, logger): + """Power Off the Host. + + :param subcloud_name: Subcloud name. + :type subcloud_name: str. + :param config_file: RVMC config file containing BMC info. + :type config_file: str. + :param logger: The logger + :type logger: logging.Logger + """ + if not subcloud_name or subcloud_name == '': + raise exceptions.RvmcException("Subcloud name is missing.") + + logging_util = LoggingUtil(logger, subcloud_name) + exit_handler = ExitHandler() + + if not (config_file and os.path.exists(config_file)): + raise exceptions.RvmcException("RVMC config file is missing.") + else: + logging_util.dlog1("Config file : %s" % config_file) + + logging_util.ilog("Starting power-off.") + + config, target_object = parse_config_file( + subcloud_name, config_file, logging_util, exit_handler) + + if target_object: + try: + if target_object.target is not None: + logging_util.ilog("BMC Target : %s" % target_object.target) + logging_util.ilog("BMC IP Addr : %s" % target_object.ip) + logging_util.ilog("Host Image : %s" % target_object.img) + target_object.poweroff_only(False, 'ForceOff') + except Exception as e: + raise e + else: + if config_file and config: + logging_util.ilog("Config File :\n%s" % config) + raise exceptions.RvmcException( + "Operation aborted ; no valid bmc information found") diff --git a/distributedcloud/dccommon/subcloud_install.py b/distributedcloud/dccommon/subcloud_install.py index 2d0f90591..c90143b84 100644 --- a/distributedcloud/dccommon/subcloud_install.py +++ b/distributedcloud/dccommon/subcloud_install.py @@ -13,41 +13,31 @@ # limitations under the License. # -import json import os import shutil import socket -import ssl import tempfile -import time from eventlet.green import subprocess import netaddr -from oslo_concurrency import lockutils from oslo_log import log as logging from six.moves.urllib import error as urllib_error from six.moves.urllib import parse from six.moves.urllib import request -import yaml from dccommon import consts from dccommon.drivers.openstack.keystone_v3 import KeystoneClient from dccommon.drivers.openstack.sysinv_v1 import SysinvClient from dccommon import exceptions -from dccommon import kubeoperator from dccommon import utils as dccommon_utils from dcmanager.common import consts as dcmanager_consts from dcmanager.common import utils + LOG = logging.getLogger(__name__) BOOT_MENU_TIMEOUT = '5' -# The RVMC_IMAGE_NAME:RVMC_IMAGE_TAG must align with the one specified -# in system images in the ansible install/upgrade playbook -RVMC_IMAGE_NAME = 'docker.io/starlingx/rvmc' -RVMC_IMAGE_TAG = 'stx.8.0-v1.0.2' - SUBCLOUD_ISO_PATH = '/opt/platform/iso' SUBCLOUD_ISO_DOWNLOAD_PATH = '/var/www/pages/iso' SUBCLOUD_FEED_PATH = '/var/www/pages/feed' @@ -60,104 +50,6 @@ NETWORK_INTERFACE_PREFIX = 'ifcfg' NETWORK_ROUTE_PREFIX = 'route' LOCAL_REGISTRY_PREFIX = 'registry.local:9001/' -# Redfish constants -ACTION_URL = '/Actions/ComputerSystem.Reset' -POWER_OFF_PAYLOAD = {'Action': 'Reset', 'ResetType': 'ForceOff'} -REDFISH_HEADER = {'Content-Type': 'application/json', - 'Accept': 'application/json'} -REDFISH_SYSTEMS_URL = '/redfish/v1/Systems' -SUCCESSFUL_STATUS_CODES = [200, 202, 204] - -RVMC_LOCK_NAME = 'dc-rvmc-install' -RVMC_NAMESPACE = 'rvmc' -KUBE_SYSTEM_NAMESPACE = 'kube-system' -DEFAULT_REGISTRY_KEY = 'default-registry-key' - - -class SubcloudShutdown(object): - """Sends a shutdown signal to a Redfish controlled subcloud - - Approach: - - To shutdown a Redfish controlled subcloud, it's needed to first - send a GET request to find the @odata.id of the member, and then - send a POST request with the shutdown signal. Since this is - intended as a way to turn off the subcloud during the deploy abort - process, only the ForceOff option is considered. - """ - def __init__(self, subcloud_name): - self.target = subcloud_name - self.rvmc_data = self._get_subcloud_data() - - def _get_subcloud_data(self): - rvmc_config_file_path = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, - self.target, consts.RVMC_CONFIG_FILE_NAME) - if not os.path.isfile(rvmc_config_file_path): - raise Exception('Missing rvmc files for %s' % self.target) - with open(os.path.abspath(rvmc_config_file_path), 'r') as f: - rvmc_data = f.read() - rvmc_config_values = yaml.load(rvmc_data, Loader=yaml.SafeLoader) - base_url = "https://" + rvmc_config_values['bmc_address'] - bmc_username = rvmc_config_values['bmc_username'] - bmc_password = rvmc_config_values['bmc_password'] - credentials = ("%s:%s" % (bmc_username.rstrip(), bmc_password)).encode("utf-8") - return {'base_url': base_url, 'credentials': credentials} - - def _make_request(self, url, credentials, method, retry=5): - if method == 'get': - payload = None - else: - payload = json.dumps(POWER_OFF_PAYLOAD).encode('utf-8') - - try: - context = ssl._create_unverified_context() - req = request.Request(url, headers=REDFISH_HEADER, method=method) - req.add_header('Authorization', 'Basic %s' % credentials) - response = request.urlopen(req, data=payload, context=context) - status_code = response.getcode() - - if status_code not in SUCCESSFUL_STATUS_CODES: - if retry <= 0: - raise exceptions.SubcloudShutdownError( - subcloud_name=self.target) - retry -= retry - time.sleep(2) - self._make_request(url, credentials, method, retry=retry) - except urllib_error.URLError: - # This occurs when the BMC is not available anymore, - # so we just ignore it. - return None - except Exception as ex: - raise ex - - return response - - def _get_data_id(self): - base_url = self.rvmc_data['base_url'] - credentials = self.rvmc_data['credentials'] - url = base_url + REDFISH_SYSTEMS_URL - response = self._make_request(url, credentials, method='GET') - if not response: - return None - r = json.loads(response.read().decode()) - - for member in r['Members']: - if member.get('@odata.id'): - url_with_id = member['@odata.id'] - break - - return url_with_id - - def send_shutdown_signal(self): - base_url = self.rvmc_data['base_url'] - credentials = self.rvmc_data['credentials'] - url_with_id = self._get_data_id() - if not url_with_id: - return None - url = base_url + url_with_id + ACTION_URL - response = self._make_request(url, credentials, method='POST') - return response - class SubcloudInstall(object): """Class to encapsulate the subcloud install operations""" @@ -256,17 +148,6 @@ class SubcloudInstall(object): return "%s://%s:%s" % (protocol, self.get_oam_address(), port) - def check_image_exists(self, image_name, image_tag): - tags = self.sysinv_client.get_registry_image_tags(image_name) - if tags: - if any(getattr(tag, 'tag') == image_tag for tag in tags): - return - msg = "Error: Image %s:%s not found in the local registry." % ( - image_name, image_tag) - LOG.error(msg) - raise exceptions.ImageNotInLocalRegistry(image_name=image_name, - image_tag=image_tag) - @staticmethod def create_rvmc_config_file(override_path, payload): @@ -279,39 +160,17 @@ class SubcloudInstall(object): if k in consts.BMC_INSTALL_VALUES or k == 'image': f_out_rvmc_config_file.write(k + ': ' + v + '\n') - @lockutils.synchronized(RVMC_LOCK_NAME) - def copy_default_registry_key(self): - """Copy default-registry-key secret for pulling rvmc image.""" - kube = kubeoperator.KubeOperator() - try: - if kube.kube_get_secret(DEFAULT_REGISTRY_KEY, RVMC_NAMESPACE) is None: - if not kube.kube_get_namespace(RVMC_NAMESPACE): - LOG.info("Creating rvmc namespace") - kube.kube_create_namespace(RVMC_NAMESPACE) - LOG.info("Copying default-registry-key secret to rvmc namespace") - kube.kube_copy_secret( - DEFAULT_REGISTRY_KEY, KUBE_SYSTEM_NAMESPACE, RVMC_NAMESPACE) - except Exception as e: - LOG.exception("Failed to copy default-registry-key secret") - raise e - def create_install_override_file(self, override_path, payload): LOG.debug("create install override file") - self.check_image_exists(RVMC_IMAGE_NAME, RVMC_IMAGE_TAG) - rvmc_image = LOCAL_REGISTRY_PREFIX + RVMC_IMAGE_NAME + ':' +\ - RVMC_IMAGE_TAG install_override_file = os.path.join(override_path, 'install_values.yml') - rvmc_name = "%s-%s" % (consts.RVMC_NAME_PREFIX, self.name) host_name = socket.gethostname() with open(install_override_file, 'w') as f_out_override_file: f_out_override_file.write( '---' '\npassword_change: true' - '\nrvmc_image: ' + rvmc_image + - '\nrvmc_name: ' + rvmc_name + '\nhost_name: ' + host_name + '\nrvmc_config_dir: ' + override_path + '\n' @@ -687,9 +546,6 @@ class SubcloudInstall(object): # create the rvmc config file self.create_rvmc_config_file(override_path, payload) - # copy the default_registry_key secret to rvmc namespace - self.copy_default_registry_key() - # remove the bmc values from the payload for k in consts.BMC_INSTALL_VALUES: if k in payload: diff --git a/distributedcloud/dccommon/utils.py b/distributedcloud/dccommon/utils.py index 444203f93..510402dda 100644 --- a/distributedcloud/dccommon/utils.py +++ b/distributedcloud/dccommon/utils.py @@ -30,6 +30,7 @@ from dccommon import consts from dccommon import exceptions from dccommon.exceptions import PlaybookExecutionFailed from dccommon.exceptions import PlaybookExecutionTimeout +from dccommon import rvmc from dccommon.subprocess_cleanup import kill_subprocess_group from dccommon.subprocess_cleanup import SubprocessCleanup from dcorch.common.i18n import _ @@ -356,3 +357,19 @@ def get_ssl_cert_ca_file(): return os.path.join( consts.SSL_CERT_CA_DIR, consts.CERT_CA_FILE_DEBIAN if is_debian() else consts.CERT_CA_FILE_CENTOS) + + +def send_subcloud_shutdown_signal(subcloud_name): + """Sends a shutdown signal to a Redfish controlled subcloud. + + :param subcloud_name: the name of the subcloud to be shut down + :type subcloud_name: str + """ + # All logs are expected to originate from the rvmc module, + # so the log churn from the 'redfish.rest.v1' module is disabled. + logging.getLogger('redfish.rest.v1').setLevel(logging.CRITICAL) + + rvmc_config_file = os.path.join(consts.ANSIBLE_OVERRIDES_PATH, + subcloud_name, + consts.RVMC_CONFIG_FILE_NAME) + rvmc.power_off(subcloud_name, rvmc_config_file, LOG) diff --git a/distributedcloud/dcmanager/common/consts.py b/distributedcloud/dcmanager/common/consts.py index 57af3dd43..7ed1d13ac 100644 --- a/distributedcloud/dcmanager/common/consts.py +++ b/distributedcloud/dcmanager/common/consts.py @@ -263,6 +263,10 @@ ERR_MSG_DICT = { "ping_bmc": "Check reachability to the BMC: ping <>", + "rvmc_process": "Ensure the previous RVMC process is terminated.", + + "rvmc_timeout": "Please check the dcmanager ansible log for details.", + "dm_pod_failed": """- Ensure you are using the correct tarball that \ corresponds to the image. - Check helm overrides files, ensure the deployment manager images exist in \ diff --git a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py index dc34733b4..57f9ca2ac 100644 --- a/distributedcloud/dcmanager/common/phased_subcloud_deploy.py +++ b/distributedcloud/dcmanager/common/phased_subcloud_deploy.py @@ -638,6 +638,16 @@ def validate_install_values(payload, subcloud=None): LOG.exception(e) pecan.abort(400, _("rd.net.timeout.ipv6dad invalid: %s") % e) + if 'rvmc_debug_level' in install_values: + try: + rvmc_debug_level = int(install_values['rvmc_debug_level']) + if rvmc_debug_level < 0 or rvmc_debug_level > 4: + pecan.abort(400, _("rvmc_debug_level must be an integer " + "between 0 and 4.")) + except ValueError as e: + LOG.exception(e) + pecan.abort(400, _("Invalid value of rvmc_debug_level: %s") % e) + def validate_k8s_version(payload): """Validate k8s version. diff --git a/distributedcloud/dcmanager/manager/subcloud_manager.py b/distributedcloud/dcmanager/manager/subcloud_manager.py index 2d3b52883..32b899b98 100644 --- a/distributedcloud/dcmanager/manager/subcloud_manager.py +++ b/distributedcloud/dcmanager/manager/subcloud_manager.py @@ -46,9 +46,9 @@ from dccommon.exceptions import PlaybookExecutionFailed from dccommon.exceptions import SubcloudNotFound from dccommon import kubeoperator from dccommon.subcloud_install import SubcloudInstall -from dccommon.subcloud_install import SubcloudShutdown from dccommon.utils import AnsiblePlaybook from dccommon.utils import LAST_SW_VERSION_IN_CENTOS +from dccommon.utils import send_subcloud_shutdown_signal from dcmanager.audit import rpcapi as dcmanager_audit_rpc_client from dcmanager.common import consts from dcmanager.common.consts import INVENTORY_FILE_POSTFIX @@ -291,7 +291,11 @@ class SubcloudManager(manager.Manager): "-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + subcloud_name + '/' + "install_values.yml", "-e", "install_release_version=%s" % - software_version if software_version else SW_VERSION] + software_version if software_version else SW_VERSION, + "-e", "rvmc_config_file=%s" % + os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, + subcloud_name, + dccommon_consts.RVMC_CONFIG_FILE_NAME)] return install_command def compose_bootstrap_command(self, subcloud_name, @@ -1107,24 +1111,11 @@ class SubcloudManager(manager.Manager): return if subcloud.deploy_status == consts.DEPLOY_STATE_ABORTING_INSTALL: - # First delete the k8s job and pod, stopping the current - # installation attempt if exists - # Then send shutdown signal to subcloud - kube = kubeoperator.KubeOperator() - shutdown_subcloud = SubcloudShutdown(subcloud.name) - namespace = dccommon_consts.RVMC_NAME_PREFIX - jobname = '%s-%s' % (namespace, subcloud.name) - pod_basename = '%s-' % jobname - all_pods = kube.get_pods_by_namespace(namespace) - desired_pod = next((s for s in all_pods if pod_basename in s), None) - if desired_pod: - kube.kube_delete_job(jobname, namespace) - kube.kube_delete_pod(desired_pod, namespace) - while kube.pod_exists(desired_pod, namespace): - time.sleep(2) - shutdown_subcloud.send_shutdown_signal() + # Send shutdown signal to subcloud + send_subcloud_shutdown_signal(subcloud.name) except Exception as ex: - LOG.error("Subcloud deploy abort failed for subcloud %s" % subcloud.name) + LOG.error("Subcloud deploy abort failed for subcloud %s: %s" % + (subcloud.name, str(ex))) utils.update_abort_status(context, subcloud.id, subcloud.deploy_status, abort_failed=True) # exception is logged above diff --git a/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py b/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py index 468ae1803..c6d90a0bd 100644 --- a/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py +++ b/distributedcloud/dcmanager/orchestrator/states/upgrade/upgrading_simplex.py @@ -356,12 +356,17 @@ class UpgradingSimplexState(BaseState): utils.create_subcloud_inventory(install_values, ansible_subcloud_inventory_file) + rvmc_config_file = os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, + strategy_step.subcloud.name, + dccommon_consts.RVMC_CONFIG_FILE_NAME) + # SubcloudInstall.prep creates data_install.yml (install overrides) install_command = [ "ansible-playbook", dccommon_consts.ANSIBLE_SUBCLOUD_INSTALL_PLAYBOOK, "-i", ansible_subcloud_inventory_file, "-e", "@%s" % dccommon_consts.ANSIBLE_OVERRIDES_PATH + "/" + - strategy_step.subcloud.name + '/' + "install_values.yml" + strategy_step.subcloud.name + '/' + "install_values.yml", + "-e", "rvmc_config_file=%s" % rvmc_config_file ] # Run the remote install playbook diff --git a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py index 4397ac621..a54435d7c 100644 --- a/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py +++ b/distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py @@ -1762,7 +1762,11 @@ class TestSubcloudManager(base.DCManagerTestCase): '-i', f'{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1_inventory.yml', '--limit', 'subcloud1', '-e', f"@{dccommon_consts.ANSIBLE_OVERRIDES_PATH}/subcloud1/install_values.yml", - '-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION + '-e', "install_release_version=%s" % FAKE_PREVIOUS_SW_VERSION, + '-e', "rvmc_config_file=%s" % + os.path.join(dccommon_consts.ANSIBLE_OVERRIDES_PATH, + 'subcloud1', + dccommon_consts.RVMC_CONFIG_FILE_NAME) ] ) diff --git a/distributedcloud/debian/deb_folder/distributedcloud-dccommon.install b/distributedcloud/debian/deb_folder/distributedcloud-dccommon.install index 48d37200e..f50df16e8 100644 --- a/distributedcloud/debian/deb_folder/distributedcloud-dccommon.install +++ b/distributedcloud/debian/deb_folder/distributedcloud-dccommon.install @@ -2,3 +2,4 @@ etc/logrotate.d/distcloud.conf etc/syslog-ng/conf.d/distcloud.conf usr/lib/python3/dist-packages/dccommon/* usr/lib/python3/dist-packages/distributedcloud-*.egg-info +usr/local/bin/rvmc_install.py \ No newline at end of file diff --git a/distributedcloud/debian/deb_folder/rules b/distributedcloud/debian/deb_folder/rules index d2acaa5a0..244d706a9 100755 --- a/distributedcloud/debian/deb_folder/rules +++ b/distributedcloud/debian/deb_folder/rules @@ -14,6 +14,9 @@ BIN_DIR = $(ROOT)/usr/bin %: dh $@ --with python3 --buildsystem=pybuild +override_dh_usrlocal: + # skip running dh_usrlocal + override_dh_install: python3 setup.py install -f --install-layout=deb \ --root=$(CURDIR)/debian/tmp @@ -77,6 +80,10 @@ override_dh_install: --output-file ./dcdbsync/dcdbsync.conf.sample install -p -D -m 640 ./dcdbsync/dcdbsync.conf.sample $(SYS_CONF_DIR)/dcdbsync/dcdbsync.conf + # install rvmc_install.py script + install -d $(ROOT)/usr/local/bin/ + install -p -D -m 700 scripts/rvmc_install.py $(ROOT)/usr/local/bin + rm -rf $(ROOT)/usr/lib/python3/dist-packages/dcmanagerclient/tests rm -rf $(ROOT)/usr/lib/python3/dist-packages/dccommon/tests rm -rf $(ROOT)/usr/lib/python3/dist-packages/dcmanager/tests diff --git a/distributedcloud/requirements.txt b/distributedcloud/requirements.txt index ffd949902..8bc5cd645 100644 --- a/distributedcloud/requirements.txt +++ b/distributedcloud/requirements.txt @@ -33,10 +33,11 @@ python-neutronclient>=6.3.0 # Apache-2.0 python-cinderclient>=2.1.0 # Apache-2.0 python-novaclient>=7.1.0 # Apache-2.0 python-keystoneclient>=3.8.0 # Apache-2.0 -routes>=2.3.1 # MIT +redfish # BSD requests!=2.12.2,!=2.13.0,>=2.10.0 # Apache-2.0 requests_toolbelt # Apache-2.0 retrying!=1.3.0,>=1.2.3 # Apache-2.0 +routes>=2.3.1 # MIT six>=1.9.0 # MIT sqlalchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT sqlalchemy-migrate>=0.11.0 # Apache-2.0 diff --git a/distributedcloud/scripts/rvmc_install.py b/distributedcloud/scripts/rvmc_install.py new file mode 100644 index 000000000..32b4abc65 --- /dev/null +++ b/distributedcloud/scripts/rvmc_install.py @@ -0,0 +1,311 @@ +#!/usr/bin/python3 +############################################################################### +# +# Copyright (c) 2019-2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Redfish Virtual Media Controller Installer Script""" +# +############################################################################### +# +# This Redfish Virtual Media Controller forces an install of a specified BMC's +# host using the Redfish Platform management protocol. +# +# To do so the following Redfish operations are performed +# +# Step 1: Client Connect ... Establish a client connection to the BMC +# Step 2: Root Query ... Learn Redfish Services offered by the BMC +# Step 3: Find CD/DVD ... Locate the virtual media CD/DVD device +# Step 4: Power Off Host ... Host power needs to be off +# Step 5: Eject Iso ... Eject iso if needed +# Step 6: Inject Iso ... Inject the URL based ISO image into CD/DVD +# Step 7: Force DVD Boot ... Set Net boot device to be CD/DVD +# Step 8: Power On Host ... Host will boot and install from DVD +# +# Note: All server starting state conditions such as the server running or +# being stuck in POST, say at the grub prompt due to previous host boot +# failure, the host needs to be in the powered off state for the ISO +# Insertion and Set DVD for Next Boot steps. +# +# Mandatory Arguments: +# +# --config_file +# +# Config file is assumed to be a yaml file with the following format. +# +# bmc_address: +# bmc_username: +# bmc_password: +# image: http://://bootimage.iso +# +# Real Example: +# +# bmc_address: 123.45.67.89 +# bmc_username: root +# bmc_password: TGk2OW51eCo= +# image: http://[2620:10a:a001:a103::81]:8080/iso/sub_cloud_bootimage.iso +# +# > rvmc_install.py --config_file +# "/opt/dc-vault/ansible/subcloud1/rvmc-config.yaml" +# +# --subcloud_name +# +# The subcloud name is used as the bmc target name, and used for tracking +# the RVMC installation process ID. The process ID file is generated under +# '/var/run/rvmc', eg. '/var/run/rvmc/subcloud1_rvmc.pid'. +# +# > rvmc_install.py --subcloud_name subcloud1 --config_file +# +# Optional Argument: +# +# --debug <0 .. 4> +# +# Note: 0 no debug info (default value) +# 1 = execution stage +# 2 + http request logs +# 3 + headers and payloads and misc other +# 4 + json output of all command responses +# +# > rvmc_install.py --debug 4 --subcloud_name +# --config_file +############################################################################### +# +# Code structure: Note: any error causes error log, session close and exit. +# +# parse command line arguments +# +# create target control object +# +# execute: +# _redfish_client_connect ... connect to bmc +# _redfish_root_query ... get base url tree +# _redfish_create_session ... authenticated session +# _redfish_get_managers ... get managers urls +# _redfish_get_systems_members .. get systems members info +# _redfish_get_vm_url ... get cd/dvd vm url +# _redfish_load_vm_actions ... get eject/insert action urls/info +# _redfish_poweroff_host ... tell bmc to power-off the host +# _redfish_eject_image ... eject current media if present +# _redfish_insert_image ... insert and verify insertion of iso +# _redfish_set_boot_override ... set boot from cd/dvd on next reset +# _redfish_poweron_host ... tell bmc to power-on the host +# +# exit code: +# 0 - Success +# 1 - Retryable failures: +# - Config file not found/opened +# - BNC IP address ping failure +# - Redfish GET query failures (managers, system members, URLs, status, +# virtual media group, etc.) +# - Power command not found +# - Host power on/off failure +# - Virtual media not supported by BMC +# - CD/DVD virtual media type not found +# - Eject target not found +# - VM state query failure +# - Image insertion/ejection timeout +# - Media insertion failure +# - Image verification failure +# - Boot override set/verification failure +# - ... +# 2 - Non-retryable failures: +# - Invalid credentials +# - Script execution time out +# - Failed to terminate the previous process +############################################################################### + +import argparse +import eventlet +import os +import signal +import sys +import time + +from dccommon import rvmc + + +# Constants +# --------- +FEATURE_NAME = 'Redfish Virtual Media Controller' +VERSION_MAJOR = 3 +VERSION_MINOR = 1 + +# The path for RVMC PID file +RVMC_PID_FILE_PATH = '/var/run/rvmc/' +RVMC_PID_FILENAME_POSTFIX = '_rvmc.pid' + +# The signals to be caught for abnormal termination +EXIT_SIGNALS = [signal.SIGTERM, signal.SIGABRT, signal.SIGINT] + +# Global variables +# ------------------ +# The logging utility +logging_util = None +# The exit handler +exit_handler = None + + +def parse_arguments(): + """Parse command line arguments. + + :returns argparse.Namespace: the arguments with name and value + """ + parser = argparse.ArgumentParser(description=FEATURE_NAME) + + parser.add_argument("--debug", type=int, required=False, default=0, + help="Optional debug level ; 0..4") + + parser.add_argument("--subcloud_name", type=str, required=False, + help="Subcloud name") + + parser.add_argument("--config_file", type=str, required=True, + help="RVMC config file") + + return parser.parse_args() + + +def prepare_execution(rvmc_pid_file): + """Terminate the previous RMVC process if it's still running. + + :param rvmc_pid_file: The RVMC PID file + :param type: str + """ + if not rvmc_pid_file: + return + + if not os.path.exists(RVMC_PID_FILE_PATH): + os.makedirs(RVMC_PID_FILE_PATH) + + # Check if the PID file exists. + # Usually, it exists only when the parent process was manually killed. + if os.path.exists(rvmc_pid_file): + with open(rvmc_pid_file, 'r') as pid_file: + pid = pid_file.read() + # Attempt to kill the previous RVMC process using SIGTERM (15) + if pid: + try: + os.kill(int(pid), 15) + except ProcessLookupError: + # Ignore the error if the process with this PID doesn't exit + logging_util.ilog( + "Process %s not found or already terminated." % pid) + except Exception: + logging_util.elog( + "Failed to terminate the previous process %s," % pid) + logging_util.alog( + "Please terminate the previous process %s " + "before running the RVMC script again." % pid) + exit_handler.exit(2) + # Give some time between reading and writing to the same PID file + time.sleep(3) + + # Get the current process ID + current_pid = os.getpid() + + # Write the PID to the file + logging_util.dlog1("Save process ID %d to the file %s." % + (current_pid, rvmc_pid_file)) + with open(rvmc_pid_file, 'w') as pid_file: + pid_file.write(str(current_pid)) + + +def signal_handler(): + """This function handles signals received by the script""" + logging_util.elog("Received exit signal.") + exit_handler.exit(1) + + +class ExitHandler(rvmc.ExitHandler): + """A utility class for handling different exit scenarios in a process. + + Provides methods to manage the process exit in various situations. + """ + def __init__(self, rvmc_pid_file): + """Handler object constructor. + + :param rvmc_pid_file: the RVMC PID file. + :type rvmc_pid_file: str. + """ + self.rvmc_pid_file = rvmc_pid_file + + def exit(self, code): + """Early fault handling. + + :param code: the exit status code + :type code: int. + """ + + if self.rvmc_pid_file and os.path.exists(self.rvmc_pid_file): + os.remove(self.rvmc_pid_file) + sys.stdout.write("\n\n") + sys.exit(code) + + +############################################################################## +# +# Main steps: +# 1. Parse script arguments. +# 2. Register the signal handler. +# 3. Load BMC target info from config file. +# 4. Insert BMC iso for the target through self.execute +# +############################################################################## +if __name__ == "__main__": + args = parse_arguments() + + # get debug level + debug = args.debug + + # get subcloud name + subcloud_name = args.subcloud_name + + # get config file + config_file = args.config_file + + # RVMC PID file + rvmc_pid_file = os.path.join( + RVMC_PID_FILE_PATH, subcloud_name + RVMC_PID_FILENAME_POSTFIX) + + # Set logging utility and exit handler + logging_util = rvmc.LoggingUtil(debug_level=debug) + exit_handler = ExitHandler(rvmc_pid_file) + + logging_util.ilog("%s version %d.%d\n" % + (FEATURE_NAME, VERSION_MAJOR, VERSION_MINOR)) + + # Register the signal handler + for sig in EXIT_SIGNALS: + signal.signal(sig, signal_handler) + + config, target_object = rvmc.parse_config_file( + subcloud_name, config_file, logging_util, exit_handler) + + if target_object: + prepare_execution(rvmc_pid_file) + # TODO(lzhu1): support --timeout option + script_timeout = eventlet.timeout.Timeout( + int(os.environ.get('RVMC_SCRIPT_TIMEOUT', 1800))) + try: + # Load the Iso for the target + logging_util.ilog("BMC Target : %s" % target_object.target) + logging_util.ilog("BMC IP Addr : %s" % target_object.ip) + logging_util.ilog("Host Image : %s" % target_object.img) + target_object.execute() + except eventlet.timeout.Timeout as e: + if e is not script_timeout: + raise + logging_util.elog("RVMC script execution timed out.") + exit_handler.exit(2) + except Exception as e: + logging_util.elog("Got exception: %s" % e) + exit_handler.exit(1) + finally: + script_timeout.cancel() + else: + logging_util.elog("Operation aborted ; no valid bmc information found") + if config_file and config: + logging_util.ilog("Config File :\n%s" % config) + exit_handler.exit(1) + + exit_handler.exit(0) diff --git a/distributedcloud/test-requirements.txt b/distributedcloud/test-requirements.txt index 00a8c6f2e..cf7e82db2 100644 --- a/distributedcloud/test-requirements.txt +++ b/distributedcloud/test-requirements.txt @@ -7,6 +7,7 @@ coverage!=4.4,>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD mock>=2.0 # BSD python-subunit>=0.0.18 # Apache-2.0/BSD +redfish # BSD requests-mock>=1.1 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT diff --git a/python/python3-redfish/debian/deb_folder/changelog b/python/python3-redfish/debian/deb_folder/changelog new file mode 100644 index 000000000..bedb227fd --- /dev/null +++ b/python/python3-redfish/debian/deb_folder/changelog @@ -0,0 +1,5 @@ +redfish (3.2.1) unstable; urgency=medium + + * Initial release. + + -- Li Zhu Fri, 20 Oct 2023 10:14:40 -0300 \ No newline at end of file diff --git a/python/python3-redfish/debian/deb_folder/control b/python/python3-redfish/debian/deb_folder/control new file mode 100644 index 000000000..3c9636382 --- /dev/null +++ b/python/python3-redfish/debian/deb_folder/control @@ -0,0 +1,18 @@ +Source: redfish +Section: admin +Priority: optional +Maintainer: StarlingX Developers +Build-Depends: debhelper-compat (= 13), + dh-python, + flake8, + python3-setuptools, + python3-all, + python3-jsonpath-rw, + python3-requests-unixsocket +Standards-Version: 4.5.1 + +Package: python3-redfish +Architecture: all +Depends: ${python3:Depends}, ${misc:Depends} +Description: Python library for Redfish operations + This package provides the Redfish Python library. diff --git a/python/python3-redfish/debian/deb_folder/copyright b/python/python3-redfish/debian/deb_folder/copyright new file mode 100644 index 000000000..3acb139f1 --- /dev/null +++ b/python/python3-redfish/debian/deb_folder/copyright @@ -0,0 +1,48 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: python3-redfish +Source: https://opendev.org/starlingx/distcloud/python/python3-redfish + +Files: * +Copyright: Copyright 2016-2022 DMTF. All rights reserved. +License: BSD-3-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Distributed Management Task Force (DMTF) nor the names + of its contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +Files: debian/* +Copyright: (c) 2023 Wind River Systems, Inc +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + https://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian-based systems the full text of the Apache version 2.0 license + can be found in `/usr/share/common-licenses/Apache-2.0'. diff --git a/python/python3-redfish/debian/deb_folder/rules b/python/python3-redfish/debian/deb_folder/rules new file mode 100755 index 000000000..c63c15110 --- /dev/null +++ b/python/python3-redfish/debian/deb_folder/rules @@ -0,0 +1,17 @@ +#!/usr/bin/make -f + +export DH_VERBOSE=1 +export PYBUILD_NAME=python3-redfish + +%: + dh $@ --with python3 --buildsystem=pybuild + +override_dh_auto_clean: + python3 setup.py clean -a + find . -name \*.pyc -exec rm {} \; + +override_dh_auto_build: + python3 setup.py build --force + +override_dh_auto_install: + python3 setup.py install --force --root=debian/python3-redfish --no-compile -O0 --install-layout=deb diff --git a/python/python3-redfish/debian/meta_data.yaml b/python/python3-redfish/debian/meta_data.yaml new file mode 100644 index 000000000..b1f8727ed --- /dev/null +++ b/python/python3-redfish/debian/meta_data.yaml @@ -0,0 +1,14 @@ +--- +debname: redfish +debver: 3.2.1 +dl_path: + name: redfish-3.2.1.tar.gz + url: > + https://files.pythonhosted.org/packages/ea/04/ + 0e08ec3ad8af6de811edc864c7f3c505e22af14a7da735c45c512dfe9e9c/ + redfish-3.2.1.tar.gz + md5sum: 8203bb777f2559e65302e988eee9f1cc + sha256sum: c10b11ff5a4e6cab0888b577c51facae689f4c6002b7ce34b1741707290aca78 +revision: + dist: $STX_DIST + PKG_GITREVCOUNT: true diff --git a/tox.ini b/tox.ini index 5fbf3f8c7..91ab251fd 100644 --- a/tox.ini +++ b/tox.ini @@ -51,10 +51,19 @@ commands = rm -rf api-ref/build sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html +[bandit] +# The following bandit tests are being skipped: +# B605: Test for starting a process with a shell +# +# Note: 'skips' entry cannot be split across multiple lines +# +skips = B605 +exclude = tests + [testenv:bandit] description = Bandit code scan for *.py files under config folder deps = bandit -commands = bandit -r {toxinidir}/ -x '**/.tox/**,**/.eggs/**' -lll +commands = bandit --ini tox.ini -r {toxinidir}/ -x '**/.tox/**,**/.eggs/**' -lll [testenv:linters] allowlist_externals = bash