# # Copyright (c) 2019, 2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # import ipaddress import re import os import time import yaml from pytest import skip from consts.auth import Tenant, HostLinuxUser from consts.stx import UUID, Prompt, SysType, EventLogID, HostAvailState from consts.proj_vars import ProjVar from consts.timeout import SysInvTimeout, MiscTimeout, HostTimeout from utils import cli, table_parser, exceptions from utils.clients.ssh import ControllerClient from utils.tis_log import LOG from testfixtures.fixture_resources import ResourceCleanup from keywords import common def get_sys_type(con_ssh=None): """ Please do NOT call this function in testcase/keyword. This is used to set global variable SYS_TYPE in ProjVar. Use ProjVar.get_var('SYS_TYPE') in testcase/keyword instead. Args: con_ssh: Returns: """ auth_info = Tenant.get('admin_platform') is_aio = is_aio_system(controller_ssh=con_ssh, auth_info=auth_info) if is_aio: sys_type = SysType.AIO_DX if len(get_controllers(con_ssh=con_ssh, auth_info=auth_info)) == 1: sys_type = SysType.AIO_SX elif get_storage_nodes(con_ssh=con_ssh): sys_type = SysType.STORAGE else: sys_type = SysType.REGULAR LOG.info("============= System type: {} ==============".format(sys_type)) return sys_type def is_storage_system(con_ssh=None, auth_info=Tenant.get('admin_platform')): sys_type = ProjVar.get_var('SYS_TYPE') if sys_type: if not (ProjVar.get_var('IS_DC') and auth_info and ProjVar.get_var('PRIMARY_SUBCLOUD') != auth_info.get('region')): return SysType.STORAGE == sys_type else: return bool(get_storage_nodes(con_ssh=con_ssh, auth_info=auth_info)) def is_aio_duplex(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Whether it is two node CPE system Args: con_ssh: auth_info Returns (bool): """ sys_type = ProjVar.get_var('SYS_TYPE') if sys_type: if not (ProjVar.get_var('IS_DC') and auth_info and ProjVar.get_var('PRIMARY_SUBCLOUD') != auth_info.get('region', None)): return SysType.AIO_DX == sys_type else: return is_aio_system(controller_ssh=con_ssh) \ and len(get_controllers(con_ssh=con_ssh)) == 2 def is_aio_simplex(con_ssh=None, auth_info=Tenant.get('admin_platform')): sys_type = ProjVar.get_var('SYS_TYPE') if sys_type: if not (con_ssh and ProjVar.get_var('IS_DC') and auth_info and ProjVar.get_var('PRIMARY_SUBCLOUD') != auth_info.get('region', None)): return SysType.AIO_SX == sys_type return is_aio_system(controller_ssh=con_ssh, auth_info=auth_info) and \ len(get_controllers(con_ssh=con_ssh, auth_info=auth_info)) == 1 def is_aio_system(controller_ssh=None, controller='controller-0', auth_info=Tenant.get('admin_platform')): """ Whether it is AIO-Duplex or AIO-Simplex system where controller has both controller and compute functions Args: controller_ssh (SSHClient): controller (str): controller to check auth_info Returns (bool): True if CPE or Simplex, else False """ sys_type = ProjVar.get_var('SYS_TYPE') if sys_type: if not (controller_ssh and ProjVar.get_var('IS_DC') and auth_info and ProjVar.get_var('PRIMARY_SUBCLOUD') != auth_info.get('region', None)): return 'aio' in sys_type.lower() subfunc = get_host_values(host=controller, fields='subfunctions', con_ssh=controller_ssh, auth_info=auth_info)[0] combined = 'controller' in subfunc and re.search('compute|worker', subfunc) str_ = 'not ' if not combined else '' LOG.info("This is {}small footprint system.".format(str_)) return combined def get_storage_nodes(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get hostnames with 'storage' personality from system host-list Args: con_ssh (SSHClient): auth_info Returns (list): list of hostnames. Empty list [] returns when no storage nodes. """ return get_hosts(personality='storage', con_ssh=con_ssh, auth_info=auth_info) def get_controllers(administrative=None, operational=None, availability=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get hostnames with 'controller' personality from system host-list Args: administrative operational availability con_ssh (SSHClient): auth_info Returns (list): list of hostnames """ return get_hosts(personality='controller', administrative=administrative, operational=operational, availability=availability, con_ssh=con_ssh, auth_info=auth_info) def get_computes(administrative=None, operational=None, availability=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get hostnames with 'compute' personality from system host-list Args: administrative operational availability con_ssh (SSHClient): auth_info Returns (list): list of hostnames. Empty list [] returns when no compute nodes. """ return get_hosts(personality='compute', administrative=administrative, operational=operational, availability=availability, con_ssh=con_ssh, auth_info=auth_info) def get_hypervisors(administrative=None, operational=None, availability=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get nodes that can be used as hypervisor/worker. e.g., in standard config, it will mean worker nodes. In DX+worker config, it will mean worker nodes and controller nodes. Args: administrative: operational: availability: con_ssh: auth_info: Returns (list): """ computes = get_computes(administrative=administrative, operational=operational, availability=availability, con_ssh=con_ssh, auth_info=auth_info) if is_aio_system(controller_ssh=con_ssh, auth_info=auth_info): computes += get_controllers(administrative=administrative, operational=operational, availability=availability, con_ssh=con_ssh, auth_info=auth_info) return computes def get_hosts(personality=None, administrative=None, operational=None, availability=None, hostname=None, strict=True, exclude=False, con_ssh=None, auth_info=Tenant.get('admin_platform'), field='hostname', rtn_dict=False): """ Get hostnames with given criteria Args: personality (None|str|tuple|list): administrative (None|str|list|tuple): operational (None|str|list|tuple): availability (None|str|list|tuple): hostname (None|tuple|list|str): filter out these hosts only strict (bool): exclude (bool): con_ssh (SSHClient|None): auth_info field (str|list|tuple) rtn_dict (bool): Whether to return dict where each field is a key, and value is a list Returns (list): hosts """ if not con_ssh: con_name = auth_info.get('region') if ( auth_info and ProjVar.get_var('IS_DC')) else None con_ssh = ControllerClient.get_active_controller(name=con_name) table_ = table_parser.table( cli.system('host-list', ssh_client=con_ssh, auth_info=auth_info)[1]) table_ = table_parser.filter_table(table_, exclude=True, hostname='None') if hostname: table_ = table_parser.filter_table(table_, hostname=hostname) if personality: compute_personality = 'compute|worker' if personality == 'compute': personality = compute_personality elif not isinstance(personality, str): personality = list(personality) if 'compute' in personality: compute_index = personality.index('compute') personality[compute_index] = compute_personality filters = {'personality': personality, 'administrative': administrative, 'operational': operational, 'availability': availability} filters = {k: v for k, v in filters.items() if v is not None} if filters: table_ = table_parser.filter_table(table_, strict=strict, exclude=exclude, regex=True, **filters) hostnames = table_parser.get_multi_values(table_, field, rtn_dict=rtn_dict) LOG.debug("Filtered hosts: {}".format(hostnames)) return hostnames def get_host_list_data(columns=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), source_rc=False): """ Args: columns con_ssh auth_info source_rc Returns (list of dict of hosts): e.g., [{'administrative': 'unlocked', 'availability': 'available', 'hostname': 'controller-0', 'id': 1, 'operational': 'enabled', 'personality': 'controller'}, {'administrative': 'unlocked', 'availability': 'available', 'hostname': 'compute-1', 'id': 2, 'operational': 'enabled', 'personality': 'worker'}, {'administrative': 'unlocked', 'availability': 'available', 'hostname': 'compute-0', 'id': 3, 'operational': 'enabled', 'personality': 'worker'}, {'administrative': 'unlocked', 'availability': 'available', 'hostname': 'controller-1', 'id': 4, 'operational': 'enabled', 'personality': 'controller'}, ] """ args = "" if columns: for col in columns: args += ' --column {}'.format(col) args += " --format yaml" code, output = cli.system('host-list', args, ssh_client=con_ssh, auth_info=auth_info, source_openrc=source_rc) if code == 0: return yaml.safe_load(output) else: LOG.error("Error with CLI command") return output def get_hosts_per_personality(availability=None, administrative=None, operational=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), source_rc=False, rtn_tuple=False): """ Args: availability administrative operational con_ssh: auth_info source_rc rtn_tuple (bool): whether to return tuple instead of dict. i.e., , , Returns (dict|tuple): e.g., {'controller': ['controller-0', 'controller-1'], 'compute': [ 'compute-0', 'compute-1], 'storage': []} """ table_ = table_parser.table( cli.system('host-list', ssh_client=con_ssh, auth_info=auth_info, source_openrc=source_rc)[1]) personalities = ('controller', 'compute', 'storage') res = {} for personality in personalities: personality_tmp = 'compute|worker' if personality == 'compute' else \ personality hosts = table_parser.get_values(table_, 'hostname', personality=personality_tmp, availability=availability, administrative=administrative, operational=operational, regex=True) hosts = [host for host in hosts if host.lower() != 'none'] res[personality] = hosts if rtn_tuple: res = res['controller'], res['compute'], res['storage'] return res def get_active_controller_name(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ This assumes system has 1 active controller Args: con_ssh: auth_info Returns: hostname of the active controller Further info such as ip, uuid can be obtained via System.CONTROLLERS[ hostname]['uuid'] """ return get_active_standby_controllers(con_ssh=con_ssh, auth_info=auth_info)[ 0] def get_standby_controller_name(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ This assumes system has 1 standby controller Args: con_ssh: auth_info Returns (str): hostname of the active controller Further info such as ip, uuid can be obtained via System.CONTROLLERS[ hostname]['uuid'] """ active, standby = get_active_standby_controllers(con_ssh=con_ssh, auth_info=auth_info) return standby if standby else '' def get_active_standby_controllers(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get active controller name and standby controller name (if any) Args: con_ssh (SSHClient): auth_info Returns (tuple): such as ('controller-0', 'controller-1'), when non-active controller is in bad state or degraded state, or any scenarios where standby controller does not exist, this function will return (, None) """ table_ = table_parser.table( cli.system('servicegroup-list', ssh_client=con_ssh, auth_info=auth_info)[1]) table_ = table_parser.filter_table(table_, service_group_name='controller-services') active_con = table_parser.get_values(table_, 'hostname', state='active', strict=False)[0] standby_con = table_parser.get_values(table_, 'hostname', state='standby', strict=False) standby_con = standby_con[0] if standby_con else None return active_con, standby_con def get_alarms_table(uuid=True, show_suppress=False, query_key=None, query_value=None, query_type=None, con_ssh=None, mgmt_affecting=None, auth_info=Tenant.get('admin_platform'), retry=0): """ Get active alarms_and_events dictionary with given criteria Args: uuid (bool): whether to show uuid show_suppress (bool): whether to show suppressed alarms_and_events query_key (str): one of these: 'event_log_id', 'entity_instance_id', 'uuid', 'severity', query_value (str): expected value for given key query_type (str): data type of value. one of these: 'string', 'integer', 'float', 'boolean' mgmt_affecting (bool) con_ssh (SSHClient): auth_info (dict): retry (None|int): number of times to retry if the alarm-list cli got rejected Returns: dict: events table in format: {'headers': , 'values': } """ args = '--nowrap' args = __process_query_args(args, query_key, query_value, query_type) if uuid: args += ' --uuid' if show_suppress: args += ' --include_suppress' if mgmt_affecting: args += ' --mgmt_affecting' fail_ok = True if not retry: fail_ok = False retry = 0 output = None for i in range(retry + 1): code, output = cli.fm('alarm-list', args, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code == 0: table_ = table_parser.table(output, combine_multiline_entry=True) return table_ if i < retry: time.sleep(5) else: raise exceptions.CLIRejected( 'fm alarm-list cli got rejected after {} retries: {}'.format( retry, output)) def get_alarms(fields=('Alarm ID', 'Entity ID'), alarm_id=None, reason_text=None, entity_id=None, severity=None, time_stamp=None, strict=False, show_suppress=False, query_key=None, query_value=None, query_type=None, mgmt_affecting=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), combine_entries=True): """ Get a list of alarms with values for specified fields. Args: fields (tuple): fields to get values for alarm_id (str): filter out the table using given alarm id ( strict=True). if None, table will not be filtered. reason_text (str): reason text to filter out the table (strict defined in param) entity_id (str): entity instance id to filter out the table (strict defined in param) severity (str): severity such as 'critical', 'major' time_stamp (str): strict (bool): whether to perform strict filter on reason text, entity_id, severity, or time_stamp show_suppress (bool): whether to show suppressed alarms. Default to False. query_key (str): key in --query = passed to fm alarm-list query_value (str): value in --query = passed to fm alarm-list query_type (str): 'string', 'integer', 'float', or 'boolean' mgmt_affecting (bool) con_ssh (SSHClient): auth_info (dict): combine_entries (bool): return list of strings when set to True, else return a list of tuples. e.g., when True, returns ["800.003::::cluster=829851fa", "250.001::::host=controller-0"] when False, returns [("800.003", "cluster=829851fa"), ("250.001", "host=controller-0")] Returns (list): list of alarms with values of specified fields """ table_ = get_alarms_table(show_suppress=show_suppress, query_key=query_key, query_value=query_value, query_type=query_type, con_ssh=con_ssh, auth_info=auth_info, mgmt_affecting=mgmt_affecting) if alarm_id: table_ = table_parser.filter_table(table_, **{'Alarm ID': alarm_id}) kwargs_dict = { 'Reason Text': reason_text, 'Entity ID': entity_id, 'Severity': severity, 'Time Stamp': time_stamp } kwargs = {} for key, value in kwargs_dict.items(): if value is not None: kwargs[key] = value if kwargs: table_ = table_parser.filter_table(table_, strict=strict, **kwargs) rtn_vals_list = [] for field in fields: vals = table_parser.get_column(table_, field) rtn_vals_list.append(vals) rtn_vals_list = zip(*rtn_vals_list) if combine_entries: rtn_vals_list = ['::::'.join(vals) for vals in rtn_vals_list] else: rtn_vals_list = list(rtn_vals_list) return rtn_vals_list def get_suppressed_alarms(uuid=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get suppressed alarms_and_events as dictionary Args: uuid (bool): whether to show uuid con_ssh (SSHClient): auth_info (dict): Returns: dict: events table in format: {'headers': , 'values': } """ args = '' if uuid: args += ' --uuid' args += ' --nowrap --nopaging' table_ = table_parser.table( cli.fm('event-suppress-list', args, ssh_client=con_ssh, auth_info=auth_info)[1]) return table_ def unsuppress_all_events(ssh_con=None, fail_ok=False, auth_info=Tenant.get('admin_platform')): """ Args: ssh_con: fail_ok: auth_info: Returns (tuple): ((int), (str)) """ LOG.info("Un-suppress all events") args = '--nowrap --nopaging' code, output = cli.fm('event-unsuppress-all', positional_args=args, ssh_client=ssh_con, fail_ok=fail_ok, auth_info=auth_info) if code == 1: return 1, output if not output: msg = "No suppressed events to un-suppress" LOG.warning(msg) return -1, msg table_ = table_parser.table(output) if not table_['values']: suppressed_list = [] else: suppressed_list = table_parser.get_values(table_, target_header="Suppressed " "Alarm ID's", **{'Status': 'suppressed'}) if suppressed_list: msg = "Unsuppress-all failed. Suppressed Alarm IDs: {}".format( suppressed_list) if fail_ok: LOG.warning(msg) return 2, msg raise exceptions.NeutronError(msg) succ_msg = "All events unsuppressed successfully." LOG.info(succ_msg) return 0, succ_msg def get_events(fields=('Event Log ID', 'Entity Instance ID'), limit=10, event_id=None, entity_id=None, severity=None, show_suppress=False, start=None, end=None, state=None, show_uuid=True, strict=False, time_stamp=None, reason_text=None, uuid=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), combine_entries=True): """ Get a list of alarms with values for specified fields. Args: fields (tuple|list|str): fields to get values for limit (int) event_id (str): filter event using event log id reason_text (str): reason text to filter out the table (strict defined in param) entity_id (str): entity instance id to filter out the table (strict defined in param) severity (str): severity such as 'critical', 'major' show_suppress (bool): whether to show suppressed events. Default to False. show_uuid (bool): Whether to show uuid in event table start (str): display events after this time stamp end (str): display events prior to this time stamp state (str): filter with events state time_stamp (str): exact timestamp for the event, filter after events displayed uuid (str) strict (bool): whether to perform strict filter on reason text, or time_stamp con_ssh (SSHClient): auth_info (dict): combine_entries (bool): return list of strings when set to True, else return a list of tuples. e.g., when True, returns ["800.003::::cluster=829851fa", "250.001::::host=controller-0"] when False, returns [("800.003", "cluster=829851fa"), ("250.001", "host=controller-0")] Returns (list): list of events with values of specified fields """ table_ = get_events_table(show_uuid=show_uuid, limit=limit, event_log_id=event_id, entity_instance_id=entity_id, show_suppress=show_suppress, con_ssh=con_ssh, auth_info=auth_info, start=start, end=end, severity=severity) kwargs_dict = { 'Reason Text': reason_text, 'Time Stamp': time_stamp, 'UUID': uuid, 'State': state, } kwargs = {} for key, value in kwargs_dict.items(): if value is not None: kwargs[key] = value if kwargs: table_ = table_parser.filter_table(table_, strict=strict, **kwargs) rtn_vals_list = [] if isinstance(fields, str): fields = (fields,) for header in fields: vals = table_parser.get_column(table_, header) if not vals: vals = [] rtn_vals_list.append(vals) LOG.warning('{}'.format(rtn_vals_list)) rtn_vals_list = list(zip(*rtn_vals_list)) if combine_entries: rtn_vals_list = ['::::'.join(vals) for vals in rtn_vals_list] return rtn_vals_list def get_events_table(limit=5, show_uuid=False, show_only=None, show_suppress=False, event_log_id=None, entity_type_id=None, entity_instance_id=None, severity=None, start=None, end=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), regex=False, **kwargs): """ Get a list of events with given criteria as dictionary Args: limit (int): max number of event logs to return show_uuid (bool): whether to show uuid show_only (str): 'alarms_and_events' or 'logs' to return only alarms_and_events or logs show_suppress (bool): whether or not to show suppressed alarms_and_events event_log_id (str|None): event log id passed to system eventlog -q event_log_id= entity_type_id (str|None): entity_type_id passed to system eventlog -q entity_type_id= entity_instance_id (str|None): entity_instance_id passed to system eventlog -q entity_instance_id= severity (str|None): start (str|None): start date/time passed to '--query' in format "20170410"/"20170410 01:23:34" end (str|None): end date/time passed to '--query' in format "20170410"/"20170410 01:23:34" con_ssh (SSHClient): auth_info (dict): regex (bool): **kwargs: filter table after table returned Returns: dict: events table in format: {'headers': , 'values': } """ args = '-l {}'.format(limit) # args = __process_query_args(args, query_key, query_value, query_type) query_dict = { 'event_log_id': event_log_id, 'entity_type_id': entity_type_id, 'entity_instance_id': entity_instance_id, 'severity': severity, 'start': '{}'.format(start) if start else None, 'end': '{}'.format(end) if end else None } queries = [] for q_key, q_val in query_dict.items(): if q_val is not None: queries.append('{}={}'.format(q_key, str(q_val))) query_string = ';'.join(queries) if query_string: args += " -q '{}'".format(query_string) args += ' --nowrap --nopaging' if show_uuid: args += ' --uuid' if show_only: args += ' --{}'.format(show_only.lower()) if show_suppress: args += ' --include_suppress' table_ = table_parser.table( cli.fm('event-list ', args, ssh_client=con_ssh, auth_info=auth_info)[1]) if kwargs: table_ = table_parser.filter_table(table_, regex=regex, **kwargs) return table_ def _compose_events_table(output, uuid=False): if not output['headers']: headers = ['UUID', 'Time Stamp', 'State', 'Event Log ID', 'Reason Text', 'Entity Instance ID', 'Severity'] if not uuid: headers.remove('UUID') values = [] output['headers'] = headers output['values'] = values return output def __process_query_args(args, query_key, query_value, query_type): if query_key: if not query_value: raise ValueError( "Query value is not supplied for key - {}".format(query_key)) data_type_arg = '' if not query_type else "{}::".format( query_type.lower()) args += ' -q {}={}"{}"'.format(query_key.lower(), data_type_arg, query_value.lower()) return args def wait_for_events(timeout=60, num=30, uuid=False, show_only=None, event_log_id=None, entity_type_id=None, entity_instance_id=None, severity=None, start=None, end=None, field='Event Log ID', regex=False, strict=True, check_interval=5, fail_ok=True, con_ssh=None, auth_info=Tenant.get('admin_platform'), **kwargs): """ Wait for event(s) to appear in fm event-list Args: timeout (int): max time to wait in seconds num (int): max number of event logs to return uuid (bool): whether to show uuid show_only (str): 'alarms_and_events' or 'logs' to return only alarms_and_events or logs fail_ok (bool): whether to return False if event(s) did not appear within timeout field (str): list of values to return. Defaults to 'Event Log ID' con_ssh (SSHClient): auth_info (dict): regex (bool): Whether to use regex or string operation to search/match the value in kwargs strict (bool): whether it's a strict match (case is always ignored regardless of this flag) check_interval (int): how often to check the event logs event_log_id (str|None): event log id passed to system eventlog -q event_log_id= entity_type_id (str|None): entity_type_id passed to system eventlog -q entity_type_id= entity_instance_id (str|None): entity_instance_id passed to system eventlog -q entity_instance_id= severity (str|None): start (str|None): start date/time passed to '--query' in format "20170410"/"20170410 01:23:34" end (str|None): end date/time passed to '--query' in format "20170410"/"20170410 01:23:34" **kwargs: criteria to filter out event(s) from the events list table Returns: list: list of event log ids (or whatever specified in rtn_value) for matching events. """ end_time = time.time() + timeout while time.time() < end_time: events_tab = get_events_table(limit=num, show_uuid=uuid, show_only=show_only, event_log_id=event_log_id, entity_type_id=entity_type_id, entity_instance_id=entity_instance_id, severity=severity, start=start, end=end, con_ssh=con_ssh, auth_info=auth_info) events_tab = table_parser.filter_table(events_tab, strict=strict, regex=regex, **kwargs) events = table_parser.get_column(events_tab, field) if events: LOG.info("Event(s) appeared in event-list: {}".format(events)) return events time.sleep(check_interval) msg = "Event(s) did not appear in fm event-list within timeout." if fail_ok: LOG.warning(msg) return [] else: raise exceptions.TimeoutException(msg) def delete_alarms(alarms=None, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Delete active alarms_and_events Args: alarms (list|str): UUID(s) of alarms_and_events to delete fail_ok (bool): whether or not to raise exception if any alarm failed to delete con_ssh (SSHClient): auth_info (dict): Returns (tuple): (rtn_code(int), message(str)) 0, "Alarms deleted successfully" 1, "Some alarm(s) still exist on system after attempt to delete: " """ if alarms is None: alarms_tab = get_alarms_table(uuid=True) alarms = [] if alarms_tab['headers']: alarms = table_parser.get_column(alarms_tab, 'UUID') if isinstance(alarms, str): alarms = [alarms] LOG.info("Deleting following alarms_and_events: {}".format(alarms)) res = {} failed_clis = [] for alarm in alarms: code, out = cli.fm('alarm-delete', alarm, ssh_client=con_ssh, auth_info=auth_info) res[alarm] = code, out if code != 0: failed_clis.append(alarm) post_alarms_tab = get_alarms_table(uuid=True) if post_alarms_tab['headers']: post_alarms = table_parser.get_column(post_alarms_tab, 'UUID') else: post_alarms = [] undeleted_alarms = list(set(alarms) & set(post_alarms)) if undeleted_alarms: err_msg = "Some alarm(s) still exist on system after attempt to " \ "delete: {}\nAlarm delete results: {}". \ format(undeleted_alarms, res) if fail_ok: return 1, err_msg raise exceptions.SysinvError(err_msg) elif failed_clis: LOG.warning( "Some alarm-delete cli(s) rejected, but alarm no longer " "exists.\nAlarm delete results: {}". format(res)) succ_msg = "Alarms deleted successfully" LOG.info(succ_msg) return 0, succ_msg def wait_for_alarm_gone(alarm_id, entity_id=None, reason_text=None, strict=False, timeout=120, check_interval=10, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for given alarm to disappear from fm alarm-list Args: alarm_id (str): such as 200.009 entity_id (str): entity instance id for the alarm (strict as defined in param) reason_text (str): reason text for the alarm (strict as defined in param) strict (bool): whether to perform strict string match on entity instance id and reason timeout (int): max seconds to wait for alarm to disappear check_interval (int): how frequent to check fail_ok (bool): whether to raise exception if alarm did not disappear within timeout con_ssh (SSHClient): auth_info (dict): Returns (bool): True if alarm is gone else False """ LOG.info( "Waiting for alarm {} to disappear from fm alarm-list".format(alarm_id)) build_ver = get_sw_version(con_ssh=con_ssh) alarmcmd = 'alarm-list' if build_ver != '15.12': alarmcmd += ' --nowrap' end_time = time.time() + timeout while time.time() < end_time: alarms_tab = table_parser.table( cli.fm(alarmcmd, ssh_client=con_ssh, auth_info=auth_info)[1]) alarm_tab = table_parser.filter_table(alarms_tab, **{'Alarm ID': alarm_id}) if table_parser.get_all_rows(alarm_tab): kwargs = {} if entity_id: kwargs['Entity ID'] = entity_id if reason_text: kwargs['Reason Text'] = reason_text if kwargs: alarms = table_parser.get_values(alarm_tab, target_header='Alarm ID', strict=strict, **kwargs) if not alarms: LOG.info( "Alarm {} with {} is not displayed in fm " "alarm-list".format( alarm_id, kwargs)) return True else: LOG.info( "Alarm {} is not displayed in fm alarm-list".format(alarm_id)) return True time.sleep(check_interval) else: err_msg = "Timed out waiting for alarm {} to disappear".format(alarm_id) if fail_ok: LOG.warning(err_msg) return False else: raise exceptions.TimeoutException(err_msg) def _get_alarms(alarms_tab): alarm_ids = table_parser.get_column(alarms_tab, 'Alarm_ID') entity_ids = table_parser.get_column(alarms_tab, 'Entity ID') alarms = list(zip(alarm_ids, entity_ids)) return alarms def wait_for_alarm(field='Alarm ID', alarm_id=None, entity_id=None, reason=None, severity=None, timeout=60, check_interval=3, regex=False, strict=False, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for given alarm to appear Args: field: alarm_id (str): such as 200.009 entity_id (str|list|tuple): entity instance id for the alarm (strict as defined in param) reason (str): reason text for the alarm (strict as defined in param) severity (str): severity of the alarm to wait for timeout (int): max seconds to wait for alarm to appear check_interval (int): how frequent to check regex (bool): whether to use regex when matching entity instance id and reason strict (bool): whether to perform strict match on entity instance id and reason fail_ok (bool): whether to raise exception if alarm did not disappear within timeout con_ssh (SSHClient): auth_info (dict): Returns (tuple): (, ). Such as (True, '200.009') or ( False, None) """ kwargs = {} if alarm_id: kwargs['Alarm ID'] = alarm_id if reason: kwargs['Reason Text'] = reason if severity: kwargs['Severity'] = severity if entity_id and isinstance(entity_id, str): entity_id = [entity_id] end_time = time.time() + timeout while time.time() < end_time: current_alarms_tab = get_alarms_table(con_ssh=con_ssh, auth_info=auth_info) if kwargs: current_alarms_tab = table_parser.filter_table( table_=current_alarms_tab, strict=strict, regex=regex, **kwargs) if entity_id: val = [] for entity in entity_id: entity_filter = {'Entity ID': entity} val_ = table_parser.get_values(current_alarms_tab, field, strict=strict, regex=regex, **entity_filter) if not val_: LOG.info( "Alarm for entity {} has not appeared".format(entity)) time.sleep(check_interval) continue val += val_ else: val = table_parser.get_values(current_alarms_tab, field) if val: LOG.info('Expected alarm appeared. Filters: {}'.format(kwargs)) return True, val time.sleep(check_interval) entity_str = ' for entity {}'.format(entity_id) if entity_id else '' err_msg = "Alarm {}{} did not appear in fm alarm-list within {} " \ "seconds".format(kwargs, entity_str, timeout) if fail_ok: LOG.warning(err_msg) return False, None raise exceptions.TimeoutException(err_msg) def wait_for_alarms_gone(alarms, timeout=120, check_interval=3, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for given alarms_and_events to be gone from fm alarm-list Args: alarms (list): list of tuple. [(, ), ...] timeout (int): check_interval (int): fail_ok (bool): con_ssh (SSHClient): auth_info (dict): Returns (tuple): (res(bool), remaining_alarms(list of tuple)) """ pre_alarms = list(alarms) # Don't update the original list LOG.info( "Waiting for alarms_and_events to disappear from fm alarm-list: " "{}".format(pre_alarms)) alarms_to_check = pre_alarms.copy() alarms_cleared = [] def _update_alarms(alarms_to_check_, alarms_cleared_): current_alarms_tab = get_alarms_table(con_ssh=con_ssh, auth_info=auth_info) current_alarms = _get_alarms(current_alarms_tab) for alarm in pre_alarms: if alarm not in current_alarms: LOG.info( "Removing alarm {} from current alarms_and_events list: " "{}".format(alarm, alarms_to_check)) alarms_to_check_.remove(alarm) alarms_cleared_.append(alarm) _update_alarms(alarms_to_check_=alarms_to_check, alarms_cleared_=alarms_cleared) if not alarms_to_check: LOG.info( "Following alarms_and_events cleared: {}".format(alarms_cleared)) return True, [] end_time = time.time() + timeout while time.time() < end_time: pre_alarms = alarms_to_check.copy() time.sleep(check_interval) _update_alarms(alarms_to_check_=alarms_to_check, alarms_cleared_=alarms_cleared) if not alarms_to_check: LOG.info("Following alarms_and_events cleared: {}".format( alarms_cleared)) return True, [] else: err_msg = "Following alarms_and_events did not clear within {} " \ "seconds: {}".format(timeout, alarms_to_check) if fail_ok: LOG.warning(err_msg) return False, alarms_to_check else: raise exceptions.TimeoutException(err_msg) def wait_for_all_alarms_gone(timeout=120, check_interval=3, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for all alarms_and_events to be cleared from fm alarm-list Args: timeout (int): check_interval (int): fail_ok (bool): con_ssh (SSHClient): auth_info (dict): Returns (tuple): (res(bool), remaining_alarms(tuple)) """ LOG.info( "Waiting for all existing alarms_and_events to disappear from fm " "alarm-list: {}".format( get_alarms())) end_time = time.time() + timeout while time.time() < end_time: current_alarms_tab = get_alarms_table(con_ssh=con_ssh, auth_info=auth_info) current_alarms = _get_alarms(current_alarms_tab) if len(current_alarms) == 0: return True, [] else: time.sleep(check_interval) else: existing_alarms = get_alarms() err_msg = "Alarms did not clear within {} seconds: {}".format( timeout, existing_alarms) if fail_ok: LOG.warning(err_msg) return False, existing_alarms else: raise exceptions.TimeoutException(err_msg) def host_exists(host, field='hostname', con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Args: host: field: con_ssh: auth_info Returns (bool): whether given host exists in system host-list """ if not field.lower() in ['hostname', 'id']: raise ValueError("field has to be either \'hostname\' or \'id\'") hosts = get_hosts(con_ssh=con_ssh, auth_info=auth_info, field=field) return host in hosts def modify_system(fail_ok=True, con_ssh=None, auth_info=Tenant.get('admin_platform'), **kwargs): """ Modify the System configs/info. Args: fail_ok (bool): con_ssh (SSHClient): auth_info (dict): **kwargs: attribute-value pairs Returns: (int, str) 0 - success 1 - error Test Steps: - Set the value via system modify = [,= action=install - anystr system dns-modify <> action=anystring... Returns (tuple): (-1, ) (0, ) (1, ) """ if not nameservers: raise ValueError("Please specify DNS server(s).") if check_first: dns_servers = get_dns_servers(con_ssh=con_ssh, auth_info=auth_info) if dns_servers == nameservers and with_action_option is None: msg = 'DNS servers already set to {}. Do nothing.'.format( dns_servers) LOG.info(msg) return -1, dns_servers args_ = 'nameservers="{}"'.format(','.join(nameservers)) if with_action_option is not None: args_ += ' action={}'.format(with_action_option) LOG.info('args_:{}'.format(args_)) code, output = cli.system('dns-modify', args_, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info, timeout=SysInvTimeout.DNS_MODIFY) if code == 1: return 1, output post_dns_servers = get_dns_servers(auth_info=auth_info, con_ssh=con_ssh) if post_dns_servers != nameservers: raise exceptions.SysinvError( 'dns servers expected: {}; actual: {}'.format(nameservers, post_dns_servers)) LOG.info("DNS servers successfully updated to: {}".format(nameservers)) return 0, nameservers def get_vm_topology_tables(*table_names, con_ssh=None, combine_multiline=False, exclude_one_col_table=True, auth_info=Tenant.get('admin')): if con_ssh is None: con_name = auth_info.get('region') if ( auth_info and ProjVar.get_var('IS_DC')) else None con_ssh = ControllerClient.get_active_controller(name=con_name) show_args = ','.join(table_names) tables_ = table_parser.tables(con_ssh.exec_sudo_cmd('vm-topology --show {}'. format(show_args), expect_timeout=30)[1], combine_multiline_entry=combine_multiline) if exclude_one_col_table: new_tables = [] for table_ in tables_: if len(table_['headers']) > 1: new_tables.append(table_) return new_tables return tables_ def __suppress_unsuppress_event(alarm_id, suppress=True, check_first=False, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ suppress/unsuppress an event by uuid Args: alarm_id (str): fail_ok (bool): con_ssh (SSHClient) suppress(bool) True or false Returns (tuple): (rtn_code, message) (0, ) """ suppressed_alarms_tab = get_suppressed_alarms(uuid=True, con_ssh=con_ssh, auth_info=auth_info) alarm_status = "unsuppressed" if suppress else "suppressed" cmd = "event-suppress" if suppress else "event-unsuppress" alarm_filter = {"Suppressed Event ID's": alarm_id} if check_first: if not suppressed_alarms_tab['values']: pre_status = "unsuppressed" else: pre_status = table_parser.get_values(table_=suppressed_alarms_tab, target_header='Status', strict=True, **alarm_filter)[0] if pre_status.lower() != alarm_status: msg = "Event is already {}. Do nothing".format(pre_status) LOG.info(msg) return -1, msg code, output = cli.fm(cmd, '--alarm_id ' + alarm_id, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code == 1: return 1, output post_suppressed_alarms_tab = get_suppressed_alarms(uuid=True, con_ssh=con_ssh) if not post_suppressed_alarms_tab['values']: post_status = ["unsuppressed"] else: post_status = table_parser.get_values(table_=post_suppressed_alarms_tab, target_header="Status", strict=True, **{"Event id": alarm_id}) expt_status = "suppressed" if suppress else "unsuppressed" if post_status[0].lower() != expt_status: msg = "Alarm {} is not {}".format(alarm_id, expt_status) if fail_ok: LOG.warning(msg) return 2, msg raise exceptions.TiSError(msg) succ_msg = "Event {} is {} successfully".format(alarm_id, expt_status) LOG.info(succ_msg) return 0, succ_msg def suppress_event(alarm_id, check_first=False, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): return __suppress_unsuppress_event(alarm_id, True, check_first=check_first, fail_ok=fail_ok, con_ssh=con_ssh, auth_info=auth_info) def unsuppress_event(alarm_id, check_first=False, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): return __suppress_unsuppress_event(alarm_id, False, check_first=check_first, fail_ok=fail_ok, con_ssh=con_ssh, auth_info=auth_info) def generate_event(event_id='300.005', state='set', severity='critical', reason_text='Generated for testing', entity_id='STXAuto', unknown_text='unknown1', unknown_two='unknown2', con_ssh=None): cmd = '''fmClientCli -c "### ###{}###{}###{}###{}### ###{}### ###{}### {}### ###True###True###"'''. \ format(event_id, state, reason_text, entity_id, severity, unknown_text, unknown_two) LOG.info("Generate system event: {}".format(cmd)) if not con_ssh: con_ssh = ControllerClient.get_active_controller() output = con_ssh.exec_cmd(cmd, fail_ok=False)[1] event_uuid = re.findall(UUID, output)[0] LOG.info("Event {} generated successfully".format(event_uuid)) return event_uuid def get_service_parameter_values(service=None, section=None, name=None, field='value', con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Returns the list of values from system service-parameter-list service, section, name can be used to filter the table Args: field (str): field to return valueds for. Default to 'value' service (str): section (str): name (str): con_ssh: auth_info Returns (list): """ kwargs = {} if service: kwargs['service'] = service if section: kwargs['section'] = section if name: kwargs['name'] = name table_ = table_parser.table( cli.system('service-parameter-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) return table_parser.get_values(table_, field, **kwargs) def create_service_parameter(service, section, name, value, con_ssh=None, fail_ok=False, check_first=True, modify_existing=True, verify=True, apply=False, auth_info=Tenant.get('admin_platform')): """ Add service-parameter system service-parameter-add (service) (section) (name)=(value) Args: service (str): Required section (str): Required name (str): Required value (str): Required con_ssh: fail_ok: check_first (bool): Check if the service parameter exists before modify_existing (bool): Whether to modify the service parameter if it already exists verify: this enables to skip the verification. sometimes not all values are displayed in the service-parameter-list, ex password apply (bool): whether to apply service parameter after add auth_info Returns (tuple): (rtn_code, err_msg or param_uuid) """ if check_first: val = get_service_parameter_values(service=service, section=section, name=name, con_ssh=con_ssh, auth_info=auth_info) if val: val = val[0] msg = "The service parameter {} {} {} already exists. value: " \ "{}".format(service, section, name, val) LOG.info(msg) if value != val and modify_existing: return modify_service_parameter(service, section, name, value, create=False, apply=apply, con_ssh=con_ssh, fail_ok=fail_ok, check_first=False, verify=verify, auth_info=auth_info) return -1, msg LOG.info("Creating service parameter") args = service + ' ' + section + ' ' + name + '=' + value res, out = cli.system('service-parameter-add', args, ssh_client=con_ssh, fail_ok=fail_ok) if res == 1: return 1, out LOG.info("Verifying the service parameter value") val = get_service_parameter_values(service=service, section=section, name=name, con_ssh=con_ssh, auth_info=auth_info)[0] value = value.strip('\"') if verify: if val != value: msg = 'The service parameter was not added with the correct ' \ 'value {} to {}'.format(val, value) if fail_ok: return 2, msg raise exceptions.SysinvError(msg) LOG.info("Service parameter was added with the correct value") uuid = get_service_parameter_values(field='uuid', service=service, section=section, name=name, con_ssh=con_ssh, auth_info=auth_info)[0] if apply: apply_service_parameters(service, wait_for_config=True, con_ssh=con_ssh, auth_info=auth_info) return 0, uuid def modify_service_parameter(service, section, name, value, apply=False, con_ssh=None, fail_ok=False, check_first=True, create=True, verify=True, auth_info=Tenant.get('admin_platform')): """ Modify a service parameter Args: service (str): Required section (str): Required name (str): Required value (str): Required apply con_ssh: fail_ok: check_first (bool): Check if the parameter exists first create (bool): Whether to create the parameter if it does not exist verify: this enables to skip the verification. sometimes not all values are displayed in the service-parameter-list, ex password auth_info Returns (tuple): (rtn_code, message) """ if check_first: val = get_service_parameter_values(service=service, section=section, name=name, con_ssh=con_ssh) if not val: msg = "The service parameter {} {} {} doesn't exist".format(service, section, name) LOG.info(msg) if create: return create_service_parameter(service, section, name, value, auth_info=auth_info, con_ssh=con_ssh, fail_ok=fail_ok, check_first=False) return -1, msg if val[0] == value: msg = "The service parameter value is already set to {}".format(val) return -1, msg LOG.info("Modifying service parameter") args = service + ' ' + section + ' ' + name + '=' + value res, out = cli.system('service-parameter-modify', args, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if res == 1: return 1, out LOG.info("Verifying the service parameter value") val = get_service_parameter_values(service=service, section=section, name=name, con_ssh=con_ssh, auth_info=auth_info)[0] value = value.strip('\"') if verify: if val != value: msg = 'The service parameter was not modified to the correct value' if fail_ok: return 2, msg raise exceptions.SysinvError(msg) msg = "Service parameter modified to {}".format(val) LOG.info(msg) if apply: apply_service_parameters(service, wait_for_config=True, con_ssh=con_ssh, auth_info=auth_info) return 0, msg def delete_service_parameter(uuid, con_ssh=None, fail_ok=False, check_first=True, auth_info=Tenant.get('admin_platform')): """ Delete a service parameter Args: uuid (str): Required con_ssh: fail_ok: check_first (bool): Check if the service parameter exists before auth_info Returns (tuple): """ if check_first: uuids = get_service_parameter_values(field='uuid', con_ssh=con_ssh, auth_info=auth_info) if uuid not in uuids: return -1, "There is no service parameter with uuid {}".format(uuid) res, out = cli.system('service-parameter-delete', uuid, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if res == 1: return 1, out LOG.info("Deleting service parameter") uuids = get_service_parameter_values(field='uuid', con_ssh=con_ssh, auth_info=auth_info) if uuid in uuids: err_msg = "Service parameter was not deleted" if fail_ok: return 2, err_msg raise exceptions.SysinvError(err_msg) msg = "The service parameter {} was deleted".format(uuid) LOG.info(msg) return 0, msg def apply_service_parameters(service, wait_for_config=True, timeout=300, con_ssh=None, fail_ok=False, auth_info=Tenant.get('admin_platform')): """ Apply service parameters Args: service (str): Required wait_for_config (bool): Wait for config out of date alarms to clear timeout (int): con_ssh: auth_info fail_ok: Returns (tuple): (rtn_code, message) """ LOG.info("Applying service parameters {}".format(service)) res, out = cli.system('service-parameter-apply', service, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if res == 1: return res, out alarm_id = '250.001' time.sleep(10) if wait_for_config: LOG.info("Waiting for config-out-of-date alarms to clear. " "There may be cli errors when active controller's config " "updates") end_time = time.time() + timeout while time.time() < end_time: table_ = get_alarms_table(uuid=True, con_ssh=con_ssh, retry=3) alarms_tab = table_parser.filter_table(table_, **{'Alarm ID': alarm_id}) uuids = table_parser.get_values(alarms_tab, 'uuid') if not uuids: LOG.info("Config has been applied") break time.sleep(5) else: err_msg = "The config has not finished applying after timeout" if fail_ok: return 2, err_msg raise exceptions.TimeoutException(err_msg) return 0, "The {} service parameter was applied".format(service) def get_system_health_query(con_ssh=None, auth_info=Tenant.get('admin_platform')): output = cli.system('health-query', ssh_client=con_ssh, fail_ok=False, auth_info=auth_info, source_openrc=True)[1] output = output.splitlines() failed = [] for line in output: if "[Fail]" in line: failed_item = line.split(sep=': ')[0] failed.append(failed_item.strip()) if failed: return 1, failed else: return 0, None def get_build_info(con_ssh=None, refresh=False): """ Get build info from /etc/build.info Args: con_ssh: refresh: Returns (dict): """ build_info = ProjVar.get_var('BUILD_INFO') if build_info and not refresh: return build_info con_client = con_ssh code, output = con_client.exec_cmd('cat /etc/build.info') build_info = {} for line in output.splitlines(): if '="' in line: key, value = re.findall('(.*)="(.*)"', line)[0] build_info[key] = value for mandatory_key in ('BUILD_ID', 'BUILD_HOST', 'BUILD_BY', 'JOB'): if mandatory_key not in build_info: build_info[mandatory_key] = '' ProjVar.set_var(BUILD_INFO=build_info) sw_version = build_info.get('SW_VERSION') if sw_version: existing_versions = ProjVar.get_var('SW_VERSION') if not (existing_versions and sw_version == existing_versions[-1]): ProjVar.set_var(append=True, SW_VERSION=sw_version) return build_info def get_sw_version(con_ssh=None, use_existing=True): """ Args: con_ssh: use_existing Returns (str): e.g., 16.10 """ sw_versions = ProjVar.get_var('SW_VERSION') if use_existing and sw_versions: return sw_versions[-1] info_dict = get_build_info(con_ssh=con_ssh, refresh=True) return info_dict.get('SW_VERSION') def install_license(license_path, timeout=30, con_ssh=None): if con_ssh is None: con_ssh = ControllerClient.get_active_controller() cmd = "test -e {}".format(license_path) rc = con_ssh.exec_cmd(cmd, fail_ok=True)[0] if rc != 0: msg = "The {} file missing from active controller".format(license_path) return rc, msg cmd = "sudo license-install " + license_path con_ssh.send(cmd) end_time = time.time() + timeout rc = 1 while time.time() < end_time: index = con_ssh.expect( [con_ssh.prompt, Prompt.PASSWORD_PROMPT, Prompt.Y_N_PROMPT], timeout=timeout) if index == 2: con_ssh.send('y') if index == 1: con_ssh.send(HostLinuxUser.get_password()) if index == 0: rc = con_ssh.exec_cmd("echo $?")[0] con_ssh.flush() break return rc def wait_for_services_enable(timeout=300, fail_ok=False, con_ssh=None): """ Wait for services to be enabled-active in system service-list Args: timeout (int): max wait time in seconds fail_ok (bool): whether return False or raise exception when some services fail to reach enabled-active state con_ssh (SSHClient): Returns (tuple): ((bool), (str)) (True, "All services are enabled-active") (False, "Some services are not enabled-active: ") Applicable if fail_ok=True """ LOG.info("Wait for services to be enabled-active in system service-list") service_list_tab = None end_time = time.time() + timeout while time.time() < end_time: service_list_tab = table_parser.table( cli.system('service-list', ssh_client=con_ssh)[1]) states = table_parser.get_column(service_list_tab, 'state') if all(state == 'enabled-active' for state in states): LOG.info("All services are enabled-active in system service-list") return True, "All services are enabled-active" LOG.warning( "Not all services are enabled-ative within {} seconds".format(timeout)) inactive_services_tab = table_parser.filter_table(service_list_tab, exclude=True, state='enabled-active') msg = "Some services are not enabled-active: {}".format( table_parser.get_all_rows(inactive_services_tab)) if fail_ok: return False, msg raise exceptions.SysinvError(msg) def enable_service(service_name, con_ssh=None, auth_info=Tenant.get('admin_platform'), fail_ok=False): """ Enable Service Args: service_name (str): con_ssh (SSHClient): auth_info (dict): fail_ok: whether return False or raise exception when some services fail to reach enabled-active state Returns (tuple): """ res, output = cli.system('service-enable', service_name, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if res == 1: return 1, output msg = "Service enabled: {}".format(service_name) LOG.info(msg) return 0, msg def disable_service(service_name, con_ssh=None, auth_info=Tenant.get('admin_platform'), fail_ok=False): """ Disable Service Args: service_name (str) con_ssh (SSHClient): auth_info (dict): fail_ok: whether return False or raise exception when some services fail to reach enabled-active state Returns (tuple): """ res, output = cli.system('service-disable', service_name, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if res == 1: return 1, output msg = "Service disabled: {}".format(service_name) LOG.info(msg) return 0, msg def get_system_networks(field='uuid', uuid=None, net_type=None, mtu=None, dynamic=None, pool_uuid=None, auth_info=Tenant.get('admin_platform'), con_ssh=None, strict=True, regex=None, **kwargs): """ Get networks values from system network-list Args: field: 'uuid' (default) uuid: net_type: mtu: dynamic: pool_uuid: auth_info: con_ssh: strict: regex: **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('network-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) args_temp = { 'uuid': uuid, 'type': net_type, 'mtu': mtu, 'dynamic': dynamic, 'pool_uuid': pool_uuid } kwargs.update({k: v for k, v in args_temp.items() if v is not None}) return table_parser.get_multi_values(table_, field, strict=strict, regex=regex, **kwargs) def get_clusters(field='uuid', uuid=None, cluster_uuid=None, ntype=None, name=None, auth_info=Tenant.get('admin_platform'), con_ssh=None, strict=True, regex=None, **kwargs): """ Get cluster values from system cluster-list Args: field: 'uuid' (default) uuid: cluster_uuid: ntype: (mapped as ntype) name: auth_info: con_ssh: strict: regex: **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('cluster-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) args_temp = { 'uuid': uuid, 'cluster_uuid': cluster_uuid, 'ntype': ntype, 'name': name, } kwargs.update({k: v for k, v in args_temp.items() if v is not None}) return table_parser.get_multi_values(table_, field, strict=strict, regex=regex, **kwargs) def get_services(field='id', service_id=None, service_name=None, hostname=None, state=None, auth_info=Tenant.get('admin_platform'), con_ssh=None, strict=True, regex=None, **kwargs): """ Get service_list through service service-list command Args: field: 'id' (default value) service_id: service_name: hostname: state: auth_info: con_ssh: strict: regex: **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('service-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) args_temp = { 'id': service_id, 'service_name': service_name, 'hostname': hostname, 'state': state } kwargs.update({k: v for k, v in args_temp.items() if v is not None}) return table_parser.get_multi_values(table_, field, strict=strict, regex=regex, **kwargs) def get_servicenodes(field='id', servicenode_id=None, name=None, operational=None, availability=None, ready_state=None, auth_info=Tenant.get('admin_platform'), con_ssh=None, strict=True, regex=None, **kwargs): """ Get servicenodes list through service servicenode-list Args: field (str|tuple|list): 'id' (default) servicenode_id: name: operational: availability: ready_state: auth_info: con_ssh: strict: regex: **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('servicenode-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) args_temp = { 'id': servicenode_id, 'name': name, 'operational': operational, 'ready_state': ready_state, 'availability': availability } kwargs.update({k: v for k, v in args_temp.items() if v is not None}) return table_parser.get_multi_values(table_, field, strict=strict, regex=regex, **kwargs) def get_servicegroups(fields='uuid', uuid=None, service_group_name=None, hostname=None, state=None, auth_info=Tenant.get('admin_platform'), con_ssh=None, strict=True, regex=None, **kwargs): """ Get servicegroups via system servicegroup-list Args: fields: 'uuid' (default) uuid: service_group_name: hostname: state: auth_info: con_ssh: strict: regex **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('servicegroup-list', ssh_client=con_ssh, auth_info=auth_info)[1]) args_temp = { 'uuid': uuid, 'service_group_name': service_group_name, 'hostname': hostname, 'state': state } kwargs.update({k: v for k, v in args_temp.items() if v is not None}) return table_parser.get_multi_values(table_, fields, strict=strict, regex=regex, **kwargs) def get_oam_values(fields=None, con_ssh=None, auth_info=Tenant.get('admin_platform'), rtn_dict=True): """ Get oam info via system oam-show Args: fields: con_ssh: auth_info: rtn_dict Returns (dict|list): """ table_ = table_parser.table( cli.system('oam-show', ssh_client=con_ssh, auth_info=auth_info)[1]) if not fields: fields = table_parser.get_column(table_, 'Property') fields = [field for field in fields if field.startswith('oam_')] return table_parser.get_multi_values_two_col_table(table_, fields, rtn_dict=rtn_dict) def modify_oam_ips(fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform'), **kwargs): """ Modify oam ip(s) Args: fail_ok: con_ssh: auth_info: Returns: """ if not kwargs: raise ValueError("Nothing is provided to modify") args = ' '.join(['{}={}'.format(key, val) for key, val in kwargs.items()]) LOG.info("Modify oam ip(s): {}".format(args)) if not con_ssh: con_ssh = ControllerClient.get_active_controller() original = get_oam_values(fields=list(kwargs.keys()), auth_info=auth_info, con_ssh=con_ssh) code, output = cli.system('oam-modify', args, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, output oam_info = get_oam_values(fields=list(kwargs.keys()), auth_info=auth_info, con_ssh=con_ssh) for field, expt_val in kwargs.items(): actual_val = oam_info[field] if expt_val != actual_val: raise exceptions.SysinvError( "{} expected: {}, actual: {}".format(field, expt_val, actual_val)) from keywords import host_helper active, standby = get_active_standby_controllers(con_ssh=con_ssh, auth_info=auth_info) standby_configured = True if standby: standby_configured = False if wait_for_alarm(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, entity_id=standby, timeout=120, con_ssh=con_ssh, fail_ok=True, auth_info=auth_info)[0]: host_helper.lock_unlock_hosts(standby, auth_info=auth_info, con_ssh=con_ssh) standby_configured = True if not standby_configured: revert_args = ' '.join( ['{}={}'.format(key, val) for key, val in original.items()]) LOG.error("Failed to modify oam ip. Revert to: {}".format(revert_args)) cli.system('oam-modify', revert_args, ssh_client=con_ssh, auth_info=auth_info) raise exceptions.SysinvError( "Config out-of-date alarm did not appear or standby controller " "lock/unlock" "failed after oam-modify.") # Update system ssh client and global var fip_field = 'oam_if' if is_aio_simplex(con_ssh=con_ssh, auth_info=auth_info) else \ 'oam_floating_ip' new_lab = ProjVar.get_var('lab') if fip_field in kwargs: new_fip = kwargs[fip_field] con_ssh.update_host() new_lab['floating ip'] = new_fip if 'oam_c0_ip' in kwargs: new_lab['controller-0 ip'] = kwargs['oam_c0_ip'] if 'oam_c1_ip' in kwargs: new_lab['controller-1 ip'] = kwargs['oam_c1_ip'] ProjVar.set_var(LAB=new_lab) host_helper.lock_unlock_hosts(active, con_ssh=con_ssh, auth_info=auth_info) LOG.info("Wait for config out-of-date alarm to clear on system") wait_for_alarm_gone(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, timeout=120, auth_info=auth_info, con_ssh=con_ssh) msg = "OAM IP(s) modified successfully." LOG.info(msg) return 0, msg def modify_spectre_meltdown_version(version='spectre_meltdown_all', check_first=True, con_ssh=None, fail_ok=False, auth_info=Tenant.get('admin_platform')): """ Modify spectre meltdown version Args: version (str): valid values: spectre_meltdown_v1, spectre_meltdown_all. Other values will be rejected by system modify cmd. check_first (bool): con_ssh: fail_ok (bool): auth_info Returns (tuple): (-1, "Security feature already set to . Do nothing") (0, "System security_feature is successfully modified to: ") (1, ) """ current_version = get_system_values(fields='security_feature')[0] if not current_version: skip('spectre_meltdown update feature is unavailable in current load') from keywords import host_helper hosts = get_hosts(con_ssh=con_ssh) check_val = 'nopti nospectre_v2 nospectre_v1' if check_first and version == current_version: LOG.info( "{} already set in 'system show'. Checking actual cmdline options " "on each host.".format( version)) hosts_to_configure = [] for host in hosts: cmdline_options = host_helper.get_host_cmdline_options(host=host) if 'v1' in version: if check_val not in cmdline_options: hosts_to_configure.append(host) elif check_val in cmdline_options: hosts_to_configure.append(host) hosts = hosts_to_configure if not hosts_to_configure: msg = 'Security feature already set to {}. Do nothing.'.format( current_version) LOG.info(msg) return -1, msg LOG.info("Set spectre_meltdown version to {}".format(version)) code, output = cli.system('modify -S {}'.format(version), ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, output conf_storage0 = False if 'storage-0' in hosts: hosts.remove('storage-0') conf_storage0 = True active_controller = get_active_controller_name(con_ssh=con_ssh, auth_info=auth_info) conf_active = False if active_controller in hosts: hosts.remove(active_controller) conf_active = True if hosts: LOG.info( "Lock/unlock unconfigured hosts other than active controller: " "{}".format(hosts)) try: for host in hosts: host_helper.lock_host(host=host, con_ssh=con_ssh, auth_info=auth_info) finally: host_helper.unlock_hosts(hosts=hosts, fail_ok=False, con_ssh=con_ssh, auth_info=auth_info) host_helper.wait_for_hosts_ready(hosts=hosts, con_ssh=con_ssh, auth_info=auth_info) if conf_storage0: LOG.info("Lock/unlock storage-0") try: host_helper.lock_host(host='storage-0', con_ssh=con_ssh, auth_info=auth_info) finally: host_helper.unlock_host(host='storage-0', con_ssh=con_ssh, auth_info=auth_info) if conf_active: LOG.info( "Lock/unlock active controller (swact first if needed): {}".format( active_controller)) try: host_helper.lock_host(host=active_controller, swact=True, con_ssh=con_ssh, auth_info=auth_info) finally: host_helper.unlock_host(host=active_controller, con_ssh=con_ssh, auth_info=auth_info) LOG.info("Check 'system show' is updated to {}".format(version)) post_version = \ get_system_values(fields='security_feature', auth_info=auth_info)[0] assert version == post_version, 'Value is not {} after system ' \ 'modify'.format(version) LOG.info('Check cmdline options are updated on each host via /proc/cmdline') hosts.append(active_controller) for host in hosts: options = host_helper.get_host_cmdline_options(host=host) if 'v1' in version: assert check_val in options, '{} not in cmdline options after set' \ ' to {}'.format(check_val, version) else: assert check_val not in options, '{} in cmdline options after set' \ ' to {}'.format(check_val, version) msg = 'System spectre meltdown version is successfully modified to: ' \ '{}'.format(version) LOG.info(msg) return 0, msg def is_avs(con_ssh=None): vswitch_type = ProjVar.get_var('VSWITCH_TYPE') if vswitch_type is None: vswitch_type = get_system_values(fields='vswitch_type', con_ssh=con_ssh)[0] ProjVar.set_var(VSWITCH_TYPE=vswitch_type) return 'ovs' not in vswitch_type def get_controller_uptime(con_ssh, auth_info=Tenant.get('admin_platform')): """ Get uptime for all controllers. If no standby controller, then we only calculate for current active controller. Args: con_ssh auth_info Returns (int): in seconds """ active_con, standby_con = get_active_standby_controllers( con_ssh=con_ssh, auth_info=auth_info) active_con_uptime = int( get_host_values(host=active_con, fields='uptime', con_ssh=con_ssh, auth_info=auth_info)[0]) con_uptime = active_con_uptime if standby_con: standby_con_uptime = int( get_host_values(host=standby_con, fields='uptime', con_ssh=con_ssh, auth_info=auth_info)[0]) con_uptime = min(active_con_uptime, standby_con_uptime) return con_uptime def add_ml2_extension_drivers(drivers, auth_info=Tenant.get('admin_platform'), con_ssh=None): """ Add given ml2 extension drivers to helm charts override if they don't currently exist Args: drivers (str|list|tuple): auth_info: con_ssh: Returns (tuple): """ return __update_ml2_extension_drivers(drivers=drivers, enable=True, auth_info=auth_info, con_ssh=con_ssh) def remove_ml2_extension_drivers(drivers, auth_info=Tenant.get('admin_platform'), con_ssh=None): """ Remove given ml2 extension drivers from helm charts override if they exist Args: drivers (str|list|tuple): auth_info: con_ssh: Returns (tuple): """ return __update_ml2_extension_drivers(drivers=drivers, enable=False, auth_info=auth_info, con_ssh=con_ssh) def __update_ml2_extension_drivers(drivers, enable=True, auth_info=Tenant.get('admin_platform'), con_ssh=None): """ Add or remove ml2 extension drivers by updating helm charts user override Args: drivers (str|list|tuple): enable (bool): whether to enable or disable given ml2 extension driver(s) auth_info: con_ssh: Returns (tuple): """ if isinstance(drivers, str): drivers = (drivers,) from keywords import container_helper known_drivers = ['port_security', 'qos', 'dns'] all_drivers = known_drivers + [driver for driver in drivers if driver not in known_drivers] chart = 'neutron' LOG.info("Check existing ml2 extension_drivers") field = 'combined_overrides' combined_overrides = \ container_helper.get_helm_override_values(chart, namespace='openstack', fields=field)[0] current_drivers = combined_overrides['conf'].get('plugins', {}).get( 'ml2_conf', {}).get('ml2', {}). \ get('extension_drivers', '').split(sep=',') if enable: expt_drivers = set(current_drivers + list(drivers)) # convert expt_drivers to ordered list by removing unwanted drivers # from ordered all_drivers list drivers_to_remove = set(all_drivers) - expt_drivers expt_drivers = [driver for driver in all_drivers if driver not in drivers_to_remove] else: expt_drivers = [driver for driver in current_drivers if driver not in drivers] if expt_drivers == current_drivers: LOG.info("ml2 extension drivers already set to {}. Do nothing.".format( expt_drivers)) return -1, current_drivers path = 'conf.plugins.ml2_conf.ml2.extension_drivers' new_value = ','.join(expt_drivers) LOG.info("Update neutron helm-override: {}={}".format(path, new_value)) if len(expt_drivers) <= 1: kw_args = {'kv_pairs': {path: new_value}} else: content = """ conf: plugins: ml2_conf: ml2: extension_drivers: {} """.format(new_value) yaml_file = os.path.join(HostLinuxUser.get_home(), 'ml2_drivers.yaml') if not con_ssh: con_ssh = ControllerClient.get_active_controller() con_ssh.exec_cmd('rm -f {}'.format(yaml_file), get_exit_code=False) con_ssh.exec_cmd("echo '{}' >> {}".format(content, yaml_file)) kw_args = {'yaml_file': yaml_file} container_helper.update_helm_override(chart=chart, namespace='openstack', auth_info=auth_info, con_ssh=con_ssh, **kw_args) post_overrides = \ container_helper.get_helm_override_values(chart, namespace='openstack', fields=field)[0] post_drivers = post_overrides['conf'].get('plugins', {}).\ get('ml2_conf', {}).get('ml2', {}).get('extension_drivers', '').\ split(sep=',') if not post_drivers == expt_drivers: raise exceptions.SysinvError( "ml2 extension_drivers override is not reflected") LOG.info("Re-apply stx-openstack application") container_helper.apply_app(app_name='stx-openstack', applied_timeout=1200, auth_info=auth_info, con_ssh=con_ssh) return 0, post_drivers def get_ptp_values(fields='mode', rtn_dict=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get values from system ptp-show table. Args: fields (str|tuple|list): rtn_dict (bool): whether to return dict or list con_ssh: auth_info Returns (list|dict): """ table_ = table_parser.table( cli.system('ptp-show', ssh_client=con_ssh, auth_info=auth_info)[1]) return table_parser.get_multi_values_two_col_table(table_, fields, rtn_dict=rtn_dict, merge_lines=True) def modify_ptp(enabled=None, mode=None, transport=None, mechanism=None, fail_ok=False, con_ssh=None, clear_alarm=True, wait_with_best_effort=False, check_first=True, auth_info=Tenant.get('admin_platform')): """ Modify ptp with given parameters Args: enabled (bool|None): mode (str|None): transport (str|None): mechanism (str|None): fail_ok (bool): clear_alarm (bool): wait_with_best_effort (bool): check_first: auth_info (dict): con_ssh: Returns: """ args_map = { 'enabled': enabled, 'mode': mode, 'transport': transport, 'mechanism': mechanism, } args_dict = {} for key, val in args_map.items(): if val is not None: args_dict[key] = str(val) if not args_dict: raise ValueError("At least one parameter has to be specified.") arg_str = ' '.join(['--{} {}'.format(k, v) for k, v in args_dict.items()]) if check_first: actual_val_list = get_ptp_values(fields=list(args_dict.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info) changeparm = False for field in args_dict: param_val = args_dict[field] actual_val = actual_val_list[field] if actual_val != param_val: changeparm = True break if not changeparm: return -1, 'No parameter chage' code, output = cli.system('ptp-modify', arg_str, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, output if clear_alarm: wait_and_clear_config_out_of_date_alarms( host_type='controller', wait_with_best_effort=wait_with_best_effort, con_ssh=con_ssh, auth_info=auth_info) post_args = get_ptp_values(fields=list(args_dict.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info) for field in args_dict: expt_val = args_dict[field] actual_val = post_args[field] if actual_val != expt_val: raise exceptions.SysinvError( "{} in ptp-show is not as expected after modify. Expt: {}; " "actual: {}". format(field, expt_val, actual_val)) msg = 'ptp modified successfully. {}'.format( 'Alarm not cleared yet.' if not clear_alarm else '') return 0, msg def get_ntp_values(fields='ntpservers', rtn_dict=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get values from system ntp-show table. Args: fields (str|tuple|list): rtn_dict (bool) con_ssh: auth_info Returns (list|dict): """ table_ = table_parser.table( cli.system('ntp-show', ssh_client=con_ssh, auth_info=auth_info)[1]) return table_parser.get_multi_values_two_col_table(table_, fields, rtn_dict=rtn_dict) def get_ntp_servers(con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get ntp servers via system ntp-show Args: con_ssh: auth_info: Returns (list): """ ntp_servers = get_ntp_values(fields='ntpservers', rtn_dict=False, con_ssh=con_ssh, auth_info=auth_info) ntp_servers = ntp_servers[0].split(',') return ntp_servers def modify_ntp(enabled=None, ntp_servers=None, check_first=True, fail_ok=False, clear_alarm=True, wait_with_best_effort=False, con_ssh=None, auth_info=Tenant.get('admin_platform'), **kwargs): """ Args: enabled (bool|None): ntp_servers (str|None|list|tuple): check_first (bool) fail_ok (bool) clear_alarm (bool): Whether to wait and lock/unlock hosts to clear alarm wait_with_best_effort (bool): whether to wait for alarm with best effort only con_ssh: check_first: auth_info: **kwargs Returns (tuple): (0, ) (1, ) # cli rejected """ arg = '' verify_args = {} if enabled is not None: arg += '--enabled {}'.format(enabled).lower() verify_args['enabled'] = str(enabled) if ntp_servers: if isinstance(ntp_servers, (tuple, list)): ntp_servers = ','.join(ntp_servers) arg += ' ntpservers="{}"'.format(ntp_servers) verify_args['ntpservers'] = ntp_servers if kwargs: for k, v in kwargs.items(): arg += ' {}={}'.format(k, v) verify_args[k] = v if not arg: raise ValueError( "Nothing to modify. enable, ntp_servers or kwwargs has to be " "provided") prev_args = None toggle_state = False if enabled is not None: prev_args = get_ntp_values(fields=list(verify_args.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info) if prev_args['enabled'] != verify_args['enabled']: toggle_state = True if check_first and not toggle_state: if not clear_alarm or (clear_alarm and not get_alarms( alarm_id=EventLogID.CONFIG_OUT_OF_DATE, con_ssh=con_ssh, entity_id='controller', auth_info=auth_info)): if not prev_args: prev_args = get_ntp_values(fields=list(verify_args.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info) for field in verify_args: expt_val = verify_args[field] actual_val = prev_args[field] if actual_val != expt_val: break else: msg = 'NTP already configured with given criteria {}. Do ' \ 'nothing.'.format(verify_args) LOG.info(msg) return -1, msg code, out = cli.system('ntp-modify', arg.strip(), ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, out if clear_alarm: # config out-of-date alarm only on controller if only ntp servers are # changed. # If ntp state changes, ALL hosts need to be lock/unlock. host_type = None if toggle_state else 'controller' wait_and_clear_config_out_of_date_alarms( host_type=host_type, con_ssh=con_ssh, auth_info=auth_info, wait_with_best_effort=wait_with_best_effort) post_args = get_ntp_values(fields=list(verify_args.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info) for field in verify_args: expt_val = verify_args[field] actual_val = post_args[field] if actual_val != expt_val: raise exceptions.SysinvError( "{} in ntp-show is not as expected after modify. Expt: {}; " "actual: {}". format(field, expt_val, actual_val)) msg = 'ntp modified successfully. {}'.format( 'Alarm not cleared yet.' if not clear_alarm else '') return 0, msg def wait_and_clear_config_out_of_date_alarms( hosts=None, host_type=None, lock_unlock=True, wait_timeout=60, wait_with_best_effort=False, clear_timeout=60, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for config out-of-date alarms on given hosts and (lock/unlock and) wait for clear Args: hosts: host_type (str|list|tuple): valid types: controller, compute, storage lock_unlock (bool) wait_timeout (int) wait_with_best_effort (bool): clear_timeout (int) con_ssh: auth_info Returns: """ from keywords.host_helper import lock_unlock_hosts if not hosts: if not host_type: host_type = ('controller', 'compute', 'storage') elif isinstance(host_type, str): host_type = (host_type,) avail_states = (HostAvailState.DEGRADED, HostAvailState.AVAILABLE, HostAvailState.ONLINE) hosts_per_type = get_hosts_per_personality(availability=avail_states, con_ssh=con_ssh, auth_info=auth_info) # host_groups: ordered list for controller, compute, storage hosts host_groups = [hosts_per_type[type_] for type_ in host_type if hosts_per_type[type_]] if not host_groups: raise exceptions.HostError( "No valid hosts found for host_type: {}".format(host_type)) else: if isinstance(hosts, str): hosts = [hosts] host_groups = [hosts] hosts_out_of_date = [] all_hosts = [] for hosts_ in host_groups: LOG.info( "Wait for config out-of-date alarms for {} with best effort".format( hosts_)) all_hosts += hosts_ if wait_for_alarm(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, entity_id=hosts_, timeout=wait_timeout, con_ssh=con_ssh, fail_ok=True, auth_info=auth_info)[0]: hosts_out_of_date += hosts_ hosts_out_of_date = list(set(hosts_out_of_date)) all_hosts = list(set(all_hosts)) LOG.info("Config out-of-date hosts: {}".format(hosts_out_of_date)) if hosts_out_of_date: if lock_unlock: LOG.info( "Wait for 60 seconds, then lock/unlock config out-of-date " "hosts: {}".format(hosts_out_of_date)) time.sleep(60) lock_unlock_hosts(hosts_out_of_date, con_ssh=con_ssh, auth_info=auth_info) LOG.info("Wait for config out-of-date alarm to clear on system") wait_for_alarm_gone(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, timeout=clear_timeout, auth_info=auth_info, con_ssh=con_ssh) if not wait_with_best_effort and all_hosts != hosts_out_of_date: raise exceptions.SysinvError( "Expect config out of date: {}; actual: {}".format( all_hosts, hosts_out_of_date)) def get_timezone(auth_info=Tenant.get('admin_platform'), con_ssh=None): return get_system_values(fields='timezone', auth_info=auth_info, con_ssh=con_ssh)[0] def modify_timezone(timezone, check_first=True, fail_ok=False, clear_alarm=True, auth_info=Tenant.get('admin_platform'), con_ssh=None): """ Modify timezone to given zone Args: timezone: check_first: fail_ok: clear_alarm: auth_info: con_ssh: Returns (tuple): """ if check_first: current_timezone = get_timezone(auth_info=auth_info, con_ssh=con_ssh) if current_timezone == timezone: msg = "Timezone is already set to {}. Do nothing.".format(timezone) LOG.info(msg) return -1, msg LOG.info("Modifying Timezone to {}".format(timezone)) code, out = modify_system(fail_ok=fail_ok, auth_info=auth_info, con_ssh=con_ssh, timezone=timezone) if code > 0: return 1, out if clear_alarm: if wait_for_alarm(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, timeout=30, con_ssh=con_ssh, fail_ok=True, auth_info=auth_info)[0]: wait_for_alarm_gone(alarm_id=EventLogID.CONFIG_OUT_OF_DATE, timeout=180, con_ssh=con_ssh, auth_info=auth_info) time.sleep(10) post_timezone = get_timezone(auth_info=auth_info, con_ssh=con_ssh) if post_timezone != timezone: msg = 'Timezone is {} instead of {} after modify'.format(post_timezone, timezone) if fail_ok: LOG.warning(msg) return 2, post_timezone raise exceptions.SysinvError(msg) LOG.info("Timezone is successfully modified to {}".format(timezone)) return 0, timezone def create_data_network(name, net_type='vlan', mode=None, mtu=None, port_num=None, multicast_group=None, ttl=None, description=None, field='uuid', fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform'), cleanup=None): """ Add a datanetwork Args: name (str): net_type (str): vlan, vxlan or flat mode (None|str|None): mtu (int|str|None): port_num (int|str|None): multicast_group (str|None): ttl (int|str|None): description (str|None): field (str): uuid or name fail_ok: con_ssh: auth_info: cleanup (str|None): function, class, module or session Returns (tuple): (0, ) (1, ) """ args_dict = { 'description': description, 'mtu': mtu, 'port_num': port_num, 'multicast_group': multicast_group, 'ttl': ttl, 'mode': mode, } args = '{} {} {}'.format(common.parse_args(args_dict), name, net_type) code, output = cli.system('datanetwork-add', args, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, output table_ = table_parser.table(output) LOG.info("data network {} is created successfully".format(name)) if cleanup: uuid = table_parser.get_value_two_col_table(table_, field='uuid') ResourceCleanup.add('datanetwork', uuid, scope=cleanup) return 0, table_parser.get_value_two_col_table(table_, field) def get_data_networks(field='name', con_ssh=None, auth_info=Tenant.get('admin_platform'), strict=True, **kwargs): """ Get values from system datanetwork-list Args: field (str|tuple|list): con_ssh: auth_info: strict: **kwargs: Returns (list): """ table_ = table_parser.table( cli.system('datanetwork-list', ssh_client=con_ssh, auth_info=auth_info)[ 1]) return table_parser.get_multi_values(table_, fields=field, strict=strict, **kwargs) def get_data_network_values(datanetwork, fields=('uuid',), fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get datanetwork values from system datanetwork-show table. Args: datanetwork (str): name or uuid of datanetwork fields (str|tuple|list): fail_ok: con_ssh: auth_info: Returns (list|None): values for given fields. None if cli is rejected. """ code, output = cli.system('datanetwork-show', datanetwork, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return None table_ = table_parser.table(output) return table_parser.get_multi_values_two_col_table(table_=table_, fields=fields) def delete_data_network(datanetwork_uuid, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Delete given datanetwork Args: datanetwork_uuid (str): fail_ok: con_ssh: auth_info: Returns (tuple): (0, "datanetwork deleted successfully") (1, ) (2, "datanetwork still exists after deletion") """ code, output = cli.system('datanetwork-delete', datanetwork_uuid, ssh_client=con_ssh, fail_ok=fail_ok, auth_info=auth_info) if code > 0: return 1, output if get_data_network_values(datanetwork=datanetwork_uuid, con_ssh=con_ssh, auth_info=auth_info, fail_ok=True): err = 'datanetwork {} still exists after deletion'.format( datanetwork_uuid) LOG.warning(err) if fail_ok: return 2, err else: raise exceptions.SysinvError(err) msg = 'datanetwork {} deleted successfully'.format(datanetwork_uuid) LOG.info(msg) return 0, msg def get_addr_pools(field, name=None, uuid=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get values from system addrpool-list Args: field (str|list|tuple): name: uuid: con_ssh: auth_info: Returns (list): """ table_ = table_parser.table( cli.system('addrpool-list --nowrap', ssh_client=con_ssh, auth_info=auth_info)[1]) kwargs = {'name': name, 'uuid': uuid} return table_parser.get_multi_values(table_=table_, fields=field, **{k: v for k, v in kwargs.items()}) def get_addr_pool_values(fields, addr_pool=None, network_type=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get values from system addrpool-show Args: fields (str|tuple|list): addr_pool: network_type: con_ssh: auth_info: Returns (list): """ if not addr_pool and not network_type: raise ValueError('addr_pool uuid or network_type has to be provided') if not addr_pool: addr_pool = \ get_system_networks(field='pool_uuid', net_type=network_type, con_ssh=con_ssh, auth_info=auth_info)[0] if not addr_pool: raise exceptions.SysinvError( "No pool_uuid found for network type {}".format(network_type)) table_ = table_parser.table( cli.system('addrpool-show', addr_pool, ssh_client=con_ssh, auth_info=auth_info)[1]) return table_parser.get_multi_values_two_col_table(table_, fields=fields) def get_system_network_cidr(network_type, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get cidr for given network type, such as mgmt, oam, cluster-host, etc. Args: network_type: con_ssh: auth_info: Returns (str): """ network, prefix = get_addr_pool_values(fields=('network', 'prefix'), network_type=network_type, con_ssh=con_ssh, auth_info=auth_info) return '{}/{}'.format(network, prefix) def get_host_values(host, fields, rtn_dict=False, merge_lines=True, auth_info=Tenant.get('admin_platform'), con_ssh=None): """ Get values from system host-show Args: host (str): fields (str|list|tuple): rtn_dict: merge_lines auth_info: con_ssh: Returns (list): """ table_ = table_parser.table( cli.system('host-show', host, ssh_client=con_ssh, auth_info=auth_info)[ 1], combine_multiline_entry=merge_lines) return table_parser.get_multi_values_two_col_table(table_, fields, rtn_dict=rtn_dict, evaluate=True) def get_hosts_values(hosts, fields, merge_lines=False, rtn_dict_per_host=True, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get values for multiple hosts via system host-show Args: hosts: fields: merge_lines: rtn_dict_per_host: con_ssh: auth_info: Returns (dict): e.g., {'controller-0': {'task': booting, 'subfunctions': ...}, 'controller-1':...} """ if isinstance(fields, str): fields = [fields] states_vals = {} for host in hosts: vals = get_host_values(host, fields, merge_lines=merge_lines, con_ssh=con_ssh, rtn_dict=rtn_dict_per_host, auth_info=auth_info) states_vals[host] = vals return states_vals def get_ntpq_status(host, mgmt_cidr=None, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Get ntp status via 'sudo ntpq -pn' Args: host (str): host to check mgmt_cidr (str): internal management ip from peer host con_ssh (SSHClient) auth_info Returns(tuple): (, ) - (0, " NTPQ is in healthy state") - (1, "No NTP server selected") - (2, "Some NTP servers are discarded") """ if not mgmt_cidr: mgmt_cidr = get_system_network_cidr('mgmt', con_ssh=con_ssh, auth_info=auth_info) cmd = 'ntpq -pn' from keywords import host_helper with host_helper.ssh_to_host(host, con_ssh=con_ssh) as host_ssh: output = host_ssh.exec_sudo_cmd(cmd, fail_ok=False)[1] output_lines = output.splitlines() server_lines = list(output_lines) for line in output_lines: server_lines.remove(line) if '======' in line: break selected = None discarded = [] for server_line in server_lines: try: # Check if its an internal mgmt net ip if ipaddress.ip_address(server_line[1:]) in ipaddress.ip_network( mgmt_cidr): continue except ValueError: pass if server_line.startswith('*'): selected = server_line elif server_line.startswith('-') or server_line.startswith( 'x') or server_line.startswith(' '): discarded.append(server_line) if not selected: return 1, "No NTP server selected" if discarded: return 2, "Some NTP servers are discarded" return 0, "{} NTPQ is in healthy state".format(host) def wait_for_ntp_sync(host, timeout=MiscTimeout.NTPQ_UPDATE, fail_ok=False, con_ssh=None, auth_info=Tenant.get('admin_platform')): """ Wait for ntp alarm inline with sudo ntpq output. Args: host: timeout: fail_ok: con_ssh: auth_info: Returns (bool): """ LOG.info("Waiting for ntp alarm to clear or sudo ntpq -pn indicate " "unhealthy server for {}".format(host)) end_time = time.time() + timeout msg = ntp_alarms = None if not con_ssh: con_name = auth_info.get('region') if ( auth_info and ProjVar.get_var('IS_DC')) else None con_ssh = ControllerClient.get_active_controller(name=con_name) mgmt_cidr = get_system_network_cidr('mgmt', con_ssh=con_ssh, auth_info=auth_info) while time.time() < end_time: ntp_alarms = get_alarms(alarm_id=EventLogID.NTP_ALARM, entity_id=host, strict=False, con_ssh=con_ssh, auth_info=auth_info) status, msg = get_ntpq_status(host, mgmt_cidr=mgmt_cidr, con_ssh=con_ssh, auth_info=auth_info) if ntp_alarms and status != 0: LOG.info("Valid NTP alarm") return True elif not ntp_alarms and status == 0: LOG.info("NTP alarm cleared and sudo ntpq shows servers healthy") return True LOG.info("NTPQ status: {}; NTP alarms: {}".format(msg, ntp_alarms)) time.sleep(30) err_msg = "Timed out waiting for NTP alarm to be in sync with ntpq " \ "output. NTPQ status: {}; NTP alarms: {}".format(msg, ntp_alarms) if fail_ok: LOG.warning(err_msg) return False raise exceptions.HostTimeout(err_msg) def __hosts_stay_in_states(hosts, duration=10, con_ssh=None, auth_info=Tenant.get('admin_platform'), **states): """ Check if hosts stay in specified state(s) for given duration. Args: hosts (list|str): hostname(s) duration (int): duration to check for in seconds con_ssh (SSHClient): **states: such as availability=[online, available] Returns: bool: True if host stayed in specified states for given duration; False if host is not in specified states anytime in the duration. """ end_time = time.time() + duration while time.time() < end_time: if not __hosts_in_states(hosts=hosts, con_ssh=con_ssh, auth_info=auth_info, **states): return False time.sleep(1) return True def wait_for_hosts_states(hosts, timeout=HostTimeout.REBOOT, check_interval=5, duration=3, con_ssh=None, fail_ok=True, auth_info=Tenant.get('admin_platform'), **states): """ Wait for hosts to go in specified states via system host-list Args: hosts (str|list): timeout (int): check_interval (int): duration (int): wait for a host to be in given state(s) for at least seconds con_ssh (SSHClient): fail_ok (bool) auth_info **states: such as availability=[online, available] Returns (bool): True if host reaches specified states within timeout, and stays in states for given duration; False otherwise """ if not hosts: raise ValueError("No host(s) provided to wait for states.") if isinstance(hosts, str): hosts = [hosts] for key, value in states.items(): if isinstance(value, str): value = [value] states[key] = value LOG.info("Waiting for {} to reach state(s): {}...".format(hosts, states)) end_time = time.time() + timeout while time.time() < end_time: if __hosts_stay_in_states(hosts, con_ssh=con_ssh, duration=duration, auth_info=auth_info, **states): LOG.info("{} have reached state(s): {}".format(hosts, states)) return True time.sleep(check_interval) else: msg = "Timed out waiting for {} in state(s) - {}".format(hosts, states) if fail_ok: LOG.warning(msg) return False raise exceptions.HostTimeout(msg) def __hosts_in_states(hosts, con_ssh=None, auth_info=Tenant.get('admin_platform'), **states): actual_values = get_hosts(hostname=hosts, field=list(states.keys()), con_ssh=con_ssh, auth_info=auth_info, rtn_dict=True) for field, expt_values in states.items(): actual_states = actual_values[field] for actual_state in actual_states: if actual_state not in expt_values: LOG.debug("At least one host from {} has {} state(s) in {} " "instead of {}".format(hosts, field, actual_state, expt_values)) return False return True def wait_for_host_values(host, timeout=HostTimeout.REBOOT, check_interval=3, strict=True, regex=False, fail_ok=True, con_ssh=None, auth_info=Tenant.get('admin_platform'), **kwargs): """ Wait for host values via system host-show Args: host: timeout: check_interval: strict: regex: fail_ok: con_ssh: auth_info **kwargs: key/value pair to wait for. Returns: """ if not kwargs: raise ValueError( "Expected host state(s) has to be specified via " "keyword argument states") LOG.info("Waiting for {} to reach state(s) - {}".format(host, kwargs)) end_time = time.time() + timeout last_vals = {} for field in kwargs: last_vals[field] = None while time.time() < end_time: actual_vals = get_host_values(host, fields=list(kwargs.keys()), con_ssh=con_ssh, rtn_dict=True, auth_info=auth_info, merge_lines=False) for field, expt_vals in kwargs.items(): actual_val = actual_vals[field] if isinstance(actual_val, list): actual_val = ' '.join(actual_val) actual_val_lower = actual_val.lower() if isinstance(expt_vals, str): expt_vals = [expt_vals] for expected_val in expt_vals: expected_val_lower = expected_val.strip().lower() found_match = False if regex: if strict: res_ = re.match(expected_val_lower, actual_val_lower) else: res_ = re.search(expected_val_lower, actual_val_lower) if res_: found_match = True else: if strict: found_match = actual_val_lower == expected_val_lower else: found_match = actual_val_lower in expected_val_lower if found_match: LOG.info( "{} {} has reached: {}".format(host, field, actual_val)) break else: # no match found. run system host-show again if last_vals[field] != actual_val_lower: LOG.info("{} {} is {}.".format(host, field, actual_val)) last_vals[field] = actual_val_lower break else: LOG.info("{} is in state(s): {}".format(host, kwargs)) return True time.sleep(check_interval) else: msg = "{} did not reach state(s) within {}s - {}".format(host, timeout, kwargs) if fail_ok: LOG.warning(msg) return False raise exceptions.TimeoutException(msg) def is_active_controller(host, con_ssh=None, auth_info=Tenant.get('admin_platform')): personality = get_host_values(host, fields='capabilities', auth_info=auth_info, merge_lines=True, con_ssh=con_ssh)[0].get('Personality', '') return personality.lower() == 'Controller-Active'.lower() def is_lowlatency_host(host): subfuncs = get_host_values(host=host, fields='subfunctions')[0] return 'lowlatency' in subfuncs def get_system_iplist(): """ Checks the ipv4 or ipv6 simplex or other and returns the ip list accordingly Return: returns the system ipv4/ipv6 list """ ip = [] out = get_oam_values() if is_aio_simplex(): ip.append(out["oam_ip"]) else: ip.extend([out["oam_floating_ip"], out["oam_c0_ip"], out["oam_c1_ip"]]) if ProjVar.get_var('IPV6_OAM'): iplist = ["[{}]".format(i) for i in ip] ip = iplist return ip