starlingx
/
test
Code Issues Proposed changes
test/automated-pytest-suite/keywords/vm_helper.py

5989 lines
220 KiB
Python
Executable File
Raw Blame History

#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
import copy
import math
import os
import random
import re
import time
import ipaddress
import pexpect
from contextlib import contextmanager
from consts.auth import Tenant, TestFileServer, HostLinuxUser
from consts.stx import VMStatus, NovaCLIOutput, EXT_IP, ImageStatus, \
VMNetwork, EventLogID, GuestImages, Networks, FlavorSpec, VimEventID
from consts.filepaths import VMPath, UserData, TestServerPath
from consts.proj_vars import ProjVar
from consts.timeout import VMTimeout, CMDTimeout
from utils import exceptions, cli, table_parser, multi_thread
from utils import local_host
from utils.clients.ssh import NATBoxClient, VMSSHClient, ControllerClient, \
Prompt, get_cli_client
from utils.clients.local import LocalHostClient
from utils.guest_scripts.scripts import TisInitServiceScript
from utils.multi_thread import MThread, Events
from utils.tis_log import LOG
from keywords import network_helper, nova_helper, cinder_helper, host_helper, \
glance_helper, common, system_helper, \
storage_helper
from testfixtures.fixture_resources import ResourceCleanup
from testfixtures.recover_hosts import HostsToRecover
def set_vm(vm_id, name=None, state=None, con_ssh=None, auth_info=None,
fail_ok=False, **properties):
"""
Set vm with given parameters - name, state, and/or properties
Args:
vm_id:
name:
state:
con_ssh:
auth_info:
fail_ok:
**properties:
Returns (tuple):
(0, )
"""
args_dict = {
'--name': name,
'--state': state.lower() if state else None,
'--property': properties,
}
args = '{} {}'.format(common.parse_args(args_dict, repeat_arg=True), vm_id)
LOG.info("Setting vm with args: {}".format(args))
code, output = cli.openstack('server set', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code > 0:
return 1, output
msg = "VM {} is set successfully.".format(vm_id)
LOG.info(msg)
return 0, msg
def unset_vm(vm_id, properties, con_ssh=None, auth_info=None, fail_ok=False):
"""
Unset given properties for VM
Args:
vm_id:
properties:
con_ssh:
auth_info:
fail_ok:
Returns (tuple):
(1, <std_err>) - cli rejected
(0, "VM <vm_id> properties unset successfully: <properties>")
"""
if isinstance(properties, str):
properties = (properties,)
args = '{} {}'.format(
common.parse_args({'--property': properties}, repeat_arg=True), vm_id)
LOG.info("Unsetting vm {} properties: {}".format(vm_id, properties))
code, output = cli.openstack('server unset', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code > 0:
return 1, output
msg = "VM {} properties unset successfully: {}".format(vm_id, properties)
LOG.info(msg)
return 0, msg
def get_any_vms(count=None, con_ssh=None, auth_info=None, all_tenants=False,
rtn_new=False):
"""
Get a list of ids of any active vms.
Args:
count (int): number of vms ids to return. If None, all vms for
specific tenant will be returned. If num of
existing vm is less than count additional vm will be created to match
the count
con_ssh (SSHClient):
auth_info (dict):
all_tenants (bool): whether to get any vms from all tenants or just
admin tenant if auth_info is set to Admin
rtn_new (bool): whether to return an extra list containing only the
newly created vms
Returns (list):
vms(list) # rtn_new=False
[vms(list), new_vms(list)] # rtn_new=True
"""
vms = get_vms(con_ssh=con_ssh, auth_info=auth_info,
all_projects=all_tenants, Status='ACTIVE')
if count is None:
if rtn_new:
vms = [vms, []]
return vms
diff = count - len(vms)
if diff <= 0:
vms = random.sample(vms, count)
if rtn_new:
vms = [vms, []]
return vms
new_vms = []
for i in range(diff):
new_vm = boot_vm(con_ssh=con_ssh, auth_info=auth_info)[1]
vms.append(new_vm)
new_vms.append(new_vm)
if rtn_new:
vms = [vms, new_vms]
return vms
def create_image_from_vm(vm_id, image_name=None, wait=True,
expt_cinder_snapshot=None,
fail_ok=False, con_ssh=None, auth_info=None,
cleanup=None):
"""
Create glance image from an existing vm
Args:
vm_id:
image_name:
wait:
expt_cinder_snapshot (bool): if vm was booted from cinder volume,
then a cinder snapshot is expected
fail_ok:
con_ssh:
auth_info:
cleanup (None|str): valid scopes: function, class, module, session
Returns (tuple):
"""
LOG.info("Creating image from vm {}".format(vm_id))
args_dict = {'--name': image_name, '--wait': wait}
args = '{} {}'.format(common.parse_args(args_dict), vm_id)
code, out = cli.openstack('server image create', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
table_ = table_parser.table(out)
image_id = table_parser.get_value_two_col_table(table_, 'id')
cinder_snapshot_id = None
if cleanup and image_id:
ResourceCleanup.add('image', image_id, scope=cleanup)
if code > 0:
return 1, out, cinder_snapshot_id
post_name = table_parser.get_value_two_col_table(table_, 'name')
if image_name and image_name != post_name:
raise exceptions.NovaError(
"Create image does not expected name. Actual {}, expected: "
"{}".format(post_name, image_name))
LOG.info(
"Wait for created image {} to reach active state".format(post_name))
glance_helper.wait_for_image_status(image_id, status=ImageStatus.ACTIVE,
con_ssh=con_ssh, auth_info=auth_info)
image_size = table_parser.get_value_two_col_table(table_, 'size')
if str(image_size) == '0' or expt_cinder_snapshot:
cinder_snapshotname = "snapshot for {}".format(post_name)
vol_snapshots = cinder_helper.get_vol_snapshots(
name=cinder_snapshotname)
if not vol_snapshots:
raise exceptions.CinderError(
"cinder snapshot expected, but was not found: {}".format(
cinder_snapshotname))
cinder_snapshot_id = vol_snapshots[0]
if cleanup:
ResourceCleanup.add('vol_snapshot', cinder_snapshot_id)
LOG.info("glance image {} successfully created from vm {}".format(post_name,
vm_id))
return 0, image_id, cinder_snapshot_id
def add_security_group(vm_id, security_group, fail_ok=False, con_ssh=None,
auth_info=None):
"""
Add given security group to vm
Args:
vm_id:
security_group:
fail_ok:
con_ssh:
auth_info:
Returns (tuple):
"""
LOG.info("Adding security group {} to vm {}".format(security_group, vm_id))
args = '{} {}'.format(vm_id, security_group)
code, output = cli.openstack('server add security group', args,
ssh_client=con_ssh, fail_ok=fail_ok,
auth_info=auth_info)
if code > 0:
return 1, output
msg = "Security group {} added to VM {} successfully".format(security_group,
vm_id)
LOG.info(msg)
return 0, msg
def wait_for_vol_attach(vm_id, vol_id, timeout=VMTimeout.VOL_ATTACH,
con_ssh=None, auth_info=None, fail_ok=False):
"""
Wait for volume attachment appear in openstack server show as well as
opentstack volume show
Args:
vm_id:
vol_id:
timeout:
con_ssh:
auth_info:
fail_ok:
Returns (bool):
"""
end_time = time.time() + timeout
while time.time() < end_time:
vols_attached = get_vm_volumes(vm_id=vm_id, con_ssh=con_ssh,
auth_info=auth_info)
if vol_id in vols_attached:
cinder_helper.wait_for_volume_status(vol_id, status='in-use',
timeout=120, fail_ok=False,
con_ssh=con_ssh,
auth_info=auth_info)
return True
time.sleep(5)
else:
msg = "Volume {} is not shown in nova show {} in {} seconds".format(
vol_id, vm_id, timeout)
LOG.warning(msg)
if not fail_ok:
raise exceptions.VMError(msg)
return False
def attach_vol_to_vm(vm_id, vol_id=None, device=None, mount=False, con_ssh=None,
auth_info=None, fail_ok=False,
cleanup=None):
"""
Attach a volume to VM
Args:
vm_id (str):
vol_id (str|None): volume to attach. When None, a non-bootable volume
will be created to attach to given vm
device (str|None): whether to specify --device in cmd
mount (bool): if True, login to vm and attempt to mount the device
after attached. Best effort only.
con_ssh:
auth_info:
fail_ok:
cleanup:
Returns:
"""
if not vol_id:
vol_id = \
cinder_helper.create_volume(bootable=False, auth_info=auth_info,
con_ssh=con_ssh, cleanup=cleanup)[1]
LOG.info("Attaching volume {} to vm {}".format(vol_id, vm_id))
args = '{}{} {}'.format('--device {} '.format(device) if device else '',
vm_id, vol_id)
code, output = cli.openstack('server add volume', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code > 0:
return 1, output
LOG.info(
"Waiting for attached volume to appear in openstack server show and "
"volume show")
wait_for_vol_attach(vm_id=vm_id, vol_id=vol_id, con_ssh=con_ssh,
auth_info=auth_info)
if mount:
LOG.info("Mount attached volume {} to vm {}".format(vol_id, vm_id))
guest = get_vm_image_name(vm_id)
if not (guest and 'cgcs-guest' in guest):
attached_devs = get_vm_volume_attachments(vm_id=vm_id,
field='device',
vol_id=vol_id,
auth_info=auth_info,
con_ssh=con_ssh)
device_name = attached_devs[0]
device = device_name.split('/')[-1]
LOG.info(
"Volume {} is attached to VM {} as {}".format(vol_id, vm_id,
device_name))
mount_attached_volume(vm_id, device, vm_image_name=guest)
return 0, vol_id
def is_attached_volume_mounted(vm_id, rootfs, vm_image_name=None, vm_ssh=None):
"""
Checks if an attached volume is mounted in VM
Args:
vm_id (str): - the vm uuid where the volume is attached to
rootfs (str) - the device name of the attached volume like vda, vdb,
vdc, ....
vm_image_name (str): - the guest image the vm is booted with
vm_ssh (VMSSHClient): ssh client session to vm
Returns: bool
"""
if vm_image_name is None:
vm_image_name = get_vm_image_name(vm_id)
cmd = "mount | grep {} | wc -l".format(rootfs)
mounted_msg = "Filesystem /dev/{} is mounted: {}".format(rootfs, vm_id)
not_mount_msg = "Filesystem /dev/{} is not mounted: {}".format(rootfs,
vm_id)
if vm_ssh:
cmd_output = vm_ssh.exec_sudo_cmd(cmd)[1]
if cmd_output != '0':
LOG.info(mounted_msg)
return True
LOG.info(not_mount_msg)
return False
with ssh_to_vm_from_natbox(vm_id, vm_image_name=vm_image_name) as vm_ssh:
cmd_output = vm_ssh.exec_sudo_cmd(cmd)[1]
if cmd_output != '0':
LOG.info(mounted_msg)
return True
LOG.info(not_mount_msg)
return False
def get_vm_volume_attachments(vm_id, vol_id=None, field='device', con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Get volume attachments for given vm
Args:
vm_id:
vol_id:
field:
con_ssh:
auth_info:
Returns (list):
"""
# No replacement in openstack client
table_ = table_parser.table(
cli.nova('volume-attachments', vm_id, ssh_client=con_ssh,
auth_info=auth_info)[1])
return table_parser.get_values(table_, field, **{'volume id': vol_id})
def mount_attached_volume(vm_id, rootfs, vm_image_name=None):
"""
Mounts an attached volume on VM
Args:
vm_id (str): - the vm uuid where the volume is attached to
rootfs (str) - the device name of the attached volume like vda, vdb,
vdc, ....
vm_image_name (str): - the guest image the vm is booted with
Returns: bool
"""
wait_for_vm_pingable_from_natbox(vm_id)
if vm_image_name is None:
vm_image_name = get_vm_image_name(vm_id)
with ssh_to_vm_from_natbox(vm_id, vm_image_name=vm_image_name) as vm_ssh:
if not is_attached_volume_mounted(vm_id, rootfs,
vm_image_name=vm_image_name,
vm_ssh=vm_ssh):
LOG.info("Creating ext4 file system on /dev/{} ".format(rootfs))
cmd = "mkfs -t ext4 /dev/{}".format(rootfs)
rc, output = vm_ssh.exec_cmd(cmd)
if rc != 0:
msg = "Failed to create filesystem on /dev/{} for vm " \
"{}: {}".format(rootfs, vm_id, output)
LOG.warning(msg)
return False
LOG.info("Mounting /dev/{} to /mnt/volume".format(rootfs))
cmd = "test -e /mnt/volume"
rc, output = vm_ssh.exec_cmd(cmd)
mount_cmd = ''
if rc == 1:
mount_cmd += "mkdir -p /mnt/volume; mount /dev/{} " \
"/mnt/volume".format(rootfs)
else:
mount_cmd += "mount /dev/{} /mnt/volume".format(rootfs)
rc, output = vm_ssh.exec_cmd(mount_cmd)
if rc != 0:
msg = "Failed to mount /dev/{} for vm {}: {}".format(rootfs,
vm_id,
output)
LOG.warning(msg)
return False
LOG.info(
"Adding /dev/{} mounting point in /etc/fstab".format(rootfs))
cmd = "echo \"/dev/{} /mnt/volume ext4 defaults 0 0\" >> " \
"/etc/fstab".format(rootfs)
rc, output = vm_ssh.exec_cmd(cmd)
if rc != 0:
msg = "Failed to add /dev/{} mount point to /etc/fstab for " \
"vm {}: {}".format(rootfs, vm_id, output)
LOG.warning(msg)
LOG.info(
"/dev/{} is mounted to /mnt/volume for vm {}".format(rootfs,
vm_id))
return True
else:
LOG.info(
"/dev/{} is already mounted to /mnt/volume for vm {}".format(
rootfs, vm_id))
return True
def get_vm_devices_via_virsh(vm_id, con_ssh=None):
"""
Get vm disks in dict format via 'virsh domblklist <instance_name>'
Args:
vm_id (str):
con_ssh:
Returns (dict): vm disks per type.
Examples:
{'root_img': {'vda': '/dev/nova-local/a746beb9-08e4-4b08-af2a
-000c8ca72851_disk'},
'attached_vol': {'vdb': '/dev/disk/by-path/ip-192.168.205.106:3260-iscsi
-iqn.2010-10.org.openstack:volume-...'},
'swap': {},
'eph': {}}
"""
vm_host = get_vm_host(vm_id=vm_id, con_ssh=con_ssh)
inst_name = get_vm_instance_name(vm_id=vm_id, con_ssh=con_ssh)
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
output = host_ssh.exec_sudo_cmd('virsh domblklist {}'.format(inst_name),
fail_ok=False)[1]
disk_lines = output.split('-------------------------------\n', 1)[
-1].splitlines()
disks = {}
root_line = disk_lines.pop(0)
root_dev, root_source = root_line.split()
if re.search('openstack:volume|cinder-volumes|/dev/sd', root_source):
disk_type = 'root_vol'
else:
disk_type = 'root_img'
disks[disk_type] = {root_dev: root_source}
LOG.info("Root disk: {}".format(disks))
disks.update({'eph': {}, 'swap': {}, 'attached_vol': {}})
for line in disk_lines:
dev, source = line.split()
if re.search('disk.swap', source):
disk_type = 'swap'
elif re.search('openstack:volume|cinder-volumes|/dev/sd', source):
disk_type = 'attached_vol'
elif re.search('disk.eph|disk.local', source):
disk_type = 'eph'
else:
raise exceptions.CommonError(
"Unknown disk in virsh: {}. Automation update "
"required.".format(
line))
disks[disk_type][dev] = source
LOG.info("disks for vm {}: {}".format(vm_id, disks))
return disks
def get_vm_boot_volume_via_virsh(vm_id, con_ssh=None):
"""
Get cinder volume id where the vm is booted from via virsh cmd.
Args:
vm_id (str):
con_ssh (SSHClient):
Returns (str|None): vol_id or None if vm is not booted from cinder volume
"""
disks = get_vm_devices_via_virsh(vm_id=vm_id, con_ssh=con_ssh)
root_vol = disks.get('root_vol', {})
if not root_vol:
LOG.info("VM is not booted from volume. Return None")
return
root_vol = list(root_vol.values())[0]
root_vol = re.findall('openstack:volume-(.*)-lun', root_vol)[0]
LOG.info("vm {} is booted from cinder volume {}".format(vm_id, root_vol))
return root_vol
def auto_mount_vm_devices(vm_id, devices, guest_os=None, check_first=True,
vm_ssh=None):
"""
Mount and auto mount devices on vm
Args:
vm_id (str): - the vm uuid where the volume is attached to
devices (str|list) - the device name(s). such as vdc or [vda, vdb]
guest_os (str): - the guest image the vm is booted with. such as
tis-centos-guest
check_first (bool): where to check if the device is already mounted
and auto mounted before mount and automount
vm_ssh (VMSSHClient):
"""
if isinstance(devices, str):
devices = [devices]
def _auto_mount(vm_ssh_):
_mounts = []
for disk in devices:
fs = '/dev/{}'.format(disk)
mount_on, fs_type = storage_helper.mount_partition(
ssh_client=vm_ssh_, disk=disk, partition=fs)
storage_helper.auto_mount_fs(ssh_client=vm_ssh_, fs=fs,
mount_on=mount_on, fs_type=fs_type,
check_first=check_first)
_mounts.append(mount_on)
return _mounts
if vm_ssh:
mounts = _auto_mount(vm_ssh_=vm_ssh)
else:
with ssh_to_vm_from_natbox(vm_id, vm_image_name=guest_os) as vm_ssh:
mounts = _auto_mount(vm_ssh_=vm_ssh)
return mounts
def touch_files(vm_id, file_dirs, file_name=None, content=None, guest_os=None):
"""
touch files from vm in specified dirs,and adds same content to all
touched files.
Args:
vm_id (str):
file_dirs (list): e.g., ['/', '/mnt/vdb']
file_name (str|None): defaults to 'test_file.txt' if set to None
content (str|None): defaults to "I'm a test file" if set to None
guest_os (str|None): default guest assumed to set to None
Returns (tuple): (<file_paths_for_touched_files>, <file_content>)
"""
if not file_name:
file_name = 'test_file.txt'
if not content:
content = "I'm a test file"
if isinstance(file_dirs, str):
file_dirs = [file_dirs]
file_paths = []
with ssh_to_vm_from_natbox(vm_id=vm_id, vm_image_name=guest_os) as vm_ssh:
for file_dir in file_dirs:
file_path = "{}/{}".format(file_dir, file_name)
file_path = file_path.replace('//', '/')
vm_ssh.exec_sudo_cmd(
'mkdir -p {}; touch {}'.format(file_dir, file_path),
fail_ok=False)
time.sleep(3)
vm_ssh.exec_sudo_cmd('echo "{}" >> {}'.format(content, file_path),
fail_ok=False)
output = \
vm_ssh.exec_sudo_cmd('cat {}'.format(file_path),
fail_ok=False)[1]
# TO DELETE: Debugging purpose only
vm_ssh.exec_sudo_cmd('mount | grep vd')
assert content in output, "Expected content {} is not in {}. " \
"Actual content: {}". \
format(content, file_path, output)
file_paths.append(file_path)
vm_ssh.exec_sudo_cmd('sync')
return file_paths, content
def auto_mount_vm_disks(vm_id, disks=None, guest_os=None):
"""
Auto mount non-root vm disks and return all the mount points including
root dir
Args:
vm_id (str):
disks (dict|None): disks returned by get_vm_devices_via_virsh()
guest_os (str|None): when None, default guest is assumed.
Returns (list): list of mount points. e.g., ['/', '/mnt/vdb']
"""
if not disks:
disks_to_check = get_vm_devices_via_virsh(vm_id=vm_id)
else:
disks_to_check = copy.deepcopy(disks)
root_disk = disks_to_check.pop('root_vol', {})
if not root_disk:
disks_to_check.pop('root_img')
# add root dir
mounted_on = ['/']
devs_to_mount = []
for val in disks_to_check.values():
devs_to_mount += list(val.keys())
LOG.info("Devices to mount: {}".format(devs_to_mount))
if devs_to_mount:
mounted_on += auto_mount_vm_devices(vm_id=vm_id, devices=devs_to_mount,
guest_os=guest_os)
else:
LOG.info("No non-root disks to mount for vm {}".format(vm_id))
return mounted_on
vif_map = {
'e1000': 'normal',
'rt18139': 'normal',
'virtio': 'normal',
'avp': 'normal',
'pci-sriov': 'direct',
'pci-passthrough': 'direct-physical'}
def _convert_vnics(nics, con_ssh, auth_info, cleanup):
"""
Conversion from wrs vif-model to upstream implementation
Args:
nics (list|tuple|dict):
con_ssh
auth_info
cleanup (None|str)
Returns (list):
"""
converted_nics = []
for nic in nics:
nic = dict(nic) # Do not modify original nic param
if 'vif-model' in nic:
vif_model = nic.pop('vif-model')
if vif_model:
vnic_type = vif_map[vif_model]
vif_model_ = vif_model if (
system_helper.is_avs() and vnic_type == 'normal')\
else None
if 'port-id' in nic:
port_id = nic['port-id']
current_vnic_type, current_vif_model = \
network_helper.get_port_values(
port=port_id,
fields=('binding_vnic_type', 'binding_profile'),
con_ssh=con_ssh,
auth_info=auth_info)
if current_vnic_type != vnic_type or (
vif_model_ and vif_model_ not in current_vif_model):
network_helper.set_port(port_id, vnic_type=vnic_type,
con_ssh=con_ssh,
auth_info=auth_info,
wrs_vif=vif_model_)
else:
net_id = nic.pop('net-id')
port_name = common.get_unique_name(
'port_{}'.format(vif_model))
port_id = network_helper.create_port(net_id, name=port_name,
wrs_vif=vif_model_,
vnic_type=vnic_type,
con_ssh=con_ssh,
auth_info=auth_info,
cleanup=cleanup)[1]
nic['port-id'] = port_id
converted_nics.append(nic)
return converted_nics
def boot_vm(name=None, flavor=None, source=None, source_id=None, image_id=None,
min_count=None, nics=None, hint=None,
max_count=None, key_name=None, swap=None, ephemeral=None,
user_data=None, block_device=None,
block_device_mapping=None, security_groups=None, vm_host=None,
avail_zone=None, file=None,
config_drive=False, meta=None, tags=None,
fail_ok=False, auth_info=None, con_ssh=None, reuse_vol=False,
guest_os='', poll=True, cleanup=None):
"""
Boot a vm with given parameters
Args:
name (str):
flavor (str):
source (str): 'image', 'volume', 'snapshot', or 'block_device'
source_id (str): id of the specified source. such as volume_id,
image_id, or snapshot_id
image_id (str): id of glance image. Will not be used if source is
image and source_id is specified
min_count (int):
max_count (int):
key_name (str):
swap (int|None):
ephemeral (int):
user_data (str|list):
vm_host (str): which host to place the vm
avail_zone (str): availability zone for vm host, Possible values:
'nova', 'stxauto', etc
block_device (dict|list|tuple): dist or list of dict, each dictionary
is a block device.
e.g, {'source': 'volume', 'volume_id': xxxx, ...}
block_device_mapping (str): Block device mapping in the format
'<dev-name>=<id>:<type>:<size(GB)>:<delete-on-
terminate>'.
auth_info (dict):
con_ssh (SSHClient):
security_groups (str|list|tuple): add nova boot option
--security-groups $(sec_group_name)
nics (list): nics to be created for the vm
each nic: <net-id=net-uuid,net-name=network-name,
v4-fixed-ip=ip-addr,v6-fixed-ip=ip-addr,
port-id=port-uuid,vif-model=model>,
vif-pci-address=pci-address>
Examples: [{'net-id': <net_id1>, 'vif-model': <vif1>}, {'net-id':
<net_id2>, 'vif-model': <vif2>}, ...]
Notes: valid vif-models:
virtio, avp, e1000, pci-passthrough, pci-sriov, rtl8139,
ne2k_pci, pcnet
hint (dict): key/value pair(s) sent to scheduler for custom use. such
as group=<server_group_id>
file (str): <dst-path=src-path> To store files from local <src-path>
to <dst-path> on the new server.
config_drive (bool): To enable config drive.
meta (dict): key/value pairs for vm meta data. e.g.,
{'sw:wrs:recovery_priority': 1, ...}
tags (None|str|tuple|list)
fail_ok (bool):
reuse_vol (bool): whether or not to reuse the existing volume
guest_os (str): Valid values: 'cgcs-guest', 'ubuntu_14', 'centos_6',
'centos_7', etc.
This will be overriden by image_id if specified.
poll (bool):
cleanup (str|None): valid values: 'module', 'session', 'function',
'class', vm (and volume) will be deleted as
part of teardown
Returns (tuple): (rtn_code(int), new_vm_id_if_any(str), message(str),
new_vol_id_if_any(str))
(0, vm_id, 'VM is booted successfully') # vm is created
successfully and in Active state.
(1, vm_id, <stderr>) # boot vm cli command failed, but vm is
still booted
(2, vm_id, "VM building is not 100% complete.") # boot vm cli
accepted, but vm building is not
100% completed. Only applicable when poll=True
(3, vm_id, "VM <uuid> did not reach ACTIVE state within <seconds>. VM
status: <status>")
# vm is not in Active state after created.
(4, '', <stderr>): create vm cli command failed, vm is not booted
"""
valid_cleanups = (None, 'function', 'class', 'module', 'session')
if cleanup not in valid_cleanups:
raise ValueError(
"Invalid scope provided. Choose from: {}".format(valid_cleanups))
LOG.info("Processing boot_vm args...")
# Handle mandatory arg - name
tenant = common.get_tenant_name(auth_info=auth_info)
if name is None:
name = 'vm'
name = "{}-{}".format(tenant, name)
name = common.get_unique_name(name, resource_type='vm')
# Handle mandatory arg - key_name
key_name = key_name if key_name is not None else get_default_keypair(
auth_info=auth_info, con_ssh=con_ssh)
# Handle mandatory arg - flavor
if flavor is None:
flavor = nova_helper.get_basic_flavor(auth_info=auth_info,
con_ssh=con_ssh,
guest_os=guest_os)
if guest_os == 'vxworks':
LOG.tc_step("Add HPET Timer extra spec to flavor")
extra_specs = {FlavorSpec.HPET_TIMER: 'True'}
nova_helper.set_flavor(flavor=flavor, **extra_specs)
# Handle mandatory arg - nics
if not nics:
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info,
con_ssh=con_ssh)
if not mgmt_net_id:
raise exceptions.NeutronError("Cannot find management network")
nics = [{'net-id': mgmt_net_id}]
if 'edge' not in guest_os and 'vxworks' not in guest_os:
tenant_net_id = network_helper.get_tenant_net_id(
auth_info=auth_info, con_ssh=con_ssh)
if tenant_net_id:
nics.append({'net-id': tenant_net_id})
if isinstance(nics, dict):
nics = [nics]
nics = _convert_vnics(nics, con_ssh=con_ssh, auth_info=auth_info,
cleanup=cleanup)
# Handle mandatory arg - boot source
volume_id = snapshot_id = image = None
if source != 'block_device':
if source is None:
if min_count is None and max_count is None:
source = 'volume'
else:
source = 'image'
if source.lower() == 'volume':
if source_id:
volume_id = source_id
else:
vol_name = 'vol-' + name
if reuse_vol:
volume_id = cinder_helper.get_any_volume(
new_name=vol_name,
auth_info=auth_info,
con_ssh=con_ssh,
cleanup=cleanup)
else:
volume_id = cinder_helper.create_volume(
name=vol_name,
source_id=image_id,
auth_info=auth_info,
con_ssh=con_ssh,
guest_image=guest_os,
cleanup=cleanup)[1]
elif source.lower() == 'image':
image = source_id if source_id else image_id
if not image:
img_name = guest_os if guest_os else GuestImages.DEFAULT[
'guest']
image = glance_helper.get_image_id_from_name(img_name,
strict=True,
fail_ok=False)
elif source.lower() == 'snapshot':
if not snapshot_id:
snapshot_id = cinder_helper.get_vol_snapshots(
auth_info=auth_info, con_ssh=con_ssh)
if not snapshot_id:
raise ValueError(
"snapshot id is required to boot vm; however no "
"snapshot exists on the system.")
snapshot_id = snapshot_id[0]
if vm_host and not avail_zone:
avail_zone = 'nova'
if avail_zone and vm_host:
avail_zone = '{}:{}'.format(avail_zone, vm_host)
if user_data is None and guest_os and not re.search(
GuestImages.TIS_GUEST_PATTERN, guest_os):
# create userdata cloud init file to run right after vm
# initialization to get ip on interfaces other than eth0.
user_data = _create_cloud_init_if_conf(guest_os, nics_num=len(nics))
if user_data and user_data.startswith('~'):
user_data = user_data.replace('~', HostLinuxUser.get_home(), 1)
if file and file.startswith('~'):
file = file.replace('~', HostLinuxUser.get_home(), 1)
# create cmd
non_repeat_args = {'--flavor': flavor,
'--image': image,
'--boot-volume': volume_id,
'--snapshot': snapshot_id,
'--min-count': str(
min_count) if min_count is not None else None,
'--max-count': str(
max_count) if max_count is not None else None,
'--key-name': key_name,
'--swap': swap,
'--user-data': user_data,
'--ephemeral': ephemeral,
'--availability-zone': avail_zone,
'--file': file,
'--config-drive': str(
config_drive) if config_drive else None,
'--block-device-mapping': block_device_mapping,
'--security-groups': security_groups,
'--tags': tags,
'--poll': poll,
}
non_repeat_args = common.parse_args(non_repeat_args, repeat_arg=False,
vals_sep=',')
repeat_args = {
'--meta': meta,
'--nic': nics,
'--hint': hint,
'--block-device': block_device,
}
repeat_args = common.parse_args(repeat_args, repeat_arg=True, vals_sep=',')
pre_boot_vms = []
if not (min_count is None and max_count is None):
name_str = name + '-'
pre_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
strict=False, name=name_str)
args_ = ' '.join([non_repeat_args, repeat_args, name])
LOG.info("Booting VM {} with args: {}".format(name, args_))
exitcode, output = cli.nova('boot', positional_args=args_,
ssh_client=con_ssh, fail_ok=True,
auth_info=auth_info,
timeout=VMTimeout.BOOT_VM)
tmout = VMTimeout.STATUS_CHANGE
if min_count is None and max_count is None:
table_ = table_parser.table(output)
vm_id = table_parser.get_value_two_col_table(table_, 'id')
if cleanup and vm_id:
ResourceCleanup.add('vm', vm_id, scope=cleanup, del_vm_vols=False)
if exitcode == 1:
if vm_id:
# print out vm show for debugging purpose
cli.openstack('server show', vm_id, ssh_client=con_ssh,
auth_info=Tenant.get('admin'))
if not fail_ok:
raise exceptions.VMOperationFailed(output)
if vm_id:
return 1, vm_id, output # vm_id = '' if cli is rejected
# without vm created
return 4, '', output
LOG.info("Post action check...")
if poll and "100% complete" not in output:
message = "VM building is not 100% complete."
if fail_ok:
LOG.warning(message)
return 2, vm_id, "VM building is not 100% complete."
else:
raise exceptions.VMOperationFailed(message)
if not wait_for_vm_status(vm_id=vm_id, status=VMStatus.ACTIVE,
timeout=tmout, con_ssh=con_ssh,
auth_info=auth_info, fail_ok=True):
vm_status = \
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
auth_info=auth_info)[0]
message = "VM {} did not reach ACTIVE state within {}. VM " \
"status: {}".format(vm_id, tmout, vm_status)
if fail_ok:
LOG.warning(message)
return 3, vm_id, message
else:
raise exceptions.VMPostCheckFailed(message)
LOG.info("VM {} is booted successfully.".format(vm_id))
return 0, vm_id, 'VM is booted successfully'
else:
name_str = name + '-'
post_boot_vms = get_vms(auth_info=auth_info, con_ssh=con_ssh,
strict=False, name=name_str)
vm_ids = list(set(post_boot_vms) - set(pre_boot_vms))
if cleanup and vm_ids:
ResourceCleanup.add('vm', vm_ids, scope=cleanup, del_vm_vols=False)
if exitcode == 1:
return 1, vm_ids, output
result, vms_in_state, vms_failed_to_reach_state = wait_for_vms_values(
vm_ids, fail_ok=True, timeout=tmout,
con_ssh=con_ssh,
auth_info=Tenant.get('admin'))
if not result:
msg = "VMs failed to reach ACTIVE state: {}".format(
vms_failed_to_reach_state)
if fail_ok:
LOG.warning(msg=msg)
return 3, vm_ids, msg
LOG.info("VMs booted successfully: {}".format(vm_ids))
return 0, vm_ids, "VMs are booted successfully"
def wait_for_vm_pingable_from_natbox(vm_id, timeout=200, fail_ok=False,
con_ssh=None, use_fip=False):
"""
Wait for ping vm from natbox succeeds.
Args:
vm_id (str): id of the vm to ping
timeout (int): max retry time for pinging vm
fail_ok (bool): whether to raise exception if vm cannot be ping'd
successfully from natbox within timeout
con_ssh (SSHClient): TiS server ssh handle
use_fip (bool): whether or not to ping floating ip only if any
Returns (bool): True if ping vm succeeded, False otherwise.
"""
ping_end_time = time.time() + timeout
while time.time() < ping_end_time:
if ping_vms_from_natbox(vm_ids=vm_id, fail_ok=True, con_ssh=con_ssh,
num_pings=3, use_fip=use_fip)[0]:
# give it sometime to settle after vm booted and became pingable
time.sleep(5)
return True
else:
msg = "Ping from NatBox to vm {} failed for {} seconds.".format(vm_id,
timeout)
if fail_ok:
LOG.warning(msg)
return False
else:
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
date_format='%Y%m%d_%H-%M')
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'),
time_stamp, ProjVar.get_var('TEST_NAME'))
common.write_to_file(f_path,
"=================={}===============\n".format(
msg))
ProjVar.set_var(PING_FAILURE=True)
get_console_logs(vm_ids=vm_id, sep_file=f_path)
network_helper.collect_networking_info(vms=vm_id, sep_file=f_path,
time_stamp=time_stamp)
raise exceptions.VMNetworkError(msg)
def __merge_dict(base_dict, merge_dict):
# identical to {**base_dict, **merge_dict} in python3.6+
d = dict(base_dict) # id() will be different, making a copy
for k in merge_dict:
d[k] = merge_dict[k]
return d
def get_default_keypair(auth_info=None, con_ssh=None):
"""
Get keypair for specific tenant.
Args:
auth_info (dict): If None, default tenant will be used.
con_ssh (SSHClient):
Returns (str): key name
"""
if auth_info is None:
auth_info = Tenant.get_primary()
keypair_name = auth_info['nova_keypair']
existing_keypairs = nova_helper.get_keypairs(name=keypair_name,
con_ssh=con_ssh,
auth_info=auth_info)
if existing_keypairs:
return existing_keypairs[0]
# Assume that public key file already exists since it should have been
# set up in session config.
# In the case of public key file does not exist, there should be existing
# nova keypair, so it should not
# reach this step. Config done via setups.setup_keypair()
keyfile_stx_final = ProjVar.get_var('STX_KEYFILE_SYS_HOME')
public_key_stx = '{}.pub'.format(keyfile_stx_final)
LOG.info("Create nova keypair {} using public key {}".format(
keypair_name, public_key_stx))
nova_helper.create_keypair(keypair_name, public_key=public_key_stx,
auth_info=auth_info, con_ssh=con_ssh)
return keypair_name
def live_migrate_vm(vm_id, destination_host='', con_ssh=None,
block_migrate=None, force=None, fail_ok=False,
auth_info=Tenant.get('admin')):
"""
Args:
vm_id (str):
destination_host (str): such as compute-0, compute-1
con_ssh (SSHClient):
block_migrate (bool): whether to add '--block-migrate' to command
force (None|bool): force live migrate
fail_ok (bool): if fail_ok, return a numerical number to indicate the
execution status
One exception is if the live-migration command exit_code > 1,
which indicating the command itself may
be incorrect. In this case CLICommandFailed exception will be
thrown regardless of the fail_ok flag.
auth_info (dict):
Returns (tuple): (return_code (int), error_msg_if_migration_rejected (str))
(0, 'Live migration is successful.'):
live migration succeeded and post migration checking passed
(1, <cli stderr>): # This scenario is changed to host did not change
as excepted
live migration request rejected as expected. e.g., no available
destination host,
or live migrate a vm with block migration
(2, <cli stderr>): live migration request rejected due to unknown
reason.
(3, 'Post action check failed: VM is in ERROR state.'):
live migration command executed successfully, but VM is in Error
state after migration
(4, 'Post action check failed: VM is not in original state.'):
live migration command executed successfully, but VM is not in
before-migration-state
(5, 'Post action check failed: VM host did not change!'): (this
scenario is removed from Newton)
live migration command executed successfully, but VM is still on
the same host after migration
(6, <cli_stderr>) This happens when vote_note_to_migrate is set for
vm, or pci device is used in vm, etc
For the first two scenarios, results will be returned regardless of the
fail_ok flag.
For scenarios other than the first two, returns are only applicable if
fail_ok=True
Examples:
1) If a test case is meant to test live migration with a specific
flavor which would block the migration, the
following call can be made:
return_code, msg = live_migrate_vm(vm_id, fail_ok=True)
expected_err_str = "your error string"
assert return_code in [1, 2]
assert expected_err_str in msg
2) For a test that needs to live migrate
"""
optional_arg = ''
if block_migrate:
optional_arg += '--block-migrate'
if force:
optional_arg += '--force'
before_host = get_vm_host(vm_id, con_ssh=con_ssh)
before_status = get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
auth_info=Tenant.get('admin'))[0]
if not before_status == VMStatus.ACTIVE:
LOG.warning("Non-active VM status before live migrate: {}".format(
before_status))
extra_str = ''
if not destination_host == '':
extra_str = ' to ' + destination_host
positional_args = ' '.join(
[optional_arg.strip(), str(vm_id), destination_host]).strip()
LOG.info(
"Live migrating VM {} from {}{} started.".format(vm_id, before_host,
extra_str))
LOG.info("nova live-migration {}".format(positional_args))
# auto host/block migration selection unavailable in openstack client
exit_code, output = cli.nova('live-migration',
positional_args=positional_args,
ssh_client=con_ssh, fail_ok=fail_ok,
auth_info=auth_info)
if exit_code == 1:
return 6, output
LOG.info("Waiting for VM status change to {} with best effort".format(
VMStatus.MIGRATING))
in_mig_state = wait_for_vm_status(vm_id, status=VMStatus.MIGRATING,
timeout=60, fail_ok=True)
if not in_mig_state:
LOG.warning(
"VM did not reach {} state after triggering live-migration".format(
VMStatus.MIGRATING))
LOG.info("Waiting for VM status change to original state {}".format(
before_status))
end_time = time.time() + VMTimeout.LIVE_MIGRATE_COMPLETE
while time.time() < end_time:
time.sleep(2)
status = get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
auth_info=Tenant.get('admin'))[0]
if status == before_status:
LOG.info("Live migrate vm {} completed".format(vm_id))
break
elif status == VMStatus.ERROR:
if fail_ok:
return 3, "Post action check failed: VM is in ERROR state."
nova_helper.get_migration_list_table(con_ssh=con_ssh,
auth_info=auth_info)
raise exceptions.VMPostCheckFailed(
"VM {} is in {} state after live migration. Original state "
"before live migration is: {}".format(vm_id, VMStatus.ERROR,
before_status))
else:
if fail_ok:
return 4, "Post action check failed: VM is not in original state."
else:
nova_helper.get_migration_list_table(con_ssh=con_ssh,
auth_info=auth_info)
raise exceptions.TimeoutException(
"VM {} did not reach original state within {} seconds after "
"live migration".format(vm_id, VMTimeout.LIVE_MIGRATE_COMPLETE))
after_host = before_host
for i in range(3):
after_host = get_vm_host(vm_id, con_ssh=con_ssh)
if after_host != before_host:
break
time.sleep(3)
if before_host == after_host:
LOG.warning(
"Live migration of vm {} failed. Checking if this is expected "
"failure...".format(
vm_id))
if _is_live_migration_allowed(vm_id, vm_host=before_host,
block_migrate=block_migrate) and \
(destination_host or get_dest_host_for_live_migrate(vm_id)):
if fail_ok:
return 1, "Unknown live migration failure"
else:
nova_helper.get_migration_list_table(con_ssh=con_ssh,
auth_info=auth_info)
raise exceptions.VMPostCheckFailed(
"Unexpected failure of live migration!")
else:
LOG.debug(
"System does not allow live migrating vm {} as "
"expected.".format(
vm_id))
return 2, "Live migration failed as expected"
LOG.info(
"VM {} successfully migrated from {} to {}".format(vm_id, before_host,
after_host))
return 0, "Live migration is successful."
def _is_live_migration_allowed(vm_id, vm_host, con_ssh=None,
block_migrate=None):
vm_info = VMInfo.get_vm_info(vm_id, con_ssh=con_ssh)
storage_backing = vm_info.get_storage_type()
if not storage_backing:
storage_backing = host_helper.get_host_instance_backing(host=vm_host,
con_ssh=con_ssh)
vm_boot_from = vm_info.boot_info['type']
if storage_backing == 'local_image':
if block_migrate and vm_boot_from == 'volume' and not \
vm_info.has_local_disks():
LOG.warning(
"Live block migration is not supported for boot-from-volume "
"vm with local_image storage")
return False
return True
elif storage_backing == 'local_lvm':
if (not block_migrate) and vm_boot_from == 'volume' and not \
vm_info.has_local_disks():
return True
else:
LOG.warning(
"Live (block) migration is not supported for local_lvm vm "
"with localdisk")
return False
else:
# remote backend
if block_migrate:
LOG.warning(
"Live block migration is not supported for vm with remote "
"storage")
return False
else:
return True
def get_dest_host_for_live_migrate(vm_id, con_ssh=None):
"""
Check whether a destination host exists with following criteria:
Criteria:
1) host has same storage backing as the vm
2) host is unlocked
3) different than current host
Args:
vm_id (str):
con_ssh (SSHClient):
Returns (str): hostname for the first host found. Or '' if no proper host
found
"""
vm_info = VMInfo.get_vm_info(vm_id, con_ssh=con_ssh)
vm_storage_backing = vm_info.get_storage_type()
current_host = vm_info.get_host_name()
if not vm_storage_backing:
vm_storage_backing = host_helper.get_host_instance_backing(
host=current_host, con_ssh=con_ssh)
candidate_hosts = host_helper.get_hosts_in_storage_backing(
storage_backing=vm_storage_backing, con_ssh=con_ssh)
hosts_table_ = table_parser.table(cli.system('host-list')[1])
for host in candidate_hosts:
if not host == current_host:
host_state = table_parser.get_values(hosts_table_, 'administrative',
hostname=host)[0]
if host_state == 'unlocked':
LOG.debug(
"At least one host - {} is available for live migrating "
"vm {}".format(
host, vm_id))
return host
LOG.warning("No valid host found for live migrating vm {}".format(vm_id))
return ''
def cold_migrate_vm(vm_id, revert=False, con_ssh=None, fail_ok=False,
auth_info=Tenant.get('admin')):
"""
Cold migrate a vm and confirm/revert
Args:
vm_id (str): vm to cold migrate
revert (bool): False to confirm resize, True to revert
con_ssh (SSHClient):
fail_ok (bool): True if fail ok. Default to False, ie., throws
exception upon cold migration fail.
auth_info (dict):
Returns (tuple): (rtn_code, message)
(0, success_msg) # Cold migration and confirm/revert succeeded. VM is
back to original state or Active state.
(1, <stderr>) # cold migration cli rejected
# (2, <stderr>) # Cold migration cli command rejected. <stderr> is
the err message returned by cli cmd.
(3, <stdout>) # Cold migration cli accepted, but not finished.
<stdout> is the output of cli cmd.
(4, timeout_message] # Cold migration command ran successfully,
but timed out waiting for VM to reach
'Verify Resize' state or Error state.
(5, err_msg) # Cold migration command ran successfully, but VM is in
Error state.
(6, err_msg) # Cold migration command ran successfully, and resize
confirm/revert performed. But VM is not in
Active state after confirm/revert.
(7, err_msg) # Cold migration and resize confirm/revert ran
successfully and vm in active state. But host for vm
is not as expected. i.e., still the same host after confirm
resize, or different host after revert resize.
(8, <stderr>) # Confirm/Revert resize cli rejected
"""
before_host = get_vm_host(vm_id, con_ssh=con_ssh)
before_status = \
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh)[0]
if not before_status == VMStatus.ACTIVE:
LOG.warning("Non-active VM status before cold migrate: {}".format(
before_status))
LOG.info("Cold migrating VM {} from {}...".format(vm_id, before_host))
exitcode, output = cli.nova('migrate --poll', vm_id, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info,
timeout=VMTimeout.COLD_MIGRATE_CONFIRM)
if exitcode == 1:
return 1, output
LOG.info(
"Waiting for VM status change to {}".format(VMStatus.VERIFY_RESIZE))
vm_status = wait_for_vm_status(vm_id=vm_id,
status=[VMStatus.VERIFY_RESIZE,
VMStatus.ERROR],
timeout=300,
fail_ok=fail_ok, con_ssh=con_ssh)
if vm_status is None:
return 4, 'Timed out waiting for Error or Verify_Resize status for ' \
'VM {}'.format(vm_id)
verify_resize_str = 'Revert' if revert else 'Confirm'
if vm_status == VMStatus.VERIFY_RESIZE:
LOG.info("{}ing resize..".format(verify_resize_str))
res, out = _confirm_or_revert_resize(vm=vm_id, revert=revert,
fail_ok=fail_ok, con_ssh=con_ssh)
if res > 0:
return 8, out
elif vm_status == VMStatus.ERROR:
err_msg = "VM {} in Error state after cold migrate. {} resize is not " \
"reached.".format(vm_id, verify_resize_str)
if fail_ok:
return 5, err_msg
nova_helper.get_migration_list_table(con_ssh=con_ssh,
auth_info=auth_info)
raise exceptions.VMPostCheckFailed(err_msg)
post_confirm_state = wait_for_vm_status(
vm_id, status=VMStatus.ACTIVE,
timeout=VMTimeout.COLD_MIGRATE_CONFIRM, fail_ok=fail_ok,
con_ssh=con_ssh)
if post_confirm_state is None:
err_msg = "VM {} is not in Active state after {} Resize".format(
vm_id, verify_resize_str)
return 6, err_msg
# Process results
after_host = get_vm_host(vm_id, con_ssh=con_ssh)
host_changed = before_host != after_host
host_change_str = "changed" if host_changed else "did not change"
operation_ok = not host_changed if revert else host_changed
if not operation_ok:
err_msg = (
"VM {} host {} after {} Resize. Before host: {}. After host: {}".
format(vm_id, host_change_str, verify_resize_str, before_host,
after_host))
if fail_ok:
return 7, err_msg
nova_helper.get_migration_list_table(con_ssh=con_ssh,
auth_info=auth_info)
raise exceptions.VMPostCheckFailed(err_msg)
success_msg = "VM {} successfully cold migrated and {}ed Resize.".format(
vm_id, verify_resize_str)
LOG.info(success_msg)
return 0, success_msg
def resize_vm(vm_id, flavor_id, revert=False, con_ssh=None, fail_ok=False,
auth_info=Tenant.get('admin')):
"""
Resize vm to given flavor
Args:
vm_id (str):
flavor_id (str): flavor to resize to
revert (bool): True to revert resize, else confirm resize
con_ssh (SSHClient):
fail_ok (bool):
auth_info (dict):
Returns (tuple): (rtn_code, msg)
(0, "VM <vm_id> successfully resized and confirmed/reverted.")
(1, <std_err>) # resize cli rejected
(2, "Timed out waiting for Error or Verify_Resize status for VM
<vm_id>")
(3, "VM <vm_id> in Error state after resizing. VERIFY_RESIZE is not
reached.")
(4, "VM <vm_id> is not in Active state after confirm/revert Resize")
(5, "Flavor is changed after revert resizing.")
(6, "VM flavor is not changed to expected after resizing.")
"""
before_flavor = get_vm_flavor(vm_id, con_ssh=con_ssh)
before_status = \
get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh)[0]
if not before_status == VMStatus.ACTIVE:
LOG.warning("Non-active VM status before cold migrate: {}".format(
before_status))
LOG.info("Resizing VM {} to flavor {}...".format(vm_id, flavor_id))
args = '--wait --flavor {} {}'.format(flavor_id, vm_id)
exitcode, output = cli.openstack('server resize', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info,
timeout=VMTimeout.COLD_MIGRATE_CONFIRM)
if exitcode > 0:
return 1, output
LOG.info(
"Waiting for VM status change to {}".format(VMStatus.VERIFY_RESIZE))
vm_status = wait_for_vm_status(vm_id=vm_id,
status=[VMStatus.VERIFY_RESIZE,
VMStatus.ERROR],
fail_ok=fail_ok,
timeout=300, con_ssh=con_ssh)
if vm_status is None:
err_msg = 'Timed out waiting for Error or Verify_Resize status for ' \
'VM {}'.format(vm_id)
LOG.error(err_msg)
return 2, err_msg
verify_resize_str = 'Revert' if revert else 'Confirm'
if vm_status == VMStatus.VERIFY_RESIZE:
LOG.info("{}ing resize..".format(verify_resize_str))
_confirm_or_revert_resize(vm=vm_id, revert=revert, con_ssh=con_ssh)
elif vm_status == VMStatus.ERROR:
err_msg = "VM {} in Error state after resizing. {} is not " \
"reached.".format(vm_id, VMStatus.VERIFY_RESIZE)
if fail_ok:
LOG.error(err_msg)
return 3, err_msg
raise exceptions.VMPostCheckFailed(err_msg)
post_confirm_state = wait_for_vm_status(
vm_id, status=VMStatus.ACTIVE, timeout=VMTimeout.COLD_MIGRATE_CONFIRM,
fail_ok=fail_ok, con_ssh=con_ssh)
if post_confirm_state is None:
err_msg = "VM {} is not in Active state after {} Resize".format(
vm_id, verify_resize_str)
LOG.error(err_msg)
return 4, err_msg
after_flavor = get_vm_flavor(vm_id)
if revert and after_flavor != before_flavor:
err_msg = "Flavor is changed after revert resizing. Before flavor: " \
"{}, after flavor: {}".format(before_flavor, after_flavor)
if fail_ok:
LOG.error(err_msg)
return 5, err_msg
raise exceptions.VMPostCheckFailed(err_msg)
if not revert and after_flavor != flavor_id:
err_msg = "VM flavor is not changed to expected after resizing. " \
"Before flavor: {}, after flavor: {}".\
format(flavor_id, before_flavor, after_flavor)
if fail_ok:
LOG.error(err_msg)
return 6, err_msg
raise exceptions.VMPostCheckFailed(err_msg)
success_msg = "VM {} successfully resized and {}ed.".format(
vm_id, verify_resize_str)
LOG.info(success_msg)
return 0, success_msg
def wait_for_vm_values(vm_id, timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
fail_ok=True, strict=True,
regex=False, con_ssh=None, auth_info=None, **kwargs):
"""
Wait for vm to reach given states.
Args:
vm_id (str): vm id
timeout (int): in seconds
check_interval (int): in seconds
fail_ok (bool): whether to return result or raise exception when vm
did not reach expected value(s).
strict (bool): whether to perform strict search(match) for the value(s)
For regular string: if True, match the whole string; if False,
find any substring match
For regex: if True, match from start of the value string; if
False, search anywhere of the value string
regex (bool): whether to use regex to find matching value(s)
con_ssh (SSHClient):
auth_info (dict):
**kwargs: field/value pair(s) to identify the waiting criteria.
Returns (tuple): (result(bool), actual_vals(dict))
"""
if not kwargs:
raise ValueError("No field/value pair is passed via kwargs")
LOG.info("Waiting for vm to reach state(s): {}".format(kwargs))
fields_to_check = list(kwargs.keys())
results = {}
end_time = time.time() + timeout
while time.time() < end_time:
actual_vals = get_vm_values(vm_id=vm_id, con_ssh=con_ssh,
auth_info=auth_info,
fields=fields_to_check)
for i in range(len(fields_to_check)):
field = fields_to_check[i]
expt_vals = kwargs[field]
actual_val = actual_vals[i]
results[field] = actual_val
if not isinstance(expt_vals, list):
expt_vals = [expt_vals]
for expt_val in expt_vals:
if regex:
match_found = re.match(expt_val,
actual_val) if strict else re.search(
expt_val, actual_val)
else:
match_found = expt_val == actual_val if strict else \
expt_val in actual_val
if match_found:
fields_to_check.remove(field)
if not fields_to_check:
LOG.info("VM has reached states: {}".format(results))
return True, results
time.sleep(check_interval)
msg = "VM {} did not reach expected states within timeout. Actual state(" \
"s): {}".format(vm_id, results)
if fail_ok:
LOG.warning(msg)
return False, results
else:
raise exceptions.VMTimeout(msg)
def wait_for_vm_status(vm_id, status=VMStatus.ACTIVE,
timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
fail_ok=False,
con_ssh=None, auth_info=Tenant.get('admin')):
"""
Args:
vm_id:
status (list|str):
timeout:
check_interval:
fail_ok (bool):
con_ssh:
auth_info:
Returns: The Status of the vm_id depend on what Status it is looking for
"""
end_time = time.time() + timeout
if isinstance(status, str):
status = [status]
current_status = get_vm_values(vm_id, 'status', strict=True,
con_ssh=con_ssh, auth_info=auth_info)[0]
while time.time() < end_time:
for expected_status in status:
if current_status == expected_status:
LOG.info("VM status has reached {}".format(expected_status))
return expected_status
time.sleep(check_interval)
current_status = get_vm_values(vm_id, 'status', strict=True,
con_ssh=con_ssh, auth_info=auth_info)[0]
err_msg = "Timed out waiting for vm status: {}. Actual vm status: " \
"{}".format(status, current_status)
if fail_ok:
LOG.warning(err_msg)
return None
else:
raise exceptions.VMTimeout(err_msg)
def _confirm_or_revert_resize(vm, revert=False, con_ssh=None, fail_ok=False):
args = '--revert' if revert else '--confirm'
args = '{} {}'.format(args, vm)
return cli.openstack('server resize', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=Tenant.get('admin'))
def _get_vms_ips(vm_ids, net_types='mgmt', exclude_nets=None, con_ssh=None,
vshell=False):
if isinstance(net_types, str):
net_types = [net_types]
if isinstance(vm_ids, str):
vm_ids = [vm_ids]
valid_net_types = ['mgmt', 'data', 'internal', 'external']
if not set(net_types) <= set(valid_net_types):
raise ValueError(
"Invalid net type(s) provided. Valid net_types: {}. net_types "
"given: {}".
format(valid_net_types, net_types))
vms_ips = []
vshell_ips_dict = dict(data=[], internal=[])
if 'mgmt' in net_types:
mgmt_ips = network_helper.get_mgmt_ips_for_vms(
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
if not mgmt_ips:
raise exceptions.VMNetworkError(
"Management net ip is not found for vms {}".format(vm_ids))
vms_ips += mgmt_ips
if 'external' in net_types:
ext_ips = network_helper.get_external_ips_for_vms(
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
if not ext_ips:
raise exceptions.VMNetworkError(
"No external network ip found for vms {}".format(vm_ids))
vms_ips += ext_ips
if 'data' in net_types:
data_ips = network_helper.get_tenant_ips_for_vms(
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
if not data_ips:
raise exceptions.VMNetworkError(
"Data network ip is not found for vms {}".format(vm_ids))
if vshell:
vshell_ips_dict['data'] = data_ips
else:
vms_ips += data_ips
if 'internal' in net_types:
internal_ips = network_helper.get_internal_ips_for_vms(
vms=vm_ids, con_ssh=con_ssh, exclude_nets=exclude_nets)
if not internal_ips:
raise exceptions.VMNetworkError(
"Internal net ip is not found for vms {}".format(vm_ids))
if vshell:
vshell_ips_dict['internal'] = internal_ips
else:
vms_ips += internal_ips
return vms_ips, vshell_ips_dict
def _ping_vms(ssh_client, vm_ids=None, con_ssh=None, num_pings=5, timeout=15,
fail_ok=False, net_types='mgmt', retry=3,
retry_interval=3, vshell=False, sep_file=None,
source_net_types=None):
"""
Args:
vm_ids (list|str): list of vms to ping
ssh_client (SSHClient): ping from this ssh client. Usually a natbox'
ssh client or another vm's ssh client
con_ssh (SSHClient): active controller ssh client to run cli command
to get all the management ips
num_pings (int): number of pings to send
timeout (int): timeout waiting for response of ping messages in seconds
fail_ok (bool): Whether it's okay to have 100% packet loss rate.
sep_file (str|None)
net_types (str|list|tuple)
source_net_types (str|list|tuple|None):
vshell specific
None: use the same net_type s as the target IPs'
str: use the specified net_type for all target IPs
tuple: (net_type_data, net_type_internal)
use net_type_data for data IPs
use net_type_internal for internal IPs
list: same as tuple
Returns (tuple): (res (bool), packet_loss_dict (dict))
Packet loss rate dictionary format:
{
ip1: packet_loss_percentile1,
ip2: packet_loss_percentile2,
...
}
"""
vms_ips, vshell_ips_dict = _get_vms_ips(vm_ids=vm_ids, net_types=net_types,
con_ssh=con_ssh, vshell=vshell)
res_bool = False
res_dict = {}
for i in range(retry + 1):
for ip in vms_ips:
packet_loss_rate = network_helper.ping_server(
server=ip, ssh_client=ssh_client, num_pings=num_pings,
timeout=timeout, fail_ok=True, vshell=False)[0]
res_dict[ip] = packet_loss_rate
for net_type, vshell_ips in vshell_ips_dict.items():
if source_net_types is None:
pass
elif isinstance(source_net_types, str):
net_type = source_net_types
else:
net_type_data, net_type_internal = source_net_types
if net_type == 'data':
net_type = net_type_data
elif net_type == 'internal':
net_type = net_type_internal
else:
raise ValueError(net_type)
for vshell_ip in vshell_ips:
packet_loss_rate = network_helper.ping_server(
server=vshell_ip, ssh_client=ssh_client,
num_pings=num_pings, timeout=timeout, fail_ok=True,
vshell=True, net_type=net_type)[0]
res_dict[vshell_ip] = packet_loss_rate
res_bool = not any(loss_rate == 100 for loss_rate in res_dict.values())
if res_bool:
LOG.info(
"Ping successful from {}: {}".format(ssh_client.host, res_dict))
return res_bool, res_dict
if i < retry:
LOG.info("Retry in {} seconds".format(retry_interval))
time.sleep(retry_interval)
if not res_dict:
raise ValueError("Ping res dict contains no result.")
err_msg = "Ping unsuccessful from vm (logged in via {}): {}".format(
ssh_client.host, res_dict)
if fail_ok:
LOG.info(err_msg)
return res_bool, res_dict
else:
if sep_file:
msg = "==========================Ping unsuccessful from vm to " \
"vms===================="
common.write_to_file(
sep_file,
content="{}\nLogged into vm via {}. Result: {}".format(
msg, ssh_client.host, res_dict))
raise exceptions.VMNetworkError(err_msg)
def configure_vm_vifs_on_same_net(vm_id, vm_ips=None, ports=None,
vm_prompt=None, restart_service=True,
reboot=False):
"""
Configure vm routes if the vm has multiple vifs on same network.
Args:
vm_id (str):
vm_ips (str|list): ips for specific vifs. Only works if vifs are up
with ips assigned
ports (list of dict): vm ports to configure.
vm_prompt (None|str)
restart_service
reboot
Returns:
"""
if isinstance(vm_ips, str):
vm_ips = [vm_ips]
vnics_info = {}
if ports:
LOG.info("Get vm interfaces' mac and ip addressess")
if isinstance(ports, str):
ports = [ports]
vm_interfaces_table = table_parser.table(
cli.openstack('port list', '--server {}'.format(vm_id))[1])
vm_interfaces_dict = table_parser.row_dict_table(
table_=vm_interfaces_table, key_header='ID')
for i in range(len(ports)):
port_id = ports[i]
vif_info = vm_interfaces_dict[port_id]
vif_ip = vif_info['fixed ip addresses']
if vif_ip and 'ip_address' in vif_ip:
vif_ip = \
re.findall("ip_address='(.*)'", vif_ip.split(sep=',')[0])[0]
else:
if not vm_ips:
raise ValueError(
"vm_ips for matching vnics has to be provided for "
"ports without ip address "
"listed in neutron port-list")
vif_ip = vm_ips[i]
cidr = vif_ip.rsplit('.', maxsplit=1)[0] + '.0/24'
vif_mac = vif_info['mac address']
vnics_info[vif_mac] = (cidr, vif_ip)
LOG.info("Configure vm routes if the vm has multiple vifs on same network.")
with ssh_to_vm_from_natbox(vm_id=vm_id, prompt=vm_prompt) as vm_ssh:
vifs_to_conf = {}
if not ports:
extra_grep = '| grep --color=never -E "{}"'.format(
'|'.join(vm_ips)) if vm_ips else ''
kernel_routes = vm_ssh.exec_cmd(
'ip route | grep --color=never "proto kernel" {}'.format(
extra_grep))[1]
cidr_dict = {}
for line in kernel_routes.splitlines():
found = re.findall(
r'^(.*/\d+)\sdev\s(.*)\sproto kernel.*\ssrc\s(.*)$', line)
cidr, dev_name, dev_ip = found[0]
if cidr not in cidr_dict:
cidr_dict[cidr] = []
cidr_dict[cidr].append((dev_name, dev_ip))
for cidr_, val in cidr_dict.items():
if not vm_ips:
val = val[1:]
for eth_info in val:
dev_name, dev_ip = eth_info
vifs_to_conf[dev_name] = \
(cidr_, dev_ip, 'stxauto_{}'.format(dev_name))
if not vifs_to_conf:
LOG.info(
"Did not find multiple vifs on same subnet. Do nothing.")
else:
for mac_addr in vnics_info:
dev_name = network_helper.get_eth_for_mac(vm_ssh,
mac_addr=mac_addr)
cidr_, dev_ip = vnics_info[mac_addr]
vifs_to_conf[dev_name] = (
cidr_, dev_ip, 'stxauto_{}'.format(dev_name))
used_tables = vm_ssh.exec_cmd(
'grep --color=never -E "^[0-9]" {}'.format(VMPath.RT_TABLES))[1]
used_tables = [int(re.split(r'[\s\t]', line_)[0].strip()) for line_ in
used_tables.splitlines()]
start_range = 110
for eth_name, eth_info in vifs_to_conf.items():
cidr_, vif_ip, table_name = eth_info
exiting_tab = vm_ssh.exec_cmd(
'grep --color=never {} {}'.format(table_name,
VMPath.RT_TABLES))[1]
if not exiting_tab:
for i in range(start_range, 250):
if i not in used_tables:
LOG.info(
"Append new routing table {} to rt_tables".
format(table_name))
vm_ssh.exec_sudo_cmd(
'echo "{} {}" >> {}'.format(i, table_name,
VMPath.RT_TABLES))
start_range = i + 1
break
else:
raise ValueError(
"Unable to get a valid table number to create route "
"for {}".format(eth_name))
LOG.info(
"Update arp_filter, arp_announce, route and rule scripts for "
"vm {} {}".format(vm_id, eth_name))
vm_ssh.exec_sudo_cmd(
'echo 2 > {}'.format(VMPath.ETH_ARP_ANNOUNCE.format(eth_name)))
vm_ssh.exec_sudo_cmd(
'echo 1 > {}'.format(VMPath.ETH_ARP_FILTER.format(eth_name)))
route = '{} dev {} proto kernel scope link src {} table {}'.format(
cidr_, eth_name, vif_ip, table_name)
vm_ssh.exec_sudo_cmd('echo "{}" > {}'.format(
route, VMPath.ETH_RT_SCRIPT.format(eth_name)))
rule = 'table {} from {}'.format(table_name, vif_ip)
vm_ssh.exec_sudo_cmd('echo "{}" > {}'.format(
rule, VMPath.ETH_RULE_SCRIPT.format(eth_name)))
if restart_service and not reboot:
LOG.info("Restart network service after configure vm routes")
vm_ssh.exec_sudo_cmd('systemctl restart network',
expect_timeout=120, get_exit_code=False)
# vm_ssh.exec_cmd('ip addr')
if reboot:
LOG.info("Reboot vm after configure vm routes")
reboot_vm(vm_id=vm_id)
def cleanup_routes_for_vifs(vm_id, vm_ips, rm_ifcfg=True, restart_service=True,
reboot=False):
"""
Cleanup the configured routes for specified vif(s). This is needed when a
vif is detached from a vm.
Args:
vm_id:
vm_ips:
rm_ifcfg
restart_service
reboot
Returns:
"""
with ssh_to_vm_from_natbox(vm_id=vm_id) as vm_ssh:
if isinstance(vm_ips, str):
vm_ips = [vm_ips]
for vm_ip in vm_ips:
LOG.info("Clean up route for dev with ip {}".format(vm_ip))
route = vm_ssh.exec_sudo_cmd(
'grep --color=never {} {}'.format(
vm_ip, VMPath.ETH_RT_SCRIPT.format('*')))[1]
if not route:
continue
pattern = '(.*) dev (.*) proto kernel .* src {} table (.*)'.format(
vm_ip)
found = re.findall(pattern, route)
if found:
cidr, eth_name, table_name = found[0]
LOG.info(
"Update arp_filter, arp_announce, route and rule scripts "
"for vm {} {}".format(vm_id, eth_name))
# vm_ssh.exec_sudo_cmd('rm -f {}'.format(
# VMPath.ETH_ARP_ANNOUNCE.format(eth_name)))
# vm_ssh.exec_sudo_cmd('rm -f {}'.format(
# VMPath.ETH_ARP_FILTER.format(eth_name)))
vm_ssh.exec_sudo_cmd(
'rm -f {}'.format(VMPath.ETH_RULE_SCRIPT.format(eth_name)))
vm_ssh.exec_sudo_cmd(
'rm -f {}'.format(VMPath.ETH_RT_SCRIPT.format(eth_name)))
vm_ssh.exec_sudo_cmd("sed -n -i '/{}/!p' {}".format(
table_name, VMPath.RT_TABLES))
if rm_ifcfg:
vm_ssh.exec_sudo_cmd('rm -f {}'.format(
VMPath.ETH_PATH_CENTOS.format(eth_name)))
if restart_service and not reboot:
LOG.info("Restart network service")
vm_ssh.exec_sudo_cmd('systemctl restart network',
get_exit_code=False, expect_timeout=60)
if reboot:
reboot_vm(vm_id=vm_id)
def ping_vms_from_natbox(vm_ids=None, natbox_client=None, con_ssh=None,
num_pings=5, timeout=30, fail_ok=False,
use_fip=False, retry=0):
"""
Args:
vm_ids: vms to ping. If None, all vms will be ping'd.
con_ssh (SSHClient): active controller client to retrieve the vm info
natbox_client (NATBoxClient): ping vms from this client
num_pings (int): number of pings to send
timeout (int): timeout waiting for response of ping messages in seconds
fail_ok (bool): When False, test will stop right away if one ping
failed. When True, test will continue to ping
the rest of the vms and return results even if pinging one vm
failed.
use_fip (bool): Whether to ping floating ip only if a vm has more
than one management ips
retry (int): number of times to retry if ping fails
Returns (tuple): (res (bool), packet_loss_dict (dict))
Packet loss rate dictionary format:
{
ip1: packet_loss_percentile1,
ip2: packet_loss_percentile2,
...
}
"""
if isinstance(vm_ids, str):
vm_ids = [vm_ids]
if not natbox_client:
natbox_client = NATBoxClient.get_natbox_client()
if not con_ssh:
con_ssh = ControllerClient.get_active_controller()
net_type = 'external' if use_fip else 'mgmt'
res_bool, res_dict = _ping_vms(ssh_client=natbox_client, vm_ids=vm_ids,
con_ssh=con_ssh, num_pings=num_pings,
timeout=timeout, fail_ok=True,
net_types=net_type, retry=retry,
vshell=False)
if not res_bool and not fail_ok:
msg = "==================Ping vm(s) from NatBox failed - Collecting " \
"extra information==============="
LOG.error(msg)
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
date_format='%Y%m%d_%H-%M')
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'),
time_stamp, ProjVar.get_var("TEST_NAME"))
common.write_to_file(file_path=f_path,
content="\n{}\nResult(s): {}\n".format(msg,
res_dict))
ProjVar.set_var(PING_FAILURE=True)
get_console_logs(vm_ids=vm_ids, sep_file=f_path)
network_helper.collect_networking_info(vms=vm_ids, sep_file=f_path,
time_stamp=time_stamp,
con_ssh=con_ssh)
raise exceptions.VMNetworkError(
"Ping failed from NatBox. Details: {}".format(res_dict))
return res_bool, res_dict
def get_console_logs(vm_ids, length=None, con_ssh=None, sep_file=None):
"""
Get console logs for given vm(s)
Args:
vm_ids (str|list):
length (int|None): how many lines to tail
con_ssh:
sep_file (str|None): write vm console logs to given sep_file if
specified.
Returns (dict): {<vm1_id>: <vm1_console>, <vm2_id>: <vm2_console>, ...}
"""
if isinstance(vm_ids, str):
vm_ids = [vm_ids]
vm_ids = list(set(vm_ids))
console_logs = {}
args = '--lines={} '.format(length) if length else ''
content = ''
for vm_id in vm_ids:
vm_args = '{}{}'.format(args, vm_id)
output = cli.openstack('console log show', vm_args, ssh_client=con_ssh,
auth_info=Tenant.get('admin'))[1]
console_logs[vm_id] = output
content += "\n#### Console log for vm {} ####\n{}\n".format(vm_id,
output)
if sep_file:
common.write_to_file(sep_file, content=content)
return console_logs
def ping_vms_from_vm(to_vms=None, from_vm=None, user=None, password=None,
prompt=None, con_ssh=None, natbox_client=None,
num_pings=5, timeout=120, fail_ok=False, from_vm_ip=None,
from_fip=False, net_types='mgmt',
retry=3, retry_interval=5, vshell=False,
source_net_types=None):
"""
Args:
from_vm (str):
to_vms (str|list|None):
user (str):
password (str):
prompt (str):
con_ssh (SSHClient):
natbox_client (SSHClient):
num_pings (int):
timeout (int): max number of seconds to wait for ssh connection to
from_vm
fail_ok (bool): When False, test will stop right away if one ping
failed. When True, test will continue to ping
the rest of the vms and return results even if pinging one vm
failed.
from_vm_ip (str): vm ip to ssh to if given. from_fip flag will be
considered only if from_vm_ip=None
from_fip (bool): whether to ssh to vm's floating ip if it has
floating ip associated with it
net_types (list|str|tuple): 'mgmt', 'data', or 'internal'
retry (int): number of times to retry
retry_interval (int): seconds to wait between each retries
vshell (bool): whether to ping vms' data interface through internal
interface.
Usage: when set to True, use 'vshell ping --count 3
<other_vm_data_ip> <internal_if_id>'
- dpdk vms should be booted from lab_setup scripts
source_net_types (str|list|tuple|None):
vshell specific
None: use the same net_type s as the target IPs'
str: use the specified net_type for all target IPs
tuple: (net_type_data, net_type_internal)
use net_type_data for data IPs
use net_type_internal for internal IPs
list: same as tuple
Returns (tuple):
A tuple in form: (res (bool), packet_loss_dict (dict))
Packet loss rate dictionary format:
{
ip1: packet_loss_percentile1,
ip2: packet_loss_percentile2,
...
}
"""
if isinstance(net_types, str):
net_types = [net_types]
if from_vm is None or to_vms is None:
vms_ips = network_helper.get_mgmt_ips_for_vms(con_ssh=con_ssh,
rtn_dict=True)
if not vms_ips:
raise exceptions.NeutronError("No management ip found for any vms")
vms_ids = list(vms_ips.keys())
if from_vm is None:
from_vm = random.choice(vms_ids)
if to_vms is None:
to_vms = vms_ids
if isinstance(to_vms, str):
to_vms = [to_vms]
if not isinstance(from_vm, str):
raise ValueError("from_vm is not a string: {}".format(from_vm))
assert from_vm and to_vms, "from_vm: {}, to_vms: {}".format(from_vm, to_vms)
time_stamp = common.get_date_in_format(ssh_client=con_ssh,
date_format='%Y%m%d_%H-%M')
f_path = '{}/{}-{}'.format(ProjVar.get_var('PING_FAILURE_DIR'), time_stamp,
ProjVar.get_var('TEST_NAME'))
try:
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user,
password=password,
natbox_client=natbox_client,
prompt=prompt, con_ssh=con_ssh,
vm_ip=from_vm_ip, use_fip=from_fip,
retry_timeout=300) as from_vm_ssh:
res = _ping_vms(ssh_client=from_vm_ssh, vm_ids=to_vms,
con_ssh=con_ssh, num_pings=num_pings,
timeout=timeout, fail_ok=fail_ok,
net_types=net_types, retry=retry,
retry_interval=retry_interval, vshell=vshell,
sep_file=f_path,
source_net_types=source_net_types)
return res
except (exceptions.TiSError, pexpect.ExceptionPexpect):
ProjVar.set_var(PING_FAILURE=True)
collect_to_vms = False if list(to_vms) == [from_vm] else True
get_console_logs(vm_ids=from_vm, length=20, sep_file=f_path)
if collect_to_vms:
get_console_logs(vm_ids=to_vms, sep_file=f_path)
network_helper.collect_networking_info(vms=to_vms, sep_file=f_path,
time_stamp=time_stamp)
try:
LOG.warning(
"Ping vm(s) from vm failed - Attempt to ssh to from_vm and "
"collect vm networking info")
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user,
password=password,
natbox_client=natbox_client,
prompt=prompt, con_ssh=con_ssh,
vm_ip=from_vm_ip,
use_fip=from_fip) as from_vm_ssh:
_collect_vm_networking_info(vm_ssh=from_vm_ssh, sep_file=f_path,
vm_id=from_vm)
if collect_to_vms:
LOG.warning(
"Ping vm(s) from vm failed - Attempt to ssh to to_vms and "
"collect vm networking info")
for vm_ in to_vms:
with ssh_to_vm_from_natbox(vm_, retry=False,
con_ssh=con_ssh) as to_ssh:
_collect_vm_networking_info(to_ssh, sep_file=f_path,
vm_id=vm_)
except (exceptions.TiSError, pexpect.ExceptionPexpect):
pass
raise
def _collect_vm_networking_info(vm_ssh, sep_file=None, vm_id=None):
vm = vm_id if vm_id else ''
content = '#### VM network info collected when logged into vm {}via {} ' \
'####'.format(vm, vm_ssh.host)
for cmd in ('ip addr', 'ip neigh', 'ip route'):
output = vm_ssh.exec_cmd(cmd, get_exit_code=False)[1]
content += '\nSent: {}\nOutput:\n{}\n'.format(cmd, output)
if sep_file:
common.write_to_file(sep_file, content=content)
def ping_ext_from_vm(from_vm, ext_ip=None, user=None, password=None,
prompt=None, con_ssh=None, natbox_client=None,
num_pings=5, timeout=30, fail_ok=False, vm_ip=None,
use_fip=False):
if ext_ip is None:
ext_ip = EXT_IP
with ssh_to_vm_from_natbox(vm_id=from_vm, username=user, password=password,
natbox_client=natbox_client,
prompt=prompt, con_ssh=con_ssh, vm_ip=vm_ip,
use_fip=use_fip) as from_vm_ssh:
from_vm_ssh.exec_cmd('ip addr', get_exit_code=False)
return network_helper.ping_server(ext_ip, ssh_client=from_vm_ssh,
num_pings=num_pings,
timeout=timeout, fail_ok=fail_ok)[0]
def scp_to_vm_from_natbox(vm_id, source_file, dest_file, timeout=60,
validate=True, natbox_client=None, sha1sum=None):
"""
scp a file to a vm from natbox
the file must be located in the natbox
the natbox must has connectivity to the VM
Args:
vm_id (str): vm to scp to
source_file (str): full pathname to the source file
dest_file (str): destination full pathname in the VM
timeout (int): scp timeout
validate (bool): verify src and dest sha1sum
natbox_client (NATBoxClient|None):
sha1sum (str|None): validates the source file prior to operation,
or None, only checked if validate=True
Returns (None):
"""
if natbox_client is None:
natbox_client = NATBoxClient.get_natbox_client()
LOG.info("scp-ing from {} to VM {}".format(natbox_client.host, vm_id))
tmp_loc = '/tmp'
fname = os.path.basename(os.path.normpath(source_file))
# ensure source file exists
natbox_client.exec_cmd('test -f {}'.format(source_file), fail_ok=False)
# calculate sha1sum
src_sha1 = None
if validate:
src_sha1 = natbox_client.exec_cmd('sha1sum {}'.format(source_file),
fail_ok=False)[1]
src_sha1 = src_sha1.split(' ')[0]
LOG.info("src: {}, sha1sum: {}".format(source_file, src_sha1))
if sha1sum is not None and src_sha1 != sha1sum:
raise ValueError(
"src sha1sum validation failed {} != {}".format(src_sha1,
sha1sum))
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
vm_ssh.exec_cmd('mkdir -p {}'.format(tmp_loc))
vm_ssh.scp_on_dest(natbox_client.user, natbox_client.host, source_file,
'/'.join([tmp_loc, fname]), natbox_client.password,
timeout=timeout)
# `mv $s $d` fails if $s == $d
if os.path.normpath(os.path.join(tmp_loc, fname)) != os.path.normpath(
dest_file):
vm_ssh.exec_sudo_cmd(
'mv -f {} {}'.format('/'.join([tmp_loc, fname]), dest_file),
fail_ok=False)
# ensure destination file exists
vm_ssh.exec_sudo_cmd('test -f {}'.format(dest_file), fail_ok=False)
# validation
if validate:
dest_sha1 = vm_ssh.exec_sudo_cmd(
'sha1sum {}'.format(dest_file), fail_ok=False)[1]
dest_sha1 = dest_sha1.split(' ')[0]
LOG.info("dst: {}, sha1sum: {}".format(dest_file, dest_sha1))
if src_sha1 != dest_sha1:
raise ValueError(
"dst sha1sum validation failed {} != {}".format(src_sha1,
dest_sha1))
LOG.info("scp completed successfully")
def scp_to_vm(vm_id, source_file, dest_file, timeout=60, validate=True,
source_ssh=None, natbox_client=None):
"""
scp a file from any SSHClient to a VM
since not all SSHClient's has connectivity to the VM, this function scps
the source file to natbox first
Args:
vm_id (str): vm to scp to
source_file (str): full pathname to the source file
dest_file (str): destination path in the VM
timeout (int): scp timeout
validate (bool): verify src and dest sha1sum
source_ssh (SSHClient|None): the source ssh session, or None to use
'localhost'
natbox_client (NATBoxClient|None):
Returns (None):
"""
if not natbox_client:
natbox_client = NATBoxClient.get_natbox_client()
close_source = False
if not source_ssh:
source_ssh = LocalHostClient()
source_ssh.connect()
close_source = True
try:
# scp-ing from natbox, forward the call
if source_ssh.host == natbox_client.host:
return scp_to_vm_from_natbox(vm_id, source_file, dest_file, timeout,
validate, natbox_client=natbox_client)
LOG.info("scp-ing from {} to natbox {}".format(source_ssh.host,
natbox_client.host))
tmp_loc = '~'
fname = os.path.basename(os.path.normpath(source_file))
# ensure source file exists
source_ssh.exec_cmd('test -f {}'.format(source_file), fail_ok=False)
# calculate sha1sum
if validate:
src_sha1 = source_ssh.exec_cmd('sha1sum {}'.format(source_file),
fail_ok=False)[1]
src_sha1 = src_sha1.split(' ')[0]
LOG.info("src: {}, sha1sum: {}".format(source_file, src_sha1))
else:
src_sha1 = None
# scp to natbox
# natbox_client.exec_cmd('mkdir -p {}'.format(tmp_loc))
source_ssh.scp_on_source(
source_file, natbox_client.user, natbox_client.host, tmp_loc,
natbox_client.password, timeout=timeout)
return scp_to_vm_from_natbox(
vm_id, '/'.join([tmp_loc, fname]), dest_file, timeout, validate,
natbox_client=natbox_client, sha1sum=src_sha1)
finally:
if close_source:
source_ssh.close()
@contextmanager
def ssh_to_vm_from_natbox(vm_id, vm_image_name=None, username=None,
password=None, prompt=None,
timeout=VMTimeout.SSH_LOGIN, natbox_client=None,
con_ssh=None, vm_ip=None,
vm_ext_port=None, use_fip=False, retry=True,
retry_timeout=120, close_ssh=True,
auth_info=Tenant.get('admin')):
"""
ssh to a vm from natbox.
Args:
vm_id (str): vm to ssh to
vm_image_name (str): such as cgcs-guest, tis-centos-guest, ubuntu_14
username (str):
password (str):
prompt (str):
timeout (int):
natbox_client (NATBoxClient):
con_ssh (SSHClient): ssh connection to TiS active controller
vm_ip (str): ssh to this ip from NatBox if given
vm_ext_port (str): port forwarding rule external port. If given this
port will be used. vm_ip must be external
router ip address.
use_fip (bool): Whether to ssh to floating ip if a vm has one
associated. Not applicable if vm_ip is given.
retry (bool): whether or not to retry if fails to connect
retry_timeout (int): max time to retry
close_ssh
auth_info (dict|None)
Yields (VMSSHClient):
ssh client of the vm
Examples:
with ssh_to_vm_from_natbox(vm_id=<id>) as vm_ssh:
vm_ssh.exec_cmd(cmd)
"""
if vm_image_name is None:
vm_image_name = get_vm_image_name(vm_id=vm_id, con_ssh=con_ssh,
auth_info=auth_info).strip().lower()
if vm_ip is None:
if use_fip:
vm_ip = network_helper.get_external_ips_for_vms(
vms=vm_id, con_ssh=con_ssh, auth_info=auth_info)[0]
else:
vm_ip = network_helper.get_mgmt_ips_for_vms(
vms=vm_id, con_ssh=con_ssh, auth_info=auth_info)[0]
if not natbox_client:
natbox_client = NATBoxClient.get_natbox_client()
try:
vm_ssh = VMSSHClient(natbox_client=natbox_client, vm_ip=vm_ip,
vm_ext_port=vm_ext_port,
vm_img_name=vm_image_name, user=username,
password=password, prompt=prompt,
timeout=timeout, retry=retry,
retry_timeout=retry_timeout)
except (exceptions.TiSError, pexpect.ExceptionPexpect):
LOG.warning(
'Failed to ssh to VM {}! Collecting vm console log'.format(vm_id))
get_console_logs(vm_ids=vm_id)
raise
try:
yield vm_ssh
finally:
if close_ssh:
vm_ssh.close()
def get_vm_pid(instance_name, host_ssh):
"""
Get instance pid on its host.
Args:
instance_name: instance name of a vm
host_ssh: ssh for the host of the given instance
Returns (str): pid of a instance on its host
"""
code, vm_pid = host_ssh.exec_sudo_cmd(
"ps aux | grep --color='never' {} | grep -v grep | awk '{{print $2}}'".
format(instance_name))
if code != 0:
raise exceptions.SSHExecCommandFailed(
"Failed to get pid for vm: {}".format(instance_name))
if not vm_pid:
LOG.warning("PID for {} is not found on host!".format(instance_name))
return vm_pid
class VMInfo:
"""
class for storing and retrieving information for specific VM using
openstack admin.
Notes: Do not use this class for vm actions, such as boot, delete,
migrate, etc as these actions should be done by
tenants.
"""
__instances = {}
active_controller_ssh = None
def __init__(self, vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
"""
Args:
vm_id:
con_ssh: floating controller ssh for the system
Returns:
"""
if con_ssh is None:
con_ssh = ControllerClient.get_active_controller()
VMInfo.active_controller_ssh = con_ssh
self.vm_id = vm_id
self.con_ssh = con_ssh
self.auth_info = auth_info
self.initial_table_ = table_parser.table(
cli.openstack('server show', vm_id, ssh_client=con_ssh,
auth_info=self.auth_info, timeout=60)[1])
self.table_ = self.initial_table_
self.name = table_parser.get_value_two_col_table(self.initial_table_,
'name', strict=True)
self.tenant_id = table_parser.get_value_two_col_table(
self.initial_table_, 'project_id')
self.user_id = table_parser.get_value_two_col_table(self.initial_table_,
'user_id')
self.boot_info = self.__get_boot_info()
self.flavor_table = None
VMInfo.__instances[
vm_id] = self # add instance to class variable for tracking
def refresh_table(self):
self.table_ = table_parser.table(
cli.openstack('server show', self.vm_id, ssh_client=self.con_ssh,
auth_info=self.auth_info, timeout=60)[1])
def get_host_name(self):
self.refresh_table()
return table_parser.get_value_two_col_table(table_=self.table_,
field=':host', strict=False)
def get_flavor_id(self):
"""
Returns: (dict) {'name': flavor_name, 'id': flavor_id}
"""
flavor = table_parser.get_value_two_col_table(self.table_, 'flavor')
flavor_id = re.findall(r'\((.*)\)', flavor)[0]
return flavor_id
def refresh_flavor_table(self):
flavor_id = self.get_flavor_id()
self.flavor_table = table_parser.table(
cli.openstack('flavor show', flavor_id, ssh_client=self.con_ssh,
auth_info=Tenant.get('admin'))[1])
return self.flavor_table
def __get_boot_info(self):
return _get_boot_info(table_=self.table_, vm_id=self.vm_id,
auth_info=self.auth_info,
con_ssh=self.con_ssh)
def get_storage_type(self):
table_ = self.flavor_table
if not table_:
table_ = self.refresh_flavor_table()
extra_specs = table_parser.get_value_two_col_table(table_, 'properties',
merge_lines=True)
extra_specs = table_parser.convert_value_to_dict(value=extra_specs)
return extra_specs.get(FlavorSpec.STORAGE_BACKING, None)
def has_local_disks(self):
if self.boot_info['type'] == 'image':
return True
table_ = self.flavor_table
if not table_:
table_ = self.refresh_flavor_table()
swap = table_parser.get_value_two_col_table(table_, 'swap')
ephemeral = table_parser.get_value_two_col_table(table_, 'ephemeral',
strict=False)
return bool(swap or int(ephemeral))
@classmethod
def get_vms_info(cls):
return tuple(cls.__instances)
@classmethod
def get_vm_info(cls, vm_id, con_ssh=None):
if vm_id not in cls.__instances:
if vm_id in get_all_vms(con_ssh=con_ssh):
return cls(vm_id, con_ssh)
else:
raise exceptions.VMError(
"VM with id {} does not exist!".format(vm_id))
instance = cls.__instances[vm_id]
instance.refresh_table()
return instance
@classmethod
def remove_instance(cls, vm_id):
cls.__instances.pop(vm_id, default="No instance found")
def delete_vms(vms=None, delete_volumes=True, check_first=True,
timeout=VMTimeout.DELETE, fail_ok=False,
stop_first=True, con_ssh=None, auth_info=Tenant.get('admin'),
remove_cleanup=None):
"""
Delete given vm(s) (and attached volume(s)). If None vms given, all vms
on the system will be deleted.
Args:
vms (list|str): list of vm ids to be deleted. If string input,
assume only one vm id is provided.
check_first (bool): Whether to check if given vm(s) exist on system
before attempt to delete
timeout (int): Max time to wait for delete cli finish and wait for
vms actually disappear from system
delete_volumes (bool): delete attached volume(s) if set to True
fail_ok (bool):
stop_first (bool): whether to stop active vm(s) first before
deleting. Best effort only
con_ssh (SSHClient):
auth_info (dict):
remove_cleanup (None|str): remove from vm cleanup list if deleted
successfully
Returns (tuple): (rtn_code(int), msg(str)) # rtn_code 1,2,3 only returns
when fail_ok=True
(-1, 'No vm(s) to delete.') # "Empty vm list/string provided and
no vm exist on system.
(-1, 'None of the given vm(s) exists on system.')
(0, "VM(s) deleted successfully.")
(1, <stderr>) # delete vm(s) cli returns stderr, some or all vms
failed to delete.
(2, "VMs deletion cmd all accepted, but some vms still exist after
deletion")
"""
existing_vms = None
if not vms:
vms = get_vms(con_ssh=con_ssh, auth_info=auth_info, all_projects=True,
long=False)
existing_vms = list(vms)
elif isinstance(vms, str):
vms = [vms]
vms = [vm for vm in vms if vm]
if not vms:
LOG.warning(
"Empty vm list/string provided and no vm exist on system. Do "
"Nothing")
return -1, 'No vm(s) to delete.'
if check_first:
if existing_vms is None:
existing_vms = get_vms(con_ssh=con_ssh, auth_info=auth_info,
all_projects=True, long=False)
vms = list(set(vms) & set(existing_vms))
if not vms:
LOG.info("None given vms exist on system. Do nothing")
return -1, 'None of the given vm(s) exists on system.'
if stop_first: # best effort only
active_vms = get_vms(vms=vms, auth_info=auth_info, con_ssh=con_ssh,
all_projects=True,
Status=VMStatus.ACTIVE)
if active_vms:
stop_vms(active_vms, fail_ok=True, con_ssh=con_ssh,
auth_info=auth_info)
vols_to_del = []
if delete_volumes:
vols_to_del = cinder_helper.get_volumes_attached_to_vms(
vms=vms, auth_info=auth_info, con_ssh=con_ssh)
LOG.info("Deleting vm(s): {}".format(vms))
vms_accepted = []
deletion_err = ''
for vm in vms:
# Deleting vm one by one due to the cmd will stop if a failure is
# encountered, causing no attempt to delete
# other vms
code, output = cli.openstack('server delete', vm, ssh_client=con_ssh,
fail_ok=True, auth_info=auth_info,
timeout=timeout)
if code > 0:
deletion_err += '{}\n'.format(output)
else:
vms_accepted.append(vm)
# check if vms are actually removed from nova list
all_deleted, vms_undeleted = _wait_for_vms_deleted(vms_accepted,
fail_ok=True,
auth_info=auth_info,
timeout=timeout,
con_ssh=con_ssh)
if remove_cleanup:
vms_deleted = list(set(vms_accepted) - set(vms_undeleted))
ResourceCleanup.remove('vm', vms_deleted, scope=remove_cleanup,
del_vm_vols=False)
# Delete volumes results will not be returned. Best effort only.
if delete_volumes:
res = cinder_helper.delete_volumes(vols_to_del, fail_ok=True,
auth_info=auth_info,
con_ssh=con_ssh)[0]
if res == 0 and remove_cleanup:
ResourceCleanup.remove('volume', vols_to_del, scope=remove_cleanup)
# Process returns
if deletion_err:
LOG.warning(deletion_err)
if fail_ok:
return 1, deletion_err
raise exceptions.CLIRejected(deletion_err)
if vms_undeleted:
msg = 'VM(s) still exsit after deletion: {}'.format(vms_undeleted)
LOG.warning(msg)
if fail_ok:
return 2, msg
raise exceptions.VMPostCheckFailed(msg)
LOG.info("VM(s) deleted successfully: {}".format(vms))
return 0, "VM(s) deleted successfully."
def _wait_for_vms_deleted(vms, timeout=VMTimeout.DELETE, fail_ok=True,
check_interval=3, con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Wait for specific vm to be removed from nova list
Args:
vms (str|list): list of vms' ids
timeout (int): in seconds
fail_ok (bool):
check_interval (int):
con_ssh (SSHClient|None):
auth_info (dict|None):
Returns (tuple): (result(bool), vms_failed_to_delete(list))
"""
if isinstance(vms, str):
vms = [vms]
vms_to_check = list(vms)
end_time = time.time() + timeout
while time.time() < end_time:
try:
vms_to_check = get_vms(vms=vms_to_check, con_ssh=con_ssh,
auth_info=auth_info)
except exceptions.CLIRejected:
pass
if not vms_to_check:
return True, []
time.sleep(check_interval)
if fail_ok:
return False, vms_to_check
raise exceptions.VMPostCheckFailed(
"Some vm(s) are not removed from nova list within {} seconds: {}".
format(timeout, vms_to_check))
def wait_for_vms_values(vms, header='Status', value=VMStatus.ACTIVE,
timeout=VMTimeout.STATUS_CHANGE, fail_ok=True,
check_interval=3, con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Wait for specific vms to reach any of the given state(s) in openstack
server list
Args:
vms (str|list): id(s) of vms to check
header (str): target header in nova list
value (str|list): expected value(s)
timeout (int): in seconds
fail_ok (bool):
check_interval (int):
con_ssh (SSHClient|None):
auth_info (dict|None):
Returns (list): [result(bool), vms_in_state(dict),
vms_failed_to_reach_state(dict)]
"""
if isinstance(vms, str):
vms = [vms]
if isinstance(value, str):
value = [value]
res_fail = res_pass = None
end_time = time.time() + timeout
while time.time() < end_time:
res_pass = {}
res_fail = {}
vms_values = get_vms(vms=vms, con_ssh=con_ssh, auth_info=auth_info,
field=header)
for i in range(len(vms)):
vm = vms[i]
vm_value = vms_values[i]
if vm_value in value:
res_pass[vm] = vm_value
else:
res_fail[vm] = vm_value
if not res_fail:
return True, res_pass, res_fail
time.sleep(check_interval)
fail_msg = "Some vm(s) did not reach given status from nova list within " \
"{} seconds: {}".format(timeout, res_fail)
if fail_ok:
LOG.warning(fail_msg)
return False, res_pass, res_fail
raise exceptions.VMPostCheckFailed(fail_msg)
def set_vm_state(vm_id, check_first=False, error_state=True, fail_ok=False,
auth_info=Tenant.get('admin'),
con_ssh=None):
"""
Set vm state to error or active via nova reset-state.
Args:
vm_id:
check_first:
error_state:
fail_ok:
auth_info:
con_ssh:
Returns (tuple):
"""
expt_vm_status = VMStatus.ERROR if error_state else VMStatus.ACTIVE
LOG.info("Setting vm {} state to: {}".format(vm_id, expt_vm_status))
if check_first:
pre_vm_status = get_vm_values(vm_id, fields='status', con_ssh=con_ssh,
auth_info=auth_info)[0]
if pre_vm_status.lower() == expt_vm_status.lower():
msg = "VM {} already in {} state. Do nothing.".format(vm_id,
pre_vm_status)
LOG.info(msg)
return -1, msg
code, out = set_vm(vm_id=vm_id, state=expt_vm_status, con_ssh=con_ssh,
auth_info=auth_info, fail_ok=fail_ok)
if code > 0:
return 1, out
result = wait_for_vm_status(vm_id, expt_vm_status, fail_ok=fail_ok)
if result is None:
msg = "VM {} did not reach expected state - {} after " \
"reset-state.".format(vm_id, expt_vm_status)
LOG.warning(msg)
return 2, msg
msg = "VM state is successfully set to: {}".format(expt_vm_status)
LOG.info(msg)
return 0, msg
def reboot_vm(vm_id, hard=False, fail_ok=False, con_ssh=None, auth_info=None,
cli_timeout=CMDTimeout.REBOOT_VM,
reboot_timeout=VMTimeout.REBOOT):
"""
reboot vm via openstack server reboot
Args:
vm_id:
hard (bool): hard or soft reboot
fail_ok:
con_ssh:
auth_info:
cli_timeout:
reboot_timeout:
Returns (tuple):
"""
vm_status = get_vm_status(vm_id, con_ssh=con_ssh)
if not vm_status.lower() == 'active':
LOG.warning(
"VM is not in active state before rebooting. VM status: {}".format(
vm_status))
extra_arg = '--hard ' if hard else ''
arg = "{}{}".format(extra_arg, vm_id)
date_format = "%Y%m%d %T"
start_time = common.get_date_in_format(date_format=date_format)
code, output = cli.openstack('server reboot', arg, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info,
timeout=cli_timeout)
if code > 0:
return 1, output
# expt_reboot = VMStatus.HARD_REBOOT if hard else VMStatus.SOFT_REBOOT
# _wait_for_vm_status(vm_id, expt_reboot, check_interval=0, fail_ok=False)
LOG.info("Wait for vm reboot events to appear in fm event-list")
expt_reason = 'hard-reboot' if hard else 'soft-reboot'
system_helper.wait_for_events(
timeout=30, num=10, entity_instance_id=vm_id,
start=start_time, fail_ok=False, strict=False,
**{'Event Log ID': EventLogID.REBOOT_VM_ISSUED,
'Reason Text': expt_reason})
system_helper.wait_for_events(
timeout=reboot_timeout, num=10, entity_instance_id=vm_id,
start=start_time, fail_ok=False,
**{'Event Log ID': EventLogID.REBOOT_VM_COMPLETE})
LOG.info("Check vm status from nova show")
actual_status = wait_for_vm_status(vm_id,
[VMStatus.ACTIVE, VMStatus.ERROR],
fail_ok=fail_ok, con_ssh=con_ssh,
timeout=30)
if not actual_status:
msg = "VM {} did not reach active state after reboot.".format(vm_id)
LOG.warning(msg)
return 2, msg
if actual_status.lower() == VMStatus.ERROR.lower():
msg = "VM is in error state after reboot."
if fail_ok:
LOG.warning(msg)
return 3, msg
raise exceptions.VMPostCheckFailed(msg)
succ_msg = "VM rebooted successfully."
LOG.info(succ_msg)
return 0, succ_msg
def __perform_vm_action(vm_id, action, expt_status,
timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
con_ssh=None,
auth_info=None):
LOG.info("{} vm {} begins...".format(action, vm_id))
code, output = cli.nova(action, vm_id, ssh_client=con_ssh, fail_ok=fail_ok,
auth_info=auth_info, timeout=120)
if code == 1:
return 1, output
actual_status = wait_for_vm_status(vm_id, [expt_status, VMStatus.ERROR],
fail_ok=fail_ok, con_ssh=con_ssh,
timeout=timeout)
if not actual_status:
msg = "VM {} did not reach expected state {} after {}.".format(
vm_id, expt_status, action)
LOG.warning(msg)
return 2, msg
if actual_status.lower() == VMStatus.ERROR.lower():
msg = "VM is in error state after {}.".format(action)
if fail_ok:
LOG.warning(msg)
return 3, msg
raise exceptions.VMPostCheckFailed(msg)
succ_msg = "{} VM succeeded.".format(action)
LOG.info(succ_msg)
return 0, succ_msg
def suspend_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
con_ssh=None, auth_info=None):
return __perform_vm_action(vm_id, 'suspend', VMStatus.SUSPENDED,
timeout=timeout, fail_ok=fail_ok,
con_ssh=con_ssh, auth_info=auth_info)
def resume_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
con_ssh=None, auth_info=None):
return __perform_vm_action(vm_id, 'resume', VMStatus.ACTIVE,
timeout=timeout, fail_ok=fail_ok,
con_ssh=con_ssh,
auth_info=auth_info)
def pause_vm(vm_id, timeout=VMTimeout.PAUSE, fail_ok=False, con_ssh=None,
auth_info=None):
return __perform_vm_action(vm_id, 'pause', VMStatus.PAUSED, timeout=timeout,
fail_ok=fail_ok, con_ssh=con_ssh,
auth_info=auth_info)
def unpause_vm(vm_id, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False,
con_ssh=None, auth_info=None):
return __perform_vm_action(vm_id, 'unpause', VMStatus.ACTIVE,
timeout=timeout, fail_ok=fail_ok,
con_ssh=con_ssh,
auth_info=auth_info)
def stop_vms(vms, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False, con_ssh=None,
auth_info=None):
return _start_or_stop_vms(vms, 'stop', VMStatus.STOPPED, timeout,
check_interval=1, fail_ok=fail_ok,
con_ssh=con_ssh, auth_info=auth_info)
def start_vms(vms, timeout=VMTimeout.STATUS_CHANGE, fail_ok=False, con_ssh=None,
auth_info=None):
return _start_or_stop_vms(vms, 'start', VMStatus.ACTIVE, timeout,
check_interval=1, fail_ok=fail_ok,
con_ssh=con_ssh, auth_info=auth_info)
def _start_or_stop_vms(vms, action, expt_status,
timeout=VMTimeout.STATUS_CHANGE, check_interval=3,
fail_ok=False,
con_ssh=None, auth_info=None):
LOG.info("{}ing vms {}...".format(action, vms))
action = action.lower()
if isinstance(vms, str):
vms = [vms]
# Not using openstack client due to stop will be aborted at first
# failure, without continue processing other vms
code, output = cli.nova(action, ' '.join(vms), ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
vms_to_check = list(vms)
if code == 1:
vms_to_check = re.findall(
NovaCLIOutput.VM_ACTION_ACCEPTED.format(action), output)
if not vms_to_check:
return 1, output
res_bool, res_pass, res_fail = wait_for_vms_values(
vms_to_check, 'Status', [expt_status, VMStatus.ERROR],
fail_ok=fail_ok, check_interval=check_interval, con_ssh=con_ssh,
timeout=timeout)
if not res_bool:
msg = "Some VM(s) did not reach expected state(s) - {}. Actual " \
"states: {}".format(expt_status, res_fail)
LOG.warning(msg)
return 2, msg
error_vms = [vm_id for vm_id in vms_to_check if
res_pass[vm_id].lower() == VMStatus.ERROR.lower()]
if error_vms:
msg = "Some VM(s) in error state after {}: {}".format(action, error_vms)
if fail_ok:
LOG.warning(msg)
return 3, msg
raise exceptions.VMPostCheckFailed(msg)
succ_msg = "Action {} performed successfully on vms.".format(action)
LOG.info(succ_msg)
return 0, succ_msg
def rebuild_vm(vm_id, image_id=None, new_name=None, preserve_ephemeral=None,
fail_ok=False, con_ssh=None,
auth_info=Tenant.get('admin'), **metadata):
if image_id is None:
image_id = glance_helper.get_image_id_from_name(
GuestImages.DEFAULT['guest'], strict=True)
args = '{} {}'.format(vm_id, image_id)
if new_name:
args += ' --name {}'.format(new_name)
if preserve_ephemeral:
args += ' --preserve-ephemeral'
for key, value in metadata.items():
args += ' --meta {}={}'.format(key, value)
LOG.info("Rebuilding vm {}".format(vm_id))
# Some features such as trusted image cert not available with openstack
# client
code, output = cli.nova('rebuild', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code == 1:
return code, output
LOG.info("Check vm status after vm rebuild")
wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, fail_ok=fail_ok,
con_ssh=con_ssh)
actual_status = wait_for_vm_status(vm_id, [VMStatus.ACTIVE, VMStatus.ERROR],
fail_ok=fail_ok, con_ssh=con_ssh,
timeout=VMTimeout.REBUILD)
if not actual_status:
msg = "VM {} did not reach active state after rebuild.".format(vm_id)
LOG.warning(msg)
return 2, msg
if actual_status.lower() == VMStatus.ERROR.lower():
msg = "VM is in error state after rebuild."
if fail_ok:
LOG.warning(msg)
return 3, msg
raise exceptions.VMPostCheckFailed(msg)
succ_msg = "VM rebuilded successfully."
LOG.info(succ_msg)
return 0, succ_msg
def get_vm_numa_nodes_via_ps(vm_id=None, instance_name=None, host=None,
con_ssh=None, auth_info=Tenant.get('admin'),
per_vcpu=False):
"""
Get numa nodes VM is currently on
Args:
vm_id:
instance_name:
host:
con_ssh:
auth_info:
per_vcpu (bool): if True, return per vcpu, e.g., if vcpu=0,1,2,
returned list will have same length [0,1,0]
Returns (list): e.g., [0], [0, 1]
"""
if not instance_name or not host:
if not vm_id:
raise ValueError('vm_id has to be provided')
instance_name, host = get_vm_values(vm_id,
fields=[":instance_name", ":host"],
strict=False, con_ssh=con_ssh,
auth_info=auth_info)
with host_helper.ssh_to_host(host, con_ssh=con_ssh) as host_ssh:
vcpu_cpu_map = get_vcpu_cpu_map(instance_names=instance_name,
host_ssh=host_ssh, con_ssh=con_ssh)[
instance_name]
cpus = []
for i in range(len(vcpu_cpu_map)):
cpus.append(vcpu_cpu_map[i])
cpu_non_dup = sorted(list(set(cpus)))
grep_str = ' '.join(
['-e "processor.*: {}$"'.format(cpu) for cpu in cpu_non_dup])
cmd = 'cat /proc/cpuinfo | grep -A 10 {} | grep --color=never ' \
'"physical id"'.format(grep_str)
physical_ids = host_ssh.exec_cmd(cmd, fail_ok=False)[1].splitlines()
physical_ids = [int(proc.split(sep=':')[-1].strip()) for proc in
physical_ids if 'physical' in proc]
if per_vcpu:
physical_ids = [physical_ids[cpu_non_dup.index(cpu)] for cpu in
cpus]
return physical_ids
def get_vm_host_and_numa_nodes(vm_id, con_ssh=None, per_vcpu=False):
"""
Get vm host and numa nodes used for the vm on the host
Args:
vm_id (str):
con_ssh (SSHClient):
per_vcpu (bool): if True, return numa nodes per vcpu, e.g., vcpu=0,1,
2, returned list can be: [0,1,0]
Returns (tuple): (<vm_hostname> (str), <numa_nodes> (list of integers))
"""
instance_name, host = get_vm_values(vm_id,
fields=[":instance_name", ":host"],
strict=False)
actual_node_vals = get_vm_numa_nodes_via_ps(vm_id=vm_id,
instance_name=instance_name,
host=host, con_ssh=con_ssh,
per_vcpu=per_vcpu)
return host, actual_node_vals
def perform_action_on_vm(vm_id, action, auth_info=Tenant.get('admin'),
con_ssh=None, **kwargs):
"""
Perform action on a given vm.
Args:
vm_id (str):
action (str): action to perform on vm. Valid_actions: 'start',
'stop', 'suspend', 'resume', 'pause', 'unpause',
'reboot', 'live_migrate', or 'cold_migrate'
auth_info (dict):
con_ssh (SSHClient):
**kwargs: extra params to pass to action function,
e.g.destination_host='compute-0' when action is live_migrate
Returns (None):
"""
action_function_map = {
'start': start_vms,
'stop': stop_vms,
'suspend': suspend_vm,
'resume': resume_vm,
'pause': pause_vm,
'unpause': unpause_vm,
'reboot': reboot_vm,
'rebuild': rebuild_vm,
'live_migrate': live_migrate_vm,
'cold_migrate': cold_migrate_vm,
'cold_mig_revert': cold_migrate_vm,
}
if not vm_id:
raise ValueError("vm id is not provided.")
valid_actions = list(action_function_map.keys())
action = action.lower().replace(' ', '_')
if action not in valid_actions:
raise ValueError(
"Invalid action provided: {}. Valid actions: {}".format(
action, valid_actions))
if action == 'cold_mig_revert':
kwargs['revert'] = True
return action_function_map[action](vm_id, con_ssh=con_ssh,
auth_info=auth_info, **kwargs)
def get_vm_nics_info(vm_id, network=None, vnic_type=None, rtn_dict=False):
"""
Get vm nics info
Args:
vm_id:
network:
vnic_type:
rtn_dict:
Returns (list of dict|dict of dict):
list or dict (port as key) of port_info_dict. Each port_info_dict
contains following info:
{
'port_id': <port_id>,
'network': <net_name>,
'network_id': <net_id>,
'vnic_type': <vnic_type>,
'mac_address': <mac_address>,
'subnet_id': <subnet_id>,
'subnet_cidr': <subnet_cidr>
}
"""
vm_ports, vm_macs, vm_ips_info = network_helper.get_ports(
server=vm_id, network=network,
field=('ID', 'MAC Address', 'Fixed IP Addresses'))
vm_subnets = []
vm_ips = []
for ip_info in vm_ips_info:
ip_info = ip_info[0]
vm_ips.append(ip_info.get('ip_address'))
vm_subnets.append(ip_info.get('subnet_id'))
indexes = list(range(len(vm_ports)))
vnic_types = []
vm_net_ids = []
for port in vm_ports:
port_vnic_type, port_net_id = network_helper.get_port_values(
port=port, fields=('binding_vnic_type', 'network_id'))
vnic_types.append(port_vnic_type)
vm_net_ids.append(port_net_id)
if vnic_type and vnic_type != port_vnic_type:
indexes.remove(list(vm_ports).index(port))
vm_net_names = []
ids_, names_, = network_helper.get_networks(field=('ID', 'Name'),
strict=False)
for net_id in vm_net_ids:
vm_net_names.append(names_[ids_.index(net_id)])
res_dict = {}
res = []
for i in indexes:
port_dict = {
'port_id': vm_ports[i],
'network': vm_net_names[i],
'network_id': vm_net_ids[i],
'vnic_type': vnic_types[i],
'mac_address': vm_macs[i],
'ip_address': vm_ips[i]
}
if rtn_dict:
res_dict[vm_ports[i]] = port_dict
else:
res.append(port_dict)
return res_dict if rtn_dict else res
def get_vm_interfaces_via_virsh(vm_id, con_ssh=None):
"""
Args:
vm_id:
con_ssh:
Returns (list of tuple):
[(mac_0, vif_model_0)...]
"""
vm_host = get_vm_host(vm_id=vm_id, con_ssh=con_ssh)
inst_name = get_vm_instance_name(vm_id=vm_id, con_ssh=con_ssh)
vm_ifs = []
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
output = host_ssh.exec_sudo_cmd('virsh domiflist {}'.format(inst_name),
fail_ok=False)[1]
if_lines = output.split('-------------------------------\n', 1)[
-1].splitlines()
for line in if_lines:
if not line.strip():
continue
interface, type_, source, model, mac = line.split()
vm_ifs.append((mac, model))
return vm_ifs
def add_vlan_for_vm_pcipt_interfaces(vm_id, net_seg_id, retry=3,
init_conf=False):
"""
Add vlan for vm pci-passthrough interface and restart networking service.
Do nothing if expected vlan interface already exists in 'ip addr'.
Args:
vm_id (str):
net_seg_id (int|str|dict): such as 1792
retry (int): max number of times to reboot vm to try to recover it
from non-exit
init_conf (bool): To workaround upstream bug where mac changes after
migrate or resize https://bugs.launchpad.net/nova/+bug/1617429
Returns: None
Raises: VMNetworkError if vlan interface is not found in 'ip addr' after
adding
Notes:
Sometimes a non-exist 'rename6' interface will be used for
pci-passthrough nic after vm maintenance
Sudo reboot from the vm as workaround.
By default will try to reboot for a maximum of 3 times
"""
if not vm_id or not net_seg_id:
raise ValueError("vm_id and/or net_seg_id not provided.")
net_seg_id_dict = None
if isinstance(net_seg_id, dict):
net_seg_id_dict = net_seg_id
net_seg_id = None
for i in range(retry):
vm_pcipt_nics = get_vm_nics_info(vm_id, vnic_type='direct-physical')
if not vm_pcipt_nics:
LOG.warning("No pci-passthrough device found for vm from nova "
"show {}".format(vm_id))
return
with ssh_to_vm_from_natbox(vm_id=vm_id) as vm_ssh:
for pcipt_nic in vm_pcipt_nics:
mac_addr = pcipt_nic['mac_address']
eth_name = network_helper.get_eth_for_mac(mac_addr=mac_addr,
ssh_client=vm_ssh)
if not eth_name:
if not init_conf:
LOG.warning(
"Interface with mac {} is not listed in 'ip addr' "
"in vm {}".format(mac_addr, vm_id))
LOG.info("Try to get first eth with mac 90:...")
eth_name = network_helper.get_eth_for_mac(
mac_addr="link/ether 90:", ssh_client=vm_ssh)
if not eth_name:
exceptions.VMNetworkError(
"No Mac starts with 90: in ip addr for vm "
"{}".format(vm_id))
else:
raise exceptions.VMNetworkError(
"Interface with mac {} is not listed in 'ip addr' "
"in vm {}".format(mac_addr, vm_id))
if 'rename' in eth_name:
LOG.warning(
"Retry {}: non-existing interface {} found on "
"pci-passthrough nic in vm {}, "
"reboot vm to try to recover".format(
i + 1, eth_name, vm_id))
sudo_reboot_from_vm(vm_id=vm_id, vm_ssh=vm_ssh)
wait_for_vm_pingable_from_natbox(vm_id)
break
else:
if net_seg_id_dict:
net_name = pcipt_nic['network']
net_seg_id = net_seg_id_dict[net_name]
LOG.info(
"Seg id for {}: {}".format(net_name, net_seg_id))
vlan_name = "{}.{}".format(eth_name, net_seg_id)
output_pre_ipaddr = \
vm_ssh.exec_cmd('ip addr', fail_ok=False)[1]
if vlan_name in output_pre_ipaddr:
LOG.info("{} already in ip addr. Skip.".format(
vlan_name))
continue
# Bring up pcipt interface and assign IP manually.
# Upstream bug causes dev name and MAC addr
# change after reboot,migrate, making it impossible to
# use DHCP or configure permanant static IP.
# https://bugs.launchpad.net/nova/+bug/1617429
wait_for_interfaces_up(vm_ssh, eth_name, set_up=True)
# 'ip link add' works for all linux guests but it does
# not persists after network service restart
vm_ssh.exec_cmd(
'ip link add link {} name {} type vlan id {}'.format(
eth_name, vlan_name,
net_seg_id))
vm_ssh.exec_cmd('ip link set {} up'.format(vlan_name))
vnic_ip = pcipt_nic['ip_address']
vm_ssh.exec_cmd(
'ip addr add {}/24 dev {}'.format(vnic_ip, vlan_name))
LOG.info(
"Check if vlan is added successfully with IP assigned")
output_post_ipaddr = \
vm_ssh.exec_cmd('ip addr', fail_ok=False)[1]
if vlan_name not in output_post_ipaddr:
raise exceptions.VMNetworkError(
"{} is not found in 'ip addr' after adding vlan "
"interface".
format(vlan_name))
time.sleep(5)
if not is_ip_assigned(vm_ssh, eth_name=vlan_name):
msg = 'No IP assigned to {} vlan interface for VM ' \
'{}'.format(vlan_name, vm_id)
LOG.warning(msg)
raise exceptions.VMNetworkError(msg)
else:
LOG.info(
"vlan {} is successfully added and an IP is "
"assigned.".format(vlan_name))
else:
# did not break, meaning no 'rename' interface detected,
# vlan either existed or successfully added
return
# 'for' loop break which means 'rename' interface detected,
# and vm reboot triggered - known issue with wrl
LOG.info("Reboot vm completed. Retry started.")
else:
raise exceptions.VMNetworkError(
"'rename' interface still exists in pci-passthrough vm {} with {} "
"reboot attempts.".format(vm_id, retry))
def is_ip_assigned(vm_ssh, eth_name):
output = vm_ssh.exec_cmd('ip addr show {}'.format(eth_name),
fail_ok=False)[1]
return re.search('inet {}'.format(Networks.IPV4_IP), output)
def wait_for_interfaces_up(vm_ssh, eth_names, check_interval=10, timeout=180,
set_up=False):
LOG.info(
"Waiting for vm interface(s) to be in UP state: {}".format(eth_names))
end_time = time.time() + timeout
if isinstance(eth_names, str):
eth_names = [eth_names]
ifs_to_check = list(eth_names)
while time.time() < end_time:
for eth in ifs_to_check:
output = \
vm_ssh.exec_cmd('ip -d link show {}'.format(eth),
fail_ok=False)[1]
if 'state UP' in output:
ifs_to_check.remove(eth)
continue
else:
if set_up:
vm_ssh.exec_cmd('ip link set {} up'.format(eth))
LOG.info(
"{} is not up - wait for {} seconds and check again".format(
eth, check_interval))
break
if not ifs_to_check:
LOG.info('interfaces are up: {}'.format(eth_names))
return
time.sleep(check_interval)
raise exceptions.VMNetworkError("Interface(s) not up for given vm")
def sudo_reboot_from_vm(vm_id, vm_ssh=None, check_host_unchanged=True,
con_ssh=None):
pre_vm_host = None
if check_host_unchanged:
pre_vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
LOG.info("Initiate sudo reboot from vm")
def _sudo_reboot(vm_ssh_):
extra_prompt = 'Broken pipe'
output = vm_ssh_.exec_sudo_cmd('reboot -f', get_exit_code=False,
extra_prompt=extra_prompt)[1]
expt_string = 'The system is going down for reboot|Broken pipe'
if re.search(expt_string, output):
# Sometimes system rebooting msg will be displayed right after
# reboot cmd sent
vm_ssh_.parent.flush()
return
try:
time.sleep(10)
vm_ssh_.send('')
index = vm_ssh_.expect([expt_string, vm_ssh_.prompt], timeout=60)
if index == 1:
raise exceptions.VMOperationFailed("Unable to reboot vm {}")
vm_ssh_.parent.flush()
except pexpect.TIMEOUT:
vm_ssh_.send_control('c')
vm_ssh_.expect()
raise
if not vm_ssh:
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
_sudo_reboot(vm_ssh)
else:
_sudo_reboot(vm_ssh)
LOG.info(
"sudo vm reboot initiated - wait for reboot completes and VM reaches "
"active state")
system_helper.wait_for_events(VMTimeout.AUTO_RECOVERY, strict=False,
fail_ok=False, con_ssh=con_ssh,
**{'Entity Instance ID': vm_id,
'Event Log ID':
EventLogID.REBOOT_VM_COMPLETE})
wait_for_vm_status(vm_id, status=VMStatus.ACTIVE, fail_ok=False,
con_ssh=con_ssh)
if check_host_unchanged:
post_vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
if not pre_vm_host == post_vm_host:
raise exceptions.HostError(
"VM host changed from {} to {} after sudo reboot vm".format(
pre_vm_host, post_vm_host))
def get_proc_nums_from_vm(vm_ssh):
total_cores = common.parse_cpus_list(
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/present', fail_ok=False)[
1])
online_cores = common.parse_cpus_list(
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/online', fail_ok=False)[1])
offline_cores = common.parse_cpus_list(
vm_ssh.exec_cmd('cat /sys/devices/system/cpu/offline', fail_ok=False)[
1])
return total_cores, online_cores, offline_cores
def get_instance_names_via_virsh(host_ssh):
"""
Get instance names via virsh list on given host
Args:
host_ssh:
Returns (list):
"""
inst_names = host_ssh.exec_sudo_cmd(
"virsh list | grep instance- | awk {{'print $2'}}",
get_exit_code=False)[1]
return [name.strip() for name in inst_names.splitlines()]
def get_vcpu_cpu_map(instance_names=None, host_ssh=None, host=None,
con_ssh=None):
"""
Get vm(s) vcpu cpu map on given host
Args:
instance_names (str|tuple|list|None):
host_ssh (SSHClient|None):
host (str|None):
con_ssh:
Returns (dict): {<instance_name>: {0: <host_log_core0>,
1: <host_log_core1>, ...}, ...}
"""
if not host and not host_ssh:
raise ValueError('host or host_ssh has to be specified')
extra_grep = ''
if instance_names:
if isinstance(instance_names, str):
instance_names = (instance_names,)
extra_grep = '|grep -E "{}"'.format('|'.join(instance_names))
cmd = 'ps-sched.sh|grep qemu{}|grep " CPU" '.format(extra_grep) + \
"""| awk '{{print $10" "$12" "$15 ;}}'"""
if host_ssh:
output = host_ssh.exec_cmd(cmd)[1]
else:
with host_helper.ssh_to_host(host, con_ssh=con_ssh) as host_ssh:
output = host_ssh.exec_cmd(cmd)[1]
vcpu_cpu_map = {}
for line in output.splitlines():
cpu, vcpu, instance_name = line.split()
instance_name = instance_name.split(sep=',')[0].split(sep='=')[1]
if instance_name not in vcpu_cpu_map:
vcpu_cpu_map[instance_name] = {}
vcpu_cpu_map[instance_name][int(vcpu.split(sep='/')[0])] = int(cpu)
return vcpu_cpu_map
def get_affined_cpus_for_vm(vm_id, host_ssh=None, vm_host=None,
instance_name=None, con_ssh=None):
"""
cpu affinity list for vm via taskset -pc
Args:
vm_id (str):
host_ssh
vm_host
instance_name
con_ssh (SSHClient):
Returns (list): such as [10, 30]
"""
cmd = "ps-sched.sh|grep qemu|grep {}|grep -v grep|awk '{{print $2;}}'" + \
'|xargs -i /bin/sh -c "taskset -pc {{}}"'
if host_ssh:
if not vm_host or not instance_name:
raise ValueError(
"vm_host and instance_name have to be provided together with "
"host_ssh")
output = host_ssh.exec_cmd(cmd.format(instance_name))[1]
else:
vm_host = get_vm_host(vm_id, con_ssh=con_ssh)
instance_name = get_vm_instance_name(vm_id, con_ssh=con_ssh)
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
output = host_ssh.exec_cmd(cmd.format(instance_name))[1]
# Sample output:
# pid 6376's current affinity list: 10
# pid 6380's current affinity list: 10
# pid 6439's current affinity list: 10
# pid 6441's current affinity list: 10
# pid 6442's current affinity list: 30
# pid 6445's current affinity list: 10
# pid 24142's current affinity list: 10
all_cpus = []
lines = output.splitlines()
for line in lines:
# skip line if below output occurs due to timing in executing cmds
# taskset: failed to get pid 17125's affinity: No such process
if "No such process" in line:
continue
cpu_str = line.split(sep=': ')[-1].strip()
cpus = common.parse_cpus_list(cpus=cpu_str)
all_cpus += cpus
all_cpus = sorted(list(set(all_cpus)))
LOG.info("Affined cpus on host {} for vm {}: {}".format(vm_host, vm_id,
all_cpus))
return all_cpus
def _scp_net_config_cloud_init(guest_os):
con_ssh = get_cli_client()
dest_dir = '{}/userdata'.format(ProjVar.get_var('USER_FILE_DIR'))
if 'ubuntu' in guest_os:
dest_name = 'ubuntu_cloud_init_if_conf.sh'
elif 'centos' in guest_os:
dest_name = 'centos_cloud_init_if_conf.sh'
else:
raise ValueError("Unknown guest_os")
dest_path = '{}/{}'.format(dest_dir, dest_name)
if con_ssh.file_exists(file_path=dest_path):
LOG.info('userdata {} already exists. Return existing path'.format(
dest_path))
return dest_path
LOG.debug('Create userdata directory if not already exists')
cmd = 'mkdir -p {}'.format(dest_dir)
con_ssh.exec_cmd(cmd, fail_ok=False)
# LOG.info('wget image from {} to {}/{}'.format(img_url, img_dest,
# new_name))
# cmd = 'wget {} --no-check-certificate -P {} -O {}'.format(img_url,
# img_dest, new_name)
# con_ssh.exec_cmd(cmd, expect_timeout=7200, fail_ok=False)
source_path = '{}/userdata/{}'.format(TestFileServer.HOME, dest_name)
LOG.info('scp image from test server to active controller')
scp_cmd = 'scp -oStrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' \
' {}@{}:{} {}'.format(TestFileServer.USER, TestFileServer.SERVER,
source_path, dest_dir)
con_ssh.send(scp_cmd)
index = con_ssh.expect(
[con_ssh.prompt, Prompt.PASSWORD_PROMPT, Prompt.ADD_HOST], timeout=3600)
if index == 2:
con_ssh.send('yes')
index = con_ssh.expect([con_ssh.prompt, Prompt.PASSWORD_PROMPT],
timeout=3600)
if index == 1:
con_ssh.send(TestFileServer.PASSWORD)
index = con_ssh.expect()
if index != 0:
raise exceptions.SSHException("Failed to scp files")
return dest_dir
def _create_cloud_init_if_conf(guest_os, nics_num):
"""
Args:
guest_os:
nics_num:
Returns (str|None): file path of the cloud init userdata file for given
guest os and number of nics
Sample file content for Centos vm:
#!/bin/bash
sudo cp /etc/sysconfig/network-scripts/ifcfg-eth0
/etc/sysconfig/network-scripts/ifcfg-eth1
sudo sed -i 's/eth0/eth1/g'
/etc/sysconfig/network-scripts/ifcfg-eth1
sudo ifup eth1
Sample file content for Ubuntu vm:
"""
file_dir = '{}/userdata'.format(ProjVar.get_var('USER_FILE_DIR'))
guest_os = guest_os.lower()
# default eth_path for non-ubuntu image
eth_path = VMPath.ETH_PATH_CENTOS
new_user = None
if 'ubuntu' in guest_os or 'trusty_uefi' in guest_os:
guest_os = 'ubuntu'
# vm_if_path = VMPath.VM_IF_PATH_UBUNTU
eth_path = VMPath.ETH_PATH_UBUNTU
new_user = 'ubuntu'
elif 'centos' in guest_os:
# vm_if_path = VMPath.VM_IF_PATH_CENTOS
new_user = 'centos'
file_name = '{}_{}nic_cloud_init_if_conf.sh'.format(guest_os, nics_num)
file_path = file_dir + file_name
con_ssh = get_cli_client()
if con_ssh.file_exists(file_path=file_path):
LOG.info('userdata {} already exists. Return existing path'.format(
file_path))
return file_path
LOG.info('Create userdata directory if not already exists')
cmd = 'mkdir -p {}'.format(file_dir)
con_ssh.exec_cmd(cmd, fail_ok=False)
tmp_dir = '{}/userdata'.format(ProjVar.get_var('TEMP_DIR'))
os.makedirs(tmp_dir, exist_ok=True)
tmp_file = tmp_dir + file_name
# No longer need to specify bash using cloud-config
# if 'centos_7' in guest_os:
# shell = '/usr/bin/bash'
# else:
# shell = '/bin/bash'
with open(tmp_file, mode='a') as f:
f.write("#cloud-config\n")
if new_user is not None:
f.write("user: {}\n"
"password: {}\n"
"chpasswd: {{ expire: False}}\n"
"ssh_pwauth: True\n\n".format(new_user, new_user))
if eth_path is not None:
eth0_path = eth_path.format('eth0')
f.write("runcmd:\n")
# f.write(" - echo '#!{}'\n".format(shell))
for i in range(nics_num - 1):
ethi_name = 'eth{}'.format(i + 1)
ethi_path = eth_path.format(ethi_name)
f.write(' - cp {} {}\n'.format(eth0_path, ethi_path))
f.write(
" - sed -i 's/eth0/{}/g' {}\n".format(ethi_name, ethi_path))
f.write(' - ifup {}\n'.format(ethi_name))
if not ProjVar.get_var('REMOTE_CLI'):
common.scp_from_localhost_to_active_controller(source_path=tmp_file,
dest_path=file_path,
is_dir=False)
LOG.info("Userdata file created: {}".format(file_path))
return file_path
def _get_cloud_config_add_user(con_ssh=None):
"""
copy the cloud-config userdata to STX server.
This userdata adds stx/li69nux user to guest
Args:
con_ssh (SSHClient):
Returns (str): STX filepath of the userdata
"""
file_dir = ProjVar.get_var('USER_FILE_DIR')
file_name = UserData.ADDUSER_TO_GUEST
file_path = file_dir + file_name
if con_ssh is None:
con_ssh = get_cli_client()
if con_ssh.file_exists(file_path=file_path):
LOG.info('userdata {} already exists. Return existing path'.format(
file_path))
return file_path
source_file = TestServerPath.USER_DATA + file_name
dest_path = common.scp_from_test_server_to_user_file_dir(
source_path=source_file, dest_dir=file_dir,
dest_name=file_name, con_ssh=con_ssh)
if dest_path is None:
raise exceptions.CommonError(
"userdata file {} does not exist after download".format(dest_path))
return dest_path
def boost_vm_cpu_usage(vm_id, end_event, new_dd_events=None, dd_event=None,
timeout=1200, con_ssh=None):
"""
Boost cpu usage on given number of cpu cores on specified vm using dd cmd
on a new thread
Args:
vm_id (str):
end_event (Events): Event for kill the dd processes
new_dd_events (list|Events): list of Event(s) for adding new dd
process(es)
dd_event (Events): Event to set after sending first dd cmd.
timeout: Max time to wait for the end_event to be set before killing dd.
con_ssh
Returns: thread
Examples:
LOG.tc_step("Boost VM cpu usage")
"""
if not new_dd_events:
new_dd_events = []
elif not isinstance(new_dd_events, list):
new_dd_events = [new_dd_events]
def _boost_cpu_in_vm():
LOG.info("Boosting cpu usage for vm {} using 'dd'".format(vm_id))
dd_cmd = 'dd if=/dev/zero of=/dev/null &'
kill_dd = 'pkill -ex dd'
with ssh_to_vm_from_natbox(vm_id, con_ssh=con_ssh, timeout=120,
auth_info=None) as vm_ssh:
LOG.info("Start first 2 dd processes in vm")
vm_ssh.exec_cmd(cmd=dd_cmd)
vm_ssh.exec_cmd(cmd=dd_cmd)
if dd_event:
dd_event.set()
end_time = time.time() + timeout
while time.time() < end_time:
if end_event.is_set():
LOG.info("End event set, kill dd processes in vm")
vm_ssh.flush()
vm_ssh.exec_cmd(kill_dd, get_exit_code=False)
return
for event in new_dd_events:
if event.is_set():
LOG.info(
"New dd event set, start 2 new dd processes in vm")
vm_ssh.exec_cmd(cmd=dd_cmd)
vm_ssh.exec_cmd(cmd=dd_cmd)
new_dd_events.remove(event)
break
time.sleep(3)
LOG.error(
"End event is not set within timeout - {}s, kill dd "
"anyways".format(
timeout))
vm_ssh.exec_cmd(kill_dd)
LOG.info(
"Creating new thread to spike cpu_usage on vm cores for vm {}".format(
vm_id))
thread = multi_thread.MThread(_boost_cpu_in_vm)
thread.start_thread(timeout=timeout + 10)
return thread
def write_in_vm(vm_id, end_event, start_event=None, expect_timeout=120,
thread_timeout=None, write_interval=5,
con_ssh=None):
"""
Continue to write in vm using dd
Args:
vm_id (str):
start_event (Events): set this event when write in vm starts
end_event (Events): if this event is set, end write right away
expect_timeout (int):
thread_timeout (int):
write_interval (int): how frequent to write. Note: 5 seconds seem to
be a good interval,
1 second interval might have noticeable impact on the performance
of pexpect.
con_ssh (SSHClient): controller ssh client
Returns (MThread): new_thread
"""
if not start_event:
start_event = Events("Write in vm {} start".format(vm_id))
write_cmd = "while (true) do date; dd if=/dev/urandom of=output.txt " \
"bs=1k count=1 conv=fsync || break; echo ; " \
"sleep {}; done 2>&1 | tee trace.txt".format(write_interval)
def _keep_writing(vm_id_):
LOG.info("starting to write to vm using dd...")
with ssh_to_vm_from_natbox(vm_id_, con_ssh=con_ssh,
close_ssh=False) as vm_ssh_:
vm_ssh_.send(cmd=write_cmd)
start_event.set()
LOG.info("Write_in_vm started")
LOG.info("Reading the dd output from vm {}".format(vm_id))
thread.res = True
try:
while True:
expt_output = '1024 bytes'
index = vm_ssh_.expect([expt_output, vm_ssh_.prompt],
timeout=expect_timeout, fail_ok=True,
searchwindowsize=100)
if index != 0:
LOG.warning(
"write has stopped or expected output-'{}' is not "
"found".format(
expt_output))
thread.res = False
break
if end_event.is_set():
LOG.info("End thread now")
break
LOG.info("Writing in vm continues...")
time.sleep(write_interval)
finally:
vm_ssh_.send_control('c')
return vm_ssh_
thread = multi_thread.MThread(_keep_writing, vm_id)
thread_timeout = expect_timeout + 30 if thread_timeout is None else \
thread_timeout
thread.start_thread(timeout=thread_timeout)
start_event.wait_for_event(timeout=thread_timeout)
return thread
def attach_interface(vm_id, port_id=None, net_id=None, fixed_ip=None,
fail_ok=False, auth_info=None,
con_ssh=None):
"""
Attach interface to a vm via port_id OR net_id
Args:
vm_id (str):
port_id (str): port to attach to vm
net_id (str): port from given net to attach to vm
fixed_ip (str): fixed ip for attached interface. Only works when
attaching interface via net_id
fail_ok (bool):
auth_info (dict):
con_ssh (SSHClient):
Returns (tuple): (<return_code>, <attached_port_id>)
(0, <port_id_attached>)
(1, <std_err>) - cli rejected
(2, "Post interface attach check failed: <reasons>") -
net_id/port_id, vif_model, or fixed_ip do not match
with
given value
"""
LOG.info("Attaching interface to VM {}".format(vm_id))
if not vm_id:
raise ValueError('vm_id is not supplied')
args = ''
args_dict = {
'--port-id': port_id,
'--net-id': net_id,
'--fixed-ip': fixed_ip,
}
for key, val in args_dict.items():
if val is not None:
args += ' {} {}'.format(key, val)
args += ' {}'.format(vm_id)
prev_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
con_ssh=con_ssh)
# Not switching to openstack client due to nova cli makes more sense.
# openstack client has separate cmds for adding
# port, network and fixed ip, while fixed ip cmd has to specify the network.
code, output = cli.nova('interface-attach', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code == 1:
return code, output
LOG.info("Post interface-attach checks started...")
post_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
con_ssh=con_ssh)
attached_port = list(set(post_ports) - set(prev_ports))
err_msgs = []
if len(attached_port) != 1:
err_msg = "NICs for vm {} is not incremented by 1".format(vm_id)
err_msgs.append(err_msg)
else:
attached_port = attached_port[0]
if net_id:
net_name = network_helper.get_net_name_from_id(net_id, con_ssh=con_ssh,
auth_info=auth_info)
net_ips = get_vm_values(vm_id, fields=net_name, strict=False,
con_ssh=con_ssh, auth_info=auth_info)[0]
if fixed_ip and fixed_ip not in net_ips.split(sep=', '):
err_msg = "specified fixed ip {} is not found in nova show " \
"{}".format(fixed_ip, vm_id)
err_msgs.append(err_msg)
elif port_id and port_id not in post_ports:
err_msg = "port {} is not associated to VM".format(port_id)
err_msgs.append(err_msg)
if err_msgs:
err_msgs_str = "Post interface attach check failed:\n{}".format(
'\n'.join(err_msgs))
if fail_ok:
LOG.warning(err_msgs_str)
return 2, attached_port
raise exceptions.NovaError(err_msgs_str)
succ_msg = "Port {} successfully attached to VM {}".format(attached_port,
vm_id)
LOG.info(succ_msg)
return 0, attached_port
def add_ifcfg_scripts(vm_id, mac_addrs, static_ips=None, ipv6='no', reboot=True,
vm_prompt=None,
**extra_configs):
"""
Args:
vm_id:
mac_addrs (list of str):
static_ips (None|str|list):
ipv6:
reboot:
vm_prompt
**extra_configs:
Returns:
"""
LOG.info('Add ifcfg script(s) to VM {}'.format(vm_id))
with ssh_to_vm_from_natbox(vm_id, prompt=vm_prompt) as vm_ssh:
vm_eths = []
for mac_addr in mac_addrs:
eth_name = network_helper.get_eth_for_mac(mac_addr=mac_addr,
ssh_client=vm_ssh)
assert eth_name, "vif not found for expected mac_address {} in vm" \
" {}".format(mac_addr, vm_id)
vm_eths.append(eth_name)
if static_ips:
if isinstance(static_ips, str):
static_ips = [static_ips]
if len(static_ips) != len(vm_eths):
raise ValueError(
"static_ips count has to be the same as vm devs to be "
"configured")
for i in range(len(vm_eths)):
eth = vm_eths[i]
if static_ips:
static_ip = static_ips[i]
script_content = VMNetwork.IFCFG_STATIC.format(eth, ipv6,
static_ip)
else:
script_content = VMNetwork.IFCFG_DHCP.format(eth, ipv6)
if extra_configs:
extra_str = '\n'.join(
['{}={}'.format(k, v) for k, v in extra_configs.items()])
script_content += '\n{}'.format(extra_str)
script_path = VMPath.ETH_PATH_CENTOS.format(eth)
vm_ssh.exec_sudo_cmd('touch {}'.format(script_path))
vm_ssh.exec_sudo_cmd(
"cat > {} << 'EOT'\n{}\nEOT".format(script_path,
script_content),
fail_ok=False)
if reboot:
reboot_vm(vm_id=vm_id)
def detach_interface(vm_id, port_id, cleanup_route=False, fail_ok=False,
auth_info=None, con_ssh=None,
verify_virsh=True):
"""
Detach a port from vm
Args:
vm_id (str):
port_id (str): existing port that is attached to given vm
fail_ok (bool):
auth_info (dict):
con_ssh (SSHClient):
cleanup_route (bool)
verify_virsh (bool): Whether to verify in virsh xmldump for detached
port
Returns (tuple): (<return_code>, <msg>)
(0, Port <port_id> is successfully detached from VM <vm_id>)
(1, <stderr>) - cli rejected
(2, "Port <port_id> is not detached from VM <vm_id>") - detached
port is still shown in nova show
"""
target_ips = None
if cleanup_route:
fixed_ips = \
network_helper.get_ports(field='Fixed IP Addresses',
port_id=port_id,
con_ssh=con_ssh, auth_info=auth_info)[0]
target_ips = [fixed_ip['ip_address'] for fixed_ip in fixed_ips]
mac_to_check = None
if verify_virsh:
prev_ports, prev_macs = network_helper.get_ports(
server=vm_id, auth_info=auth_info, con_ssh=con_ssh,
field=('ID', 'MAC Address'))
for prev_port in prev_ports:
if port_id == prev_port:
mac_to_check = prev_macs[list(prev_ports).index(prev_port)]
break
LOG.info("Detaching port {} from vm {}".format(port_id, vm_id))
args = '{} {}'.format(vm_id, port_id)
code, output = cli.nova('interface-detach', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code == 1:
return code, output
post_ports = network_helper.get_ports(server=vm_id, auth_info=auth_info,
con_ssh=con_ssh)
if port_id in post_ports:
err_msg = "Port {} is not detached from VM {}".format(port_id, vm_id)
if fail_ok:
return 2, err_msg
else:
raise exceptions.NeutronError(
'port {} is still listed for vm {} after detaching'.format(
port_id, vm_id))
succ_msg = "Port {} is successfully detached from VM {}".format(port_id,
vm_id)
LOG.info(succ_msg)
if cleanup_route and target_ips:
cleanup_routes_for_vifs(vm_id=vm_id, vm_ips=target_ips, reboot=True)
if verify_virsh and mac_to_check:
if not (cleanup_route and target_ips):
reboot_vm(vm_id=vm_id, auth_info=auth_info, con_ssh=con_ssh)
check_devs_detached(vm_id=vm_id, mac_addrs=mac_to_check,
con_ssh=con_ssh)
return 0, succ_msg
def check_devs_detached(vm_id, mac_addrs, con_ssh=None):
if isinstance(mac_addrs, str):
mac_addrs = [mac_addrs]
wait_for_vm_pingable_from_natbox(vm_id, con_ssh=con_ssh)
LOG.info("Check dev detached from vm")
vm_err = ''
with ssh_to_vm_from_natbox(vm_id=vm_id, con_ssh=con_ssh,
retry_timeout=180) as vm_ssh:
for mac_addr in mac_addrs:
if vm_ssh.exec_cmd('ip addr | grep -B 1 "{}"'.format(mac_addr))[0] \
== 0:
vm_err += 'Interface with mac address {} still exists in ' \
'vm\n'.format(mac_addr)
LOG.info("Check virsh xmldump on compute host")
inst_name, vm_host = get_vm_values(vm_id,
fields=[":instance_name", ":host"],
strict=False)
host_err = ''
with host_helper.ssh_to_host(vm_host, con_ssh=con_ssh) as host_ssh:
for mac_addr in mac_addrs:
if host_ssh.exec_sudo_cmd(
'virsh dumpxml {} | grep -B 1 -A 1 "{}"'.format(
inst_name, mac_addr))[0] == 0:
host_err += 'VM interface with mac address {} still exists in' \
' virsh\n'.format(mac_addr)
assert not host_err, host_err
assert not vm_err, vm_err
def evacuate_vms(host, vms_to_check, con_ssh=None, timeout=600,
wait_for_host_up=False, fail_ok=False, post_host=None,
force=True, ping_vms=False):
"""
Evacuate given vms by rebooting their host. VMs should be on specified
host already when this keyword called.
Args:
host (str): host to reboot
vms_to_check (list): vms to check status for after host reboot
con_ssh (SSHClient):
timeout (int): Max time to wait for vms to reach active state after
reboot -f initiated on host
wait_for_host_up (bool): whether to wait for host reboot completes
before checking vm status
fail_ok (bool): whether to return or to fail test when vm(s) failed
to evacuate
post_host (str): expected host for vms to be evacuated to
force (bool): whether to use 'reboot -f'. This param is only used if
vlm=False.
ping_vms (bool): whether to ping vms after evacuation
Returns (tuple): (<code> (int), <vms_failed_to_evac> (list))
- (0, []) all vms evacuated successfully. i.e., active state,
host changed, pingable from NatBox
- (1, <inactive_vms>) some vms did not reach active state after
host reboot
- (2, <vms_host_err>) some vms' host did not change after host reboot
"""
if isinstance(vms_to_check, str):
vms_to_check = [vms_to_check]
HostsToRecover.add(host)
is_swacted = False
standby = None
if wait_for_host_up:
active, standby = system_helper.get_active_standby_controllers(
con_ssh=con_ssh)
if standby and active == host:
is_swacted = True
is_sx = system_helper.is_aio_simplex()
LOG.tc_step("'sudo reboot -f' from {}".format(host))
host_helper.reboot_hosts(host, wait_for_offline=True,
wait_for_reboot_finish=False, force_reboot=force,
con_ssh=con_ssh)
if is_sx:
host_helper.wait_for_hosts_ready(hosts=host, con_ssh=con_ssh)
try:
LOG.tc_step(
"Wait for vms to reach ERROR or REBUILD state with best effort")
if not is_sx:
wait_for_vms_values(vms_to_check,
value=[VMStatus.ERROR, VMStatus.REBUILD],
fail_ok=True, timeout=120,
con_ssh=con_ssh)
LOG.tc_step(
"Check vms are in Active state and moved to other host(s) ("
"non-sx) after host failure")
res, active_vms, inactive_vms = wait_for_vms_values(
vms=vms_to_check, value=VMStatus.ACTIVE, timeout=timeout,
con_ssh=con_ssh)
if not is_sx:
vms_host_err = []
for vm in vms_to_check:
if post_host:
if get_vm_host(vm) != post_host:
vms_host_err.append(vm)
else:
if get_vm_host(vm) == host:
vms_host_err.append(vm)
if vms_host_err:
if post_host:
err_msg = "Following VMs is not moved to expected host " \
"{} from {}: {}\nVMs did not reach Active " \
"state: {}".format(post_host, host, vms_host_err,
inactive_vms)
else:
err_msg = "Following VMs stayed on the same host {}: " \
"{}\nVMs did not reach Active state: {}".\
format(host, vms_host_err, inactive_vms)
if fail_ok:
LOG.warning(err_msg)
return 1, vms_host_err
raise exceptions.VMError(err_msg)
if inactive_vms:
err_msg = "VMs did not reach Active state after vm host rebooted:" \
" {}".format(inactive_vms)
if fail_ok:
LOG.warning(err_msg)
return 2, inactive_vms
raise exceptions.VMError(err_msg)
if ping_vms:
LOG.tc_step("Ping vms after evacuated")
for vm_ in vms_to_check:
wait_for_vm_pingable_from_natbox(vm_id=vm_,
timeout=VMTimeout.DHCP_RETRY)
LOG.info("All vms are successfully evacuated to other host")
return 0, []
finally:
if wait_for_host_up:
LOG.tc_step("Waiting for {} to recover".format(host))
host_helper.wait_for_hosts_ready(host, con_ssh=con_ssh)
# Do not fail the test due to task affining incomplete for now to
# unblock test case.
host_helper.wait_for_tasks_affined(host=host, con_ssh=con_ssh,
fail_ok=True)
if is_swacted:
host_helper.wait_for_tasks_affined(standby, con_ssh=con_ssh,
fail_ok=True)
time.sleep(60) # Give some idle time before continue.
if system_helper.is_aio_duplex(con_ssh=con_ssh):
system_helper.wait_for_alarm_gone(
alarm_id=EventLogID.CPU_USAGE_HIGH, fail_ok=True,
check_interval=30)
def boot_vms_various_types(storage_backing=None, target_host=None,
cleanup='function', avail_zone='nova', vms_num=5):
"""
Boot following 5 vms and ensure they are pingable from NatBox:
- vm1: ephemeral=0, swap=0, boot_from_volume
- vm2: ephemeral=1, swap=1, boot_from_volume
- vm3: ephemeral=0, swap=0, boot_from_image
- vm4: ephemeral=0, swap=0, boot_from_image, attach_volume
- vm5: ephemeral=1, swap=1, boot_from_image
Args:
storage_backing (str|None): storage backing to set in flavor spec.
When None, storage backing which used by
most up hypervisors will be used.
target_host (str|None): Boot vm on target_host when specified. (admin
role has to be added to tenant under test)
cleanup (str|None): Scope for resource cleanup, valid values:
'function', 'class', 'module', None.
When None, vms/volumes/flavors will be kept on system
avail_zone (str): availability zone to boot the vms
vms_num
Returns (list): list of vm ids
"""
LOG.info("Create a flavor without ephemeral or swap disks")
flavor_1 = \
nova_helper.create_flavor('flv_rootdisk',
storage_backing=storage_backing,
cleanup=cleanup)[1]
LOG.info("Create another flavor with ephemeral and swap disks")
flavor_2 = nova_helper.create_flavor('flv_ephemswap', ephemeral=1, swap=512,
storage_backing=storage_backing,
cleanup=cleanup)[1]
launched_vms = []
for i in range(int(math.ceil(vms_num / 5.0))):
LOG.info(
"Boot vm1 from volume with flavor flv_rootdisk and wait for it "
"pingable from NatBox")
vm1_name = "vol_root"
vm1 = boot_vm(vm1_name, flavor=flavor_1, source='volume',
avail_zone=avail_zone, vm_host=target_host,
cleanup=cleanup)[1]
wait_for_vm_pingable_from_natbox(vm1)
launched_vms.append(vm1)
if len(launched_vms) == vms_num:
break
LOG.info(
"Boot vm2 from volume with flavor flv_localdisk and wait for it "
"pingable from NatBox")
vm2_name = "vol_ephemswap"
vm2 = boot_vm(vm2_name, flavor=flavor_2, source='volume',
avail_zone=avail_zone, vm_host=target_host,
cleanup=cleanup)[1]
wait_for_vm_pingable_from_natbox(vm2)
launched_vms.append(vm2)
if len(launched_vms) == vms_num:
break
LOG.info(
"Boot vm3 from image with flavor flv_rootdisk and wait for it "
"pingable from NatBox")
vm3_name = "image_root"
vm3 = boot_vm(vm3_name, flavor=flavor_1, source='image',
avail_zone=avail_zone, vm_host=target_host,
cleanup=cleanup)[1]
wait_for_vm_pingable_from_natbox(vm3)
launched_vms.append(vm3)
if len(launched_vms) == vms_num:
break
LOG.info(
"Boot vm4 from image with flavor flv_rootdisk, attach a volume to "
"it and wait for it "
"pingable from NatBox")
vm4_name = 'image_root_attachvol'
vm4 = boot_vm(vm4_name, flavor_1, source='image', avail_zone=avail_zone,
vm_host=target_host,
cleanup=cleanup)[1]
vol = cinder_helper.create_volume(bootable=False, cleanup=cleanup)[1]
attach_vol_to_vm(vm4, vol_id=vol, cleanup=cleanup)
wait_for_vm_pingable_from_natbox(vm4)
launched_vms.append(vm4)
if len(launched_vms) == vms_num:
break
LOG.info(
"Boot vm5 from image with flavor flv_localdisk and wait for it "
"pingable from NatBox")
vm5_name = 'image_ephemswap'
vm5 = boot_vm(vm5_name, flavor_2, source='image', avail_zone=avail_zone,
vm_host=target_host,
cleanup=cleanup)[1]
wait_for_vm_pingable_from_natbox(vm5)
launched_vms.append(vm5)
if len(launched_vms) == vms_num:
break
assert len(launched_vms) == vms_num
return launched_vms
def get_vcpu_model(vm_id, guest_os=None, con_ssh=None):
"""
Get vcpu model of given vm. e.g., Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz
Args:
vm_id (str):
guest_os (str):
con_ssh (SSHClient):
Returns (str):
"""
with ssh_to_vm_from_natbox(vm_id, vm_image_name=guest_os,
con_ssh=con_ssh) as vm_ssh:
out = vm_ssh.exec_cmd("cat /proc/cpuinfo | grep --color=never "
"'model name'", fail_ok=False)[1]
vcpu_model = out.strip().splitlines()[0].split(sep=': ')[1].strip()
LOG.info("VM {} cpu model: {}".format(vm_id, vcpu_model))
return vcpu_model
def get_quotas(quotas, default=False, tenant=None, auth_info=None,
con_ssh=None):
"""
Get openstack quotas
Args:
quotas (str|list|tuple):
default (bool)
tenant (str|None): Only used if admin user is used in auth_info
auth_info (dict):
con_ssh:
Returns (list):
"""
if auth_info is None:
auth_info = Tenant.get_primary()
args = ''
if default:
args += '--default'
if tenant and auth_info['user'] == 'admin':
args += ' {}'.format(tenant)
if isinstance(quotas, str):
quotas = [quotas]
table_ = table_parser.table(
cli.openstack('quota show', args, ssh_client=con_ssh,
auth_info=auth_info)[1])
values = []
for item in quotas:
val = table_parser.get_value_two_col_table(table_, item)
try:
val = eval(val)
except (NameError, SyntaxError):
pass
values.append(val)
return values
def get_quota_details_info(component='compute', tenant=None, detail=True,
resources=None,
auth_info=Tenant.get('admin'), con_ssh=None):
"""
Get quota details table from openstack quota list --detail
Args:
component (str): compute, network or volume
tenant:
detail (bool)
resources (str|list|tuple|None): filter out table. Used only if
detail is True and component is not volume
auth_info:
con_ssh:
Returns (dict): All keys are converted to lower case.
e.g.,
{'server_groups': {'in use': 0, 'reserved': 1, 'limit': 10},
...}
"""
valid_components = ('compute', 'network', 'volume')
if component not in valid_components:
raise ValueError(
"Please specify a valid component: {}".format(valid_components))
if not tenant:
tenant = Tenant.get_primary()['tenant']
detail_str = ' --detail' if detail and component != 'volume' else ''
args = '--project={} --{}{}'.format(tenant, component, detail_str)
table_ = table_parser.table(
cli.openstack('quota list', args, ssh_client=con_ssh,
auth_info=auth_info)[1])
key_header = 'Project ID'
if detail_str:
if resources:
table_ = table_parser.filter_table(table_, Resource=resources)
key_header = 'resource'
table_ = table_parser.row_dict_table(table_, key_header=key_header,
lower_case=True,
eliminate_keys=key_header)
return {k: int(v) for k, v in table_.items()}
def set_quotas(tenant=None, auth_info=Tenant.get('admin'), con_ssh=None,
sys_con_for_dc=True, fail_ok=False,
**kwargs):
"""
Set openstack quotas
Args:
tenant (str):
auth_info (dict):
con_ssh:
sys_con_for_dc (bool):
fail_ok (bool):
**kwargs: quotas to set. e.g., **{'instances': 10, 'volumes': 20}
Returns (tuple):
"""
if not tenant:
tenant = Tenant.get_primary()['tenant']
if not auth_info:
auth_info = Tenant.get_primary()
if ProjVar.get_var('IS_DC') and sys_con_for_dc and auth_info['region'] \
!= 'SystemController':
auth_info = Tenant.get(auth_info['user'], dc_region='SystemController')
args = common.parse_args(
args_dict={k.replace('_', '-'): v for k, v in kwargs.items()})
args = '{} {}'.format(args, tenant)
code, output = cli.openstack('quota set', args, ssh_client=con_ssh,
fail_ok=fail_ok, auth_info=auth_info)
if code > 0:
return 1, output
msg = '{} quotas set successfully'.format(tenant)
LOG.info(msg)
return 0, msg
def ensure_vms_quotas(vms_num=10, cores_num=None, vols_num=None, ram=None,
tenant=None, auth_info=Tenant.get('admin'),
con_ssh=None):
"""
Update instances, cores, volumes quotas to given numbers
Args:
vms_num (int): max number of instances allowed for given tenant
cores_num (int|None): twice of the vms quota when None
vols_num (int|None): twice of the vms quota when None
ram (int|None)
tenant (None|str):
auth_info (dict): auth info for admin user
con_ssh (SSHClient):
"""
if not vols_num:
vols_num = 2 * vms_num
if not cores_num:
cores_num = 2 * vms_num
if not ram:
ram = 2048 * vms_num
if not tenant:
tenant = Tenant.get_primary()['tenant']
volumes_quota, vms_quota, cores_quota, ram_quota = get_quotas(
quotas=['volumes', 'instances', 'cores', 'ram'],
con_ssh=con_ssh, tenant=tenant, auth_info=auth_info)
kwargs = {}
if vms_num > vms_quota:
kwargs['instances'] = vms_num
if cores_num > cores_quota:
kwargs['cores'] = cores_num
if vols_num > volumes_quota:
kwargs['volumes'] = vols_num
if ram > ram_quota:
kwargs['ram'] = ram
if kwargs:
set_quotas(con_ssh=con_ssh, tenant=tenant, auth_info=auth_info,
**kwargs)
def launch_vms(vm_type, count=1, nics=None, flavor=None, storage_backing=None,
image=None, boot_source=None,
guest_os=None, avail_zone=None, target_host=None, ping_vms=False,
con_ssh=None, auth_info=None,
cleanup='function', **boot_vm_kwargs):
"""
Args:
vm_type:
count:
nics:
flavor:
storage_backing (str):
storage backend for flavor to be created
only used if flavor is None
image:
boot_source:
guest_os
avail_zone:
target_host:
ping_vms
con_ssh:
auth_info:
cleanup:
boot_vm_kwargs (dict):
additional kwargs to pass to boot_vm
Returns:
"""
if not flavor:
flavor = nova_helper.create_flavor(name=vm_type, vcpus=2,
storage_backing=storage_backing,
cleanup=cleanup)[1]
extra_specs = {FlavorSpec.CPU_POLICY: 'dedicated'}
if vm_type in ['vswitch', 'dpdk', 'vhost']:
extra_specs.update({FlavorSpec.VCPU_MODEL: 'SandyBridge',
FlavorSpec.MEM_PAGE_SIZE: '2048'})
nova_helper.set_flavor(flavor=flavor, **extra_specs)
resource_id = None
boot_source = boot_source if boot_source else 'volume'
if image:
if boot_source == 'volume':
resource_id = \
cinder_helper.create_volume(name=vm_type, source_id=image,
auth_info=auth_info,
guest_image=guest_os)[1]
if cleanup:
ResourceCleanup.add('volume', resource_id, scope=cleanup)
else:
resource_id = image
if not nics:
if vm_type in ['pci-sriov', 'pci-passthrough']:
raise NotImplemented("nics has to be provided for pci-sriov and "
"pci-passthrough")
if vm_type in ['vswitch', 'dpdk', 'vhost']:
vif_model = 'avp'
else:
vif_model = vm_type
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info)
tenant_net_id = network_helper.get_tenant_net_id(auth_info=auth_info)
internal_net_id = network_helper.get_internal_net_id(
auth_info=auth_info)
nics = [{'net-id': mgmt_net_id},
{'net-id': tenant_net_id, 'vif-model': vif_model},
{'net-id': internal_net_id, 'vif-model': vif_model}]
user_data = None
if vm_type in ['vswitch', 'dpdk', 'vhost']:
user_data = network_helper.get_dpdk_user_data(con_ssh=con_ssh)
vms = []
for i in range(count):
vm_id = boot_vm(name="{}-{}".format(vm_type, i), flavor=flavor,
source=boot_source, source_id=resource_id,
nics=nics, guest_os=guest_os, avail_zone=avail_zone,
vm_host=target_host, user_data=user_data,
auth_info=auth_info, con_ssh=con_ssh, cleanup=cleanup,
**boot_vm_kwargs)[1]
vms.append(vm_id)
if ping_vms:
wait_for_vm_pingable_from_natbox(vm_id=vm_id, con_ssh=con_ssh)
return vms, nics
def get_ping_loss_duration_between_vms(from_vm, to_vm, net_type='data',
timeout=600, ipv6=False,
start_event=None,
end_event=None, con_ssh=None,
ping_interval=1):
"""
Get ping loss duration in milliseconds from one vm to another
Args:
from_vm (str): id of the ping source vm
to_vm (str): id of the ping destination vm
net_type (str): e.g., data, internal, mgmt
timeout (int): max time to wait for ping loss before force end it
ipv6 (bool): whether to use ping -6 for ipv6 address
start_event (Event): set given event to signal ping has started
end_event (Event): stop ping loss detection if given event is set
con_ssh (SSHClient):
ping_interval (int|float): timeout of ping cmd in seconds
Returns (int): milliseconds of ping loss duration
"""
to_vm_ip = _get_vms_ips(vm_ids=to_vm, net_types=net_type,
con_ssh=con_ssh)[0][0]
with ssh_to_vm_from_natbox(vm_id=from_vm, con_ssh=con_ssh) as from_vm_ssh:
duration = network_helper.get_ping_failure_duration(
server=to_vm_ip, ssh_client=from_vm_ssh, timeout=timeout,
ipv6=ipv6, start_event=start_event, end_event=end_event,
ping_interval=ping_interval)
return duration
def get_ping_loss_duration_from_natbox(vm_id, timeout=900, start_event=None,
end_event=None, con_ssh=None,
ping_interval=0.5):
vm_ip = _get_vms_ips(vm_ids=vm_id, net_types='mgmt', con_ssh=con_ssh)[0][0]
natbox_client = NATBoxClient.get_natbox_client()
duration = network_helper.get_ping_failure_duration(
server=vm_ip, ssh_client=natbox_client, timeout=timeout,
start_event=start_event, end_event=end_event,
ping_interval=ping_interval)
return duration
def get_ping_loss_duration_on_operation(vm_id, timeout, ping_interval,
oper_func, *func_args, **func_kwargs):
LOG.tc_step("Start pinging vm {} from NatBox on a new thread".format(vm_id))
start_event = Events("Ping started")
end_event = Events("Operation completed")
ping_thread = MThread(get_ping_loss_duration_from_natbox, vm_id=vm_id,
timeout=timeout,
start_event=start_event, end_event=end_event,
ping_interval=ping_interval)
ping_thread.start_thread(timeout=timeout + 30)
try:
if start_event.wait_for_event(timeout=60):
LOG.tc_step(
"Perform operation on vm and ensure it's reachable after that")
oper_func(*func_args, **func_kwargs)
# Operation completed. Set end flag so ping thread can end properly
time.sleep(3)
end_event.set()
# Expect ping thread to end in less than 1 minute after
# live-migration complete
duration = ping_thread.get_output(timeout=60)
# assert duration, "No ping loss detected"
if duration == 0:
LOG.warning("No ping loss detected")
return duration
assert False, "Ping failed since start"
finally:
ping_thread.wait_for_thread_end(timeout=5)
def collect_guest_logs(vm_id):
LOG.info("Attempt to collect guest logs with best effort")
log_names = ['messages', 'user.log']
try:
res = _recover_vm(vm_id=vm_id)
if not res:
LOG.info(
"VM {} in unrecoverable state, skip collect guest logs.".format(
vm_id))
return
with ssh_to_vm_from_natbox(vm_id) as vm_ssh:
for log_name in log_names:
log_path = '/var/log/{}'.format(log_name)
if not vm_ssh.file_exists(log_path):
continue
local_log_path = '{}/{}_{}'.format(
ProjVar.get_var('GUEST_LOGS_DIR'), log_name, vm_id)
current_user = local_host.get_user()
if current_user == TestFileServer.USER:
vm_ssh.exec_sudo_cmd('chmod -R 755 {}'.format(log_path),
fail_ok=True)
vm_ssh.scp_on_source_to_localhost(
source_file=log_path,
dest_user=current_user,
dest_password=TestFileServer.PASSWORD,
dest_path=local_log_path)
else:
output = vm_ssh.exec_cmd('tail -n 200 {}'.format(log_path),
fail_ok=False)[1]
with open(local_log_path, mode='w') as f:
f.write(output)
return
except Exception as e:
LOG.warning("Failed to collect guest logs: {}".format(e))
def _recover_vm(vm_id, con_ssh=None):
status = get_vm_status(vm_id=vm_id, con_ssh=con_ssh)
if status == VMStatus.ACTIVE:
return True
elif status == VMStatus.STOPPED:
code, msg = start_vms(vms=vm_id, fail_ok=True)
return code == 0
elif status == VMStatus.PAUSED:
code, msg = unpause_vm(vm_id=vm_id, fail_ok=True, con_ssh=con_ssh)
if code > 0:
code, msg = resume_vm(vm_id, fail_ok=True, con_ssh=con_ssh)
if code > 0:
return False
return True
else:
return False
def get_vim_events(vm_id, event_ids=None, controller=None, con_ssh=None):
"""
Get vim events from nfv-vim-events.log
Args:
vm_id (str):
event_ids (None|str|list|tuple): return only given vim events when
specified
controller (None|str): controller where vim log is on. Use current
active controller if None.
con_ssh (SSHClient):
Returns (list): list of dictionaries, each dictionary is one event. e.g.,:
[{'log-id': '47', 'event-id': 'instance-live-migrate-begin', ... ,
'timestamp': '2018-03-04 01:34:28.915008'},
{'log-id': '49', 'event-id': 'instance-live-migrated', ... ,
'timestamp': '2018-03-04 01:35:34.043094'}]
"""
if not controller:
controller = system_helper.get_active_controller_name()
if isinstance(event_ids, str):
event_ids = [event_ids]
with host_helper.ssh_to_host(controller, con_ssh=con_ssh) as controller_ssh:
vm_logs = controller_ssh.exec_cmd(
'grep --color=never -A 4 -B 6 -E "entity .*{}" '
'/var/log/nfv-vim-events.log'.
format(vm_id))[1]
log_lines = vm_logs.splitlines()
vm_events = []
vm_event = {}
for line in log_lines:
if re.search(' = ', line):
if line.startswith('log-id') and vm_event:
if not event_ids or vm_event['event-id'] in event_ids:
vm_events.append(vm_event)
vm_event = {}
key, val = re.findall('(.*)= (.*)', line)[0]
vm_event[key.strip()] = val.strip()
if vm_event and (not event_ids or vm_event['event-id'] in event_ids):
vm_events.append(vm_event)
LOG.info("VM events: {}".format(vm_events))
return vm_events
def get_live_migrate_duration(vm_id, con_ssh=None):
LOG.info(
"Get live migration duration from nfv-vim-events.log for vm {}".format(
vm_id))
events = (VimEventID.LIVE_MIG_BEGIN, VimEventID.LIVE_MIG_END)
live_mig_begin, live_mig_end = get_vim_events(vm_id=vm_id, event_ids=events,
con_ssh=con_ssh)
start_time = live_mig_begin['timestamp']
end_time = live_mig_end['timestamp']
duration = common.get_timedelta_for_isotimes(time1=start_time,
time2=end_time).total_seconds()
LOG.info("Live migration for vm {} took {} seconds".format(vm_id, duration))
return duration
def get_cold_migrate_duration(vm_id, con_ssh=None):
LOG.info("Get cold migration duration from vim-event-log for vm {}".format(
vm_id))
events = (VimEventID.COLD_MIG_BEGIN, VimEventID.COLD_MIG_END,
VimEventID.COLD_MIG_CONFIRM_BEGIN, VimEventID.COLD_MIG_CONFIRMED)
cold_mig_begin, cold_mig_end, cold_mig_confirm_begin, \
cold_mig_confirm_end = get_vim_events(vm_id=vm_id, event_ids=events,
con_ssh=con_ssh)
duration_cold_mig = common.get_timedelta_for_isotimes(
time1=cold_mig_begin['timestamp'],
time2=cold_mig_end['timestamp']).total_seconds()
duration_confirm = common.get_timedelta_for_isotimes(
time1=cold_mig_confirm_begin['timestamp'],
time2=cold_mig_confirm_end['timestamp']).total_seconds()
duration = duration_cold_mig + duration_confirm
LOG.info("Cold migrate and confirm for vm {} took {} seconds".format(
vm_id, duration))
return duration
def live_migrate_force_complete(vm_id, migration_id=None, timeout=300,
fail_ok=False, con_ssh=None):
"""
Run nova live-migration-force-complete against given vm and migration
session.
Args:
vm_id (str):
migration_id (str|int):
timeout:
fail_ok:
con_ssh:
Returns (tuple):
(0, 'VM is successfully live-migrated after
live-migration-force-complete')
(1, <err_msg>) # nova live-migration-force-complete cmd
rejected. Only returns if fail_ok=True.
"""
if not migration_id:
migration_id = get_vm_migration_values(vm_id=vm_id, fail_ok=False,
con_ssh=con_ssh)[0]
# No replacement in openstack client
code, output = cli.nova('live-migration-force-complete',
'{} {}'.format(vm_id, migration_id),
ssh_client=con_ssh,
fail_ok=fail_ok)
if code > 0:
return 1, output
wait_for_vm_migration_status(vm_id=vm_id, migration_id=migration_id,
fail_ok=False, timeout=timeout,
con_ssh=con_ssh)
msg = "VM is successfully live-migrated after live-migration-force-complete"
LOG.info(msg)
return 0, msg
def get_vm_migration_values(vm_id, field='Id', migration_type='live-migration',
fail_ok=True, con_ssh=None, **kwargs):
"""
Get values for given vm via nova migration-list
Args:
vm_id (str):
field (str):
migration_type(str): valid types: live-migration, migration
fail_ok:
con_ssh:
**kwargs:
Returns (list):
"""
migration_tab = nova_helper.get_migration_list_table(con_ssh=con_ssh)
filters = {'Instance UUID': vm_id, 'Type': migration_type}
if kwargs:
filters.update(kwargs)
mig_ids = table_parser.get_values(migration_tab, target_header=field,
**filters)
if not mig_ids and not fail_ok:
raise exceptions.VMError(
"{} has no {} session with filters: {}".format(vm_id,
migration_type,
kwargs))
return mig_ids
def wait_for_vm_migration_status(vm_id, migration_id=None, migration_type=None,
expt_status='completed',
fail_ok=False, timeout=300, check_interval=5,
con_ssh=None):
"""
Wait for a migration session to reach given status in nova mgiration-list
Args:
vm_id (str):
migration_id (str|int):
migration_type (str): valid types: live-migration, migration
expt_status (str): migration status to wait for. such as completed,
running, etc
fail_ok (bool):
timeout (int): max time to wait for the state
check_interval (int):
con_ssh:
Returns (tuple):
(0, <expt_status>) # migration status reached as expected
(1, <current_status>) # did not reach given status. This only
returns if fail_ok=True
"""
if not migration_id:
migration_id = get_vm_migration_values(
vm_id=vm_id, migration_type=migration_type, fail_ok=False,
con_ssh=con_ssh)[0]
LOG.info("Waiting for migration {} for vm {} to reach {} status".format(
migration_id, vm_id, expt_status))
end_time = time.time() + timeout
prev_state = None
while time.time() < end_time:
mig_status = get_vm_migration_values(vm_id=vm_id, field='Status',
**{'Id': migration_id})[0]
if mig_status == expt_status:
LOG.info(
"Migration {} for vm {} reached status: {}".format(migration_id,
vm_id,
expt_status))
return True, expt_status
if mig_status != prev_state:
LOG.info(
"Migration {} for vm {} is in status - {}".format(migration_id,
vm_id,
mig_status))
prev_state = mig_status
time.sleep(check_interval)
msg = 'Migration {} for vm {} did not reach {} status within {} seconds. ' \
'It is in {} status.'.format(migration_id, vm_id, expt_status,
timeout, prev_state)
if fail_ok:
LOG.warning(msg)
return False, prev_state
else:
raise exceptions.VMError(msg)
def get_vms_ports_info(vms, rtn_subnet_id=False):
"""
Get VMs' ports' (ip_addr, subnet_cidr_or_id, mac_addr).
Args:
vms (str|list):
vm_id, or a list of vm_ids
rtn_subnet_id (bool):
replaces cidr with subnet_id in result
Returns (dict):
{vms[0]: [(ip_addr, subnet, ...], vms[1]: [...], ...}
"""
if not issubclass(type(vms), (list, tuple)):
vms = [vms]
info = {}
subnet_tab_ = table_parser.table(
cli.openstack('subnet list', auth_info=Tenant.get('admin'))[1])
for vm in vms:
info[vm] = []
vm_ports, vm_macs, vm_fixed_ips = network_helper.get_ports(
server=vm, field=('ID', 'MAC Address', 'Fixed IP Addresses'))
for i in range(len(vm_ports)):
port = vm_ports[i]
mac = vm_macs[i]
fixed_ips = vm_fixed_ips[i]
if not isinstance(fixed_ips, list):
fixed_ips = [fixed_ips]
for fixed_ip in fixed_ips:
subnet_id = fixed_ip['subnet_id']
ip_addr = fixed_ip['ip_address']
subnet = subnet_id if rtn_subnet_id else \
table_parser.get_values(subnet_tab_, 'Subnet',
id=subnet_id)[0]
net_id = table_parser.get_values(subnet_tab_, 'Network',
id=subnet_id)[0]
LOG.info(
"VM {} port {}: mac={} ip={} subnet={} net_id={}".format(
vm, port, mac, ip_addr, subnet, net_id))
info[vm].append((port, ip_addr, subnet, mac, net_id))
return info
def _set_vm_route(vm_id, target_subnet, via_ip, dev_or_mac, persist=True):
# returns True if the targeted VM is vswitch-enabled
# for vswitch-enabled VMs, it must be setup with TisInitServiceScript if
# persist=True
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
vshell, msg = ssh_client.exec_cmd("vshell port-list", fail_ok=True)
vshell = not vshell
if ':' in dev_or_mac:
dev = network_helper.get_eth_for_mac(ssh_client, dev_or_mac,
vshell=vshell)
else:
dev = dev_or_mac
if not vshell: # not avs managed
param = target_subnet, via_ip, dev
LOG.info("Routing {} via {} on interface {}".format(*param))
ssh_client.exec_sudo_cmd(
"route add -net {} gw {} {}".format(*param), fail_ok=False)
if persist:
LOG.info("Setting persistent route")
ssh_client.exec_sudo_cmd(
"echo -e \"{} via {}\" > "
"/etc/sysconfig/network-scripts/route-{}".format(
*param), fail_ok=False)
return False
else:
param = target_subnet, via_ip, dev
LOG.info(
"Routing {} via {} on interface {}, AVS-enabled".format(*param))
ssh_client.exec_sudo_cmd(
"sed -i $'s,quit,route add {} {} {} 1\\\\nquit,"
"g' /etc/vswitch/vswitch.cmds.default".format(
target_subnet, dev, via_ip), fail_ok=False)
# reload vswitch
ssh_client.exec_sudo_cmd("/etc/init.d/vswitch restart",
fail_ok=False)
if persist:
LOG.info("Setting persistent route")
ssh_client.exec_sudo_cmd(
# ROUTING_STUB
# "192.168.1.0/24,192.168.111.1,eth0"
"sed -i $'s@#ROUTING_STUB@\"{},{},"
"{}\"\\\\n#ROUTING_STUB@g' {}".format(
target_subnet, via_ip, dev,
TisInitServiceScript.configuration_path
), fail_ok=False)
return True
def route_vm_pair(vm1, vm2, bidirectional=True, validate=True):
"""
Route the pair of VMs' data interfaces through internal interfaces
If multiple interfaces available on either of the VMs, the last one is used
If no interfaces available for data/internal network for either VM,
raises IndexError
The internal interfaces for the pair VM must be on the same gateway
no fail_ok option, since if failed, the vm's state is undefined
Args:
vm1 (str):
vm_id, src if bidirectional=False
vm2 (str):
vm_id, dest if bidirectional=False
bidirectional (bool):
if True, also routes from vm2 to vm1
validate (bool):
validate pings between the pair over the data network
Returns (dict):
the interfaces used for routing,
{vm_id: {'data': {'ip', 'cidr', 'mac'},
'internal':{'ip', 'cidr', 'mac'}}}
"""
if vm1 == vm2:
raise ValueError("cannot route to a VM itself")
auth_info = Tenant.get('admin')
LOG.info("Collecting VMs' networks")
interfaces = {
vm1: {"data": network_helper.get_tenant_ips_for_vms(
vm1, auth_info=auth_info),
"internal": network_helper.get_internal_ips_for_vms(vm1)},
vm2: {"data": network_helper.get_tenant_ips_for_vms(
vm2, auth_info=auth_info),
"internal": network_helper.get_internal_ips_for_vms(vm2)},
}
for vm, info in get_vms_ports_info([vm1, vm2]).items():
for port, ip, cidr, mac, net_id in info:
# expect one data and one internal
if ip in interfaces[vm]['data']:
interfaces[vm]['data'] = {'ip': ip, 'cidr': cidr, 'mac': mac,
'port': port}
elif ip in interfaces[vm]['internal']:
interfaces[vm]['internal'] = {'ip': ip, 'cidr': cidr,
'mac': mac, 'port': port}
if interfaces[vm1]['internal']['cidr'] != \
interfaces[vm2]['internal']['cidr']:
raise ValueError(
"the internal interfaces for the VM pair is not on the same "
"gateway")
vshell_ = _set_vm_route(
vm1,
interfaces[vm2]['data']['cidr'], interfaces[vm2]['internal']['ip'],
interfaces[vm1]['internal']['mac'])
if bidirectional:
_set_vm_route(vm2, interfaces[vm1]['data']['cidr'],
interfaces[vm1]['internal']['ip'],
interfaces[vm2]['internal']['mac'])
for vm in (vm1, vm2):
LOG.info("Add vms' data network ip as allowed address for internal "
"network port")
network_helper.set_port(
port_id=interfaces[vm]['internal']['port'],
auth_info=auth_info,
allowed_addr_pairs={'ip-address': interfaces[vm]['data']['ip']})
if validate:
LOG.info("Validating route(s) across data")
ping_between_routed_vms(to_vm=vm2, from_vm=vm1, vshell=vshell_,
bidirectional=bidirectional)
return interfaces
def ping_between_routed_vms(to_vm, from_vm, vshell=True, bidirectional=True,
timeout=120):
"""
Ping between routed vm pair
Args:
to_vm:
from_vm:
vshell:
bidirectional:
timeout:
Returns:
"""
ping_vms_from_vm(to_vms=to_vm, from_vm=from_vm, timeout=timeout,
net_types='data', vshell=vshell,
source_net_types='internal')
if bidirectional:
ping_vms_from_vm(to_vms=from_vm, from_vm=to_vm, timeout=timeout,
net_types='data', vshell=vshell,
source_net_types='internal')
def setup_kernel_routing(vm_id, **kwargs):
"""
Setup kernel routing function for the specified VM
replicates the operation as in wrs_guest_setup.sh (and comes
with the same assumptions)
in order to persist kernel routing after reboots, the operation has to be
stored in /etc/init.d
see TisInitServiceScript for script details
no fail_ok option, since if failed, the vm's state is undefined
Args:
vm_id (str):
the VM to be configured
kwargs (dict):
kwargs for TisInitServiceScript.configure
"""
LOG.info(
"Setting up kernel routing for VM {}, kwargs={}".format(vm_id, kwargs))
scp_to_vm(vm_id, TisInitServiceScript.src(), TisInitServiceScript.dst())
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
r, msg = ssh_client.exec_cmd("cat /proc/sys/net/ipv4/ip_forward",
fail_ok=False)
if msg == "1":
LOG.warn(
"VM {} has ip_forward enabled already, skipping".format(vm_id))
return
TisInitServiceScript.configure(ssh_client, **kwargs)
TisInitServiceScript.enable(ssh_client)
TisInitServiceScript.start(ssh_client)
def setup_avr_routing(vm_id, mtu=1500, vm_type='vswitch', **kwargs):
"""
Setup avr routing (vswitch L3) function for the specified VM
replciates the operation as in wrs_guest_setup.sh (and comes with
the same assumptions)
in order to persist kernel routing after reboots, the operation has to be
stored in /etc/init.d
see TisInitServiceScript for script details
no fail_ok option, since if failed, the vm's state is undefined
Args:
vm_id (str):
the VM to be configured
mtu (int):
1500 by default
for jumbo frames (9000), tenant net support is required
vm_type (str):
PCI NIC_DEVICE
vhost: "${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_VIRTIO}:
${PCI_SUBDEVICE_NET}"
any other: "${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_MEMORY}:
${PCI_SUBDEVICE_AVP}" (default)
kwargs (dict):
kwargs for TisInitServiceScript.configure
"""
LOG.info(
"Setting up avr routing for VM {}, kwargs={}".format(vm_id, kwargs))
datas = network_helper.get_tenant_ips_for_vms(vm_id)
data_dict = dict()
try:
internals = network_helper.get_internal_ips_for_vms(vm_id)
except ValueError:
internals = list()
internal_dict = dict()
for vm, info in get_vms_ports_info([vm_id]).items():
for port, ip, cidr, mac, net_id in info:
if ip in datas:
data_dict[ip] = ipaddress.ip_network(cidr).netmask
elif ip in internals:
internal_dict[ip] = ipaddress.ip_network(cidr).netmask
interfaces = list()
items = list(data_dict.items()) + list(internal_dict.items())
if len(items) > 2:
LOG.warn(
"wrs_guest_setup/tis_automation_init does not support more than "
"two DPDK NICs")
LOG.warn("stripping {} from interfaces".format(items[2:]))
items = items[:2]
for (ip, netmask), ct in zip(items, range(len(items))):
interfaces.append(
"""\"{},{},eth{},{}\"""".format(ip, netmask, ct, str(mtu)))
nic_device = ""
if vm_type == 'vhost':
nic_device = "\"${PCI_VENDOR_VIRTIO}:${PCI_DEVICE_VIRTIO}:" \
"${PCI_SUBDEVICE_NET}\""
scp_to_vm(vm_id, TisInitServiceScript.src(), TisInitServiceScript.dst())
with ssh_to_vm_from_natbox(vm_id) as ssh_client:
TisInitServiceScript.configure(
ssh_client, NIC_DEVICE=nic_device,
NIC_COUNT=str(len(items)), FUNCTIONS="avr,",
ROUTES="""(
#ROUTING_STUB
)""",
ADDRESSES="""(
{}
)
""".format("\n ".join(interfaces)), **kwargs)
TisInitServiceScript.enable(ssh_client)
TisInitServiceScript.start(ssh_client)
def launch_vm_pair(vm_type='virtio', primary_kwargs=None, secondary_kwargs=None,
**launch_vms_kwargs):
"""
Launch a pair of routed VMs
one on the primary tenant, and the other on the secondary tenant
Args:
vm_type (str):
one of 'virtio', 'avp', 'dpdk'
primary_kwargs (dict):
launch_vms_kwargs for the VM launched under the primary tenant
secondary_kwargs (dict):
launch_vms_kwargs for the VM launched under the secondary tenant
**launch_vms_kwargs:
additional keyword arguments for launch_vms for both tenants
overlapping keys will be overridden by primary_kwargs and
secondary_kwargs
shall not specify count, ping_vms, auth_info
Returns (tuple):
(vm_id_on_primary_tenant, vm_id_on_secondary_tenant)
"""
LOG.info("Launch a {} test-observer pair of VMs".format(vm_type))
for invalid_key in ('count', 'ping_vms'):
if invalid_key in launch_vms_kwargs:
launch_vms_kwargs.pop(invalid_key)
primary_kwargs = dict() if not primary_kwargs else primary_kwargs
secondary_kwargs = dict() if not secondary_kwargs else secondary_kwargs
if 'auth_info' not in primary_kwargs:
primary_kwargs['auth_info'] = Tenant.get_primary()
if 'auth_info' not in secondary_kwargs:
secondary_kwargs['auth_info'] = Tenant.get_secondary()
if 'nics' not in primary_kwargs or 'nics' not in secondary_kwargs:
if vm_type in ['pci-sriov', 'pci-passthrough']:
raise NotImplemented(
"nics has to be provided for pci-sriov and pci-passthrough")
if vm_type in ['vswitch', 'dpdk', 'vhost']:
vif_model = 'avp'
else:
vif_model = vm_type
internal_net_id = network_helper.get_internal_net_id()
for tenant_info in (primary_kwargs, secondary_kwargs):
auth_info_ = tenant_info['auth_info']
mgmt_net_id = network_helper.get_mgmt_net_id(auth_info=auth_info_)
tenant_net_id = network_helper.get_tenant_net_id(
auth_info=auth_info_)
nics = [{'net-id': mgmt_net_id},
{'net-id': tenant_net_id, 'vif-model': vif_model},
{'net-id': internal_net_id, 'vif-model': vif_model}]
tenant_info['nics'] = nics
vm_test = launch_vms(vm_type=vm_type, count=1, ping_vms=True,
**__merge_dict(launch_vms_kwargs, primary_kwargs)
)[0][0]
vm_observer = launch_vms(vm_type=vm_type, count=1, ping_vms=True,
**__merge_dict(launch_vms_kwargs,
secondary_kwargs))[0][0]
LOG.info("Route the {} test-observer VM pair".format(vm_type))
if vm_type in ('dpdk', 'vhost', 'vswitch'):
setup_avr_routing(vm_test, vm_type=vm_type)
setup_avr_routing(vm_observer, vm_type=vm_type)
else:
# vm_type in ('virtio', 'avp'):
setup_kernel_routing(vm_test)
setup_kernel_routing(vm_observer)
route_vm_pair(vm_test, vm_observer)
return vm_test, vm_observer
def get_all_vms(field='ID', con_ssh=None, auth_info=Tenant.get('admin')):
"""
Get VMs for all tenants in the systems
Args:
field:
con_ssh:
auth_info
Returns (list): list of all vms on the system
"""
return get_vms(field=field, all_projects=True, long=False, con_ssh=con_ssh,
auth_info=auth_info)
def get_vms_info(fields, vms=None, con_ssh=None, long=True, all_projects=True,
host=None,
auth_info=Tenant.get('admin')):
"""
Get vms values for given fields
Args:
fields (str|list|tuple):
vms:
con_ssh:
long:
all_projects:
host
auth_info:
Returns (dict): vm as key, values for given fields as value
Examples:
input: fields = [field1, field2]
output: {vm_1: [vm1_field1_value, vm1_field2_value],
vm_2: [vm2_field1_value, vm2_field2_value]}
"""
if isinstance(fields, str):
fields = (fields,)
fields = ['ID'] + list(fields)
values = get_vms(vms=vms, field=fields, con_ssh=con_ssh, long=long,
all_projects=all_projects, host=host,
auth_info=auth_info)
vm_ids = values.pop(0)
values = list(zip(*values))
results = {vm_ids[i]: values[i] for i in range(len(vm_ids))}
return results
def get_vms(vms=None, field='ID', long=False, all_projects=True, host=None,
project=None, project_domain=None,
strict=True, regex=False, con_ssh=None, auth_info=None, **kwargs):
"""
get a list of VM IDs or Names for given tenant in auth_info param.
Args:
vms (list): filter vms from this list if not None
field (str|tuple|list): 'ID' or 'Name'
con_ssh (SSHClient): controller SSHClient.
auth_info (dict): such as ones in auth.py: auth.ADMIN, auth.TENANT1
long (bool): whether to use --long in cmd
project (str)
project_domain (str)
all_projects (bool): whether to use --a in cmd
host (str): value for --host arg in cmd
strict (bool): applies to search for value(s) specified in kwargs
regex (bool): whether to use regular expression to search for the
kwargs value(s)
**kwargs: header/value pair to filter out the vms
Returns (list): list of VMs for tenant(s).
"""
args_dict = {'--long': long,
'--a': all_projects if auth_info and auth_info[
'user'] == 'admin' else None,
'--host': host,
'--project': project,
'--project-domain': project_domain}
args = common.parse_args(args_dict)
table_ = table_parser.table(
cli.openstack('server list', args, ssh_client=con_ssh,
auth_info=auth_info)[1])
if vms:
table_ = table_parser.filter_table(table_, ID=vms)
return table_parser.get_multi_values(table_, field, strict=strict,
regex=regex, **kwargs)
def get_vm_status(vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
return get_vm_values(vm_id, 'status', strict=True, con_ssh=con_ssh,
auth_info=auth_info)[0]
def get_vm_id_from_name(vm_name, con_ssh=None, strict=True, regex=False,
fail_ok=False, auth_info=Tenant.get('admin')):
if not auth_info:
auth_info = Tenant.get_primary()
vm_ids = get_vms(name=vm_name, strict=strict, regex=regex, con_ssh=con_ssh,
auth_info=auth_info)
if not vm_ids:
err_msg = "No vm found with name: {}".format(vm_name)
LOG.info(err_msg)
if fail_ok:
return ''
raise exceptions.VMError(err_msg)
return vm_ids[0]
def get_vm_name_from_id(vm_id, con_ssh=None, auth_info=None):
return get_vm_values(vm_id, fields='name', con_ssh=con_ssh,
auth_info=auth_info)[0]
def get_vm_volumes(vm_id, con_ssh=None, auth_info=None):
"""
Get volume ids attached to given vm.
Args:
vm_id (str):
con_ssh (SSHClient):
auth_info (dict):
Returns (tuple): list of volume ids attached to specific vm
"""
table_ = table_parser.table(
cli.openstack('server show', vm_id, ssh_client=con_ssh,
auth_info=auth_info)[1])
return _get_vm_volumes(table_)
def get_vm_values(vm_id, fields, strict=True, con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Get vm values via openstack server show
Args:
vm_id (str):
fields (str|list|tuple): fields in openstack server show table
strict (bool): whether to perform a strict search on given field name
con_ssh (SSHClient):
auth_info (dict|None):
Returns (list): values for given fields
"""
if isinstance(fields, str):
fields = [fields]
table_ = table_parser.table(
cli.openstack('server show', vm_id, ssh_client=con_ssh,
auth_info=auth_info)[1])
values = []
for field in fields:
merge = False
if field in ('fault',):
merge = True
value = table_parser.get_value_two_col_table(table_, field, strict,
merge_lines=merge)
if field in ('properties',):
value = table_parser.convert_value_to_dict(value)
elif field in ('security_groups',):
if isinstance(value, str):
value = [value]
value = [re.findall("name='(.*)'", v)[0] for v in value]
values.append(value)
return values
def get_vm_fault_message(vm_id, con_ssh=None, auth_info=None):
return get_vm_values(vm_id=vm_id, fields='fault', con_ssh=con_ssh,
auth_info=auth_info)[0]
def get_vm_flavor(vm_id, field='id', con_ssh=None, auth_info=None):
"""
Get flavor id of given vm
Args:
vm_id (str):
field (str): id or name
con_ssh (SSHClient):
auth_info (dict):
Returns (str):
"""
flavor = get_vm_values(vm_id, fields='flavor', strict=True, con_ssh=con_ssh,
auth_info=auth_info)[0]
flavor_name, flavor_id = flavor.split('(')
if field == 'id':
flavor = flavor_id.strip().split(')')[0]
else:
flavor = flavor_name.strip()
return flavor
def get_vm_host(vm_id, con_ssh=None, auth_info=Tenant.get('admin')):
"""
Get host of given vm via openstack server show
Args:
vm_id:
con_ssh:
auth_info
Returns (str):
"""
return get_vm_values(vm_id, ':host', strict=False, con_ssh=con_ssh,
auth_info=auth_info)[0]
def get_vms_hosts(vm_ids, con_ssh=None, auth_info=Tenant.get('admin')):
"""
Get vms' hosts via openstack server list
Args:
vm_ids:
con_ssh:
auth_info
Returns:
"""
vms_hosts = get_vms_info(vms=vm_ids, fields='host', auth_info=auth_info,
con_ssh=con_ssh)
vms_hosts = [vms_hosts[vm][0] for vm in vm_ids]
return vms_hosts
def get_vms_on_host(hostname, field='ID', con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Get vms on given host
Args:
field: ID or Name
hostname (str):Name of a compute node
con_ssh:
auth_info
Returns (list): A list of VMs' ID under a hypervisor
"""
vms = get_vms(host=hostname, all_projects=True, long=False, con_ssh=con_ssh,
auth_info=auth_info, field=field)
return vms
def get_vms_per_host(vms=None, con_ssh=None, auth_info=Tenant.get('admin')):
"""
Get vms per host
Args:
vms
con_ssh (SSHClient):
auth_info (dict)
Returns (dict):return a dictionary where the host(hypervisor) is the key
and value are a list of VMs under the host
"""
vms_hosts = get_vms_info(vms=vms, fields='host', auth_info=auth_info,
con_ssh=con_ssh, long=True, all_projects=True)
vms_per_host = {}
for vm in vms_hosts:
host = vms_hosts[vm][0]
if host in vms_per_host:
vms_per_host[host].append(vm)
else:
vms_per_host[host] = [vm]
return vms_per_host
def _get_boot_info(table_, vm_id, auth_info=None, con_ssh=None):
image = table_parser.get_value_two_col_table(table_, 'image')
if not image:
volumes = _get_vm_volumes(table_)
if len(volumes) == 0:
raise exceptions.VMError(
"Booted from volume, but no volume id found.")
from keywords import cinder_helper
if len(volumes) == 1:
vol_id = volumes[0]
vol_name, image_info = cinder_helper.get_volume_show_values(
vol_id, auth_info=auth_info, con_ssh=con_ssh,
fields=('name', 'volume_image_metadata'))
LOG.info("VM booted from volume.")
return {'type': 'volume', 'id': vol_id, 'volume_name': vol_name,
'image_name': image_info['image_name']}
else:
LOG.info(
"VM booted from volume. Multiple volumes found, taking the "
"first boot-able volume.")
for volume in volumes:
bootable, vol_name, image_info = \
cinder_helper.get_volume_show_values(
volume,
fields=('bootable', 'name', 'volume_image_metadata'),
auth_info=auth_info, con_ssh=con_ssh)
if str(bootable).lower() == 'true':
return {'type': 'volume', 'id': volume,
'volume_name': vol_name,
'image_name': image_info['image_name']}
raise exceptions.VMError(
"VM {} has no bootable volume attached.".format(vm_id))
else:
name, img_uuid = image.strip().split(sep='(')
return {'type': 'image', 'id': img_uuid.split(sep=')')[0],
'image_name': name.strip()}
def get_vm_boot_info(vm_id, auth_info=None, con_ssh=None):
"""
Get vm boot source and id.
Args:
vm_id (str):
auth_info (dict|None):
con_ssh (SSHClient):
Returns (dict): VM boot info dict.
Format: {'type': <boot_source>, 'id': <source_id>}.
<boot_source> is either 'volume' or 'image'
"""
table_ = table_parser.table(
cli.openstack('server show', vm_id, ssh_client=con_ssh,
auth_info=auth_info)[1])
return _get_boot_info(table_, vm_id=vm_id, auth_info=auth_info,
con_ssh=con_ssh)
def get_vm_image_name(vm_id, auth_info=None, con_ssh=None):
"""
Args:
vm_id (str):
auth_info (dict):
con_ssh (SSHClient):
Returns (str): image name for the vm. If vm booted from volume,
then image name in volume image metadata will be returned.
"""
boot_info = get_vm_boot_info(vm_id, auth_info=auth_info, con_ssh=con_ssh)
return boot_info['image_name']
def _get_vm_volumes(table_):
"""
Args:
table_ (dict):
Returns (list: A list of volume ids from the novashow_table.
"""
volumes = table_parser.get_value_two_col_table(table_, 'volumes_attached',
merge_lines=False)
if not volumes:
return []
if isinstance(volumes, str):
volumes = [volumes]
return [re.findall("id='(.*)'", volume)[0] for volume in volumes]
def get_vm_instance_name(vm_id, con_ssh=None):
return get_vm_values(vm_id, ":instance_name", strict=False,
con_ssh=con_ssh)[0]
View Git Blame Copy Permalink