490 lines
20 KiB
Python
490 lines
20 KiB
Python
#
|
|
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
import copy
|
|
import os
|
|
|
|
from sysinv.common import constants
|
|
from sysinv.common import exception
|
|
from sysinv.common import interface
|
|
from sysinv.common import utils
|
|
from sysinv.openstack.common import log as logging
|
|
from sysinv.helm import common
|
|
from sysinv.helm import openstack
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
DEFAULT_NOVA_PCI_ALIAS = [
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_NAME},
|
|
{"class_id": constants.NOVA_PCI_ALIAS_GPU_CLASS,
|
|
"name": constants.NOVA_PCI_ALIAS_GPU_NAME}
|
|
]
|
|
|
|
SERVICE_PARAM_NOVA_PCI_ALIAS = [
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER]
|
|
|
|
|
|
class NovaHelm(openstack.OpenstackBaseHelm):
|
|
"""Class to encapsulate helm operations for the nova chart"""
|
|
|
|
CHART = constants.HELM_CHART_NOVA
|
|
|
|
SERVICE_NAME = 'nova'
|
|
AUTH_USERS = ['nova', 'placement']
|
|
SERVICE_USERS = ['neutron', 'ironic']
|
|
|
|
def get_overrides(self, namespace=None):
|
|
|
|
ssh_privatekey, ssh_publickey = \
|
|
self._get_or_generate_ssh_keys(self.SERVICE_NAME, common.HELM_NS_OPENSTACK)
|
|
overrides = {
|
|
common.HELM_NS_OPENSTACK: {
|
|
'pod': {
|
|
'replicas': {
|
|
'api_metadata': self._num_controllers(),
|
|
'placement': self._num_controllers(),
|
|
'osapi': self._num_controllers(),
|
|
'conductor': self._num_controllers(),
|
|
'consoleauth': self._num_controllers(),
|
|
'scheduler': self._num_controllers(),
|
|
# set replicas for novncproxy once it's validated.
|
|
}
|
|
},
|
|
'conf': {
|
|
'nova': {
|
|
'libvirt': {
|
|
'virt_type': self._get_virt_type(),
|
|
},
|
|
'vnc': {
|
|
'novncproxy_base_url': self._get_novncproxy_base_url(),
|
|
},
|
|
'pci': self._get_pci_alias(),
|
|
},
|
|
'overrides': {
|
|
'nova_compute': {
|
|
'hosts': self._get_per_host_overrides()
|
|
}
|
|
},
|
|
'ssh_private': ssh_privatekey,
|
|
'ssh_public': ssh_publickey,
|
|
},
|
|
'endpoints': self._get_endpoints_overrides(),
|
|
'images': self._get_images_overrides(),
|
|
'network': {
|
|
'sshd': {
|
|
'from_subnet': self._get_ssh_subnet(),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if namespace in self.SUPPORTED_NAMESPACES:
|
|
return overrides[namespace]
|
|
elif namespace:
|
|
raise exception.InvalidHelmNamespace(chart=self.CHART,
|
|
namespace=namespace)
|
|
else:
|
|
return overrides
|
|
|
|
def _get_images_overrides(self):
|
|
heat_image = self._operator.chart_operators[
|
|
constants.HELM_CHART_HEAT].docker_image
|
|
return {
|
|
'tags': {
|
|
'bootstrap': heat_image,
|
|
'db_drop': heat_image,
|
|
'db_init': heat_image,
|
|
'ks_user': heat_image,
|
|
'ks_service': heat_image,
|
|
'ks_endpoints': heat_image,
|
|
'nova_api': self.docker_image,
|
|
'nova_cell_setup': self.docker_image,
|
|
'nova_cell_setup_init': heat_image,
|
|
'nova_compute': self.docker_image,
|
|
'nova_compute_ironic': self.docker_image,
|
|
'nova_compute_ssh': self.docker_image,
|
|
'nova_conductor': self.docker_image,
|
|
'nova_consoleauth': self.docker_image,
|
|
'nova_db_sync': self.docker_image,
|
|
'nova_novncproxy': self.docker_image,
|
|
'nova_placement': self.docker_image,
|
|
'nova_scheduler': self.docker_image,
|
|
'nova_spiceproxy': self.docker_image,
|
|
'nova_spiceproxy_assets': self.docker_image
|
|
}
|
|
}
|
|
|
|
def _get_endpoints_overrides(self):
|
|
overrides = {
|
|
'identity': {
|
|
'name': 'keystone',
|
|
'auth': self._get_endpoints_identity_overrides(
|
|
self.SERVICE_NAME, self.AUTH_USERS),
|
|
},
|
|
'oslo_cache': {
|
|
'auth': {
|
|
'memcached_secret_key':
|
|
self._get_common_password('auth_memcache_key')
|
|
}
|
|
},
|
|
'oslo_messaging': {
|
|
'auth': self._get_endpoints_oslo_messaging_overrides(
|
|
self.SERVICE_NAME, [self.SERVICE_NAME])
|
|
},
|
|
}
|
|
|
|
db_passwords = {'auth': self._get_endpoints_oslo_db_overrides(
|
|
self.SERVICE_NAME, [self.SERVICE_NAME])}
|
|
overrides.update({
|
|
'oslo_db': db_passwords,
|
|
'oslo_db_api': copy.deepcopy(db_passwords),
|
|
'oslo_db_cell0': copy.deepcopy(db_passwords),
|
|
})
|
|
|
|
# Service user passwords already exist in other chart overrides
|
|
for user in self.SERVICE_USERS:
|
|
overrides['identity']['auth'].update({
|
|
user: {
|
|
'region_name': self._region_name(),
|
|
'password': self._get_or_generate_password(
|
|
user, common.HELM_NS_OPENSTACK, user)
|
|
}
|
|
})
|
|
|
|
return overrides
|
|
|
|
def _get_novncproxy_base_url(self):
|
|
oam_addr = self._get_oam_address(),
|
|
url = "http://%s:6080/vnc_auto.html" % oam_addr
|
|
return url
|
|
|
|
def _get_virt_type(self):
|
|
if utils.is_virtual():
|
|
return 'qemu'
|
|
else:
|
|
return 'kvm'
|
|
|
|
def _get_host_cpu_list(self, host, function=None, threads=False):
|
|
"""
|
|
Retreive a list of CPUs for the host, filtered by function and thread
|
|
siblings (if supplied)
|
|
"""
|
|
cpus = []
|
|
for c in self.dbapi.icpu_get_by_ihost(host.id):
|
|
if c.thread != 0 and not threads:
|
|
continue
|
|
if c.allocated_function == function or not function:
|
|
cpus.append(c)
|
|
return cpus
|
|
|
|
def _update_host_cpu_maps(self, host, default_config):
|
|
host_cpus = self._get_host_cpu_list(host, threads=True)
|
|
if host_cpus:
|
|
vm_cpus = self._get_host_cpu_list(
|
|
host, function=constants.APPLICATION_FUNCTION, threads=True)
|
|
vm_cpu_list = [c.cpu for c in vm_cpus]
|
|
vm_cpu_fmt = "\"%s\"" % utils.format_range_set(vm_cpu_list)
|
|
default_config.update({'vcpu_pin_set': vm_cpu_fmt})
|
|
|
|
shared_cpus = self._get_host_cpu_list(
|
|
host, function=constants.SHARED_FUNCTION, threads=True)
|
|
shared_cpu_map = {c.numa_node: c.cpu for c in shared_cpus}
|
|
shared_cpu_fmt = "\"%s\"" % ','.join(
|
|
"%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items())
|
|
default_config.update({'shared_pcpu_map': shared_cpu_fmt})
|
|
|
|
def _get_pci_pt_whitelist(self, host, iface_context):
|
|
# Process all configured PCI passthrough interfaces and add them to
|
|
# the list of devices to whitelist
|
|
devices = []
|
|
for iface in iface_context['interfaces'].values():
|
|
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]:
|
|
port = interface.get_interface_port(iface_context, iface)
|
|
dnames = interface._get_datanetwork_names(iface_context, iface)
|
|
device = {
|
|
'address': port['pciaddr'],
|
|
'physical_network': dnames,
|
|
}
|
|
LOG.debug('_get_pci_pt_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
# Process all enabled PCI devices configured for PT and SRIOV and
|
|
# add them to the list of devices to whitelist.
|
|
# Since we are now properly initializing the qat driver and
|
|
# restarting sysinv, we need to add VF devices to the regular
|
|
# whitelist instead of the sriov whitelist
|
|
pci_devices = self.dbapi.pci_device_get_by_host(host.id)
|
|
for pci_device in pci_devices:
|
|
if pci_device.enabled:
|
|
device = {
|
|
'address': pci_device.pciaddr,
|
|
'class_id': pci_device.pclass_id
|
|
}
|
|
LOG.debug('_get_pci_pt_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
return devices
|
|
|
|
def _get_pci_sriov_whitelist(self, host, iface_context):
|
|
# Process all configured SRIOV interfaces and add each VF
|
|
# to the list of devices to whitelist
|
|
devices = []
|
|
for iface in iface_context['interfaces'].values():
|
|
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]:
|
|
port = interface.get_interface_port(iface_context, iface)
|
|
dnames = interface._get_datanetwork_names(iface_context, iface)
|
|
vf_addrs = port['sriov_vfs_pci_address']
|
|
if vf_addrs:
|
|
for vf_addr in vf_addrs.split(","):
|
|
device = {
|
|
'address': vf_addr,
|
|
'physical_network': dnames,
|
|
}
|
|
LOG.debug('_get_pci_sriov_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
return devices
|
|
|
|
def _get_pci_alias(self):
|
|
"""
|
|
Generate multistring values containing global PCI alias
|
|
configuration for QAT and GPU devices.
|
|
|
|
The multistring type with list of JSON string values is used
|
|
to generate one-line-per-entry formatting, since JSON list of
|
|
dict is not supported by nova.
|
|
"""
|
|
service_parameters = self._get_service_parameter_configs(
|
|
constants.SERVICE_TYPE_NOVA)
|
|
|
|
alias_config = DEFAULT_NOVA_PCI_ALIAS[:]
|
|
|
|
if service_parameters is not None:
|
|
for p in SERVICE_PARAM_NOVA_PCI_ALIAS:
|
|
value = self._service_parameter_lookup_one(
|
|
service_parameters,
|
|
constants.SERVICE_PARAM_SECTION_NOVA_PCI_ALIAS,
|
|
p, None)
|
|
if value is not None:
|
|
# Replace any references to device_id with product_id
|
|
# This is to align with the requirements of the
|
|
# Nova PCI request alias schema.
|
|
# (sysinv used device_id, nova uses product_id)
|
|
value = value.replace("device_id", "product_id")
|
|
|
|
aliases = value.rstrip(';').split(';')
|
|
for alias_str in aliases:
|
|
alias = dict((str(k), str(v)) for k, v in
|
|
(x.split('=') for x in
|
|
alias_str.split(',')))
|
|
alias_config.append(alias)
|
|
|
|
LOG.debug('_get_pci_alias: aliases = %s', alias_config)
|
|
multistring = self._oslo_multistring_override(
|
|
name='alias', values=alias_config)
|
|
return multistring
|
|
|
|
def _update_host_pci_whitelist(self, host, pci_config):
|
|
"""
|
|
Generate multistring values containing PCI passthrough
|
|
and SR-IOV devices.
|
|
|
|
The multistring type with list of JSON string values is used
|
|
to generate one-line-per-entry pretty formatting.
|
|
"""
|
|
# obtain interface information specific to this host
|
|
iface_context = {
|
|
'ports': interface._get_port_interface_id_index(
|
|
self.dbapi, host),
|
|
'interfaces': interface._get_interface_name_index(
|
|
self.dbapi, host),
|
|
'interfaces_datanets': interface._get_interface_name_datanets(
|
|
self.dbapi, host),
|
|
'addresses': interface._get_address_interface_name_index(
|
|
self.dbapi, host),
|
|
}
|
|
|
|
# This host's list of PCI passthrough and SR-IOV device dictionaries
|
|
devices = []
|
|
devices.extend(self._get_pci_pt_whitelist(host, iface_context))
|
|
devices.extend(self._get_pci_sriov_whitelist(host, iface_context))
|
|
if not devices:
|
|
return
|
|
|
|
# Convert device list into passthrough_whitelist multistring
|
|
multistring = self._oslo_multistring_override(
|
|
name='passthrough_whitelist', values=devices)
|
|
if multistring is not None:
|
|
pci_config.update(multistring)
|
|
|
|
def _update_host_storage(self, host, default_config, libvirt_config):
|
|
remote_storage = False
|
|
labels = self.dbapi.label_get_all(host.id)
|
|
for label in labels:
|
|
if (label.label_key == common.LABEL_REMOTE_STORAGE and
|
|
label.label_value == common.LABEL_VALUE_ENABLED):
|
|
remote_storage = True
|
|
break
|
|
|
|
rbd_pool = constants.CEPH_POOL_EPHEMERAL_NAME
|
|
rbd_ceph_conf = os.path.join(constants.CEPH_CONF_PATH,
|
|
constants.SB_TYPE_CEPH_CONF_FILENAME)
|
|
|
|
# If NOVA is a service on a ceph-external backend, use the ephemeral_pool
|
|
# and ceph_conf file that are stored in that DB entry.
|
|
# If NOVA is not on any ceph-external backend, it must be on the internal
|
|
# ceph backend with default "ephemeral" pool and default "/etc/ceph/ceph.conf"
|
|
# config file
|
|
sb_list = self.dbapi.storage_backend_get_list_by_type(
|
|
backend_type=constants.SB_TYPE_CEPH_EXTERNAL)
|
|
if sb_list:
|
|
for sb in sb_list:
|
|
if constants.SB_SVC_NOVA in sb.services:
|
|
ceph_ext_obj = self.dbapi.storage_ceph_external_get(sb.id)
|
|
rbd_pool = sb.capabilities.get('ephemeral_pool')
|
|
rbd_ceph_conf = \
|
|
constants.CEPH_CONF_PATH + os.path.basename(ceph_ext_obj.ceph_conf)
|
|
|
|
if remote_storage:
|
|
libvirt_config.update({'images_type': 'rbd',
|
|
'images_rbd_pool': rbd_pool,
|
|
'images_rbd_ceph_conf': rbd_ceph_conf})
|
|
else:
|
|
libvirt_config.update({'images_type': 'default'})
|
|
|
|
def _update_host_addresses(self, host, default_config, vnc_config, libvirt_config):
|
|
interfaces = self.dbapi.iinterface_get_by_ihost(host.id)
|
|
addresses = self.dbapi.addresses_get_by_host(host.id)
|
|
cluster_host_network = self.dbapi.network_get_by_type(
|
|
constants.NETWORK_TYPE_CLUSTER_HOST)
|
|
cluster_host_iface = None
|
|
for iface in interfaces:
|
|
interface_network = {'interface_id': iface.id,
|
|
'network_id': cluster_host_network.id}
|
|
try:
|
|
self.dbapi.interface_network_query(interface_network)
|
|
cluster_host_iface = iface
|
|
except exception.InterfaceNetworkNotFoundByHostInterfaceNetwork:
|
|
pass
|
|
|
|
if cluster_host_iface is None:
|
|
return
|
|
cluster_host_ip = None
|
|
ip_family = None
|
|
for addr in addresses:
|
|
if addr.interface_uuid == cluster_host_iface.uuid:
|
|
cluster_host_ip = addr.address
|
|
ip_family = addr.family
|
|
|
|
default_config.update({'my_ip': cluster_host_ip})
|
|
if ip_family == 4:
|
|
vnc_config.update({'vncserver_listen': '0.0.0.0'})
|
|
elif ip_family == 6:
|
|
vnc_config.update({'vncserver_listen': '::0'})
|
|
|
|
libvirt_config.update({'live_migration_inbound_addr': cluster_host_ip})
|
|
vnc_config.update({'vncserver_proxyclient_address': cluster_host_ip})
|
|
|
|
def _get_ssh_subnet(self):
|
|
cluster_host_network = self.dbapi.network_get_by_type(
|
|
constants.NETWORK_TYPE_CLUSTER_HOST)
|
|
address_pool = self.dbapi.address_pool_get(cluster_host_network.pool_uuid)
|
|
return '%s/%s' % (str(address_pool.network), str(address_pool.prefix))
|
|
|
|
def _update_host_memory(self, host, default_config):
|
|
vswitch_2M_pages = []
|
|
vswitch_1G_pages = []
|
|
vm_4K_pages = []
|
|
# The retrieved information is not necessarily ordered by numa node.
|
|
host_memory = self.dbapi.imemory_get_by_ihost(host.id)
|
|
# This makes it ordered by numa node.
|
|
memory_numa_list = utils.get_numa_index_list(host_memory)
|
|
# Process them in order of numa node.
|
|
for node, memory_list in memory_numa_list.items():
|
|
memory = memory_list[0]
|
|
# first the 4K memory
|
|
vm_hugepages_nr_4K = memory.vm_hugepages_nr_4K if (
|
|
memory.vm_hugepages_nr_4K is not None) else 0
|
|
vm_4K_pages.append(vm_hugepages_nr_4K)
|
|
# Now the vswitch memory of each hugepage size.
|
|
vswitch_2M_page = 0
|
|
vswitch_1G_page = 0
|
|
if memory.vswitch_hugepages_size_mib == constants.MIB_2M:
|
|
vswitch_2M_page = memory.vswitch_hugepages_nr
|
|
elif memory.vswitch_hugepages_size_mib == constants.MIB_1G:
|
|
vswitch_1G_page = memory.vswitch_hugepages_nr
|
|
vswitch_2M_pages.append(vswitch_2M_page)
|
|
vswitch_1G_pages.append(vswitch_1G_page)
|
|
# Build up the config values.
|
|
vswitch_2M = "\"%s\"" % ','.join([str(i) for i in vswitch_2M_pages])
|
|
vswitch_1G = "\"%s\"" % ','.join([str(i) for i in vswitch_1G_pages])
|
|
vm_4K = "\"%s\"" % ','.join([str(i) for i in vm_4K_pages])
|
|
# Add the new entries to the DEFAULT config section.
|
|
default_config.update({
|
|
'compute_vm_4K_pages': vm_4K,
|
|
'compute_vswitch_2M_pages': vswitch_2M,
|
|
'compute_vswitch_1G_pages': vswitch_1G,
|
|
})
|
|
|
|
def _get_per_host_overrides(self):
|
|
host_list = []
|
|
hosts = self.dbapi.ihost_get_list()
|
|
|
|
for host in hosts:
|
|
if (host.invprovision in [constants.PROVISIONED,
|
|
constants.PROVISIONING]):
|
|
if constants.WORKER in utils.get_personalities(host):
|
|
|
|
hostname = str(host.hostname)
|
|
default_config = {}
|
|
vnc_config = {}
|
|
libvirt_config = {}
|
|
pci_config = {}
|
|
self._update_host_cpu_maps(host, default_config)
|
|
self._update_host_storage(host, default_config, libvirt_config)
|
|
self._update_host_addresses(host, default_config, vnc_config,
|
|
libvirt_config)
|
|
self._update_host_memory(host, default_config)
|
|
self._update_host_pci_whitelist(host, pci_config)
|
|
host_nova = {
|
|
'name': hostname,
|
|
'conf': {
|
|
'nova': {
|
|
'DEFAULT': default_config,
|
|
'vnc': vnc_config,
|
|
'libvirt': libvirt_config,
|
|
'pci': pci_config if pci_config else None,
|
|
}
|
|
}
|
|
}
|
|
host_list.append(host_nova)
|
|
return host_list
|
|
|
|
def get_region_name(self):
|
|
return self._get_service_region_name(self.SERVICE_NAME)
|