510 lines
21 KiB
Python
510 lines
21 KiB
Python
#
|
|
# Copyright (c) 2018-2019 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
import copy
|
|
import os
|
|
|
|
from sysinv.common import constants
|
|
from sysinv.common import exception
|
|
from sysinv.common import interface
|
|
from sysinv.common import utils
|
|
from sysinv.common.storage_backend_conf import StorageBackendConfig
|
|
from sysinv.openstack.common import log as logging
|
|
from sysinv.helm import common
|
|
from sysinv.helm import openstack
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# Align ephemeral rbd_user with the cinder rbd_user so that the same libvirt
|
|
# secret can be used for accessing both pools. This also aligns with the
|
|
# behavior defined in nova/virt/libvirt/volume/net.py:_set_auth_config_rbd()
|
|
RBD_POOL_USER = "cinder"
|
|
|
|
DEFAULT_NOVA_PCI_ALIAS = [
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_NAME},
|
|
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
|
|
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE,
|
|
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_NAME},
|
|
{"class_id": constants.NOVA_PCI_ALIAS_GPU_CLASS,
|
|
"name": constants.NOVA_PCI_ALIAS_GPU_NAME}
|
|
]
|
|
|
|
SERVICE_PARAM_NOVA_PCI_ALIAS = [
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_PF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_VF,
|
|
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER]
|
|
|
|
|
|
class NovaHelm(openstack.OpenstackBaseHelm):
|
|
"""Class to encapsulate helm operations for the nova chart"""
|
|
|
|
CHART = constants.HELM_CHART_NOVA
|
|
|
|
SERVICE_NAME = 'nova'
|
|
AUTH_USERS = ['nova', 'placement']
|
|
SERVICE_USERS = ['neutron', 'ironic']
|
|
NOVNCPROXY_SERVICE_NAME = 'novncproxy'
|
|
|
|
def get_overrides(self, namespace=None):
|
|
|
|
ssh_privatekey, ssh_publickey = \
|
|
self._get_or_generate_ssh_keys(self.SERVICE_NAME, common.HELM_NS_OPENSTACK)
|
|
overrides = {
|
|
common.HELM_NS_OPENSTACK: {
|
|
'manifests': self._get_compute_ironic_manifests(),
|
|
'pod': {
|
|
'mounts': {
|
|
'nova_compute': {
|
|
'nova_compute': self._get_mount_overrides()
|
|
}
|
|
},
|
|
'replicas': {
|
|
'api_metadata': self._num_controllers(),
|
|
'placement': self._num_controllers(),
|
|
'osapi': self._num_controllers(),
|
|
'conductor': self._num_controllers(),
|
|
'consoleauth': self._num_controllers(),
|
|
'scheduler': self._num_controllers(),
|
|
'novncproxy': self._num_controllers()
|
|
}
|
|
},
|
|
'conf': {
|
|
'ceph': {
|
|
'ephemeral_storage': self._get_rbd_ephemeral_storage()
|
|
},
|
|
'nova': {
|
|
'libvirt': {
|
|
'virt_type': self._get_virt_type(),
|
|
},
|
|
'vnc': {
|
|
'novncproxy_base_url': self._get_novncproxy_base_url(),
|
|
},
|
|
'pci': self._get_pci_alias(),
|
|
},
|
|
'overrides': {
|
|
'nova_compute': {
|
|
'hosts': self._get_per_host_overrides()
|
|
}
|
|
},
|
|
'ssh_private': ssh_privatekey,
|
|
'ssh_public': ssh_publickey,
|
|
},
|
|
'endpoints': self._get_endpoints_overrides(),
|
|
'network': {
|
|
'sshd': {
|
|
'from_subnet': self._get_ssh_subnet(),
|
|
}
|
|
},
|
|
'ceph_client': self._get_ceph_client_overrides(),
|
|
}
|
|
}
|
|
|
|
if namespace in self.SUPPORTED_NAMESPACES:
|
|
return overrides[namespace]
|
|
elif namespace:
|
|
raise exception.InvalidHelmNamespace(chart=self.CHART,
|
|
namespace=namespace)
|
|
else:
|
|
return overrides
|
|
|
|
def _get_mount_overrides(self):
|
|
overrides = self._get_mount_uefi_overrides()
|
|
# mount /dev/pts in order to get console log
|
|
overrides['volumes'].append({
|
|
'name': 'dev-pts',
|
|
'hostPath': {'path': '/dev/pts'}
|
|
})
|
|
overrides['volumeMounts'].append({
|
|
'name': 'dev-pts',
|
|
'mountPath': '/dev/pts'
|
|
})
|
|
return overrides
|
|
|
|
def _compute_ironic_manifests(self, is_labeled):
|
|
manifests = {
|
|
'statefulset_compute_ironic': is_labeled
|
|
}
|
|
return manifests
|
|
|
|
def _get_compute_ironic_manifests(self):
|
|
ironic_label = self._is_labeled(common.LABEL_IRONIC, 'enabled')
|
|
return self._compute_ironic_manifests(ironic_label)
|
|
|
|
def _get_endpoints_overrides(self):
|
|
overrides = {
|
|
'identity': {
|
|
'name': 'keystone',
|
|
'auth': self._get_endpoints_identity_overrides(
|
|
self.SERVICE_NAME, self.AUTH_USERS),
|
|
},
|
|
'compute': {
|
|
'host_fqdn_override':
|
|
self._get_endpoints_host_fqdn_overrides(self.SERVICE_NAME),
|
|
'port': self._get_endpoints_port_api_public_overrides(),
|
|
'scheme': self._get_endpoints_scheme_public_overrides(),
|
|
},
|
|
'compute_novnc_proxy': {
|
|
'host_fqdn_override':
|
|
self._get_endpoints_host_fqdn_overrides(
|
|
self.NOVNCPROXY_SERVICE_NAME),
|
|
'port': self._get_endpoints_port_api_public_overrides(),
|
|
'scheme': self._get_endpoints_scheme_public_overrides(),
|
|
},
|
|
'oslo_cache': {
|
|
'auth': {
|
|
'memcache_secret_key':
|
|
self._get_common_password('auth_memcache_key')
|
|
}
|
|
},
|
|
'oslo_messaging': {
|
|
'auth': self._get_endpoints_oslo_messaging_overrides(
|
|
self.SERVICE_NAME, [self.SERVICE_NAME])
|
|
},
|
|
}
|
|
|
|
db_passwords = {'auth': self._get_endpoints_oslo_db_overrides(
|
|
self.SERVICE_NAME, [self.SERVICE_NAME])}
|
|
overrides.update({
|
|
'oslo_db': db_passwords,
|
|
'oslo_db_api': copy.deepcopy(db_passwords),
|
|
'oslo_db_cell0': copy.deepcopy(db_passwords),
|
|
})
|
|
|
|
# Service user passwords already exist in other chart overrides
|
|
for user in self.SERVICE_USERS:
|
|
overrides['identity']['auth'].update({
|
|
user: {
|
|
'region_name': self._region_name(),
|
|
'password': self._get_or_generate_password(
|
|
user, common.HELM_NS_OPENSTACK, user)
|
|
}
|
|
})
|
|
|
|
return overrides
|
|
|
|
def _get_novncproxy_base_url(self):
|
|
# Get the openstack endpoint public domain name
|
|
endpoint_domain = self._get_service_parameter(
|
|
constants.SERVICE_TYPE_OPENSTACK,
|
|
constants.SERVICE_PARAM_SECTION_OPENSTACK_HELM,
|
|
constants.SERVICE_PARAM_NAME_ENDPOINT_DOMAIN)
|
|
if endpoint_domain is not None:
|
|
location = "%s.%s" % (self.NOVNCPROXY_SERVICE_NAME,
|
|
str(endpoint_domain.value).lower())
|
|
else:
|
|
location = self._get_service_default_dns_name(
|
|
self.NOVNCPROXY_SERVICE_NAME)
|
|
|
|
url = "%s://%s/vnc_auto.html" % (self._get_public_protocol(),
|
|
location)
|
|
return url
|
|
|
|
def _get_virt_type(self):
|
|
if utils.is_virtual():
|
|
return 'qemu'
|
|
else:
|
|
return 'kvm'
|
|
|
|
def _update_host_cpu_maps(self, host, default_config):
|
|
host_cpus = self._get_host_cpu_list(host, threads=True)
|
|
if host_cpus:
|
|
vm_cpus = self._get_host_cpu_list(
|
|
host, function=constants.APPLICATION_FUNCTION, threads=True)
|
|
vm_cpu_list = [c.cpu for c in vm_cpus]
|
|
vm_cpu_fmt = "\"%s\"" % utils.format_range_set(vm_cpu_list)
|
|
default_config.update({'vcpu_pin_set': vm_cpu_fmt})
|
|
|
|
shared_cpus = self._get_host_cpu_list(
|
|
host, function=constants.SHARED_FUNCTION, threads=True)
|
|
shared_cpu_map = {c.numa_node: c.cpu for c in shared_cpus}
|
|
shared_cpu_fmt = "\"%s\"" % ','.join(
|
|
"%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items())
|
|
default_config.update({'shared_pcpu_map': shared_cpu_fmt})
|
|
|
|
def _get_pci_pt_whitelist(self, host, iface_context):
|
|
# Process all configured PCI passthrough interfaces and add them to
|
|
# the list of devices to whitelist
|
|
devices = []
|
|
for iface in iface_context['interfaces'].values():
|
|
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]:
|
|
port = interface.get_interface_port(iface_context, iface)
|
|
dnames = interface._get_datanetwork_names(iface_context, iface)
|
|
device = {
|
|
'address': port['pciaddr'],
|
|
'physical_network': dnames,
|
|
}
|
|
LOG.debug('_get_pci_pt_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
# Process all enabled PCI devices configured for PT and SRIOV and
|
|
# add them to the list of devices to whitelist.
|
|
# Since we are now properly initializing the qat driver and
|
|
# restarting sysinv, we need to add VF devices to the regular
|
|
# whitelist instead of the sriov whitelist
|
|
pci_devices = self.dbapi.pci_device_get_by_host(host.id)
|
|
for pci_device in pci_devices:
|
|
if pci_device.enabled:
|
|
device = {
|
|
'address': pci_device.pciaddr,
|
|
'class_id': pci_device.pclass_id
|
|
}
|
|
LOG.debug('_get_pci_pt_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
return devices
|
|
|
|
def _get_pci_sriov_whitelist(self, host, iface_context):
|
|
# Process all configured SRIOV interfaces and add each VF
|
|
# to the list of devices to whitelist
|
|
devices = []
|
|
for iface in iface_context['interfaces'].values():
|
|
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]:
|
|
port = interface.get_interface_port(iface_context, iface)
|
|
dnames = interface._get_datanetwork_names(iface_context, iface)
|
|
vf_addrs = port['sriov_vfs_pci_address']
|
|
if vf_addrs:
|
|
for vf_addr in vf_addrs.split(","):
|
|
device = {
|
|
'address': vf_addr,
|
|
'physical_network': dnames,
|
|
}
|
|
LOG.debug('_get_pci_sriov_whitelist '
|
|
'host=%s, device=%s', host.hostname, device)
|
|
devices.append(device)
|
|
|
|
return devices
|
|
|
|
def _get_pci_alias(self):
|
|
"""
|
|
Generate multistring values containing global PCI alias
|
|
configuration for QAT and GPU devices.
|
|
|
|
The multistring type with list of JSON string values is used
|
|
to generate one-line-per-entry formatting, since JSON list of
|
|
dict is not supported by nova.
|
|
"""
|
|
service_parameters = self._get_service_parameter_configs(
|
|
constants.SERVICE_TYPE_NOVA)
|
|
|
|
alias_config = DEFAULT_NOVA_PCI_ALIAS[:]
|
|
|
|
if service_parameters is not None:
|
|
for p in SERVICE_PARAM_NOVA_PCI_ALIAS:
|
|
value = self._service_parameter_lookup_one(
|
|
service_parameters,
|
|
constants.SERVICE_PARAM_SECTION_NOVA_PCI_ALIAS,
|
|
p, None)
|
|
if value is not None:
|
|
# Replace any references to device_id with product_id
|
|
# This is to align with the requirements of the
|
|
# Nova PCI request alias schema.
|
|
# (sysinv used device_id, nova uses product_id)
|
|
value = value.replace("device_id", "product_id")
|
|
|
|
aliases = value.rstrip(';').split(';')
|
|
for alias_str in aliases:
|
|
alias = dict((str(k), str(v)) for k, v in
|
|
(x.split('=') for x in
|
|
alias_str.split(',')))
|
|
alias_config.append(alias)
|
|
|
|
LOG.debug('_get_pci_alias: aliases = %s', alias_config)
|
|
multistring = self._oslo_multistring_override(
|
|
name='alias', values=alias_config)
|
|
return multistring
|
|
|
|
def _update_host_pci_whitelist(self, host, pci_config):
|
|
"""
|
|
Generate multistring values containing PCI passthrough
|
|
and SR-IOV devices.
|
|
|
|
The multistring type with list of JSON string values is used
|
|
to generate one-line-per-entry pretty formatting.
|
|
"""
|
|
# obtain interface information specific to this host
|
|
iface_context = {
|
|
'ports': interface._get_port_interface_id_index(
|
|
self.dbapi, host),
|
|
'interfaces': interface._get_interface_name_index(
|
|
self.dbapi, host),
|
|
'interfaces_datanets': interface._get_interface_name_datanets(
|
|
self.dbapi, host),
|
|
'addresses': interface._get_address_interface_name_index(
|
|
self.dbapi, host),
|
|
}
|
|
|
|
# This host's list of PCI passthrough and SR-IOV device dictionaries
|
|
devices = []
|
|
devices.extend(self._get_pci_pt_whitelist(host, iface_context))
|
|
devices.extend(self._get_pci_sriov_whitelist(host, iface_context))
|
|
if not devices:
|
|
return
|
|
|
|
# Convert device list into passthrough_whitelist multistring
|
|
multistring = self._oslo_multistring_override(
|
|
name='passthrough_whitelist', values=devices)
|
|
if multistring is not None:
|
|
pci_config.update(multistring)
|
|
|
|
def _update_host_storage(self, host, default_config, libvirt_config):
|
|
remote_storage = False
|
|
labels = self.dbapi.label_get_all(host.id)
|
|
for label in labels:
|
|
if (label.label_key == common.LABEL_REMOTE_STORAGE and
|
|
label.label_value == common.LABEL_VALUE_ENABLED):
|
|
remote_storage = True
|
|
break
|
|
|
|
rbd_pool = constants.CEPH_POOL_EPHEMERAL_NAME
|
|
rbd_ceph_conf = os.path.join(constants.CEPH_CONF_PATH,
|
|
constants.SB_TYPE_CEPH_CONF_FILENAME)
|
|
|
|
# If NOVA is a service on a ceph-external backend, use the ephemeral_pool
|
|
# and ceph_conf file that are stored in that DB entry.
|
|
# If NOVA is not on any ceph-external backend, it must be on the internal
|
|
# ceph backend with default "ephemeral" pool and default "/etc/ceph/ceph.conf"
|
|
# config file
|
|
sb_list = self.dbapi.storage_backend_get_list_by_type(
|
|
backend_type=constants.SB_TYPE_CEPH_EXTERNAL)
|
|
if sb_list:
|
|
for sb in sb_list:
|
|
if constants.SB_SVC_NOVA in sb.services:
|
|
ceph_ext_obj = self.dbapi.storage_ceph_external_get(sb.id)
|
|
rbd_pool = sb.capabilities.get('ephemeral_pool')
|
|
rbd_ceph_conf = \
|
|
constants.CEPH_CONF_PATH + os.path.basename(ceph_ext_obj.ceph_conf)
|
|
|
|
if remote_storage:
|
|
libvirt_config.update({'images_type': 'rbd',
|
|
'images_rbd_pool': rbd_pool,
|
|
'images_rbd_ceph_conf': rbd_ceph_conf})
|
|
else:
|
|
libvirt_config.update({'images_type': 'default'})
|
|
|
|
def _update_host_addresses(self, host, default_config, vnc_config, libvirt_config):
|
|
interfaces = self.dbapi.iinterface_get_by_ihost(host.id)
|
|
addresses = self.dbapi.addresses_get_by_host(host.id)
|
|
cluster_host_network = self.dbapi.network_get_by_type(
|
|
constants.NETWORK_TYPE_CLUSTER_HOST)
|
|
cluster_host_iface = None
|
|
for iface in interfaces:
|
|
interface_network = {'interface_id': iface.id,
|
|
'network_id': cluster_host_network.id}
|
|
try:
|
|
self.dbapi.interface_network_query(interface_network)
|
|
cluster_host_iface = iface
|
|
except exception.InterfaceNetworkNotFoundByHostInterfaceNetwork:
|
|
pass
|
|
|
|
if cluster_host_iface is None:
|
|
return
|
|
cluster_host_ip = None
|
|
ip_family = None
|
|
for addr in addresses:
|
|
if addr.interface_uuid == cluster_host_iface.uuid:
|
|
cluster_host_ip = addr.address
|
|
ip_family = addr.family
|
|
|
|
default_config.update({'my_ip': cluster_host_ip})
|
|
if ip_family == 4:
|
|
vnc_config.update({'vncserver_listen': '0.0.0.0'})
|
|
elif ip_family == 6:
|
|
vnc_config.update({'vncserver_listen': '::0'})
|
|
|
|
libvirt_config.update({'live_migration_inbound_addr': cluster_host_ip})
|
|
vnc_config.update({'vncserver_proxyclient_address': cluster_host_ip})
|
|
|
|
def _get_ssh_subnet(self):
|
|
cluster_host_network = self.dbapi.network_get_by_type(
|
|
constants.NETWORK_TYPE_CLUSTER_HOST)
|
|
address_pool = self.dbapi.address_pool_get(cluster_host_network.pool_uuid)
|
|
return '%s/%s' % (str(address_pool.network), str(address_pool.prefix))
|
|
|
|
def _get_per_host_overrides(self):
|
|
host_list = []
|
|
hosts = self.dbapi.ihost_get_list()
|
|
|
|
for host in hosts:
|
|
if (host.invprovision in [constants.PROVISIONED,
|
|
constants.PROVISIONING]):
|
|
if constants.WORKER in utils.get_personalities(host):
|
|
|
|
hostname = str(host.hostname)
|
|
default_config = {}
|
|
vnc_config = {}
|
|
libvirt_config = {}
|
|
pci_config = {}
|
|
self._update_host_cpu_maps(host, default_config)
|
|
self._update_host_storage(host, default_config, libvirt_config)
|
|
self._update_host_addresses(host, default_config, vnc_config,
|
|
libvirt_config)
|
|
self._update_host_pci_whitelist(host, pci_config)
|
|
host_nova = {
|
|
'name': hostname,
|
|
'conf': {
|
|
'nova': {
|
|
'DEFAULT': default_config,
|
|
'vnc': vnc_config,
|
|
'libvirt': libvirt_config,
|
|
'pci': pci_config if pci_config else None,
|
|
}
|
|
}
|
|
}
|
|
host_list.append(host_nova)
|
|
return host_list
|
|
|
|
def get_region_name(self):
|
|
return self._get_service_region_name(self.SERVICE_NAME)
|
|
|
|
def _get_rbd_ephemeral_storage(self):
|
|
ephemeral_storage_conf = {}
|
|
ephemeral_pools = []
|
|
|
|
# Get the values for replication and min replication from the storage
|
|
# backend attributes.
|
|
replication, min_replication = \
|
|
StorageBackendConfig.get_ceph_pool_replication(self.dbapi)
|
|
|
|
# For now, the ephemeral pool will only be on the primary Ceph tier
|
|
rule_name = "{0}{1}{2}".format(
|
|
constants.SB_TIER_DEFAULT_NAMES[
|
|
constants.SB_TIER_TYPE_CEPH],
|
|
constants.CEPH_CRUSH_TIER_SUFFIX,
|
|
"-ruleset").replace('-', '_')
|
|
|
|
# Form the dictionary with the info for the ephemeral pool.
|
|
# If needed, multiple pools can be specified.
|
|
ephemeral_pool = {
|
|
'rbd_pool_name': constants.CEPH_POOL_EPHEMERAL_NAME,
|
|
'rbd_user': RBD_POOL_USER,
|
|
'rbd_crush_rule': rule_name,
|
|
'rbd_replication': replication,
|
|
'rbd_chunk_size': constants.CEPH_POOL_EPHEMERAL_PG_NUM
|
|
}
|
|
ephemeral_pools.append(ephemeral_pool)
|
|
|
|
ephemeral_storage_conf = {
|
|
'type': 'rbd',
|
|
'rbd_pools': ephemeral_pools
|
|
}
|
|
|
|
return ephemeral_storage_conf
|