Update nova helm overrides for PCI alias, passthrough, and SR-IOV

This adds generation of nova.conf overrides to configure:
- global PCI aliases for QAT and GPU devices
- per-host PCI passthrough whitelist (contains both
  passthrough and SR-IOV devices)

Helm multistring dictionary is created for PCI alias, and created for
PCI passthrough whitelist. These multistring are OSLO.conf compatible
with oslo_config.MultiStringOpt() multiple input values.
Each multistring contains a list of JSON encoded strings.

The generation of these overrides assumes that all host PCI device
PFs and VFs are already provisioned, and that the resulting
pci addresses are queryable using sysinv DB methods.

Story: 2003909
Task: 29071

Change-Id: I8b96e471f7dcff6277cca107cbd0668ffd67c7b7
Signed-off-by: Jim Gauld <james.gauld@windriver.com>
This commit is contained in:
Jim Gauld 2019-01-24 13:54:43 -05:00
parent e1dd3de752
commit f12b0bc669
6 changed files with 348 additions and 62 deletions

View File

@ -0,0 +1,104 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
"""Common interface utility and helper functions."""
import collections
from sysinv.common import constants
from sysinv.openstack.common import log
LOG = log.getLogger(__name__)
def _get_port_interface_id_index(dbapi, host):
"""
Builds a dictionary of ports indexed by interface id.
"""
ports = {}
for port in dbapi.ethernet_port_get_by_host(host.id):
ports[port.interface_id] = port
return ports
def _get_interface_name_index(dbapi, host):
"""
Builds a dictionary of interfaces indexed by interface name.
"""
interfaces = {}
for iface in dbapi.iinterface_get_by_ihost(host.id):
interfaces[iface.ifname] = iface
return interfaces
def _get_interface_name_datanets(dbapi, host):
"""
Builds a dictionary of datanets indexed by interface name.
"""
datanets = {}
for iface in dbapi.iinterface_get_by_ihost(host.id):
ifdatanets = dbapi.interface_datanetwork_get_by_interface(iface.uuid)
datanetworks = []
for ifdatanet in ifdatanets:
datanetworks.append(ifdatanet.datanetwork_uuid)
datanetworks_list = []
for datanetwork in datanetworks:
dn = dbapi.datanetwork_get(datanetwork)
datanetwork_dict = \
{'name': dn.name,
'uuid': dn.uuid,
'network_type': dn.network_type,
'mtu': dn.mtu}
if dn.network_type == constants.DATANETWORK_TYPE_VXLAN:
datanetwork_dict.update(
{'multicast_group': dn.multicast_group,
'port_num': dn.port_num,
'ttl': dn.ttl,
'mode': dn.mode})
datanetworks_list.append(datanetwork_dict)
datanets[iface.ifname] = datanetworks_list
LOG.debug('_get_interface_name_datanets '
'host=%s, datanets=%s', host.hostname, datanets)
return datanets
def _get_address_interface_name_index(dbapi, host):
"""
Builds a dictionary of address lists indexed by interface name.
"""
addresses = collections.defaultdict(list)
for address in dbapi.addresses_get_by_host(host.id):
addresses[address.ifname].append(address)
return addresses
def get_interface_datanets(context, iface):
"""
Return the list of data networks of the supplied interface
"""
return context['interfaces_datanets'][iface.ifname]
def _get_datanetwork_names(context, iface):
"""
Return the CSV list of data networks of the supplied interface
"""
dnets = get_interface_datanets(context, iface)
dnames_list = [dnet['name'] for dnet in dnets]
dnames = ",".join(dnames_list)
return dnames
def get_interface_port(context, iface):
"""
Determine the port of the underlying device.
"""
assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET
return context['ports'][iface['id']]

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -189,8 +189,8 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
if brname:
datanets = self._get_interface_datanets(iface)
for datanet in datanets:
LOG.info("_get_dynamic_ovs_agent_config datanet %s" %
datanet)
LOG.debug('_get_dynamic_ovs_agent_config '
'host=%s datanet=%s', host.hostname, datanet)
address = self._get_interface_primary_address(
self.context, host, iface)
if address:

View File

@ -9,6 +9,7 @@ import os
from sysinv.common import constants
from sysinv.common import exception
from sysinv.common import interface
from sysinv.common import utils
from sysinv.openstack.common import log as logging
from sysinv.helm import common
@ -17,6 +18,34 @@ from sysinv.helm import openstack
LOG = logging.getLogger(__name__)
DEFAULT_NOVA_PCI_ALIAS = [
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE,
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME},
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
"product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE,
"name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME},
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR,
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE,
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_NAME},
{"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR,
"product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE,
"name": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_NAME},
{"class_id": constants.NOVA_PCI_ALIAS_GPU_CLASS,
"name": constants.NOVA_PCI_ALIAS_GPU_NAME}
]
SERVICE_PARAM_NOVA_PCI_ALIAS = [
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_PF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_VF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_PF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_VF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_PF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_VF,
constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER]
class NovaHelm(openstack.OpenstackBaseHelm):
"""Class to encapsulate helm operations for the nova chart"""
@ -50,7 +79,8 @@ class NovaHelm(openstack.OpenstackBaseHelm):
},
'vnc': {
'novncproxy_base_url': self._get_novncproxy_base_url(),
}
},
'pci': self._get_pci_alias(),
},
'overrides': {
'nova_compute': {
@ -185,6 +215,133 @@ class NovaHelm(openstack.OpenstackBaseHelm):
"%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items())
default_config.update({'shared_pcpu_map': shared_cpu_fmt})
def _get_pci_pt_whitelist(self, host, iface_context):
# Process all configured PCI passthrough interfaces and add them to
# the list of devices to whitelist
devices = []
for iface in iface_context['interfaces'].values():
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]:
port = interface.get_interface_port(iface_context, iface)
dnames = interface._get_datanetwork_names(iface_context, iface)
device = {
'address': port['pciaddr'],
'physical_network': dnames,
}
LOG.debug('_get_pci_pt_whitelist '
'host=%s, device=%s', host.hostname, device)
devices.append(device)
# Process all enabled PCI devices configured for PT and SRIOV and
# add them to the list of devices to whitelist.
# Since we are now properly initializing the qat driver and
# restarting sysinv, we need to add VF devices to the regular
# whitelist instead of the sriov whitelist
pci_devices = self.dbapi.pci_device_get_by_host(host.id)
for pci_device in pci_devices:
if pci_device.enabled:
device = {
'address': pci_device.pciaddr,
'class_id': pci_device.pclass_id
}
LOG.debug('_get_pci_pt_whitelist '
'host=%s, device=%s', host.hostname, device)
devices.append(device)
return devices
def _get_pci_sriov_whitelist(self, host, iface_context):
# Process all configured SRIOV interfaces and add each VF
# to the list of devices to whitelist
devices = []
for iface in iface_context['interfaces'].values():
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]:
port = interface.get_interface_port(iface_context, iface)
dnames = interface._get_datanetwork_names(iface_context, iface)
vf_addrs = port['sriov_vfs_pci_address']
if vf_addrs:
for vf_addr in vf_addrs.split(","):
device = {
'address': vf_addr,
'physical_network': dnames,
}
LOG.debug('_get_pci_sriov_whitelist '
'host=%s, device=%s', host.hostname, device)
devices.append(device)
return devices
def _get_pci_alias(self):
"""
Generate multistring values containing global PCI alias
configuration for QAT and GPU devices.
The multistring type with list of JSON string values is used
to generate one-line-per-entry formatting, since JSON list of
dict is not supported by nova.
"""
service_parameters = self._get_service_parameter_configs(
constants.SERVICE_TYPE_NOVA)
alias_config = DEFAULT_NOVA_PCI_ALIAS[:]
if service_parameters is not None:
for p in SERVICE_PARAM_NOVA_PCI_ALIAS:
value = self._service_parameter_lookup_one(
service_parameters,
constants.SERVICE_PARAM_SECTION_NOVA_PCI_ALIAS,
p, None)
if value is not None:
# Replace any references to device_id with product_id
# This is to align with the requirements of the
# Nova PCI request alias schema.
# (sysinv used device_id, nova uses product_id)
value = value.replace("device_id", "product_id")
aliases = value.rstrip(';').split(';')
for alias_str in aliases:
alias = dict((str(k), str(v)) for k, v in
(x.split('=') for x in
alias_str.split(',')))
alias_config.append(alias)
LOG.debug('_get_pci_alias: aliases = %s', alias_config)
multistring = self._oslo_multistring_override(
name='alias', values=alias_config)
return multistring
def _update_host_pci_whitelist(self, host, pci_config):
"""
Generate multistring values containing PCI passthrough
and SR-IOV devices.
The multistring type with list of JSON string values is used
to generate one-line-per-entry pretty formatting.
"""
# obtain interface information specific to this host
iface_context = {
'ports': interface._get_port_interface_id_index(
self.dbapi, host),
'interfaces': interface._get_interface_name_index(
self.dbapi, host),
'interfaces_datanets': interface._get_interface_name_datanets(
self.dbapi, host),
'addresses': interface._get_address_interface_name_index(
self.dbapi, host),
}
# This host's list of PCI passthrough and SR-IOV device dictionaries
devices = []
devices.extend(self._get_pci_pt_whitelist(host, iface_context))
devices.extend(self._get_pci_sriov_whitelist(host, iface_context))
if not devices:
return
# Convert device list into passthrough_whitelist multistring
multistring = self._oslo_multistring_override(
name='passthrough_whitelist', values=devices)
if multistring is not None:
pci_config.update(multistring)
def _update_host_storage(self, host, default_config, libvirt_config):
remote_storage = False
labels = self.dbapi.label_get_all(host.id)
@ -307,11 +464,13 @@ class NovaHelm(openstack.OpenstackBaseHelm):
default_config = {}
vnc_config = {}
libvirt_config = {}
pci_config = {}
self._update_host_cpu_maps(host, default_config)
self._update_host_storage(host, default_config, libvirt_config)
self._update_host_addresses(host, default_config, vnc_config,
libvirt_config)
self._update_host_memory(host, default_config)
self._update_host_pci_whitelist(host, pci_config)
host_nova = {
'name': hostname,
'conf': {
@ -319,6 +478,7 @@ class NovaHelm(openstack.OpenstackBaseHelm):
'DEFAULT': default_config,
'vnc': vnc_config,
'libvirt': libvirt_config,
'pci': pci_config if pci_config else None,
}
}
}

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -12,8 +12,10 @@ from sysinv.helm import base
from sysinv.helm import common
from oslo_log import log
from oslo_serialization import jsonutils
from sysinv.common import constants
from sysinv.common import exception
from sqlalchemy.orm.exc import NoResultFound
LOG = log.getLogger(__name__)
@ -30,6 +32,36 @@ class OpenstackBaseHelm(base.BaseHelm):
configs[service] = self._get_service(service)
return configs[service]
def _get_service_parameters(self, service=None):
service_parameters = []
if self.dbapi is None:
return service_parameters
try:
service_parameters = self.dbapi.service_parameter_get_all(
service=service)
# the service parameter has not been added
except NoResultFound:
pass
return service_parameters
def _get_service_parameter_configs(self, service):
configs = self.context.setdefault('_service_params', {})
if service not in configs:
params = self._get_service_parameters(service)
if params:
configs[service] = params
else:
return None
return configs[service]
@staticmethod
def _service_parameter_lookup_one(service_parameters, section, name,
default):
for param in service_parameters:
if param['section'] == section and param['name'] == name:
return param['value']
return default
def _get_admin_user_name(self):
keystone_operator = self._operator.chart_operators[
constants.HELM_CHART_KEYSTONE]
@ -262,3 +294,32 @@ class OpenstackBaseHelm(base.BaseHelm):
name=chart, namespace=namespace, values=values)
return newprivatekey, newpublickey
def _oslo_multistring_override(self, name=None, values=[]):
"""
Generate helm multistring dictionary override for specified option
name with multiple values.
This generates oslo_config.MultiStringOpt() compatible config
with multiple input values. This routine JSON encodes each value for
complex types (eg, dict, list, set).
Return a multistring type formatted dictionary override.
"""
override = None
if name is None or not values:
return override
mvalues = []
for value in values:
if isinstance(value, (dict, list, set)):
mvalues.append(jsonutils.dumps(value))
else:
mvalues.append(value)
override = {
name: {'type': 'multistring',
'values': mvalues,
}
}
return override

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -13,6 +13,7 @@ from netaddr import IPNetwork
from sysinv.common import constants
from sysinv.common import exception
from sysinv.common import interface
from sysinv.common import utils
from sysinv.conductor import openstack
from sysinv.openstack.common import log
@ -152,54 +153,19 @@ class InterfacePuppet(base.BasePuppet):
"""
Builds a dictionary of ports indexed by interface id.
"""
ports = {}
for port in self.dbapi.ethernet_port_get_by_host(host.id):
ports[port.interface_id] = port
return ports
return interface._get_port_interface_id_index(self.dbapi, host)
def _get_interface_name_index(self, host):
"""
Builds a dictionary of interfaces indexed by interface name.
"""
interfaces = {}
for iface in self.dbapi.iinterface_get_by_ihost(host.id):
interfaces[iface.ifname] = iface
return interfaces
return interface._get_interface_name_index(self.dbapi, host)
def _get_interface_name_datanets(self, host):
"""
Builds a dictionary of datanets indexed by interface name.
"""
interfaces = {}
for iface in self.dbapi.iinterface_get_by_ihost(host.id):
ifdatanets = self.dbapi.interface_datanetwork_get_by_interface(
iface.uuid)
datanetworks = []
for ifdatanet in ifdatanets:
datanetworks.append(ifdatanet.datanetwork_uuid)
datanetworks_list = []
for datanetwork in datanetworks:
dn = self.dbapi.datanetwork_get(datanetwork)
datanetwork_dict = \
{'name': dn.name,
'uuid': dn.uuid,
'network_type': dn.network_type,
'mtu': dn.mtu}
if dn.network_type == constants.DATANETWORK_TYPE_VXLAN:
datanetwork_dict.update(
{'multicast_group': dn.multicast_group,
'port_num': dn.port_num,
'ttl': dn.ttl,
'mode': dn.mode})
datanetworks_list.append(datanetwork_dict)
interfaces[iface.ifname] = datanetworks_list
LOG.debug("_get_interface_name_datanets ifdatanet=%s" % interfaces)
return interfaces
return interface._get_interface_name_datanets(self.dbapi, host)
def _get_port_pciaddr_index(self, host):
"""
@ -214,10 +180,7 @@ class InterfacePuppet(base.BasePuppet):
"""
Builds a dictionary of address lists indexed by interface name.
"""
addresses = collections.defaultdict(list)
for address in self.dbapi.addresses_get_by_host(host.id):
addresses[address.ifname].append(address)
return addresses
return interface._get_address_interface_name_index(self.dbapi, host)
def _get_routes_interface_name_index(self, host):
"""
@ -498,15 +461,21 @@ def get_interface_datanets(context, iface):
"""
Return the list of data networks of the supplied interface
"""
return context['interfaces_datanets'][iface.ifname]
return interface.get_interface_datanets(context, iface)
def _get_datanetwork_names(context, iface):
"""
Return the CSV list of data networks of the supplied interface
"""
return interface._get_datanetwork_names(context, iface)
def get_interface_port(context, iface):
"""
Determine the port of the underlying device.
"""
assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET
return context['ports'][iface['id']]
return interface.get_interface_port(context, iface)
def get_interface_port_name(context, iface):

View File

@ -583,13 +583,6 @@ class NovaPuppet(openstack.OpenstackBasePuppet):
return "\"%s\"" % ','.join(
"%r:%r" % (node, cpu) for node, cpu in cpu_map.items())
def _get_datanetwork_names(self, iface):
dnets = interface.get_interface_datanets(
self.context, iface)
dnames_list = [dnet['name'] for dnet in dnets]
dnames = ",".join(dnames_list)
return dnames
def _get_pci_pt_whitelist(self, host):
# Process all configured PCI passthrough interfaces and add them to
# the list of devices to whitelist
@ -597,8 +590,7 @@ class NovaPuppet(openstack.OpenstackBasePuppet):
for iface in self.context['interfaces'].values():
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]:
port = interface.get_interface_port(self.context, iface)
dnames = self._get_datanetwork_names(iface)
dnames = interface._get_datanetwork_names(self.context, iface)
device = {
'address': port['pciaddr'],
'physical_network': dnames
@ -629,13 +621,13 @@ class NovaPuppet(openstack.OpenstackBasePuppet):
for iface in self.context['interfaces'].values():
if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]:
port = interface.get_interface_port(self.context, iface)
dnames = self._get_datanetwork_names(iface)
dnames = interface._get_datanetwork_names(self.context, iface)
device = {
'address': port['pciaddr'],
'physical_network': dnames,
'sriov_numvfs': iface['sriov_numvfs']
}
LOG.info("_get_pci_sriov_whitelist device=%s" % device)
LOG.debug("_get_pci_sriov_whitelist device=%s" % device)
devices.append(device)
return json.dumps(devices) if devices else None