From f12b0bc669ce13cf8a639d1d09dfc1f70e4bc0e7 Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 24 Jan 2019 13:54:43 -0500 Subject: [PATCH] Update nova helm overrides for PCI alias, passthrough, and SR-IOV This adds generation of nova.conf overrides to configure: - global PCI aliases for QAT and GPU devices - per-host PCI passthrough whitelist (contains both passthrough and SR-IOV devices) Helm multistring dictionary is created for PCI alias, and created for PCI passthrough whitelist. These multistring are OSLO.conf compatible with oslo_config.MultiStringOpt() multiple input values. Each multistring contains a list of JSON encoded strings. The generation of these overrides assumes that all host PCI device PFs and VFs are already provisioned, and that the resulting pci addresses are queryable using sysinv DB methods. Story: 2003909 Task: 29071 Change-Id: I8b96e471f7dcff6277cca107cbd0668ffd67c7b7 Signed-off-by: Jim Gauld --- .../sysinv/sysinv/sysinv/common/interface.py | 104 +++++++++++ sysinv/sysinv/sysinv/sysinv/helm/neutron.py | 6 +- sysinv/sysinv/sysinv/sysinv/helm/nova.py | 162 +++++++++++++++++- sysinv/sysinv/sysinv/sysinv/helm/openstack.py | 63 ++++++- .../sysinv/sysinv/sysinv/puppet/interface.py | 61 ++----- sysinv/sysinv/sysinv/sysinv/puppet/nova.py | 14 +- 6 files changed, 348 insertions(+), 62 deletions(-) create mode 100644 sysinv/sysinv/sysinv/sysinv/common/interface.py diff --git a/sysinv/sysinv/sysinv/sysinv/common/interface.py b/sysinv/sysinv/sysinv/sysinv/common/interface.py new file mode 100644 index 0000000000..0ef20350ef --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/common/interface.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Common interface utility and helper functions.""" + +import collections + +from sysinv.common import constants +from sysinv.openstack.common import log + +LOG = log.getLogger(__name__) + + +def _get_port_interface_id_index(dbapi, host): + """ + Builds a dictionary of ports indexed by interface id. + """ + ports = {} + for port in dbapi.ethernet_port_get_by_host(host.id): + ports[port.interface_id] = port + return ports + + +def _get_interface_name_index(dbapi, host): + """ + Builds a dictionary of interfaces indexed by interface name. + """ + interfaces = {} + for iface in dbapi.iinterface_get_by_ihost(host.id): + interfaces[iface.ifname] = iface + return interfaces + + +def _get_interface_name_datanets(dbapi, host): + """ + Builds a dictionary of datanets indexed by interface name. + """ + datanets = {} + for iface in dbapi.iinterface_get_by_ihost(host.id): + ifdatanets = dbapi.interface_datanetwork_get_by_interface(iface.uuid) + + datanetworks = [] + for ifdatanet in ifdatanets: + datanetworks.append(ifdatanet.datanetwork_uuid) + + datanetworks_list = [] + for datanetwork in datanetworks: + dn = dbapi.datanetwork_get(datanetwork) + datanetwork_dict = \ + {'name': dn.name, + 'uuid': dn.uuid, + 'network_type': dn.network_type, + 'mtu': dn.mtu} + if dn.network_type == constants.DATANETWORK_TYPE_VXLAN: + datanetwork_dict.update( + {'multicast_group': dn.multicast_group, + 'port_num': dn.port_num, + 'ttl': dn.ttl, + 'mode': dn.mode}) + datanetworks_list.append(datanetwork_dict) + datanets[iface.ifname] = datanetworks_list + + LOG.debug('_get_interface_name_datanets ' + 'host=%s, datanets=%s', host.hostname, datanets) + + return datanets + + +def _get_address_interface_name_index(dbapi, host): + """ + Builds a dictionary of address lists indexed by interface name. + """ + addresses = collections.defaultdict(list) + for address in dbapi.addresses_get_by_host(host.id): + addresses[address.ifname].append(address) + return addresses + + +def get_interface_datanets(context, iface): + """ + Return the list of data networks of the supplied interface + """ + return context['interfaces_datanets'][iface.ifname] + + +def _get_datanetwork_names(context, iface): + """ + Return the CSV list of data networks of the supplied interface + """ + dnets = get_interface_datanets(context, iface) + dnames_list = [dnet['name'] for dnet in dnets] + dnames = ",".join(dnames_list) + return dnames + + +def get_interface_port(context, iface): + """ + Determine the port of the underlying device. + """ + assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET + return context['ports'][iface['id']] diff --git a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py index c900d70472..169e7173e0 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -189,8 +189,8 @@ class NeutronHelm(openstack.OpenstackBaseHelm): if brname: datanets = self._get_interface_datanets(iface) for datanet in datanets: - LOG.info("_get_dynamic_ovs_agent_config datanet %s" % - datanet) + LOG.debug('_get_dynamic_ovs_agent_config ' + 'host=%s datanet=%s', host.hostname, datanet) address = self._get_interface_primary_address( self.context, host, iface) if address: diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova.py b/sysinv/sysinv/sysinv/sysinv/helm/nova.py index c8f76988df..b756e6b50a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova.py @@ -9,6 +9,7 @@ import os from sysinv.common import constants from sysinv.common import exception +from sysinv.common import interface from sysinv.common import utils from sysinv.openstack.common import log as logging from sysinv.helm import common @@ -17,6 +18,34 @@ from sysinv.helm import openstack LOG = logging.getLogger(__name__) +DEFAULT_NOVA_PCI_ALIAS = [ + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_NAME}, + {"class_id": constants.NOVA_PCI_ALIAS_GPU_CLASS, + "name": constants.NOVA_PCI_ALIAS_GPU_NAME} +] + +SERVICE_PARAM_NOVA_PCI_ALIAS = [ + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER] + + class NovaHelm(openstack.OpenstackBaseHelm): """Class to encapsulate helm operations for the nova chart""" @@ -50,7 +79,8 @@ class NovaHelm(openstack.OpenstackBaseHelm): }, 'vnc': { 'novncproxy_base_url': self._get_novncproxy_base_url(), - } + }, + 'pci': self._get_pci_alias(), }, 'overrides': { 'nova_compute': { @@ -185,6 +215,133 @@ class NovaHelm(openstack.OpenstackBaseHelm): "%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items()) default_config.update({'shared_pcpu_map': shared_cpu_fmt}) + def _get_pci_pt_whitelist(self, host, iface_context): + # Process all configured PCI passthrough interfaces and add them to + # the list of devices to whitelist + devices = [] + for iface in iface_context['interfaces'].values(): + if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]: + port = interface.get_interface_port(iface_context, iface) + dnames = interface._get_datanetwork_names(iface_context, iface) + device = { + 'address': port['pciaddr'], + 'physical_network': dnames, + } + LOG.debug('_get_pci_pt_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + # Process all enabled PCI devices configured for PT and SRIOV and + # add them to the list of devices to whitelist. + # Since we are now properly initializing the qat driver and + # restarting sysinv, we need to add VF devices to the regular + # whitelist instead of the sriov whitelist + pci_devices = self.dbapi.pci_device_get_by_host(host.id) + for pci_device in pci_devices: + if pci_device.enabled: + device = { + 'address': pci_device.pciaddr, + 'class_id': pci_device.pclass_id + } + LOG.debug('_get_pci_pt_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + return devices + + def _get_pci_sriov_whitelist(self, host, iface_context): + # Process all configured SRIOV interfaces and add each VF + # to the list of devices to whitelist + devices = [] + for iface in iface_context['interfaces'].values(): + if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]: + port = interface.get_interface_port(iface_context, iface) + dnames = interface._get_datanetwork_names(iface_context, iface) + vf_addrs = port['sriov_vfs_pci_address'] + if vf_addrs: + for vf_addr in vf_addrs.split(","): + device = { + 'address': vf_addr, + 'physical_network': dnames, + } + LOG.debug('_get_pci_sriov_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + return devices + + def _get_pci_alias(self): + """ + Generate multistring values containing global PCI alias + configuration for QAT and GPU devices. + + The multistring type with list of JSON string values is used + to generate one-line-per-entry formatting, since JSON list of + dict is not supported by nova. + """ + service_parameters = self._get_service_parameter_configs( + constants.SERVICE_TYPE_NOVA) + + alias_config = DEFAULT_NOVA_PCI_ALIAS[:] + + if service_parameters is not None: + for p in SERVICE_PARAM_NOVA_PCI_ALIAS: + value = self._service_parameter_lookup_one( + service_parameters, + constants.SERVICE_PARAM_SECTION_NOVA_PCI_ALIAS, + p, None) + if value is not None: + # Replace any references to device_id with product_id + # This is to align with the requirements of the + # Nova PCI request alias schema. + # (sysinv used device_id, nova uses product_id) + value = value.replace("device_id", "product_id") + + aliases = value.rstrip(';').split(';') + for alias_str in aliases: + alias = dict((str(k), str(v)) for k, v in + (x.split('=') for x in + alias_str.split(','))) + alias_config.append(alias) + + LOG.debug('_get_pci_alias: aliases = %s', alias_config) + multistring = self._oslo_multistring_override( + name='alias', values=alias_config) + return multistring + + def _update_host_pci_whitelist(self, host, pci_config): + """ + Generate multistring values containing PCI passthrough + and SR-IOV devices. + + The multistring type with list of JSON string values is used + to generate one-line-per-entry pretty formatting. + """ + # obtain interface information specific to this host + iface_context = { + 'ports': interface._get_port_interface_id_index( + self.dbapi, host), + 'interfaces': interface._get_interface_name_index( + self.dbapi, host), + 'interfaces_datanets': interface._get_interface_name_datanets( + self.dbapi, host), + 'addresses': interface._get_address_interface_name_index( + self.dbapi, host), + } + + # This host's list of PCI passthrough and SR-IOV device dictionaries + devices = [] + devices.extend(self._get_pci_pt_whitelist(host, iface_context)) + devices.extend(self._get_pci_sriov_whitelist(host, iface_context)) + if not devices: + return + + # Convert device list into passthrough_whitelist multistring + multistring = self._oslo_multistring_override( + name='passthrough_whitelist', values=devices) + if multistring is not None: + pci_config.update(multistring) + def _update_host_storage(self, host, default_config, libvirt_config): remote_storage = False labels = self.dbapi.label_get_all(host.id) @@ -307,11 +464,13 @@ class NovaHelm(openstack.OpenstackBaseHelm): default_config = {} vnc_config = {} libvirt_config = {} + pci_config = {} self._update_host_cpu_maps(host, default_config) self._update_host_storage(host, default_config, libvirt_config) self._update_host_addresses(host, default_config, vnc_config, libvirt_config) self._update_host_memory(host, default_config) + self._update_host_pci_whitelist(host, pci_config) host_nova = { 'name': hostname, 'conf': { @@ -319,6 +478,7 @@ class NovaHelm(openstack.OpenstackBaseHelm): 'DEFAULT': default_config, 'vnc': vnc_config, 'libvirt': libvirt_config, + 'pci': pci_config if pci_config else None, } } } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py index 7166bb9cc0..c5c8d52679 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -12,8 +12,10 @@ from sysinv.helm import base from sysinv.helm import common from oslo_log import log +from oslo_serialization import jsonutils from sysinv.common import constants from sysinv.common import exception +from sqlalchemy.orm.exc import NoResultFound LOG = log.getLogger(__name__) @@ -30,6 +32,36 @@ class OpenstackBaseHelm(base.BaseHelm): configs[service] = self._get_service(service) return configs[service] + def _get_service_parameters(self, service=None): + service_parameters = [] + if self.dbapi is None: + return service_parameters + try: + service_parameters = self.dbapi.service_parameter_get_all( + service=service) + # the service parameter has not been added + except NoResultFound: + pass + return service_parameters + + def _get_service_parameter_configs(self, service): + configs = self.context.setdefault('_service_params', {}) + if service not in configs: + params = self._get_service_parameters(service) + if params: + configs[service] = params + else: + return None + return configs[service] + + @staticmethod + def _service_parameter_lookup_one(service_parameters, section, name, + default): + for param in service_parameters: + if param['section'] == section and param['name'] == name: + return param['value'] + return default + def _get_admin_user_name(self): keystone_operator = self._operator.chart_operators[ constants.HELM_CHART_KEYSTONE] @@ -262,3 +294,32 @@ class OpenstackBaseHelm(base.BaseHelm): name=chart, namespace=namespace, values=values) return newprivatekey, newpublickey + + def _oslo_multistring_override(self, name=None, values=[]): + """ + Generate helm multistring dictionary override for specified option + name with multiple values. + + This generates oslo_config.MultiStringOpt() compatible config + with multiple input values. This routine JSON encodes each value for + complex types (eg, dict, list, set). + + Return a multistring type formatted dictionary override. + """ + override = None + if name is None or not values: + return override + + mvalues = [] + for value in values: + if isinstance(value, (dict, list, set)): + mvalues.append(jsonutils.dumps(value)) + else: + mvalues.append(value) + + override = { + name: {'type': 'multistring', + 'values': mvalues, + } + } + return override diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py index 4423422fac..dc33199a0e 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2017 Wind River Systems, Inc. +# Copyright (c) 2017-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -13,6 +13,7 @@ from netaddr import IPNetwork from sysinv.common import constants from sysinv.common import exception +from sysinv.common import interface from sysinv.common import utils from sysinv.conductor import openstack from sysinv.openstack.common import log @@ -152,54 +153,19 @@ class InterfacePuppet(base.BasePuppet): """ Builds a dictionary of ports indexed by interface id. """ - ports = {} - for port in self.dbapi.ethernet_port_get_by_host(host.id): - ports[port.interface_id] = port - return ports + return interface._get_port_interface_id_index(self.dbapi, host) def _get_interface_name_index(self, host): """ Builds a dictionary of interfaces indexed by interface name. """ - interfaces = {} - for iface in self.dbapi.iinterface_get_by_ihost(host.id): - interfaces[iface.ifname] = iface - - return interfaces + return interface._get_interface_name_index(self.dbapi, host) def _get_interface_name_datanets(self, host): """ Builds a dictionary of datanets indexed by interface name. """ - interfaces = {} - for iface in self.dbapi.iinterface_get_by_ihost(host.id): - ifdatanets = self.dbapi.interface_datanetwork_get_by_interface( - iface.uuid) - - datanetworks = [] - for ifdatanet in ifdatanets: - datanetworks.append(ifdatanet.datanetwork_uuid) - - datanetworks_list = [] - for datanetwork in datanetworks: - dn = self.dbapi.datanetwork_get(datanetwork) - datanetwork_dict = \ - {'name': dn.name, - 'uuid': dn.uuid, - 'network_type': dn.network_type, - 'mtu': dn.mtu} - if dn.network_type == constants.DATANETWORK_TYPE_VXLAN: - datanetwork_dict.update( - {'multicast_group': dn.multicast_group, - 'port_num': dn.port_num, - 'ttl': dn.ttl, - 'mode': dn.mode}) - datanetworks_list.append(datanetwork_dict) - interfaces[iface.ifname] = datanetworks_list - - LOG.debug("_get_interface_name_datanets ifdatanet=%s" % interfaces) - - return interfaces + return interface._get_interface_name_datanets(self.dbapi, host) def _get_port_pciaddr_index(self, host): """ @@ -214,10 +180,7 @@ class InterfacePuppet(base.BasePuppet): """ Builds a dictionary of address lists indexed by interface name. """ - addresses = collections.defaultdict(list) - for address in self.dbapi.addresses_get_by_host(host.id): - addresses[address.ifname].append(address) - return addresses + return interface._get_address_interface_name_index(self.dbapi, host) def _get_routes_interface_name_index(self, host): """ @@ -498,15 +461,21 @@ def get_interface_datanets(context, iface): """ Return the list of data networks of the supplied interface """ - return context['interfaces_datanets'][iface.ifname] + return interface.get_interface_datanets(context, iface) + + +def _get_datanetwork_names(context, iface): + """ + Return the CSV list of data networks of the supplied interface + """ + return interface._get_datanetwork_names(context, iface) def get_interface_port(context, iface): """ Determine the port of the underlying device. """ - assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET - return context['ports'][iface['id']] + return interface.get_interface_port(context, iface) def get_interface_port_name(context, iface): diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/nova.py b/sysinv/sysinv/sysinv/sysinv/puppet/nova.py index 80412b25c4..3e42e01ab8 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/nova.py @@ -583,13 +583,6 @@ class NovaPuppet(openstack.OpenstackBasePuppet): return "\"%s\"" % ','.join( "%r:%r" % (node, cpu) for node, cpu in cpu_map.items()) - def _get_datanetwork_names(self, iface): - dnets = interface.get_interface_datanets( - self.context, iface) - dnames_list = [dnet['name'] for dnet in dnets] - dnames = ",".join(dnames_list) - return dnames - def _get_pci_pt_whitelist(self, host): # Process all configured PCI passthrough interfaces and add them to # the list of devices to whitelist @@ -597,8 +590,7 @@ class NovaPuppet(openstack.OpenstackBasePuppet): for iface in self.context['interfaces'].values(): if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]: port = interface.get_interface_port(self.context, iface) - - dnames = self._get_datanetwork_names(iface) + dnames = interface._get_datanetwork_names(self.context, iface) device = { 'address': port['pciaddr'], 'physical_network': dnames @@ -629,13 +621,13 @@ class NovaPuppet(openstack.OpenstackBasePuppet): for iface in self.context['interfaces'].values(): if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]: port = interface.get_interface_port(self.context, iface) - dnames = self._get_datanetwork_names(iface) + dnames = interface._get_datanetwork_names(self.context, iface) device = { 'address': port['pciaddr'], 'physical_network': dnames, 'sriov_numvfs': iface['sriov_numvfs'] } - LOG.info("_get_pci_sriov_whitelist device=%s" % device) + LOG.debug("_get_pci_sriov_whitelist device=%s" % device) devices.append(device) return json.dumps(devices) if devices else None