diff --git a/sysinv/sysinv/sysinv/sysinv/common/interface.py b/sysinv/sysinv/sysinv/sysinv/common/interface.py new file mode 100644 index 0000000000..0ef20350ef --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/common/interface.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Common interface utility and helper functions.""" + +import collections + +from sysinv.common import constants +from sysinv.openstack.common import log + +LOG = log.getLogger(__name__) + + +def _get_port_interface_id_index(dbapi, host): + """ + Builds a dictionary of ports indexed by interface id. + """ + ports = {} + for port in dbapi.ethernet_port_get_by_host(host.id): + ports[port.interface_id] = port + return ports + + +def _get_interface_name_index(dbapi, host): + """ + Builds a dictionary of interfaces indexed by interface name. + """ + interfaces = {} + for iface in dbapi.iinterface_get_by_ihost(host.id): + interfaces[iface.ifname] = iface + return interfaces + + +def _get_interface_name_datanets(dbapi, host): + """ + Builds a dictionary of datanets indexed by interface name. + """ + datanets = {} + for iface in dbapi.iinterface_get_by_ihost(host.id): + ifdatanets = dbapi.interface_datanetwork_get_by_interface(iface.uuid) + + datanetworks = [] + for ifdatanet in ifdatanets: + datanetworks.append(ifdatanet.datanetwork_uuid) + + datanetworks_list = [] + for datanetwork in datanetworks: + dn = dbapi.datanetwork_get(datanetwork) + datanetwork_dict = \ + {'name': dn.name, + 'uuid': dn.uuid, + 'network_type': dn.network_type, + 'mtu': dn.mtu} + if dn.network_type == constants.DATANETWORK_TYPE_VXLAN: + datanetwork_dict.update( + {'multicast_group': dn.multicast_group, + 'port_num': dn.port_num, + 'ttl': dn.ttl, + 'mode': dn.mode}) + datanetworks_list.append(datanetwork_dict) + datanets[iface.ifname] = datanetworks_list + + LOG.debug('_get_interface_name_datanets ' + 'host=%s, datanets=%s', host.hostname, datanets) + + return datanets + + +def _get_address_interface_name_index(dbapi, host): + """ + Builds a dictionary of address lists indexed by interface name. + """ + addresses = collections.defaultdict(list) + for address in dbapi.addresses_get_by_host(host.id): + addresses[address.ifname].append(address) + return addresses + + +def get_interface_datanets(context, iface): + """ + Return the list of data networks of the supplied interface + """ + return context['interfaces_datanets'][iface.ifname] + + +def _get_datanetwork_names(context, iface): + """ + Return the CSV list of data networks of the supplied interface + """ + dnets = get_interface_datanets(context, iface) + dnames_list = [dnet['name'] for dnet in dnets] + dnames = ",".join(dnames_list) + return dnames + + +def get_interface_port(context, iface): + """ + Determine the port of the underlying device. + """ + assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET + return context['ports'][iface['id']] diff --git a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py index c900d70472..169e7173e0 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -189,8 +189,8 @@ class NeutronHelm(openstack.OpenstackBaseHelm): if brname: datanets = self._get_interface_datanets(iface) for datanet in datanets: - LOG.info("_get_dynamic_ovs_agent_config datanet %s" % - datanet) + LOG.debug('_get_dynamic_ovs_agent_config ' + 'host=%s datanet=%s', host.hostname, datanet) address = self._get_interface_primary_address( self.context, host, iface) if address: diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova.py b/sysinv/sysinv/sysinv/sysinv/helm/nova.py index c8f76988df..b756e6b50a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova.py @@ -9,6 +9,7 @@ import os from sysinv.common import constants from sysinv.common import exception +from sysinv.common import interface from sysinv.common import utils from sysinv.openstack.common import log as logging from sysinv.helm import common @@ -17,6 +18,34 @@ from sysinv.helm import openstack LOG = logging.getLogger(__name__) +DEFAULT_NOVA_PCI_ALIAS = [ + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_C62X_PF_NAME}, + {"vendor_id": constants.NOVA_PCI_ALIAS_QAT_VF_VENDOR, + "product_id": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE, + "name": constants.NOVA_PCI_ALIAS_QAT_C62X_VF_NAME}, + {"class_id": constants.NOVA_PCI_ALIAS_GPU_CLASS, + "name": constants.NOVA_PCI_ALIAS_GPU_NAME} +] + +SERVICE_PARAM_NOVA_PCI_ALIAS = [ + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_GPU_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_DH895XCC_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_PF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_QAT_C62X_VF, + constants.SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER] + + class NovaHelm(openstack.OpenstackBaseHelm): """Class to encapsulate helm operations for the nova chart""" @@ -50,7 +79,8 @@ class NovaHelm(openstack.OpenstackBaseHelm): }, 'vnc': { 'novncproxy_base_url': self._get_novncproxy_base_url(), - } + }, + 'pci': self._get_pci_alias(), }, 'overrides': { 'nova_compute': { @@ -185,6 +215,133 @@ class NovaHelm(openstack.OpenstackBaseHelm): "%r:%r" % (node, cpu) for node, cpu in shared_cpu_map.items()) default_config.update({'shared_pcpu_map': shared_cpu_fmt}) + def _get_pci_pt_whitelist(self, host, iface_context): + # Process all configured PCI passthrough interfaces and add them to + # the list of devices to whitelist + devices = [] + for iface in iface_context['interfaces'].values(): + if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]: + port = interface.get_interface_port(iface_context, iface) + dnames = interface._get_datanetwork_names(iface_context, iface) + device = { + 'address': port['pciaddr'], + 'physical_network': dnames, + } + LOG.debug('_get_pci_pt_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + # Process all enabled PCI devices configured for PT and SRIOV and + # add them to the list of devices to whitelist. + # Since we are now properly initializing the qat driver and + # restarting sysinv, we need to add VF devices to the regular + # whitelist instead of the sriov whitelist + pci_devices = self.dbapi.pci_device_get_by_host(host.id) + for pci_device in pci_devices: + if pci_device.enabled: + device = { + 'address': pci_device.pciaddr, + 'class_id': pci_device.pclass_id + } + LOG.debug('_get_pci_pt_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + return devices + + def _get_pci_sriov_whitelist(self, host, iface_context): + # Process all configured SRIOV interfaces and add each VF + # to the list of devices to whitelist + devices = [] + for iface in iface_context['interfaces'].values(): + if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]: + port = interface.get_interface_port(iface_context, iface) + dnames = interface._get_datanetwork_names(iface_context, iface) + vf_addrs = port['sriov_vfs_pci_address'] + if vf_addrs: + for vf_addr in vf_addrs.split(","): + device = { + 'address': vf_addr, + 'physical_network': dnames, + } + LOG.debug('_get_pci_sriov_whitelist ' + 'host=%s, device=%s', host.hostname, device) + devices.append(device) + + return devices + + def _get_pci_alias(self): + """ + Generate multistring values containing global PCI alias + configuration for QAT and GPU devices. + + The multistring type with list of JSON string values is used + to generate one-line-per-entry formatting, since JSON list of + dict is not supported by nova. + """ + service_parameters = self._get_service_parameter_configs( + constants.SERVICE_TYPE_NOVA) + + alias_config = DEFAULT_NOVA_PCI_ALIAS[:] + + if service_parameters is not None: + for p in SERVICE_PARAM_NOVA_PCI_ALIAS: + value = self._service_parameter_lookup_one( + service_parameters, + constants.SERVICE_PARAM_SECTION_NOVA_PCI_ALIAS, + p, None) + if value is not None: + # Replace any references to device_id with product_id + # This is to align with the requirements of the + # Nova PCI request alias schema. + # (sysinv used device_id, nova uses product_id) + value = value.replace("device_id", "product_id") + + aliases = value.rstrip(';').split(';') + for alias_str in aliases: + alias = dict((str(k), str(v)) for k, v in + (x.split('=') for x in + alias_str.split(','))) + alias_config.append(alias) + + LOG.debug('_get_pci_alias: aliases = %s', alias_config) + multistring = self._oslo_multistring_override( + name='alias', values=alias_config) + return multistring + + def _update_host_pci_whitelist(self, host, pci_config): + """ + Generate multistring values containing PCI passthrough + and SR-IOV devices. + + The multistring type with list of JSON string values is used + to generate one-line-per-entry pretty formatting. + """ + # obtain interface information specific to this host + iface_context = { + 'ports': interface._get_port_interface_id_index( + self.dbapi, host), + 'interfaces': interface._get_interface_name_index( + self.dbapi, host), + 'interfaces_datanets': interface._get_interface_name_datanets( + self.dbapi, host), + 'addresses': interface._get_address_interface_name_index( + self.dbapi, host), + } + + # This host's list of PCI passthrough and SR-IOV device dictionaries + devices = [] + devices.extend(self._get_pci_pt_whitelist(host, iface_context)) + devices.extend(self._get_pci_sriov_whitelist(host, iface_context)) + if not devices: + return + + # Convert device list into passthrough_whitelist multistring + multistring = self._oslo_multistring_override( + name='passthrough_whitelist', values=devices) + if multistring is not None: + pci_config.update(multistring) + def _update_host_storage(self, host, default_config, libvirt_config): remote_storage = False labels = self.dbapi.label_get_all(host.id) @@ -307,11 +464,13 @@ class NovaHelm(openstack.OpenstackBaseHelm): default_config = {} vnc_config = {} libvirt_config = {} + pci_config = {} self._update_host_cpu_maps(host, default_config) self._update_host_storage(host, default_config, libvirt_config) self._update_host_addresses(host, default_config, vnc_config, libvirt_config) self._update_host_memory(host, default_config) + self._update_host_pci_whitelist(host, pci_config) host_nova = { 'name': hostname, 'conf': { @@ -319,6 +478,7 @@ class NovaHelm(openstack.OpenstackBaseHelm): 'DEFAULT': default_config, 'vnc': vnc_config, 'libvirt': libvirt_config, + 'pci': pci_config if pci_config else None, } } } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py index 7166bb9cc0..c5c8d52679 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -12,8 +12,10 @@ from sysinv.helm import base from sysinv.helm import common from oslo_log import log +from oslo_serialization import jsonutils from sysinv.common import constants from sysinv.common import exception +from sqlalchemy.orm.exc import NoResultFound LOG = log.getLogger(__name__) @@ -30,6 +32,36 @@ class OpenstackBaseHelm(base.BaseHelm): configs[service] = self._get_service(service) return configs[service] + def _get_service_parameters(self, service=None): + service_parameters = [] + if self.dbapi is None: + return service_parameters + try: + service_parameters = self.dbapi.service_parameter_get_all( + service=service) + # the service parameter has not been added + except NoResultFound: + pass + return service_parameters + + def _get_service_parameter_configs(self, service): + configs = self.context.setdefault('_service_params', {}) + if service not in configs: + params = self._get_service_parameters(service) + if params: + configs[service] = params + else: + return None + return configs[service] + + @staticmethod + def _service_parameter_lookup_one(service_parameters, section, name, + default): + for param in service_parameters: + if param['section'] == section and param['name'] == name: + return param['value'] + return default + def _get_admin_user_name(self): keystone_operator = self._operator.chart_operators[ constants.HELM_CHART_KEYSTONE] @@ -262,3 +294,32 @@ class OpenstackBaseHelm(base.BaseHelm): name=chart, namespace=namespace, values=values) return newprivatekey, newpublickey + + def _oslo_multistring_override(self, name=None, values=[]): + """ + Generate helm multistring dictionary override for specified option + name with multiple values. + + This generates oslo_config.MultiStringOpt() compatible config + with multiple input values. This routine JSON encodes each value for + complex types (eg, dict, list, set). + + Return a multistring type formatted dictionary override. + """ + override = None + if name is None or not values: + return override + + mvalues = [] + for value in values: + if isinstance(value, (dict, list, set)): + mvalues.append(jsonutils.dumps(value)) + else: + mvalues.append(value) + + override = { + name: {'type': 'multistring', + 'values': mvalues, + } + } + return override diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py index 4423422fac..dc33199a0e 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2017 Wind River Systems, Inc. +# Copyright (c) 2017-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -13,6 +13,7 @@ from netaddr import IPNetwork from sysinv.common import constants from sysinv.common import exception +from sysinv.common import interface from sysinv.common import utils from sysinv.conductor import openstack from sysinv.openstack.common import log @@ -152,54 +153,19 @@ class InterfacePuppet(base.BasePuppet): """ Builds a dictionary of ports indexed by interface id. """ - ports = {} - for port in self.dbapi.ethernet_port_get_by_host(host.id): - ports[port.interface_id] = port - return ports + return interface._get_port_interface_id_index(self.dbapi, host) def _get_interface_name_index(self, host): """ Builds a dictionary of interfaces indexed by interface name. """ - interfaces = {} - for iface in self.dbapi.iinterface_get_by_ihost(host.id): - interfaces[iface.ifname] = iface - - return interfaces + return interface._get_interface_name_index(self.dbapi, host) def _get_interface_name_datanets(self, host): """ Builds a dictionary of datanets indexed by interface name. """ - interfaces = {} - for iface in self.dbapi.iinterface_get_by_ihost(host.id): - ifdatanets = self.dbapi.interface_datanetwork_get_by_interface( - iface.uuid) - - datanetworks = [] - for ifdatanet in ifdatanets: - datanetworks.append(ifdatanet.datanetwork_uuid) - - datanetworks_list = [] - for datanetwork in datanetworks: - dn = self.dbapi.datanetwork_get(datanetwork) - datanetwork_dict = \ - {'name': dn.name, - 'uuid': dn.uuid, - 'network_type': dn.network_type, - 'mtu': dn.mtu} - if dn.network_type == constants.DATANETWORK_TYPE_VXLAN: - datanetwork_dict.update( - {'multicast_group': dn.multicast_group, - 'port_num': dn.port_num, - 'ttl': dn.ttl, - 'mode': dn.mode}) - datanetworks_list.append(datanetwork_dict) - interfaces[iface.ifname] = datanetworks_list - - LOG.debug("_get_interface_name_datanets ifdatanet=%s" % interfaces) - - return interfaces + return interface._get_interface_name_datanets(self.dbapi, host) def _get_port_pciaddr_index(self, host): """ @@ -214,10 +180,7 @@ class InterfacePuppet(base.BasePuppet): """ Builds a dictionary of address lists indexed by interface name. """ - addresses = collections.defaultdict(list) - for address in self.dbapi.addresses_get_by_host(host.id): - addresses[address.ifname].append(address) - return addresses + return interface._get_address_interface_name_index(self.dbapi, host) def _get_routes_interface_name_index(self, host): """ @@ -498,15 +461,21 @@ def get_interface_datanets(context, iface): """ Return the list of data networks of the supplied interface """ - return context['interfaces_datanets'][iface.ifname] + return interface.get_interface_datanets(context, iface) + + +def _get_datanetwork_names(context, iface): + """ + Return the CSV list of data networks of the supplied interface + """ + return interface._get_datanetwork_names(context, iface) def get_interface_port(context, iface): """ Determine the port of the underlying device. """ - assert iface['iftype'] == constants.INTERFACE_TYPE_ETHERNET - return context['ports'][iface['id']] + return interface.get_interface_port(context, iface) def get_interface_port_name(context, iface): diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/nova.py b/sysinv/sysinv/sysinv/sysinv/puppet/nova.py index 80412b25c4..3e42e01ab8 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/nova.py @@ -583,13 +583,6 @@ class NovaPuppet(openstack.OpenstackBasePuppet): return "\"%s\"" % ','.join( "%r:%r" % (node, cpu) for node, cpu in cpu_map.items()) - def _get_datanetwork_names(self, iface): - dnets = interface.get_interface_datanets( - self.context, iface) - dnames_list = [dnet['name'] for dnet in dnets] - dnames = ",".join(dnames_list) - return dnames - def _get_pci_pt_whitelist(self, host): # Process all configured PCI passthrough interfaces and add them to # the list of devices to whitelist @@ -597,8 +590,7 @@ class NovaPuppet(openstack.OpenstackBasePuppet): for iface in self.context['interfaces'].values(): if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_PASSTHROUGH]: port = interface.get_interface_port(self.context, iface) - - dnames = self._get_datanetwork_names(iface) + dnames = interface._get_datanetwork_names(self.context, iface) device = { 'address': port['pciaddr'], 'physical_network': dnames @@ -629,13 +621,13 @@ class NovaPuppet(openstack.OpenstackBasePuppet): for iface in self.context['interfaces'].values(): if iface['ifclass'] in [constants.INTERFACE_CLASS_PCI_SRIOV]: port = interface.get_interface_port(self.context, iface) - dnames = self._get_datanetwork_names(iface) + dnames = interface._get_datanetwork_names(self.context, iface) device = { 'address': port['pciaddr'], 'physical_network': dnames, 'sriov_numvfs': iface['sriov_numvfs'] } - LOG.info("_get_pci_sriov_whitelist device=%s" % device) + LOG.debug("_get_pci_sriov_whitelist device=%s" % device) devices.append(device) return json.dumps(devices) if devices else None