From c66853f7e6a932fa2fd1202511f3202bce589dfd Mon Sep 17 00:00:00 2001 From: Heitor Matsui Date: Thu, 14 Oct 2021 12:45:15 -0300 Subject: [PATCH] Create containerized pci-irq-affinity-agent This commit makes the necessary changes to convert the pci-irq-affinity-agent into a container, to be deployed along with OpenStack application. What was modified: - Agent logging facility changed from syslog to stdout - Agent session creation includes CA certificate to work with TLS - Agent can now use environment variables with properties from the host running the containerized agent - Files changed to build a wheel for the agent - Files added to build the container image Test Plan: PASS: Verify that logging facility was changed to stdout PASS: Verify that agent can communicate with Nova using TLS PASS: Verify that agent image was built successfully PASS: Verify that agent can be run properly in a container PASS: Verify that agent can load and use environment variables with properties from the host like hostname and filepaths Regression: PASS: Verify that agent logs no errors while running in a container PASS: Verify that agent connects to rabbit and nova properly while running in a container Story: 2009299 Task: 43650 Signed-off-by: Heitor Matsui Change-Id: I7680e39de734e16affebff893df21c6b5089dc76 --- centos_dev_docker_images.inc | 2 ++ centos_dev_wheels.inc | 1 + centos_stable_docker_images.inc | 1 + centos_stable_wheels.inc | 1 + .../centos/pci-irq-affinity.dev_docker_image | 23 +++++++++++++++++++ .../centos/pci-irq-affinity.spec | 14 +++++++++++ .../pci-irq-affinity.stable_docker_image | 23 +++++++++++++++++++ .../pci_irq_affinity/affinity.py | 7 +++++- .../pci_irq_affinity/agent.py | 4 ++-- .../pci_irq_affinity/pci_irq_affinity/log.py | 10 +++----- .../pci_irq_affinity/nova_provider.py | 16 +++++++++---- .../pci_irq_affinity/utils.py | 14 +++++++---- 12 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 centos_dev_docker_images.inc create mode 100644 utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.dev_docker_image create mode 100644 utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.stable_docker_image diff --git a/centos_dev_docker_images.inc b/centos_dev_docker_images.inc new file mode 100644 index 00000000..baa83863 --- /dev/null +++ b/centos_dev_docker_images.inc @@ -0,0 +1,2 @@ +ceph/ceph-manager +utilities/pci-irq-affinity-agent \ No newline at end of file diff --git a/centos_dev_wheels.inc b/centos_dev_wheels.inc index 314a4ebc..9b90e276 100644 --- a/centos_dev_wheels.inc +++ b/centos_dev_wheels.inc @@ -1 +1,2 @@ python-cephclient-wheels +pci-irq-affinity-agent-wheels diff --git a/centos_stable_docker_images.inc b/centos_stable_docker_images.inc index efc47163..2104c00a 100644 --- a/centos_stable_docker_images.inc +++ b/centos_stable_docker_images.inc @@ -1 +1,2 @@ ceph/ceph-manager +utilities/pci-irq-affinity-agent diff --git a/centos_stable_wheels.inc b/centos_stable_wheels.inc index 314a4ebc..9b90e276 100644 --- a/centos_stable_wheels.inc +++ b/centos_stable_wheels.inc @@ -1 +1,2 @@ python-cephclient-wheels +pci-irq-affinity-agent-wheels diff --git a/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.dev_docker_image b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.dev_docker_image new file mode 100644 index 00000000..0d762b81 --- /dev/null +++ b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.dev_docker_image @@ -0,0 +1,23 @@ +BUILDER=loci +LABEL=stx-pci-irq-affinity-agent +PROJECT=infra +PROJECT_REPO=nil +DIST_PACKAGES="libvirt-python python-devel gcc" +PIP_PACKAGES="pci-irq-affinity-agent \ + eventlet \ + keyring \ + keyrings.alt \ + pycrypto \ + keystoneauth1 \ + pkgconfig \ + libvirt-python \ + oslo.concurrency \ + oslo.config \ + oslo.i18n \ + oslo.log \ + oslo.messaging \ + oslo.service \ + python-daemon \ + python-novaclient \ + retrying \ + psutil" diff --git a/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.spec b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.spec index fc8d9853..22ba618f 100644 --- a/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.spec +++ b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.spec @@ -12,6 +12,7 @@ Source0: %{name}-%{version}.tar.gz Requires: python-novaclient BuildRequires: python-setuptools BuildRequires: systemd-devel +BuildRequires: python2-wheel %description StarlingX PCI Interrupt Affinity Agent Package @@ -29,6 +30,7 @@ rm -rf *.egg-info %build %{__python} setup.py build +%{__python} setup.py bdist_wheel %install %{__python} setup.py install --root=%{buildroot} \ @@ -37,6 +39,9 @@ rm -rf *.egg-info --install-data=/usr/share \ --single-version-externally-managed +mkdir -p $RPM_BUILD_ROOT/wheels +%{__install} -m 644 dist/*.whl $RPM_BUILD_ROOT/wheels/ + %{__install} -d -m 755 %{buildroot}%{local_etc_initd} %{__install} -p -D -m 755 pci-irq-affinity-agent %{buildroot}%{local_etc_initd}/pci-irq-affinity-agent @@ -68,3 +73,12 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/pci-irq-affinity-agent %{_bindir}/nova-sriov %config(noreplace) %{_sysconfdir}/pci_irq_affinity/config.ini + +%package wheels +Summary: %{name} wheels + +%description wheels +Contains python wheels for %{name} + +%files wheels +/wheels/* diff --git a/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.stable_docker_image b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.stable_docker_image new file mode 100644 index 00000000..0d762b81 --- /dev/null +++ b/utilities/pci-irq-affinity-agent/centos/pci-irq-affinity.stable_docker_image @@ -0,0 +1,23 @@ +BUILDER=loci +LABEL=stx-pci-irq-affinity-agent +PROJECT=infra +PROJECT_REPO=nil +DIST_PACKAGES="libvirt-python python-devel gcc" +PIP_PACKAGES="pci-irq-affinity-agent \ + eventlet \ + keyring \ + keyrings.alt \ + pycrypto \ + keystoneauth1 \ + pkgconfig \ + libvirt-python \ + oslo.concurrency \ + oslo.config \ + oslo.i18n \ + oslo.log \ + oslo.messaging \ + oslo.service \ + python-daemon \ + python-novaclient \ + retrying \ + psutil" diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py index a4484aa8..c84a8eea 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py @@ -10,14 +10,18 @@ # """ Define pci_irq_affinity_provider class""" +import os import pci_irq_affinity.utils as pci_utils from pci_irq_affinity.driver import AffinePciIrqDriver from pci_irq_affinity.nova_provider import novaClient from pci_irq_affinity.log import LOG +COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/") + class pci_irq_affinity_provider: + def __init__(self): self.affinePciIrqDriver = AffinePciIrqDriver() self.inst_dict = {} @@ -28,6 +32,7 @@ class pci_irq_affinity_provider: The instance has already been deleted or related PCI not used by it anymore. """ + if irqs or msi_irqs: # reset irq affinity for specified irqs _irqs = irqs @@ -42,7 +47,7 @@ class pci_irq_affinity_provider: return try: - with open('/proc/irq/default_smp_affinity') as f: + with open('%s/default_smp_affinity' % COMPUTE_IRQ) as f: cpulist = f.readline().strip() LOG.debug("default smp affinity bitmap:%s" % cpulist) diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py index 8ac50dc2..ab619386 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 StarlingX. +# Copyright (c) 2019-2021 StarlingX. # # SPDX-License-Identifier: Apache-2.0 # @@ -195,7 +195,7 @@ def process_main(): sys.exit(200) finally: - LOG.error("proces_main finalized!!!") + LOG.error("process_main finalized!!!") if openstack_enabled == 'true': novaClient.close_libvirt_connect() audit_srv.tg.stop() diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/log.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/log.py index 878ef98d..2eae90e6 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/log.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/log.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 StarlingX. +# Copyright (c) 2019-2021 StarlingX. # # SPDX-License-Identifier: Apache-2.0 # @@ -12,19 +12,15 @@ """ Define Logger class for this agent""" import logging -import logging.handlers - +import sys from pci_irq_affinity.config import CONF -_syslog_facility = 'local1' - LOG = logging.getLogger("pci-interrupt-affinity") formatter = logging.Formatter("%(asctime)s %(threadName)s[%(process)d] " "%(name)s.%(pathname)s.%(lineno)d - %(levelname)s " "%(message)s") -handler = logging.handlers.SysLogHandler(address='/dev/log', - facility=_syslog_facility) +handler = logging.StreamHandler(stream=sys.stdout) handler.setFormatter(formatter) LOG.addHandler(handler) LOG.setLevel(CONF.log_level) diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py index 14c72080..9bbcbdef 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py @@ -18,9 +18,9 @@ import keyring from novaclient import client from keystoneauth1 import loading from keystoneauth1 import session +import os import socket from pci_irq_affinity.log import LOG -from pci_irq_affinity.config import CONF from pci_irq_affinity.config import sysconfig from pci_irq_affinity import instance from pci_irq_affinity import guest @@ -31,11 +31,12 @@ class NovaProvider: def __init__(self): self._creds = self._get_keystone_creds() self._auth = self._get_auth(self._creds) + self._cacert = self._get_cacert() self._hostname = self.get_hostname() self._conn = None def get_hostname(self): - return socket.gethostname() + return os.getenv("COMPUTE_HOSTNAME", default=socket.gethostname()) def _get_keystone_creds(self): creds = {} @@ -57,16 +58,23 @@ class NovaProvider: return creds def _get_auth(self, creds): - if creds is not None: loader = loading.get_plugin_loader('password') auth = loader.load_from_options(**creds) return auth return None + def _get_cacert(self): + if ( + sysconfig.has_option('openstack', 'cacert') + and len(sysconfig.get('openstack', 'cacert')) + ): + return sysconfig.get('openstack', 'cacert') + return None + def get_nova(self): try: - sess = session.Session(auth=self._auth) + sess = session.Session(auth=self._auth, verify=self._cacert) nova = client.Client('2.1', session=sess) return nova except Exception as e: diff --git a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/utils.py b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/utils.py index aa18da5f..5ce14ac1 100644 --- a/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/utils.py +++ b/utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/utils.py @@ -19,6 +19,10 @@ from pci_irq_affinity.log import LOG import pci_irq_affinity.instance +COMPUTE_PCI_DEVICES = os.getenv("COMPUTE_PCI_DEVICES", default="/sys/bus/pci/devices") +COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq") + + def list_to_range(input_list=None): """Convert a list into a string of comma separate ranges. @@ -123,7 +127,7 @@ def get_irqs_by_pci_address(pci_addr): irqs = set() msi_irqs = set() - dev_path = "/sys/bus/pci/devices/%s" % (pci_addr) + dev_path = "%s/%s" % (COMPUTE_PCI_DEVICES, pci_addr) if not os.path.isdir(dev_path): raise Exception("PciDeviceNotFoundById id = %r" % pci_addr) @@ -160,11 +164,11 @@ def get_irqs_by_pci_address(pci_addr): # Return only configured irqs, ignore any that are missing. for irq in _irqs: - irq_path = "/proc/irq/%s" % (irq) + irq_path = "%s/%s" % (COMPUTE_IRQ, irq) if os.path.isdir(irq_path): irqs.update([irq]) for irq in _msi_irqs: - irq_path = "/proc/irq/%s" % (irq) + irq_path = "%s/%s" % (COMPUTE_IRQ, irq) if os.path.isdir(irq_path): msi_irqs.update([irq]) return (irqs, msi_irqs) @@ -223,7 +227,7 @@ def set_irq_affinity(set_bitmap, irqs, cpulist): filename = 'smp_affinity_list' for irq in irqs: - irq_aff_path = "/proc/irq/%s/%s" % (irq, filename) + irq_aff_path = "%s/%s/%s" % (COMPUTE_IRQ, irq, filename) try: with open(irq_aff_path, 'w') as f: f.write(cpulist) @@ -262,7 +266,7 @@ def set_irqs_affinity_by_pci_address(pci_addr, extra_spec=None, LOG.debug("pci: %s, irqs: %s, msi_irqs: %s" % (pci_addr, _irqs, _msi_irqs)) # Obtain physical numa_node for this pci addr - numa_path = "/sys/bus/pci/devices/%s/numa_node" % (pci_addr) + numa_path = "%s/%s/numa_node" % (COMPUTE_PCI_DEVICES, pci_addr) try: with open(numa_path) as f: numa_node = [int(x) for x in f.readline().split()][0]