Refactoring novaClient instantiation and unittests

And adding infra for unittest execution.

Partial-Bug: 1951579
Signed-off-by: Thiago Brito <thiago.brito@windriver.com>
Change-Id: I58cefac9076db52333b41633bf2cbaa5441dc98c

.gitignore

@@ -2,3 +2,5 @@
 *.egg-info
 *.swp
 .idea
+__pycache__
+*.py[cod]

.zuul.yaml

@@ -6,12 +6,18 @@
         - stx-tox-pylint
         - utilities-ceph-manager-tox-bandit
         - utilities-ceph-client-tox-bandit
+        - utilities-pci-irq-affinity-agent-tox-py27
+        - utilities-pci-irq-affinity-agent-tox-py39
+        - utilities-pci-irq-affinity-agent-tox-flake8
     gate:
       jobs:
         - openstack-tox-linters
         - stx-tox-pylint
         - utilities-ceph-manager-tox-bandit
         - utilities-ceph-client-tox-bandit
+        - utilities-pci-irq-affinity-agent-tox-py27
+        - utilities-pci-irq-affinity-agent-tox-py39
+        - utilities-pci-irq-affinity-agent-tox-flake8
     post:
       jobs:
         - stx-utilities-upload-git-mirror
@@ -63,6 +69,44 @@
       tox_envlist: bandit
       tox_extra_args: -c ./ceph/python-cephclient/python-cephclient/tox.ini
 
+- job:
+    name: utilities-pci-irq-affinity-agent-tox-py27
+    parent: tox
+    description: |
+      Run py27 unittests for utilities/pci-irq-affinity-agent
+    nodeset: ubuntu-bionic
+    files:
+      - ./utilities/pci-irq-affinity-agent/*
+    vars:
+      tox_envlist: py27
+      tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
+
+- job:
+    name: utilities-pci-irq-affinity-agent-tox-py39
+    voting: false
+    parent: tox
+    description: |
+      Run py39 unittests for utilities/pci-irq-affinity-agent
+    nodeset: debian-bullseye
+    files:
+      - ./utilities/pci-irq-affinity-agent/*
+    vars:
+      tox_envlist: py39
+      python_version: 3.9
+      tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
+
+- job:
+    name: utilities-pci-irq-affinity-agent-tox-flake8
+    parent: tox
+    description: |
+      Run flake8 checks for utilities/pci-irq-affinity-agent
+    nodeset: ubuntu-bionic
+    files:
+      - ./utilities/pci-irq-affinity-agent/*
+    vars:
+      tox_envlist: flake8
+      tox_extra_args: -c ./utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini
+
 - secret:
     name: stx-utilities-github-secret
     data:

utilities/pci-irq-affinity-agent/pci_irq_affinity/bindep.txt

@@ -0,0 +1,13 @@
+# This is a cross-platform list tracking distribution packages needed for install and tests;
+# see https://docs.openstack.org/infra/bindep/ for additional information.
+
+build-essential                      [platform:dpkg test]
+gcc                                  [platform:rpm test]
+libvirt-dev                          [platform:dpkg]
+python3-dev                          [platform:dpkg (requirements python3)]
+python3-devel                        [platform:rpm (requirements python3)]
+python-dev                           [platform:dpkg test]
+python3-all                          [platform:dpkg]
+python3-all-dev                      [platform:dpkg]
+python3                              [platform:rpm test]
+python3-devel                        [platform:rpm test]

utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/affinity.py

@@ -12,15 +12,15 @@
 """ Define pci_irq_affinity_provider class"""
 import os
 
-import pci_irq_affinity.utils as pci_utils
 from pci_irq_affinity.driver import AffinePciIrqDriver
-from pci_irq_affinity.nova_provider import novaClient
 from pci_irq_affinity.log import LOG
+from pci_irq_affinity import nova_provider
+from pci_irq_affinity import utils as pci_utils
 
 COMPUTE_IRQ = os.getenv("COMPUTE_IRQ", default="/proc/irq/")
 
 
-class pci_irq_affinity_provider:
+class PciIrqAffinityProvider:
 
     def __init__(self):
         self.affinePciIrqDriver = AffinePciIrqDriver()
@@ -90,9 +90,10 @@ class pci_irq_affinity_provider:
                    'task_state': None,
                    'deleted': False,
                    'all_tenants': True}
-        instances = novaClient.get_instances(filters)
+        nova_client = nova_provider.get_nova_client()
+        instances = nova_client.get_instances(filters)
         for inst in instances:
             self.affine_pci_dev_instance(inst, wait_for_irqs=False)
 
 
-pciIrqAffinity = pci_irq_affinity_provider()
+pci_irq_affinity = PciIrqAffinityProvider()

utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/agent.py

@@ -23,11 +23,12 @@ from oslo_service import periodic_task
 from oslo_service import service
 import oslo_messaging
 
-from pci_irq_affinity.affinity import pciIrqAffinity
+from pci_irq_affinity import affinity
 from pci_irq_affinity.config import CONF
 from pci_irq_affinity.log import LOG
-from pci_irq_affinity.nova_provider import novaClient
+from pci_irq_affinity import nova_provider
-import pci_irq_affinity.utils as pci_utils
+from pci_irq_affinity import utils as pci_utils
+
 
 stay_on = True
 
@@ -44,7 +45,8 @@ def process_signal_handler(signum, frame):
 
 def get_inst(instance_uuid, callback):
     # get instance info from nova
-    inst = novaClient.get_instance(instance_uuid)
+    nova_client = nova_provider.get_nova_client()
+    inst = nova_client.get_instance(instance_uuid)
     if inst is not None:
         LOG.debug("inst:%s" % inst)
         callback(inst)
@@ -52,12 +54,12 @@ def get_inst(instance_uuid, callback):
 
 def query_instance_callback(inst):
     LOG.debug("query inst:%s" % inst)
-    pciIrqAffinity.affine_pci_dev_instance(inst)
+    affinity.pci_irq_affinity.affine_pci_dev_instance(inst)
 
 
 @periodic_task.periodic_task(spacing=CONF.parameters.pci_affine_interval)
 def audit_affinity(self, context):
-    pciIrqAffinity.audit_pci_irq_affinity()
+    affinity.pci_irq_affinity.audit_pci_irq_affinity()
 
 
 def audit_work(srv, callback):
@@ -150,7 +152,7 @@ class InstanceOfflineNotificationEndpoint(BaseInstanceEndpoint):
         if instance_uuid:
             LOG.info("Instance offline: uuid=%s, instance_host=%s, event_type=%s" % (
                 instance_uuid, instance_host, event_type))
-            pciIrqAffinity.reset_irq_affinity(instance_uuid)
+            affinity.pci_irq_affinity.reset_irq_affinity(instance_uuid)
 
 
 def rpc_work(srv):
@@ -194,11 +196,12 @@ def process_main():
 
     LOG.info("Enter PCIInterruptAffinity Agent")
 
+    nova_client = nova_provider.get_nova_client()
     try:
         signal.signal(signal.SIGTSTP, process_signal_handler)
         openstack_enabled = CONF.openstack.openstack_enabled
         if openstack_enabled:
-            novaClient.open_libvirt_connect()
+            nova_client.open_libvirt_connect()
             audit_srv = audits_initialize()
             rabbit_client = start_rabbitmq_client()
 
@@ -216,7 +219,7 @@ def process_main():
     finally:
         LOG.error("process_main finalized!!!")
         if openstack_enabled:
-            novaClient.close_libvirt_connect()
+            nova_client.close_libvirt_connect()
             audit_srv.tg.stop()
             rabbit_client.stop()
 

utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/driver.py

@@ -14,10 +14,10 @@
 from oslo_service import loopingcall
 from oslo_concurrency import lockutils
 
-import pci_irq_affinity.utils as pci_utils
 from pci_irq_affinity.config import CONF
 from pci_irq_affinity.log import LOG
-from pci_irq_affinity.nova_provider import novaClient
+from pci_irq_affinity import nova_provider
+from pci_irq_affinity import utils as pci_utils
 
 synchronized = lockutils.synchronized_with_prefix('pci_irq_affinity-')
 
@@ -94,7 +94,8 @@ class AffinePciIrqDriver:
             _msi_irqs = set()
             # refresh instance info.
             if refresh_need:
-                _inst = novaClient.get_instance(inst.uuid)
+                nova_client = nova_provider.get_nova_client()
+                _inst = nova_client.get_instance(inst.uuid)
             if _inst is None:
                 return
 

utilities/pci-irq-affinity-agent/pci_irq_affinity/pci_irq_affinity/nova_provider.py

@@ -14,18 +14,18 @@ This class wraps novaclient access interface and expose get_instance() and
 get_instances() to other agent classes.
 """
 
+import keyring
 import os
 import socket
-import keyring
 
 from keystoneauth1 import loading
 from keystoneauth1 import session
 from novaclient import client
 
-from pci_irq_affinity.log import LOG
 from pci_irq_affinity.config import CONF
-from pci_irq_affinity import instance
 from pci_irq_affinity import guest
+from pci_irq_affinity import instance
+from pci_irq_affinity.log import LOG
 
 
 class NovaProvider:
@@ -142,7 +142,8 @@ class NovaProvider:
         return instances
 
 
-if CONF.openstack.openstack_enabled:
+def get_nova_client():
-    novaClient = NovaProvider()
+    if CONF.openstack.openstack_enabled:
-else:
+        return NovaProvider()
-    novaClient = None
+    else:
+        return None

utilities/pci-irq-affinity-agent/pci_irq_affinity/requirements.txt

@@ -0,0 +1,19 @@
+eventlet==0.33.0
+keyring==18.0.1
+keyrings.alt==3.2.0
+pycrypto==2.6.1
+keystoneauth1==4.0.1
+kombu==4.6.11
+pkgconfig==1.5.2
+libvirt-python;python_version>="3.0" # GPLv2
+libvirt-python<6.0.0;python_version<"3.0" # GPLv2
+oslo.concurrency==3.31.0
+oslo.config==7.0.0
+oslo.i18n==3.25.1
+oslo.log==3.45.2
+oslo.messaging==10.5.0
+oslo.service==1.41.1
+python-daemon==2.3.0
+python-novaclient==16.0.0
+retrying==1.3.3
+psutil==5.8.0

utilities/pci-irq-affinity-agent/pci_irq_affinity/test-requirements.txt

@@ -0,0 +1,8 @@
+# hacking pulls in flake8
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+bashate >= 0.2
+isort<5;python_version>="3.0"
+pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint<2.3.0;python_version>="3.0" # GPLv2
+pytest
+mock

utilities/pci-irq-affinity-agent/pci_irq_affinity/tests/test_nova_provider.py

@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2021 StarlingX.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# All Rights Reserved.
+#
+
+import mock
+import unittest
+
+from pci_irq_affinity.config import CONF
+from pci_irq_affinity import nova_provider
+
+
+class TestNovaProvider(unittest.TestCase):
+    AUTH_URL = "http://keystone.local"
+    USERNAME = "my-name"
+    CACERT = "cacert.pem"
+    KEYRING_SERVICE = "CGCS"
+    PASSWORD = "my-pass"
+
+    def setUp(self):
+        CONF.set_override("auth_url", self.AUTH_URL, group="openstack")
+        CONF.set_override("username", self.USERNAME, group="openstack")
+        CONF.set_override("cacert", self.CACERT, group="openstack")
+        CONF.set_override("keyring_service", self.KEYRING_SERVICE,
+                          group="openstack")
+
+    def tearDown(self):
+        CONF.clear_override("auth_url", group="openstack")
+        CONF.clear_override("username", group="openstack")
+        CONF.clear_override("cacert", group="openstack")
+        CONF.clear_override("keyring_service", group="openstack")
+        CONF.clear_override("password", group="openstack")
+
+    def test__get_keystone_creds(self):
+        CONF.set_override("password", "my-pass", group="openstack")
+
+        instance = nova_provider.NovaProvider()
+        creds = instance._get_keystone_creds()
+
+        expected = {
+            "auth_url": self.AUTH_URL,
+            "username": self.USERNAME,
+            "password": self.PASSWORD,
+            # "project_domain_name": None,
+            # "project_name": None,
+            # "user_domain_name": None
+        }
+        self.assertDictEqual(expected, creds)
+
+    @mock.patch.object(nova_provider, "keyring")
+    def test__get_keystone_creds_no_password(self, mock_keyring):
+        keyring_pass = "keyring-pass"
+        mock_keyring.get_password.return_value = keyring_pass
+        instance = nova_provider.NovaProvider()
+        mock_keyring.get_password.reset_mock()
+
+        creds = instance._get_keystone_creds()
+
+        expected = {
+            "auth_url": self.AUTH_URL,
+            "username": self.USERNAME,
+            "password": keyring_pass,
+            # "project_domain_name": None,
+            # "project_name": None,
+        }
+        self.assertDictEqual(expected, creds)
+        mock_keyring.get_password.assert_called_once_with(
+            self.KEYRING_SERVICE,
+            self.USERNAME,
+        )

utilities/pci-irq-affinity-agent/pci_irq_affinity/tox.ini

@@ -0,0 +1,82 @@
+[tox]
+envlist = py27,flake8
+minversion = 2.3
+skipsdist = True
+
+[testenv]
+sitepackages = False
+basepython = python2.7
+install_command = pip install -vvv  -U {opts} {packages}
+setenv =
+    VIRTUAL_ENV={envdir}
+    OS_STDOUT_CAPTURE=1
+    OS_STDERR_CAPTURE=1
+    OS_DEBUG=1
+    OS_LOG_CAPTURE=1
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+whitelist_externals =
+    bash
+commands =
+    pytest tests/
+
+[testenv:py36]
+basepython = python3.6
+
+[testenv:py39]
+basepython = python3.9
+
+[flake8]
+# Note: hacking pulls in flake8 2.5.5 which can not parse an ignore list spanning multiple lines
+# F errors are high priority to fix. W are warnings. E series are pep8, H series are hacking
+# F401 'FOO' imported but unused
+# F841 local variable 'FOO' is assigned to but never used
+# W291 trailing whitespace
+# W391 blank line at end of file
+# W503 line break before binary operator
+# W504 line break after binary operator
+# W605 invalid escape sequence '\s'
+# E128 continuation line under-indented for visual indent
+# E221 multiple spaces before operator
+# E226 missing whitespace around arithmetic operator
+# E227 missing whitespace around bitwise or shift operator
+# E241 multiple spaces after ','
+# E265 block comment should start with '# '
+# E302 expected 2 blank lines, found 1
+# E305 expected 2 blank lines after class or function definition, found 1
+# E501 line too long
+# E502 the backslash is redundant between brackets
+# E702 multiple statements on one line (semicolon)
+# E722 do not use bare 'except'
+# H101 is TODO
+# H104  File contains nothing but comments
+# H201  no 'except:' at least use 'except Exception:'
+# H238  old style class declaration, use new style (inherit from `object`)
+# H306  imports not in alphabetical order (sys, re)
+# H401  docstring should not start with a space
+# H405  multi line docstring summary not separated with an empty line
+ignore = F401,F841,W291,W391,W503,W504,W605,E128,E221,E226,E227,E241,E265,E302,E305,E501,E502,E702,E722,H101,H104,H201,H238,H306,H401,H405
+# max-line-length is not referenced if E501 is suppressed
+max-line-length=80
+
+[testenv:flake8]
+commands =
+  flake8
+
+[testenv:pylint]
+commands = pylint {posargs} pci_irq_affinity --rcfile=../../../pylint.rc
+
+[testenv:bandit]
+description = Dummy environment to allow bandit to be run in subdir tox
+
+[testenv:bindep]
+# Do not install any requirements. We want this to be fast and work even if
+# system dependencies are missing, since it's used to tell you what system
+# dependencies are missing! This also means that bindep must be installed
+# separately, outside of the requirements files, and develop mode disabled
+# explicitly to avoid unnecessarily installing the checked-out repo too (this
+# further relies on "tox.skipsdist = True" above).
+deps = bindep
+commands = bindep test
+usedevelop = False