Fix f/centos8 zuul gate

This commit fixes various gate issues on f/centos8, this is being done
so we can merge the master branch into the f/centos8 due to an issue
with merging branches and zuul. The fixes have been cherry-picked from
the master branch:

- "Not require recreate of tox env when running tox" - align the version
of hacking and pylint as the same of other starlingx repos.
(eafee58e62)
- "Specify the nodeset for zuul jobs" - py2.7 jobs need to specify
xenial, changed py3.7 to py3.6 and specify bionic.
(5ff24cf13f)

Testing was done by running the pylint, py27, py36, and pep8 jobs
locally.

Change-Id: I9a16449c7194bea3c0dede213f7209bb7558f82c

.zuul.yaml

@@ -9,7 +9,7 @@
         - openstack-tox-linters
         - nfv-tox-pep8
         - nfv-tox-py27
-        - nfv-tox-py37
+        - nfv-tox-py36
         - nfv-tox-pylint
         - nova-api-proxy-tox-pep8
         - nova-api-proxy-tox-pylint
@@ -18,7 +18,7 @@
         - openstack-tox-linters
         - nfv-tox-pep8
         - nfv-tox-py27
-        - nfv-tox-py37
+        - nfv-tox-py36
         - nfv-tox-pylint
         - nova-api-proxy-tox-pep8
         - nova-api-proxy-tox-pylint
@@ -29,6 +29,7 @@
 - job:
     name: nfv-tox-py27
     parent: tox
+    nodeset: ubuntu-xenial
     description: Run py27 for nfv
     required-projects:
       - starlingx/fault
@@ -37,19 +38,21 @@
       tox_extra_args: -c nfv/tox.ini
 
 - job:
-    name: nfv-tox-py37
+    name: nfv-tox-py36
     parent: tox
-    description: Run py37 for nfv
+    nodeset: ubuntu-bionic
+    description: Run py36 for nfv
     required-projects:
       - starlingx/fault
     vars:
-      tox_envlist: py37
+      tox_envlist: py36
       tox_extra_args: -c nfv/tox.ini
-      bindep_profile: test py37
-      python_version: 3.7
+      bindep_profile: test py36
+      python_version: 3.6
 
 - job:
     name: nfv-tox-pep8
+    nodeset: ubuntu-xenial
     parent: tox
     description: Run pep8 for nfv
     required-projects:
@@ -60,6 +63,7 @@
 
 - job:
     name: nfv-tox-pylint
+    nodeset: ubuntu-xenial
     parent: tox
     description: Run pylint for nfv
     required-projects:
@@ -80,6 +84,7 @@
     name: nova-api-proxy-tox-pylint
     parent: tox
     description: Run pylint for nova-api-proxy
+    nodeset: ubuntu-xenial
     vars:
       tox_envlist: pylint
       tox_extra_args: -c nova-api-proxy/tox.ini

nfv/nfv-debug-tools/histogram_analysis/plotter.py

@@ -348,7 +348,7 @@ def setFilename(graphName):
         os.makedirs("Graphs/")
     os.chdir(os.path.join(dir, 'Graphs/'))
     if not graphName:
-            graphName = time.strftime("%m-%d-%Y")
+        graphName = time.strftime("%m-%d-%Y")
     if os.path.exists(str(graphName + ".html")):
         n = 1
         while(not validName):

nfv/nfv-tests/nfv_unit_tests/tests/test_database_upgrades.py

@@ -4,7 +4,9 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 import os
+import shutil
 import subprocess
+import tempfile
 
 from nfv_vim import database
 from nfv_vim import tables
@@ -14,24 +16,34 @@ from . import testcase  # noqa: H304
 
 class TestNFVDatabaseUpgrade(testcase.NFVTestCase):
 
+    def setUp(self):
+        super(TestNFVDatabaseUpgrade, self).setUp()
+        root_dir = os.environ['VIRTUAL_ENV']
+        # create a directory to hold the DB, randomly named, under the tox env
+        self.db_dir = tempfile.mkdtemp(dir=root_dir)
+
+    def tearDown(self):
+        super(TestNFVDatabaseUpgrade, self).tearDown()
+        shutil.rmtree(self.db_dir)
+
     def test_nfv_vim_database_upgrade_from_19_12(self):
         """
         Test VIM database upgrades from stx 19_12
         """
         root_dir = os.environ['VIRTUAL_ENV']
-
+        # stage some old data
         devnull = open(os.devnull, 'w')
         try:
             vim_cmd = ("nfv-vim-manage db-load-data -d %s "
-                       "-f %s/nfv_vim_db_stx_19.12" % (root_dir, root_dir))
+                       "-f %s/nfv_vim_db_stx_19.12" % (self.db_dir, root_dir))
 
             subprocess.check_call([vim_cmd], shell=True, stderr=devnull)
-
         except subprocess.CalledProcessError:
             raise
 
+        # migrate the old data
         config = dict()
-        config['database_dir'] = root_dir
+        config['database_dir'] = self.db_dir
         database.database_initialize(config)
         database.database_migrate_data()
         tables.tables_initialize()

nfv/nfv-vim/nfv_vim/api/openstack/_objects.py

@@ -60,6 +60,7 @@ class OpenStackServices(Constants):
     HEAT = Constant('heat')
     FM = Constant('fm')
 
+
 # OpenStack Services Constant
 OPENSTACK_SERVICE = OpenStackServices()
 

nfv/nfv-vim/nfv_vim/network_rebalance/_dhcp_rebalance.py

@@ -27,6 +27,7 @@ class AgentType(Constants):
     L3 = Constant('L3 agent')
     DHCP = Constant('DHCP agent')
 
+
 AGENT_TYPE = AgentType()
 
 
@@ -43,6 +44,7 @@ class DHCPRebalanceState(Constants):
     HOLD_OFF = Constant('HOLD_OFF')
     DONE = Constant('DONE')
 
+
 DHCP_REBALANCE_STATE = DHCPRebalanceState()
 
 

nfv/nfv-vim/nfv_vim/network_rebalance/_network_rebalance.py

@@ -27,6 +27,7 @@ class AgentType(Constants):
     L3 = Constant('L3 agent')
     DHCP = Constant('DHCP agent')
 
+
 AGENT_TYPE = AgentType()
 
 
@@ -46,6 +47,7 @@ class L3RebalanceState(Constants):
     HOLD_OFF = Constant('HOLD_OFF')
     DONE = Constant('DONE')
 
+
 L3_REBALANCE_STATE = L3RebalanceState()
 
 

nfv/nfv-vim/nfv_vim/strategy/_strategy_defs.py

@@ -50,4 +50,5 @@ class FirmwareUpdateLabels(object):
     DEVICE_IMAGE_UPDATE_FAILED = Constant('failed')
     DEVICE_IMAGE_UPDATE_IN_PROGRESS_ABORTED = Constant('in-progress-aborted')
 
+
 FW_UPDATE_LABEL = FirmwareUpdateLabels()

nfv/test-requirements.txt

@@ -1 +1,13 @@
-hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid<2.0;python_version<"3.0" # GPLv2
+astroid<= 2.2.5;python_version>="3.0" # GPLv2
+bandit<1.6.0;python_version>="3.0"
+coverage>=3.6
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0.0 # BSD
+pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint<2.4.0;python_version>="3.0" # GPLv2
+testtools>=2.2.0 # MIT

nfv/tox.ini

@@ -15,12 +15,14 @@ skipsdist = True
 stxdir = {toxinidir}/../..
 
 [testenv]
-recreate = True
 install_command = pip install \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
     {opts} {packages}
 whitelist_externals = cp
 
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [nfv]
 nfv_base_dir = .
 nfv_client_dir = ./nfv-client
@@ -57,30 +59,35 @@ deps = {[nfv]nfv_client_dir}
 verbosity=2
 
 [flake8]
-# Temporarily ignoring these warnings
-# E121 continuation line under-indented for hanging indent
-# E122 continuation line missing indentation or outdented
-# E123 closing bracket does not match indentation of opening bracket
-# E124 closing bracket does not match visual indentation
-# E126 continuation line over-indented for hanging indent
-# E127 continuation line over-indented for visual indent
-# E128 continuation line under-indented for visual indent
-# E129 visually indented line with same indent as next logical line
-# E501 line too long
 # - hacking codes -
 # H104: File contains nothing but comments
 # H404: multi line docstring should start without a leading new line
 # H405: multi line docstring summary not separated with an empty line
 # H501: Do not use self.__dict__ for string formatting
-# - flake8 codes -
-# F823 local variable referenced before assignment
+# warnings
+# W503 line break before binary operator
+# W504  line break after binary operator
+# W605 invalid escape sequence
+# errors
+# E121 continuation line under-indented for hanging indent
+# E122 continuation line missing indentation or outdented
+# E124 closing bracket does not match visual indentation
+# E126 continuation line over-indented for hanging indent
+# E127 continuation line over-indented for visual indent
+# E128 continuation line under-indented for visual indent
+# E129 visually indented line with same indent as next logical line
+# flake8
 # F841 local variable 'e' is assigned to but never used
-ignore = E121,E122,E123,E124,E126,E127,E128,E129,E501,H104,H404,H405,H501,F823,F841
+ignore = H104,H404,H405,H501,
+         W503,W504,W605,
+         E121,E122,E124,E126,E127,E128,E129,
+         F841
 # H106 Don’t put vim configuration in source files (off by default).
 # H203 Use assertIs(Not)None to check for None (off by default).
 # TODO: enable: H904 Delay string interpolations at logging calls (off by default).
 enable-extensions = H106,H203
-max-line-length = 84
+# rather than suppress E501 line too long, set max line length and reduce later
+max-line-length = 185
 
 [testenv:pep8]
 basepython = python3
@@ -94,12 +101,9 @@ commands =
 [testenv:pylint]
 basepython = python2.7
 deps = {[nfv]deps}
+       -r{toxinidir}/test-requirements.txt
        {[nfv]nfv_test_dir}/nfv_scenario_tests
-       fixtures
        jinja2
-       mock
-       testtools
-       pylint
 commands = pylint {[nfv]nfv_client_src_dir}  \
                   {[nfv]nfv_common_src_dir}  \
                   {[nfv]nfv_plugins_src_dir} \
@@ -120,8 +124,8 @@ exclude = ./nfv-docs,./nfv-tests,./nfv-debug-tools,unit_test
 
 [testenv:bandit]
 deps = {[nfv]deps}
-       bandit
-basepython = python2.7
+       -r{toxinidir}/test-requirements.txt
+basepython = python3
 commands = bandit --ini tox.ini -f txt -r {[nfv]nfv_base_dir}
 
 [testenv:py27]
@@ -137,8 +141,8 @@ commands = cp -v {[nfv]nfv_base_dir}/nfv-tests/nfv_unit_tests/test_data/nfv_vim_
            stestr --test-path={[nfv]nfv_base_dir}/nfv-tests/nfv_unit_tests/tests run '{posargs}'
            stestr slowest
 
-[testenv:py37]
-basepython = python3.7
+[testenv:py36]
+basepython = python3.6
 deps = {[nfv]deps}
        coverage
        fixtures

nova-api-proxy/test-requirements.txt

@@ -0,0 +1,12 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+hacking>=1.1.0,<=2.0.0 # Apache-2.0
+astroid<2.0;python_version<"3.0" # GPLv2
+astroid<= 2.2.5;python_version>="3.0" # GPLv2
+bandit<1.6.0;python_version>="3.0"
+coverage>=3.6
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0.0 # BSD
+pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint<2.4.0;python_version>="3.0" # GPLv2

nova-api-proxy/tox.ini

@@ -5,6 +5,7 @@ skipsdist = True
 
 [testenv]
 basepython = python3
+sitepackages = False
 install_command = pip install \
     -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
     {opts} {packages}
@@ -16,6 +17,8 @@ setenv = VIRTUAL_ENV={envdir}
 commands = find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -not -path '*/__pycache__/*' -name '*.py[c|o]' -delete
 whitelist_externals = find
 
+deps=-r{toxinidir}/test-requirements.txt
+
 [testenv:venv]
 commands = {posargs}
 
@@ -36,15 +39,13 @@ max-line-length=84
 [testenv:pep8]
 usedevelop = False
 skip_install = True
-deps =
-    hacking
-    flake8-bugbear
 commands =
     flake8
 
 [testenv:pylint]
 basepython = python2.7
-deps =
+deps=
+    -r{toxinidir}/test-requirements.txt
     eventlet
     oslo.config
     oslo.log
@@ -52,6 +53,5 @@ deps =
     PasteDeploy
     routes
     webob
-    pylint
 commands = pylint nova-api-proxy/nova_api_proxy --rcfile=pylint.rc
 

test-requirements.txt

@@ -1,6 +1,4 @@
 bashate < 1.0.0
 PyYAML >= 3.1.0
-yamllint >= 0.5.2
-# The following are for py27 and py35
-mock  # BSD
-nose  # LGPL
+yamllint<1.26.1;python_version>="3.0" # GPLv2
+

tox.ini

@@ -27,7 +27,7 @@ basepython = python3
 description = Dummy environment to allow pep8 to be run in subdir tox
 
 [testenv:pylint]
-basepython = python3
+basepython = python2.7
 description = Dummy environment to allow pylint to be run in subdir tox
 
 [testenv:linters]