Update sysinv tox for python3.9

sysinv will now use python3.9 for pylint, flake8, bandit

basepython is now set to python3.9

The zuul jobs for older versions are now updated or removed
The nodeset for python 3.9 jobs is set to debian-bullseye

pylint.rc changes include:
 - Upversion to latest pylint (2.16)
 - remove suppressions that no longer exist.
 - add suppressions for new checks added in recent pylint
 - update pylint.rc settings that no longer exist.
 - remove explicit enabling of pylint checkers from python2
   that are on by default in python3

Note: some of the newly suppressed pylint error codes will
be un-suppressed in later commits.

A subtle issue has also been fixed where a variable was
shadowing an imported method, and would have led to a
an exception when logging an error. The variable has been
renamed.

Example CLI service-parameter command for that code path:
 system service-parameter-modify kubernetes \
  kube_apiserver_volumes \
  default-audit-policy-file='{"hostPath":"x", "name":"y"}'

The CLI of the previous code would show the following
in its exception case without the fix
   'int' object is not callable

Test Plan:
  PASS: run tox in a debian build environment.
  PASS: build/install/unlock AIO-SX
  PASS: service-parameter CLI command still works

Story: 2010531
Task: 47279

Change-Id: I989ec6e75463be0a4dcd2c7727cd69584c2a20a2
Signed-off-by: Al Bailey <al.bailey@windriver.com>

.zuul.yaml

@@ -9,7 +9,6 @@
     check:
       jobs:
         - openstack-tox-linters
-        - sysinv-tox-py27
         - sysinv-tox-py39
         - sysinv-tox-flake8
         - sysinv-tox-pylint
@@ -29,7 +28,6 @@
     gate:
       jobs:
         - openstack-tox-linters
-        - sysinv-tox-py27
         - sysinv-tox-py39
         - sysinv-tox-flake8
         - sysinv-tox-pylint
@@ -50,22 +48,6 @@
       jobs:
         - stx-config-upload-git-mirror
 
-- job:
-    name: sysinv-tox-py27
-    parent: tox
-    description: |
-      Run py27 test for sysinv
-    nodeset: ubuntu-xenial
-    required-projects:
-      - starlingx/fault
-      - starlingx/utilities
-      - starlingx/update
-    files:
-      - sysinv/sysinv/*
-    vars:
-      tox_envlist: py27
-      tox_extra_args: -c sysinv/sysinv/sysinv/tox.ini
-
 - job:
     name: sysinv-tox-py39
     parent: tox-py39
@@ -88,7 +70,7 @@
     parent: tox
     description: |
       Run flake8 test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
     files:
       - sysinv/sysinv/*
     vars:
@@ -100,7 +82,7 @@
     parent: tox
     description: |
       Run pylint test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
     required-projects:
       - starlingx/fault
       - starlingx/utilities
@@ -116,7 +98,7 @@
     parent: tox
     description: |
       Run bandit test for sysinv
-    nodeset: ubuntu-bionic
+    nodeset: debian-bullseye
     files:
       - sysinv/sysinv/*
     vars:

sysinv/sysinv/sysinv/pylint.rc

@@ -31,97 +31,13 @@ extension-pkg-whitelist=lxml.etree,greenlet
 
 
 [MESSAGES CONTROL]
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time.
-#
-# Python3 checker:
-#
-# E1601: print-statement
-# E1602: parameter-unpacking
-# E1603: unpacking-in-except
-# E1604: old-raise-syntax
-# E1605: backtick
-# E1606: long-suffix
-# E1607: old-ne-operator
-# E1608: old-octal-literal
-# E1609: import-star-module-level
-# E1610: non-ascii-bytes-literal
-# E1611: invalid-unicode-literal
-# W1601: apply-builtin
-# W1602: basestring-builtin
-# W1603: buffer-builtin
-# W1604: cmp-builtin
-# W1605: coerce-builtin
-# W1606: execfile-builtin
-# W1607: file-builtin
-# W1608: long-builtin
-# W1609: raw_input-builtin
-# W1610: reduce-builtin
-# W1611: standarderror-builtin
-# W1612: unicode-builtin
-# W1613: xrange-builtin
-# W1614: coerce-method
-# W1615: delslice-method
-# W1616: getslice-method
-# W1617: setslice-method
-# W1618: no-absolute-import
-# W1619: old-division
-# W1620: dict-iter-method
-# W1621: dict-view-method
-# W1622: next-method-called
-# W1623: metaclass-assignment
-# W1624: indexing-exception
-# W1625: raising-string
-# W1626: reload-builtin
-# W1627: oct-method
-# W1628: hex-method
-# W1629: nonzero-method
-# W1630: cmp-method
-# W1632: input-builtin
-# W1633: round-builtin
-# W1634: intern-builtin
-# W1635: unichr-builtin
-# W1636: map-builtin-not-iterating
-# W1637: zip-builtin-not-iterating
-# W1638: range-builtin-not-iterating
-# W1639: filter-builtin-not-iterating
-# W1640: using-cmp-argument
-# W1641: eq-without-hash
-# W1642: div-method
-# W1643: idiv-method
-# W1644: rdiv-method
-# W1645: exception-message-attribute
-# W1646: invalid-str-codec
-# W1647: sys-max-int
-# W1648: bad-python3-import
-# W1649: deprecated-string-function
-# W1650: deprecated-str-translate-call
-# W1651: deprecated-itertools-function
-# W1652: deprecated-types-field
-# W1653: next-method-defined
-# W1654: dict-items-not-iterating
-# W1655: dict-keys-not-iterating
-# W1656: dict-values-not-iterating
-# W1657: deprecated-operator-function
-# W1658: deprecated-urllib-function
-# W1659: xreadlines-attribute
-# W1660: deprecated-sys-function
-# W1661: exception-escape
-# W1662: comprehension-escape
-enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
-       W1651,W1649,W1657,W1660,W1658,W1659,W1623,W1622,W1620,W1621,W1645,W1641,
-       W1624,W1648,W1625,W1611,W1662,W1661,W1650,W1640,W1630,W1614,W1615,W1642,
-       W1616,W1628,W1643,W1629,W1627,W1644,W1617,W1601,W1602,W1603,W1604,W1605,
-       W1654,W1655,W1656,W1619,W1606,W1607,W1639,W1618,W1632,W1634,W1608,W1636,
-       W1653,W1646,W1638,W1609,W1610,W1626,W1633,W1647,W1635,W1612,W1613,W1637
-
 # Disable the message, report, category or checker with the given id(s). You
 # can either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once).
 # See "Messages Control" section of
 # https://pylint.readthedocs.io/en/latest/user_guide
+
 # We are disabling (C)onvention
 # We are disabling (R)efactor
 # We are selectively disabling (W)arning
@@ -133,8 +49,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0106: expression-not-assigned
 # W0107: unnecessary-pass
 # W0108: unnecessary-lambda
-# W0110: deprecated-lambda
+# W0109: duplicate-key                  !!!
 # W0123: eval-used
+# W0125: using-constant-test            !!!
+# W0133: pointless-exception-statement  !!!
+# W0143: comparison-with-callable       !!!
 # W0150: lost-exception
 # W0201: attribute-defined-outside-init
 # W0211: bad-staticmethod-argument
@@ -143,10 +62,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0223: abstract-method
 # W0231: super-init-not-called
 # W0235: useless-super-delegation
+# W0237: arguments-renamed  !!!
 # W0311: bad-indentation
 # W0402: deprecated-module
-# W0403: relative-import
 # W0404: reimported
+# W0602: global-variable-not-assigned  !!!
 # W0603: global-statement
 # W0612: unused-variable
 # W0613: unused-argument
@@ -154,34 +74,46 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652,
 # W0622: redefined-builtin
 # W0631: undefined-loop-variable
 # W0632: unbalanced-tuple-unpacking
-# W0703: broad-except
+# W0703: broad-except (pylint 2.16 renamed to broad-except-caught)
 # W0706: try-except-raise
+# W0707: raise-missing-from
+# W0719: broad-exception-raised
 # W1113: keyword-arg-before-vararg
 # W1201: logging-not-lazy
+# W1310: format-string-without-interpolation  !!!
 # W1401: anomalous-backslash-in-string
+# W1406: redundant-u-string-prefix
 # W1501: subprocess-popen-preexec-fn
 # W1505: deprecated-method
 # W1509: subprocess-popen-preexec-fn
+# W1514: unspecified-encoding
 # W1618: no-absolute-import
+# W3101: missing-timeout
 # All these errors should be fixed:
+# E0601: used-before-assignment  !!!
+# E0605: invalid-all-format      !!!
 # E1101: no-member
 # E1111: assignment-from-no-return
-disable=C, R, fixme, W0101, W0105, W0106, W0107, W0108, W0110, W0123, W0150,
+# E1120: no-value-for-parameter  !!!
-        W0201, W0211, W0212, W0221, W0223, W0231, W0235, W0311, W0402, W0403,
+# E1121: too-many-function-args  !!!
-        W0404, W0603, W0612, W0613, W0621, W0622, W0631, W0632, W0701, W0703,
+# E1123: unexpected-keyword-arg  !!!
-        W0706, W1113, W1201, W1401, W1505, W1509, W1618,
+# E1136: unsubscriptable-object  !!!
-        E1101, E1111
+disable=C, R, fixme,
+        W0101, W0105, W0106, W0107, W0108, W0109, W0123,
+        W0125, W0133, W0143, W0150, W0201, W0211, W0212,
+        W0221, W0223, W0231, W0235, W0237, W0311, W0402,
+        W0404, W0602, W0603, W0612, W0613, W0621, W0622,
+        W0631, W0632, W0701, W0703, W0706, W0707, W0719,
+        W1113, W1201, W1310, W1401, W1406, W1505, W1509,
+        W1514, W1618, W3101,
+        E0601, E0605, E1101, E1111, E1120, E1121, E1123,
+        E1136
 
 [REPORTS]
 # Set the output format. Available formats are text, parseable, colorized, msvs
 # (visual studio) and html
 output-format=text
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=yes
 
@@ -240,9 +172,6 @@ generated-members=REQUEST,acl_users,aq_parent
 
 
 [BASIC]
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input
-
 # Regular expression which should only match correct module names
 module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
@@ -332,7 +261,7 @@ max-locals=15
 max-returns=6
 
 # Maximum number of branch for function / method body
-max-branchs=12
+max-branches=12
 
 # Maximum number of statements in function / method body
 max-statements=50
@@ -359,6 +288,6 @@ valid-classmethod-first-arg=cls
 
 
 [EXCEPTIONS]
-# Exceptions that will emit a warning when being caught. Defaults to
+# Exceptions that will emit a warning when caught.
-# "Exception"
+overgeneral-exceptions=builtins.BaseException,builtins.Exception
-overgeneral-exceptions=Exception
+

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py

@@ -61,7 +61,7 @@ def delete_k8s_configmap(parameter, kube_operator):
     ------
     wsme.exc.ClientSideError
     """
-    _volume, _ = service_parameter.parse_volume_string_to_dict(parameter)
+    _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter)
 
     # only delete configmaps for 'File' type since
     # 'DirectoryorCreate' type has no associated configmaps
@@ -160,7 +160,7 @@ def update_k8s_configmap(parameter, kube_operator):
     ------
     wsme.exc.ClientSideError
     """
-    _volume, _ = service_parameter.parse_volume_string_to_dict(parameter)
+    _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter)
 
     # only update configmaps for 'File' type since
     # 'DirectoryorCreate' type has no associated configmaps

sysinv/sysinv/sysinv/test-requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 hacking>=1.1.0,<=2.0.0 # Apache-2.0
-astroid <= 2.2.5
+astroid
 bandit<1.7.2;python_version>="3.0"
 coverage>=3.6
 fixtures>=3.0.0 # Apache-2.0/BSD
@@ -16,6 +16,5 @@ stestr>=1.0.0 # Apache-2.0
 testrepository>=0.0.18
 testtools!=1.2.0,>=0.9.36
 isort<5;python_version>="3.0"
-pylint<2.1.0;python_version<"3.0" # GPLv2
+pylint
-pylint<2.4.0;python_version>="3.0" # GPLv2
 pycryptodomex

sysinv/sysinv/sysinv/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py27,py36,py39,pylint
+envlist = flake8,py39,pylint
 minversion = 1.6
 skipsdist = True
 
@@ -10,7 +10,7 @@ stxdir = {toxinidir}/../../../..
 distshare={toxworkdir}/.tox/distshare
 
 [testenv]
-basepython = python3
+basepython = python3.9
 usedevelop = True
 
 # tox is silly... these need to be separated by a newline....
@@ -18,7 +18,7 @@ allowlist_externals = bash
                       find
 
 install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
     {opts} {packages}
 
 # Note the hash seed is set to 0 until can be tested with a
@@ -42,7 +42,7 @@ deps = -r{toxinidir}/requirements.txt
        -e{[tox]stxdir}/fault/fm-api/source
        -e{[tox]stxdir}/fault/python-fmclient/fmclient
        -e{[tox]stxdir}/config/controllerconfig/controllerconfig
-       -e{[tox]stxdir}/update/cgcs-patch/cgcs-patch
+       -e{[tox]stxdir}/update/sw-patch/cgcs-patch
        -e{[tox]stxdir}/utilities/utilities/platform-util/platform-util
        -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient
        -e{[tox]stxdir}/config/sysinv/cgts-client/cgts-client
@@ -95,42 +95,19 @@ exclude = build,dist,tools,.eggs
 max-line-length=120
 
 [testenv:flake8]
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands =
   flake8 {posargs} . \
     scripts/manage-partitions \
     scripts/query_pci_id
 
-[testenv:py27]
-basepython = python2.7
-passenv = CURL_CA_BUNDLE
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
-[testenv:py36]
-basepython = python3.6
-commands =
-  {[testenv]commands}
-  stestr run {posargs}
-  stestr slowest
-
 [testenv:py39]
-basepython = python3.9
-
-install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
-    {opts} {packages}
 commands =
   stestr run {posargs}
   stestr slowest
 
-
 [testenv:pep8]
 # testenv:flake8 clone
-basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = {[testenv:flake8]commands}
 
@@ -170,20 +147,18 @@ skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B4
 exclude = tests
 
 [testenv:bandit]
-basepython = python3.6
 deps = -r{toxinidir}/test-requirements.txt
 commands = bandit --ini tox.ini -n 5 -r sysinv
 
 [testenv:pylint]
-basepython = python3.6
+install_command = pip install -v -v -v \
+    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
+    {opts} {packages}
 commands =
      pylint {posargs} sysinv --rcfile=./pylint.rc
 
 [testenv:cover]
-basepython = python3.9
+# not sure is passenv is still needed
-install_command = pip install -v -v -v \
-    -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \
-    {opts} {packages}
 passenv = CURL_CA_BUNDLE
 deps = {[testenv]deps}
 setenv = {[testenv]setenv}