From 4927790c0da5f18009f89afb0e2e3552b7667778 Mon Sep 17 00:00:00 2001 From: Al Bailey Date: Thu, 21 Jul 2022 15:11:52 +0000 Subject: [PATCH] Update sysinv tox for python3.9 sysinv will now use python3.9 for pylint, flake8, bandit basepython is now set to python3.9 The zuul jobs for older versions are now updated or removed The nodeset for python 3.9 jobs is set to debian-bullseye pylint.rc changes include: - Upversion to latest pylint (2.16) - remove suppressions that no longer exist. - add suppressions for new checks added in recent pylint - update pylint.rc settings that no longer exist. - remove explicit enabling of pylint checkers from python2 that are on by default in python3 Note: some of the newly suppressed pylint error codes will be un-suppressed in later commits. A subtle issue has also been fixed where a variable was shadowing an imported method, and would have led to a an exception when logging an error. The variable has been renamed. Example CLI service-parameter command for that code path: system service-parameter-modify kubernetes \ kube_apiserver_volumes \ default-audit-policy-file='{"hostPath":"x", "name":"y"}' The CLI of the previous code would show the following in its exception case without the fix 'int' object is not callable Test Plan: PASS: run tox in a debian build environment. PASS: build/install/unlock AIO-SX PASS: service-parameter CLI command still works Story: 2010531 Task: 47279 Change-Id: I989ec6e75463be0a4dcd2c7727cd69584c2a20a2 Signed-off-by: Al Bailey --- .zuul.yaml | 24 +-- sysinv/sysinv/sysinv/pylint.rc | 139 +++++------------- .../api/controllers/v1/service_parameter.py | 4 +- sysinv/sysinv/sysinv/test-requirements.txt | 5 +- sysinv/sysinv/sysinv/tox.ini | 41 +----- 5 files changed, 49 insertions(+), 164 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 817327b451..27cde3f998 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -9,7 +9,6 @@ check: jobs: - openstack-tox-linters - - sysinv-tox-py27 - sysinv-tox-py39 - sysinv-tox-flake8 - sysinv-tox-pylint @@ -29,7 +28,6 @@ gate: jobs: - openstack-tox-linters - - sysinv-tox-py27 - sysinv-tox-py39 - sysinv-tox-flake8 - sysinv-tox-pylint @@ -50,22 +48,6 @@ jobs: - stx-config-upload-git-mirror -- job: - name: sysinv-tox-py27 - parent: tox - description: | - Run py27 test for sysinv - nodeset: ubuntu-xenial - required-projects: - - starlingx/fault - - starlingx/utilities - - starlingx/update - files: - - sysinv/sysinv/* - vars: - tox_envlist: py27 - tox_extra_args: -c sysinv/sysinv/sysinv/tox.ini - - job: name: sysinv-tox-py39 parent: tox-py39 @@ -88,7 +70,7 @@ parent: tox description: | Run flake8 test for sysinv - nodeset: ubuntu-bionic + nodeset: debian-bullseye files: - sysinv/sysinv/* vars: @@ -100,7 +82,7 @@ parent: tox description: | Run pylint test for sysinv - nodeset: ubuntu-bionic + nodeset: debian-bullseye required-projects: - starlingx/fault - starlingx/utilities @@ -116,7 +98,7 @@ parent: tox description: | Run bandit test for sysinv - nodeset: ubuntu-bionic + nodeset: debian-bullseye files: - sysinv/sysinv/* vars: diff --git a/sysinv/sysinv/sysinv/pylint.rc b/sysinv/sysinv/sysinv/pylint.rc index 444e33cc30..1fd6bcf480 100755 --- a/sysinv/sysinv/sysinv/pylint.rc +++ b/sysinv/sysinv/sysinv/pylint.rc @@ -31,97 +31,13 @@ extension-pkg-whitelist=lxml.etree,greenlet [MESSAGES CONTROL] -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. -# -# Python3 checker: -# -# E1601: print-statement -# E1602: parameter-unpacking -# E1603: unpacking-in-except -# E1604: old-raise-syntax -# E1605: backtick -# E1606: long-suffix -# E1607: old-ne-operator -# E1608: old-octal-literal -# E1609: import-star-module-level -# E1610: non-ascii-bytes-literal -# E1611: invalid-unicode-literal -# W1601: apply-builtin -# W1602: basestring-builtin -# W1603: buffer-builtin -# W1604: cmp-builtin -# W1605: coerce-builtin -# W1606: execfile-builtin -# W1607: file-builtin -# W1608: long-builtin -# W1609: raw_input-builtin -# W1610: reduce-builtin -# W1611: standarderror-builtin -# W1612: unicode-builtin -# W1613: xrange-builtin -# W1614: coerce-method -# W1615: delslice-method -# W1616: getslice-method -# W1617: setslice-method -# W1618: no-absolute-import -# W1619: old-division -# W1620: dict-iter-method -# W1621: dict-view-method -# W1622: next-method-called -# W1623: metaclass-assignment -# W1624: indexing-exception -# W1625: raising-string -# W1626: reload-builtin -# W1627: oct-method -# W1628: hex-method -# W1629: nonzero-method -# W1630: cmp-method -# W1632: input-builtin -# W1633: round-builtin -# W1634: intern-builtin -# W1635: unichr-builtin -# W1636: map-builtin-not-iterating -# W1637: zip-builtin-not-iterating -# W1638: range-builtin-not-iterating -# W1639: filter-builtin-not-iterating -# W1640: using-cmp-argument -# W1641: eq-without-hash -# W1642: div-method -# W1643: idiv-method -# W1644: rdiv-method -# W1645: exception-message-attribute -# W1646: invalid-str-codec -# W1647: sys-max-int -# W1648: bad-python3-import -# W1649: deprecated-string-function -# W1650: deprecated-str-translate-call -# W1651: deprecated-itertools-function -# W1652: deprecated-types-field -# W1653: next-method-defined -# W1654: dict-items-not-iterating -# W1655: dict-keys-not-iterating -# W1656: dict-values-not-iterating -# W1657: deprecated-operator-function -# W1658: deprecated-urllib-function -# W1659: xreadlines-attribute -# W1660: deprecated-sys-function -# W1661: exception-escape -# W1662: comprehension-escape -enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652, - W1651,W1649,W1657,W1660,W1658,W1659,W1623,W1622,W1620,W1621,W1645,W1641, - W1624,W1648,W1625,W1611,W1662,W1661,W1650,W1640,W1630,W1614,W1615,W1642, - W1616,W1628,W1643,W1629,W1627,W1644,W1617,W1601,W1602,W1603,W1604,W1605, - W1654,W1655,W1656,W1619,W1606,W1607,W1639,W1618,W1632,W1634,W1608,W1636, - W1653,W1646,W1638,W1609,W1610,W1626,W1633,W1647,W1635,W1612,W1613,W1637 - # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). # See "Messages Control" section of # https://pylint.readthedocs.io/en/latest/user_guide + # We are disabling (C)onvention # We are disabling (R)efactor # We are selectively disabling (W)arning @@ -133,8 +49,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652, # W0106: expression-not-assigned # W0107: unnecessary-pass # W0108: unnecessary-lambda -# W0110: deprecated-lambda +# W0109: duplicate-key !!! # W0123: eval-used +# W0125: using-constant-test !!! +# W0133: pointless-exception-statement !!! +# W0143: comparison-with-callable !!! # W0150: lost-exception # W0201: attribute-defined-outside-init # W0211: bad-staticmethod-argument @@ -143,10 +62,11 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652, # W0223: abstract-method # W0231: super-init-not-called # W0235: useless-super-delegation +# W0237: arguments-renamed !!! # W0311: bad-indentation # W0402: deprecated-module -# W0403: relative-import # W0404: reimported +# W0602: global-variable-not-assigned !!! # W0603: global-statement # W0612: unused-variable # W0613: unused-argument @@ -154,34 +74,46 @@ enable=E1603,E1609,E1610,E1602,E1606,E1608,E1607,E1605,E1604,E1601,E1611,W1652, # W0622: redefined-builtin # W0631: undefined-loop-variable # W0632: unbalanced-tuple-unpacking -# W0703: broad-except +# W0703: broad-except (pylint 2.16 renamed to broad-except-caught) # W0706: try-except-raise +# W0707: raise-missing-from +# W0719: broad-exception-raised # W1113: keyword-arg-before-vararg # W1201: logging-not-lazy +# W1310: format-string-without-interpolation !!! # W1401: anomalous-backslash-in-string +# W1406: redundant-u-string-prefix # W1501: subprocess-popen-preexec-fn # W1505: deprecated-method # W1509: subprocess-popen-preexec-fn +# W1514: unspecified-encoding # W1618: no-absolute-import +# W3101: missing-timeout # All these errors should be fixed: +# E0601: used-before-assignment !!! +# E0605: invalid-all-format !!! # E1101: no-member # E1111: assignment-from-no-return -disable=C, R, fixme, W0101, W0105, W0106, W0107, W0108, W0110, W0123, W0150, - W0201, W0211, W0212, W0221, W0223, W0231, W0235, W0311, W0402, W0403, - W0404, W0603, W0612, W0613, W0621, W0622, W0631, W0632, W0701, W0703, - W0706, W1113, W1201, W1401, W1505, W1509, W1618, - E1101, E1111 +# E1120: no-value-for-parameter !!! +# E1121: too-many-function-args !!! +# E1123: unexpected-keyword-arg !!! +# E1136: unsubscriptable-object !!! +disable=C, R, fixme, + W0101, W0105, W0106, W0107, W0108, W0109, W0123, + W0125, W0133, W0143, W0150, W0201, W0211, W0212, + W0221, W0223, W0231, W0235, W0237, W0311, W0402, + W0404, W0602, W0603, W0612, W0613, W0621, W0622, + W0631, W0632, W0701, W0703, W0706, W0707, W0719, + W1113, W1201, W1310, W1401, W1406, W1505, W1509, + W1514, W1618, W3101, + E0601, E0605, E1101, E1111, E1120, E1121, E1123, + E1136 [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html output-format=text -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - # Tells whether to display a full report or only the messages reports=yes @@ -240,9 +172,6 @@ generated-members=REQUEST,acl_users,aq_parent [BASIC] -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input - # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ @@ -332,7 +261,7 @@ max-locals=15 max-returns=6 # Maximum number of branch for function / method body -max-branchs=12 +max-branches=12 # Maximum number of statements in function / method body max-statements=50 @@ -359,6 +288,6 @@ valid-classmethod-first-arg=cls [EXCEPTIONS] -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py index 807f494b03..9baa917e65 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py @@ -61,7 +61,7 @@ def delete_k8s_configmap(parameter, kube_operator): ------ wsme.exc.ClientSideError """ - _volume, _ = service_parameter.parse_volume_string_to_dict(parameter) + _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter) # only delete configmaps for 'File' type since # 'DirectoryorCreate' type has no associated configmaps @@ -160,7 +160,7 @@ def update_k8s_configmap(parameter, kube_operator): ------ wsme.exc.ClientSideError """ - _volume, _ = service_parameter.parse_volume_string_to_dict(parameter) + _volume, _noConfigMap = service_parameter.parse_volume_string_to_dict(parameter) # only update configmaps for 'File' type since # 'DirectoryorCreate' type has no associated configmaps diff --git a/sysinv/sysinv/sysinv/test-requirements.txt b/sysinv/sysinv/sysinv/test-requirements.txt index ef0601ec04..a140bcc1ae 100644 --- a/sysinv/sysinv/sysinv/test-requirements.txt +++ b/sysinv/sysinv/sysinv/test-requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. hacking>=1.1.0,<=2.0.0 # Apache-2.0 -astroid <= 2.2.5 +astroid bandit<1.7.2;python_version>="3.0" coverage>=3.6 fixtures>=3.0.0 # Apache-2.0/BSD @@ -16,6 +16,5 @@ stestr>=1.0.0 # Apache-2.0 testrepository>=0.0.18 testtools!=1.2.0,>=0.9.36 isort<5;python_version>="3.0" -pylint<2.1.0;python_version<"3.0" # GPLv2 -pylint<2.4.0;python_version>="3.0" # GPLv2 +pylint pycryptodomex diff --git a/sysinv/sysinv/sysinv/tox.ini b/sysinv/sysinv/sysinv/tox.ini index 8cd445f777..7a1a91ce6a 100644 --- a/sysinv/sysinv/sysinv/tox.ini +++ b/sysinv/sysinv/sysinv/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,py27,py36,py39,pylint +envlist = flake8,py39,pylint minversion = 1.6 skipsdist = True @@ -10,7 +10,7 @@ stxdir = {toxinidir}/../../../.. distshare={toxworkdir}/.tox/distshare [testenv] -basepython = python3 +basepython = python3.9 usedevelop = True # tox is silly... these need to be separated by a newline.... @@ -18,7 +18,7 @@ allowlist_externals = bash find install_command = pip install -v -v -v \ - -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/stable/stein/upper-constraints.txt} \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ {opts} {packages} # Note the hash seed is set to 0 until can be tested with a @@ -42,7 +42,7 @@ deps = -r{toxinidir}/requirements.txt -e{[tox]stxdir}/fault/fm-api/source -e{[tox]stxdir}/fault/python-fmclient/fmclient -e{[tox]stxdir}/config/controllerconfig/controllerconfig - -e{[tox]stxdir}/update/cgcs-patch/cgcs-patch + -e{[tox]stxdir}/update/sw-patch/cgcs-patch -e{[tox]stxdir}/utilities/utilities/platform-util/platform-util -e{[tox]stxdir}/utilities/ceph/python-cephclient/python-cephclient -e{[tox]stxdir}/config/sysinv/cgts-client/cgts-client @@ -95,42 +95,19 @@ exclude = build,dist,tools,.eggs max-line-length=120 [testenv:flake8] -basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = flake8 {posargs} . \ scripts/manage-partitions \ scripts/query_pci_id -[testenv:py27] -basepython = python2.7 -passenv = CURL_CA_BUNDLE -commands = - {[testenv]commands} - stestr run {posargs} - stestr slowest - -[testenv:py36] -basepython = python3.6 -commands = - {[testenv]commands} - stestr run {posargs} - stestr slowest - [testenv:py39] -basepython = python3.9 - -install_command = pip install -v -v -v \ - -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ - {opts} {packages} commands = stestr run {posargs} stestr slowest - [testenv:pep8] # testenv:flake8 clone -basepython = python3 deps = -r{toxinidir}/test-requirements.txt commands = {[testenv:flake8]commands} @@ -170,20 +147,18 @@ skips = B101,B103,B104,B105,B108,B110,B303,B307,B310,B311,B314,B318,B320,B404,B4 exclude = tests [testenv:bandit] -basepython = python3.6 deps = -r{toxinidir}/test-requirements.txt commands = bandit --ini tox.ini -n 5 -r sysinv [testenv:pylint] -basepython = python3.6 +install_command = pip install -v -v -v \ + -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ + {opts} {packages} commands = pylint {posargs} sysinv --rcfile=./pylint.rc [testenv:cover] -basepython = python3.9 -install_command = pip install -v -v -v \ - -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/starlingx/root/raw/branch/master/build-tools/requirements/debian/upper-constraints.txt} \ - {opts} {packages} +# not sure is passenv is still needed passenv = CURL_CA_BUNDLE deps = {[testenv]deps} setenv = {[testenv]setenv}