From 6b4e8efc26b092b1d17caadcfd5c742df2d28d11 Mon Sep 17 00:00:00 2001 From: Al Bailey Date: Mon, 20 Jan 2020 10:41:11 -0600 Subject: [PATCH] Updates for cgtsclient to work with python3 Deprecate cgtsclient.openstack.common files since those files were mostly replaced by oslo. Fix a wrapping_formatters import which was using a relative path. A recursion error with deepcopy has been fixed which allows the py36 unit tests to be enabled. Newer flake8, hacking and bugbear are now enabled since python3 is able to be tested now. Change-Id: I4978ab33b09e13b933b7f0fc9da22ed97e671938 Story: 2007082 Task: 38314 Signed-off-by: Al Bailey --- .zuul.yaml | 15 ++ sysinv/cgts-client/centos/cgts-client.spec | 4 + sysinv/cgts-client/cgts-client/.coveragerc | 8 + .../cgts-client/cgtsclient/_i18n.py | 32 +++ .../cgts-client/cgtsclient/client.py | 2 +- .../cgts-client/cgtsclient/common/base.py | 16 ++ .../cgts-client/cgtsclient/common/http.py | 2 +- .../cgts-client/cgtsclient/common/utils.py | 11 +- .../cgtsclient/common/wrapping_formatters.py | 1 + .../cgtsclient/openstack/__init__.py | 14 - .../cgtsclient/openstack/common/__init__.py | 14 - .../openstack/common/config/generator.py | 254 ------------------ .../openstack/common/gettextutils.py | 50 ---- .../openstack/common/importutils.py | 66 ----- .../openstack/common/rootwrap/__init__.py | 16 -- .../openstack/common/rootwrap/cmd.py | 118 -------- .../openstack/common/rootwrap/filters.py | 228 ---------------- .../openstack/common/rootwrap/wrapper.py | 151 ----------- .../cgts-client/cgtsclient/shell.py | 1 + .../cgts-client/cgtsclient/v1/iHost_shell.py | 3 +- .../cgts-client/cgtsclient/v1/icpu.py | 2 +- .../cgts-client/cgts-client/requirements.txt | 3 + .../cgts-client/test-requirements.txt | 3 +- sysinv/cgts-client/cgts-client/tox.ini | 42 ++- 24 files changed, 121 insertions(+), 935 deletions(-) create mode 100644 sysinv/cgts-client/cgts-client/.coveragerc create mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/_i18n.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/__init__.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/__init__.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/config/generator.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/gettextutils.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/importutils.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/__init__.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/cmd.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/filters.py delete mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/wrapper.py diff --git a/.zuul.yaml b/.zuul.yaml index 61555c1447..076fa78f8e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -16,6 +16,7 @@ - controllerconfig-tox-py27 - controllerconfig-tox-pylint - cgtsclient-tox-py27 + - cgtsclient-tox-py36 - cgtsclient-tox-pep8 - cgtsclient-tox-pylint gate: @@ -30,6 +31,7 @@ - controllerconfig-tox-py27 - controllerconfig-tox-pylint - cgtsclient-tox-py27 + - cgtsclient-tox-py36 - cgtsclient-tox-pep8 - cgtsclient-tox-pylint @@ -171,6 +173,19 @@ tox_envlist: py27 tox_extra_args: -c sysinv/cgts-client/cgts-client/tox.ini +- job: + name: cgtsclient-tox-py36 + parent: tox + description: | + Run py36 test for cgts-client + nodeset: ubuntu-bionic + files: + - sysinv/cgts-client/* + vars: + tox_envlist: py36 + tox_extra_args: -c sysinv/cgts-client/cgts-client/tox.ini + + - job: name: cgtsclient-tox-pep8 parent: tox diff --git a/sysinv/cgts-client/centos/cgts-client.spec b/sysinv/cgts-client/centos/cgts-client.spec index 0f46393430..4e108d4b3d 100644 --- a/sysinv/cgts-client/centos/cgts-client.spec +++ b/sysinv/cgts-client/centos/cgts-client.spec @@ -18,6 +18,10 @@ Requires: python-prettytable Requires: bash-completion Requires: python-neutronclient Requires: python-keystoneclient +Requires: python2-oslo-i18n +Requires: python2-oslo-serialization +Requires: python2-oslo-utils + # Needed for python2 and python3 compatible Requires: python-six diff --git a/sysinv/cgts-client/cgts-client/.coveragerc b/sysinv/cgts-client/cgts-client/.coveragerc new file mode 100644 index 0000000000..b8f4e06728 --- /dev/null +++ b/sysinv/cgts-client/cgts-client/.coveragerc @@ -0,0 +1,8 @@ +[run] +branch = True +source = cgtsclient +omit = cgtsclient/tests/* + +[report] +ignore_errors = True + diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/_i18n.py b/sysinv/cgts-client/cgts-client/cgtsclient/_i18n.py new file mode 100644 index 0000000000..fe3dba1d19 --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/_i18n.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""oslo.i18n integration module. + +See https://docs.openstack.org/oslo.i18n/latest/user/usage.html + +""" + +import oslo_i18n + +DOMAIN = 'python-cgtsclient' + +_translators = oslo_i18n.TranslatorFactory(domain=DOMAIN) + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# The contextual translation function using the name "_C" +# requires oslo.i18n >=2.1.0 +_C = _translators.contextual_form + +# The plural translation function using the name "_P" +# requires oslo.i18n >=2.1.0 +_P = _translators.plural_form + + +def get_available_languages(): + return oslo_i18n.get_available_languages(DOMAIN) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/client.py b/sysinv/cgts-client/cgts-client/cgtsclient/client.py index f093b396ea..d1241a0afa 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/client.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/client.py @@ -4,9 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # +from cgtsclient._i18n import _ from cgtsclient.common import utils from cgtsclient import exc -from cgtsclient.openstack.common.gettextutils import _ def _get_ksclient(**kwargs): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/common/base.py b/sysinv/cgts-client/cgts-client/cgtsclient/common/base.py index df638d5fce..bdd9e8c55f 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/common/base.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/common/base.py @@ -119,6 +119,22 @@ class Resource(object): else: return self.__dict__[k] + # deepcopy is invoked on this object which causes infinite recursion in python3 + # unless the copy and deepcopy methods are overridden + def __copy__(self): + cls = self.__class__ + result = cls.__new__(cls) + result.__dict__.update(self.__dict__) + return result + + def __deepcopy__(self, memo): + cls = self.__class__ + result = cls.__new__(cls) + memo[id(self)] = result + for k, v in self.__dict__.items(): + setattr(result, k, copy.deepcopy(v, memo)) + return result + def __repr__(self): reprkeys = sorted(k for k in self.__dict__.keys() if k[0] != '_' and k != 'manager') diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py b/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py index 48f3037363..9235df91b1 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py @@ -528,7 +528,7 @@ class ResponseBodyIterator(object): def __iter__(self): while True: - yield self.next() + yield six.next() def next(self): chunk = self.resp.read(CHUNKSIZE) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/common/utils.py b/sysinv/cgts-client/cgts-client/cgtsclient/common/utils.py index f214efdbda..1fa3f6dc12 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/common/utils.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/common/utils.py @@ -41,13 +41,10 @@ from prettytable import NONE from datetime import datetime from dateutil import parser - -from cgtsclient import exc -from cgtsclient.openstack.common import importutils from functools import wraps -# noinspection PyProtectedMember -from wrapping_formatters import _get_width +from cgtsclient import exc +from oslo_utils import importutils from cgtsclient.common import wrapping_formatters from six.moves import input @@ -300,7 +297,7 @@ def pt_builder(field_labels, fields, formatters, paging, printer=default_printer output = self.pt.get_string() if wrapping_formatters.is_nowrap_set(): return output - output_width = _get_width(output) + output_width = wrapping_formatters._get_width(output) if output_width <= self.terminal_width: return output # At this point pretty Table (self.pt) does not fit the terminal width so let's @@ -476,7 +473,7 @@ def print_dict_with_format(data, wrap=0, output_format=None): def print_dict_value(d): # Print values on a single line separated by spaces # e.g. 'available ntp' - print (' '.join(map(str, d.values()))) + print(' '.join(map(str, d.values()))) def print_dict(d, dict_property="Property", wrap=0): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/common/wrapping_formatters.py b/sysinv/cgts-client/cgts-client/cgtsclient/common/wrapping_formatters.py index e1a5981e18..2f53265c91 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/common/wrapping_formatters.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/common/wrapping_formatters.py @@ -801,6 +801,7 @@ def _simpleTestHarness(no_wrap): print("nowrap = {}".format(is_nowrap_set())) + if __name__ == "__main__": _simpleTestHarness(True) _simpleTestHarness(False) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/__init__.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/__init__.py deleted file mode 100644 index 265c2d9f65..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/__init__.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/__init__.py deleted file mode 100644 index 265c2d9f65..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/config/generator.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/config/generator.py deleted file mode 100644 index 526f71ef01..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/config/generator.py +++ /dev/null @@ -1,254 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 SINA Corporation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# @author: Zhongyue Luo, SINA Corporation. -# -"""Extracts OpenStack config option info from module(s).""" - -import imp -import os -import re -import six -import socket -import sys -import textwrap - -from oslo_config import cfg - -from cgtsclient.openstack.common import gettextutils -from cgtsclient.openstack.common import importutils - -gettextutils.install('python-cgtsclient') - -STROPT = "StrOpt" -BOOLOPT = "BoolOpt" -INTOPT = "IntOpt" -FLOATOPT = "FloatOpt" -LISTOPT = "ListOpt" -MULTISTROPT = "MultiStrOpt" - -OPT_TYPES = { - STROPT: 'string value', - BOOLOPT: 'boolean value', - INTOPT: 'integer value', - FLOATOPT: 'floating point value', - LISTOPT: 'list value', - MULTISTROPT: 'multi valued', -} - -OPTION_COUNT = 0 -OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, - FLOATOPT, LISTOPT, - MULTISTROPT])) - -PY_EXT = ".py" -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - "../../../../")) -WORDWRAP_WIDTH = 60 - - -def generate(srcfiles): - mods_by_pkg = dict() - for filepath in srcfiles: - pkg_name = filepath.split(os.sep)[1] - mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), - os.path.basename(filepath).split('.')[0]]) - mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) - # NOTE(lzyeval): place top level modules before packages - pkg_names = [x for x in mods_by_pkg.keys() if x.endswith(PY_EXT)] - pkg_names.sort() - ext_names = [x for x in mods_by_pkg.keys() if x not in pkg_names] - ext_names.sort() - pkg_names.extend(ext_names) - - # opts_by_group is a mapping of group name to an options list - # The options list is a list of (module, options) tuples - opts_by_group = {'DEFAULT': []} - - for pkg_name in pkg_names: - mods = mods_by_pkg.get(pkg_name) - mods.sort() - for mod_str in mods: - if mod_str.endswith('.__init__'): - mod_str = mod_str[:mod_str.rfind(".")] - - mod_obj = _import_module(mod_str) - if not mod_obj: - continue - - for group, opts in _list_opts(mod_obj): - opts_by_group.setdefault(group, []).append((mod_str, opts)) - - print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) - for group, opts in opts_by_group.items(): - print_group_opts(group, opts) - - print("# Total option count: %d" % OPTION_COUNT) - - -def _import_module(mod_str): - try: - if mod_str.startswith('bin.'): - imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) - return sys.modules[mod_str[4:]] - else: - return importutils.import_module(mod_str) - except ImportError as ie: - sys.stderr.write("%s\n" % str(ie)) - return None - except Exception: - return None - - -def _is_in_group(opt, group): - "Check if opt is in group." - for key, value in group._opts.items(): - if value['opt'] == opt: - return True - return False - - -def _guess_groups(opt, mod_obj): - # is it in the DEFAULT group? - if _is_in_group(opt, cfg.CONF): - return 'DEFAULT' - - # what other groups is it in? - for key, value in cfg.CONF.items(): - if isinstance(value, cfg.CONF.GroupAttr): - if _is_in_group(opt, value._group): - return value._group.name - - raise RuntimeError( - "Unable to find group for option %s, " - "maybe it's defined twice in the same group?" - % opt.name - ) - - -def _list_opts(obj): - def is_opt(o): - return (isinstance(o, cfg.Opt) and - not isinstance(o, cfg.SubCommandOpt)) - - opts = list() - for attr_str in dir(obj): - attr_obj = getattr(obj, attr_str) - if is_opt(attr_obj): - opts.append(attr_obj) - elif (isinstance(attr_obj, list) and - all([is_opt(x) for x in attr_obj])): - opts.extend(attr_obj) - - ret = {} - for opt in opts: - ret.setdefault(_guess_groups(opt, obj), []).append(opt) - return ret.items() - - -def print_group_opts(group, opts_by_module): - print("[%s]" % group) - print('') - global OPTION_COUNT - for mod, opts in opts_by_module: - OPTION_COUNT += len(opts) - print('#') - print('# Options defined in %s' % mod) - print('#') - print('') - for opt in opts: - _print_opt(opt) - print('') - - -def _get_my_ip(): - try: - csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - csock.connect(('8.8.8.8', 80)) - (addr, port) = csock.getsockname() - csock.close() - return addr - except socket.error: - return None - - -def _sanitize_default(s): - """Set up a reasonably sensible default for pybasedir, my_ip and host.""" - if s.startswith(BASEDIR): - return s.replace(BASEDIR, '/usr/lib/python/site-packages') - elif BASEDIR in s: - return s.replace(BASEDIR, '') - elif s == _get_my_ip(): - return '10.0.0.1' - elif s == socket.gethostname(): - return 'python-cgtsclient' - elif s.strip() != s: - return '"%s"' % s - return s - - -def _print_opt(opt): - opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help - if not opt_help: - sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) - opt_type = None - try: - opt_type = OPTION_REGEX.search(str(type(opt))).group(0) - except (ValueError, AttributeError) as err: - sys.stderr.write("%s\n" % str(err)) - sys.exit(1) - opt_help += ' (' + OPT_TYPES[opt_type] + ')' - print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) - try: - if opt_default is None: - print('#%s=' % opt_name) - elif opt_type == STROPT: - assert(isinstance(opt_default, six.string_types)) - print('#%s=%s' % (opt_name, _sanitize_default(opt_default))) - elif opt_type == BOOLOPT: - assert(isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, str(opt_default).lower())) - elif opt_type == INTOPT: - assert(isinstance(opt_default, int) and - not isinstance(opt_default, bool)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == FLOATOPT: - assert(isinstance(opt_default, float)) - print('#%s=%s' % (opt_name, opt_default)) - elif opt_type == LISTOPT: - assert(isinstance(opt_default, list)) - print('#%s=%s' % (opt_name, ','.join(opt_default))) - elif opt_type == MULTISTROPT: - assert(isinstance(opt_default, list)) - if not opt_default: - opt_default = [''] - for default in opt_default: - print('#%s=%s' % (opt_name, default)) - print('') - except Exception: - sys.stderr.write('Error in option "%s"\n' % opt_name) - sys.exit(1) - - -def main(): - if len(sys.argv) < 2: - print("usage: %s [srcfile]...\n" % sys.argv[0]) - sys.exit(0) - generate(sys.argv[1:]) - -if __name__ == '__main__': - main() diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/gettextutils.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/gettextutils.py deleted file mode 100644 index 15962e6979..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/gettextutils.py +++ /dev/null @@ -1,50 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 Red Hat, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -gettext for openstack-common modules. - -Usual usage in an openstack.common module: - - from cgts.openstack.common.gettextutils import _ -""" - -import gettext -import os - -_localedir = os.environ.get('cgtsclient'.upper() + '_LOCALEDIR') -_t = gettext.translation('cgtsclient', localedir=_localedir, fallback=True) - - -def _(msg): - return _t.ugettext(msg) - - -def install(domain): - """Install a _() function using the given translation domain. - - Given a translation domain, install a _() function using gettext's - install() function. - - The main difference from gettext.install() is that we allow - overriding the default localedir (e.g. /usr/share/locale) using - a translation-domain-specific environment variable (e.g. - NOVA_LOCALEDIR). - """ - gettext.install(domain, - localedir=os.environ.get(domain.upper() + '_LOCALEDIR'), - unicode=True) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/importutils.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/importutils.py deleted file mode 100644 index f19403bcc8..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/importutils.py +++ /dev/null @@ -1,66 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Import related utilities and helper functions. -""" - -import sys -import traceback - - -def import_class(import_str): - """Returns a class from a string including module and class""" - mod_str, _sep, class_str = import_str.rpartition('.') - try: - __import__(mod_str) - return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): - raise ImportError('Class %s cannot be found (%s)' % - (class_str, - traceback.format_exception(*sys.exc_info()))) - - -def import_object(import_str, *args, **kwargs): - """Import a class and return an instance of it.""" - return import_class(import_str)(*args, **kwargs) - - -def import_object_ns(name_space, import_str, *args, **kwargs): - """Import a class and return an instance of it, first by trying - to find the class in a default namespace, then failing back to - a full path if not found in the default namespace. - """ - import_value = "%s.%s" % (name_space, import_str) - try: - return import_class(import_value)(*args, **kwargs) - except ImportError: - return import_class(import_str)(*args, **kwargs) - - -def import_module(import_str): - """Import a module.""" - __import__(import_str) - return sys.modules[import_str] - - -def try_import(import_str, default=None): - """Try to import a module and if it fails return default.""" - try: - return import_module(import_str) - except ImportError: - return default diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/__init__.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/__init__.py deleted file mode 100644 index 2d32e4ef31..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/cmd.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/cmd.py deleted file mode 100644 index dcc373efc1..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/cmd.py +++ /dev/null @@ -1,118 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Root wrapper for OpenStack services - -""" - -from __future__ import print_function - -import logging -import os -import pwd -import signal -import subprocess -import sys - -from six.moves import configparser - -RC_UNAUTHORIZED = 99 -RC_NOCOMMAND = 98 -RC_BADCONFIG = 97 -RC_NOEXECFOUND = 96 - - -def _subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - - -def _exit_error(execname, message, errorcode, log=True): - print("%s: %s" % (execname, message)) - if log: - logging.error(message) - sys.exit(errorcode) - - -def main(): - # Split arguments, require at least a command - execname = sys.argv.pop(0) - if len(sys.argv) < 2: - _exit_error(execname, "No command specified", RC_NOCOMMAND, log=False) - - configfile = sys.argv.pop(0) - userargs = sys.argv[:] - - # Add ../ to sys.path to allow running from branch - possible_topdir = os.path.normpath(os.path.join(os.path.abspath(execname), - os.pardir, os.pardir)) - if os.path.exists(os.path.join(possible_topdir, "cgtsclient", - "__init__.py")): - sys.path.insert(0, possible_topdir) - - from cgtsclient.openstack.common.rootwrap import wrapper - - # Load configuration - try: - rawconfig = configparser.RawConfigParser() - rawconfig.read(configfile) - config = wrapper.RootwrapConfig(rawconfig) - except ValueError as exc: - msg = "Incorrect value in %s: %s" % (configfile, exc.message) - _exit_error(execname, msg, RC_BADCONFIG, log=False) - except configparser.Error: - _exit_error(execname, "Incorrect configuration file: %s" % configfile, - RC_BADCONFIG, log=False) - - if config.use_syslog: - wrapper.setup_syslog(execname, - config.syslog_log_facility, - config.syslog_log_level) - - # Execute command if it matches any of the loaded filters - filters = wrapper.load_filters(config.filters_path) - try: - filtermatch = wrapper.match_filter(filters, userargs, - exec_dirs=config.exec_dirs) - if filtermatch: - command = filtermatch.get_command(userargs, - exec_dirs=config.exec_dirs) - if config.use_syslog: - logging.info("(%s > %s) Executing %s (filter match = %s)" % ( - os.getlogin(), pwd.getpwuid(os.getuid())[0], - command, filtermatch.name)) - - obj = subprocess.Popen(command, - stdin=sys.stdin, - stdout=sys.stdout, - stderr=sys.stderr, - preexec_fn=_subprocess_setup, - env=filtermatch.get_environment(userargs)) - obj.wait() - sys.exit(obj.returncode) - - except wrapper.FilterMatchNotExecutable as exc: - msg = ("Executable not found: %s (filter match = %s)" - % (exc.match.exec_path, exc.match.name)) - _exit_error(execname, msg, RC_NOEXECFOUND, log=config.use_syslog) - - except wrapper.NoFilterMatched: - msg = ("Unauthorized command: %s (no filter matched)" - % ' '.join(userargs)) - _exit_error(execname, msg, RC_UNAUTHORIZED, log=config.use_syslog) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/filters.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/filters.py deleted file mode 100644 index ae7c62cada..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/filters.py +++ /dev/null @@ -1,228 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import re - - -class CommandFilter(object): - """Command filter only checking that the 1st argument matches exec_path.""" - - def __init__(self, exec_path, run_as, *args): - self.name = '' - self.exec_path = exec_path - self.run_as = run_as - self.args = args - self.real_exec = None - - def get_exec(self, exec_dirs=[]): - """Returns existing executable, or empty string if none found.""" - if self.real_exec is not None: - return self.real_exec - self.real_exec = "" - if self.exec_path.startswith('/'): - if os.access(self.exec_path, os.X_OK): - self.real_exec = self.exec_path - else: - for binary_path in exec_dirs: - expanded_path = os.path.join(binary_path, self.exec_path) - if os.access(expanded_path, os.X_OK): - self.real_exec = expanded_path - break - return self.real_exec - - def match(self, userargs): - """Only check that the first argument (command) matches exec_path.""" - return os.path.basename(self.exec_path) == userargs[0] - - def get_command(self, userargs, exec_dirs=[]): - """Returns command to execute (with sudo -u if run_as != root).""" - to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path - if (self.run_as != 'root'): - # Used to run commands at lesser privileges - return ['sudo', '-u', self.run_as, to_exec] + userargs[1:] - return [to_exec] + userargs[1:] - - def get_environment(self, userargs): - """Returns specific environment to set, None if none.""" - return None - - -class RegExpFilter(CommandFilter): - """Command filter doing regexp matching for every argument.""" - - def match(self, userargs): - # Early skip if command or number of args don't match - if (len(self.args) != len(userargs)): - # DENY: argument numbers don't match - return False - # Compare each arg (anchoring pattern explicitly at end of string) - for (pattern, arg) in zip(self.args, userargs): - try: - if not re.match(pattern + '$', arg): - break - except re.error: - # DENY: Badly-formed filter - return False - else: - # ALLOW: All arguments matched - return True - - # DENY: Some arguments did not match - return False - - -class PathFilter(CommandFilter): - """Command filter checking that path arguments are within given dirs - - One can specify the following constraints for command arguments: - 1) pass - pass an argument as is to the resulting command - 2) some_str - check if an argument is equal to the given string - 3) abs path - check if a path argument is within the given base dir - - A typical rootwrapper filter entry looks like this: - # cmdname: filter name, raw command, user, arg_i_constraint [, ...] - chown: PathFilter, /bin/chown, root, nova, /var/lib/images - - """ - - def match(self, userargs): - command, arguments = userargs[0], userargs[1:] - - equal_args_num = len(self.args) == len(arguments) - exec_is_valid = super(PathFilter, self).match(userargs) - args_equal_or_pass = all( - arg == 'pass' or arg == value - for arg, value in zip(self.args, arguments) - if not os.path.isabs(arg) # arguments not specifying abs paths - ) - paths_are_within_base_dirs = all( - os.path.commonprefix([arg, os.path.realpath(value)]) == arg - for arg, value in zip(self.args, arguments) - if os.path.isabs(arg) # arguments specifying abs paths - ) - - return (equal_args_num and - exec_is_valid and - args_equal_or_pass and - paths_are_within_base_dirs) - - def get_command(self, userargs, exec_dirs=[]): - command, arguments = userargs[0], userargs[1:] - - # convert path values to canonical ones; copy other args as is - args = [os.path.realpath(value) if os.path.isabs(arg) else value - for arg, value in zip(self.args, arguments)] - - return super(PathFilter, self).get_command([command] + args, - exec_dirs) - - -class DnsmasqFilter(CommandFilter): - """Specific filter for the dnsmasq call (which includes env).""" - - CONFIG_FILE_ARG = 'CONFIG_FILE' - - def match(self, userargs): - if (userargs[0] == 'env' and - userargs[1].startswith(self.CONFIG_FILE_ARG) and - userargs[2].startswith('NETWORK_ID=') and - userargs[3] == 'dnsmasq'): - return True - return False - - def get_command(self, userargs, exec_dirs=[]): - to_exec = self.get_exec(exec_dirs=exec_dirs) or self.exec_path - dnsmasq_pos = userargs.index('dnsmasq') - return [to_exec] + userargs[dnsmasq_pos + 1:] - - def get_environment(self, userargs): - env = os.environ.copy() - env[self.CONFIG_FILE_ARG] = userargs[1].split('=')[-1] - env['NETWORK_ID'] = userargs[2].split('=')[-1] - return env - - -class DeprecatedDnsmasqFilter(DnsmasqFilter): - """Variant of dnsmasq filter to support old-style FLAGFILE.""" - CONFIG_FILE_ARG = 'FLAGFILE' - - -class KillFilter(CommandFilter): - """Specific filter for the kill calls. - 1st argument is the user to run /bin/kill under - 2nd argument is the location of the affected executable - Subsequent arguments list the accepted signals (if any) - - This filter relies on /proc to accurately determine affected - executable, so it will only work on procfs-capable systems (not OSX). - """ - - def __init__(self, *args): - super(KillFilter, self).__init__("/bin/kill", *args) - - def match(self, userargs): - if userargs[0] != "kill": - return False - args = list(userargs) - if len(args) == 3: - # A specific signal is requested - signal = args.pop(1) - if signal not in self.args[1:]: - # Requested signal not in accepted list - return False - else: - if len(args) != 2: - # Incorrect number of arguments - return False - if len(self.args) > 1: - # No signal requested, but filter requires specific signal - return False - try: - command = os.readlink("/proc/%d/exe" % int(args[1])) - # NOTE(yufang521247): /proc/PID/exe may have '\0' on the - # end, because python doen't stop at '\0' when read the - # target path. - command = command.split('\0')[0] - # NOTE(dprince): /proc/PID/exe may have ' (deleted)' on - # the end if an executable is updated or deleted - if command.endswith(" (deleted)"): - command = command[:command.rindex(" ")] - if command != self.args[0]: - # Affected executable does not match - return False - except (ValueError, OSError): - # Incorrect PID - return False - return True - - -class ReadFileFilter(CommandFilter): - """Specific filter for the utils.read_file_as_root call.""" - - def __init__(self, file_path, *args): - self.file_path = file_path - super(ReadFileFilter, self).__init__("/bin/cat", "root", *args) - - def match(self, userargs): - if userargs[0] != 'cat': - return False - if userargs[1] != self.file_path: - return False - if len(userargs) != 2: - return False - return True diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/wrapper.py b/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/wrapper.py deleted file mode 100644 index 0117b308ee..0000000000 --- a/sysinv/cgts-client/cgts-client/cgtsclient/openstack/common/rootwrap/wrapper.py +++ /dev/null @@ -1,151 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import logging -import logging.handlers -import os -import string - -from cgtsclient.openstack.common.rootwrap import filters -from six.moves import configparser - - -class NoFilterMatched(Exception): - """This exception is raised when no filter matched.""" - pass - - -class FilterMatchNotExecutable(Exception): - """raise if filter matche but not executable - - This exception is raised when a filter matched but no executable was - found. - """ - def __init__(self, match=None, **kwargs): - self.match = match - - -class RootwrapConfig(object): - - def __init__(self, config): - # filters_path - self.filters_path = config.get("DEFAULT", "filters_path").split(",") - - # exec_dirs - if config.has_option("DEFAULT", "exec_dirs"): - self.exec_dirs = config.get("DEFAULT", "exec_dirs").split(",") - else: - # Use system PATH if exec_dirs is not specified - self.exec_dirs = os.environ["PATH"].split(':') - - # syslog_log_facility - if config.has_option("DEFAULT", "syslog_log_facility"): - v = config.get("DEFAULT", "syslog_log_facility") - facility_names = logging.handlers.SysLogHandler.facility_names - self.syslog_log_facility = getattr(logging.handlers.SysLogHandler, - v, None) - if self.syslog_log_facility is None and v in facility_names: - self.syslog_log_facility = facility_names.get(v) - if self.syslog_log_facility is None: - raise ValueError('Unexpected syslog_log_facility: %s' % v) - else: - default_facility = logging.handlers.SysLogHandler.LOG_SYSLOG - self.syslog_log_facility = default_facility - - # syslog_log_level - if config.has_option("DEFAULT", "syslog_log_level"): - v = config.get("DEFAULT", "syslog_log_level") - self.syslog_log_level = logging.getLevelName(v.upper()) - if (self.syslog_log_level == "Level %s" % v.upper()): - raise ValueError('Unexepected syslog_log_level: %s' % v) - else: - self.syslog_log_level = logging.ERROR - - # use_syslog - if config.has_option("DEFAULT", "use_syslog"): - self.use_syslog = config.getboolean("DEFAULT", "use_syslog") - else: - self.use_syslog = False - - -def setup_syslog(execname, facility, level): - rootwrap_logger = logging.getLogger() - rootwrap_logger.setLevel(level) - handler = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - handler.setFormatter(logging.Formatter( - os.path.basename(execname) + ': %(message)s')) - rootwrap_logger.addHandler(handler) - - -def build_filter(class_name, *args): - """Returns a filter object of class class_name.""" - if not hasattr(filters, class_name): - logging.warning("Skipping unknown filter class (%s) specified " - "in filter definitions" % class_name) - return None - filterclass = getattr(filters, class_name) - return filterclass(*args) - - -def load_filters(filters_path): - """Load filters from a list of directories.""" - filterlist = [] - for filterdir in filters_path: - if not os.path.isdir(filterdir): - continue - for filterfile in os.listdir(filterdir): - filterconfig = configparser.RawConfigParser() - filterconfig.read(os.path.join(filterdir, filterfile)) - for (name, value) in filterconfig.items("Filters"): - filterdefinition = [string.strip(s) for s in value.split(',')] - newfilter = build_filter(*filterdefinition) - if newfilter is None: - continue - newfilter.name = name - filterlist.append(newfilter) - return filterlist - - -def match_filter(filter_list, userargs, exec_dirs=[]): - """check user command and args - - Checks user command and arguments through command filters and - returns the first matching filter. - Raises NoFilterMatched if no filter matched. - Raises FilterMatchNotExecutable if no executable was found for the - best filter match. - """ - first_not_executable_filter = None - - for f in filter_list: - if f.match(userargs): - # Try other filters if executable is absent - if not f.get_exec(exec_dirs=exec_dirs): - if not first_not_executable_filter: - first_not_executable_filter = f - continue - # Otherwise return matching filter for execution - return f - - if first_not_executable_filter: - # A filter matched, but no executable was found for it - raise FilterMatchNotExecutable(match=first_not_executable_filter) - - # No filter matched - raise NoFilterMatched() diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/shell.py index 3b249ac605..75ccba814f 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/shell.py @@ -349,5 +349,6 @@ def main(): print(e, file=sys.stderr) sys.exit(1) + if __name__ == "__main__": main() diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py index f3f29071de..6421885907 100755 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/iHost_shell.py @@ -13,10 +13,10 @@ from collections import OrderedDict import datetime import os +from cgtsclient._i18n import _ from cgtsclient.common import constants from cgtsclient.common import utils from cgtsclient import exc -from cgtsclient.openstack.common.gettextutils import _ from cgtsclient.v1 import icpu as icpu_utils from cgtsclient.v1 import ihost as ihost_utils from cgtsclient.v1 import iinterface as iinterface_utils @@ -513,6 +513,7 @@ def _list_storage(cc, host): fields = ['uuid', 'lvm_pv_name', 'disk_or_part_device_path', 'lvm_vg_name'] utils.print_list(ipvs, fields, field_labels, sortby=0) + """ NOTE (neid): all three "do_host_apply_profile" methods can be replaced diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu.py index ff4bbd5694..60fe5fdb90 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/icpu.py @@ -6,9 +6,9 @@ # +from cgtsclient._i18n import _ from cgtsclient.common import base from cgtsclient import exc -from cgtsclient.openstack.common.gettextutils import _ CREATION_ATTRIBUTES = ['ihost_uuid', 'inode_uuid', 'cpu', 'core', 'thread', diff --git a/sysinv/cgts-client/cgts-client/requirements.txt b/sysinv/cgts-client/cgts-client/requirements.txt index fae039f467..be7aee85cf 100644 --- a/sysinv/cgts-client/cgts-client/requirements.txt +++ b/sysinv/cgts-client/cgts-client/requirements.txt @@ -1,2 +1,5 @@ python-neutronclient keyring +oslo.i18n # Apache-2.0 +oslo.serialization>=1.10.0,!=2.19.1 # Apache-2.0 +oslo.utils>=3.5.0 # Apache-2.0 diff --git a/sysinv/cgts-client/cgts-client/test-requirements.txt b/sysinv/cgts-client/cgts-client/test-requirements.txt index ad0a4024f7..9beb4450d2 100644 --- a/sysinv/cgts-client/cgts-client/test-requirements.txt +++ b/sysinv/cgts-client/cgts-client/test-requirements.txt @@ -3,7 +3,8 @@ # process, which may cause wedges in the gate later. # Hacking already pins down pep8, pyflakes and flake8 -hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 +hacking>=1.1.0,<=2.0.0 # Apache-2.0 +pycodestyle>=2.0.0 # MIT License bandit>=1.1.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD diff --git a/sysinv/cgts-client/cgts-client/tox.ini b/sysinv/cgts-client/cgts-client/tox.ini index a32c792bbe..a3c44394d3 100644 --- a/sysinv/cgts-client/cgts-client/tox.ini +++ b/sysinv/cgts-client/cgts-client/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pep8,pylint +envlist = py27,py36,pep8,pylint minversion = 1.6 #skipsdist = True @@ -47,9 +47,11 @@ commands = stestr slowest [testenv:pep8] +basepython = python3 deps = -r{toxinidir}/test-requirements.txt + flake8-bugbear commands = - flake8 cgtsclient + flake8 cgtsclient [testenv:venv] commands = {posargs} @@ -75,16 +77,32 @@ commands = show-source = true exclude=.*,dist,*lib/python*,*egg,build max-complexity=25 -ignore = H102,H104,H105,H238,H404,H405,E501,F841 -#H102 Apache 2.0 license header not found -#H104 File contains nothing but comments -#H105 Don't use author tags -#H238 old style class declaration, use new style (inherit from `object`) -#H404 multi line docstring should start without a leading new line -#H405 multi line docstring summary not separated with an empty line -#E501 line too long -#F841 local variable 'X' is assigned to but never used - +# H102 Apache 2.0 license header not found +# H104 File contains nothing but comments +# H105 Don't use author tags +# H238 old style class declaration, use new style (inherit from `object`) +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +# -B- codes are bugbear +# B004 Using `hasattr(x, '__call__')` to test if `x` is callable is unreliable. +# B005 Using .strip() with multi-character strings is misleading the reader. +# B006 Do not use mutable data structures for argument defaults +# B009 Do not call getattr with a constant attribute value +# B010 Do not call setattr with a constant attribute value +# -W- codes are warnings +# W503 line break before binary operator +# W504 line break after binary operator +# W605 invalid escape sequence +# -E- codes are errors +# E501 line too long +# E731 do not assign a lambda expression, use a def +# -F- codes are errors +# F841 local variable 'X' is assigned to but never used +ignore = H102,H104,H105,H238,H404,H405, + B004,B005,B006,B009,B010, + W503,W504,W605, + E501,E731, + F841 [testenv:pylint] basepython = python2.7