Fix python3 compatibility

- Use six.moves for urllib2
- Use six.moves.xmlrpc_client
- Use six.moves.zip
- Adapt gettextutils
- Fix set() order dismatch for python 2/3
- Change dict key to list

Test:
- Build RPM with applied changes.
- Installed simplex controller checked to see if sm-api was running.

Story: 2006796
Task: 42388

Signed-off-by: Charles Short <charles.short@windriver.com>
Change-Id: I29a81755f732b55f67321748604b2e5d951935c9
Signed-off-by: Mihnea Saracin <Mihnea.Saracin@windriver.com>
(cherry picked from commit cdc4757a46)

service-mgmt-api/sm-api/sm_api/api/controllers/v1/servicenode.py

@@ -24,10 +24,10 @@ import pecan
 from pecan import rest
 import wsme
 import six
+from six.moves import urllib
 from wsme import types as wtypes
 import wsmeext.pecan as wsme_pecan
 import socket
-import urllib2
 import json
 
 from sm_api.api.controllers.v1 import base
@@ -128,7 +128,7 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
 
     response = None
     try:
-        request_info = urllib2.Request(api_cmd)
+        request_info = urllib.request.Request(api_cmd)
         request_info.get_method = lambda: method
         if token:
             request_info.add_header("X-Auth-Token", token)
@@ -139,9 +139,9 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
                 request_info.add_header(header_type, header_value)
 
         if api_cmd_payload is not None:
-            request_info.add_data(api_cmd_payload)
+            request_info.data = api_cmd_payload
 
-        request = urllib2.urlopen(request_info, timeout=timeout)
+        request = urllib.request.urlopen(request_info, timeout=timeout)
         response = request.read()
 
         if response == "":
@@ -152,7 +152,7 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
 
         LOG.info("Response=%s" % response)
 
-    except urllib2.HTTPError as e:
+    except urllib.error.HTTPError as e:
         LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e))
         if hasattr(e, 'msg') and e.msg:
             response = json.loads(e.msg)
@@ -160,7 +160,7 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
             response = {}
 
         LOG.info("HTTPError response=%s" % (response))
-    except urllib2.URLError as urle:
+    except urllib.error.URLError as urle:
         LOG.debug("Connection refused")
 
     return response

service-mgmt-api/sm-api/sm_api/common/safe_utils.py

@@ -24,6 +24,7 @@
 """Utilities and helper functions that won't produce circular imports."""
 
 import inspect
+from six.moves import zip
 
 
 def getcallargs(function, *args, **kwargs):

service-mgmt-api/sm-api/sm_api/common/utils.py

@@ -47,6 +47,7 @@ from oslo_config import cfg
 
 from sm_api.common import exception
 from sm_api.openstack.common import log as logging
+from six.moves import range
 
 utils_opts = [
     cfg.StrOpt('rootwrap_config',
@@ -125,7 +126,7 @@ def execute(*cmd, **kwargs):
     if run_as_root and os.geteuid() != 0:
         cmd = ['sudo', 'sm_api-rootwrap', CONF.rootwrap_config] + list(cmd)
 
-    cmd = map(str, cmd)
+    cmd = [str(c) for c in cmd]
 
     while attempts > 0:
         attempts -= 1
@@ -146,7 +147,8 @@ def execute(*cmd, **kwargs):
                                    stderr=_PIPE,
                                    close_fds=close_fds,
                                    preexec_fn=preexec_fn,
-                                   shell=shell)
+                                   shell=shell,
+                                   universal_newlines=True)
             result = None
             if process_input is not None:
                 result = obj.communicate(process_input)
@@ -445,7 +447,7 @@ def file_open(*args, **kwargs):
           be able to provide a stub module that doesn't alter system
           state at all (for unit tests)
     """
-    return file(*args, **kwargs)
+    return open(*args, **kwargs)
 
 
 def hash_file(file_like_object):

service-mgmt-api/sm-api/sm_api/objects/base.py

@@ -333,7 +333,7 @@ class Sm_apiObject(object):
 
         NOTE(danms): May be removed in the future.
         """
-        for name in self.fields.keys() + self.obj_extra_fields:
+        for name in list(self.fields.keys()) + self.obj_extra_fields:
             if (hasattr(self, get_attrname(name)) or
                     name in self.obj_extra_fields):
                 yield name, getattr(self, name)

service-mgmt-api/sm-api/sm_api/openstack/common/config/generator.py

@@ -155,13 +155,13 @@ def _list_opts(obj):
         if is_opt(attr_obj):
             opts.append(attr_obj)
         elif (isinstance(attr_obj, list) and
-              all(map(lambda x: is_opt(x), attr_obj))):
+              all([is_opt(x) for x in attr_obj])):
             opts.extend(attr_obj)
 
     ret = {}
     for opt in opts:
         ret.setdefault(_guess_groups(opt, obj), []).append(opt)
-    return ret.items()
+    return list(ret.items())
 
 
 def print_group_opts(group, opts_by_module):

service-mgmt-api/sm-api/sm_api/openstack/common/context.py

@@ -79,7 +79,7 @@ def get_context_from_function_and_args(function, args, kwargs):
        know much about the function we're wrapping.
     """
 
-    for arg in itertools.chain(kwargs.values(), args):
+    for arg in itertools.chain(list(kwargs.values()), args):
         if isinstance(arg, RequestContext):
             return arg
 

service-mgmt-api/sm-api/sm_api/openstack/common/db/sqlalchemy/utils.py

@@ -28,6 +28,8 @@ import sqlalchemy
 
 from sm_api.openstack.common.gettextutils import _
 from sm_api.openstack.common import log as logging
+from six.moves import range
+from six.moves import zip
 
 
 LOG = logging.getLogger(__name__)

service-mgmt-api/sm-api/sm_api/openstack/common/fileutils.py

@@ -110,4 +110,4 @@ def file_open(*args, **kwargs):
     be able to provide a stub module that doesn't alter system
     state at all (for unit tests)
     """
-    return file(*args, **kwargs)
+    return open(*args, **kwargs)

service-mgmt-api/sm-api/sm_api/openstack/common/gettextutils.py

@@ -30,12 +30,17 @@ Usual usage in an openstack.common module:
 import gettext
 import os
 
+import six
+
 _localedir = os.environ.get('sm_api'.upper() + '_LOCALEDIR')
 _t = gettext.translation('sm_api', localedir=_localedir, fallback=True)
 
 
 def _(msg):
-    return _t.ugettext(msg)
+    if six.PY2:
+        return _t.ugettext(msg)
+    if six.PY3:
+        return _t.gettext(msg)
 
 
 def install(domain):
@@ -49,6 +54,10 @@ def install(domain):
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
     """
-    gettext.install(domain,
+    if six.PY2:
-                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+        gettext.install(domain,
-                    unicode=True)
+                        localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
+                        unicode=True)
+    if six.PY3:
+        gettext.install(domain,
+                        localedir=os.environ.get(domain.upper() + '_LOCALEDIR'))

service-mgmt-api/sm-api/sm_api/openstack/common/jsonutils.py

@@ -42,9 +42,9 @@ import functools
 import inspect
 import itertools
 import json
-import xmlrpclib
 
 import six
+import six.moves.xmlrpc_client as xmlrpclib
 
 from sm_api.openstack.common import timeutils
 

service-mgmt-api/sm-api/sm_api/openstack/common/log.py

@@ -34,7 +34,7 @@ It also allows setting of formatting information through conf.
 """
 
 from six.moves import configparser
-import cStringIO
+from six.moves import cStringIO as StringIO
 import inspect
 import itertools
 import logging
@@ -524,7 +524,7 @@ class ContextFormatter(logging.Formatter):
         if not record:
             return logging.Formatter.formatException(self, exc_info)
 
-        stringbuffer = cStringIO.StringIO()
+        stringbuffer = StringIO()
         traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
                                   None, stringbuffer)
         lines = stringbuffer.getvalue().split('\n')

service-mgmt-api/sm-api/sm_api/openstack/common/policy.py

@@ -62,14 +62,14 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import re
-import urllib
 
 import six
-import urllib2
 
 from sm_api.openstack.common.gettextutils import _
 from sm_api.openstack.common import jsonutils
 from sm_api.openstack.common import log as logging
+from six.moves import urllib
+from six.moves import range
 
 
 LOG = logging.getLogger(__name__)
@@ -758,8 +758,8 @@ class HttpCheck(Check):
         url = ('http:' + self.match) % target
         data = {'target': jsonutils.dumps(target),
                 'credentials': jsonutils.dumps(creds)}
-        post_data = urllib.urlencode(data)
+        post_data = urllib.parse.urlencode(data)
-        f = urllib2.urlopen(url, post_data)
+        f = urllib.request.urlopen(url, post_data)
         return f.read() == "True"
 
 

service-mgmt-api/sm-api/sm_api/openstack/common/processutils.py

@@ -138,7 +138,7 @@ def execute(*cmd, **kwargs):
                          'helper.'))
         cmd = shlex.split(root_helper) + list(cmd)
 
-    cmd = map(str, cmd)
+    cmd = [str(c) for c in cmd]
 
     while attempts > 0:
         attempts -= 1
@@ -159,7 +159,8 @@ def execute(*cmd, **kwargs):
                                    stderr=_PIPE,
                                    close_fds=close_fds,
                                    preexec_fn=preexec_fn,
-                                   shell=shell)
+                                   shell=shell,
+                                   universal_newlines=True)
             result = None
             if process_input is not None:
                 result = obj.communicate(process_input)

service-mgmt-api/sm-api/sm_api/openstack/common/rootwrap/cmd.py

@@ -119,7 +119,8 @@ def main():
                                    stdout=sys.stdout,
                                    stderr=sys.stderr,
                                    preexec_fn=_subprocess_setup,
-                                   env=filtermatch.get_environment(userargs))
+                                   env=filtermatch.get_environment(userargs),
+                                   universal_newlines=True)
             obj.wait()
             sys.exit(obj.returncode)
 

service-mgmt-api/sm-api/sm_api/openstack/common/rootwrap/filters.py

@@ -21,6 +21,7 @@
 
 import os
 import re
+from six.moves import zip
 
 
 class CommandFilter(object):

service-mgmt-api/sm-api/sm_api/openstack/common/rpc/common.py

@@ -507,7 +507,7 @@ def deserialize_msg(msg):
         return msg
 
     base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY)
-    if not all(map(lambda key: key in msg, base_envelope_keys)):
+    if not all([key in msg for key in base_envelope_keys]):
         #  See #1.b above.
         return msg
 

service-mgmt-api/sm-api/sm_api/openstack/common/rpc/impl_fake.py

@@ -31,7 +31,7 @@ import time
 
 import eventlet
 
-from six import reraise as raise_
+import six
 from sm_api.openstack.common.rpc import common as rpc_common
 
 CONSUMERS = {}
@@ -74,7 +74,7 @@ class Consumer(object):
                 # Caller might have called ctxt.reply() manually
                 for (reply, failure) in ctxt._response:
                     if failure:
-                        raise_(failure[0], failure[1], failure[2])
+                        six.reraise(failure[0], failure[1], failure[2])
                     res.append(reply)
                 # if ending not 'sent'...we might have more data to
                 # return from the function itself

service-mgmt-api/sm-api/sm_api/openstack/common/rpc/impl_zmq.py

@@ -37,6 +37,7 @@ from sm_api.openstack.common import jsonutils
 from sm_api.openstack.common import processutils as utils
 from sm_api.openstack.common.rpc import common as rpc_common
 from functools import reduce
+from six.moves import map
 
 zmq = importutils.try_import('eventlet.green.zmq')
 
@@ -158,7 +159,7 @@ class ZmqSocket(object):
         """Get socket type as string."""
         t_enum = ('PUSH', 'PULL', 'PUB', 'SUB', 'REP', 'REQ', 'ROUTER',
                   'DEALER')
-        return dict(map(lambda t: (getattr(zmq, t), t), t_enum))[self.type]
+        return dict([(getattr(zmq, t), t) for t in t_enum])[self.type]
 
     def subscribe(self, msg_filter):
         """Subscribe."""
@@ -227,14 +228,14 @@ class ZmqClient(object):
         msg_id = msg_id or 0
 
         if not envelope:
-            self.outq.send(map(bytes,
+            self.outq.send([bytes(x) for x in
-                               (msg_id, topic, 'cast', _serialize(data))))
+                           (msg_id, topic, 'cast', _serialize(data))])
             return
 
         rpc_envelope = rpc_common.serialize_msg(data[1], envelope)
-        zmq_msg = reduce(lambda x, y: x + y, rpc_envelope.items())
+        zmq_msg = reduce(lambda x, y: x + y, list(rpc_envelope.items()))
-        self.outq.send(map(bytes,
+        self.outq.send([bytes(x) for x in
-                           (msg_id, topic, 'impl_zmq_v2', data[0]) + zmq_msg))
+                       ((msg_id, topic, 'impl_zmq_v2', data[0]) + zmq_msg)])
 
     def close(self):
         self.outq.close()

service-mgmt-api/sm-api/sm_api/openstack/common/rpc/matchmaker_ring.py

@@ -107,7 +107,7 @@ class FanoutRingExchange(RingExchange):
                   "see ringfile") % (nkey, )
             )
             return []
-        return map(lambda x: (key + '.' + x, x), self.ring[nkey])
+        return [(key + '.' + x, x) for x in self.ring[nkey]]
 
 
 class MatchMakerRing(mm.MatchMakerBase):

service-mgmt-api/sm-api/sm_api/openstack/common/setup.py

@@ -125,11 +125,13 @@ def _run_shell_command(cmd, throw_on_error=False):
     if os.name == 'nt':
         output = subprocess.Popen(["cmd.exe", "/C", cmd],
                                   stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
+                                  stderr=subprocess.PIPE,
+                                  universal_newlines=True)
     else:
         output = subprocess.Popen(["/bin/sh", "-c", cmd],
                                   stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
+                                  stderr=subprocess.PIPE,
+                                  universal_newlines=True)
     out = output.communicate()
     if output.returncode and throw_on_error:
         raise Exception("%s returned %d" % cmd, output.returncode)

service-mgmt-client/sm-client/sm_client/common/http.py

@@ -18,7 +18,7 @@
 
 
 import copy
-import httplib
+import six.moves.http_client
 import logging
 import os
 import socket
@@ -66,7 +66,7 @@ class HTTPClient(object):
             _kwargs['key_file'] = kwargs.get('key_file', None)
             _kwargs['insecure'] = kwargs.get('insecure', False)
         elif parts.scheme == 'http':
-            _class = httplib.HTTPConnection
+            _class = six.moves.http_client.HTTPConnection
         else:
             msg = 'Unsupported scheme: %s' % parts.scheme
             raise exc.InvalidEndpoint(msg)
@@ -78,7 +78,7 @@ class HTTPClient(object):
         try:
             return _class(*self.connection_params[1][0:2],
                           **self.connection_params[2])
-        except httplib.InvalidURL:
+        except six.moves.http_client.InvalidURL:
             raise exc.InvalidEndpoint()
 
     def log_curl_request(self, method, url, kwargs):
@@ -216,7 +216,7 @@ class HTTPClient(object):
         return self._http_request(url, method, **kwargs)
 
 
-class VerifiedHTTPSConnection(httplib.HTTPSConnection):
+class VerifiedHTTPSConnection(six.moves.http_client.HTTPSConnection):
     """httplib-compatibile connection using client-side SSL authentication
 
     :see http://code.activestate.com/recipes/
@@ -225,8 +225,8 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection):
 
     def __init__(self, host, port, key_file=None, cert_file=None,
                  ca_file=None, timeout=None, insecure=False):
-        httplib.HTTPSConnection.__init__(self, host, port, key_file=key_file,
+        six.moves.http_client.HTTPSConnection.__init__(self, host, port, key_file=key_file,
-                                         cert_file=cert_file)
+                                                       cert_file=cert_file)
         self.key_file = key_file
         self.cert_file = cert_file
         if ca_file is not None:

service-mgmt-client/sm-client/sm_client/common/utils.py

@@ -28,6 +28,7 @@ import six
 
 from sm_client import exc
 from sm_client.openstack.common import importutils
+from six.moves import zip
 
 
 class HelpFormatter(argparse.HelpFormatter):

service-mgmt-client/sm-client/sm_client/openstack/common/config/generator.py

@@ -154,13 +154,13 @@ def _list_opts(obj):
         if is_opt(attr_obj):
             opts.append(attr_obj)
         elif (isinstance(attr_obj, list) and
-              all(map(lambda x: is_opt(x), attr_obj))):
+              all([is_opt(x) for x in attr_obj])):
             opts.extend(attr_obj)
 
     ret = {}
     for opt in opts:
         ret.setdefault(_guess_groups(opt, obj), []).append(opt)
-    return ret.items()
+    return list(ret.items())
 
 
 def print_group_opts(group, opts_by_module):

service-mgmt-client/sm-client/tools_junk/install_venv_common.py

@@ -67,7 +67,7 @@ class InstallVenv(object):
         else:
             stdout = None
 
-        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
+        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout, universal_newlines=True)
         output = proc.communicate()[0]
         if check_exit_code and proc.returncode != 0:
             self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)

service-mgmt-tools/sm-tools/sm_tools/sm_action.py

@@ -8,9 +8,9 @@ import sys
 import argparse
 import sqlite3
 
-from sm_api_msg_utils import restart_service as restart_service
+from sm_tools.sm_api_msg_utils import restart_service as restart_service
-from sm_api_msg_utils import restart_service_safe as restart_service_safe
+from sm_tools.sm_api_msg_utils import restart_service_safe as restart_service_safe
-from sm_api_msg_utils import database_running_name as database_name
+from sm_tools.sm_api_msg_utils import database_running_name as database_name
 
 
 def main():

service-mgmt-tools/sm-tools/sm_tools/sm_api_msg_utils.py

@@ -39,7 +39,7 @@ def _send_msg_to_sm(sm_api_msg):
     s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
     try:
         s.setblocking(True)
-        s.sendto(sm_api_msg, SM_API_SERVER_ADDR)
+        s.sendto(sm_api_msg.encode("ascii", "ignore"), SM_API_SERVER_ADDR)
         time.sleep(1)
 
     except socket.error as e:

service-mgmt-tools/sm-tools/sm_tools/sm_configure.py

@@ -7,7 +7,8 @@ import sys
 import argparse
 import sqlite3
 from netaddr import IPNetwork
-from sm_api_msg_utils import database_name as database_name
+from sm_tools.sm_api_msg_utils import database_name as database_name
+from six.moves import range
 
 cpe_duplex = "duplex"
 cpe_duplex_direct = "duplex-direct"

service-mgmt-tools/sm-tools/sm_tools/sm_provision.py

@@ -7,8 +7,8 @@ import os
 import sys
 import argparse
 import sqlite3
-from sm_api_msg_utils import provision_service
+from sm_tools.sm_api_msg_utils import provision_service
-from sm_api_msg_utils import deprovision_service
+from sm_tools.sm_api_msg_utils import deprovision_service
 
 database_name = "/var/lib/sm/sm.db"
 runtime_db_name = "/var/run/sm/sm.db"