679 lines
20 KiB
Python
679 lines
20 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright 2010 United States Government as represented by the
|
|
# Administrator of the National Aeronautics and Space Administration.
|
|
# Copyright 2011 Justin Santa Barbara
|
|
# Copyright (c) 2012 NTT DOCOMO, INC.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
# Copyright (c) 2013-2014 Wind River Systems, Inc.
|
|
#
|
|
|
|
|
|
"""Utilities and helper functions."""
|
|
|
|
import contextlib
|
|
import errno
|
|
import hashlib
|
|
import json
|
|
import os
|
|
#import paramiko
|
|
import random
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import six
|
|
import socket
|
|
import tempfile
|
|
import uuid
|
|
|
|
from eventlet.green import subprocess
|
|
from eventlet import greenthread
|
|
import netaddr
|
|
|
|
from oslo_config import cfg
|
|
|
|
from sm_api.common import exception
|
|
from sm_api.openstack.common import log as logging
|
|
|
|
utils_opts = [
|
|
cfg.StrOpt('rootwrap_config',
|
|
default="/etc/sm_api/rootwrap.conf",
|
|
help='Path to the rootwrap configuration file to use for '
|
|
'running commands as root'),
|
|
cfg.StrOpt('tempdir',
|
|
default=None,
|
|
help='Explicitly specify the temporary working directory'),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(utils_opts)
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# Used for looking up extensions of text
|
|
# to their 'multiplied' byte amount
|
|
BYTE_MULTIPLIERS = {
|
|
'': 1,
|
|
't': 1024 ** 4,
|
|
'g': 1024 ** 3,
|
|
'm': 1024 ** 2,
|
|
'k': 1024,
|
|
}
|
|
|
|
|
|
def _subprocess_setup():
|
|
# Python installs a SIGPIPE handler by default. This is usually not what
|
|
# non-Python subprocesses expect.
|
|
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
|
|
|
|
|
|
def execute(*cmd, **kwargs):
|
|
"""Helper method to execute command with optional retry.
|
|
|
|
If you add a run_as_root=True command, don't forget to add the
|
|
corresponding filter to etc/sm_api/rootwrap.d !
|
|
|
|
:param cmd: Passed to subprocess.Popen.
|
|
:param process_input: Send to opened process.
|
|
:param check_exit_code: Single bool, int, or list of allowed exit
|
|
codes. Defaults to [0]. Raise
|
|
exception.ProcessExecutionError unless
|
|
program exits with one of these code.
|
|
:param delay_on_retry: True | False. Defaults to True. If set to
|
|
True, wait a short amount of time
|
|
before retrying.
|
|
:param attempts: How many times to retry cmd.
|
|
:param run_as_root: True | False. Defaults to False. If set to True,
|
|
the command is run with rootwrap.
|
|
|
|
:raises exception.SmApiException: on receiving unknown arguments
|
|
:raises exception.ProcessExecutionError:
|
|
|
|
:returns: a tuple, (stdout, stderr) from the spawned process, or None if
|
|
the command fails.
|
|
"""
|
|
process_input = kwargs.pop('process_input', None)
|
|
check_exit_code = kwargs.pop('check_exit_code', [0])
|
|
ignore_exit_code = False
|
|
if isinstance(check_exit_code, bool):
|
|
ignore_exit_code = not check_exit_code
|
|
check_exit_code = [0]
|
|
elif isinstance(check_exit_code, int):
|
|
check_exit_code = [check_exit_code]
|
|
delay_on_retry = kwargs.pop('delay_on_retry', True)
|
|
attempts = kwargs.pop('attempts', 1)
|
|
run_as_root = kwargs.pop('run_as_root', False)
|
|
shell = kwargs.pop('shell', False)
|
|
|
|
if len(kwargs):
|
|
raise exception.SmApiException(_('Got unknown keyword args '
|
|
'to utils.execute: %r') % kwargs)
|
|
|
|
if run_as_root and os.geteuid() != 0:
|
|
cmd = ['sudo', 'sm_api-rootwrap', CONF.rootwrap_config] + list(cmd)
|
|
|
|
cmd = map(str, cmd)
|
|
|
|
while attempts > 0:
|
|
attempts -= 1
|
|
try:
|
|
LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd))
|
|
_PIPE = subprocess.PIPE # pylint: disable=E1101
|
|
|
|
if os.name == 'nt':
|
|
preexec_fn = None
|
|
close_fds = False
|
|
else:
|
|
preexec_fn = _subprocess_setup
|
|
close_fds = True
|
|
|
|
obj = subprocess.Popen(cmd,
|
|
stdin=_PIPE,
|
|
stdout=_PIPE,
|
|
stderr=_PIPE,
|
|
close_fds=close_fds,
|
|
preexec_fn=preexec_fn,
|
|
shell=shell)
|
|
result = None
|
|
if process_input is not None:
|
|
result = obj.communicate(process_input)
|
|
else:
|
|
result = obj.communicate()
|
|
obj.stdin.close() # pylint: disable=E1101
|
|
_returncode = obj.returncode # pylint: disable=E1101
|
|
LOG.debug(_('Result was %s') % _returncode)
|
|
if not ignore_exit_code and _returncode not in check_exit_code:
|
|
(stdout, stderr) = result
|
|
raise exception.ProcessExecutionError(
|
|
exit_code=_returncode,
|
|
stdout=stdout,
|
|
stderr=stderr,
|
|
cmd=' '.join(cmd))
|
|
return result
|
|
except exception.ProcessExecutionError:
|
|
if not attempts:
|
|
raise
|
|
else:
|
|
LOG.debug(_('%r failed. Retrying.'), cmd)
|
|
if delay_on_retry:
|
|
greenthread.sleep(random.randint(20, 200) / 100.0)
|
|
finally:
|
|
# NOTE(termie): this appears to be necessary to let the subprocess
|
|
# call clean something up in between calls, without
|
|
# it two execute calls in a row hangs the second one
|
|
greenthread.sleep(0)
|
|
|
|
|
|
def trycmd(*args, **kwargs):
|
|
"""A wrapper around execute() to more easily handle warnings and errors.
|
|
|
|
Returns an (out, err) tuple of strings containing the output of
|
|
the command's stdout and stderr. If 'err' is not empty then the
|
|
command can be considered to have failed.
|
|
|
|
:discard_warnings True | False. Defaults to False. If set to True,
|
|
then for succeeding commands, stderr is cleared
|
|
|
|
"""
|
|
discard_warnings = kwargs.pop('discard_warnings', False)
|
|
|
|
try:
|
|
out, err = execute(*args, **kwargs)
|
|
failed = False
|
|
except exception.ProcessExecutionError as exn:
|
|
out, err = '', str(exn)
|
|
failed = True
|
|
|
|
if not failed and discard_warnings and err:
|
|
# Handle commands that output to stderr but otherwise succeed
|
|
err = ''
|
|
|
|
return out, err
|
|
|
|
|
|
# def ssh_connect(connection):
|
|
# """Method to connect to a remote system using ssh protocol.
|
|
#
|
|
# :param connection: a dict of connection parameters.
|
|
# :returns: paramiko.SSHClient -- an active ssh connection.
|
|
# :raises: SSHConnectFailed
|
|
#
|
|
# """
|
|
# try:
|
|
# ssh = paramiko.SSHClient()
|
|
# ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
# ssh.connect(connection.get('host'),
|
|
# username=connection.get('username'),
|
|
# password=connection.get('password', None),
|
|
# port=connection.get('port', 22),
|
|
# key_filename=connection.get('key_filename', None),
|
|
# timeout=connection.get('timeout', 10))
|
|
#
|
|
# # send TCP keepalive packets every 20 seconds
|
|
# ssh.get_transport().set_keepalive(20)
|
|
# except Exception:
|
|
# raise exception.SSHConnectFailed(host=connection.get('host'))
|
|
#
|
|
# return ssh
|
|
|
|
|
|
def generate_uid(topic, size=8):
|
|
characters = '01234567890abcdefghijklmnopqrstuvwxyz'
|
|
choices = [random.choice(characters) for _x in xrange(size)]
|
|
return '%s-%s' % (topic, ''.join(choices))
|
|
|
|
|
|
def random_alnum(size=32):
|
|
characters = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
|
return ''.join(random.choice(characters) for _ in xrange(size))
|
|
|
|
|
|
class LazyPluggable(object):
|
|
"""A pluggable backend loaded lazily based on some value."""
|
|
|
|
def __init__(self, pivot, config_group=None, **backends):
|
|
self.__backends = backends
|
|
self.__pivot = pivot
|
|
self.__backend = None
|
|
self.__config_group = config_group
|
|
|
|
def __get_backend(self):
|
|
if not self.__backend:
|
|
if self.__config_group is None:
|
|
backend_name = CONF[self.__pivot]
|
|
else:
|
|
backend_name = CONF[self.__config_group][self.__pivot]
|
|
if backend_name not in self.__backends:
|
|
msg = _('Invalid backend: %s') % backend_name
|
|
raise exception.SmApiException(msg)
|
|
|
|
backend = self.__backends[backend_name]
|
|
if isinstance(backend, tuple):
|
|
name = backend[0]
|
|
fromlist = backend[1]
|
|
else:
|
|
name = backend
|
|
fromlist = backend
|
|
|
|
self.__backend = __import__(name, None, None, fromlist)
|
|
return self.__backend
|
|
|
|
def __getattr__(self, key):
|
|
backend = self.__get_backend()
|
|
return getattr(backend, key)
|
|
|
|
|
|
def delete_if_exists(pathname):
|
|
"""delete a file, but ignore file not found error."""
|
|
|
|
try:
|
|
os.unlink(pathname)
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
return
|
|
else:
|
|
raise
|
|
|
|
|
|
def is_int_like(val):
|
|
"""Check if a value looks like an int."""
|
|
try:
|
|
return str(int(val)) == str(val)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def is_valid_boolstr(val):
|
|
"""Check if the provided string is a valid bool string or not."""
|
|
boolstrs = ('true', 'false', 'yes', 'no', 'y', 'n', '1', '0')
|
|
return str(val).lower() in boolstrs
|
|
|
|
|
|
def is_valid_mac(address):
|
|
"""Verify the format of a MAC addres."""
|
|
m = "[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$"
|
|
if isinstance(address, six.string_types) and re.match(m, address.lower()):
|
|
return True
|
|
return False
|
|
|
|
|
|
def validate_and_normalize_mac(address):
|
|
"""Validate a MAC address and return normalized form.
|
|
|
|
Checks whether the supplied MAC address is formally correct and
|
|
normalize it to all lower case.
|
|
|
|
:param address: MAC address to be validated and normalized.
|
|
:returns: Normalized and validated MAC address.
|
|
:raises: InvalidMAC If the MAC address is not valid.
|
|
|
|
"""
|
|
if not is_valid_mac(address):
|
|
raise exception.InvalidMAC(mac=address)
|
|
return address.lower()
|
|
|
|
|
|
def is_valid_ipv4(address):
|
|
"""Verify that address represents a valid IPv4 address."""
|
|
try:
|
|
return netaddr.valid_ipv4(address)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def is_valid_ipv6(address):
|
|
try:
|
|
return netaddr.valid_ipv6(address)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def is_valid_ipv6_cidr(address):
|
|
try:
|
|
str(netaddr.IPNetwork(address, version=6).cidr)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def get_shortened_ipv6(address):
|
|
addr = netaddr.IPAddress(address, version=6)
|
|
return str(addr.ipv6())
|
|
|
|
|
|
def get_shortened_ipv6_cidr(address):
|
|
net = netaddr.IPNetwork(address, version=6)
|
|
return str(net.cidr)
|
|
|
|
|
|
def is_valid_cidr(address):
|
|
"""Check if the provided ipv4 or ipv6 address is a valid CIDR address."""
|
|
try:
|
|
# Validate the correct CIDR Address
|
|
netaddr.IPNetwork(address)
|
|
except netaddr.core.AddrFormatError:
|
|
return False
|
|
except UnboundLocalError:
|
|
# NOTE(MotoKen): work around bug in netaddr 0.7.5 (see detail in
|
|
# https://github.com/drkjam/netaddr/issues/2)
|
|
return False
|
|
|
|
# Prior validation partially verify /xx part
|
|
# Verify it here
|
|
ip_segment = address.split('/')
|
|
|
|
if (len(ip_segment) <= 1 or
|
|
ip_segment[1] == ''):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def get_ip_version(network):
|
|
"""Returns the IP version of a network (IPv4 or IPv6).
|
|
|
|
:raises: AddrFormatError if invalid network.
|
|
"""
|
|
if netaddr.IPNetwork(network).version == 6:
|
|
return "IPv6"
|
|
elif netaddr.IPNetwork(network).version == 4:
|
|
return "IPv4"
|
|
|
|
|
|
def convert_to_list_dict(lst, label):
|
|
"""Convert a value or list into a list of dicts."""
|
|
if not lst:
|
|
return None
|
|
if not isinstance(lst, list):
|
|
lst = [lst]
|
|
return [{label: x} for x in lst]
|
|
|
|
|
|
def sanitize_hostname(hostname):
|
|
"""Return a hostname which conforms to RFC-952 and RFC-1123 specs."""
|
|
if isinstance(hostname, unicode):
|
|
hostname = hostname.encode('latin-1', 'ignore')
|
|
|
|
hostname = re.sub('[ _]', '-', hostname)
|
|
hostname = re.sub('[^\w.-]+', '', hostname)
|
|
hostname = hostname.lower()
|
|
hostname = hostname.strip('.-')
|
|
|
|
return hostname
|
|
|
|
|
|
def read_cached_file(filename, cache_info, reload_func=None):
|
|
"""Read from a file if it has been modified.
|
|
|
|
:param cache_info: dictionary to hold opaque cache.
|
|
:param reload_func: optional function to be called with data when
|
|
file is reloaded due to a modification.
|
|
|
|
:returns: data from file
|
|
|
|
"""
|
|
mtime = os.path.getmtime(filename)
|
|
if not cache_info or mtime != cache_info.get('mtime'):
|
|
LOG.debug(_("Reloading cached file %s") % filename)
|
|
with open(filename) as fap:
|
|
cache_info['data'] = fap.read()
|
|
cache_info['mtime'] = mtime
|
|
if reload_func:
|
|
reload_func(cache_info['data'])
|
|
return cache_info['data']
|
|
|
|
|
|
def file_open(*args, **kwargs):
|
|
"""Open file
|
|
|
|
see built-in file() documentation for more details
|
|
|
|
Note: The reason this is kept in a separate module is to easily
|
|
be able to provide a stub module that doesn't alter system
|
|
state at all (for unit tests)
|
|
"""
|
|
return file(*args, **kwargs)
|
|
|
|
|
|
def hash_file(file_like_object):
|
|
"""Generate a hash for the contents of a file."""
|
|
checksum = hashlib.sha1()
|
|
for chunk in iter(lambda: file_like_object.read(32768), b''):
|
|
checksum.update(chunk)
|
|
return checksum.hexdigest()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def temporary_mutation(obj, **kwargs):
|
|
"""Temporarily set the attr on a particular object to a given value then
|
|
revert when finished.
|
|
|
|
One use of this is to temporarily set the read_deleted flag on a context
|
|
object:
|
|
|
|
with temporary_mutation(context, read_deleted="yes"):
|
|
do_something_that_needed_deleted_objects()
|
|
"""
|
|
def is_dict_like(thing):
|
|
return hasattr(thing, 'has_key')
|
|
|
|
def get(thing, attr, default):
|
|
if is_dict_like(thing):
|
|
return thing.get(attr, default)
|
|
else:
|
|
return getattr(thing, attr, default)
|
|
|
|
def set_value(thing, attr, val):
|
|
if is_dict_like(thing):
|
|
thing[attr] = val
|
|
else:
|
|
setattr(thing, attr, val)
|
|
|
|
def delete(thing, attr):
|
|
if is_dict_like(thing):
|
|
del thing[attr]
|
|
else:
|
|
delattr(thing, attr)
|
|
|
|
NOT_PRESENT = object()
|
|
|
|
old_values = {}
|
|
for attr, new_value in kwargs.items():
|
|
old_values[attr] = get(obj, attr, NOT_PRESENT)
|
|
set_value(obj, attr, new_value)
|
|
|
|
try:
|
|
yield
|
|
finally:
|
|
for attr, old_value in old_values.items():
|
|
if old_value is NOT_PRESENT:
|
|
delete(obj, attr)
|
|
else:
|
|
set_value(obj, attr, old_value)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def tempdir(**kwargs):
|
|
tempfile.tempdir = CONF.tempdir
|
|
tmpdir = tempfile.mkdtemp(**kwargs)
|
|
try:
|
|
yield tmpdir
|
|
finally:
|
|
try:
|
|
shutil.rmtree(tmpdir)
|
|
except OSError as e:
|
|
LOG.error(_('Could not remove tmpdir: %s'), str(e))
|
|
|
|
|
|
def mkfs(fs, path, label=None):
|
|
"""Format a file or block device
|
|
|
|
:param fs: Filesystem type (examples include 'swap', 'ext3', 'ext4'
|
|
'btrfs', etc.)
|
|
:param path: Path to file or block device to format
|
|
:param label: Volume label to use
|
|
"""
|
|
if fs == 'swap':
|
|
args = ['mkswap']
|
|
else:
|
|
args = ['mkfs', '-t', fs]
|
|
#add -F to force no interactive execute on non-block device.
|
|
if fs in ('ext3', 'ext4'):
|
|
args.extend(['-F'])
|
|
if label:
|
|
if fs in ('msdos', 'vfat'):
|
|
label_opt = '-n'
|
|
else:
|
|
label_opt = '-L'
|
|
args.extend([label_opt, label])
|
|
args.append(path)
|
|
execute(*args)
|
|
|
|
|
|
# TODO(deva): Make these work in SmApi.
|
|
# Either copy nova/virt/utils (bad),
|
|
# or reimplement as a common lib,
|
|
# or make a driver that doesn't need to do this.
|
|
#
|
|
#def cache_image(context, target, image_id, user_id, project_id):
|
|
# if not os.path.exists(target):
|
|
# libvirt_utils.fetch_image(context, target, image_id,
|
|
# user_id, project_id)
|
|
#
|
|
#
|
|
#def inject_into_image(image, key, net, metadata, admin_password,
|
|
# files, partition, use_cow=False):
|
|
# try:
|
|
# disk_api.inject_data(image, key, net, metadata, admin_password,
|
|
# files, partition, use_cow)
|
|
# except Exception as e:
|
|
# LOG.warn(_("Failed to inject data into image %(image)s. "
|
|
# "Error: %(e)s") % locals())
|
|
|
|
|
|
def unlink_without_raise(path):
|
|
try:
|
|
os.unlink(path)
|
|
except OSError as e:
|
|
if e.errno == errno.ENOENT:
|
|
return
|
|
else:
|
|
LOG.warn(_("Failed to unlink %(path)s, error: %(e)s") %
|
|
{'path': path, 'e': e})
|
|
|
|
|
|
def rmtree_without_raise(path):
|
|
try:
|
|
if os.path.isdir(path):
|
|
shutil.rmtree(path)
|
|
except OSError as e:
|
|
LOG.warn(_("Failed to remove dir %(path)s, error: %(e)s") %
|
|
{'path': path, 'e': e})
|
|
|
|
|
|
def write_to_file(path, contents):
|
|
with open(path, 'w') as f:
|
|
f.write(contents)
|
|
|
|
|
|
def create_link_without_raise(source, link):
|
|
try:
|
|
os.symlink(source, link)
|
|
except OSError as e:
|
|
if e.errno == errno.EEXIST:
|
|
return
|
|
else:
|
|
LOG.warn(_("Failed to create symlink from %(source)s to %(link)s"
|
|
", error: %(e)s") %
|
|
{'source': source, 'link': link, 'e': e})
|
|
|
|
|
|
def safe_rstrip(value, chars=None):
|
|
"""Removes trailing characters from a string if that does not make it empty
|
|
|
|
:param value: A string value that will be stripped.
|
|
:param chars: Characters to remove.
|
|
:return: Stripped value.
|
|
|
|
"""
|
|
if not isinstance(value, six.string_types):
|
|
LOG.warn(_("Failed to remove trailing character. Returning original "
|
|
"object. Supplied object is not a string: %s,") % value)
|
|
return value
|
|
|
|
return value.rstrip(chars) or value
|
|
|
|
|
|
def generate_uuid():
|
|
return str(uuid.uuid4())
|
|
|
|
|
|
def is_uuid_like(val):
|
|
"""Returns validation of a value as a UUID.
|
|
|
|
For our purposes, a UUID is a canonical form string:
|
|
aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
|
|
|
|
"""
|
|
try:
|
|
return str(uuid.UUID(val)) == val
|
|
except (TypeError, ValueError, AttributeError):
|
|
return False
|
|
|
|
|
|
def removekey(d, key):
|
|
r = dict(d)
|
|
del r[key]
|
|
return r
|
|
|
|
|
|
def notify_mtc_and_recv(mtc_address, mtc_port, idict):
|
|
mtc_response_dict = {}
|
|
mtc_response_dict['status'] = None
|
|
|
|
serialized_idict = json.dumps(idict)
|
|
|
|
# notify mtc this ihost has been added
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
try:
|
|
s.setblocking(1) # blocking, timeout must be specified
|
|
s.settimeout(6) # give mtc a few secs to respond
|
|
s.connect((mtc_address, mtc_port))
|
|
LOG.warning("Mtc Command : %s" % serialized_idict)
|
|
s.sendall(serialized_idict)
|
|
|
|
mtc_response = s.recv(1024) # check if mtc allows
|
|
try:
|
|
mtc_response_dict = json.loads(mtc_response)
|
|
LOG.warning("Mtc Response: %s" % mtc_response_dict)
|
|
except:
|
|
LOG.exception("Mtc Response Error: %s" % mtc_response)
|
|
pass
|
|
|
|
except socket.error, e:
|
|
LOG.exception(_("Socket Error: %s on %s:%s for %s") % (e,
|
|
mtc_address, mtc_port, serialized_idict))
|
|
# if e not in [errno.EWOULDBLOCK, errno.EINTR]:
|
|
# raise exception.CommunicationError(_(
|
|
# "Socket error: address=%s port=%s error=%s ") % (
|
|
# self._mtc_address, self._mtc_port, e))
|
|
pass
|
|
|
|
finally:
|
|
s.close()
|
|
|
|
return mtc_response_dict
|