starlingx
/
config
Code Issues Proposed changes

Merge "Fix Stevedore plugin usage for Debian OS" into f/centos8

This commit is contained in:
Zuul 2021-08-17 14:13:41 +00:00 committed by Gerrit Code Review
parent 579bc18fed 357e4dba76
commit 5677c883f4
3 changed files with 171 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
sysinv/sysinv/sysinv/sysinv/common/utils.py
View File

@ -46,6 +46,7 @@ import json
import keyring
import math
import os
import pathlib
import pwd
import random
import re
@ -56,9 +57,11 @@ import six
import socket
import stat
import string
import sys
import tempfile
import time
import tsconfig.tsconfig as tsc
import types
import uuid
import wsme
import yaml
@ -89,6 +92,15 @@ except ImportError:
SW_VERSION = "unknown"
if six.PY3:
USE_IMPORTLIB_METADATA_STDLIB = False
try:
import importlib.metadata
USE_IMPORTLIB_METADATA_STDLIB = True
except ImportError:
import importlib_metadata
utils_opts = [
cfg.StrOpt('rootwrap_config',
default="/etc/sysinv/rootwrap.conf",
@ -2891,3 +2903,117 @@ def TempDirectory():
shutil.rmtree(tmpdir)
except OSError as e:
LOG.error(_('Could not remove tmpdir: %s'), str(e))
def get_stevedore_major_version():
if six.PY2:
# Hardcode Stevedore 1.25.0 for CentOS7 that has Python2.
# Support for Python2 will be dropped soon, and this removed.
return 1
package = 'stevedore'
if USE_IMPORTLIB_METADATA_STDLIB:
distribution = importlib.metadata.distribution
else:
distribution = importlib_metadata.distribution
return int(distribution(package).version.split('.')[0])
def get_distribution_from_entry_point(entry_point):
"""
With Stevedore 3.0.0 the entry_point object was changed.
https://docs.openstack.org/releasenotes/stevedore/victoria.html
This affects some of our Stevedore based logic on Debian Bullseye which
currently uses Stevedore 3.2.2.
In Python3.9.2 used on Debian Bullseye the EntryPoint returned by
importlib does not hold a reference to a Distribution object.
https://bugs.python.org/issue42382
Determine the missing information by parsing all modules in Python3 envs.
This can be removed when Python will be patched or upgraded.
:param entry_point: An EntryPoint object
:return: A Distribution object
:raises exception.SysinvException: If distribution could not be found
"""
# Just a refactor on this path
if get_stevedore_major_version() < 3:
return entry_point.dist
if six.PY2:
raise exception.SysinvException(_(
"Python2 + Stevedore 3 and later support not implemented: "
"parsing modules in Python2 not implemented."))
loaded_entry_point = entry_point.load()
if isinstance(loaded_entry_point, types.ModuleType):
module_path = loaded_entry_point.__file__
else:
module_path = sys.modules[loaded_entry_point.__module__].__file__
if USE_IMPORTLIB_METADATA_STDLIB:
distributions = importlib.metadata.distributions
else:
distributions = importlib_metadata.distributions
for distribution in distributions():
try:
relative = pathlib.Path(module_path).relative_to(
distribution.locate_file("")
)
except ValueError:
pass
else:
if relative in distribution.files:
return distribution
raise exception.SysinvException(_(
"Distribution information for entry point {} "
"could not be found.".format(entry_point)))
def get_project_name_and_location_from_distribution(distribution):
"""
With Stevedore 3.0.0 the entry_point object was changed.
https://docs.openstack.org/releasenotes/stevedore/victoria.html
This affects some of our Stevedore based logic on Debian Bullseye which
currently uses Stevedore 3.2.2.
Determine the missing information by parsing the Distribution object.
:param distribution: A Distribution object
:return: Tuple of project name and project location. Location being
the parent of directory named <project name>
"""
# Just a refactor on this path
if get_stevedore_major_version() < 3:
return (distribution.project_name, distribution.location)
project_name = distribution.metadata.get('Name')
project_location = str(distribution._path.parent)
return (project_name, project_location)
def get_module_name_from_entry_point(entry_point):
"""
With Stevedore 3.0.0 the entry_point object was changed.
https://docs.openstack.org/releasenotes/stevedore/victoria.html
This affects some of our Stevedore based logic on Debian Bullseye which
currently uses Stevedore 3.2.2.
:param entry_point: An EntryPoint object
:return: Module name
:raises exception.SysinvException: If module name could not be found
"""
if 'module_name' in dir(entry_point):
return entry_point.module_name
elif 'module' in dir(entry_point):
return entry_point.module
raise exception.SysinvException(_(
"Module name for entry point {} "
"could not be determined.".format(entry_point)))

59
sysinv/sysinv/sysinv/sysinv/helm/helm.py
View File

@ -117,28 +117,44 @@ class HelmOperator(object):
def purge_cache_by_location(self, install_location):
"""Purge the stevedore entry point cache."""
for lifecycle_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE]:
if lifecycle_ep.dist.location == install_location:
lifecycle_distribution = utils.get_distribution_from_entry_point(lifecycle_ep)
(project_name, project_location) = \
utils.get_project_name_and_location_from_distribution(lifecycle_distribution)
if project_location == install_location:
extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE].remove(lifecycle_ep)
break
else:
LOG.info("Couldn't find endpoint distribution located at %s for "
"%s" % (install_location, lifecycle_ep.dist))
"%s" % (install_location, lifecycle_distribution))
for armada_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA]:
if armada_ep.dist.location == install_location:
armada_distribution = utils.get_distribution_from_entry_point(armada_ep)
(project_name, project_location) = \
utils.get_project_name_and_location_from_distribution(armada_distribution)
if project_location == install_location:
extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA].remove(armada_ep)
break
else:
LOG.info("Couldn't find endpoint distribution located at %s for "
"%s" % (install_location, armada_ep.dist))
"%s" % (install_location, armada_distribution))
for app_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]:
if app_ep.dist.location == install_location:
namespace = app_ep.module_name
app_distribution = utils.get_distribution_from_entry_point(app_ep)
(app_project_name, app_project_location) = \
utils.get_project_name_and_location_from_distribution(app_distribution)
if app_project_location == install_location:
namespace = utils.get_module_name_from_entry_point(app_ep)
purged_list = []
for helm_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[namespace]:
if helm_ep.dist.location != install_location:
helm_distribution = utils.get_distribution_from_entry_point(helm_ep)
(helm_project_name, helm_project_location) = \
utils.get_project_name_and_location_from_distribution(helm_distribution)
if helm_project_location != install_location:
purged_list.append(helm_ep)
if purged_list:
@ -152,7 +168,7 @@ class HelmOperator(object):
"""Purge the stevedore entry point cache."""
if self.STEVEDORE_APPS in extension.ExtensionManager.ENTRY_POINT_CACHE:
for entry_point in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]:
namespace = entry_point.module_name
namespace = utils.get_module_name_from_entry_point(entry_point)
try:
del extension.ExtensionManager.ENTRY_POINT_CACHE[namespace]
LOG.debug("Deleted entry points for %s." % namespace)
@ -201,10 +217,14 @@ class HelmOperator(object):
operator_name = operator.name
operators_dict[operator_name] = operator.obj
distribution = utils.get_distribution_from_entry_point(operator.entry_point)
(project_name, project_location) = \
utils.get_project_name_and_location_from_distribution(distribution)
# Extract distribution information for logging
dist_info_dict[operator_name] = {
'name': operator.entry_point.dist.project_name,
'location': operator.entry_point.dist.location,
'name': project_name,
'location': project_location,
}
return operators_dict
@ -240,10 +260,14 @@ class HelmOperator(object):
op_name = op.name
operators_dict[op_name] = op.obj
distribution = utils.get_distribution_from_entry_point(op.entry_point)
(project_name, project_location) = \
utils.get_project_name_and_location_from_distribution(distribution)
# Extract distribution information for logging
dist_info_dict[op_name] = {
'name': op.entry_point.dist.project_name,
'location': op.entry_point.dist.location,
'name': project_name,
'location': project_location,
}
# Provide some log feedback on plugins being used
@ -271,7 +295,8 @@ class HelmOperator(object):
on_load_failure_callback=suppress_stevedore_errors
)
for entry_point in helm_applications.list_entry_points():
helm_application_dict[entry_point.name] = entry_point.module_name
helm_application_dict[entry_point.name] = \
utils.get_module_name_from_entry_point(entry_point)
supported_helm_applications = {}
for name, namespace in helm_application_dict.items():
@ -280,10 +305,14 @@ class HelmOperator(object):
namespace=namespace, invoke_on_load=True, invoke_args=(self,))
sorted_helm_plugins = sorted(helm_plugins.extensions, key=lambda x: x.name)
for plugin in sorted_helm_plugins:
distribution = utils.get_distribution_from_entry_point(plugin.entry_point)
(project_name, project_location) = \
utils.get_project_name_and_location_from_distribution(distribution)
LOG.debug("%s: helm plugin %s loaded from %s - %s." % (name,
plugin.name,
plugin.entry_point.dist.project_name,
plugin.entry_point.dist.location))
project_name,
project_location))
plugin_name = plugin.name[HELM_PLUGIN_PREFIX_LENGTH:]
self.chart_operators.update({plugin_name: plugin.obj})

1
sysinv/sysinv/sysinv/test-requirements.txt
View File

@ -19,3 +19,4 @@ isort<5;python_version>="3.0"
pylint<2.1.0;python_version<"3.0" # GPLv2
pylint<2.4.0;python_version>="3.0" # GPLv2
pycryptodomex
pathlib;python_version<"3.0"