diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 747560b948..31e6b3b888 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -46,6 +46,7 @@ import json import keyring import math import os +import pathlib import pwd import random import re @@ -56,9 +57,11 @@ import six import socket import stat import string +import sys import tempfile import time import tsconfig.tsconfig as tsc +import types import uuid import wsme import yaml @@ -89,6 +92,15 @@ except ImportError: SW_VERSION = "unknown" +if six.PY3: + USE_IMPORTLIB_METADATA_STDLIB = False + try: + import importlib.metadata + USE_IMPORTLIB_METADATA_STDLIB = True + except ImportError: + import importlib_metadata + + utils_opts = [ cfg.StrOpt('rootwrap_config', default="/etc/sysinv/rootwrap.conf", @@ -2891,3 +2903,117 @@ def TempDirectory(): shutil.rmtree(tmpdir) except OSError as e: LOG.error(_('Could not remove tmpdir: %s'), str(e)) + + +def get_stevedore_major_version(): + if six.PY2: + # Hardcode Stevedore 1.25.0 for CentOS7 that has Python2. + # Support for Python2 will be dropped soon, and this removed. + return 1 + + package = 'stevedore' + if USE_IMPORTLIB_METADATA_STDLIB: + distribution = importlib.metadata.distribution + else: + distribution = importlib_metadata.distribution + + return int(distribution(package).version.split('.')[0]) + + +def get_distribution_from_entry_point(entry_point): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + In Python3.9.2 used on Debian Bullseye the EntryPoint returned by + importlib does not hold a reference to a Distribution object. + https://bugs.python.org/issue42382 + + Determine the missing information by parsing all modules in Python3 envs. + This can be removed when Python will be patched or upgraded. + + :param entry_point: An EntryPoint object + :return: A Distribution object + :raises exception.SysinvException: If distribution could not be found + """ + # Just a refactor on this path + if get_stevedore_major_version() < 3: + return entry_point.dist + + if six.PY2: + raise exception.SysinvException(_( + "Python2 + Stevedore 3 and later support not implemented: " + "parsing modules in Python2 not implemented.")) + + loaded_entry_point = entry_point.load() + if isinstance(loaded_entry_point, types.ModuleType): + module_path = loaded_entry_point.__file__ + else: + module_path = sys.modules[loaded_entry_point.__module__].__file__ + if USE_IMPORTLIB_METADATA_STDLIB: + distributions = importlib.metadata.distributions + else: + distributions = importlib_metadata.distributions + + for distribution in distributions(): + try: + relative = pathlib.Path(module_path).relative_to( + distribution.locate_file("") + ) + except ValueError: + pass + else: + if relative in distribution.files: + return distribution + + raise exception.SysinvException(_( + "Distribution information for entry point {} " + "could not be found.".format(entry_point))) + + +def get_project_name_and_location_from_distribution(distribution): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + Determine the missing information by parsing the Distribution object. + + :param distribution: A Distribution object + :return: Tuple of project name and project location. Location being + the parent of directory named + """ + # Just a refactor on this path + if get_stevedore_major_version() < 3: + return (distribution.project_name, distribution.location) + + project_name = distribution.metadata.get('Name') + project_location = str(distribution._path.parent) + return (project_name, project_location) + + +def get_module_name_from_entry_point(entry_point): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + :param entry_point: An EntryPoint object + :return: Module name + :raises exception.SysinvException: If module name could not be found + """ + if 'module_name' in dir(entry_point): + return entry_point.module_name + elif 'module' in dir(entry_point): + return entry_point.module + + raise exception.SysinvException(_( + "Module name for entry point {} " + "could not be determined.".format(entry_point))) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/helm.py b/sysinv/sysinv/sysinv/sysinv/helm/helm.py index 6af62fa879..f4a85033d5 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/helm.py @@ -117,28 +117,44 @@ class HelmOperator(object): def purge_cache_by_location(self, install_location): """Purge the stevedore entry point cache.""" for lifecycle_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE]: - if lifecycle_ep.dist.location == install_location: + lifecycle_distribution = utils.get_distribution_from_entry_point(lifecycle_ep) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(lifecycle_distribution) + + if project_location == install_location: extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE].remove(lifecycle_ep) break else: LOG.info("Couldn't find endpoint distribution located at %s for " - "%s" % (install_location, lifecycle_ep.dist)) + "%s" % (install_location, lifecycle_distribution)) for armada_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA]: - if armada_ep.dist.location == install_location: + armada_distribution = utils.get_distribution_from_entry_point(armada_ep) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(armada_distribution) + + if project_location == install_location: extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA].remove(armada_ep) break else: LOG.info("Couldn't find endpoint distribution located at %s for " - "%s" % (install_location, armada_ep.dist)) + "%s" % (install_location, armada_distribution)) for app_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]: - if app_ep.dist.location == install_location: - namespace = app_ep.module_name + app_distribution = utils.get_distribution_from_entry_point(app_ep) + (app_project_name, app_project_location) = \ + utils.get_project_name_and_location_from_distribution(app_distribution) + + if app_project_location == install_location: + namespace = utils.get_module_name_from_entry_point(app_ep) purged_list = [] for helm_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[namespace]: - if helm_ep.dist.location != install_location: + helm_distribution = utils.get_distribution_from_entry_point(helm_ep) + (helm_project_name, helm_project_location) = \ + utils.get_project_name_and_location_from_distribution(helm_distribution) + + if helm_project_location != install_location: purged_list.append(helm_ep) if purged_list: @@ -152,7 +168,7 @@ class HelmOperator(object): """Purge the stevedore entry point cache.""" if self.STEVEDORE_APPS in extension.ExtensionManager.ENTRY_POINT_CACHE: for entry_point in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]: - namespace = entry_point.module_name + namespace = utils.get_module_name_from_entry_point(entry_point) try: del extension.ExtensionManager.ENTRY_POINT_CACHE[namespace] LOG.debug("Deleted entry points for %s." % namespace) @@ -201,10 +217,14 @@ class HelmOperator(object): operator_name = operator.name operators_dict[operator_name] = operator.obj + distribution = utils.get_distribution_from_entry_point(operator.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + # Extract distribution information for logging dist_info_dict[operator_name] = { - 'name': operator.entry_point.dist.project_name, - 'location': operator.entry_point.dist.location, + 'name': project_name, + 'location': project_location, } return operators_dict @@ -240,10 +260,14 @@ class HelmOperator(object): op_name = op.name operators_dict[op_name] = op.obj + distribution = utils.get_distribution_from_entry_point(op.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + # Extract distribution information for logging dist_info_dict[op_name] = { - 'name': op.entry_point.dist.project_name, - 'location': op.entry_point.dist.location, + 'name': project_name, + 'location': project_location, } # Provide some log feedback on plugins being used @@ -271,7 +295,8 @@ class HelmOperator(object): on_load_failure_callback=suppress_stevedore_errors ) for entry_point in helm_applications.list_entry_points(): - helm_application_dict[entry_point.name] = entry_point.module_name + helm_application_dict[entry_point.name] = \ + utils.get_module_name_from_entry_point(entry_point) supported_helm_applications = {} for name, namespace in helm_application_dict.items(): @@ -280,10 +305,14 @@ class HelmOperator(object): namespace=namespace, invoke_on_load=True, invoke_args=(self,)) sorted_helm_plugins = sorted(helm_plugins.extensions, key=lambda x: x.name) for plugin in sorted_helm_plugins: + distribution = utils.get_distribution_from_entry_point(plugin.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + LOG.debug("%s: helm plugin %s loaded from %s - %s." % (name, plugin.name, - plugin.entry_point.dist.project_name, - plugin.entry_point.dist.location)) + project_name, + project_location)) plugin_name = plugin.name[HELM_PLUGIN_PREFIX_LENGTH:] self.chart_operators.update({plugin_name: plugin.obj}) diff --git a/sysinv/sysinv/sysinv/test-requirements.txt b/sysinv/sysinv/sysinv/test-requirements.txt index 685aa6e289..280a1556ff 100644 --- a/sysinv/sysinv/sysinv/test-requirements.txt +++ b/sysinv/sysinv/sysinv/test-requirements.txt @@ -19,3 +19,4 @@ isort<5;python_version>="3.0" pylint<2.1.0;python_version<"3.0" # GPLv2 pylint<2.4.0;python_version>="3.0" # GPLv2 pycryptodomex +pathlib;python_version<"3.0"