From 357e4dba7642b9999bd8299f42b203a1b10a5c3b Mon Sep 17 00:00:00 2001 From: Dan Voiculeasa Date: Thu, 5 Aug 2021 12:39:35 +0300 Subject: [PATCH] Fix Stevedore plugin usage for Debian OS API was changed between Stevedore and Python versions. Stevedore entry_point object was changed and importlib.metadata.entry_point doesn't reference a Distribution object. https://docs.openstack.org/releasenotes/stevedore/victoria.html https://bugs.python.org/issue42382 Added wrapper code to handle different scenarios. Support parsing available modules for Python3 environments to determine the Distribution object. Need to parse available modules only for Stevedore 3. This keeps compatibility with: Stevedore 1.25 + Python 2.7 (CentOS7), Stevedore 1.31 + Python 3.6 (CentOS8). This adds logic needed to be run on: Stevedore 3.2.2 + Python 3.9 (Debian Bullseye). Story: 2009101 Task: 42950 Signed-off-by: Dan Voiculeasa Change-Id: Id88951104a93af7be3c169c3ba91ace2b7300a34 --- sysinv/sysinv/sysinv/sysinv/common/utils.py | 126 ++++++++++++++++++++ sysinv/sysinv/sysinv/sysinv/helm/helm.py | 59 ++++++--- sysinv/sysinv/sysinv/test-requirements.txt | 1 + 3 files changed, 171 insertions(+), 15 deletions(-) diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 747560b948..31e6b3b888 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -46,6 +46,7 @@ import json import keyring import math import os +import pathlib import pwd import random import re @@ -56,9 +57,11 @@ import six import socket import stat import string +import sys import tempfile import time import tsconfig.tsconfig as tsc +import types import uuid import wsme import yaml @@ -89,6 +92,15 @@ except ImportError: SW_VERSION = "unknown" +if six.PY3: + USE_IMPORTLIB_METADATA_STDLIB = False + try: + import importlib.metadata + USE_IMPORTLIB_METADATA_STDLIB = True + except ImportError: + import importlib_metadata + + utils_opts = [ cfg.StrOpt('rootwrap_config', default="/etc/sysinv/rootwrap.conf", @@ -2891,3 +2903,117 @@ def TempDirectory(): shutil.rmtree(tmpdir) except OSError as e: LOG.error(_('Could not remove tmpdir: %s'), str(e)) + + +def get_stevedore_major_version(): + if six.PY2: + # Hardcode Stevedore 1.25.0 for CentOS7 that has Python2. + # Support for Python2 will be dropped soon, and this removed. + return 1 + + package = 'stevedore' + if USE_IMPORTLIB_METADATA_STDLIB: + distribution = importlib.metadata.distribution + else: + distribution = importlib_metadata.distribution + + return int(distribution(package).version.split('.')[0]) + + +def get_distribution_from_entry_point(entry_point): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + In Python3.9.2 used on Debian Bullseye the EntryPoint returned by + importlib does not hold a reference to a Distribution object. + https://bugs.python.org/issue42382 + + Determine the missing information by parsing all modules in Python3 envs. + This can be removed when Python will be patched or upgraded. + + :param entry_point: An EntryPoint object + :return: A Distribution object + :raises exception.SysinvException: If distribution could not be found + """ + # Just a refactor on this path + if get_stevedore_major_version() < 3: + return entry_point.dist + + if six.PY2: + raise exception.SysinvException(_( + "Python2 + Stevedore 3 and later support not implemented: " + "parsing modules in Python2 not implemented.")) + + loaded_entry_point = entry_point.load() + if isinstance(loaded_entry_point, types.ModuleType): + module_path = loaded_entry_point.__file__ + else: + module_path = sys.modules[loaded_entry_point.__module__].__file__ + if USE_IMPORTLIB_METADATA_STDLIB: + distributions = importlib.metadata.distributions + else: + distributions = importlib_metadata.distributions + + for distribution in distributions(): + try: + relative = pathlib.Path(module_path).relative_to( + distribution.locate_file("") + ) + except ValueError: + pass + else: + if relative in distribution.files: + return distribution + + raise exception.SysinvException(_( + "Distribution information for entry point {} " + "could not be found.".format(entry_point))) + + +def get_project_name_and_location_from_distribution(distribution): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + Determine the missing information by parsing the Distribution object. + + :param distribution: A Distribution object + :return: Tuple of project name and project location. Location being + the parent of directory named + """ + # Just a refactor on this path + if get_stevedore_major_version() < 3: + return (distribution.project_name, distribution.location) + + project_name = distribution.metadata.get('Name') + project_location = str(distribution._path.parent) + return (project_name, project_location) + + +def get_module_name_from_entry_point(entry_point): + """ + With Stevedore 3.0.0 the entry_point object was changed. + https://docs.openstack.org/releasenotes/stevedore/victoria.html + + This affects some of our Stevedore based logic on Debian Bullseye which + currently uses Stevedore 3.2.2. + + :param entry_point: An EntryPoint object + :return: Module name + :raises exception.SysinvException: If module name could not be found + """ + if 'module_name' in dir(entry_point): + return entry_point.module_name + elif 'module' in dir(entry_point): + return entry_point.module + + raise exception.SysinvException(_( + "Module name for entry point {} " + "could not be determined.".format(entry_point))) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/helm.py b/sysinv/sysinv/sysinv/sysinv/helm/helm.py index 6af62fa879..f4a85033d5 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/helm.py @@ -117,28 +117,44 @@ class HelmOperator(object): def purge_cache_by_location(self, install_location): """Purge the stevedore entry point cache.""" for lifecycle_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE]: - if lifecycle_ep.dist.location == install_location: + lifecycle_distribution = utils.get_distribution_from_entry_point(lifecycle_ep) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(lifecycle_distribution) + + if project_location == install_location: extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_LIFECYCLE].remove(lifecycle_ep) break else: LOG.info("Couldn't find endpoint distribution located at %s for " - "%s" % (install_location, lifecycle_ep.dist)) + "%s" % (install_location, lifecycle_distribution)) for armada_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA]: - if armada_ep.dist.location == install_location: + armada_distribution = utils.get_distribution_from_entry_point(armada_ep) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(armada_distribution) + + if project_location == install_location: extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_ARMADA].remove(armada_ep) break else: LOG.info("Couldn't find endpoint distribution located at %s for " - "%s" % (install_location, armada_ep.dist)) + "%s" % (install_location, armada_distribution)) for app_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]: - if app_ep.dist.location == install_location: - namespace = app_ep.module_name + app_distribution = utils.get_distribution_from_entry_point(app_ep) + (app_project_name, app_project_location) = \ + utils.get_project_name_and_location_from_distribution(app_distribution) + + if app_project_location == install_location: + namespace = utils.get_module_name_from_entry_point(app_ep) purged_list = [] for helm_ep in extension.ExtensionManager.ENTRY_POINT_CACHE[namespace]: - if helm_ep.dist.location != install_location: + helm_distribution = utils.get_distribution_from_entry_point(helm_ep) + (helm_project_name, helm_project_location) = \ + utils.get_project_name_and_location_from_distribution(helm_distribution) + + if helm_project_location != install_location: purged_list.append(helm_ep) if purged_list: @@ -152,7 +168,7 @@ class HelmOperator(object): """Purge the stevedore entry point cache.""" if self.STEVEDORE_APPS in extension.ExtensionManager.ENTRY_POINT_CACHE: for entry_point in extension.ExtensionManager.ENTRY_POINT_CACHE[self.STEVEDORE_APPS]: - namespace = entry_point.module_name + namespace = utils.get_module_name_from_entry_point(entry_point) try: del extension.ExtensionManager.ENTRY_POINT_CACHE[namespace] LOG.debug("Deleted entry points for %s." % namespace) @@ -201,10 +217,14 @@ class HelmOperator(object): operator_name = operator.name operators_dict[operator_name] = operator.obj + distribution = utils.get_distribution_from_entry_point(operator.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + # Extract distribution information for logging dist_info_dict[operator_name] = { - 'name': operator.entry_point.dist.project_name, - 'location': operator.entry_point.dist.location, + 'name': project_name, + 'location': project_location, } return operators_dict @@ -240,10 +260,14 @@ class HelmOperator(object): op_name = op.name operators_dict[op_name] = op.obj + distribution = utils.get_distribution_from_entry_point(op.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + # Extract distribution information for logging dist_info_dict[op_name] = { - 'name': op.entry_point.dist.project_name, - 'location': op.entry_point.dist.location, + 'name': project_name, + 'location': project_location, } # Provide some log feedback on plugins being used @@ -271,7 +295,8 @@ class HelmOperator(object): on_load_failure_callback=suppress_stevedore_errors ) for entry_point in helm_applications.list_entry_points(): - helm_application_dict[entry_point.name] = entry_point.module_name + helm_application_dict[entry_point.name] = \ + utils.get_module_name_from_entry_point(entry_point) supported_helm_applications = {} for name, namespace in helm_application_dict.items(): @@ -280,10 +305,14 @@ class HelmOperator(object): namespace=namespace, invoke_on_load=True, invoke_args=(self,)) sorted_helm_plugins = sorted(helm_plugins.extensions, key=lambda x: x.name) for plugin in sorted_helm_plugins: + distribution = utils.get_distribution_from_entry_point(plugin.entry_point) + (project_name, project_location) = \ + utils.get_project_name_and_location_from_distribution(distribution) + LOG.debug("%s: helm plugin %s loaded from %s - %s." % (name, plugin.name, - plugin.entry_point.dist.project_name, - plugin.entry_point.dist.location)) + project_name, + project_location)) plugin_name = plugin.name[HELM_PLUGIN_PREFIX_LENGTH:] self.chart_operators.update({plugin_name: plugin.obj}) diff --git a/sysinv/sysinv/sysinv/test-requirements.txt b/sysinv/sysinv/sysinv/test-requirements.txt index 685aa6e289..280a1556ff 100644 --- a/sysinv/sysinv/sysinv/test-requirements.txt +++ b/sysinv/sysinv/sysinv/test-requirements.txt @@ -19,3 +19,4 @@ isort<5;python_version>="3.0" pylint<2.1.0;python_version<"3.0" # GPLv2 pylint<2.4.0;python_version>="3.0" # GPLv2 pycryptodomex +pathlib;python_version<"3.0"