diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py index 17770cb5d0..44e9092cd1 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py @@ -77,13 +77,12 @@ class HelmChartsController(rest.RestController): except exception.HelmOverrideNotFound: user_overrides = None - system_apps = pecan.request.rpcapi.get_helm_applications( - pecan.request.context) - if app_name in system_apps: + if pecan.request.rpcapi.app_has_system_plugins(pecan.request.context, + app_name): # Get any system overrides for system app. try: system_overrides = pecan.request.rpcapi.get_helm_chart_overrides( - pecan.request.context, name, namespace) + pecan.request.context, app_name, name, namespace) system_overrides = yaml.safe_dump(system_overrides) \ if system_overrides else None except Exception as e: diff --git a/sysinv/sysinv/sysinv/sysinv/cmd/helm.py b/sysinv/sysinv/sysinv/sysinv/cmd/helm.py index 63cb75fdee..e329ab4f3a 100644 --- a/sysinv/sysinv/sysinv/sysinv/cmd/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/cmd/helm.py @@ -14,7 +14,10 @@ import sys from oslo_config import cfg from oslo_log import log +from sysinv.common import constants +from sysinv.common import exception from sysinv.common import service +from sysinv.conductor import kube_app from sysinv.db import api from sysinv.helm import helm @@ -25,35 +28,56 @@ LOG = log.getLogger(__name__) def create_app_overrides_action(path, app_name=None, namespace=None): dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi) - system_apps = operator.get_helm_applications() - if app_name not in system_apps: + + try: + db_app = dbapi.kube_app_get(app_name) + except exception.KubeAppNotFound: + LOG.info("Application %s not found" % app_name) + return + + helm_operator = helm.HelmOperator(dbapi=dbapi) + app_operator = kube_app.AppOperator(dbapi, helm_operator) + + if not app_operator.app_has_system_plugins(db_app): LOG.info("Overrides generation for application %s is " "not supported via this command." % app_name) else: - operator.generate_helm_application_overrides(path, app_name, mode=None, - cnamespace=namespace) + if db_app.status == constants.APP_UPLOAD_SUCCESS: + app_operator.activate_app_plugins(db_app) + helm_operator.generate_helm_application_overrides( + path, app_name, mode=None, cnamespace=namespace) + app_operator.deactivate_app_plugins(db_app) + else: + helm_operator.generate_helm_application_overrides( + path, app_name, mode=None, cnamespace=namespace) def create_armada_app_overrides_action(path, app_name=None, namespace=None): dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi) - system_apps = operator.get_helm_applications() - if app_name not in system_apps: + + try: + db_app = dbapi.kube_app_get(app_name) + except exception.KubeAppNotFound: + LOG.info("Application %s not found" % app_name) + return + + helm_operator = helm.HelmOperator(dbapi=dbapi) + app_operator = kube_app.AppOperator(dbapi, helm_operator) + + if not app_operator.app_has_system_plugins(db_app): LOG.info("Overrides generation for application %s is " "not supported via this command." % app_name) else: - operator.generate_helm_application_overrides(path, app_name, mode=None, - cnamespace=namespace, - armada_format=True, - armada_chart_info=None, - combined=False) - - -def create_chart_override_action(path, chart_name=None, namespace=None): - dbapi = api.get_instance() - operator = helm.HelmOperator(dbapi=dbapi) - operator.generate_helm_chart_overrides(path, chart_name, namespace) + if db_app.status == constants.APP_UPLOAD_SUCCESS: + app_operator.activate_app_plugins(db_app) + helm_operator.generate_helm_application_overrides( + path, app_name, mode=None, cnamespace=namespace, + armada_format=True, armada_chart_info=None, combined=False) + app_operator.deactivate_app_plugins(db_app) + else: + helm_operator.generate_helm_application_overrides( + path, app_name, mode=None, cnamespace=namespace, + armada_format=True, armada_chart_info=None, combined=False) def add_action_parsers(subparsers): @@ -69,12 +93,6 @@ def add_action_parsers(subparsers): parser.add_argument('app_name', nargs='?') parser.add_argument('namespace', nargs='?') - parser = subparsers.add_parser('create-chart-overrides') - parser.set_defaults(func=create_chart_override_action) - parser.add_argument('path', nargs='?') - parser.add_argument('chart_name', nargs='?') - parser.add_argument('namespace', nargs='?') - CONF.register_cli_opt( cfg.SubCommandOpt('action', @@ -103,10 +121,3 @@ def main(): CONF.action.func(CONF.action.path, CONF.action.app_name, CONF.action.namespace) - elif CONF.action.name == 'create-chart-overrides': - try: - CONF.action.func(CONF.action.path, - CONF.action.chart_name, - CONF.action.namespace) - except Exception as e: - print(e) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 480a1ca91d..cd1361e9d8 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -160,6 +160,21 @@ class AppOperator(object): # applications self._plugins.audit_plugins() + def activate_app_plugins(self, rpc_app): + app = AppOperator.Application(rpc_app) + self._plugins.activate_plugins(app) + + def deactivate_app_plugins(self, rpc_app): + app = AppOperator.Application(rpc_app) + self._plugins.deactivate_plugins(app) + + def app_has_system_plugins(self, rpc_app): + app = AppOperator.Application(rpc_app) + # TODO(rchurch): Update once apps are decoupled + return (app.system_app or + app.name == constants.HELM_APP_CERT_MANAGER or + app.name == constants.HELM_APP_OIDC_AUTH) + def _clear_armada_locks(self): lock_name = "{}.{}.{}".format(ARMADA_LOCK_PLURAL, ARMADA_LOCK_GROUP, @@ -3228,7 +3243,8 @@ class PluginHelper(object): # An enabled plugin will have a python path configuration file name with the # following format: stx_app-platform-integ-apps-1.0-8.pth PTH_PREFIX = 'stx_app-' - PTH_PATTERN = re.compile("{}/([\w-]+)/(\d+\.\d+-\d+)".format(common.HELM_OVERRIDES_PATH)) + PTH_PATTERN = re.compile("{}/([\w-]+)/(\d+\.\d+-\d+.*)/plugins".format( + common.HELM_OVERRIDES_PATH)) def __init__(self, dbapi, helm_op): self._dbapi = dbapi @@ -3267,6 +3283,7 @@ class PluginHelper(object): discoverable_pths = glob.glob(pattern) LOG.debug("PluginHelper: Discoverable app plugins: %s" % discoverable_pths) + # Examine existing pth files to make sure they are still valid for pth in discoverable_pths: with open(pth, 'r') as f: contents = f.readlines() @@ -3286,7 +3303,8 @@ class PluginHelper(object): else: LOG.warning("PluginHelper: Stale plugin pth file " "found %s: Wrong plugin version " - "enabled %s." % (pth, ver)) + "enabled %s != %s." % ( + pth, ver, app_obj.app_version)) except exception.KubeAppNotFound: LOG.warning("PluginHelper: Stale plugin pth file found" " %s: App is not active." % pth) @@ -3300,6 +3318,17 @@ class PluginHelper(object): LOG.info("PluginHelper: Removing invalid plugin pth: %s" % pth) os.remove(pth) + # Examine existing applications in an applying state and make sure they + # are activated + apps = self._dbapi.kube_app_get_all() + for app in apps: + # If the app is in some form of apply the the plugins should be + # enabled + if app.status in [constants.APP_APPLY_IN_PROGRESS, + constants.APP_APPLY_SUCCESS, + constants.APP_APPLY_FAILURE]: + self.activate_plugins(AppOperator.Application(app)) + def install_plugins(self, app): """ Install application plugins. """ @@ -3340,9 +3369,15 @@ class PluginHelper(object): "need to remove." % app.sync_plugins_dir) def activate_plugins(self, app): + pth_fqpn = self._get_pth_fqpn(app) + + # If this isn't an app with plugins or the plugin path is already + # active, skip activation + if not app.system_app or os.path.isfile(pth_fqpn): + return + # Add a .pth file to a site-packages directory so the plugin is picked # automatically on a conductor restart - pth_fqpn = self._get_pth_fqpn(app) with open(pth_fqpn, 'w') as f: f.write(app.sync_plugins_dir + '\n') LOG.info("PluginHelper: Enabled plugin directory %s: created %s" % ( @@ -3362,6 +3397,10 @@ class PluginHelper(object): self._helm_op.discover_plugins() def deactivate_plugins(self, app): + # If the application doesn't have any plugins, skip deactivation + if not app.system_app: + return + pth_fqpn = self._get_pth_fqpn(app) if os.path.exists(pth_fqpn): # Remove the pth file, so on a conductor restart this installed diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index dd29d78bf9..483015f49e 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -10834,7 +10834,8 @@ class ConductorManager(service.PeriodicService): """ return self._helm.get_helm_chart_namespaces(chart_name) - def get_helm_chart_overrides(self, context, chart_name, cnamespace=None): + def get_helm_chart_overrides(self, context, app_name, chart_name, + cnamespace=None): """Get the overrides for a supported chart. This method retrieves overrides for a supported chart. Overrides for @@ -10842,6 +10843,7 @@ class ConductorManager(service.PeriodicService): is requested. :param context: request context. + :param app_name: name of a supported application :param chart_name: name of a supported chart :param cnamespace: (optional) namespace :returns: dict of overrides. @@ -10871,16 +10873,30 @@ class ConductorManager(service.PeriodicService): } } """ - return self._helm.get_helm_chart_overrides(chart_name, - cnamespace) - def get_helm_applications(self, context): - """Get supported applications. + app = kubeapp_obj.get_by_name(context, app_name) + if app.status in [constants.APP_APPLY_IN_PROGRESS, + constants.APP_APPLY_SUCCESS, + constants.APP_APPLY_FAILURE]: + overrides = self._helm.get_helm_chart_overrides(chart_name, + cnamespace) + else: + self._app.activate_app_plugins(app) + overrides = self._helm.get_helm_chart_overrides(chart_name, + cnamespace) + self._app.deactivate_app_plugins(app) - :returns: a list of suppotred applications that associated overrides may - be provided. + return overrides + + def app_has_system_plugins(self, context, app_name): + + """Determine if the application has system plugin support. + + :returns: True if the application has system plugins and can generate + system overrides. """ - return self._helm.get_helm_applications() + app = kubeapp_obj.get_by_name(context, app_name) + return self._app.app_has_system_plugins(app) def get_helm_application_namespaces(self, context, app_name): """Get supported application namespaces. diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py index c01d854711..aff30ddf9d 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py @@ -1610,10 +1610,12 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy): self.make_msg('get_helm_chart_namespaces', chart_name=chart_name)) - def get_helm_chart_overrides(self, context, chart_name, cnamespace=None): + def get_helm_chart_overrides(self, context, app_name, chart_name, + cnamespace=None): """Get the overrides for a supported chart. :param context: request context. + :param app_name: name of a supported application :param chart_name: name of a supported chart :param cnamespace: (optional) namespace :returns: dict of overrides. @@ -1621,18 +1623,20 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy): """ return self.call(context, self.make_msg('get_helm_chart_overrides', + app_name=app_name, chart_name=chart_name, cnamespace=cnamespace)) - def get_helm_applications(self, context): + def app_has_system_plugins(self, context, app_name): - """Get supported applications. + """Determine if the application has system plugin support. - :returns: a list of suppotred applications that associated overrides may - be provided. + :returns: True if the application has system plugins and can generate + system overrides. """ return self.call(context, - self.make_msg('get_helm_applications')) + self.make_msg('app_has_system_plugins', + app_name=app_name)) def get_helm_application_namespaces(self, context, app_name): """Get supported application namespaces. diff --git a/sysinv/sysinv/sysinv/sysinv/helm/helm.py b/sysinv/sysinv/sysinv/sysinv/helm/helm.py index 8aa7f8e422..7861473053 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/helm.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/helm.py @@ -222,8 +222,8 @@ class HelmOperator(object): return supported_helm_applications - def get_helm_applications(self): - """ Get the system applications and charts """ + def get_active_helm_applications(self): + """ Get the active system applications and charts """ return self.helm_system_applications @property diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_helm_charts.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_helm_charts.py index 552ab6489b..03be16b71d 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_helm_charts.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_helm_charts.py @@ -18,8 +18,9 @@ from sysinv.tests.db import utils as dbutils class FakeConductorAPI(object): def __init__(self): + self.app_has_system_plugins = mock.MagicMock() self.get_helm_application_namespaces = mock.MagicMock() - self.get_helm_applications = mock.MagicMock() + self.get_active_helm_applications = mock.MagicMock() self.get_helm_chart_overrides = mock.MagicMock() self.merge_overrides = mock.MagicMock() @@ -72,10 +73,11 @@ class ApiHelmChartTestCaseMixin(base.FunctionalTest, chart_namespace='kube-system', system_override_attr={"enabled": False}, user_override="global:\n replicas: \"3\"\n") - self.fake_helm_apps = self.fake_conductor_api.get_helm_applications + self.fake_helm_apps = self.fake_conductor_api.get_active_helm_applications self.fake_ns = self.fake_conductor_api.get_helm_application_namespaces self.fake_override = self.fake_conductor_api.get_helm_chart_overrides self.fake_merge_overrides = self.fake_conductor_api.merge_overrides + self.fake_system_app = self.fake_conductor_api.app_has_system_plugins def exception_helm_override(self): print('Raised a fake exception') @@ -169,6 +171,7 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin): super(ApiHelmChartShowTestSuiteMixin, self).setUp() def test_no_system_override(self): + self.fake_system_app.return_value = False url = self.get_single_url_helm_override('platform-integ-apps', 'ceph-pools-audit', 'kube-system') response = self.get_json(url) @@ -190,6 +193,8 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin): response.json['error_message']) def test_fetch_helm_override_show_invalid_helm_chart(self): + self.fake_system_app.return_value = False + url = self.get_single_url_helm_override('platform-integ-apps', 'invalid_value', 'kube-system') response = self.get_json(url, expect_errors=True) @@ -202,6 +207,7 @@ class ApiHelmChartShowTestSuiteMixin(ApiHelmChartTestCaseMixin): response.json['error_message']) def test_fetch_helm_override_show_invalid_namespace(self): + self.fake_system_app.return_value = False url = self.get_single_url_helm_override('platform-integ-apps', 'ceph-pools-audit', 'invalid_value') @@ -287,6 +293,7 @@ class ApiHelmChartDeleteTestSuiteMixin(ApiHelmChartTestCaseMixin): # Test that a valid DELETE operation is successful def test_delete_helm_override_success(self): + self.fake_system_app.return_value = False # Verify that user override exists initially url = self.get_single_url_helm_override('platform-integ-apps', @@ -494,6 +501,7 @@ class ApiHelmChartPatchTestSuiteMixin(ApiHelmChartTestCaseMixin): 'set': ['global.replicas=2']}}, headers=self.API_HEADERS, expect_errors=True) + self.fake_system_app.return_value = False response = self.get_json(url, expect_errors=True) self.assertEqual(response.status_code, http_client.OK) # Verify the values of the response with the values in database