Add support for helm chart override namespaces

There are some charts which we know will be installed multiple times in
different namespaces, and we may require separate system and user
overrides depending on the namespace.

Accordingly, we need to change over to using a "chart_name/namespace"
tuple instead of just "chart_name" to locate a given set of overrides.

This updates the DB and all APIs to use the name/namespace tuple,
replaces do_helm_chart_list() with do_helm_override_list() for
consistency, and adds a list of known user override namespaces to the
list of known charts.

Story: 2002876
Task: 22831

Change-Id: I538893bae6644e158404cfc5e94c470991df024d
Signed-off-by: Jack Ding <jack.ding@windriver.com>

sysinv/cgts-client/cgts-client/cgtsclient/v1/helm.py

@@ -22,25 +22,33 @@ class HelmManager(base.Manager):
         return '/v1/helm_charts/%s' % name
 
     def list_charts(self):
+        """Get list of charts
+
+        For each chart it will show any overrides for that chart along
+        with the namespace of the overrides.
+        """
         return self._list(self._path(), 'charts')
 
-    def get_overrides(self, name):
+    def get_overrides(self, name, namespace):
         """Get overrides for a given chart.
 
         :param name: name of the chart
+        :param namespace: namespace for the chart overrides
 
         This will return the end-user, system, and combined overrides for the
         specified chart.
         """
         try:
-            return self._list(self._path(name))[0]
+            return self._list(self._path(name) + '?namespace=' + namespace)[0]
         except IndexError:
             return None
 
-    def update_overrides(self, name, flag='reset', override_values={}):
+    def update_overrides(self, name, namespace,
+                         flag='reset', override_values={}):
         """Update overrides for a given chart.
 
         :param name: name of the chart
+        :param namespace: namespace for the chart overrides
         :param flag: 'reuse' or 'reset' to indicate how to handle existing
                      user overrides for this chart
         :param override_values: a dict representing the overrides
@@ -48,11 +56,12 @@ class HelmManager(base.Manager):
         This will return the end-user overrides for the specified chart.
         """
         body = {'flag': flag, 'values': override_values}
-        return self._update(self._path(name), body)
+        return self._update(self._path(name) + '?namespace=' + namespace, body)
 
-    def delete_overrides(self, name):
+    def delete_overrides(self, name, namespace):
         """Delete overrides for a given chart.
 
         :param name: name of the chart
+        :param namespace: namespace for the chart overrides
         """
-        return self._delete(self._path(name))
+        return self._delete(self._path(name) + '?namespace=' + namespace)

sysinv/cgts-client/cgts-client/cgtsclient/v1/helm_shell.py

@@ -22,37 +22,50 @@ def _print_helm_chart(chart):
     utils.print_dict(ordereddata)
 
 
-def do_helm_chart_list(cc, args):
+def do_helm_override_list(cc, args):
     """List system helm charts."""
     charts = cc.helm.list_charts()
-    utils.print_list(charts, ['name'], ['chart name'], sortby=0)
+    utils.print_list(charts, ['name', 'namespaces'], ['chart name', 'overrides namespaces'], sortby=0)
 
 
 @utils.arg('chart', metavar='<chart name>',
            help="Name of chart")
+@utils.arg('namespace',
+           metavar='<namespace>',
+           help="namespace of chart overrides")
 def do_helm_override_show(cc, args):
     """Show overrides for chart."""
-    chart = cc.helm.get_overrides(args.chart)
+    try:
-    _print_helm_chart(chart)
+        chart = cc.helm.get_overrides(args.chart, args.namespace)
+        _print_helm_chart(chart)
+    except exc.HTTPNotFound:
+        raise exc.CommandError('chart overrides not found: %s:%s' % (
+            args.chart, args.namespace))
 
 
 @utils.arg('chart',
            metavar='<chart name>',
-           nargs='+',
            help="Name of chart")
+@utils.arg('namespace',
+           metavar='<namespace>',
+           help="namespace of chart overrides")
 def do_helm_override_delete(cc, args):
-    """Delete overrides for one or more charts."""
+    """Delete overrides for a chart."""
-    for chart in args.chart:
+    try:
-        try:
+        cc.helm.delete_overrides(args.chart, args.namespace)
-            cc.helm.delete_overrides(chart)
+        print 'Deleted chart overrides for %s:%s' % (
-            print 'Deleted chart %s' % chart
+            args.chart, args.namespace)
-        except exc.HTTPNotFound:
+    except exc.HTTPNotFound:
-            raise exc.CommandError('chart not found: %s' % chart)
+        raise exc.CommandError('chart overrides not found: %s:%s' % (
+            args.chart, args.namespace))
 
 
 @utils.arg('chart',
            metavar='<chart name>',
            help="Name of chart")
+@utils.arg('namespace',
+           metavar='<namespace>',
+           help="namespace of chart overrides")
 @utils.arg('--reuse-values', action='store_true', default=False,
            help='Should we reuse existing helm chart user override values. '
                 'If --reset-values is set this is ignored')
@@ -102,7 +115,9 @@ def do_helm_override_update(cc, args):
     }
 
     try:
-        chart = cc.helm.update_overrides(args.chart, flag, overrides)
+        chart = cc.helm.update_overrides(args.chart, args.namespace,
+                                         flag, overrides)
     except exc.HTTPNotFound:
-        raise exc.CommandError('helm chart not found: %s' % args.chart)
+        raise exc.CommandError('helm chart not found: %s:%s' % (
+            args.chart, args.namespace))
     _print_helm_chart(chart)

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py

@@ -30,18 +30,26 @@ class HelmChartsController(rest.RestController):
     @wsme_pecan.wsexpose(wtypes.text)
     def get_all(self):
         """Provides information about the available charts to override."""
-        charts = [{'name': chart} for chart in SYSTEM_CHARTS]
+
+        overrides = pecan.request.dbapi.helm_override_get_all()
+
+        charts = [{'name': chart, 'namespaces': []} for chart in SYSTEM_CHARTS]
+        for chart in charts:
+            namespaces = [x['namespace'] for x in overrides
+                          if x['name'] == chart['name']]
+            chart['namespaces'] = namespaces
         return {'charts': charts}
 
-    @wsme_pecan.wsexpose(wtypes.text, wtypes.text)
+    @wsme_pecan.wsexpose(wtypes.text, wtypes.text, wtypes.text)
-    def get_one(self, name):
+    def get_one(self, name, namespace):
         """Retrieve information about the given event_log.
 
         :param name: name of helm chart
+        :param namespace: namespace of chart overrides
         """
         try:
             db_chart = objects.helm_overrides.get_by_name(
-                pecan.request.context, name)
+                pecan.request.context, name, namespace)
             overrides = db_chart.user_overrides
         except exception.HelmOverrideNotFound:
             if name in SYSTEM_CHARTS:
@@ -50,32 +58,44 @@ class HelmChartsController(rest.RestController):
                 raise
 
         rpc_chart = {'name': name,
+                     'namespace': namespace,
                      'system_overrides': {},
                      'user_overrides': overrides}
 
         return rpc_chart
 
-    @wsme_pecan.wsexpose(wtypes.text, wtypes.text, wtypes.text, wtypes.text)
+    def validate_name_and_namespace(self, name, namespace):
-    def patch(self, name, flag, values):
+        if not name:
+                raise wsme.exc.ClientSideError(_(
+                    "Helm-override-update rejected: name must be specified"))
+        if not namespace:
+            raise wsme.exc.ClientSideError(_(
+                "Helm-override-update rejected: namespace must be specified"))
+
+    @wsme_pecan.wsexpose(wtypes.text, wtypes.text, wtypes.text, wtypes.text, wtypes.text)
+    def patch(self, name, namespace, flag, values):
         """ Update user overrides.
 
         :param name: chart name
+        :param namespace: namespace of chart overrides
         :param flag: one of "reuse" or "reset", describes how to handle
                      previous user overrides
         :param values: a dict of different types of user override values
         """
+        self.validate_name_and_namespace(name, namespace)
 
         try:
             db_chart = objects.helm_overrides.get_by_name(
-                pecan.request.context, name)
+                pecan.request.context, name, namespace)
             db_values = db_chart.user_overrides
         except exception.HelmOverrideNotFound:
             if name in SYSTEM_CHARTS:
                 pecan.request.dbapi.helm_override_create({
                     'name': name,
+                    'namespace': namespace,
                     'user_overrides': ''})
                 db_chart = objects.helm_overrides.get_by_name(
-                    pecan.request.context, name)
+                    pecan.request.context, name, namespace)
             else:
                 raise
         db_values = db_chart.user_overrides
@@ -144,17 +164,20 @@ class HelmChartsController(rest.RestController):
         db_chart.user_overrides = values
         db_chart.save()
 
-        chart = {'name': name, 'user_overrides': values}
+        chart = {'name': name, 'namespace': namespace,
+                 'user_overrides': values}
 
         return chart
 
-    @wsme_pecan.wsexpose(None, unicode, status_code=204)
+    @wsme_pecan.wsexpose(None, wtypes.text, wtypes.text, status_code=204)
-    def delete(self, name):
+    def delete(self, name, namespace):
         """Delete user overrides for a chart
 
         :param name: chart name.
+        :param namespace: namespace of chart overrides
         """
+        self.validate_name_and_namespace(name, namespace)
         try:
-            pecan.request.dbapi.helm_override_destroy(name)
+            pecan.request.dbapi.helm_override_destroy(name, namespace)
         except exception.HelmOverrideNotFound:
             pass

sysinv/sysinv/sysinv/sysinv/common/exception.py

@@ -579,7 +579,8 @@ class CertificateAlreadyExists(Conflict):
 
 
 class HelmOverrideAlreadyExists(Conflict):
-    message = _("A HelmOverride with name %(name)s already exists.")
+    message = _("A HelmOverride with name %(name)s and namespace "
+                "%(namespace)s already exists.")
 
 
 class InstanceDeployFailure(Invalid):
@@ -892,7 +893,8 @@ class CertificateNotFound(NotFound):
 
 
 class HelmOverrideNotFound(NotFound):
-    message = _("No helm override with name %(name)s")
+    message = _("No helm override with name %(name)s and namespace "
+                "%(namespace)s")
 
 
 class CertificateTypeNotFound(NotFound):

sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py

@@ -7437,13 +7437,14 @@ class Connection(api.Connection):
                 raise exception.CertificateNotFound(uuid)
             query.delete()
 
-    def _helm_override_get(self, name):
+    def _helm_override_get(self, name, namespace):
         query = model_query(models.HelmOverrides)
-        query = query.filter_by(name=name)
+        query = query.filter_by(name=name, namespace=namespace)
         try:
             return query.one()
         except NoResultFound:
-            raise exception.HelmOverrideNotFound(name)
+            raise exception.HelmOverrideNotFound(name=name,
+                                                 namespace=namespace)
 
     @objects.objectify(objects.helm_overrides)
     def helm_override_create(self, values):
@@ -7458,31 +7459,40 @@ class Connection(api.Connection):
                 LOG.error("Failed to add HelmOverrides %s. "
                           "Already exists with this name" %
                           (values['name']))
-                raise exception.HelmOverrideAlreadyExists(name=values['name'])
+                raise exception.HelmOverrideAlreadyExists(
-            return self._helm_override_get(values['name'])
+                    name=values['name'], namespace=values['namespace'])
+            return self._helm_override_get(values['name'],
+                                           values['namespace'])
 
     @objects.objectify(objects.helm_overrides)
-    def helm_override_get(self, name):
+    def helm_override_get(self, name, namespace):
-        return self._helm_override_get(name)
+        return self._helm_override_get(name, namespace)
 
     @objects.objectify(objects.helm_overrides)
-    def helm_override_update(self, name, values):
+    def helm_override_get_all(self):
+        query = model_query(models.HelmOverrides, read_deleted="no")
+        return query.all()
+
+    @objects.objectify(objects.helm_overrides)
+    def helm_override_update(self, name, namespace, values):
         with _session_for_write() as session:
             query = model_query(models.HelmOverrides, session=session)
-            query = query.filter_by(name=name)
+            query = query.filter_by(name=name, namespace=namespace)
 
             count = query.update(values, synchronize_session='fetch')
             if count == 0:
-                raise exception.HelmOverrideNotFound(name)
+                raise exception.HelmOverrideNotFound(name=name,
+                                                     namespace=namespace)
             return query.one()
 
-    def helm_override_destroy(self, name):
+    def helm_override_destroy(self, name, namespace):
         with _session_for_write() as session:
             query = model_query(models.HelmOverrides, session=session)
-            query = query.filter_by(name=name)
+            query = query.filter_by(name=name, namespace=namespace)
 
             try:
                 query.one()
             except NoResultFound:
-                raise exception.HelmOverrideNotFound(name)
+                raise exception.HelmOverrideNotFound(name=name,
+                                                     namespace=namespace)
             query.delete()

sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/073_helm_overrides.py

@@ -5,8 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from sqlalchemy import DateTime, String, Text
+from sqlalchemy import DateTime, String, Text, Integer
-from sqlalchemy import Column, MetaData, Table
+from sqlalchemy import Column, MetaData, Table, UniqueConstraint
 
 from sysinv.openstack.common import log
 
@@ -32,8 +32,11 @@ def upgrade(migrate_engine):
         Column('created_at', DateTime),
         Column('updated_at', DateTime),
         Column('deleted_at', DateTime),
-        Column('name', String(255), unique=True, index=True),
+        Column('id', Integer, primary_key=True),
+        Column('name', String(255), nullable=False),
+        Column('namespace', String(255), nullable=False),
         Column('user_overrides', Text, nullable=True),
+        UniqueConstraint('name', 'namespace', name='u_name_namespace'),
 
         mysql_engine=ENGINE,
         mysql_charset=CHARSET,

sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py

@@ -1640,5 +1640,8 @@ class certificate(Base):
 class HelmOverrides(Base):
     __tablename__ = 'helm_overrides'
 
-    name = Column(String(255), primary_key=True, unique=True)
+    id = Column(Integer, primary_key=True)
+    name = Column(String(255), nullable=False)
+    namespace = Column(String(255), nullable=False)
     user_overrides = Column(Text, nullable=True)
+    UniqueConstraint('name', 'namespace', name='u_name_namespace')

sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py

@@ -18,12 +18,13 @@ class HelmOverrides(base.SysinvObject):
     dbapi = db_api.get_instance()
 
     fields = {'name': utils.str_or_none,
+              'namespace': utils.str_or_none,
               'user_overrides': utils.str_or_none,
               }
 
     @base.remotable_classmethod
-    def get_by_name(cls, context, name):
+    def get_by_name(cls, context, name, namespace):
-        return cls.dbapi.helm_override_get(name)
+        return cls.dbapi.helm_override_get(name, namespace)
 
     def save_changes(self, context, updates):
-        self.dbapi.helm_override_update(self.name, updates)
+        self.dbapi.helm_override_update(self.name, self.namespace, updates)