diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py index ff51c2d28d..3db8595dad 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py @@ -30,6 +30,7 @@ from cgtsclient.v1 import event_log from cgtsclient.v1 import event_suppression from cgtsclient.v1 import firewallrules from cgtsclient.v1 import health +from cgtsclient.v1 import helm from cgtsclient.v1 import ialarm from cgtsclient.v1 import icommunity from cgtsclient.v1 import icpu @@ -148,3 +149,4 @@ class Client(http.HTTPClient): self.storage_tier = storage_tier.StorageTierManager(self) self.storage_ceph_external = \ storage_ceph_external.StorageCephExternalManager(self) + self.helm = helm.HelmManager(self) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm.py new file mode 100644 index 0000000000..56e8a0a993 --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# -*- encoding: utf-8 -*- +# + +from cgtsclient.common import base + + +class Helm(base.Resource): + def __repr__(self): + return "" % self._info + + +class HelmManager(base.Manager): + resource_class = Helm + + @staticmethod + def _path(name=''): + return '/v1/helm_charts/%s' % name + + def list_charts(self): + return self._list(self._path(), 'charts') + + def get_overrides(self, name): + """Get overrides for a given chart. + + :param name: name of the chart + + This will return the end-user, system, and combined overrides for the + specified chart. + """ + try: + return self._list(self._path(name))[0] + except IndexError: + return None + + def update_overrides(self, name, flag='reset', override_values={}): + """Update overrides for a given chart. + + :param name: name of the chart + :param flag: 'reuse' or 'reset' to indicate how to handle existing + user overrides for this chart + :param override_values: a dict representing the overrides + + This will return the end-user overrides for the specified chart. + """ + body = {'flag': flag, 'values': override_values} + return self._update(self._path(name), body) + + def delete_overrides(self, name): + """Delete overrides for a given chart. + + :param name: name of the chart + """ + return self._delete(self._path(name)) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm_shell.py new file mode 100755 index 0000000000..71136919bb --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/helm_shell.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from collections import OrderedDict +import yaml + +from cgtsclient.common import utils +from cgtsclient import exc + + +def _print_helm_chart(chart): + # We only want to print the fields that are in the response, that way + # we can reuse this function for printing the override update output + # where the system overrides aren't included. + ordereddata = OrderedDict(sorted(chart.to_dict().items(), + key=lambda t: t[0])) + utils.print_dict(ordereddata) + + +def do_helm_chart_list(cc, args): + """List system helm charts.""" + charts = cc.helm.list_charts() + utils.print_list(charts, ['name'], ['chart name'], sortby=0) + + +@utils.arg('chart', metavar='', + help="Name of chart") +def do_helm_override_show(cc, args): + """Show overrides for chart.""" + chart = cc.helm.get_overrides(args.chart) + _print_helm_chart(chart) + + +@utils.arg('chart', + metavar='', + nargs='+', + help="Name of chart") +def do_helm_override_delete(cc, args): + """Delete overrides for one or more charts.""" + for chart in args.chart: + try: + cc.helm.delete_overrides(chart) + print 'Deleted chart %s' % chart + except exc.HTTPNotFound: + raise exc.CommandError('chart not found: %s' % chart) + + +@utils.arg('chart', + metavar='', + help="Name of chart") +@utils.arg('--reuse-values', action='store_true', default=False, + help='Should we reuse existing helm chart user override values. ' + 'If --reset-values is set this is ignored') +@utils.arg('--reset-values', action='store_true', default=False, + help='Replace any existing helm chart overrides with the ones ' + 'specified.') +@utils.arg('--values', metavar='', action='append', dest='files', + default=[], + help='Specify a YAML file containing helm chart override values. ' + 'Can specify multiple times.') +@utils.arg('--set', metavar='', action='append', + default=[], + help='Set helm chart override values on the command line (can ' + 'specify multiple times or separate values with commas: ' + 'key1=val1,key2=val2). These are processed after "--values" ' + 'files.') +def do_helm_override_update(cc, args): + """Update helm chart user overrides.""" + + # This logic results in similar behaviour to "helm upgrade". + flag = 'reset' + if args.reuse_values and not args.reset_values: + flag = 'reuse' + + # Overrides can be specified three different ways. To preserve helm's + # behaviour we will process all "--values" files first, then all "--set" + # values, then finally all "--set-string" values. + + override_files = [] + # need to handle missing files + if args.files: + try: + for filename in args.files: + with open(filename, 'r') as input_file: + overrides = yaml.load(input_file) + override_files.append(yaml.dump(overrides)) + except IOError as ex: + raise exc.CommandError('error opening values file: %s' % ex) + + override_set = [] + for override in args.set: + override_set.append(override) + + overrides = { + 'files': override_files, + 'set': override_set, + } + + try: + chart = cc.helm.update_overrides(args.chart, flag, overrides) + except exc.HTTPNotFound: + raise exc.CommandError('helm chart not found: %s' % args.chart) + _print_helm_chart(chart) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py index fe06b3a3cf..8784f8f8ac 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py @@ -18,6 +18,7 @@ from cgtsclient.v1 import event_log_shell from cgtsclient.v1 import event_suppression_shell from cgtsclient.v1 import firewallrules_shell from cgtsclient.v1 import health_shell +from cgtsclient.v1 import helm_shell from cgtsclient.v1 import ialarm_shell from cgtsclient.v1 import icommunity_shell @@ -111,6 +112,7 @@ COMMAND_MODULES = [ license_shell, certificate_shell, storage_tier_shell, + helm_shell, ] diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py index 9fc22682d8..e25669da16 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py @@ -37,6 +37,7 @@ from sysinv.api.controllers.v1 import event_log from sysinv.api.controllers.v1 import event_suppression from sysinv.api.controllers.v1 import firewallrules from sysinv.api.controllers.v1 import health +from sysinv.api.controllers.v1 import helm_charts from sysinv.api.controllers.v1 import host from sysinv.api.controllers.v1 import interface from sysinv.api.controllers.v1 import link @@ -109,6 +110,9 @@ class V1(base.APIBase): ihosts = [link.Link] "Links to the ihosts resource" + helm_charts = [link.Link] + "Links to the helm resource" + inode = [link.Link] "Links to the inode resource" @@ -260,6 +264,14 @@ class V1(base.APIBase): bookmark=True) ] + v1.helm_charts = [link.Link.make_link('self', pecan.request.host_url, + 'helm_charts', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'helm_charts', '', + bookmark=True) + ] + v1.inode = [link.Link.make_link('self', pecan.request.host_url, 'inode', ''), link.Link.make_link('bookmark', @@ -722,6 +734,7 @@ class Controller(rest.RestController): isystems = system.SystemController() ihosts = host.HostController() + helm_charts = helm_charts.HelmChartsController() inodes = node.NodeController() icpus = cpu.CPUController() imemorys = memory.MemoryController() diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py new file mode 100644 index 0000000000..6c5d9bbdbe --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/helm_charts.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import pecan +from pecan import rest +import subprocess +import tempfile + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from sysinv import objects +from sysinv.common import exception +from sysinv.openstack.common import log +from sysinv.openstack.common.gettextutils import _ + + +LOG = log.getLogger(__name__) + +SYSTEM_CHARTS = ['mariadb', 'rabbitmq', 'ingress'] + + +class HelmChartsController(rest.RestController): + + @wsme_pecan.wsexpose(wtypes.text) + def get_all(self): + """Provides information about the available charts to override.""" + charts = [{'name': chart} for chart in SYSTEM_CHARTS] + return {'charts': charts} + + @wsme_pecan.wsexpose(wtypes.text, wtypes.text) + def get_one(self, name): + """Retrieve information about the given event_log. + + :param name: name of helm chart + """ + try: + db_chart = objects.helm_overrides.get_by_name( + pecan.request.context, name) + overrides = db_chart.user_overrides + except exception.HelmOverrideNotFound: + if name in SYSTEM_CHARTS: + overrides = {} + else: + raise + + rpc_chart = {'name': name, + 'system_overrides': {}, + 'user_overrides': overrides} + + return rpc_chart + + @wsme_pecan.wsexpose(wtypes.text, wtypes.text, wtypes.text, wtypes.text) + def patch(self, name, flag, values): + """ Update user overrides. + + :param name: chart name + :param flag: one of "reuse" or "reset", describes how to handle + previous user overrides + :param values: a dict of different types of user override values + """ + + try: + db_chart = objects.helm_overrides.get_by_name( + pecan.request.context, name) + db_values = db_chart.user_overrides + except exception.HelmOverrideNotFound: + if name in SYSTEM_CHARTS: + pecan.request.dbapi.helm_override_create({ + 'name': name, + 'user_overrides': ''}) + db_chart = objects.helm_overrides.get_by_name( + pecan.request.context, name) + else: + raise + db_values = db_chart.user_overrides + + # At this point we have potentially two separate types of overrides + # specified by the user, values from files and values passed in via + # --set . We need to ensure that we call helm using the same + # mechanisms to ensure the same behaviour. + cmd = ['helm', 'install', '--dry-run', '--debug'] + + if flag == 'reuse': + values['files'].insert(0, db_values) + elif flag == 'reset': + pass + else: + raise wsme.exc.ClientSideError(_("Invalid flag: %s must be either " + "'reuse' or 'reset'.") % flag) + + # Now process the newly-passed-in override values + tmpfiles = [] + for value_file in values['files']: + # For values passed in from files, write them back out to + # temporary files. + tmpfile = tempfile.NamedTemporaryFile(delete=False) + tmpfile.write(value_file) + tmpfile.close() + tmpfiles.append(tmpfile.name) + cmd.extend(['--values', tmpfile.name]) + + for value_set in values['set']: + cmd.extend(['--set', value_set]) + + env = os.environ.copy() + env['KUBECONFIG'] = '/etc/kubernetes/admin.conf' + + # Make a temporary directory with a fake chart in it + try: + tmpdir = tempfile.mkdtemp() + chartfile = tmpdir + '/Chart.yaml' + with open(chartfile, 'w') as tmpchart: + tmpchart.write('name: mychart\napiVersion: v1\n' + 'version: 0.1.0\n') + cmd.append(tmpdir) + + # Apply changes by calling out to helm to do values merge + # using a dummy chart. + # NOTE: this requires running sysinv-api as root, will fix it + # to use RPC in a followup patch. + output = subprocess.check_output(cmd, env=env) + + # Check output for failure + + # Extract the info we want. + values = output.split('USER-SUPPLIED VALUES:\n')[1].split( + '\nCOMPUTED VALUES:')[0] + except: + raise + finally: + os.remove(chartfile) + os.rmdir(tmpdir) + + for tmpfile in tmpfiles: + os.remove(tmpfile) + + # save chart overrides back to DB + db_chart.user_overrides = values + db_chart.save() + + chart = {'name': name, 'user_overrides': values} + + return chart + + @wsme_pecan.wsexpose(None, unicode, status_code=204) + def delete(self, name): + """Delete user overrides for a chart + + :param name: chart name. + """ + try: + pecan.request.dbapi.helm_override_destroy(name) + except exception.HelmOverrideNotFound: + pass diff --git a/sysinv/sysinv/sysinv/sysinv/common/exception.py b/sysinv/sysinv/sysinv/sysinv/common/exception.py index e680ca344a..32a6572831 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/exception.py +++ b/sysinv/sysinv/sysinv/sysinv/common/exception.py @@ -578,6 +578,10 @@ class CertificateAlreadyExists(Conflict): message = _("A Certificate with uuid %(uuid)s already exists.") +class HelmOverrideAlreadyExists(Conflict): + message = _("A HelmOverride with name %(name)s already exists.") + + class InstanceDeployFailure(Invalid): message = _("Failed to deploy instance: %(reason)s") @@ -887,6 +891,10 @@ class CertificateNotFound(NotFound): message = _("No certificate with uuid %(uuid)s") +class HelmOverrideNotFound(NotFound): + message = _("No helm override with name %(name)s") + + class CertificateTypeNotFound(NotFound): message = _("No certificate type of %(certtype)s") diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index fdde6bec7f..281e0bf22a 100755 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -7436,3 +7436,53 @@ class Connection(api.Connection): except NoResultFound: raise exception.CertificateNotFound(uuid) query.delete() + + def _helm_override_get(self, name): + query = model_query(models.HelmOverrides) + query = query.filter_by(name=name) + try: + return query.one() + except NoResultFound: + raise exception.HelmOverrideNotFound(name) + + @objects.objectify(objects.helm_overrides) + def helm_override_create(self, values): + + overrides = models.HelmOverrides() + overrides.update(values) + with _session_for_write() as session: + try: + session.add(overrides) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add HelmOverrides %s. " + "Already exists with this name" % + (values['name'])) + raise exception.HelmOverrideAlreadyExists(name=values['name']) + return self._helm_override_get(values['name']) + + @objects.objectify(objects.helm_overrides) + def helm_override_get(self, name): + return self._helm_override_get(name) + + @objects.objectify(objects.helm_overrides) + def helm_override_update(self, name, values): + with _session_for_write() as session: + query = model_query(models.HelmOverrides, session=session) + query = query.filter_by(name=name) + + count = query.update(values, synchronize_session='fetch') + if count == 0: + raise exception.HelmOverrideNotFound(name) + return query.one() + + def helm_override_destroy(self, name): + with _session_for_write() as session: + query = model_query(models.HelmOverrides, session=session) + query = query.filter_by(name=name) + + try: + query.one() + except NoResultFound: + raise exception.HelmOverrideNotFound(name) + query.delete() diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/073_helm_overrides.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/073_helm_overrides.py new file mode 100644 index 0000000000..edf8009ec0 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/073_helm_overrides.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import DateTime, String, Text +from sqlalchemy import Column, MetaData, Table + +from sysinv.openstack.common import log + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + +LOG = log.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + This database upgrade creates a new table for storing helm chart + user-specified override values. + """ + + meta = MetaData() + meta.bind = migrate_engine + + # Define and create the helm_overrides table. + helm_overrides = Table( + 'helm_overrides', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('name', String(255), unique=True, index=True), + Column('user_overrides', Text, nullable=True), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + helm_overrides.create() + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # As per other openstack components, downgrade is + # unsupported in this release. + raise NotImplementedError('SysInv database downgrade is unsupported.') diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index d5c8409013..78c5d59066 100755 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -1635,3 +1635,10 @@ class certificate(Base): start_date = Column(DateTime(timezone=False)) expiry_date = Column(DateTime(timezone=False)) capabilities = Column(JSONEncodedDict) + + +class HelmOverrides(Base): + __tablename__ = 'helm_overrides' + + name = Column(String(255), primary_key=True, unique=True) + user_overrides = Column(Text, nullable=True) diff --git a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py index 95344a0fe0..ab2d508913 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py @@ -36,6 +36,7 @@ from sysinv.objects import drbdconfig from sysinv.objects import port_ethernet from sysinv.objects import event_log from sysinv.objects import event_suppression +from sysinv.objects import helm_overrides from sysinv.objects import host from sysinv.objects import host_upgrade from sysinv.objects import network_infra @@ -179,6 +180,7 @@ storage_file = storage_file.StorageFile storage_external = storage_external.StorageExternal storage_tier = storage_tier.StorageTier storage_ceph_external = storage_ceph_external.StorageCephExternal +helm_overrides = helm_overrides.HelmOverrides __all__ = (system, cluster, @@ -245,6 +247,7 @@ __all__ = (system, storage_external, storage_tier, storage_ceph_external, + helm_overrides, # alias objects for RPC compatibility ihost, ilvg, diff --git a/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py b/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py new file mode 100644 index 0000000000..bede8824a5 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py @@ -0,0 +1,29 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sysinv.db import api as db_api +from sysinv.objects import base +from sysinv.objects import utils + + +class HelmOverrides(base.SysinvObject): + # VERSION 1.0: Initial version + VERSION = '1.0' + + dbapi = db_api.get_instance() + + fields = {'name': utils.str_or_none, + 'user_overrides': utils.str_or_none, + } + + @base.remotable_classmethod + def get_by_name(cls, context, name): + return cls.dbapi.helm_override_get(name) + + def save_changes(self, context, updates): + self.dbapi.helm_override_update(self.name, updates)