# # Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # import os import pecan from pecan import rest import shutil import tempfile import wsme from wsme import types as wtypes import wsmeext.pecan as wsme_pecan from contextlib import contextmanager from oslo_log import log from sysinv import objects from sysinv.api.controllers.v1 import base from sysinv.api.controllers.v1 import collection from sysinv.api.controllers.v1 import patch_api from sysinv.api.controllers.v1 import types from sysinv.common import constants from sysinv.common import exception from sysinv.common import utils as cutils from sysinv.helm import common as helm_common from sysinv.openstack.common.gettextutils import _ import cgcs_patch.constants as patch_constants LOG = log.getLogger(__name__) @contextmanager def TempDirectory(): tmpdir = tempfile.mkdtemp() saved_umask = os.umask(0o077) try: yield tmpdir finally: LOG.debug("Cleaning up temp directory %s" % tmpdir) os.umask(saved_umask) shutil.rmtree(tmpdir) class KubeApp(base.APIBase): """API representation of a containerized application.""" id = int "Unique ID for this application" name = wtypes.text "Represents the name of the application" app_version = wtypes.text "Represents the version of the application" created_at = wtypes.datetime.datetime "Represents the time the application was uploaded" updated_at = wtypes.datetime.datetime "Represents the time the application was updated" manifest_name = wtypes.text "Represents the name of the application manifest" manifest_file = wtypes.text "Represents the filename of the application manifest" status = wtypes.text "Represents the installation status of the application" progress = wtypes.text "Represents the installation progress of the application" active = bool "Represents the application is active" def __init__(self, **kwargs): self.fields = objects.kube_app.fields.keys() for k in self.fields: if not hasattr(self, k): continue setattr(self, k, kwargs.get(k, wtypes.Unset)) @classmethod def convert_with_links(cls, rpc_app, expand=True): app = KubeApp(**rpc_app.as_dict()) if not expand: app.unset_fields_except(['name', 'app_version', 'manifest_name', 'manifest_file', 'status', 'progress']) # skip the id app.id = wtypes.Unset return app class KubeAppCollection(collection.Collection): """API representation of a collection of Helm applications.""" apps = [KubeApp] "A list containing application objects" def __init__(self, **kwargs): self._type = 'apps' @classmethod def convert_with_links(cls, rpc_apps, expand=False): collection = KubeAppCollection() collection.apps = [KubeApp.convert_with_links(n, expand) for n in rpc_apps] return collection LOCK_NAME = 'KubeAppController' class KubeAppController(rest.RestController): """REST controller for Helm applications.""" _custom_actions = { 'update': ['POST'], } def __init__(self, parent=None, **kwargs): self._parent = parent def _check_tarfile(self, app_tarfile, app_name, app_version, operation): def _handle_upload_failure(reason): raise wsme.exc.ClientSideError(_( "Application-{} rejected: ".format(operation) + reason)) if app_tarfile: if cutils.is_url(app_tarfile): # For tarfile that is downloaded remotely, defer the checksum, manifest # and tarfile content validations to sysinv-conductor as download can # take some time depending on network traffic, target server and file # size. if not app_name: app_name = constants.APP_NAME_PLACEHOLDER if not app_version: app_version = constants.APP_VERSION_PLACEHOLDER mname = constants.APP_MANIFEST_NAME_PLACEHOLDER mfile = constants.APP_TARFILE_NAME_PLACEHOLDER return app_name, app_version, mname, mfile if not os.path.isfile(app_tarfile): _handle_upload_failure( "application tar file {} does not exist.".format(app_tarfile)) if (not app_tarfile.endswith('.tgz') and not app_tarfile.endswith('.tar.gz')): _handle_upload_failure( "{} has unrecognizable tar file extension. Supported " "extensions are: .tgz and .tar.gz.".format(app_tarfile)) with TempDirectory() as app_path: if not cutils.extract_tarfile(app_path, app_tarfile): _handle_upload_failure( "failed to extract tar file {}.".format(os.path.basename(app_tarfile))) # If checksum file is included in the tarball, verify its contents. if not cutils.verify_checksum(app_path): _handle_upload_failure("checksum validation failed.") app_helper = KubeAppHelper(pecan.request.dbapi) try: name, version, patches = app_helper._verify_metadata_file( app_path, app_name, app_version) mname, mfile = app_helper._find_manifest_file(app_path) app_helper._extract_helm_charts(app_path) LOG.info("Tar file of application %s verified." % name) return name, version, mname, mfile except exception.SysinvException as e: _handle_upload_failure(str(e)) else: raise ValueError(_( "Application-{} rejected: tar file must be specified.".format(operation))) def _get_one(self, app_name): # can result in KubeAppNotFound kube_app = objects.kube_app.get_by_name( pecan.request.context, app_name) return KubeApp.convert_with_links(kube_app) @wsme_pecan.wsexpose(KubeAppCollection) def get_all(self): apps = pecan.request.dbapi.kube_app_get_all() return KubeAppCollection.convert_with_links(apps) @wsme_pecan.wsexpose(KubeApp, wtypes.text) def get_one(self, app_name): """Retrieve a single application.""" return self._get_one(app_name) @staticmethod def _check_controller_labels(chosts): def _check_monitor_controller_labels(host_uuid, hostname): labels = pecan.request.dbapi.label_get_by_host(host_uuid) required_labels = { helm_common.LABEL_MONITOR_CONTROLLER: helm_common.LABEL_VALUE_ENABLED, helm_common.LABEL_MONITOR_DATA: helm_common.LABEL_VALUE_ENABLED, helm_common.LABEL_MONITOR_CLIENT: helm_common.LABEL_VALUE_ENABLED} assigned_labels = {} for label in labels: if label.label_key in required_labels: if label.label_value == required_labels[label.label_key]: assigned_labels.update( {label.label_key: label.label_value}) missing_labels = {k: required_labels[k] for k in set(required_labels) - set(assigned_labels)} msg = "" if missing_labels: for k, v in missing_labels.items(): msg += "%s=%s " % (k, v) if msg: msg = " 'system host-label-assign {} {}'".format( hostname, msg) return msg client_msg = "" for chost in chosts: msg = _check_monitor_controller_labels( chost.uuid, chost.hostname) if msg: client_msg += msg if client_msg: raise wsme.exc.ClientSideError( _("Operation rejected: application stx-monitor " "requires labels on controllers. {}".format(client_msg))) def _semantic_check(self, db_app): """Semantic check for application deployment """ if db_app.name == constants.HELM_APP_MONITOR: chosts = pecan.request.dbapi.ihost_get_by_personality( constants.CONTROLLER) if not cutils.is_aio_simplex_system(pecan.request.dbapi): if chosts and len(chosts) < 2: raise wsme.exc.ClientSideError(_( "Operation rejected: application {} requires 2 " "controllers".format(db_app.name))) self._check_controller_labels(chosts) for chost in chosts: if (chost.administrative != constants.ADMIN_UNLOCKED or chost.operational != constants.OPERATIONAL_ENABLED): raise wsme.exc.ClientSideError(_( "Operation rejected: application {} requires {} to be " "unlocked-enabled".format( db_app.name, chost.hostname))) @cutils.synchronized(LOCK_NAME) @wsme_pecan.wsexpose(KubeApp, body=types.apidict) def post(self, body): """Uploading an application to be deployed by Armada""" tarfile = body.get('tarfile') name = body.get('name', '') version = body.get('app_version', '') name, version, mname, mfile = self._check_tarfile(tarfile, name, version, constants.APP_UPLOAD_OP) try: objects.kube_app.get_by_name(pecan.request.context, name) raise wsme.exc.ClientSideError(_( "Application-upload rejected: application {} already exists.".format( name))) except exception.KubeAppNotFound: pass # Create a database entry and make an rpc async request to upload # the application app_data = {'name': name, 'app_version': version, 'manifest_name': mname, 'manifest_file': os.path.basename(mfile), 'status': constants.APP_UPLOAD_IN_PROGRESS} try: new_app = pecan.request.dbapi.kube_app_create(app_data) except exception.SysinvException as e: LOG.exception(e) raise pecan.request.rpcapi.perform_app_upload(pecan.request.context, new_app, tarfile) return KubeApp.convert_with_links(new_app) @cutils.synchronized(LOCK_NAME) @wsme_pecan.wsexpose(KubeApp, wtypes.text, wtypes.text, wtypes.text) def patch(self, name, directive, values): """Install/update the specified application :param name: application name :param directive: either 'apply' (fresh install/update), 'remove' or 'abort' """ if directive not in ['apply', 'remove', 'abort']: raise exception.OperationNotPermitted try: db_app = objects.kube_app.get_by_name(pecan.request.context, name) except exception.KubeAppNotFound: LOG.error("Received a request to %s app %s which does not exist." % (directive, name)) raise wsme.exc.ClientSideError(_( "Application-{} rejected: application not found.".format(directive))) if directive == 'apply': if not values: mode = None elif name not in constants.HELM_APP_APPLY_MODES.keys(): raise wsme.exc.ClientSideError(_( "Application-apply rejected: Mode is not supported " "for app {}.".format(name))) elif (values['mode'] and values['mode'] not in constants.HELM_APP_APPLY_MODES[name]): raise wsme.exc.ClientSideError(_( "Application-apply rejected: Mode {} for app {} is not " "valid. Valid modes are {}.".format( values['mode'], name, constants.HELM_APP_APPLY_MODES[name]))) else: mode = values['mode'] self._semantic_check(db_app) if db_app.status == constants.APP_APPLY_IN_PROGRESS: raise wsme.exc.ClientSideError(_( "Application-apply rejected: install/update is already " "in progress.")) elif db_app.status not in [constants.APP_UPLOAD_SUCCESS, constants.APP_APPLY_FAILURE, constants.APP_APPLY_SUCCESS]: raise wsme.exc.ClientSideError(_( "Application-apply rejected: operation is not allowed " "while the current status is {}.".format(db_app.status))) db_app.status = constants.APP_APPLY_IN_PROGRESS db_app.progress = None db_app.recovery_attempts = 0 db_app.save() pecan.request.rpcapi.perform_app_apply(pecan.request.context, db_app, mode=mode) elif directive == 'remove': if db_app.status not in [constants.APP_APPLY_SUCCESS, constants.APP_APPLY_FAILURE, constants.APP_REMOVE_FAILURE]: raise wsme.exc.ClientSideError(_( "Application-remove rejected: operation is not allowed while " "the current status is {}.".format(db_app.status))) db_app.status = constants.APP_REMOVE_IN_PROGRESS db_app.progress = None db_app.save() pecan.request.rpcapi.perform_app_remove(pecan.request.context, db_app) else: if db_app.status not in [constants.APP_APPLY_IN_PROGRESS, constants.APP_UPDATE_IN_PROGRESS, constants.APP_REMOVE_IN_PROGRESS]: raise wsme.exc.ClientSideError(_( "Application-abort rejected: operation is not allowed while " "the current status is {}.".format(db_app.status))) pecan.request.rpcapi.perform_app_abort(pecan.request.context, db_app) return KubeApp.convert_with_links(db_app) @cutils.synchronized(LOCK_NAME) @wsme_pecan.wsexpose(KubeApp, body=types.apidict) def update(self, body): """Update the applied application to a different version""" tarfile = body.get('tarfile') name = body.get('name', '') version = body.get('app_version', '') name, version, mname, mfile = self._check_tarfile(tarfile, name, version, constants.APP_UPDATE_OP) try: applied_app = objects.kube_app.get_by_name(pecan.request.context, name) except exception.KubeAppNotFound: LOG.error("Received a request to update app %s which does not exist." % name) raise wsme.exc.ClientSideError(_( "Application-update rejected: application not found.")) if applied_app.status == constants.APP_UPDATE_IN_PROGRESS: raise wsme.exc.ClientSideError(_( "Application-update rejected: update is already " "in progress.")) elif applied_app.status != constants.APP_APPLY_SUCCESS: raise wsme.exc.ClientSideError(_( "Application-update rejected: operation is not allowed " "while the current status is {}.".format(applied_app.status))) if applied_app.app_version == version: raise wsme.exc.ClientSideError(_( "Application-update rejected: the version %s is already " "applied." % version)) # Set the status for the current applied app to inactive applied_app.status = constants.APP_INACTIVE_STATE applied_app.progress = None applied_app.save() # If the version has ever applied before(inactive app found), # use armada rollback to apply application later, otherwise, # use armada apply. # On the AIO-SX, always use armada apply even it was applied # before, issue on AIO-SX(replicas is 1) to leverage rollback, # armada/helm rollback --wait does not wait for pods to be # ready before it returns. # related to helm issue, # https://github.com/helm/helm/issues/4210 # https://github.com/helm/helm/issues/2006 try: target_app = objects.kube_app.get_inactive_app_by_name_version( pecan.request.context, name, version) target_app.status = constants.APP_UPDATE_IN_PROGRESS target_app.save() if cutils.is_aio_simplex_system(pecan.request.dbapi): operation = constants.APP_APPLY_OP else: operation = constants.APP_ROLLBACK_OP except exception.KubeAppInactiveNotFound: target_app_data = { 'name': name, 'app_version': version, 'manifest_name': mname, 'manifest_file': os.path.basename(mfile), 'status': constants.APP_UPDATE_IN_PROGRESS, 'active': True } operation = constants.APP_APPLY_OP try: target_app = pecan.request.dbapi.kube_app_create(target_app_data) except exception.KubeAppAlreadyExists as e: applied_app.status = constants.APP_APPLY_SUCCESS applied_app.progress = constants.APP_PROGRESS_COMPLETED applied_app.save() LOG.exception(e) raise wsme.exc.ClientSideError(_( "Application-update failed: Unable to start application update, " "application info update failed.")) pecan.request.rpcapi.perform_app_update(pecan.request.context, applied_app, target_app, tarfile, operation) return KubeApp.convert_with_links(target_app) @cutils.synchronized(LOCK_NAME) @wsme_pecan.wsexpose(None, wtypes.text, status_code=204) def delete(self, name): """Delete the application with the given name :param name: application name """ try: db_app = objects.kube_app.get_by_name(pecan.request.context, name) except exception.KubeAppNotFound: LOG.error("Received a request to delete app %s which does not " "exist." % name) raise wsme.exc.ClientSideError(_( "Application-delete rejected: application not found.")) if db_app.status not in [constants.APP_UPLOAD_SUCCESS, constants.APP_UPLOAD_FAILURE]: raise wsme.exc.ClientSideError(_( "Application-delete rejected: operation is not allowed " "while the current status is {}.".format(db_app.status))) response = pecan.request.rpcapi.perform_app_delete( pecan.request.context, db_app) if response: raise wsme.exc.ClientSideError(_( "%s." % response)) class KubeAppHelper(object): def __init__(self, dbapi): self._dbapi = dbapi def _check_patching_operation(self): try: system = self._dbapi.isystem_get_one() response = patch_api.patch_query( token=None, timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, region_name=system.region_name ) query_patches = response['pd'] except Exception as e: # Assume that a patching operation is underway, raise an exception. LOG.error(_("No response from patch api: %s" % e)) raise for patch in query_patches: patch_state = query_patches[patch].get('patchstate', None) if (patch_state == patch_constants.PARTIAL_APPLY or patch_state == patch_constants.PARTIAL_REMOVE): raise exception.SysinvException(_( "Patching operation is in progress.")) def _check_patch_is_applied(self, patches): try: system = self._dbapi.isystem_get_one() response = patch_api.patch_is_applied( token=None, timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, region_name=system.region_name, patches=patches ) except Exception as e: LOG.error(e) raise exception.SysinvException(_( "Error while querying patch-controller for the " "state of the patch(es).")) return response def _patch_report_app_dependencies(self, name, patches=None): if patches is None: patches = [] try: system = self._dbapi.isystem_get_one() patch_api.patch_report_app_dependencies( token=None, timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS, region_name=system.region_name, patches=patches, app_name=name ) except Exception as e: LOG.error(e) raise exception.SysinvException( "Error while reporting the patch dependencies " "to patch-controller.") def _find_manifest_file(self, app_path): # It is expected that there is only one manifest file # per application and the file exists at top level of # the application path. mfiles = cutils.find_manifest_file(app_path) if mfiles is None: raise exception.SysinvException(_( "manifest file is corrupted.")) if mfiles: if len(mfiles) == 1: return mfiles[0] else: raise exception.SysinvException(_( "Application-upload rejected: tar file contains more " "than one manifest file.")) else: raise exception.SysinvException(_( "Application-upload rejected: manifest file is missing.")) def _verify_metadata_file(self, app_path, app_name, app_version): try: name, version, patches = cutils.find_metadata_file( app_path, constants.APP_METADATA_FILE) except exception.SysinvException as e: raise exception.SysinvException(_( "metadata validation failed. {}".format(e))) if not name: name = app_name if not version: version = app_version if (not name or not version or name == constants.APP_VERSION_PLACEHOLDER or version == constants.APP_VERSION_PLACEHOLDER): raise exception.SysinvException(_( "application name or/and version is/are not included " "in the tar file. Please specify the application name " "via --app-name or/and version via --app-version.")) if patches: try: self._check_patching_operation() except exception.SysinvException as e: raise exception.SysinvException(_( "{}. Please upload after the patching operation " "is completed.".format(e))) except Exception as e: raise exception.SysinvException(_( "{}. Communication Error with patching subsytem. " "Preventing application upload.".format(e))) applied = self._check_patch_is_applied(patches) if not applied: raise exception.SysinvException(_( "the required patch(es) for application {} ({}) " "must be applied".format(name, version))) LOG.info("The required patch(es) for application {} ({}) " "has/have applied.".format(name, version)) else: LOG.info("No patch required for application {} ({}).".format(name, version)) return name, version, patches def _extract_helm_charts(self, app_path, demote_user=False): charts_dir = os.path.join(app_path, 'charts') if os.path.isdir(charts_dir): tar_filelist = cutils.get_files_matching(app_path, '.tgz') if len(os.listdir(charts_dir)) == 0: raise exception.SysinvException(_( "tar file contains no Helm charts.")) if not tar_filelist: raise exception.SysinvException(_( "tar file contains no Helm charts of " "expected file extension (.tgz).")) for p, f in tar_filelist: if not cutils.extract_tarfile( p, os.path.join(p, f), demote_user): raise exception.SysinvException(_( "failed to extract tar file {}.".format(os.path.basename(f))))