diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 50e4d69ccc..5c7992f215 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1474,7 +1474,7 @@ K8S_RBD_PROV_STOR_CLASS_NAME = 'general' ################################## # Working paths APP_INSTALL_PATH = '/scratch/apps' -APP_MANIFEST_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION) +APP_SYNCED_DATA_PATH = os.path.join(tsc.PLATFORM_PATH, 'armada', tsc.SW_VERSION) # State constants APP_UPLOAD_IN_PROGRESS = 'uploading' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 22c4608ac3..d3f45a05f0 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -49,6 +49,7 @@ MAX_DOWNLOAD_THREAD = 20 INSTALLATION_TIMEOUT = 3600 APPLY_SEARCH_PATTERN = 'Processing Chart,' DELETE_SEARCH_PATTERN = 'Deleting release' +ARMADA_MANIFEST_APPLY_SUCCESS_MSG = 'Done applying manifest' CONTAINER_ABNORMAL_EXIT_CODE = 137 @@ -72,10 +73,12 @@ class AppOperator(object): os.path.exists(os.path.join(app.path, 'metadata.yaml'))): self._process_node_labels(app, op=constants.LABEL_REMOVE_OP) if app.system_app and app.status != constants.APP_UPLOAD_FAILURE: - self._remove_chart_overrides(app.mfile_abs) + self._remove_chart_overrides(app.armada_mfile_abs) os.unlink(app.armada_mfile_abs) - shutil.rmtree(app.path) + os.unlink(app.imgfile_abs) + if os.path.exists(app.path): + shutil.rmtree(app.path) except OSError as e: LOG.exception(e) @@ -94,7 +97,7 @@ class AppOperator(object): def _abort_operation(self, app, operation): if (app.status == constants.APP_UPLOAD_IN_PROGRESS): self._update_app_status(app, constants.APP_UPLOAD_FAILURE, - constants.APP_PRORESS_ABORTED) + constants.APP_PROGRESS_ABORTED) elif (app.status == constants.APP_APPLY_IN_PROGRESS): self._update_app_status(app, constants.APP_APPLY_FAILURE, constants.APP_PROGRESS_ABORTED) @@ -109,10 +112,13 @@ class AppOperator(object): name=app.name, reason="failed to extract tarfile content.") try: + # One time set up of install path per controller if not os.path.isdir(constants.APP_INSTALL_PATH): - # One time set up os.makedirs(constants.APP_INSTALL_PATH) - os.makedirs(constants.APP_MANIFEST_PATH) + + # One time set up of Armada manifest path for the system + if not os.path.isdir(constants.APP_SYNCED_DATA_PATH): + os.makedirs(constants.APP_SYNCED_DATA_PATH) if not os.path.isdir(app.path): os.makedirs(app.path) @@ -171,9 +177,10 @@ class AppOperator(object): # Either this chart does not have overrides file or image tags are # not in its overrides file, walk the chart path to find image tags chart_path = os.path.join(app_path, chart.name) - tags = self._get_image_tags_by_path(chart_path) - if tags: - image_tags.extend(tags) + if os.path.exists(chart_path): + tags = self._get_image_tags_by_path(chart_path) + if tags: + image_tags.extend(tags) return list(set(image_tags)) @@ -193,11 +200,15 @@ class AppOperator(object): name=app.name, reason="embedded images are not yet supported.") - def _download_images(self, app): - if os.path.isdir(app.images_dir): - return self._register_embedded_images(app) - + def _save_images_list(self, app): + # Extract the list of images from the charts and overrides where + # applicable. Save the list to the same location as the armada manifest + # so it can be sync'ed. if app.system_app: + LOG.info("Generating application overrides...") + self._helm.generate_helm_application_overrides( + app.name, cnamespace=None, armada_format=True, combined=True) + app.charts = self._get_list_of_charts(app.armada_mfile_abs) # Grab the image tags from the overrides. If they don't exist # then mine them from the chart paths. images_to_download = self._get_image_tags_by_charts(app.charts_dir, @@ -206,11 +217,40 @@ class AppOperator(object): # For custom apps, mine image tags from application path images_to_download = self._get_image_tags_by_path(app.path) - if images_to_download is None: - raise exception.KubeAppApplyFailure( + if not images_to_download: + raise exception.KubeAppUploadFailure( name=app.name, reason="charts specify no docker images.") + with open(app.imgfile_abs, 'wb') as f: + yaml.safe_dump(images_to_download, f, explicit_start=True, + default_flow_style=False) + + def _retrieve_images_list(self, app_images_file): + with open(app_images_file, 'rb') as f: + images_list = yaml.load(f) + return images_list + + def _download_images(self, app): + if os.path.isdir(app.images_dir): + return self._register_embedded_images(app) + + if app.system_app: + # Some images could have been overwritten via user overrides + # between upload and apply, or between applies. Refresh the + # saved images list. + saved_images_list = self._retrieve_images_list(app.imgfile_abs) + combined_images_list = list(saved_images_list) + combined_images_list.extend( + self._get_image_tags_by_charts(app.charts_dir, app.charts)) + images_to_download = list(set(combined_images_list)) + if saved_images_list != images_to_download: + with open(app.imgfile_abs, 'wb') as f: + yaml.safe_dump(images_to_download, f, explicit_start=True, + default_flow_style=False) + else: + images_to_download = self._retrieve_images_list(app.imgfile_abs) + total_count = len(images_to_download) threads = min(MAX_DOWNLOAD_THREAD, total_count) failed_downloads = [] @@ -577,6 +617,8 @@ class AppOperator(object): # user can upload helm charts os.chmod('/scratch', 0o755) self._upload_helm_charts(app) + + self._save_images_list(app) self._update_app_status(app, constants.APP_UPLOAD_SUCCESS) LOG.info("Application (%s) upload completed." % app.name) except Exception as e: @@ -613,7 +655,7 @@ class AppOperator(object): overrides_str = '' ready = True try: - app.charts = self._get_list_of_charts(app.mfile_abs) + app.charts = self._get_list_of_charts(app.armada_mfile_abs) if app.system_app: self._update_app_status( app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES) @@ -670,7 +712,7 @@ class AppOperator(object): app = AppOperator.Application(rpc_app) LOG.info("Application (%s) remove started." % app.name) - app.charts = self._get_list_of_charts(app.mfile_abs) + app.charts = self._get_list_of_charts(app.armada_mfile_abs) self._update_app_status( app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST) @@ -743,13 +785,16 @@ class AppOperator(object): os.path.join('/manifests', self._kube_app.get('name') + "-" + self._kube_app.get('manifest_file')) self.armada_mfile_abs =\ - os.path.join(constants.APP_MANIFEST_PATH, + os.path.join(constants.APP_SYNCED_DATA_PATH, self._kube_app.get('name') + "-" + self._kube_app.get('manifest_file')) self.mfile_abs =\ os.path.join(constants.APP_INSTALL_PATH, self._kube_app.get('name'), self._kube_app.get('manifest_file')) + self.imgfile_abs =\ + os.path.join(constants.APP_SYNCED_DATA_PATH, + self._kube_app.get('name') + "-images.yaml") self.charts = [] @property @@ -790,13 +835,13 @@ class DockerHelper(object): try: # First make kubernetes config accessible to Armada. This # is a work around the permission issue in Armada container. - install_dir = constants.APP_INSTALL_PATH - kube_config = os.path.join(install_dir, 'admin.conf') + kube_config = os.path.join(constants.APP_SYNCED_DATA_PATH, + 'admin.conf') shutil.copy('/etc/kubernetes/admin.conf', kube_config) os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid) overrides_dir = common.HELM_OVERRIDES_PATH - manifests_dir = constants.APP_MANIFEST_PATH + manifests_dir = constants.APP_SYNCED_DATA_PATH LOG.info("kube_config=%s, manifests_dir=%s, " "overrides_dir=%s." % (kube_config, manifests_dir, overrides_dir)) @@ -857,8 +902,14 @@ class DockerHelper(object): LOG.info("Armada apply command = %s" % cmd) (exit_code, exec_logs) = armada_svc.exec_run(cmd) if exit_code == 0: - LOG.info("Application manifest %s was successfully " - "applied/re-applied." % manifest_file) + if ARMADA_MANIFEST_APPLY_SUCCESS_MSG in exec_logs: + LOG.info("Application manifest %s was successfully " + "applied/re-applied." % manifest_file) + else: + rc = False + LOG.error("Received a false positive response from " + "Docker/Armada. Failed to apply application " + "manifest %s: %s." % (manifest_file, exec_logs)) else: rc = False if exit_code == CONTAINER_ABNORMAL_EXIT_CODE: