Merge "Fix misleading app status after failed override update"

sysinv/sysinv/sysinv/etc/sysinv/sysinv.conf

@@ -20,3 +20,6 @@ rabbit_port = 5672
 
 [lldp]
 drivers=lldpd
+
+[app_framework]
+fluxcd_hr_reconcile_check_delay=60

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -41,6 +41,7 @@ from eventlet import Timeout
 from fm_api import constants as fm_constants
 from fm_api import fm_api
 from oslo_log import log as logging
+from oslo_config import cfg
 from oslo_serialization import base64
 from sysinv._i18n import _
 from sysinv.api.controllers.v1 import kube_app
@@ -57,6 +58,8 @@ from sysinv.helm import utils as helm_utils
 from sysinv.helm.lifecycle_constants import LifecycleConstants
 from sysinv.helm.lifecycle_hook import LifecycleHookInfo
 
+CONF = cfg.CONF
+
 
 # Log and config
 LOG = logging.getLogger(__name__)
@@ -1632,6 +1635,38 @@ class AppOperator(object):
 
                     raise exception.ApplicationApplyFailure(name=app.name)
 
+        def _apply_delay_to_check_progress(kubectl_apply_time):
+            """ Apply delay before checking the charts installation progress.
+                The purpose of this is to ensure that reconciliation has been
+                initiated before checking the helmrelease status of each chart.
+
+                The variable fluxcd_hr_reconcile_check_delay has been set in
+                sysinv.conf to the default amount of time. This value will be
+                override if this variable is present in the application's
+                metadata.
+
+                :param kubectl_apply_time: date when "kubectl apply -k" was called
+            """
+            fluxcd_hr_reconcile_check_delay = CONF.app_framework.fluxcd_hr_reconcile_check_delay
+
+            # Override default time delay if app metadata have
+            # fluxcd_hr_reconcile_check_delay variable
+            if "fluxcd_hr_reconcile_check_delay" in app.app_metadata:
+                fluxcd_hr_reconcile_check_delay = app.app_metadata[
+                    "fluxcd_hr_reconcile_check_delay"]
+
+            # If fluxcd_hr_reconcile_check_delay is 0, no delay will need to be added
+            # and the _check_progress function must be called normally.
+            if fluxcd_hr_reconcile_check_delay > 0:
+                now = time.time()
+                elapsed_time = now - kubectl_apply_time
+                delay_time = fluxcd_hr_reconcile_check_delay - elapsed_time
+
+                if fluxcd_hr_reconcile_check_delay > elapsed_time:
+                    LOG.info("Waiting {} seconds to make sure "
+                             "the reconciliation was called".format(int(delay_time)))
+                    time.sleep(delay_time)
+
         def _check_progress():
             tadjust = 0
             last_successful_chart = None
@@ -1755,10 +1790,17 @@ class AppOperator(object):
             with Timeout(constants.APP_INSTALLATION_TIMEOUT,
                          exception.KubeAppProgressMonitorTimeout()):
 
-                rc = self._fluxcd.make_fluxcd_operation(request, app.sync_fluxcd_manifest)
+                rc, kubectl_apply_time = self._fluxcd.make_fluxcd_operation(
+                    request,
+                    app.sync_fluxcd_manifest)
 
                 # check progress only for apply for now
                 if rc and request == constants.APP_APPLY_OP:
+                    # Because reconciliation is not called immediately after running the
+                    # "kubectl apply -k" command, a delay is applied before _check_progress
+                    # function so that it does not return old values.
+                    _apply_delay_to_check_progress(kubectl_apply_time)
+
                     rc = _check_progress()
         except (exception.ApplicationApplyFailure):
             raise
@@ -2002,8 +2044,9 @@ class AppOperator(object):
                     manifest_sync_dir_path, constants.APP_METADATA_FILE)
                 shutil.copy(inst_metadata_file, sync_metadata_file)
 
-            if not validate_function(constants.APP_VALIDATE_OP,
-                                     validate_manifest):
+            validation_result, _ = validate_function(constants.APP_VALIDATE_OP,
+                                                     validate_manifest)
+            if not validation_result:
                 raise exception.KubeAppUploadFailure(
                     name=app.name,
                     version=app.version,
@@ -4013,7 +4056,9 @@ class FluxCDHelper(object):
             LOG.error("FluxCD operation %s failed for manifest %s : %s" %
                       (operation, manifest_dir, e))
             rc = False
-        return rc
+
+        kubectl_apply_time = time.time()
+        return rc, kubectl_apply_time
 
     def run_kubectl_kustomize(self, operation_type, manifest_dir):
         if operation_type == constants.KUBECTL_KUSTOMIZE_VALIDATE:

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -187,9 +187,16 @@ audit_intervals_opts = [
        cfg.IntOpt('k8s_cluster_health', default=180),
                   ]
 
+app_framework_opts = [
+    cfg.IntOpt('fluxcd_hr_reconcile_check_delay',
+        default=60,
+        help='Delay time to check progress of helmrelease'),
+]
+
 CONF = cfg.CONF
 CONF.register_opts(conductor_opts, 'conductor')
 CONF.register_opts(audit_intervals_opts, 'conductor_periodic_task_intervals')
+CONF.register_opts(app_framework_opts, 'app_framework')
 
 # doesn't work otherwise for ceph-manager RPC calls; reply is lost
 #