Optimizing image downloads

In this commit, we obtain a list of images already present in
containerd to avoid unnecessary checks and pulls, reducing CPU
consumption.

TEST PLAN:
PASS: Lock/Unlock controllers
PASS: Successfully swact between controllers
PASS: Successfully recover after power down and up both controllers
PASS: Successfully bootstrap (Simplex and Duplex)
PASS: Successfully recover after active controller goes down
PASS: Successfully application lifecycle

Story: 2010985
Task: 49228

Change-Id: I58dd11c8d590b60ab100f79a03e17c5921e3721b
Signed-off-by: Thiago Miranda <tmarques@windriver.com>
Co-authored-by: Eduardo Juliano Alberti <eduardo.alberti@windriver.com>

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -2070,6 +2070,9 @@ APP_VERSION_PLACEHOLDER = 'app-version-placeholder'
 APP_MANIFEST_NAME_PLACEHOLDER = 'manifest-placeholder'
 APP_TARFILE_NAME_PLACEHOLDER = 'tarfile-placeholder'
 
+# Application constants
+APP_INSTALLATION_TIMEOUT = 3600
+
 # Default node labels
 CONTROL_PLANE_LABEL = 'openstack-control-plane=enabled'
 COMPUTE_NODE_LABEL = 'openstack-compute-node=enabled'

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -16,6 +16,7 @@ from eventlet.green import subprocess
 import glob
 import grp
 import functools
+import json
 import io
 import os
 import pkg_resources
@@ -65,7 +66,6 @@ APPLY_SEARCH_PATTERN = 'Processing Chart,'
 CONTAINER_ABNORMAL_EXIT_CODE = 137
 DELETE_SEARCH_PATTERN = 'Deleting release|no release to delete'
 ROLLBACK_SEARCH_PATTERN = 'Helm rollback of release'
-INSTALLATION_TIMEOUT = 3600
 MAX_DOWNLOAD_THREAD = 5
 MAX_DOWNLOAD_ATTEMPTS = 3
 DOWNLOAD_WAIT_BEFORE_RETRY = 15
@@ -821,6 +821,8 @@ class AppOperator(object):
         total_count = len(images_to_download)
         threads = min(MAX_DOWNLOAD_THREAD, total_count)
 
+        self._docker.set_crictl_image_list([])
+
         start = time.time()
         try:
             registries_info = self._docker.retrieve_specified_registries()
@@ -1750,7 +1752,7 @@ class AppOperator(object):
         lifecycle_hook_info.lifecycle_type = constants.APP_LIFECYCLE_TYPE_FLUXCD_REQUEST
         self.app_lifecycle_actions(None, None, app._kube_app, lifecycle_hook_info)
         try:
-            with Timeout(INSTALLATION_TIMEOUT,
+            with Timeout(constants.APP_INSTALLATION_TIMEOUT,
                          exception.KubeAppProgressMonitorTimeout()):
 
                 rc = self._fluxcd.make_fluxcd_operation(request, app.sync_fluxcd_manifest)
@@ -3327,6 +3329,27 @@ class DockerHelper(object):
 
     def __init__(self, dbapi):
         self._dbapi = dbapi
+        self._crictl_image_list = []
+
+    def _get_crictl_image_list(self):
+        cmd = ['crictl', 'images', '--output=json']
+        try:
+            output = subprocess.check_output(  # pylint: disable=not-callable
+                cmd, stderr=subprocess.STDOUT)
+            crictl_output = json.loads(output)
+        except json.JSONDecodeError as e:
+            LOG.error('Could not parse json output, error=%s', e)
+        except subprocess.CalledProcessError as e:
+            LOG.error('Could not list images, error=%s', e)
+        else:
+            self._crictl_image_list = []
+            for img in crictl_output['images']:
+                self._crictl_image_list.extend(img['repoTags'])
+
+        return self._crictl_image_list
+
+    def set_crictl_image_list(self, image_list):
+        self._crictl_image_list = image_list
 
     def _parse_barbican_secret(self, secret_ref):
         """Get the registry credentials from the
@@ -3451,6 +3474,9 @@ class DockerHelper(object):
 
         rc = True
 
+        if not self._crictl_image_list:
+            self._get_crictl_image_list()
+
         start = time.time()
         if img_tag.startswith(constants.DOCKER_REGISTRY_HOST):
             try:
@@ -3458,12 +3484,15 @@ class DockerHelper(object):
                     LOG.info("User aborted. Skipping download of image %s " % img_tag)
                     return img_tag, False
 
-                LOG.info("Image %s download started from local registry" % img_tag)
+                if img_tag not in self._crictl_image_list:
-                client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
+                    LOG.info("Image %s download started from local registry" % img_tag)
-                local_registry_auth = cutils.get_local_docker_registry_auth()
+                    local_registry_auth = cutils.get_local_docker_registry_auth()
-                auth = '{0}:{1}'.format(local_registry_auth['username'],
+                    auth = '{0}:{1}'.format(local_registry_auth['username'],
-                                        local_registry_auth['password'])
+                                            local_registry_auth['password'])
-                subprocess.check_call(["crictl", "pull", "--creds", auth, img_tag])  # pylint: disable=not-callable
+                    subprocess.check_call(  # pylint: disable=not-callable
+                        ["crictl", "pull", "--creds", auth, img_tag])
+                else:
+                    LOG.info("Image %s exists in the local registry" % img_tag)
             except subprocess.CalledProcessError:
                 try:
                     # Pull the image from the public/private registry
@@ -3477,6 +3506,8 @@ class DockerHelper(object):
                     target_img_tag, registry_auth = \
                         self._get_img_tag_with_registry(pub_img_tag, registries_info)
 
+                    client = docker.APIClient(
+                        timeout=constants.APP_INSTALLATION_TIMEOUT)
                     client.pull(target_img_tag, auth_config=registry_auth)
 
                 except Exception as e:
@@ -3515,7 +3546,7 @@ class DockerHelper(object):
         else:
             try:
                 LOG.info("Image %s download started from public/private registry" % img_tag)
-                client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
+                client = docker.APIClient(timeout=constants.APP_INSTALLATION_TIMEOUT)
                 target_img_tag, registry_auth = \
                     self._get_img_tag_with_registry(img_tag, registries_info)
                 client.pull(target_img_tag, auth_config=registry_auth)