diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data index 0210427cbd..1d0342a790 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data @@ -1,3 +1,3 @@ SRC_DIR="stx-openstack-helm" COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner $PKG_BASE/../../../helm-charts/garbd $PKG_BASE/../../../helm-charts/ceph-pools-audit" -TIS_PATCH_VER=4 +TIS_PATCH_VER=5 diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml index d18f66412f..dd36a9fbf7 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml @@ -965,6 +965,10 @@ data: release_group: osh-openstack-heat component: test values: + endpoints: + oslo_cache: + hosts: + default: heat-memcached labels: api: node_selector_key: openstack-control-plane @@ -1141,6 +1145,45 @@ data: anti: type: default: requiredDuringSchedulingIgnoredDuringExecution + gnocchi: + indexer: + driver: mariadb + keystone_authtoken: + interface: internal + dependencies: + static: + db_sync: + jobs: + - gnocchi-storage-init + - gnocchi-db-init + services: + - endpoint: internal + service: oslo_db + metricd: + services: + - endpoint: internal + service: oslo_db + - endpoint: internal + service: oslo_cache + - endpoint: internal + service: metric + tests: + services: + - endpoint: internal + service: identity + - endpoint: internal + service: oslo_db + - endpoint: internal + service: metric + manifests: + daemonset_statsd: false + job_db_init_indexer: false + secret_db_indexer: false + service_statsd: false + endpoints: + oslo_cache: + hosts: + default: memcached source: type: tar location: http://172.17.0.1/helm_charts/gnocchi-0.1.0.tgz diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index 0e5e8617c9..348a5c4e31 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -965,6 +965,10 @@ data: release_group: osh-openstack-heat component: test values: + endpoints: + oslo_cache: + hosts: + default: heat-memcached labels: api: node_selector_key: openstack-control-plane @@ -1141,6 +1145,45 @@ data: anti: type: default: requiredDuringSchedulingIgnoredDuringExecution + gnocchi: + indexer: + driver: mariadb + keystone_authtoken: + interface: internal + dependencies: + static: + db_sync: + jobs: + - gnocchi-storage-init + - gnocchi-db-init + services: + - endpoint: internal + service: oslo_db + metricd: + services: + - endpoint: internal + service: oslo_db + - endpoint: internal + service: oslo_cache + - endpoint: internal + service: metric + tests: + services: + - endpoint: internal + service: identity + - endpoint: internal + service: oslo_db + - endpoint: internal + service: metric + manifests: + daemonset_statsd: false + job_db_init_indexer: false + secret_db_indexer: false + service_statsd: false + endpoints: + oslo_cache: + hosts: + default: memcached source: type: tar location: http://172.17.0.1/helm_charts/gnocchi-0.1.0.tgz diff --git a/sysinv/sysinv/sysinv/sysinv/agent/manager.py b/sysinv/sysinv/sysinv/sysinv/agent/manager.py index 6ee010f05c..08f1d6b640 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/manager.py @@ -1291,6 +1291,12 @@ class AgentManager(service.PeriodicService): fileinput.close() os.rename(temp_platform_conf_file, tsc.PLATFORM_CONF_FILE) + def _retry_on_personality_is_none(ex): + LOG.info('Caught exception. Retrying... Exception: {}'.format(ex)) + return isinstance(ex, exception.LocalManagementPersonalityNotFound) + + @retrying.retry(wait_fixed=10 * 1000, stop_max_delay=300 * 1000, + retry_on_exception=_retry_on_personality_is_none) @utils.synchronized(LOCK_AGENT_ACTION, external=False) def iconfig_update_file(self, context, iconfig_uuid, iconfig_dict): """Configure the iiconfig_uuid, by updating file based upon @@ -1314,6 +1320,11 @@ class AgentManager(service.PeriodicService): if not permissions: permissions = constants.CONFIG_FILE_PERMISSION_DEFAULT + if not self._ihost_personality: + raise exception.LocalManagementPersonalityNotFound( + config_uuid=iconfig_uuid, config_dict=iconfig_dict, + host_personality=self._ihost_personality) + if self._ihost_personality in iconfig_dict['personalities']: file_content = iconfig_dict['file_content'] diff --git a/sysinv/sysinv/sysinv/sysinv/common/exception.py b/sysinv/sysinv/sysinv/sysinv/common/exception.py index 3b96cb78b6..dd9f0d4527 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/exception.py +++ b/sysinv/sysinv/sysinv/sysinv/common/exception.py @@ -1298,6 +1298,12 @@ class InvalidHelmNamespace(Invalid): message = _("Invalid helm overrides namespace (%(namespace)s) for chart %(chart)s.") +class LocalManagementPersonalityNotFound(NotFound): + message = _("Local management personality is None: " + "config_uuid=%(config_uuid)s, config_dict=%(config_dict)s, " + "host_personality=%(host_personality)s") + + class LocalManagementIpNotFound(NotFound): message = _("Local management IP not found: " "config_uuid=%(config_uuid)s, config_dict=%(config_dict)s, " diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 9c5d7e2ea9..41c3b9b961 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -1183,6 +1183,37 @@ class DockerHelper(object): def __init__(self, dbapi): self._dbapi = dbapi + self.k8s_registry = None + self.gcr_registry = None + self.quay_registry = None + self.docker_registry = None + + def _get_registry_parameters(self): + try: + registry = self._dbapi.service_parameter_get_all( + service=constants.SERVICE_TYPE_DOCKER, + section=constants.SERVICE_PARAM_SECTION_DOCKER_REGISTRY, + ) + return registry + except Exception: + return None + + def _retrieve_specified_registries(self): + registry_params = self._get_registry_parameters() + if registry_params: + for param in registry_params: + if param.name == \ + constants.SERVICE_PARAM_NAME_DOCKER_K8S_REGISTRY: + self.k8s_registry = str(param.value) + if param.name == \ + constants.SERVICE_PARAM_NAME_DOCKER_GCR_REGISTRY: + self.gcr_registry = str(param.value) + if param.name == \ + constants.SERVICE_PARAM_NAME_DOCKER_QUAY_REGISTRY: + self.quay_registry = str(param.value) + if param.name == \ + constants.SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY: + self.docker_registry = str(param.value) def _start_armada_service(self, client): try: @@ -1317,9 +1348,50 @@ class DockerHelper(object): registry_server = '{}:{}'.format(registry_ip, common.REGISTRY_PORT) return registry_server + def _get_img_tag_with_registry(self, pub_img_tag): + registry_name = pub_img_tag[0:1 + pub_img_tag.find('/')] + img_name = pub_img_tag[1 + pub_img_tag.find('/'):] + if registry_name: + if 'k8s.gcr.io' in registry_name: + registry = self.k8s_registry + elif 'gcr.io' in registry_name: + registry = self.gcr_registry + elif 'quay.io' in registry_name: + registry = self.quay_registry + elif 'docker.io' in registry_name: + registry = self.docker_registry + else: + # try docker.io registry as default + # if other registries newly added + # or docker.io repository detected + LOG.info("Registry %s not recognized or docker.io repository " + "detected. Pulling from public/private registry" + % registry_name) + registry = self.docker_registry + if registry: + return str(registry) + '/' + pub_img_tag + else: + return pub_img_tag + + # replace registry + if registry: + return str(registry) + '/' + img_name + else: + return pub_img_tag + else: + # docker.io registry as default + # if no registries specified in img tag + registry = self.docker_registry + if registry: + return str(registry) + '/' + pub_img_tag + else: + return pub_img_tag + def download_an_image(self, img_tag): rc = True + # retrieve user specified registries first + self._retrieve_specified_registries() local_registry_server = self.get_local_docker_registry_server() start = time.time() @@ -1333,17 +1405,20 @@ class DockerHelper(object): try: # Pull the image from the public registry LOG.info("Image %s is not available in local registry, " - "download started from public registry" % img_tag) + "download started from public/private registry" + % img_tag) pub_img_tag = img_tag.replace(local_registry_server + "/", "") - client.pull(pub_img_tag) + target_img_tag = self._get_img_tag_with_registry(pub_img_tag) + client.pull(target_img_tag) except Exception as e: rc = False - LOG.error("Image %s download failed from public registry: %s" % (pub_img_tag, e)) + LOG.error("Image %s download failed from public/private" + "registry: %s" % (target_img_tag, e)) return img_tag, rc try: # Tag and push the image to the local registry - client.tag(pub_img_tag, img_tag) + client.tag(target_img_tag, img_tag) client.push(img_tag, auth_config=local_registry_auth) except Exception as e: rc = False @@ -1354,9 +1429,11 @@ class DockerHelper(object): else: try: - LOG.info("Image %s download started from public registry" % img_tag) + LOG.info("Image %s download started from public/private registry" % img_tag) client = docker.APIClient(timeout=INSTALLATION_TIMEOUT) - client.pull(img_tag) + target_img_tag = self._get_img_tag_with_registry(img_tag) + client.pull(target_img_tag) + client.tag(target_img_tag, img_tag) except Exception as e: rc = False LOG.error("Image %s download failed from public registry: %s" % (img_tag, e)) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py index 40ff6b51d3..829fe2d47c 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # diff --git a/sysinv/sysinv/sysinv/sysinv/helm/glance.py b/sysinv/sysinv/sysinv/sysinv/helm/glance.py index 737d8e63b7..b3fc0d4392 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/glance.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/glance.py @@ -36,6 +36,7 @@ class GlanceHelm(openstack.OpenstackBaseHelm): 'storage': self._get_storage_overrides(), 'conf': self._get_conf_overrides(), 'images': self._get_images_overrides(), + 'bootstrap': self._get_bootstrap_overrides() } } @@ -161,6 +162,15 @@ class GlanceHelm(openstack.OpenstackBaseHelm): return conf + def _get_bootstrap_overrides(self): + # By default, prevent the download and creation of the Cirros image. + # TODO: Remove if/when pulling from external registries is supported. + bootstrap = { + 'enabled': False + } + + return bootstrap + def _get_primary_ceph_backend(self): try: backend = self.dbapi.storage_backend_get_by_name( diff --git a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py index 4d08caa3c8..9595b19faf 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -26,11 +26,6 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): common.HELM_NS_OPENSTACK: { 'images': self._get_images_overrides(), 'pod': self._get_pod_overrides(), - 'conf': self._get_conf_overrides(), - 'dependencies': { - 'static': self._get_static_dependencies_overrides() - }, - 'manifests': self._get_manifests_overrides(), 'endpoints': self._get_endpoints_overrides(), } } @@ -68,53 +63,6 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): } } - def _get_static_dependencies_overrides(self): - return { - 'db_sync': { - 'jobs': [ - 'gnocchi-storage-init', - 'gnocchi-db-init', - ], - 'services': [ - {'endpoint': 'internal', 'service': 'oslo_db'} - ] - }, - 'metricd': { - 'services': [ - {'endpoint': 'internal', 'service': 'oslo_db'}, - {'endpoint': 'internal', 'service': 'oslo_cache'}, - {'endpoint': 'internal', 'service': 'metric'} - ] - }, - 'tests': { - 'services': [ - {'endpoint': 'internal', 'service': 'identity'}, - {'endpoint': 'internal', 'service': 'oslo_db'}, - {'endpoint': 'internal', 'service': 'metric'} - ] - } - } - - def _get_manifests_overrides(self): - return { - 'daemonset_statsd': False, - 'service_statsd': False, - 'job_db_init_indexer': False, - 'secret_db_indexer': False, - } - - def _get_conf_overrides(self): - return { - 'gnocchi': { - 'indexer': { - 'driver': 'mariadb' - }, - 'keystone_authtoken': { - 'interface': 'internal' - } - } - } - def _get_endpoints_overrides(self): return { 'identity': { @@ -125,9 +73,6 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): 'auth': { 'memcached_secret_key': self._get_common_password('auth_memcache_key') - }, - 'hosts': { - 'default': 'memcached' } }, 'oslo_db': { diff --git a/sysinv/sysinv/sysinv/sysinv/helm/heat.py b/sysinv/sysinv/sysinv/sysinv/helm/heat.py index 8a5af46f7a..d988b88b60 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/heat.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/heat.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -73,11 +73,6 @@ class HeatHelm(openstack.OpenstackBaseHelm): 'auth': self._get_endpoints_identity_overrides( self.SERVICE_NAME, self.AUTH_USERS), }, - 'oslo_cache': { - 'hosts': { - 'default': 'heat-memcached' - } - }, 'oslo_db': { 'auth': self._get_endpoints_oslo_db_overrides( self.SERVICE_NAME, [self.SERVICE_NAME])