diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/081_helm_system_overrides.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/081_helm_system_overrides.py new file mode 100644 index 0000000000..28c1084889 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/081_helm_system_overrides.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import Text +from sqlalchemy import Column, MetaData, Table + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + """ + This database upgrade creates a new column for storing passwords + on the helm chart override table. + """ + + meta = MetaData() + meta.bind = migrate_engine + + helm_overrides = Table('helm_overrides', meta, autoload=True) + helm_overrides.create_column(Column('system_overrides', Text, + nullable=True)) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + helm_overrides = Table('helm_overrides', meta, autoload=True) + helm_overrides.drop_column('system_overrides') diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index 9f7819c426..8d12133ccf 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -1619,6 +1619,7 @@ class HelmOverrides(Base): name = Column(String(255), nullable=False) namespace = Column(String(255), nullable=False) user_overrides = Column(Text, nullable=True) + system_overrides = Column(JSONEncodedDict, nullable=True) UniqueConstraint('name', 'namespace', name='u_name_namespace') diff --git a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py index 25a0caa890..33a0ce5eeb 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/ceilometer.py @@ -129,23 +129,20 @@ class CeilometerHelm(openstack.OpenstackBaseHelm): notification_overrides.update({'batch_timeout': batch_timeout}) return notification_overrides - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides() + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/cinder.py b/sysinv/sysinv/sysinv/sysinv/helm/cinder.py index 463eaaec3a..ac05ee140e 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/cinder.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/cinder.py @@ -182,26 +182,28 @@ class CinderHelm(openstack.OpenstackBaseHelm): return conf_backends - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, - user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides(), + 'identity': { + 'auth': + self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, + } def _get_images_overrides(self): diff --git a/sysinv/sysinv/sysinv/sysinv/helm/glance.py b/sysinv/sysinv/sysinv/sysinv/helm/glance.py index d9f28e8247..4d847b469a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/glance.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/glance.py @@ -85,26 +85,27 @@ class GlanceHelm(openstack.OpenstackBaseHelm): } } - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, - user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides(), + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, + } def _get_storage_overrides(self): diff --git a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py index 1eb86637b4..14aa20a5cd 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/gnocchi.py @@ -126,31 +126,23 @@ class GnocchiHelm(openstack.OpenstackBaseHelm): } } - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - - def _get_endpoints_cache_overrides(self): - return { - 'hosts': { - 'default': 'memcached' - } - } - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides(), - 'oslo_cache': self._get_endpoints_cache_overrides() + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + }, + 'hosts': { + 'default': 'memcached' + } + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/heat.py b/sysinv/sysinv/sysinv/sysinv/helm/heat.py index fa1e717321..f1975739e5 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/heat.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/heat.py @@ -80,23 +80,23 @@ class HeatHelm(openstack.OpenstackBaseHelm): } } - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides(), + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'hosts': { + 'default': 'heat-memcached' + } + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/horizon.py b/sysinv/sysinv/sysinv/sysinv/helm/horizon.py index 6429d1899d..84bf0ab12d 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/horizon.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/horizon.py @@ -67,7 +67,8 @@ class HorizonHelm(openstack.OpenstackBaseHelm): } } } - } + }, + 'endpoints': self._get_endpoints_overrides() } } @@ -79,6 +80,14 @@ class HorizonHelm(openstack.OpenstackBaseHelm): else: return overrides + def _get_endpoints_overrides(self): + return { + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, + } + def _get_images_overrides(self): heat_image = self._operator.chart_operators[ constants.HELM_CHART_HEAT].docker_image @@ -101,6 +110,9 @@ class HorizonHelm(openstack.OpenstackBaseHelm): 'https_enabled': 'False', 'lockout_period_sec': '300', 'lockout_retries_num': '3', + 'horizon_secret_key': self._get_or_generate_password( + self.SERVICE_NAME, common.HELM_NS_OPENSTACK, + 'horizon_secret_key'), # Optional Services 'enable_murano': 'False', diff --git a/sysinv/sysinv/sysinv/sysinv/helm/keystone.py b/sysinv/sysinv/sysinv/sysinv/helm/keystone.py index 2dd9e89585..8969ae653f 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/keystone.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/keystone.py @@ -246,33 +246,26 @@ class KeystoneHelm(openstack.OpenstackBaseHelm): else: return super(KeystoneHelm, self)._region_config() - def _get_endpoints_identity_users_overrides(self): - return self._get_common_users_overrides(self.SERVICE_NAME) - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - - def _get_endpoints_oslo_db_overrides(self): - # Keep the default overrides for db scheme/host/port/path - - # dbpass = self._get_database_password(self.SERVICE_NAME) - return { - 'auth': { - 'admin': { - 'username': 'root', - # 'password': dbpass - }, - 'keystone': { - 'username': self._get_database_username(self.SERVICE_NAME), - # 'password': dbpass - } - } - } - def _get_endpoints_overrides(self): return { - 'identity': self._get_endpoints_identity_overrides(), - 'oslo_db': self._get_endpoints_oslo_db_overrides(), + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, []), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, } def get_admin_user_name(self): diff --git a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py index 1559a00b5f..08584fe245 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/mariadb.py @@ -8,12 +8,12 @@ from sysinv.common import constants from sysinv.common import exception from sysinv.openstack.common import log as logging from . import common -from . import base +from . import openstack LOG = logging.getLogger(__name__) -class MariadbHelm(base.BaseHelm): +class MariadbHelm(openstack.OpenstackBaseHelm): """Class to encapsulate helm operations for the mariadb chart""" CHART = constants.HELM_CHART_MARIADB @@ -31,7 +31,8 @@ class MariadbHelm(base.BaseHelm): 'replicas': { 'server': 1 } - } + }, + 'endpoints': self._get_endpoints_overrides(), } } @@ -42,3 +43,11 @@ class MariadbHelm(base.BaseHelm): namespace=namespace) else: return overrides + + def _get_endpoints_overrides(self): + return { + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.CHART, []) + } + } diff --git a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py index 5dff34ae56..998d290294 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/neutron.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/neutron.py @@ -331,30 +331,36 @@ class NeutronHelm(openstack.OpenstackBaseHelm): } } - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, user) - } - }) - for user in self.SERVICE_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(user, user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): - return { - 'identity': self._get_endpoints_identity_overrides(), + overrides = { + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_db': { + 'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, self.AUTH_USERS) + }, } + + # Service user passwords already exist in other chart overrides + for user in self.SERVICE_USERS: + overrides['identity']['auth'].update({ + user: { + 'region_name': self._region_name(), + 'password': self._get_or_generate_password( + user, common.HELM_NS_OPENSTACK, user) + } + }) + + return overrides diff --git a/sysinv/sysinv/sysinv/sysinv/helm/nova.py b/sysinv/sysinv/sysinv/sysinv/helm/nova.py index a2d416ed59..35f9924a39 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/nova.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/nova.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # +import copy import os from sysinv.common import constants @@ -190,34 +191,45 @@ class NovaHelm(openstack.OpenstackBaseHelm): } } - def _get_endpoints_identity_users_overrides(self): - overrides = {} - overrides.update(self._get_common_users_overrides(self.SERVICE_NAME)) - - for user in self.AUTH_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(self.SERVICE_NAME, user) - } - }) - for user in self.SERVICE_USERS: - overrides.update({ - user: { - 'region_name': self._region_name(), - 'password': self._get_keyring_password(user, user) - } - }) - return overrides - - def _get_endpoints_identity_overrides(self): - return {'auth': self._get_endpoints_identity_users_overrides()} - def _get_endpoints_overrides(self): - return { - 'identity': self._get_endpoints_identity_overrides(), + overrides = { + 'identity': { + 'name': 'keystone', + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'oslo_cache': { + 'auth': { + 'memcached_secret_key': + self._get_common_password('auth_memcache_key') + } + }, + 'oslo_messaging': { + 'auth': self._get_endpoints_oslo_messaging_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME]) + }, } + db_passwords = {'auth': self._get_endpoints_oslo_db_overrides( + self.SERVICE_NAME, [self.SERVICE_NAME])} + overrides.update({ + 'oslo_db': db_passwords, + 'oslo_db_api': copy.deepcopy(db_passwords), + 'oslo_db_cell0': copy.deepcopy(db_passwords), + }) + + # Service user passwords already exist in other chart overrides + for user in self.SERVICE_USERS: + overrides['identity']['auth'].update({ + user: { + 'region_name': self._region_name(), + 'password': self._get_or_generate_password( + user, common.HELM_NS_OPENSTACK, user) + } + }) + + return overrides + def _get_virt_type(self): if utils.is_virtual(): return 'qemu' diff --git a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py index 15c171b00e..78012dbc63 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/openstack.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/openstack.py @@ -10,9 +10,12 @@ import subprocess from . import base from . import common +from oslo_log import log from sysinv.common import constants from sysinv.common import exception +LOG = log.getLogger(__name__) + class OpenstackBaseHelm(base.BaseHelm): """Class to encapsulate Openstack service operations for helm""" @@ -38,13 +41,6 @@ class OpenstackBaseHelm(base.BaseHelm): return passwords[service][user] - def _get_database_password(self, service): - passwords = self.context.setdefault('_database_passwords', {}) - if service not in passwords: - passwords[service] = self._get_keyring_password(service, - 'database') - return passwords[service] - def _get_database_username(self, service): return 'admin-%s' % service @@ -80,6 +76,103 @@ class OpenstackBaseHelm(base.BaseHelm): return self._region_name() + def _get_or_generate_password(self, chart, namespace, field): + # Get password from the db for the specified chart overrides + if not self.dbapi: + return None + + try: + override = self.dbapi.helm_override_get(name=chart, + namespace=namespace) + except exception.HelmOverrideNotFound: + # Override for this chart not found, so create one + try: + values = { + 'name': chart, + 'namespace': namespace, + } + override = self.dbapi.helm_override_create(values=values) + except exception as e: + LOG.exception(e) + return None + + password = override.system_overrides.get(field, None) + if password: + return password.encode('utf8', 'strict') + + # The password is not present, so generate one and store it to + # the override + password = self._generate_random_password() + values = {'system_overrides': override.system_overrides} + values['system_overrides'].update({ + field: password, + }) + try: + self.dbapi.helm_override_update( + name=chart, namespace=namespace, values=values) + except exception as e: + LOG.exception(e) + + return password.encode('utf8', 'strict') + + def _get_endpoints_identity_overrides(self, service_name, users): + # Returns overrides for admin and individual users + overrides = {} + overrides.update(self._get_common_users_overrides(service_name)) + + for user in users: + overrides.update({ + user: { + 'region_name': self._region_name(), + 'password': self._get_or_generate_password( + service_name, common.HELM_NS_OPENSTACK, user) + } + }) + return overrides + + def _get_endpoints_oslo_db_overrides(self, service_name, users): + overrides = { + 'admin': { + 'password': self._get_common_password('admin_db'), + } + } + + for user in users: + overrides.update({ + user: { + 'password': self._get_or_generate_password( + service_name, common.HELM_NS_OPENSTACK, + user + '_db'), + } + }) + + return overrides + + def _get_endpoints_oslo_messaging_overrides(self, service_name, users): + overrides = { + 'admin': { + 'username': 'rabbitmq-admin', + 'password': self._get_common_password('rabbitmq-admin') + } + } + + for user in users: + overrides.update({ + user: { + 'username': user + '-rabbitmq-user', + 'password': self._get_or_generate_password( + service_name, common.HELM_NS_OPENSTACK, + user + '_rabbit') + } + }) + + return overrides + + def _get_common_password(self, name): + # Admin passwords are stored on keystone's helm override entry + return self._get_or_generate_password( + 'keystone', common.HELM_NS_OPENSTACK, name) + def _get_common_users_overrides(self, service): overrides = {} for user in common.USERS: diff --git a/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py b/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py index 31515cf980..6229ee0b25 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/rabbitmq.py @@ -31,7 +31,8 @@ class RabbitmqHelm(openstack.OpenstackBaseHelm): 'replicas': { 'server': self._num_controllers() } - } + }, + 'endpoints': self._get_endpoints_overrides(), } } @@ -42,3 +43,15 @@ class RabbitmqHelm(openstack.OpenstackBaseHelm): namespace=namespace) else: return overrides + + def _get_endpoints_overrides(self): + credentials = self._get_endpoints_oslo_messaging_overrides( + self.CHART, []) + overrides = { + 'oslo_messaging': { + 'auth': { + 'user': credentials['admin'] + } + }, + } + return overrides diff --git a/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py b/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py index 594c8827a9..8b2811f9bf 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/helm_overrides.py @@ -20,6 +20,7 @@ class HelmOverrides(base.SysinvObject): fields = {'name': utils.str_or_none, 'namespace': utils.str_or_none, 'user_overrides': utils.str_or_none, + 'system_overrides': utils.dict_or_none, } @base.remotable_classmethod