From ce88b281c5bec315235383ed2c2605db968a5e54 Mon Sep 17 00:00:00 2001 From: Elena Taivan Date: Mon, 6 May 2019 13:39:11 +0000 Subject: [PATCH] Get swift working on containerized openstack In order to get swift working on containerized openstack, changes were needed both on platform and application side. From platform side, settings from ceph.conf file were replaced. A runtime manifest was added to update ceph.conf after a successful application apply: 1. Keystone auth url was updated with keystone openstack url 2. 'rgw_keystone_admin_domain' and 'rgw_keystone_project' settings were updated with 'service'. From application side the following changes have been implemented: 1. Ceph-rgw chart from openstack-helm-infra repo was included in stx-openstack 2. A chart schema for ceph-rgw was added 3. An override file was generated Signed-off-by: Elena Taivan Story: 2003909 Task: 30606 Change-Id: I01f7cf412264394f4f9bfb31f3c5a5ebd73f49dc --- .../manifests/manifest.yaml | 86 +++++++++++++++++++ .../modules/openstack/manifests/keystone.pp | 1 + .../src/modules/platform/manifests/ceph.pp | 53 ++++++++++++ sysinv/sysinv/sysinv/setup.cfg | 1 + .../sysinv/sysinv/sysinv/common/constants.py | 1 + .../sysinv/sysinv/sysinv/conductor/manager.py | 32 +++++++ sysinv/sysinv/sysinv/sysinv/helm/swift.py | 59 +++++++++++++ sysinv/sysinv/sysinv/sysinv/puppet/base.py | 4 + sysinv/sysinv/sysinv/sysinv/puppet/ceph.py | 20 ++++- .../sysinv/sysinv/sysinv/puppet/keystone.py | 17 ++++ .../sysinv/sysinv/sysinv/puppet/openstack.py | 11 +++ 11 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 sysinv/sysinv/sysinv/sysinv/helm/swift.py diff --git a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index ed9854b983..07715cf249 100644 --- a/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -470,6 +470,81 @@ data: - helm-toolkit --- schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: openstack-ceph-rgw +data: + chart_name: ceph-rgw + release: openstack-ceph-rgw + namespace: openstack + wait: + timeout: 1800 + labels: + release_group: osh-openstack-ceph-rgw + test: + enabled: false + install: + no_hooks: false + upgrade: + no_hooks: false + pre: + delete: + - type: job + labels: + release_group: osh-openstack-ceph-rgw + - type: pod + labels: + release_group: osh-openstack-ceph-rgw + component: test + values: + conf: + ceph: + global: + cephx: false + rgw_ks: + enabled: true + endpoints: + object_store: + path: + default: '/swift/v1' + port: + api: + default: null + admin: 7480 + internal: 7480 + public: 7480 + images: + tags: + ks_endpoints: docker.io/starlingx/stx-heat:master-centos-stable-latest + ks_service: docker.io/starlingx/stx-heat:master-centos-stable-latest + ks_user: docker.io/starlingx/stx-heat:master-centos-stable-latest + labels: + api: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + registry: + node_selector_key: openstack-control-plane + node_selector_value: enabled + pod: + replicas: + api: 2 + registry: 2 + affinity: + anti: + type: + default: requiredDuringSchedulingIgnoredDuringExecution + source: + type: tar + location: http://172.17.0.1/helm_charts/starlingx/ceph-rgw-0.1.0.tgz + subpath: ceph-rgw + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: openstack-glance @@ -2968,6 +3043,16 @@ data: - openstack-glance --- schema: armada/ChartGroup/v1 +metadata: + schema: metadata/Document/v1 + name: openstack-ceph-rgw +data: + description: "Deploy swift" + sequenced: true + chart_group: + - openstack-ceph-rgw +--- +schema: armada/ChartGroup/v1 metadata: schema: metadata/Document/v1 name: openstack-cinder @@ -3039,6 +3124,7 @@ data: - openstack-keystone - openstack-barbican - openstack-glance + - openstack-ceph-rgw - openstack-compute-kit - openstack-heat - openstack-horizon diff --git a/puppet-manifests/src/modules/openstack/manifests/keystone.pp b/puppet-manifests/src/modules/openstack/manifests/keystone.pp index 8aeff4dd0e..4b922b5d38 100644 --- a/puppet-manifests/src/modules/openstack/manifests/keystone.pp +++ b/puppet-manifests/src/modules/openstack/manifests/keystone.pp @@ -3,6 +3,7 @@ class openstack::keystone::params( $identity_uri, $auth_uri, $host_url, + $openstack_auth_uri = undef, $api_port = 5000, $admin_port = 5000, $region_name = undef, diff --git a/puppet-manifests/src/modules/platform/manifests/ceph.pp b/puppet-manifests/src/modules/platform/manifests/ceph.pp index 487a280ba4..f98f438b4d 100644 --- a/puppet-manifests/src/modules/platform/manifests/ceph.pp +++ b/puppet-manifests/src/modules/platform/manifests/ceph.pp @@ -26,6 +26,9 @@ class platform::ceph::params( $rgw_frontend_type = 'civetweb', $rgw_port = 7480, $rgw_log_file = '/var/log/radosgw/radosgw.log', + $rgw_service_domain = undef, + $rgw_service_project = undef, + $rgw_service_password = undef, $rgw_admin_domain = undef, $rgw_admin_project = undef, $rgw_admin_user = 'swift', @@ -406,6 +409,56 @@ class platform::ceph::haproxy } } +class platform::ceph::rgw::runtime + inherits ::platform::ceph::params { + if $service_enabled { + include ::platform::params + + include ::openstack::keystone::params + + ceph::rgw::keystone { $rgw_client_name: + rgw_keystone_admin_token => '', + rgw_keystone_url => $::openstack::keystone::params::openstack_auth_uri, + rgw_keystone_version => $::openstack::keystone::params::api_version, + rgw_keystone_accepted_roles => 'admin,_member_', + user => $rgw_user_name, + use_pki => false, + rgw_keystone_admin_domain => $rgw_service_domain, + rgw_keystone_admin_project => $rgw_service_project, + rgw_keystone_admin_user => $rgw_admin_user, + rgw_keystone_admin_password => $rgw_service_password, + } + exec { 'sm-restart-safe service ceph-radosgw': + command => 'sm-restart-safe service ceph-radosgw' + } + } +} + +class platform::ceph::rgw::runtime_revert + inherits ::platform::ceph::params { + if $service_enabled { + include ::platform::params + + include ::openstack::keystone::params + + ceph::rgw::keystone { $rgw_client_name: + rgw_keystone_admin_token => '', + rgw_keystone_url => $::openstack::keystone::params::auth_uri, + rgw_keystone_version => $::openstack::keystone::params::api_version, + rgw_keystone_accepted_roles => 'admin,_member_', + user => $rgw_user_name, + use_pki => false, + rgw_keystone_admin_domain => $rgw_admin_domain, + rgw_keystone_admin_project => $rgw_admin_project, + rgw_keystone_admin_user => $rgw_admin_user, + rgw_keystone_admin_password => $rgw_admin_password, + } + exec { 'sm-restart-safe service ceph-radosgw': + command => 'sm-restart-safe service ceph-radosgw' + } + } +} + class platform::ceph::rgw inherits ::platform::ceph::params { diff --git a/sysinv/sysinv/sysinv/setup.cfg b/sysinv/sysinv/sysinv/setup.cfg index 04739a098f..7e54a84720 100644 --- a/sysinv/sysinv/sysinv/setup.cfg +++ b/sysinv/sysinv/sysinv/setup.cfg @@ -104,6 +104,7 @@ systemconfig.helm_plugins.stx_openstack = 020_helm-toolkit = sysinv.helm.helm_toolkit:HelmToolkitHelm 021_barbican = sysinv.helm.barbican:BarbicanHelm 022_keystone-api-proxy = sysinv.helm.keystone_api_proxy:KeystoneApiProxyHelm + 023_ceph-rgw = sysinv.helm.swift:SwiftHelm sysinv.agent.lldp.drivers = lldpd = sysinv.agent.lldp.drivers.lldpd.driver:SysinvLldpdAgentDriver diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 9f93350622..01831ed857 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1450,6 +1450,7 @@ HELM_CHART_RBD_PROVISIONER = 'rbd-provisioner' HELM_CHART_CEPH_POOLS_AUDIT = 'ceph-pools-audit' HELM_CHART_HELM_TOOLKIT = 'helm-toolkit' HELM_CHART_KEYSTONE_API_PROXY = 'keystone-api-proxy' +HELM_CHART_SWIFT = 'ceph-rgw' # Helm: Supported application (aka chart bundles) HELM_APP_OPENSTACK = 'stx-openstack' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 23e0758986..a46de4ff3f 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -6483,6 +6483,21 @@ class ConductorManager(service.PeriodicService): config_uuid, config_dict) + def _revert_cephrgw_config(self, context): + """ Revert ceph rgw configuration. """ + personalities = [constants.CONTROLLER] + + config_uuid = self._config_update_hosts(context, personalities) + + config_dict = { + "personalities": personalities, + "classes": ['platform::ceph::rgw::runtime_revert'] + } + + self._config_apply_runtime_manifest(context, + config_uuid, + config_dict) + def _update_config_for_stx_openstack(self, context): """ Update the runtime configurations that are required for stx-openstack application @@ -6501,6 +6516,21 @@ class ConductorManager(service.PeriodicService): config_uuid, config_dict) + def _update_cephrgw_config(self, context): + """ Update ceph rgw configuration. """ + personalities = [constants.CONTROLLER] + + config_uuid = self._config_update_hosts(context, personalities) + + config_dict = { + "personalities": personalities, + "classes": ['platform::ceph::rgw::runtime'] + } + + self._config_apply_runtime_manifest(context, + config_uuid, + config_dict) + def report_lvm_cinder_config_success(self, context, host_uuid): """ Callback for Sysinv Agent @@ -10795,6 +10825,7 @@ class ConductorManager(service.PeriodicService): appname = self._app.get_appname(rpc_app) if constants.HELM_APP_OPENSTACK == appname and app_applied \ and not was_applied: + self._update_cephrgw_config(context) # apply any runtime configurations that are needed for # stx_openstack application self._update_config_for_stx_openstack(context) @@ -10820,6 +10851,7 @@ class ConductorManager(service.PeriodicService): app_removed = self._app.perform_app_remove(rpc_app) if constants.HELM_APP_OPENSTACK == appname and app_removed: + self._revert_cephrgw_config(context) # Update the VIM and PciIrqAffinity configuration. self._update_vim_config(context) self._update_pciirqaffinity_config(context) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/swift.py b/sysinv/sysinv/sysinv/sysinv/helm/swift.py new file mode 100644 index 0000000000..0dcc12b671 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/helm/swift.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sysinv.common import constants +from sysinv.common import exception + +from sysinv.openstack.common import log as logging + +from sysinv.helm import common +from sysinv.helm import openstack + +LOG = logging.getLogger(__name__) + + +class SwiftHelm(openstack.OpenstackBaseHelm): + """Class to encapsulate helm operations for the swift chart""" + + CHART = constants.HELM_CHART_SWIFT + + SERVICE_NAME = 'swift' + SERVICE_TYPE = 'object-store' + AUTH_USERS = ['swift'] + + def get_overrides(self, namespace=None): + overrides = { + common.HELM_NS_OPENSTACK: { + 'endpoints': self._get_endpoints_overrides(), + } + } + + if namespace in self.SUPPORTED_NAMESPACES: + return overrides[namespace] + elif namespace: + raise exception.InvalidHelmNamespace(chart=self.CHART, + namespace=namespace) + else: + return overrides + + def _get_object_store_overrides(self): + return { + 'hosts': { + 'default': 'null', + 'admin': self._get_management_address(), + 'internal': self._get_management_address(), + 'public': self._get_oam_address() + }, + } + + def _get_endpoints_overrides(self): + return { + 'identity': { + 'auth': self._get_endpoints_identity_overrides( + self.SERVICE_NAME, self.AUTH_USERS), + }, + 'object_store': self._get_object_store_overrides(), + } diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/base.py b/sysinv/sysinv/sysinv/sysinv/puppet/base.py index 7ee02a6682..4cb29342dd 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/base.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/base.py @@ -24,6 +24,7 @@ class BasePuppet(object): CONFIG_WORKDIR = '/tmp/config' DEFAULT_REGION_NAME = 'RegionOne' DEFAULT_SERVICE_PROJECT_NAME = 'services' + SWIFT_SERVICE_PROJECT_NAME = 'service' DEFAULT_KERNEL_OPTIONS = constants.SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_DEFAULT_OPTS SYSTEM_CONTROLLER_SERVICES = [ @@ -132,6 +133,9 @@ class BasePuppet(object): system = self._get_system() return system.region_name + def _get_swift_service_project_name(self): + return self.SWIFT_SERVICE_PROJECT_NAME + def _get_service_project_name(self): if self.dbapi is None: return self.DEFAULT_SERVICE_PROJECT_NAME diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py b/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py index 46a9529426..a0ec896767 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py @@ -11,6 +11,7 @@ from sysinv.common import constants from sysinv.common import exception from sysinv.common import utils from sysinv.common.storage_backend_conf import StorageBackendConfig +from sysinv.helm import common from sysinv.puppet import openstack @@ -82,7 +83,7 @@ class CephPuppet(openstack.OpenstackBasePuppet): ksuser = self._get_service_user_name(self.SERVICE_NAME_RGW) - return { + config = { 'ceph::ms_bind_ipv6': ms_bind_ipv6, 'platform::ceph::params::service_enabled': True, @@ -113,6 +114,23 @@ class CephPuppet(openstack.OpenstackBasePuppet): self._get_service_tenant_name(), } + if utils.is_openstack_installed(self.dbapi): + override = self.dbapi.helm_override_get( + self.SERVICE_NAME_RGW, common.HELM_NS_OPENSTACK) + password = override.system_overrides.get( + self.SERVICE_NAME_RGW, None) + if password: + swift_auth_password = password.encode('utf8', 'strict') + config['platform::ceph::params::rgw_service_password'] = \ + swift_auth_password + + config['platform::ceph::params::rgw_service_domain'] = \ + self._get_swift_service_user_domain_name() + config['platform::ceph::params::rgw_service_project'] = \ + self._get_swift_service_tenant_name() + + return config + def _is_ceph_mon_required(self, host, operator): # Two conditions that we need to check for: # 1) If cinder is a shared service and it has a ceph backend diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py b/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py index d0246bd16c..9465af6f2c 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py @@ -7,6 +7,7 @@ from six.moves import configparser import os +from sysinv.common import utils from sysinv.common import constants from tsconfig import tsconfig @@ -30,6 +31,7 @@ class KeystonePuppet(openstack.OpenstackBasePuppet): ADMIN_USER = 'admin' DEFAULT_DOMAIN_NAME = 'Default' + SWIFT_DOMAIN_NAME = 'service' def _region_config(self): # A wrapper over the Base region_config check. @@ -125,6 +127,10 @@ class KeystonePuppet(openstack.OpenstackBasePuppet): 'CONFIG_KEYSTONE_ADMIN_USERNAME': self.get_admin_user_name(), } + if utils.is_openstack_installed(self.dbapi): + config['openstack::keystone::params::openstack_auth_uri'] = \ + self.get_openstack_auth_uri() + config.update(self._get_service_parameter_config()) config.update(self._get_password_rule()) return config @@ -298,6 +304,14 @@ class KeystonePuppet(openstack.OpenstackBasePuppet): return "http://%s:5000" % self._format_url_address( self._get_management_address()) + def get_openstack_auth_uri(self): + location = self._get_service_default_dns_name( + self.SERVICE_NAME) + + url = "%s://%s:80" % (self._get_public_protocol(), + location) + return url + def get_identity_uri(self): if self._region_config(): service_config = self._get_service_config(self.SERVICE_NAME) @@ -349,6 +363,9 @@ class KeystonePuppet(openstack.OpenstackBasePuppet): return service_config.capabilities.get('admin_project_domain') return self.DEFAULT_DOMAIN_NAME + def get_swift_service_user_domain(self): + return self.SWIFT_DOMAIN_NAME + def get_service_user_domain(self): if self._region_config(): service_config = self._get_service_config(self.SERVICE_NAME) diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/openstack.py b/sysinv/sysinv/sysinv/sysinv/puppet/openstack.py index 1ce36ebda0..ea53747546 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/openstack.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/openstack.py @@ -10,6 +10,7 @@ import keyring from sysinv.common import constants from sysinv.puppet import base +from sysinv.helm import common class OpenstackBasePuppet(base.BasePuppet): @@ -108,6 +109,10 @@ class OpenstackBasePuppet(base.BasePuppet): def _get_public_protocol(self): return 'https' if self._https_enabled() else 'http' + def _get_service_default_dns_name(self, service): + return "{}.{}.svc.{}".format(service, common.HELM_NS_OPENSTACK, + constants.DEFAULT_DNS_SERVICE_DOMAIN) + def _get_private_protocol(self): return 'http' @@ -155,6 +160,9 @@ class OpenstackBasePuppet(base.BasePuppet): return self._region_name() + def _get_swift_service_tenant_name(self): + return self._get_swift_service_project_name() + def _get_service_tenant_name(self): return self._get_service_project_name() @@ -183,6 +191,9 @@ class OpenstackBasePuppet(base.BasePuppet): return service_config.capabilities.get(stype) return None + def _get_swift_service_user_domain_name(self): + return self._operator.keystone.get_swift_service_user_domain() + def _get_service_user_domain_name(self): return self._operator.keystone.get_service_user_domain()