From 61ac82ee828e8a2c5fb0e67cf03c06fae5964b65 Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Wed, 13 Jun 2018 01:21:21 -0400 Subject: [PATCH] Integrate gnocchi for OpenStack Telemetry This commit integrates gnocchi as metric storage backend because of the deprecation of ceilometer metering backend. With gnocchi integrated, ceilometer api and collector processes are removed, ceilometer agent nofitication and polling processes are still running to collect and normalise data from other openstack services. Ceilometer notification agent also has the responsiblity to send samples to gnocchi backend through its gnocchi publisher and events to panko backend. Two gnocchi processes are introduced, gnocchi api is REST API service and gnocchi metricd is for metric computing and gnocchi CLI is supported to query measures/metric from gnocchi backend. Ceilometer service and user still need to be registered in keystone, but stop creating endpoints. Gnocchi is a seperate user/service in keystone and run on active controller licensing on port 8041 Changes: - puppet changes to add gnocchi user and endpoints to keystone, gnocchi port to haproxy and firewall, remote logging as well as other necessary configurations for gnocchi - backup restore for gnocchi postgres db (except metric and resource tables) - sm changes to manage gnocchi processes through created lsb scripts - support for multi-region setup - distributed cloud - do not enable service and create endpoints for gnocchi in system controller - add gnocchi logs in syslog - support no reboot patching for gnocchi processes - add gnocchiclient in remote CLI - add NotIn validation method in python-voluptuous which needed in gnocchi - collect scripts updates - cleanup ceilometer api and collector in puppet configuration, sm, service parameter, syslog, remotelogging, patch-restart-processes - remote ceilometerclient from remote CLI - remote openstack client changes for ceilometer CLI extensions - remove ceilometer-api and ceilometer-collector packages from iso Story: 2002825 Task: 22871 Depends-On: https://review.openstack.org/587419 Change-Id: Ia46c67eb6543c3694dff03435dc2b00fb2b760e5 Signed-off-by: Jack Ding --- .../configutilities/common/utils.py | 4 +- .../configutilities/common/validator.py | 11 +- .../configutilities/configfiletool.py | 7 + .../controllerconfig/backup_restore.py | 11 +- .../controllerconfig/configassistant.py | 25 ++++ .../controllerconfig/regionconfig.py | 13 +- .../TiS_region_config.share.keystoneonly | 2 + ...iS_region_config.share.keystoneonly.result | 2 + .../tests/files/TiS_region_config.shareall | 2 + .../files/TiS_region_config.shareall.result | 2 + .../tests/files/cgcs_config.region | 2 + .../tests/files/cgcs_config.region_nuage_vrs | 2 + .../tests/files/region_config.lag.vlan | 2 + .../tests/files/region_config.lag.vlan.result | 2 + .../tests/files/region_config.nuage_vrs | 2 + .../files/region_config.nuage_vrs.result | 2 + .../tests/files/region_config.security | 2 + .../tests/files/region_config.security.result | 2 + .../tests/files/region_config.simple | 2 + .../tests/files/region_config.simple.can_ips | 2 + .../tests/files/region_config.simple.result | 2 + puppet-manifests/centos/puppet-manifests.spec | 1 + .../src/hieradata/controller.yaml | 26 ++-- puppet-manifests/src/manifests/controller.pp | 7 +- .../modules/openstack/manifests/ceilometer.pp | 99 ++------------ .../modules/openstack/manifests/gnocchi.pp | 115 ++++++++++++++++ .../modules/openstack/manifests/keystone.pp | 5 + .../openstack/templates/gnocchi-api.conf.erb | 3 + .../src/modules/platform/manifests/haproxy.pp | 2 +- .../modules/platform/manifests/postgresql.pp | 2 +- .../src/modules/platform/manifests/sm.pp | 38 ++++-- .../platform/templates/remotelogging.conf.erb | 8 +- .../cgts-client/cgtsclient/v1/event_log.py | 2 +- .../cgtsclient/v1/event_log_shell.py | 2 +- .../cgtsclient/v1/event_suppression.py | 2 +- .../cgtsclient/v1/event_suppression_shell.py | 2 +- .../cgts-client/cgtsclient/v1/ialarm.py | 2 +- .../cgts-client/cgtsclient/v1/ialarm_shell.py | 4 +- .../cgts-client/cgtsclient/v1/options.py | 124 ++++++++++++++++++ .../cgtsclient/v1/service_parameter.py | 2 +- .../cgtsclient/v1/service_parameter_shell.py | 2 +- .../cgts-client/cgts-client/requirements.txt | 1 - .../sysinv/sysinv/sysinv/common/constants.py | 8 +- .../sysinv/sysinv/common/service_parameter.py | 33 ----- .../sysinv/sysinv/sysinv/conductor/manager.py | 12 -- ...072_remove_ceilometer_service_parameter.py | 46 +++++++ .../sysinv/sysinv/sysinv/puppet/ceilometer.py | 22 ---- sysinv/sysinv/sysinv/sysinv/puppet/puppet.py | 8 +- 48 files changed, 458 insertions(+), 221 deletions(-) create mode 100644 puppet-manifests/src/modules/openstack/manifests/gnocchi.pp create mode 100644 puppet-manifests/src/modules/openstack/templates/gnocchi-api.conf.erb create mode 100644 sysinv/cgts-client/cgts-client/cgtsclient/v1/options.py create mode 100644 sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/072_remove_ceilometer_service_parameter.py diff --git a/configutilities/configutilities/configutilities/common/utils.py b/configutilities/configutilities/configutilities/common/utils.py index 6eed480f23..e966f4a4e0 100644 --- a/configutilities/configutilities/configutilities/common/utils.py +++ b/configutilities/configutilities/configutilities/common/utils.py @@ -40,7 +40,9 @@ EXPECTED_SERVICE_NAME_AND_TYPE = ( "AODH_SERVICE_NAME": "aodh", "AODH_SERVICE_TYPE": "alarming", "PANKO_SERVICE_NAME": "panko", - "PANKO_SERVICE_TYPE": "event"}) + "PANKO_SERVICE_TYPE": "event", + "GNOCCHI_SERVICE_NAME": "gnocchi", + "GNOCCHI_SERVICE_TYPE": "metric"}) def is_valid_vlan(vlan): diff --git a/configutilities/configutilities/configutilities/common/validator.py b/configutilities/configutilities/configutilities/common/validator.py index f831311b97..fa8e9ff04e 100755 --- a/configutilities/configutilities/configutilities/common/validator.py +++ b/configutilities/configutilities/configutilities/common/validator.py @@ -782,7 +782,7 @@ class ConfigValidator(object): self.conf.get('REGION_2_SERVICES', 'CREATE') == 'Y'): password_fields = [ 'NOVA', 'CEILOMETER', 'PATCHING', 'SYSINV', 'HEAT', - 'HEAT_ADMIN', 'PLACEMENT' + 'HEAT_ADMIN', 'PLACEMENT', 'AODH', 'PANKO', 'GNOCCHI' ] for pw in password_fields: if not self.conf.has_option('REGION_2_SERVICES', @@ -943,6 +943,12 @@ class ConfigValidator(object): panko_password = get_optional(self.conf, 'REGION_2_SERVICES', 'PANKO_PASSWORD') + # validate gnocchi service name and type + gnocchi_user_name = self.conf.get('REGION_2_SERVICES', + 'GNOCCHI_USER_NAME') + gnocchi_password = get_optional(self.conf, 'REGION_2_SERVICES', + 'GNOCCHI_PASSWORD') + if self.conf.has_option('REGION_2_SERVICES', 'USER_DOMAIN_NAME'): user_domain = self.conf.get('REGION_2_SERVICES', 'USER_DOMAIN_NAME') @@ -1048,6 +1054,9 @@ class ConfigValidator(object): self.cgcs_conf.set('cREGION', 'MTCE_PASSWORD', mtce_password) self.cgcs_conf.set('cREGION', 'PANKO_USER_NAME', panko_user_name) self.cgcs_conf.set('cREGION', 'PANKO_PASSWORD', panko_password) + self.cgcs_conf.set('cREGION', 'GNOCCHI_USER_NAME', + gnocchi_user_name) + self.cgcs_conf.set('cREGION', 'GNOCCHI_PASSWORD', gnocchi_password) self.cgcs_conf.set('cREGION', 'USER_DOMAIN_NAME', user_domain) diff --git a/configutilities/configutilities/configutilities/configfiletool.py b/configutilities/configutilities/configutilities/configfiletool.py index 07168e2215..4b4343cb29 100755 --- a/configutilities/configutilities/configutilities/configfiletool.py +++ b/configutilities/configutilities/configutilities/configfiletool.py @@ -721,6 +721,13 @@ class REG2SERVICESPage2(ConfigPage): text="Placement user password", type=TYPES.string, initial="") + self.fields['GNOCCHI_USER_NAME'] = Field( + text="GNOCCHI username", + type=TYPES.string, initial="gnocchi") + self.fields['GNOCCHI_PASSWORD'] = Field( + text="GNOCCHI user password", + type=TYPES.string, initial="") + def validate_page(self): self.prev.validate_page() super(REG2SERVICESPage2, self).validate_page() diff --git a/controllerconfig/controllerconfig/controllerconfig/backup_restore.py b/controllerconfig/controllerconfig/controllerconfig/backup_restore.py index a14d10cac3..7a0ce788d3 100644 --- a/controllerconfig/controllerconfig/controllerconfig/backup_restore.py +++ b/controllerconfig/controllerconfig/controllerconfig/backup_restore.py @@ -68,9 +68,9 @@ def get_backup_databases(cinder_config=False): # Databases common to all configurations REGION_LOCAL_DATABASES = ('postgres', 'template1', 'nova', 'sysinv', - 'ceilometer', 'neutron', 'heat', 'nova_api', + 'neutron', 'heat', 'nova_api', 'aodh', 'murano', 'magnum', 'panko', 'ironic', - 'nova_cell0') + 'nova_cell0', 'gnocchi') REGION_SHARED_DATABASES = ('glance', 'keystone') if cinder_config: @@ -79,12 +79,7 @@ def get_backup_databases(cinder_config=False): # Indicates which tables have to be dropped for a certain database. DB_TABLE_SKIP_MAPPING = { 'sysinv': ('i_alarm',), - 'ceilometer': ('metadata_bool', - 'metadata_float', - 'metadata_int', - 'metadata_text', - 'meter', 'sample', 'fault', - 'resource'), + 'gnocchi': ('metric', 'resource'), 'dcorch': ('orch_job', 'orch_request', 'resource', diff --git a/controllerconfig/controllerconfig/controllerconfig/configassistant.py b/controllerconfig/controllerconfig/controllerconfig/configassistant.py index abbf90966e..2bd5d27058 100644 --- a/controllerconfig/controllerconfig/controllerconfig/configassistant.py +++ b/controllerconfig/controllerconfig/controllerconfig/configassistant.py @@ -500,6 +500,8 @@ class ConfigAssistant(): self.aodh_ks_password = "" self.panko_ks_user_name = "" self.panko_ks_password = "" + self.gnocchi_ks_user_name = "" + self.gnocchi_ks_password = "" self.mtce_ks_user_name = "" self.mtce_ks_password = "" self.nfv_ks_user_name = "" @@ -2783,6 +2785,14 @@ class ConfigAssistant(): 'cREGION', 'PANKO_PASSWORD') self.add_password_for_validation('PANKO_PASSWORD', self.panko_ks_password) + + self.gnocchi_ks_user_name = config.get( + 'cREGION', 'GNOCCHI_USER_NAME') + self.gnocchi_ks_password = config.get( + 'cREGION', 'GNOCCHI_PASSWORD') + self.add_password_for_validation('GNOCCHI_PASSWORD', + self.gnocchi_ks_password) + self.mtce_ks_user_name = config.get( 'cREGION', 'MTCE_USER_NAME') self.mtce_ks_password = config.get( @@ -3360,6 +3370,10 @@ class ConfigAssistant(): self.panko_ks_user_name) f.write("PANKO_PASSWORD=%s\n" % self.panko_ks_password) + f.write("GNOCCHI_USER_NAME=%s\n" % + self.gnocchi_ks_user_name) + f.write("GNOCCHI_PASSWORD=%s\n" % + self.gnocchi_ks_password) f.write("MTCE_USER_NAME=%s\n" % self.mtce_ks_user_name) f.write("MTCE_PASSWORD=%s\n" % @@ -3834,6 +3848,14 @@ class ConfigAssistant(): 'capabilities': capabilities} client.sysinv.sm_service.service_create(**values) + # gnocchi service config + capabilities = {'user_name': self.gnocchi_ks_user_name} + values = {'name': "gnocchi", + 'enabled': True, + 'region_name': self.region_2_name, + 'capabilities': capabilities} + client.sysinv.sm_service.service_create(**values) + # mtc service config capabilities = {'user_name': self.mtce_ks_user_name} values = {'name': "mtce", @@ -3899,6 +3921,9 @@ class ConfigAssistant(): keyring.set_password('panko', constants.DEFAULT_SERVICE_PROJECT_NAME, self.panko_ks_password) + keyring.set_password('gnocchi', constants.DEFAULT_SERVICE_PROJECT_NAME, + self.gnocchi_ks_password) + keyring.set_password('mtce', constants.DEFAULT_SERVICE_PROJECT_NAME, self.mtce_ks_password) diff --git a/controllerconfig/controllerconfig/controllerconfig/regionconfig.py b/controllerconfig/controllerconfig/controllerconfig/regionconfig.py index 4425716898..7ae7f3e773 100755 --- a/controllerconfig/controllerconfig/controllerconfig/regionconfig.py +++ b/controllerconfig/controllerconfig/controllerconfig/regionconfig.py @@ -54,7 +54,8 @@ EXPECTED_USERS = [ ('REGION_2_SERVICES', 'NFV', 'vim'), ('REGION_2_SERVICES', 'AODH', 'aodh'), ('REGION_2_SERVICES', 'MTCE', 'mtce'), - ('REGION_2_SERVICES', 'PANKO', 'panko')] + ('REGION_2_SERVICES', 'PANKO', 'panko'), + ('REGION_2_SERVICES', 'GNOCCHI', 'gnocchi')] EXPECTED_SHARED_SERVICES_NEUTRON_USER = ('SHARED_SERVICES', 'NEUTRON', 'neutron') @@ -108,11 +109,6 @@ EXPECTED_REGION2_ENDPOINTS = [ 'http://{}:8000/v1/', 'http://{}:8000/v1/', 'Openstack Cloudformation Service'), - ('CEILOMETER_SERVICE_NAME', 'CEILOMETER_SERVICE_TYPE', - 'http://{}:8777', - 'http://{}:8777', - 'http://{}:8777', - 'Openstack Metering Service'), ('NFV_SERVICE_NAME', 'NFV_SERVICE_TYPE', 'http://{}:4545', 'http://{}:4545', @@ -128,6 +124,11 @@ EXPECTED_REGION2_ENDPOINTS = [ 'http://{}:8977', 'http://{}:8977', 'OpenStack Event Service'), + ('GNOCCHI_SERVICE_NAME', 'GNOCCHI_SERVICE_TYPE', + 'http://{}:8041', + 'http://{}:8041', + 'http://{}:8041', + 'OpenStack Metric Service'), ] EXPECTED_NEUTRON_ENDPOINT = ( diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly index 62fa15bfd7..688f03b616 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly @@ -121,6 +121,8 @@ MTCE_USER_NAME=mtceTWO MTCE_PASSWORD=password2WO* PANKO_USER_NAME=pankoTWO PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchiTWO +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly.result index 9f3b6a48a0..93d3cf614d 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.share.keystoneonly.result @@ -108,6 +108,8 @@ MTCE_USER_NAME = mtceTWO MTCE_PASSWORD = password2WO* PANKO_USER_NAME = pankoTWO PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchiTWO +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = service_domain PROJECT_DOMAIN_NAME = service_domain KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall index eb30345927..d8e08ad3f3 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall @@ -115,6 +115,8 @@ MTCE_USER_NAME=mtceTWO MTCE_PASSWORD=password2WO* PANKO_USER_NAME=pankoTWO PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchiTWO +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall.result index 3345c5b280..d8209813a4 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/TiS_region_config.shareall.result @@ -106,6 +106,8 @@ MTCE_USER_NAME = mtceTWO MTCE_PASSWORD = password2WO* PANKO_USER_NAME = pankoTWO PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchiTWO +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = Default PROJECT_DOMAIN_NAME = Default KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region index 9c57b722e5..8b63a314f9 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region @@ -127,6 +127,8 @@ AODH_USER_NAME=aodh AODH_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region_nuage_vrs b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region_nuage_vrs index 8af04c5933..bc10893b90 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region_nuage_vrs +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/cgcs_config.region_nuage_vrs @@ -139,6 +139,8 @@ AODH_USER_NAME=aodh AODH_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan index 1127edf62d..3ae5c93db1 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan @@ -111,6 +111,8 @@ MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan.result index 1d8f7b406e..e7473b39fa 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.lag.vlan.result @@ -111,6 +111,8 @@ MTCE_USER_NAME = mtce MTCE_PASSWORD = password2WO* PANKO_USER_NAME = panko PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchi +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = Default PROJECT_DOMAIN_NAME = Default KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs index c6530cc1e2..918cc17f52 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs @@ -121,6 +121,8 @@ MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs.result index 24b4e1bd00..fc4b7b4f28 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.nuage_vrs.result @@ -101,6 +101,8 @@ MTCE_USER_NAME = mtce MTCE_PASSWORD = password2WO* PANKO_USER_NAME = panko PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchi +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = Default PROJECT_DOMAIN_NAME = Default KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security index 243f6f1b5e..d6d319ee96 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security @@ -117,6 +117,8 @@ MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security.result index c50f422d34..a5b75bea1e 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.security.result @@ -89,6 +89,8 @@ MTCE_USER_NAME = mtce MTCE_PASSWORD = password2WO* PANKO_USER_NAME = panko PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchi +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = Default PROJECT_DOMAIN_NAME = Default KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple index 8f76ccc919..bb233507c6 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple @@ -117,6 +117,8 @@ MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.can_ips b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.can_ips index 7818cb2622..52f1b600e6 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.can_ips +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.can_ips @@ -118,6 +118,8 @@ MTCE_USER_NAME=mtce MTCE_PASSWORD=password2WO* PANKO_USER_NAME=panko PANKO_PASSWORD=password2WO* +GNOCCHI_USER_NAME=gnocchi +GNOCCHI_PASSWORD=password2WO* [VERSION] RELEASE = 18.04 diff --git a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.result b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.result index c50f422d34..a5b75bea1e 100755 --- a/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.result +++ b/controllerconfig/controllerconfig/controllerconfig/tests/files/region_config.simple.result @@ -89,6 +89,8 @@ MTCE_USER_NAME = mtce MTCE_PASSWORD = password2WO* PANKO_USER_NAME = panko PANKO_PASSWORD = password2WO* +GNOCCHI_USER_NAME = gnocchi +GNOCCHI_PASSWORD = password2WO* USER_DOMAIN_NAME = Default PROJECT_DOMAIN_NAME = Default KEYSTONE_AUTH_URI = http://192.168.204.12:8081/keystone/main/v2.0 diff --git a/puppet-manifests/centos/puppet-manifests.spec b/puppet-manifests/centos/puppet-manifests.spec index b84e3a2b35..7dbaca964e 100644 --- a/puppet-manifests/centos/puppet-manifests.spec +++ b/puppet-manifests/centos/puppet-manifests.spec @@ -41,6 +41,7 @@ Requires: puppet-magnum Requires: puppet-ironic Requires: puppet-panko Requires: puppet-memcached +Requires: puppet-gnocchi # Puppetlabs puppet modules Requires: puppet-concat diff --git a/puppet-manifests/src/hieradata/controller.yaml b/puppet-manifests/src/hieradata/controller.yaml index d1bbe11169..e12e2eed1e 100644 --- a/puppet-manifests/src/hieradata/controller.yaml +++ b/puppet-manifests/src/hieradata/controller.yaml @@ -292,18 +292,6 @@ neutron::services::bgpvpn::service_providers: # ceilometer -ceilometer::metering_time_to_live: 86400 - -ceilometer::api::enabled: false -ceilometer::api::service_name: 'openstack-ceilometer-api' - -ceilometer::db::database_idle_timeout: 60 -ceilometer::db::database_max_pool_size: 1 -ceilometer::db::database_max_overflow: 10 - -ceilometer::collector::enabled: false -ceilometer::collector::meter_dispatchers: ['database'] - ceilometer::agent::notification::enabled: false ceilometer::agent::notification::disable_non_metric_meters: false @@ -311,9 +299,17 @@ ceilometer::agent::polling::central_namespace: true ceilometer::agent::polling::compute_namespace: false ceilometer::agent::polling::ipmi_namespace: true -ceilometer::expirer::minute: 1 -ceilometer::expirer::hour: '*' -ceilometer::expirer::monthday: '*' +# Do not create endpoints for ceilometer as ceilometer-api is removed +ceilometer::keystone::auth::configure_endpoint: false + + +# gnocchi +gnocchi::api::enabled: false +gnocchi::api::service_name: 'openstack-gnocchi-api' +gnocchi::api::enable_proxy_headers_parsing: true +gnocchi::metricd::enabled: false +gnocchi::storage::file::file_basepath: '/opt/cgcs/ceilometer/data' +gnocchi::db::sync::user: 'root' # aodh diff --git a/puppet-manifests/src/manifests/controller.pp b/puppet-manifests/src/manifests/controller.pp index 496edd8495..d01485b7f5 100644 --- a/puppet-manifests/src/manifests/controller.pp +++ b/puppet-manifests/src/manifests/controller.pp @@ -82,9 +82,12 @@ include ::openstack::nova::network include ::openstack::nova::controller include ::openstack::nova::placement +include ::openstack::gnocchi +include ::openstack::gnocchi::api +include ::openstack::gnocchi::metricd + include ::openstack::ceilometer -include ::openstack::ceilometer::api -include ::openstack::ceilometer::collector +include ::openstack::ceilometer::agent::notification include ::openstack::ceilometer::polling include ::openstack::aodh diff --git a/puppet-manifests/src/modules/openstack/manifests/ceilometer.pp b/puppet-manifests/src/modules/openstack/manifests/ceilometer.pp index 09c9e9f48f..dc16425a2f 100644 --- a/puppet-manifests/src/modules/openstack/manifests/ceilometer.pp +++ b/puppet-manifests/src/modules/openstack/manifests/ceilometer.pp @@ -15,6 +15,11 @@ class openstack::ceilometer { rabbit_qos_prefetch_count => 100, } + if ($::openstack::ceilometer::params::service_create and + $::platform::params::init_keystone) { + include ::ceilometer::keystone::auth + } + include ::ceilometer::agent::auth include ::platform::params include ::openstack::ceilometer::params @@ -29,10 +34,14 @@ class openstack::ceilometer { 'oslo_messaging_rabbit/rpc_conn_pool_size': value => 10; 'oslo_messaging_rabbit/socket_timeout': value => 1.00; 'compute/resource_update_interval': value => 60; - 'service_credentials/os_endpoint_type': value => 'internalURL'; 'DEFAULT/region_name_for_services': value => $::openstack::ceilometer::params::region_name; } + oslo::cache { 'ceilometer_config': + enabled => true, + backend => 'dogpile.cache.memory', + } + if $::platform::params::region_config { if $::openstack::glance::params::region_name != $::platform::params::region_2_name { $shared_service_glance = [$::openstack::glance::params::service_type] @@ -58,15 +67,9 @@ class openstack::ceilometer { } -class openstack::ceilometer::collector { +class openstack::ceilometer::agent::notification { include ::platform::params - if $::platform::params::init_database { - include ::ceilometer::db::postgresql - } - include ::ceilometer::keystone::authtoken - include ::ceilometer::expirer - $cgcs_fs_directory = '/opt/cgcs' $ceilometer_directory = "${cgcs_fs_directory}/ceilometer" $ceilometer_directory_csv = "${ceilometer_directory}/csv" @@ -98,22 +101,6 @@ class openstack::ceilometer::collector { mode => '0640', } - class { '::ceilometer::db': - sync_db => $::platform::params::init_database, - } - - include ::openstack::panko::params - if $::openstack::panko::params::service_enabled { - $event_dispatcher = ['panko'] - } else { - $event_dispatcher = undef - } - - class { '::ceilometer::collector': - collector_workers => $::platform::params::eng_workers_by_2, - event_dispatchers => $event_dispatcher - } - class { '::ceilometer::agent::notification': notification_workers => $::platform::params::eng_workers_by_2, } @@ -177,67 +164,3 @@ class openstack::ceilometer::polling { compute_namespace => $compute_namespace, } } - - -class openstack::ceilometer::firewall - inherits ::openstack::ceilometer::params { - - platform::firewall::rule { 'ceilometer-api': - service_name => 'ceilometer', - ports => $api_port, - } -} - - -class openstack::ceilometer::haproxy - inherits ::openstack::ceilometer::params { - - platform::haproxy::proxy { 'ceilometer-restapi': - server_name => 's-ceilometer', - public_port => $api_port, - private_port => $api_port, - } -} - - -class openstack::ceilometer::api - inherits ::openstack::ceilometer::params { - - include ::platform::params - $api_workers = $::platform::params::eng_workers_by_2 - - include ::platform::network::mgmt::params - $api_host = $::platform::network::mgmt::params::controller_address - $url_host = $::platform::network::mgmt::params::controller_address_url - - if ($::openstack::ceilometer::params::service_create and - $::platform::params::init_keystone) { - include ::ceilometer::keystone::auth - } - - file { '/usr/share/ceilometer/ceilometer-api.conf': - ensure => file, - content => template('openstack/ceilometer-api.conf.erb'), - owner => 'root', - group => 'root', - mode => '0640', - } -> - class { '::ceilometer::api': - host => $api_host, - api_workers => $api_workers, - enable_proxy_headers_parsing => true, - } - - include ::openstack::ceilometer::firewall - include ::openstack::ceilometer::haproxy -} - - -class openstack::ceilometer::runtime { - include ::platform::amqp::params - - class { '::ceilometer': - rabbit_use_ssl => $::platform::amqp::params::ssl_enabled, - default_transport_url => $::platform::amqp::params::transport_url, - } -} diff --git a/puppet-manifests/src/modules/openstack/manifests/gnocchi.pp b/puppet-manifests/src/modules/openstack/manifests/gnocchi.pp new file mode 100644 index 0000000000..cb79f4df8e --- /dev/null +++ b/puppet-manifests/src/modules/openstack/manifests/gnocchi.pp @@ -0,0 +1,115 @@ +class openstack::gnocchi::params ( + $api_port = 8041, + $region_name = undef, + $service_name = 'openstack-gnocchi', + $service_create = false, + $service_enabled = true, +) { } + +class openstack::gnocchi + inherits ::openstack::gnocchi::params { + + if $service_enabled { + + include ::platform::params + + include ::gnocchi + include ::gnocchi::api + include ::gnocchi::client + include ::gnocchi::keystone::authtoken + include ::gnocchi::storage::file + + if $::platform::params::init_database { + include ::gnocchi::db::postgresql + } + + } +} + + +class openstack::gnocchi::firewall + inherits ::openstack::gnocchi::params { + + platform::firewall::rule { 'gnocchi-api': + service_name => 'gnocchi', + ports => $api_port, + } +} + +class openstack::gnocchi::haproxy + inherits ::openstack::gnocchi::params { + + platform::haproxy::proxy { 'gnocchi-restapi': + server_name => 's-gnocchi-restapi', + public_port => $api_port, + private_port => $api_port, + } +} + +class openstack::gnocchi::metricd + inherits ::openstack::gnocchi::params { + + if $service_enabled { + + include ::platform::params + $metricd_workers = $::platform::params::eng_workers_by_2 + + class { '::gnocchi::metricd': + workers => $metricd_workers + } + + if $metricd_workers > 1 { + gnocchi_config { + 'DEFAULT/coordination_url': value => 'file:///opt/cgcs/ceilometer/data/locks'; + } + } + + } +} + +class openstack::gnocchi::api + inherits ::openstack::gnocchi::params { + + include ::platform::params + + # The gnocchi user and service are always required and they + # are used by subclouds when the service itself is disabled + # on System Controller + # whether it creates the endpoint is determined by + # gnocchi::keystone::auth::configure_endpoint which is + # set via sysinv puppet + if $::openstack::gnocchi::params::service_create and + $::platform::params::init_keystone { + include ::gnocchi::keystone::auth + } + + if $service_enabled { + + $api_workers = $::platform::params::eng_workers_by_2 + + include ::platform::network::mgmt::params + $url_host = $::platform::network::mgmt::params::controller_address_url + + file { '/usr/share/gnocchi/gnocchi-api.conf': + ensure => file, + content => template('openstack/gnocchi-api.conf.erb'), + owner => 'root', + group => 'root', + mode => '0640', + } + + if $::platform::params::init_database { + include ::openstack::gnocchi::metricd + $sacks_number = $::openstack::gnocchi::metricd::metricd_workers + 2 + + class { '::gnocchi::db::sync': + extra_opts => "--sacks-number $sacks_number" + } + } + + include ::openstack::gnocchi::firewall + include ::openstack::gnocchi::haproxy + } +} + + diff --git a/puppet-manifests/src/modules/openstack/manifests/keystone.pp b/puppet-manifests/src/modules/openstack/manifests/keystone.pp index 98b08b1b3e..f0d12d6d9f 100644 --- a/puppet-manifests/src/modules/openstack/manifests/keystone.pp +++ b/puppet-manifests/src/modules/openstack/manifests/keystone.pp @@ -276,6 +276,11 @@ class openstack::keystone::endpoint::runtime { include ::panko::keystone::auth } + include ::openstack::gnocchi::params + if $::openstack::gnocchi::params::service_enabled { + include ::gnocchi::keystone::auth + } + include ::openstack::cinder::params if $::openstack::cinder::params::service_enabled { include ::cinder::keystone::auth diff --git a/puppet-manifests/src/modules/openstack/templates/gnocchi-api.conf.erb b/puppet-manifests/src/modules/openstack/templates/gnocchi-api.conf.erb new file mode 100644 index 0000000000..763aac83ed --- /dev/null +++ b/puppet-manifests/src/modules/openstack/templates/gnocchi-api.conf.erb @@ -0,0 +1,3 @@ +bind='<%= @url_host %>:<%= @api_port %>' +workers=<%= @api_workers %> + diff --git a/puppet-manifests/src/modules/platform/manifests/haproxy.pp b/puppet-manifests/src/modules/platform/manifests/haproxy.pp index beb4db7a59..0430a566c5 100644 --- a/puppet-manifests/src/modules/platform/manifests/haproxy.pp +++ b/puppet-manifests/src/modules/platform/manifests/haproxy.pp @@ -145,12 +145,12 @@ class platform::haproxy::runtime { include ::openstack::glance::haproxy include ::openstack::cinder::haproxy include ::openstack::aodh::haproxy - include ::openstack::ceilometer::haproxy include ::openstack::heat::haproxy include ::openstack::murano::haproxy include ::openstack::magnum::haproxy include ::openstack::ironic::haproxy include ::openstack::panko::haproxy + include ::openstack::gnocchi::haproxy class {'::platform::haproxy::reload': stage => post diff --git a/puppet-manifests/src/modules/platform/manifests/postgresql.pp b/puppet-manifests/src/modules/platform/manifests/postgresql.pp index 499c4d7cc2..e283c117f0 100644 --- a/puppet-manifests/src/modules/platform/manifests/postgresql.pp +++ b/puppet-manifests/src/modules/platform/manifests/postgresql.pp @@ -198,9 +198,9 @@ class platform::postgresql::upgrade } include ::aodh::db::postgresql - include ::ceilometer::db::postgresql include ::cinder::db::postgresql include ::glance::db::postgresql + include ::gnocchi::db::postgresql include ::heat::db::postgresql include ::murano::db::postgresql include ::magnum::db::postgresql diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index 5de3ee6388..f6f4e42e19 100755 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -646,14 +646,6 @@ class platform::sm command => "sm-configure service_instance nova-novnc nova-novnc \"config=/etc/nova/nova.conf,user=root,console_port=${novnc_console_port}\"", } - exec { 'Configure OpenStack - Ceilometer Collector': - command => "sm-configure service_instance ceilometer-collector ceilometer-collector \"config=/etc/ceilometer/ceilometer.conf\"", - } - - exec { 'Configure OpenStack - Ceilometer API': - command => "sm-configure service_instance ceilometer-api ceilometer-api \"config=/etc/ceilometer/ceilometer.conf\"", - } - exec { 'Configure OpenStack - Ceilometer Agent Notification': command => "sm-configure service_instance ceilometer-agent-notification ceilometer-agent-notification \"config=/etc/ceilometer/ceilometer.conf\"", } @@ -712,6 +704,36 @@ class platform::sm } } + # Gnocchi + if $::openstack::gnocchi::params::service_enabled { + + exec { 'Configure OpenStack - Gnocchi API': + command => "sm-configure service_instance gnocchi-api gnocchi-api \"config=/etc/gnocchi/gnocchi.conf\"", + } + + exec { 'Configure OpenStack - Gnocchi metricd': + command => "sm-configure service_instance gnocchi-metricd gnocchi-metricd \"config=/etc/gnocchi/gnocchi.conf\"", + } + } else { + exec { 'Deprovision OpenStack - Gnocchi API (service-group-member)': + path => [ '/usr/bin', '/usr/sbin', '/usr/local/bin', '/etc', '/sbin', '/bin' ], + command => "sm-deprovision service-group-member cloud-services gnocchi-api", + } -> + exec { 'Deprovision OpenStack - Gnocchi API (service)': + path => [ '/usr/bin', '/usr/sbin', '/usr/local/bin', '/etc', '/sbin', '/bin' ], + command => "sm-deprovision service gnocchi-api", + } + + exec { 'Deprovision OpenStack - Gnocchi metricd (service-group-member)': + path => [ '/usr/bin', '/usr/sbin', '/usr/local/bin', '/etc', '/sbin', '/bin' ], + command => "sm-deprovision service-group-member cloud-services gnocchi-metricd", + } -> + exec { 'Deprovision OpenStack - Gnocchi metricd (service)': + path => [ '/usr/bin', '/usr/sbin', '/usr/local/bin', '/etc', '/sbin', '/bin' ], + command => "sm-deprovision service gnocchi-metricd", + } + } + # AODH if $::openstack::aodh::params::service_enabled { diff --git a/puppet-manifests/src/modules/platform/templates/remotelogging.conf.erb b/puppet-manifests/src/modules/platform/templates/remotelogging.conf.erb index 7781f27925..3353fccf84 100644 --- a/puppet-manifests/src/modules/platform/templates/remotelogging.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/remotelogging.conf.erb @@ -18,12 +18,8 @@ rewrite r_rewrite_set{ set("<%= @system_name %> aodh-notifier.log ${HOST}", value("HOST") condition(filter(f_aodhnotifier))); set("<%= @system_name %> auth.log ${HOST}", value("HOST") condition(filter(f_auth))); set("<%= @system_name %> bash.log ${HOST}", value("HOST") condition(filter(f_bash))); - set("<%= @system_name %> ceilometer-agent-central.log ${HOST}", value("HOST") condition(filter(f_ceilometeragentcentral))); set("<%= @system_name %> ceilometer-agent-notification.log ${HOST}", value("HOST") condition(filter(f_ceilometeragentnotification))); - set("<%= @system_name %> ceilometer-alarm-evaluator.log ${HOST}", value("HOST") condition(filter(f_ceilometeralarmevaluator))); - set("<%= @system_name %> ceilometer-alarm-notifier.log ${HOST}", value("HOST") condition(filter(f_ceilometeralarmnotifier))); - set("<%= @system_name %> ceilometer-api.log ${HOST}", value("HOST") condition(filter(f_ceilometerapi))); - set("<%= @system_name %> ceilometer-collector.log ${HOST}", value("HOST") condition(filter(f_ceilometercollector))); + set("<%= @system_name %> ceilometer-upgrade.log ${HOST}", value("HOST") condition(filter(f_ceilometerupgrade))); set("<%= @system_name %> cinder-api.log ${HOST}", value("HOST") condition(filter(f_cinderapi))); set("<%= @system_name %> cinder-scheduler.log ${HOST}", value("HOST") condition(filter(f_cinderscheduler))); set("<%= @system_name %> cinder-volume.log ${HOST}", value("HOST") condition(filter(f_cindervolume))); @@ -99,6 +95,8 @@ rewrite r_rewrite_set{ set("<%= @system_name %> user.log ${HOST}", value("HOST") condition(filter(f_user))); set("<%= @system_name %> uucp.log ${HOST}", value("HOST") condition(filter(f_uucp))); set("<%= @system_name %> snmp-api.log ${HOST}", value("HOST") condition(filter(f_snmpat))); + set("<%= @system_name %> api.log ${HOST}", value("HOST") condition(filter(f_gnocchiapi))); + set("<%= @system_name %> metricd.log ${HOST}", value("HOST") condition(filter(f_gnocchimetricd))); # Most logs write log level to the message field. some writes it to the PRIORITY field # The priority field is not sent remotely. This is because tcp/udp destinations don't # work well with templates, which we use to write the priority field to log files on the diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log.py index 63ba19a3ab..a3f8eae4e8 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log.py @@ -4,8 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # -from ceilometerclient.v2 import options from cgtsclient.common import base +from cgtsclient.v1 import options class EventLog(base.Resource): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log_shell.py index c52fa9b802..36803c709e 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_log_shell.py @@ -9,10 +9,10 @@ # All Rights Reserved. # -from ceilometerclient.v2 import options from cgtsclient.common import utils from cgtsclient.common import wrapping_formatters from cgtsclient import exc +from cgtsclient.v1 import options def _display_event(log): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression.py index d457550170..b445c51c51 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression.py @@ -4,8 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # -from ceilometerclient.v2 import options from cgtsclient.common import base +from cgtsclient.v1 import options class EventSuppression(base.Resource): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression_shell.py index d621c7f065..282a4a5d46 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/event_suppression_shell.py @@ -8,9 +8,9 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # All Rights Reserved. # -from ceilometerclient.v2 import options from cgtsclient.common import utils from cgtsclient.common import wrapping_formatters +from cgtsclient.v1 import options def _get_display_config(includeUUID): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm.py index dcd14d7254..344e780fb5 100755 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm.py @@ -4,8 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # -from ceilometerclient.v2 import options from cgtsclient.common import base +from cgtsclient.v1 import options class ialarm(base.Resource): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm_shell.py index f1fa354726..db15adfbf9 100755 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/ialarm_shell.py @@ -9,11 +9,11 @@ # All Rights Reserved. # -from ceilometerclient.common import utils -from ceilometerclient.v2 import options +from cgtsclient.common import utils from cgtsclient.common import utils as cgts_utils from cgtsclient.common import wrapping_formatters from cgtsclient import exc +from cgtsclient.v1 import options def _display_fault(fault): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/options.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/options.py new file mode 100644 index 0000000000..bb8d08fee1 --- /dev/null +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/options.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import re + +from six.moves import urllib + +OP_LOOKUP = {'!=': 'ne', + '>=': 'ge', + '<=': 'le', + '>': 'gt', + '<': 'lt', + '=': 'eq'} + +OP_LOOKUP_KEYS = '|'.join(sorted(OP_LOOKUP.keys(), key=len, reverse=True)) +OP_SPLIT_RE = re.compile(r'(%s)' % OP_LOOKUP_KEYS) + +DATA_TYPE_RE = re.compile(r'^(string|integer|float|datetime|boolean)(::)(.+)$') + + +def build_url(path, q, params=None): + """Convert list of dicts and a list of params to query url format. + + This will convert the following: + "[{field=this,op=le,value=34}, + {field=that,op=eq,value=foo,type=string}], + ['foo=bar','sna=fu']" + to: + "?q.field=this&q.field=that& + q.op=le&q.op=eq& + q.type=&q.type=string& + q.value=34&q.value=foo& + foo=bar&sna=fu" + """ + if q: + query_params = {'q.field': [], + 'q.value': [], + 'q.op': [], + 'q.type': []} + + for query in q: + for name in ['field', 'op', 'value', 'type']: + query_params['q.%s' % name].append(query.get(name, '')) + + # Transform the dict to a sequence of two-element tuples in fixed + # order, then the encoded string will be consistent in Python 2&3. + new_qparams = sorted(query_params.items(), key=lambda x: x[0]) + path += "?" + urllib.parse.urlencode(new_qparams, doseq=True) + + if params: + for p in params: + path += '&%s' % p + elif params: + path += '?%s' % params[0] + for p in params[1:]: + path += '&%s' % p + return path + + +def cli_to_array(cli_query): + """Convert CLI list of queries to the Python API format. + + This will convert the following: + "this<=34;that=string::foo" + to + "[{field=this,op=le,value=34,type=''}, + {field=that,op=eq,value=foo,type=string}]" + + """ + + if cli_query is None: + return None + + def split_by_op(query): + """Split a single query string to field, operator, value.""" + + def _value_error(message): + raise ValueError('invalid query %(query)s: missing %(message)s' % + {'query': query, 'message': message}) + + try: + field, operator, value = OP_SPLIT_RE.split(query, maxsplit=1) + except ValueError: + _value_error('operator') + + if not len(field): + _value_error('field') + + if not len(value): + _value_error('value') + + return field.strip(), operator, value.strip() + + def split_by_data_type(query_value): + frags = DATA_TYPE_RE.match(query_value) + + # The second match is the separator. Return a list without it if + # a type identifier was found. + return frags.group(1, 3) if frags else None + + opts = [] + queries = cli_query.split(';') + for q in queries: + query = split_by_op(q) + opt = {} + opt['field'] = query[0] + opt['op'] = OP_LOOKUP[query[1]] + + # Allow the data type of the value to be specified via ::, + # where type can be one of integer, string, float, datetime, boolean + value_frags = split_by_data_type(query[2]) + if not value_frags: + opt['value'] = query[2] + opt['type'] = '' + else: + opt['type'] = value_frags[0] + opt['value'] = value_frags[1] + opts.append(opt) + return opts diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter.py index 4165186668..e291f93b7f 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter.py @@ -7,8 +7,8 @@ # -*- encoding: utf-8 -*- # -from ceilometerclient.v2 import options from cgtsclient.common import base +from cgtsclient.v1 import options class ServiceParameter(base.Resource): diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter_shell.py index d7dbf9ff42..583b5a740f 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/service_parameter_shell.py @@ -8,9 +8,9 @@ # All Rights Reserved. # -from ceilometerclient.v2 import options from cgtsclient.common import utils from cgtsclient import exc +from cgtsclient.v1 import options def _print_service_parameter_show(obj): diff --git a/sysinv/cgts-client/cgts-client/requirements.txt b/sysinv/cgts-client/cgts-client/requirements.txt index 013144dd9a..616bfb5b31 100644 --- a/sysinv/cgts-client/cgts-client/requirements.txt +++ b/sysinv/cgts-client/cgts-client/requirements.txt @@ -1,2 +1 @@ python-neutronclient -python-ceilometerclient diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index c14148dc37..9a7e60a89b 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -776,7 +776,6 @@ SERVICE_TYPE_PLATFORM = 'platform' SERVICE_TYPE_NOVA = 'nova' SERVICE_TYPE_SWIFT = 'swift' SERVICE_TYPE_IRONIC = 'ironic' -SERVICE_TYPE_CEILOMETER = 'ceilometer' SERVICE_TYPE_PANKO = 'panko' SERVICE_TYPE_AODH = 'aodh' SERVICE_TYPE_GLANCE = 'glance' @@ -949,15 +948,12 @@ SERVICE_PARAM_NAME_NOVA_PCI_ALIAS_USER = NOVA_PCI_ALIAS_USER_NAME # default time to live seconds PM_TTL_DEFAULT = 86400 -# Ceilometer Service Parameters -SERVICE_PARAM_SECTION_CEILOMETER_DATABASE = "database" -SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE = "metering_time_to_live" -SERVICE_PARAM_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_DEFAULT = PM_TTL_DEFAULT - +# Panko Service Parameters SERVICE_PARAM_SECTION_PANKO_DATABASE = "database" SERVICE_PARAM_NAME_PANKO_DATABASE_EVENT_TIME_TO_LIVE = "event_time_to_live" SERVICE_PARAM_PANKO_DATABASE_EVENT_TIME_TO_LIVE_DEFAULT = PM_TTL_DEFAULT +# AODH Service Parameters SERVICE_PARAM_SECTION_AODH_DATABASE = "database" SERVICE_PARAM_NAME_AODH_DATABASE_ALARM_HISTORY_TIME_TO_LIVE = "alarm_history_time_to_live" SERVICE_PARAM_AODH_DATABASE_ALARM_HISTORY_TIME_TO_LIVE_DEFAULT = PM_TTL_DEFAULT diff --git a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py index 726b076c2b..405ffc9ddd 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py @@ -543,13 +543,6 @@ def _validate_hbs_degrade_threshold(name, value): SERVICE_PARAM_PLAT_MTCE_HBS_DEGRADE_THRESHOLD_MAX) -# Validate range of Performance Monitoring Metering 'time to live" value -def _validate_metering_time_to_live_range(name, value): - _validate_range(name, value, - SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_MIN, - SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_MAX) - - # Validate range of Performance Monitoring Event 'time to live" value def _validate_event_time_to_live_range(name, value): _validate_range(name, value, @@ -1337,25 +1330,6 @@ PLATFORM_MTCE_PARAMETER_RESOURCE = { constants.SERVICE_PARAM_PLAT_MTCE_HBS_DEGRADE_THRESHOLD: 'platform::mtce::params::heartbeat_degrade_threshold', } -# Ceilometer Metering TTL range from 1 hour to 1 year -SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_MIN = 3600 -SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_MAX = 31536000 - -# Ceilometer Service Parameters -CEILOMETER_PARAMETER_MANDATORY = [ - constants.SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE, -] - -CEILOMETER_PARAMETER_VALIDATOR = { - constants.SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE: - _validate_metering_time_to_live_range, -} - -CEILOMETER_PARAMETER_RESOURCE = { - constants.SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE: - 'ceilometer::metering_time_to_live', -} - # Panko Event TTL range from 1 hour to 1 year SERVICE_PARAM_NAME_PANKO_DATABASE_EVENT_TIME_TO_LIVE_MIN = 3600 SERVICE_PARAM_NAME_PANKO_DATABASE_EVENT_TIME_TO_LIVE_MAX = 31536000 @@ -1548,13 +1522,6 @@ SERVICE_PARAMETER_SCHEMA = { SERVICE_PARAM_DATA_FORMAT: NOVA_PCI_ALIAS_PARAMETER_DATA_FORMAT, }, }, - constants.SERVICE_TYPE_CEILOMETER: { - constants.SERVICE_PARAM_SECTION_CEILOMETER_DATABASE: { - SERVICE_PARAM_MANDATORY: CEILOMETER_PARAMETER_MANDATORY, - SERVICE_PARAM_VALIDATOR: CEILOMETER_PARAMETER_VALIDATOR, - SERVICE_PARAM_RESOURCE: CEILOMETER_PARAMETER_RESOURCE, - }, - }, constants.SERVICE_TYPE_PANKO: { constants.SERVICE_PARAM_SECTION_PANKO_DATABASE: { SERVICE_PARAM_MANDATORY: PANKO_PARAMETER_MANDATORY, diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 8ca797550f..820670e35f 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -475,11 +475,6 @@ class ConductorManager(service.PeriodicService): 'name': constants.SERVICE_PARAM_PLAT_MTCE_HBS_DEGRADE_THRESHOLD, 'value': constants.SERVICE_PARAM_PLAT_MTCE_HBS_DEGRADE_THRESHOLD_DEFAULT, }, - {'service': constants.SERVICE_TYPE_CEILOMETER, - 'section': constants.SERVICE_PARAM_SECTION_CEILOMETER_DATABASE, - 'name': constants.SERVICE_PARAM_NAME_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE, - 'value': constants.SERVICE_PARAM_CEILOMETER_DATABASE_METERING_TIME_TO_LIVE_DEFAULT, - }, {'service': constants.SERVICE_TYPE_PANKO, 'section': constants.SERVICE_PARAM_SECTION_PANKO_DATABASE, 'name': constants.SERVICE_PARAM_NAME_PANKO_DATABASE_EVENT_TIME_TO_LIVE, @@ -6827,13 +6822,6 @@ class ConductorManager(service.PeriodicService): } self._config_apply_runtime_manifest(context, config_uuid, config_dict) - elif service == constants.SERVICE_TYPE_CEILOMETER: - config_dict = { - "personalities": personalities, - "classes": ['openstack::ceilometer::runtime'] - } - self._config_apply_runtime_manifest(context, config_uuid, config_dict) - elif service == constants.SERVICE_TYPE_PANKO: config_dict = { "personalities": personalities, diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/072_remove_ceilometer_service_parameter.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/072_remove_ceilometer_service_parameter.py new file mode 100644 index 0000000000..bf8f11ea23 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/072_remove_ceilometer_service_parameter.py @@ -0,0 +1,46 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import Column, MetaData, Table, Integer + +from sysinv.openstack.common import log + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + +LOG = log.getLogger(__name__) + + +def upgrade(migrate_engine): + """ + This database upgrade deletes the ceilometer metering_time_to_live + service parameter. + """ + + meta = MetaData() + meta.bind = migrate_engine + + LOG.info("Deleting ceilometer metering_time_to_live service parameter") + if migrate_engine.dialect.has_table(migrate_engine, "service_parameter"): + + sp_t = Table('service_parameter', + meta, + Column('id', Integer, primary_key=True, nullable=False), + mysql_engine=ENGINE, + mysql_charset=CHARSET, + autoload=True) + + ceilometer_metering_time_to_live_delete = sp_t.delete().where( + sp_t.c.service == 'ceilometer') + ceilometer_metering_time_to_live_delete.execute() + return True + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + raise NotImplementedError('SysInv database downgrade is unsupported.') diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/ceilometer.py b/sysinv/sysinv/sysinv/sysinv/puppet/ceilometer.py index bb7afb753c..8f1c5167c8 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/ceilometer.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/ceilometer.py @@ -21,15 +21,10 @@ class CeilometerPuppet(openstack.OpenstackBasePuppet): } def get_secure_static_config(self): - dbpass = self._get_database_password(self.SERVICE_NAME) kspass = self._get_service_password(self.SERVICE_NAME) return { - 'ceilometer::db::postgresql::password': dbpass, - 'ceilometer::keystone::auth::password': kspass, - 'ceilometer::keystone::authtoken::password': kspass, - 'ceilometer::agent::auth::auth_password': kspass, } @@ -37,27 +32,10 @@ class CeilometerPuppet(openstack.OpenstackBasePuppet): ksuser = self._get_service_user_name(self.SERVICE_NAME) config = { - 'ceilometer::keystone::auth::public_url': self.get_public_url(), - 'ceilometer::keystone::auth::internal_url': self.get_internal_url(), - 'ceilometer::keystone::auth::admin_url': self.get_admin_url(), 'ceilometer::keystone::auth::auth_name': ksuser, 'ceilometer::keystone::auth::region': self._region_name(), 'ceilometer::keystone::auth::tenant': self._get_service_tenant_name(), - 'ceilometer::keystone::authtoken::auth_url': - self._keystone_identity_uri(), - 'ceilometer::keystone::authtoken::auth_uri': - self._keystone_auth_uri(), - 'ceilometer::keystone::authtoken::user_domain_name': - self._get_service_user_domain_name(), - 'ceilometer::keystone::authtoken::project_domain_name': - self._get_service_project_domain_name(), - 'ceilometer::keystone::authtoken::project_name': - self._get_service_tenant_name(), - 'ceilometer::keystone::authtoken::region_name': - self._keystone_region_name(), - 'ceilometer::keystone::authtoken::username': ksuser, - 'ceilometer::agent::auth::auth_url': self._keystone_auth_uri(), 'ceilometer::agent::auth::auth_user': ksuser, diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py index 0c9021f256..6f15660bca 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py @@ -26,6 +26,7 @@ from . import common from . import dcmanager from . import dcorch from . import glance +from . import gnocchi from . import heat from . import horizon from . import interface @@ -79,6 +80,7 @@ class PuppetOperator(object): self.dcmanager = dcmanager.DCManagerPuppet(self) self.dcorch = dcorch.DCOrchPuppet(self) self.glance = glance.GlancePuppet(self) + self.gnocchi = gnocchi.GnocchiPuppet(self) self.heat = heat.HeatPuppet(self) self.horizon = horizon.HorizonPuppet(self) self.interface = interface.InterfacePuppet(self) @@ -130,8 +132,8 @@ class PuppetOperator(object): config.update(self.nova.get_static_config()) config.update(self.neutron.get_static_config()) config.update(self.glance.get_static_config()) + config.update(self.gnocchi.get_static_config()) config.update(self.cinder.get_static_config()) - config.update(self.ceilometer.get_static_config()) config.update(self.aodh.get_static_config()) config.update(self.heat.get_static_config()) config.update(self.magnum.get_static_config()) @@ -173,6 +175,7 @@ class PuppetOperator(object): config.update(self.neutron.get_secure_static_config()) config.update(self.horizon.get_secure_static_config()) config.update(self.glance.get_secure_static_config()) + config.update(self.gnocchi.get_secure_static_config()) config.update(self.cinder.get_secure_static_config()) config.update(self.ceilometer.get_secure_static_config()) config.update(self.aodh.get_secure_static_config()) @@ -208,6 +211,7 @@ class PuppetOperator(object): config.update(self.neutron.get_system_config()) config.update(self.horizon.get_system_config()) config.update(self.glance.get_system_config()) + config.update(self.gnocchi.get_system_config()) config.update(self.cinder.get_system_config()) config.update(self.ceilometer.get_system_config()) config.update(self.aodh.get_system_config()) @@ -241,8 +245,8 @@ class PuppetOperator(object): config.update(self.nova.get_secure_system_config()) config.update(self.neutron.get_secure_system_config()) config.update(self.glance.get_secure_system_config()) + config.update(self.gnocchi.get_secure_system_config()) config.update(self.cinder.get_secure_system_config()) - config.update(self.ceilometer.get_secure_system_config()) config.update(self.aodh.get_secure_system_config()) config.update(self.heat.get_secure_system_config()) config.update(self.magnum.get_secure_system_config())