From c1385750620d77590d9ca5a6c2a5eb952cf0eeb6 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Fri, 8 Feb 2019 14:20:04 +0200 Subject: [PATCH] Ceph initialization on AIO is done only in 'controller' manifests On AIO deployments puppet is run twice with two different manifests: 1. 'controller': to configure controller services 2. 'worker': to configure worker services. Ceph is configured when 'controller' manifests are applied, there is no need to run them a second time, when 'worker' set is applied. Commit adds new puppet classes to encapsulate ceph configuration based on node personality and adds a check to not apply it a 2nd time on controllers. If the ceph manifests are executed a second time then we get into a racing issue between SM's process monitoring and 'worker' puppet manifests triggering a restart of ceph-mon as part of reconfiguration After a reboot on AIO, SM takes control of ceph-mon monitoring after 'controller' puppet manifests finish applying. As part of this, SM monitors processes death notification and gets the pid from the .pid file. And periodically executes '/etc/init.d/ceph status mon.controller' for a more advanced monitoring. When the 'worker' manifests are executed, they trigger a restart of ceph-mon through /etc/init.d/ceph restart that has two steps: 'stop' in which ceph-mon is stopped, and 'start' in which it is restarted. In the first step, stopping ceph-mon leads to the death of ceph-mon process and removal of its PID file. This is promptly detected by SM which immediately triggers a start of ceph-mon that creates a new pid file. Problem is that ceph-mon was already in a restart, and at the end of the 'stop' step the init script cleans up the new pid file instead of the old. This leads to controllers swacting a couple of times before the system gets rid of the rogue process. Change-Id: I2a0df3bab716a553e71e322e1515bee2bb2f700d Co-authored-by: Ovidiu Poncea Story: 2002844 Task: 29214 Signed-off-by: Ovidiu Poncea --- puppet-manifests/centos/build_srpm.data | 2 +- puppet-manifests/src/manifests/controller.pp | 4 +--- puppet-manifests/src/manifests/storage.pp | 4 ---- puppet-manifests/src/manifests/worker.pp | 5 +---- .../src/modules/platform/manifests/ceph.pp | 21 +++++++++++++++++-- sysinv/sysinv/sysinv/sysinv/puppet/ceph.py | 4 ++-- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/puppet-manifests/centos/build_srpm.data b/puppet-manifests/centos/build_srpm.data index ca47755dad..9d22d52d33 100644 --- a/puppet-manifests/centos/build_srpm.data +++ b/puppet-manifests/centos/build_srpm.data @@ -1,2 +1,2 @@ SRC_DIR="src" -TIS_PATCH_VER=79 +TIS_PATCH_VER=80 diff --git a/puppet-manifests/src/manifests/controller.pp b/puppet-manifests/src/manifests/controller.pp index 49a88be4d9..0154b13430 100644 --- a/puppet-manifests/src/manifests/controller.pp +++ b/puppet-manifests/src/manifests/controller.pp @@ -55,9 +55,7 @@ include ::platform::memcached include ::platform::nfv include ::platform::nfv::api -include ::platform::ceph -include ::platform::ceph::monitor -include ::platform::ceph::storage +include ::platform::ceph::controller include ::platform::ceph::rgw include ::platform::influxdb diff --git a/puppet-manifests/src/manifests/storage.pp b/puppet-manifests/src/manifests/storage.pp index 9855560929..fde2af783e 100644 --- a/puppet-manifests/src/manifests/storage.pp +++ b/puppet-manifests/src/manifests/storage.pp @@ -28,11 +28,7 @@ include ::platform::grub include ::platform::collectd include ::platform::filesystem::storage include ::platform::docker - -include ::platform::ceph -include ::platform::ceph::monitor include ::platform::ceph::storage - include ::openstack::ceilometer include ::openstack::ceilometer::polling diff --git a/puppet-manifests/src/manifests/worker.pp b/puppet-manifests/src/manifests/worker.pp index 4d1c87c010..84fb8a6595 100644 --- a/puppet-manifests/src/manifests/worker.pp +++ b/puppet-manifests/src/manifests/worker.pp @@ -35,10 +35,7 @@ include ::platform::dockerdistribution::compute include ::platform::kubernetes::worker include ::platform::multipath include ::platform::client - -include ::platform::ceph -include ::platform::ceph::monitor - +include ::platform::ceph::worker include ::openstack::client include ::openstack::neutron include ::openstack::neutron::agents diff --git a/puppet-manifests/src/modules/platform/manifests/ceph.pp b/puppet-manifests/src/modules/platform/manifests/ceph.pp index 37b9b5f9bb..3cdb77a4ea 100644 --- a/puppet-manifests/src/modules/platform/manifests/ceph.pp +++ b/puppet-manifests/src/modules/platform/manifests/ceph.pp @@ -341,11 +341,10 @@ define platform_ceph_journal( } -class platform::ceph::storage( +class platform::ceph::osds( $osd_config = {}, $journal_config = {}, ) inherits ::platform::ceph::params { - # Ensure partitions update prior to ceph storage configuration Class['::platform::partitions'] -> Class[$name] @@ -484,6 +483,24 @@ class platform::ceph::rgw::keystone::auth( } } +class platform::ceph::worker { + if $::personality == 'worker' { + include ::platform::ceph + include ::platform::ceph::monitor + } +} + +class platform::ceph::storage { + include ::platform::ceph + include ::platform::ceph::monitor + include ::platform::ceph::osds +} + +class platform::ceph::controller { + include ::platform::ceph + include ::platform::ceph::monitor + include ::platform::ceph::osds +} class platform::ceph::runtime { include ::platform::ceph::monitor diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py b/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py index bd64515857..f157e63cb2 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/ceph.py @@ -271,8 +271,8 @@ class CephPuppet(openstack.OpenstackBasePuppet): osd_config.update({name: osd}) return { - 'platform::ceph::storage::osd_config': osd_config, - 'platform::ceph::storage::journal_config': journal_config, + 'platform::ceph::osds::osd_config': osd_config, + 'platform::ceph::osds::journal_config': journal_config, } def _format_ceph_mon_address(self, ip_address):