diff --git a/puppet-manifests/src/manifests/bootstrap.pp b/puppet-manifests/src/manifests/bootstrap.pp index c23c673fc2..c53ac5a44d 100644 --- a/puppet-manifests/src/manifests/bootstrap.pp +++ b/puppet-manifests/src/manifests/bootstrap.pp @@ -13,7 +13,6 @@ include ::platform::ldap::bootstrap include ::platform::drbd::bootstrap include ::platform::postgresql::bootstrap include ::platform::amqp::bootstrap -include ::platform::etcd::bootstrap include ::openstack::keystone::bootstrap include ::openstack::client::bootstrap diff --git a/puppet-manifests/src/modules/platform/manifests/drbd.pp b/puppet-manifests/src/modules/platform/manifests/drbd.pp index c695eb72e7..5dc16a5f2d 100644 --- a/puppet-manifests/src/modules/platform/manifests/drbd.pp +++ b/puppet-manifests/src/modules/platform/manifests/drbd.pp @@ -311,6 +311,52 @@ class platform::drbd::patch_vault ( } } +class platform::drbd::etcd::params ( + #$service_enable = false, + $device = '/dev/drbd7', + $lv_name = 'etcd-lv', + $lv_size = '5', + $mountpoint = '/opt/etcd', + $port = '7797', + $resource_name = 'drbd-etcd', + $vg_name = 'cgts-vg', +) {} + + +class platform::drbd::etcd ( +) inherits ::platform::drbd::etcd::params { + + include ::platform::kubernetes::params + + if str2bool($::is_initial_config_primary) { + $drbd_primary = true + $drbd_initial = true + $drbd_automount = true + $drbd_manage = true + } else { + $drbd_primary = undef + $drbd_initial = undef + $drbd_automount = undef + $drbd_manage = undef + } + + if $::platform::kubernetes::params::enabled { + platform::drbd::filesystem { $resource_name: + vg_name => $vg_name, + lv_name => $lv_name, + lv_size => $lv_size, + port => $port, + device => $device, + mountpoint => $mountpoint, + resync_after => undef, + manage_override => $drbd_manage, + ha_primary_override => $drbd_primary, + initial_setup_override => $drbd_initial, + automount_override => $drbd_automount, + } + } +} + class platform::drbd( $service_enable = false, $service_ensure = 'stopped', @@ -342,6 +388,7 @@ class platform::drbd( include ::platform::drbd::cgcs include ::platform::drbd::extension include ::platform::drbd::patch_vault + include ::platform::drbd::etcd # network changes need to be applied prior to DRBD resources Anchor['platform::networking'] -> @@ -403,3 +450,8 @@ class platform::drbd::patch_vault::runtime { include ::platform::drbd::params include ::platform::drbd::patch_vault } + +class platform::drbd::etcd::runtime { + include ::platform::drbd::params + include ::platform::drbd::etcd +} diff --git a/puppet-manifests/src/modules/platform/manifests/etcd.pp b/puppet-manifests/src/modules/platform/manifests/etcd.pp index 0153e286e8..a6eade8185 100644 --- a/puppet-manifests/src/modules/platform/manifests/etcd.pp +++ b/puppet-manifests/src/modules/platform/manifests/etcd.pp @@ -7,7 +7,7 @@ class platform::etcd::params ( include ::platform::params $sw_version = $::platform::params::software_version - $etcd_basedir = "/opt/cgcs/etcd" + $etcd_basedir = "/opt/etcd" $etcd_versioned_dir = "${etcd_basedir}/${sw_version}" } @@ -69,30 +69,26 @@ class platform::etcd include ::platform::kubernetes::params - Class['::platform::drbd::cgcs'] -> Class[$name] + Class['::platform::drbd::etcd'] -> Class[$name] if $::platform::kubernetes::params::enabled { + include ::platform::etcd::datadir include ::platform::etcd::setup include ::platform::etcd::init + Class['::platform::etcd::datadir'] -> Class['::platform::etcd::setup'] -> Class['::platform::etcd::init'] } } -class platform::etcd::bootstrap +class platform::etcd::datadir inherits ::platform::etcd::params { - Class['::platform::drbd::cgcs'] -> Class[$name] + Class['::platform::drbd::etcd'] -> Class[$name] if $::platform::params::init_database { - file { "${etcd_basedir}": - ensure => 'directory', - owner => 'root', - group => 'root', - mode => '0755', - } -> file { "${etcd_versioned_dir}": ensure => 'directory', owner => 'root', diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index f6f4e42e19..f469e91578 100755 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -69,6 +69,11 @@ class platform::sm $patch_fs_device = $::platform::drbd::patch_vault::params::device $patch_fs_directory = $::platform::drbd::patch_vault::params::mountpoint + include ::platform::drbd::etcd::params + $etcd_drbd_resource = $::platform::drbd::etcd::params::resource_name + $etcd_fs_device = $::platform::drbd::etcd::params::device + $etcd_fs_directory = $::platform::drbd::etcd::params::mountpoint + include ::openstack::keystone::params $keystone_api_version = $::openstack::keystone::params::api_version $keystone_identity_uri = $::openstack::keystone::params::identity_uri @@ -371,6 +376,16 @@ class platform::sm } } + if $kubernetes_enabled { + exec { 'Configure ETCD DRBD': + command => "sm-configure service_instance drbd-etcd drbd-etcd:${hostunit} drbd_resource=${etcd_drbd_resource}", + } + + exec { 'Configure ETCD DRBD FileSystem': + command => "sm-configure service_instance etcd-fs etcd-fs \"device=${etcd_fs_device},directory=${etcd_fs_directory},options=noatime,nodiratime,fstype=ext4,check_level=20\"", + } + } + if $system_mode == 'duplex-direct' or $system_mode == 'simplex' { exec { 'Configure CGCS NFS': command => "sm-configure service_instance cgcs-nfs-ip cgcs-nfs-ip \"ip=${cgcs_nfs_ip_param_ip},cidr_netmask=${cgcs_nfs_ip_param_mask},nic=${cgcs_nfs_ip_interface},arp_count=7,dc=yes\"", @@ -988,7 +1003,19 @@ class platform::sm } # Configure ETCD for Kubernetes - if $kubernetes_enabled { + if $kubernetes_enabled { + exec { 'Provision etcd-fs (service-group-member)': + command => "sm-provision service-group-member controller-services etcd-fs", + } -> + exec { 'Provision etcd-fs (service)': + command => "sm-provision service etcd-fs", + } -> + exec { 'Provision drbd-etcd (service-group-member)': + command => "sm-provision service-group-member controller-services drbd-etcd", + } -> + exec { 'Provision drbd-etcd (service)': + command => "sm-provision service drbd-etcd", + } -> exec { 'Provision ETCD (service-group-member)': command => "sm-provision service-group-member controller-services etcd", } -> diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 9a7e60a89b..21c55dbc9f 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -263,9 +263,11 @@ DEFAULT_VIRTUAL_IMG_CONVERSION_STOR_SIZE = 8 DEFAULT_VIRTUAL_BACKUP_STOR_SIZE = 5 DEFAULT_EXTENSION_STOR_SIZE = 1 DEFAULT_PATCH_VAULT_STOR_SIZE = 8 +DEFAULT_ETCD_STORE_SIZE = 1 # Docker lv size when Kubernetes is configured KUBERNETES_DOCKER_STOR_SIZE = 10 +ETCD_STOR_SIZE = 5 # Openstack Interface names OS_INTERFACE_PUBLIC = 'public' @@ -419,6 +421,7 @@ DRBD_PGSQL = 'pgsql' DRBD_CGCS = 'cgcs' DRBD_EXTENSION = 'extension' DRBD_PATCH_VAULT = 'patch-vault' +DRBD_ETCD = 'etcd' # File system names FILESYSTEM_NAME_BACKUP = 'backup' @@ -430,6 +433,7 @@ FILESYSTEM_NAME_IMG_CONVERSIONS = 'img-conversions' FILESYSTEM_NAME_SCRATCH = 'scratch' FILESYSTEM_NAME_DOCKER = 'docker' FILESYSTEM_NAME_EXTENSION = 'extension' +FILESYSTEM_NAME_ETCD = 'etcd' FILESYSTEM_NAME_PATCH_VAULT = 'patch-vault' FILESYSTEM_LV_DICT = { @@ -440,6 +444,7 @@ FILESYSTEM_LV_DICT = { FILESYSTEM_NAME_IMG_CONVERSIONS: 'img-conversions-lv', FILESYSTEM_NAME_DATABASE: 'pgsql-lv', FILESYSTEM_NAME_EXTENSION: 'extension-lv', + FILESYSTEM_NAME_ETCD: 'etcd-lv', FILESYSTEM_NAME_PATCH_VAULT: 'patch-vault-lv' } @@ -455,6 +460,7 @@ SUPPORTED_FILEYSTEM_LIST = [ FILESYSTEM_NAME_SCRATCH, FILESYSTEM_NAME_DOCKER, FILESYSTEM_NAME_PATCH_VAULT, + FILESYSTEM_NAME_ETCD, ] SUPPORTED_REPLICATED_FILEYSTEM_LIST = [ @@ -462,6 +468,7 @@ SUPPORTED_REPLICATED_FILEYSTEM_LIST = [ FILESYSTEM_NAME_DATABASE, FILESYSTEM_NAME_EXTENSION, FILESYSTEM_NAME_PATCH_VAULT, + FILESYSTEM_NAME_ETCD, ] # Storage: Volume Group Types diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 9663a58a9b..cb119680c1 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -5298,6 +5298,8 @@ class ConductorManager(service.PeriodicService): 'platform::drbd::extension::runtime', constants.FILESYSTEM_NAME_PATCH_VAULT: 'platform::drbd::patch_vault::runtime', + constants.FILESYSTEM_NAME_ETCD: + 'platform::drbd::etcd::runtime', } puppet_class = None @@ -6517,7 +6519,6 @@ class ConductorManager(service.PeriodicService): # 0.5 G - /boot # 20.0 G - / # 219.5 G - cgts-vg PV - # # 2) AIO - will leave unused space for further partitioning # 0.5 G - /boot # 20.0 G - / @@ -6619,6 +6620,20 @@ class ConductorManager(service.PeriodicService): data['name'], data['logical_volume'], data['size'])) self.dbapi.controller_fs_create(data) + # ETCD fs added to cgts-lv + etcd_lv_size = constants.ETCD_STOR_SIZE + + data_etcd = { + 'name': constants.FILESYSTEM_NAME_ETCD, + 'size': etcd_lv_size, + 'logical_volume': constants.FILESYSTEM_LV_DICT[ + constants.FILESYSTEM_NAME_ETCD], + 'replicated': True, + } + LOG.info("Creating FS:%s:%s %d" % ( + data_etcd['name'], data_etcd['logical_volume'], data_etcd['size'])) + self.dbapi.controller_fs_create(data_etcd) + if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and tsc.system_type != constants.TIS_AIO_BUILD): data = { @@ -7108,6 +7123,8 @@ class ConductorManager(service.PeriodicService): fs.append(constants.DRBD_EXTENSION) if "drbd-patch-vault" in row and ("SyncSource" in row or "PausedSyncS" in row): fs.append(constants.DRBD_PATCH_VAULT) + if "drbd-etcd" in row and ("SyncSource" in row or "PausedSyncS" in row): + fs.append(constants.DRBD_ETCD) return fs def _drbd_fs_updated(self, context): @@ -7117,6 +7134,9 @@ class ConductorManager(service.PeriodicService): drbd_patch_size = 0 patch_lv_size = 0 + drbd_etcd_size = 0 + etcd_lv_size = 0 + for row in drbd_dict: if "sync\'ed" not in row: try: @@ -7141,6 +7161,8 @@ class ConductorManager(service.PeriodicService): drbd_extension_size = size if 'drbd-patch-vault' in row: drbd_patch_size = size + if 'drbd-etcd' in row: + drbd_etcd_size = size lvdisplay_dict = self.get_controllerfs_lv_sizes(context) if lvdisplay_dict.get('pgsql-lv', None): @@ -7151,9 +7173,11 @@ class ConductorManager(service.PeriodicService): extension_lv_size = round(float(lvdisplay_dict['extension-lv'])) if lvdisplay_dict.get('patch-vault-lv', None): patch_lv_size = round(float(lvdisplay_dict['patch-vault-lv'])) + if lvdisplay_dict.get('etcd-lv', None): + etcd_lv_size = round(float(lvdisplay_dict['etcd-lv'])) - LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size) - LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size) + LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size) + LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size) drbd_fs_updated = [] if drbd_pgsql_size < pgsql_lv_size: @@ -7164,6 +7188,8 @@ class ConductorManager(service.PeriodicService): drbd_fs_updated.append(constants.DRBD_EXTENSION) if drbd_patch_size < patch_lv_size: drbd_fs_updated.append(constants.DRBD_PATCH_VAULT) + if drbd_etcd_size < etcd_lv_size: + drbd_fs_updated.append(constants.DRBD_ETCD) return drbd_fs_updated @@ -7199,6 +7225,7 @@ class ConductorManager(service.PeriodicService): cgcs_resized = False extension_resized = False patch_resized = False + etcd_resized = False loop_timeout = 0 drbd_fs_updated = self._drbd_fs_updated(context) if drbd_fs_updated: @@ -7244,6 +7271,17 @@ class ConductorManager(service.PeriodicService): LOG.info("Performed %s" % progress) patch_resized = True + if constants.DRBD_ETCD in drbd_fs_updated: + if (not etcd_resized and + (not standby_host or (standby_host and + constants.DRBD_ETCD in self._drbd_fs_sync()))): + # patch_gib /opt/etcd + progress = "resize2fs drbd7" + cmd = ["resize2fs", "/dev/drbd7"] + stdout, __ = cutils.execute(*cmd, attempts=retry_attempts, run_as_root=True) + LOG.info("Performed %s" % progress) + etcd_resized = True + if not standby_host: break @@ -7257,6 +7295,8 @@ class ConductorManager(service.PeriodicService): all_resized = False elif drbd == constants.DRBD_PATCH_VAULT and not patch_resized: all_resized = False + elif drbd == constants.DRBD_ETCD and not etcd_resized: + all_resized = False if all_resized: break @@ -8968,6 +9008,7 @@ class ConductorManager(service.PeriodicService): if kubernetes_config: lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/docker-lv ' + lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/etcd-lv' if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and tsc.system_type != constants.TIS_AIO_BUILD): diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/storage.py b/sysinv/sysinv/sysinv/sysinv/puppet/storage.py index 4b3a40608f..6e4dcae41d 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/storage.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/storage.py @@ -71,6 +71,11 @@ class StoragePuppet(base.BasePuppet): 'platform::filesystem::docker::params::lv_size': controller_fs.size }) + elif controller_fs.name == constants.FILESYSTEM_NAME_ETCD: + config.update({ + 'platform::drbd::etcd::params::lv_size': + controller_fs.size + }) return config