starlingx
/
config
Code Issues Proposed changes

Create a new drbd synch'd filesystem for ETCD 

The new drbd fs for etcd will be managed by SM.
Etcd data directory will be created on this new etcd-fs for kubernetes
config.

Story: 2002841
Task: 22784

Change-Id: I5a11a17e8292a074104a73052ea27d979cd68f57
Signed-off-by: Jack Ding <jack.ding@windriver.com>
This commit is contained in:
Shoaib Nasir 2018-06-27 11:00:50 -04:00 committed by Jack Ding
parent 17f1c84f7e
commit e028fd70dd
7 changed files with 142 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
puppet-manifests/src/manifests/bootstrap.pp
View File

@ -13,7 +13,6 @@ include ::platform::ldap::bootstrap
include ::platform::drbd::bootstrap
include ::platform::postgresql::bootstrap
include ::platform::amqp::bootstrap
include ::platform::etcd::bootstrap
include ::openstack::keystone::bootstrap
include ::openstack::client::bootstrap

52
puppet-manifests/src/modules/platform/manifests/drbd.pp
View File

@ -311,6 +311,52 @@ class platform::drbd::patch_vault (
}
}
class platform::drbd::etcd::params (
#$service_enable = false,
$device = '/dev/drbd7',
$lv_name = 'etcd-lv',
$lv_size = '5',
$mountpoint = '/opt/etcd',
$port = '7797',
$resource_name = 'drbd-etcd',
$vg_name = 'cgts-vg',
) {}
class platform::drbd::etcd (
) inherits ::platform::drbd::etcd::params {
include ::platform::kubernetes::params
if str2bool($::is_initial_config_primary) {
$drbd_primary = true
$drbd_initial = true
$drbd_automount = true
$drbd_manage = true
} else {
$drbd_primary = undef
$drbd_initial = undef
$drbd_automount = undef
$drbd_manage = undef
}
if $::platform::kubernetes::params::enabled {
platform::drbd::filesystem { $resource_name:
vg_name => $vg_name,
lv_name => $lv_name,
lv_size => $lv_size,
port => $port,
device => $device,
mountpoint => $mountpoint,
resync_after => undef,
manage_override => $drbd_manage,
ha_primary_override => $drbd_primary,
initial_setup_override => $drbd_initial,
automount_override => $drbd_automount,
}
}
}
class platform::drbd(
$service_enable = false,
$service_ensure = 'stopped',
@ -342,6 +388,7 @@ class platform::drbd(
include ::platform::drbd::cgcs
include ::platform::drbd::extension
include ::platform::drbd::patch_vault
include ::platform::drbd::etcd
# network changes need to be applied prior to DRBD resources
Anchor['platform::networking'] ->
@ -403,3 +450,8 @@ class platform::drbd::patch_vault::runtime {
include ::platform::drbd::params
include ::platform::drbd::patch_vault
}
class platform::drbd::etcd::runtime {
include ::platform::drbd::params
include ::platform::drbd::etcd
}

16
puppet-manifests/src/modules/platform/manifests/etcd.pp
View File

@ -7,7 +7,7 @@ class platform::etcd::params (
include ::platform::params
$sw_version = $::platform::params::software_version
$etcd_basedir = "/opt/cgcs/etcd"
$etcd_basedir = "/opt/etcd"
$etcd_versioned_dir = "${etcd_basedir}/${sw_version}"
}
@ -69,30 +69,26 @@ class platform::etcd
include ::platform::kubernetes::params
Class['::platform::drbd::cgcs'] -> Class[$name]
Class['::platform::drbd::etcd'] -> Class[$name]
if $::platform::kubernetes::params::enabled {
include ::platform::etcd::datadir
include ::platform::etcd::setup
include ::platform::etcd::init
Class['::platform::etcd::datadir'] ->
Class['::platform::etcd::setup'] ->
Class['::platform::etcd::init']
}
}
class platform::etcd::bootstrap
class platform::etcd::datadir
inherits ::platform::etcd::params {
Class['::platform::drbd::cgcs'] -> Class[$name]
Class['::platform::drbd::etcd'] -> Class[$name]
if $::platform::params::init_database {
file { "${etcd_basedir}":
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0755',
} ->
file { "${etcd_versioned_dir}":
ensure => 'directory',
owner => 'root',

29
puppet-manifests/src/modules/platform/manifests/sm.pp
View File

@ -69,6 +69,11 @@ class platform::sm
$patch_fs_device = $::platform::drbd::patch_vault::params::device
$patch_fs_directory = $::platform::drbd::patch_vault::params::mountpoint
include ::platform::drbd::etcd::params
$etcd_drbd_resource = $::platform::drbd::etcd::params::resource_name
$etcd_fs_device = $::platform::drbd::etcd::params::device
$etcd_fs_directory = $::platform::drbd::etcd::params::mountpoint
include ::openstack::keystone::params
$keystone_api_version = $::openstack::keystone::params::api_version
$keystone_identity_uri = $::openstack::keystone::params::identity_uri
@ -371,6 +376,16 @@ class platform::sm
}
}
if $kubernetes_enabled {
exec { 'Configure ETCD DRBD':
command => "sm-configure service_instance drbd-etcd drbd-etcd:${hostunit} drbd_resource=${etcd_drbd_resource}",
}
exec { 'Configure ETCD DRBD FileSystem':
command => "sm-configure service_instance etcd-fs etcd-fs \"device=${etcd_fs_device},directory=${etcd_fs_directory},options=noatime,nodiratime,fstype=ext4,check_level=20\"",
}
}
if $system_mode == 'duplex-direct' or $system_mode == 'simplex' {
exec { 'Configure CGCS NFS':
command => "sm-configure service_instance cgcs-nfs-ip cgcs-nfs-ip \"ip=${cgcs_nfs_ip_param_ip},cidr_netmask=${cgcs_nfs_ip_param_mask},nic=${cgcs_nfs_ip_interface},arp_count=7,dc=yes\"",
@ -988,7 +1003,19 @@ class platform::sm
}
# Configure ETCD for Kubernetes
if $kubernetes_enabled {
if $kubernetes_enabled {
exec { 'Provision etcd-fs (service-group-member)':
command => "sm-provision service-group-member controller-services etcd-fs",
} ->
exec { 'Provision etcd-fs (service)':
command => "sm-provision service etcd-fs",
} ->
exec { 'Provision drbd-etcd (service-group-member)':
command => "sm-provision service-group-member controller-services drbd-etcd",
} ->
exec { 'Provision drbd-etcd (service)':
command => "sm-provision service drbd-etcd",
} ->
exec { 'Provision ETCD (service-group-member)':
command => "sm-provision service-group-member controller-services etcd",
} ->

7
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -263,9 +263,11 @@ DEFAULT_VIRTUAL_IMG_CONVERSION_STOR_SIZE = 8
DEFAULT_VIRTUAL_BACKUP_STOR_SIZE = 5
DEFAULT_EXTENSION_STOR_SIZE = 1
DEFAULT_PATCH_VAULT_STOR_SIZE = 8
DEFAULT_ETCD_STORE_SIZE = 1
# Docker lv size when Kubernetes is configured
KUBERNETES_DOCKER_STOR_SIZE = 10
ETCD_STOR_SIZE = 5
# Openstack Interface names
OS_INTERFACE_PUBLIC = 'public'
@ -419,6 +421,7 @@ DRBD_PGSQL = 'pgsql'
DRBD_CGCS = 'cgcs'
DRBD_EXTENSION = 'extension'
DRBD_PATCH_VAULT = 'patch-vault'
DRBD_ETCD = 'etcd'
# File system names
FILESYSTEM_NAME_BACKUP = 'backup'
@ -430,6 +433,7 @@ FILESYSTEM_NAME_IMG_CONVERSIONS = 'img-conversions'
FILESYSTEM_NAME_SCRATCH = 'scratch'
FILESYSTEM_NAME_DOCKER = 'docker'
FILESYSTEM_NAME_EXTENSION = 'extension'
FILESYSTEM_NAME_ETCD = 'etcd'
FILESYSTEM_NAME_PATCH_VAULT = 'patch-vault'
FILESYSTEM_LV_DICT = {
@ -440,6 +444,7 @@ FILESYSTEM_LV_DICT = {
FILESYSTEM_NAME_IMG_CONVERSIONS: 'img-conversions-lv',
FILESYSTEM_NAME_DATABASE: 'pgsql-lv',
FILESYSTEM_NAME_EXTENSION: 'extension-lv',
FILESYSTEM_NAME_ETCD: 'etcd-lv',
FILESYSTEM_NAME_PATCH_VAULT: 'patch-vault-lv'
}
@ -455,6 +460,7 @@ SUPPORTED_FILEYSTEM_LIST = [
FILESYSTEM_NAME_SCRATCH,
FILESYSTEM_NAME_DOCKER,
FILESYSTEM_NAME_PATCH_VAULT,
FILESYSTEM_NAME_ETCD,
]
SUPPORTED_REPLICATED_FILEYSTEM_LIST = [
@ -462,6 +468,7 @@ SUPPORTED_REPLICATED_FILEYSTEM_LIST = [
FILESYSTEM_NAME_DATABASE,
FILESYSTEM_NAME_EXTENSION,
FILESYSTEM_NAME_PATCH_VAULT,
FILESYSTEM_NAME_ETCD,
]
# Storage: Volume Group Types

47
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -5298,6 +5298,8 @@ class ConductorManager(service.PeriodicService):
'platform::drbd::extension::runtime',
constants.FILESYSTEM_NAME_PATCH_VAULT:
'platform::drbd::patch_vault::runtime',
constants.FILESYSTEM_NAME_ETCD:
'platform::drbd::etcd::runtime',
}
puppet_class = None
@ -6517,7 +6519,6 @@ class ConductorManager(service.PeriodicService):
# 0.5 G - /boot
# 20.0 G - /
# 219.5 G - cgts-vg PV
#
# 2) AIO - will leave unused space for further partitioning
# 0.5 G - /boot
# 20.0 G - /
@ -6619,6 +6620,20 @@ class ConductorManager(service.PeriodicService):
data['name'], data['logical_volume'], data['size']))
self.dbapi.controller_fs_create(data)
# ETCD fs added to cgts-lv
etcd_lv_size = constants.ETCD_STOR_SIZE
data_etcd = {
'name': constants.FILESYSTEM_NAME_ETCD,
'size': etcd_lv_size,
'logical_volume': constants.FILESYSTEM_LV_DICT[
constants.FILESYSTEM_NAME_ETCD],
'replicated': True,
}
LOG.info("Creating FS:%s:%s %d" % (
data_etcd['name'], data_etcd['logical_volume'], data_etcd['size']))
self.dbapi.controller_fs_create(data_etcd)
if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and
tsc.system_type != constants.TIS_AIO_BUILD):
data = {
@ -7108,6 +7123,8 @@ class ConductorManager(service.PeriodicService):
fs.append(constants.DRBD_EXTENSION)
if "drbd-patch-vault" in row and ("SyncSource" in row or "PausedSyncS" in row):
fs.append(constants.DRBD_PATCH_VAULT)
if "drbd-etcd" in row and ("SyncSource" in row or "PausedSyncS" in row):
fs.append(constants.DRBD_ETCD)
return fs
def _drbd_fs_updated(self, context):
@ -7117,6 +7134,9 @@ class ConductorManager(service.PeriodicService):
drbd_patch_size = 0
patch_lv_size = 0
drbd_etcd_size = 0
etcd_lv_size = 0
for row in drbd_dict:
if "sync\'ed" not in row:
try:
@ -7141,6 +7161,8 @@ class ConductorManager(service.PeriodicService):
drbd_extension_size = size
if 'drbd-patch-vault' in row:
drbd_patch_size = size
if 'drbd-etcd' in row:
drbd_etcd_size = size
lvdisplay_dict = self.get_controllerfs_lv_sizes(context)
if lvdisplay_dict.get('pgsql-lv', None):
@ -7151,9 +7173,11 @@ class ConductorManager(service.PeriodicService):
extension_lv_size = round(float(lvdisplay_dict['extension-lv']))
if lvdisplay_dict.get('patch-vault-lv', None):
patch_lv_size = round(float(lvdisplay_dict['patch-vault-lv']))
if lvdisplay_dict.get('etcd-lv', None):
etcd_lv_size = round(float(lvdisplay_dict['etcd-lv']))
LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size)
LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size)
LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size)
LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size)
drbd_fs_updated = []
if drbd_pgsql_size < pgsql_lv_size:
@ -7164,6 +7188,8 @@ class ConductorManager(service.PeriodicService):
drbd_fs_updated.append(constants.DRBD_EXTENSION)
if drbd_patch_size < patch_lv_size:
drbd_fs_updated.append(constants.DRBD_PATCH_VAULT)
if drbd_etcd_size < etcd_lv_size:
drbd_fs_updated.append(constants.DRBD_ETCD)
return drbd_fs_updated
@ -7199,6 +7225,7 @@ class ConductorManager(service.PeriodicService):
cgcs_resized = False
extension_resized = False
patch_resized = False
etcd_resized = False
loop_timeout = 0
drbd_fs_updated = self._drbd_fs_updated(context)
if drbd_fs_updated:
@ -7244,6 +7271,17 @@ class ConductorManager(service.PeriodicService):
LOG.info("Performed %s" % progress)
patch_resized = True
if constants.DRBD_ETCD in drbd_fs_updated:
if (not etcd_resized and
(not standby_host or (standby_host and
constants.DRBD_ETCD in self._drbd_fs_sync()))):
# patch_gib /opt/etcd
progress = "resize2fs drbd7"
cmd = ["resize2fs", "/dev/drbd7"]
stdout, __ = cutils.execute(*cmd, attempts=retry_attempts, run_as_root=True)
LOG.info("Performed %s" % progress)
etcd_resized = True
if not standby_host:
break
@ -7257,6 +7295,8 @@ class ConductorManager(service.PeriodicService):
all_resized = False
elif drbd == constants.DRBD_PATCH_VAULT and not patch_resized:
all_resized = False
elif drbd == constants.DRBD_ETCD and not etcd_resized:
all_resized = False
if all_resized:
break
@ -8968,6 +9008,7 @@ class ConductorManager(service.PeriodicService):
if kubernetes_config:
lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/docker-lv '
lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/etcd-lv'
if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and
tsc.system_type != constants.TIS_AIO_BUILD):

5
sysinv/sysinv/sysinv/sysinv/puppet/storage.py
View File

@ -71,6 +71,11 @@ class StoragePuppet(base.BasePuppet):
'platform::filesystem::docker::params::lv_size':
controller_fs.size
})
elif controller_fs.name == constants.FILESYSTEM_NAME_ETCD:
config.update({
'platform::drbd::etcd::params::lv_size':
controller_fs.size
})
return config