From f64fa55aa4fa9f0eec5cdee36a247f71964c3cdb Mon Sep 17 00:00:00 2001 From: Shoaib Nasir Date: Fri, 25 May 2018 11:06:55 -0400 Subject: [PATCH] ETCD Integration and Management - create and run etcd manifest as part of config_controller --kubernetes - etcd bootstrapped to drbd-cgcs fs - configure SM to manage the service on active controller - allow ETCD to log data using forward-journald tool Journald forwards these etcd logs to /var/log/kern.log Story: 2002841 Task: 22784 Story: 2002843 Task: 22789 Change-Id: I5d4444133ee7143ce2b7953c6f328651e3b1baa6 Signed-off-by: Jack Ding --- puppet-manifests/src/manifests/bootstrap.pp | 1 + puppet-manifests/src/manifests/controller.pp | 1 + .../src/modules/platform/files/etcd | 103 ++++++++++++++++++ .../modules/platform/files/etcd-override.conf | 9 ++ .../src/modules/platform/manifests/etcd.pp | 103 ++++++++++++++++++ .../modules/platform/manifests/kubernetes.pp | 2 + .../src/modules/platform/manifests/sm.pp | 27 +++++ .../platform/templates/kubeadm.yaml.erb | 5 +- .../sysinv/sysinv/sysinv/puppet/kubernetes.py | 9 ++ 9 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 puppet-manifests/src/modules/platform/files/etcd create mode 100644 puppet-manifests/src/modules/platform/files/etcd-override.conf create mode 100644 puppet-manifests/src/modules/platform/manifests/etcd.pp diff --git a/puppet-manifests/src/manifests/bootstrap.pp b/puppet-manifests/src/manifests/bootstrap.pp index c53ac5a44d..c23c673fc2 100644 --- a/puppet-manifests/src/manifests/bootstrap.pp +++ b/puppet-manifests/src/manifests/bootstrap.pp @@ -13,6 +13,7 @@ include ::platform::ldap::bootstrap include ::platform::drbd::bootstrap include ::platform::postgresql::bootstrap include ::platform::amqp::bootstrap +include ::platform::etcd::bootstrap include ::openstack::keystone::bootstrap include ::openstack::client::bootstrap diff --git a/puppet-manifests/src/manifests/controller.pp b/puppet-manifests/src/manifests/controller.pp index fe7c767bbe..f1da36ed4e 100644 --- a/puppet-manifests/src/manifests/controller.pp +++ b/puppet-manifests/src/manifests/controller.pp @@ -30,6 +30,7 @@ include ::platform::amqp::rabbitmq include ::platform::postgresql::server include ::platform::haproxy::server include ::platform::grub +include ::platform::etcd include ::platform::docker include ::platform::dockerdistribution include ::platform::kubernetes::master diff --git a/puppet-manifests/src/modules/platform/files/etcd b/puppet-manifests/src/modules/platform/files/etcd new file mode 100644 index 0000000000..adbee5fc0b --- /dev/null +++ b/puppet-manifests/src/modules/platform/files/etcd @@ -0,0 +1,103 @@ +#!/bin/bash +# +# Startup script for etcd +# +# chkconfig: 2345 20 80 +# description: Starts and stops etcd systemd service + +### BEGIN INIT INFO +# Provides: etcd +# Required-Start: $local_fs $network +# Required-Stop: $local_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start up the etcd service +# Description: A highly-available key value store for shared configuration +### END INIT INFO + + +DESC="ETCD highly-available key value database" +SERVICE="etcd.service" +PIDFILE="/var/run/etcd.pid" + + +status() +{ + if [ "`systemctl is-active etcd.service`" = "active" ]; then + RETVAL=0 + echo "$DESC is running" + return + else + echo "$DESC is Not running" + RETVAL=1 + fi +} + +start() +{ + if [ -e $PIDFILE ]; then + PIDDIR=/proc/$(cat $PIDFILE) + if [ -d $PIDDIR ]; then + echo "$DESC already running." + return + else + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi + fi + + echo "Starting $SERVICE..." + + systemctl start $SERVICE + + if [ $? -eq 0 ]; then + echo "Started $SERVICE successfully" + RETVAL=0 + else + echo "$SERVICE failed!" + RETVAL=1 + fi + +} + +stop() +{ + echo -n "Stopping $SERVICE..." + systemctl stop $SERVICE + if [ $? -eq 0 ]; then + echo "$SERVICE stopped." + else + echo "failed to stop $SERVICE!" + fi + + if [ -e $PIDFILE ]; then + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi +} + + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + restart) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac + +exit $RETVAL + + + diff --git a/puppet-manifests/src/modules/platform/files/etcd-override.conf b/puppet-manifests/src/modules/platform/files/etcd-override.conf new file mode 100644 index 0000000000..09d2ed47af --- /dev/null +++ b/puppet-manifests/src/modules/platform/files/etcd-override.conf @@ -0,0 +1,9 @@ +[Service] +EnvironmentFile=-/etc/etcd/etcd.conf +User=root +NotifyAccess=all +Type=notify +ExecStart= +ExecStart=-/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" 2>&1 | /usr/bin/forward-journald -tag etcd" +ExecStartPost=/bin/bash -c 'echo $MAINPID >/var/run/etcd.pid' +ExecStopPost=/bin/bash/rm -f /var/run/etcd.pid diff --git a/puppet-manifests/src/modules/platform/manifests/etcd.pp b/puppet-manifests/src/modules/platform/manifests/etcd.pp new file mode 100644 index 0000000000..0153e286e8 --- /dev/null +++ b/puppet-manifests/src/modules/platform/manifests/etcd.pp @@ -0,0 +1,103 @@ +class platform::etcd::params ( + $bind_address = '0.0.0.0', + $port = 2379, + $node = "controller", +) +{ + include ::platform::params + + $sw_version = $::platform::params::software_version + $etcd_basedir = "/opt/cgcs/etcd" + $etcd_versioned_dir = "${etcd_basedir}/${sw_version}" +} + +# Modify the systemd service file for etcd and +# create an init.d script for SM to manage the service +class platform::etcd::setup { + + file {'etcd_override_dir': + path => '/etc/systemd/system/etcd.service.d', + ensure => directory, + mode => '0755', + } -> + file {'etcd_override': + path => '/etc/systemd/system/etcd.service.d/etcd-override.conf', + ensure => present, + mode => '0644', + source => "puppet:///modules/${module_name}/etcd-override.conf" + } -> + file {'etcd_initd_script': + path => '/etc/init.d/etcd', + ensure => 'present', + mode => '0755', + source => "puppet:///modules/${module_name}/etcd" + } -> + exec { 'systemd-reload-daemon': + command => '/usr/bin/systemctl daemon-reload', + } -> + Service['etcd'] +} + +class platform::etcd::init + inherits ::platform::etcd::params { + + $client_url = "http://${bind_address}:${port}" + + if str2bool($::is_initial_config_primary) { + $service_ensure = 'running' + } + else { + $service_ensure = 'stopped' + } + + class { 'etcd': + ensure => 'present', + etcd_name => $node, + service_enable => false, + service_ensure => $service_ensure, + cluster_enabled => false, + listen_client_urls => $client_url, + advertise_client_urls => $client_url, + data_dir => "${etcd_versioned_dir}/${node}.etcd", + proxy => "off", + } +} + + +class platform::etcd + inherits ::platform::etcd::params { + + include ::platform::kubernetes::params + + Class['::platform::drbd::cgcs'] -> Class[$name] + + if $::platform::kubernetes::params::enabled { + include ::platform::etcd::setup + include ::platform::etcd::init + + Class['::platform::etcd::setup'] -> + Class['::platform::etcd::init'] + + } +} + +class platform::etcd::bootstrap + inherits ::platform::etcd::params { + + Class['::platform::drbd::cgcs'] -> Class[$name] + + if $::platform::params::init_database { + file { "${etcd_basedir}": + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } -> + file { "${etcd_versioned_dir}": + ensure => 'directory', + owner => 'root', + group => 'root', + mode => '0755', + } + } +} diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 7537425c65..e87b0fe25a 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -2,6 +2,7 @@ class platform::kubernetes::params ( $enabled = false, $pod_network_cidr = undef, $apiserver_advertise_address = undef, + $etcd_endpoint = undef, ) { } class platform::kubernetes::kubeadm { @@ -124,6 +125,7 @@ class platform::kubernetes::master include ::platform::kubernetes::kubeadm include ::platform::kubernetes::master::init + Class['::platform::etcd'] -> Class[$name] Class['::platform::docker::config'] -> Class[$name] Class['::platform::kubernetes::kubeadm'] -> Class['::platform::kubernetes::master::init'] diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index 3eebfe419e..63e068491c 100644 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -97,6 +97,9 @@ class platform::sm $rabbitmq_server = '/usr/lib/rabbitmq/bin/rabbitmq-server' $rabbitmqctl = '/usr/lib/rabbitmq/bin/rabbitmqctl' + include ::platform::kubernetes::params + $kubernetes_enabled = $::platform::kubernetes::params::enabled + ############ NFS Parameters ################ # Platform NFS network is over the management network @@ -831,6 +834,12 @@ class platform::sm command => "sm-configure service_instance platform-export-fs platform-export-fs \"fsid=0,directory=${platform_fs_directory},options=rw,sync,no_root_squash,no_subtree_check,clientspec=${platform_nfs_subnet_url},unlock_on_stop=true\"", } + # etcd + exec { 'Configure ETCD': + command => "sm-configure service_instance etcd etcd \"config=/etc/etcd/etcd.conf,user=root\"", + } + + if $system_mode == 'duplex-direct' or $system_mode == 'simplex' { exec { 'Configure Platform NFS': command => "sm-configure service_instance platform-nfs-ip platform-nfs-ip \"ip=${platform_nfs_ip_param_ip},cidr_netmask=${platform_nfs_ip_param_mask},nic=${mgmt_ip_interface},arp_count=7,dc=yes\"", @@ -946,6 +955,24 @@ class platform::sm command => "sm-provision service drbd-patch-vault", } } + + # Configure ETCD for Kubernetes + if $kubernetes_enabled { + exec { 'Provision ETCD (service-group-member)': + command => "sm-provision service-group-member controller-services etcd", + } -> + exec { 'Provision ETCD (service)': + command => "sm-provision service etcd", + } + } + else { + exec { 'Deprovision ETCD (service-group-member)': + command => "sm-deprovision service-group-member controller-services etcd", + } -> + exec { 'Deprovision ETCD (service)': + command => "sm-deprovision service etcd", + } + } exec { 'Configure Murano Rabbit': command => "sm-configure service_instance murano-rabbit murano-rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},nodename=${murano_rabbit_node_name},mnesia_base=${murano_rabbit_mnesia_base},ip=${oam_ip_param_ip},config_file=${murano_rabbit_config_file},env_config_file=${murano_rabbit_env_config_file},pid_file=${murano_rabbit_pid},dist_port=${murano_rabbit_dist_port}\"", diff --git a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb index 9338362d96..94c88b56cc 100644 --- a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb @@ -1,6 +1,9 @@ apiVersion: kubeadm.k8s.io/v1alpha1 kind: MasterConfiguration api: - advertiseAddress: <%= @apiserver_advertise_address %> + advertiseAddress: <%= @apiserver_advertise_address %> +etcd: + endpoints: + - <%= @etcd_endpoint %> networking: podSubnet: <%= @pod_network_cidr %> diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index 6842c91ee3..75da881c3c 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -17,6 +17,7 @@ LOG = logging.getLogger(__name__) class KubernetesPuppet(base.BasePuppet): """Class to encapsulate puppet operations for kubernetes configuration""" + ETCD_SERVICE_PORT = '2379' def get_system_config(self): config = {} @@ -27,6 +28,8 @@ class KubernetesPuppet(base.BasePuppet): '192.168.0.0/16', 'platform::kubernetes::params::apiserver_advertise_address': self._get_management_address(), + 'platform::kubernetes::params::etcd_endpoint': + self._get_etcd_endpoint(), }) return config @@ -63,3 +66,9 @@ class KubernetesPuppet(base.BasePuppet): 'Failed to generate bootstrap token') return config + + def _get_etcd_endpoint(self): + addr = self._format_url_address(self._get_management_address()) + protocol = "http" + url = "%s://%s:%s" % (protocol, str(addr), str(self.ETCD_SERVICE_PORT)) + return url