ETCD Integration and Management

- create and run etcd manifest as part of config_controller --kubernetes
- etcd bootstrapped to drbd-cgcs fs
- configure SM to manage the service on active controller
- allow ETCD to log data using forward-journald tool Journald forwards
  these etcd logs to /var/log/kern.log

Story: 2002841
Task: 22784

Story: 2002843
Task: 22789

Change-Id: I5d4444133ee7143ce2b7953c6f328651e3b1baa6
Signed-off-by: Jack Ding <jack.ding@windriver.com>

puppet-manifests/src/manifests/bootstrap.pp

@@ -13,6 +13,7 @@ include ::platform::ldap::bootstrap
 include ::platform::drbd::bootstrap
 include ::platform::postgresql::bootstrap
 include ::platform::amqp::bootstrap
+include ::platform::etcd::bootstrap
 
 include ::openstack::keystone::bootstrap
 include ::openstack::client::bootstrap

puppet-manifests/src/manifests/controller.pp

@@ -30,6 +30,7 @@ include ::platform::amqp::rabbitmq
 include ::platform::postgresql::server
 include ::platform::haproxy::server
 include ::platform::grub
+include ::platform::etcd
 include ::platform::docker
 include ::platform::dockerdistribution
 include ::platform::kubernetes::master

puppet-manifests/src/modules/platform/files/etcd

@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Startup script for etcd
+#
+# chkconfig: 2345 20 80
+# description: Starts and stops etcd systemd service
+
+### BEGIN INIT INFO
+# Provides: etcd
+# Required-Start: $local_fs $network
+# Required-Stop: $local_fs
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: Start up the etcd service
+# Description:       A highly-available key value store for shared configuration
+### END INIT INFO
+
+
+DESC="ETCD highly-available key value database"
+SERVICE="etcd.service"
+PIDFILE="/var/run/etcd.pid"
+
+
+status()
+{
+    if [ "`systemctl is-active etcd.service`" = "active" ]; then
+        RETVAL=0
+        echo "$DESC is running"
+        return
+    else    
+        echo "$DESC is Not running"
+        RETVAL=1
+    fi
+}
+
+start()
+{
+    if [ -e $PIDFILE ]; then
+        PIDDIR=/proc/$(cat $PIDFILE)
+        if [ -d $PIDDIR ]; then
+            echo "$DESC already running."
+            return
+        else
+            echo "Removing stale PID file $PIDFILE"
+            rm -f $PIDFILE
+        fi
+    fi
+    
+    echo "Starting $SERVICE..."
+
+    systemctl start $SERVICE
+
+    if [ $? -eq 0 ]; then
+        echo "Started $SERVICE successfully"
+        RETVAL=0
+    else
+        echo "$SERVICE failed!"
+        RETVAL=1
+    fi
+    
+}
+
+stop()
+{
+    echo -n "Stopping $SERVICE..."
+    systemctl stop $SERVICE
+    if [ $? -eq 0 ]; then
+        echo "$SERVICE stopped."
+    else
+        echo "failed to stop $SERVICE!"
+    fi
+
+    if [ -e $PIDFILE ]; then
+        echo "Removing stale PID file $PIDFILE"
+        rm -f $PIDFILE
+    fi
+}
+
+
+case "$1" in
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    status)
+        status
+        ;;
+    restart)
+        stop
+        start
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|status|restart}"
+        exit 1
+        ;;
+esac
+
+exit $RETVAL
+
+
+

puppet-manifests/src/modules/platform/files/etcd-override.conf

@@ -0,0 +1,9 @@
+[Service]
+EnvironmentFile=-/etc/etcd/etcd.conf
+User=root
+NotifyAccess=all
+Type=notify
+ExecStart=
+ExecStart=-/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\" 2>&1 | /usr/bin/forward-journald -tag etcd"
+ExecStartPost=/bin/bash -c 'echo $MAINPID >/var/run/etcd.pid'
+ExecStopPost=/bin/bash/rm -f /var/run/etcd.pid

puppet-manifests/src/modules/platform/manifests/etcd.pp

@@ -0,0 +1,103 @@
+class platform::etcd::params (
+  $bind_address = '0.0.0.0',
+  $port    = 2379,
+  $node   = "controller",
+)
+{
+  include ::platform::params
+
+  $sw_version = $::platform::params::software_version
+  $etcd_basedir = "/opt/cgcs/etcd"
+  $etcd_versioned_dir = "${etcd_basedir}/${sw_version}"
+}
+
+# Modify the systemd service file for etcd and 
+# create an init.d script for SM to manage the service
+class platform::etcd::setup {
+
+  file {'etcd_override_dir':
+    path   => '/etc/systemd/system/etcd.service.d',
+    ensure => directory,
+    mode   => '0755',
+  } ->
+  file {'etcd_override':
+    path    => '/etc/systemd/system/etcd.service.d/etcd-override.conf',
+    ensure  => present,
+    mode    => '0644',
+    source  => "puppet:///modules/${module_name}/etcd-override.conf"
+  } ->
+  file {'etcd_initd_script':
+    path    => '/etc/init.d/etcd',
+    ensure  => 'present',
+    mode    => '0755',
+    source  => "puppet:///modules/${module_name}/etcd"
+  } ->
+  exec { 'systemd-reload-daemon':
+    command     => '/usr/bin/systemctl daemon-reload',
+  } ->
+  Service['etcd']
+}
+
+class platform::etcd::init
+  inherits ::platform::etcd::params {
+  
+  $client_url = "http://${bind_address}:${port}"
+
+  if str2bool($::is_initial_config_primary) {
+    $service_ensure = 'running'
+  }
+  else {
+    $service_ensure = 'stopped'
+  }
+
+  class { 'etcd':
+    ensure              => 'present',
+    etcd_name           => $node,
+    service_enable      => false,
+    service_ensure      => $service_ensure,
+    cluster_enabled     => false,
+    listen_client_urls  => $client_url,
+    advertise_client_urls => $client_url,
+    data_dir            => "${etcd_versioned_dir}/${node}.etcd",
+    proxy               => "off",
+  } 
+}
+
+
+class platform::etcd
+  inherits ::platform::etcd::params {
+
+  include ::platform::kubernetes::params
+
+  Class['::platform::drbd::cgcs'] -> Class[$name]
+  
+  if $::platform::kubernetes::params::enabled {
+    include ::platform::etcd::setup
+    include ::platform::etcd::init
+
+    Class['::platform::etcd::setup'] ->
+    Class['::platform::etcd::init']
+
+  }  
+}
+
+class platform::etcd::bootstrap 
+  inherits ::platform::etcd::params {
+
+  Class['::platform::drbd::cgcs'] -> Class[$name]
+
+  if $::platform::params::init_database {
+    file { "${etcd_basedir}":
+        ensure  => 'directory',
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0755',
+    } ->
+    file { "${etcd_versioned_dir}":
+        ensure  => 'directory',
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0755',
+    }
+  }
+}

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -2,6 +2,7 @@ class platform::kubernetes::params (
   $enabled = false,
   $pod_network_cidr = undef,
   $apiserver_advertise_address = undef,
+  $etcd_endpoint = undef,
 ) { }
 
 class platform::kubernetes::kubeadm {
@@ -124,6 +125,7 @@ class platform::kubernetes::master
     include ::platform::kubernetes::kubeadm
     include ::platform::kubernetes::master::init
 
+    Class['::platform::etcd'] -> Class[$name]
     Class['::platform::docker::config'] -> Class[$name]
     Class['::platform::kubernetes::kubeadm'] ->
     Class['::platform::kubernetes::master::init']

puppet-manifests/src/modules/platform/manifests/sm.pp

@@ -97,6 +97,9 @@ class platform::sm
   $rabbitmq_server = '/usr/lib/rabbitmq/bin/rabbitmq-server'
   $rabbitmqctl     = '/usr/lib/rabbitmq/bin/rabbitmqctl'
 
+  include ::platform::kubernetes::params
+  $kubernetes_enabled             = $::platform::kubernetes::params::enabled
+
   ############ NFS Parameters ################
 
   # Platform NFS network is over the management network
@@ -831,6 +834,12 @@ class platform::sm
     command => "sm-configure service_instance platform-export-fs platform-export-fs \"fsid=0,directory=${platform_fs_directory},options=rw,sync,no_root_squash,no_subtree_check,clientspec=${platform_nfs_subnet_url},unlock_on_stop=true\"",
   }
 
+  # etcd
+  exec { 'Configure ETCD':
+    command => "sm-configure service_instance etcd etcd \"config=/etc/etcd/etcd.conf,user=root\"",
+  }
+
+
   if $system_mode == 'duplex-direct' or $system_mode == 'simplex' {
       exec { 'Configure Platform NFS':
         command => "sm-configure service_instance platform-nfs-ip platform-nfs-ip \"ip=${platform_nfs_ip_param_ip},cidr_netmask=${platform_nfs_ip_param_mask},nic=${mgmt_ip_interface},arp_count=7,dc=yes\"",
@@ -946,6 +955,24 @@ class platform::sm
       command => "sm-provision service drbd-patch-vault",
     }
   }
+  
+  # Configure ETCD for Kubernetes
+  if $kubernetes_enabled {    
+    exec { 'Provision ETCD (service-group-member)': 
+        command => "sm-provision service-group-member controller-services etcd",
+    } ->
+    exec { 'Provision ETCD (service)':
+      command => "sm-provision service etcd",
+    } 
+  }
+  else {
+    exec { 'Deprovision ETCD (service-group-member)':
+      command => "sm-deprovision service-group-member controller-services etcd",
+    } ->
+    exec { 'Deprovision ETCD (service)':
+      command => "sm-deprovision service etcd",
+    }
+  }
 
   exec { 'Configure Murano Rabbit':
     command => "sm-configure service_instance murano-rabbit murano-rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},nodename=${murano_rabbit_node_name},mnesia_base=${murano_rabbit_mnesia_base},ip=${oam_ip_param_ip},config_file=${murano_rabbit_config_file},env_config_file=${murano_rabbit_env_config_file},pid_file=${murano_rabbit_pid},dist_port=${murano_rabbit_dist_port}\"",

puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb

@@ -1,6 +1,9 @@
 apiVersion: kubeadm.k8s.io/v1alpha1
 kind: MasterConfiguration
 api:
-  advertiseAddress: <%= @apiserver_advertise_address %>
+    advertiseAddress: <%= @apiserver_advertise_address %>
+etcd:
+    endpoints:
+    - <%= @etcd_endpoint %>
 networking:
   podSubnet: <%= @pod_network_cidr %>

sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py

@@ -17,6 +17,7 @@ LOG = logging.getLogger(__name__)
 
 class KubernetesPuppet(base.BasePuppet):
     """Class to encapsulate puppet operations for kubernetes configuration"""
+    ETCD_SERVICE_PORT = '2379'
 
     def get_system_config(self):
         config = {}
@@ -27,6 +28,8 @@ class KubernetesPuppet(base.BasePuppet):
                      '192.168.0.0/16',
                  'platform::kubernetes::params::apiserver_advertise_address':
                      self._get_management_address(),
+                 'platform::kubernetes::params::etcd_endpoint':
+                     self._get_etcd_endpoint(),
                  })
 
         return config
@@ -63,3 +66,9 @@ class KubernetesPuppet(base.BasePuppet):
                             'Failed to generate bootstrap token')
 
         return config
+
+    def _get_etcd_endpoint(self):
+        addr = self._format_url_address(self._get_management_address())
+        protocol = "http"
+        url = "%s://%s:%s" % (protocol, str(addr), str(self.ETCD_SERVICE_PORT))
+        return url