From c34dcbb2dad010a175278f8d544c132995545739 Mon Sep 17 00:00:00 2001
From: Bin Qian <bin.qian@windriver.com>
Date: Tue, 5 Feb 2019 09:50:16 -0500
Subject: [PATCH] Configure kubelet.service to be monitored by pmond

Use puppet manifest to setup the following:
1. create kubelet service override to turn off systemd
   auto restart
2. create kubelet pmon conf file to setup pmond
   to monitor kubelet

Story: 2002843
Task:  29255

Depends-On I41206c7ea14d79b5d0cbca945e7a6488eda9b7bb
Change-Id: I94659dd26b664c643270baa2c013f0910a07bf12
Signed-off-by: Bin Qian <bin.qian@windriver.com>
---
 .../modules/platform/manifests/kubernetes.pp  | 33 +++++++++++++++++--
 .../templates/kube-stx-override.conf.erb      |  7 +++-
 .../platform/templates/kubelet-pmond-conf.erb | 15 +++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)
 create mode 100644 puppet-manifests/src/modules/platform/templates/kubelet-pmond-conf.erb

diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
index 319d23b626..2045f2791d 100644
--- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
+++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
@@ -124,7 +124,7 @@ class platform::kubernetes::master::init
       logoutput => true,
     }
 
-    # Add a dependency to kubelet on config so it doesn't enter a bad state on subsequent boots
+    # Add kubelet service override
     -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
       ensure  => file,
       content => template('platform/kube-stx-override.conf.erb'),
@@ -133,6 +133,15 @@ class platform::kubernetes::master::init
       mode    => '0644',
     }
 
+    # set kubelet monitored by pmond
+    -> file { '/etc/pmon.d/kubelet.conf':
+      ensure  => file,
+      content => template('platform/kubelet-pmond-conf.erb'),
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0644',
+    }
+
     # Reload systemd
     -> exec { 'perform systemctl daemon reload for kubelet override':
       command   => 'systemctl daemon-reload',
@@ -221,7 +230,7 @@ class platform::kubernetes::master::init
         logoutput => true,
       }
 
-      # Add a dependency to kubelet on config so it doesn't enter a bad state on subsequent boots
+      # Add kubelet service override
       -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
         ensure  => file,
         content => template('platform/kube-stx-override.conf.erb'),
@@ -230,6 +239,15 @@ class platform::kubernetes::master::init
         mode    => '0644',
       }
 
+      # set kubelet monitored by pmond
+      -> file { '/etc/pmon.d/kubelet.conf':
+        ensure  => file,
+        content => template('platform/kubelet-pmond-conf.erb'),
+        owner   => 'root',
+        group   => 'root',
+        mode    => '0644',
+      }
+
       # Reload systemd
       -> exec { 'perform systemctl daemon reload for kubelet override':
         command   => 'systemctl daemon-reload',
@@ -272,7 +290,7 @@ class platform::kubernetes::worker::init
     unless    => 'test -f /etc/kubernetes/kubelet.conf',
   }
 
-  # Add a dependency to kubelet on config so it doesn't enter a bad state
+  # Add kubelet service override
   -> file { '/etc/systemd/system/kubelet.service.d/kube-stx-override.conf':
     ensure  => file,
     content => template('platform/kube-stx-override.conf.erb'),
@@ -281,6 +299,15 @@ class platform::kubernetes::worker::init
     mode    => '0644',
   }
 
+  # set kubelet monitored by pmond
+  -> file { '/etc/pmon.d/kubelet.conf':
+    ensure  => file,
+    content => template('platform/kubelet-pmond-conf.erb'),
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+  }
+
   # Reload systemd
   -> exec { 'perform systemctl daemon reload for kubelet override':
     command   => 'systemctl daemon-reload',
diff --git a/puppet-manifests/src/modules/platform/templates/kube-stx-override.conf.erb b/puppet-manifests/src/modules/platform/templates/kube-stx-override.conf.erb
index 69f59043f2..0bb3ca38d9 100644
--- a/puppet-manifests/src/modules/platform/templates/kube-stx-override.conf.erb
+++ b/puppet-manifests/src/modules/platform/templates/kube-stx-override.conf.erb
@@ -1,2 +1,7 @@
 [Unit]
-After=config.service
\ No newline at end of file
+# Add a dependency to kubelet on config so it doesn't enter a bad state on subsequent boots
+After=config.service
+
+[Service]
+# pmond monitors kubelet.service
+Restart=no
diff --git a/puppet-manifests/src/modules/platform/templates/kubelet-pmond-conf.erb b/puppet-manifests/src/modules/platform/templates/kubelet-pmond-conf.erb
new file mode 100644
index 0000000000..5ad44668e3
--- /dev/null
+++ b/puppet-manifests/src/modules/platform/templates/kubelet-pmond-conf.erb
@@ -0,0 +1,15 @@
+;
+; Copyright (c) 2019 Wind River Systems, Inc.
+;
+; SPDX-License-Identifier: Apache-2.0
+;
+[process]
+process  = kubelet
+service  = kubelet
+pidfile  = /var/run/kubelet.pid
+style    = lsb            ; lsb
+severity = critical       ; minor, major, critical
+restarts = 3              ; restarts before error assertion
+startuptime = 5           ; seconds to wait after process start
+interval = 5              ; number of seconds to wait between restarts
+debounce = 20             ; number of seconds to wait before degrade clear