Add customer-specified certificates for kubernetes

We need the ability to update the Kubernetes ApiServer RootCA at
ansible-bootstrap-time. This includes the ability of being able to
specify the apiServerCertSANs such that user can specify additional
DNS:<FQDN> and/or IP Records for the auto-generated
apiServerCertificate.

This adds support for storing the apiServerCertSANs in the sysinv
database and modifies the puppet manifest to support user supplied SAN
records.

Partial-Bug: 1837079
Change-Id: I4d23828b31ced55d55b1c6932d0cfd6b59727288
Signed-off-by: David Sullivan <david.sullivan@windriver.com>

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -17,6 +17,7 @@ class platform::kubernetes::params (
   $k8s_nodeset = undef,
   $k8s_reserved_cpus = undef,
   $k8s_reserved_mem = undef,
+  $apiserver_cert_san = []
 
 ) { }
 
@@ -189,6 +190,8 @@ class platform::kubernetes::master::init
     6 => '::1',
   }
 
+  $apiserver_certsans = concat($apiserver_cert_san, $apiserver_loopback_address, $apiserver_advertise_address)
+
   # This is used for imageRepository in template kubeadm.yaml.erb
   if $::platform::docker::params::k8s_registry {
     $k8s_registry = $::platform::docker::params::k8s_registry

puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb

@@ -3,28 +3,32 @@ kind: InitConfiguration
 apiEndpoint:
   advertiseAddress: <%= @apiserver_advertise_address %>
 ---
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 kubernetesVersion: 1.13.5
+
+apiServer:
+  certSANs:
+<% @apiserver_certsans.each do |item| -%>
+  - <%= item %>
+<% end -%>
+  extraArgs:
+    default-not-ready-toleration-seconds: "30"
+    default-unreachable-toleration-seconds: "30"
+controllerManager:
+  extraArgs:
+    node-monitor-period: "2s"
+    node-monitor-grace-period: "20s"
+    pod-eviction-timeout: "30s"
 etcd:
   external:
     endpoints:
     - <%= @etcd_endpoint %>
-apiServerExtraArgs:
-  default-not-ready-toleration-seconds: "30"
-  default-unreachable-toleration-seconds: "30"
-apiServerCertSANs:
-- "<%= @apiserver_advertise_address %>"
-- "<%= @apiserver_loopback_address %>"
+imageRepository: "<%= @k8s_registry %>"
 networking:
   dnsDomain: <%= @service_domain %>
   podSubnet: <%= @pod_network_cidr %>
   serviceSubnet: <%= @service_network_cidr %>
-controllerManagerExtraArgs:
-  node-monitor-period: "2s"
-  node-monitor-grace-period: "20s"
-  pod-eviction-timeout: "30s"
-imageRepository: "<%= @k8s_registry %>"
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -910,6 +910,7 @@ SERVICE_TYPE_BARBICAN = 'barbican'
 SERVICE_TYPE_DOCKER = 'docker'
 SERVICE_TYPE_HTTP = 'http'
 SERVICE_TYPE_OPENSTACK = 'openstack'
+SERVICE_TYPE_KUBERNETES = 'kubernetes'
 
 SERVICE_PARAM_SECTION_IDENTITY_CONFIG = 'config'
 
@@ -968,6 +969,10 @@ SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker'
 SERVICE_PARAM_NAME_DOCKER_REGISTRIES = 'registries'
 SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry'
 
+# kubernetes parameters
+SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates'
+SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan'
+
 # default filesystem size to 25 MB
 SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25
 
@@ -1199,6 +1204,8 @@ SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/"
 SSL_CERT_CA_FILE = os.path.join(SSL_CERT_CA_DIR, CERT_CA_FILE)
 SSL_CERT_CA_FILE_SHARED = os.path.join(tsc.CONFIG_PATH, CERT_CA_FILE)
 
+KUBERNETES_PKI_SHARED_DIR = os.path.join(tsc.CONFIG_PATH, "kubernetes/pki")
+
 CERT_OPENSTACK_DIR = "/etc/ssl/private/openstack"
 CERT_OPENSTACK_SHARED_DIR = os.path.join(tsc.CONFIG_PATH, 'openstack')
 OPENSTACK_CERT_FILE = os.path.join(CERT_OPENSTACK_DIR, CERT_FILE)

sysinv/sysinv/sysinv/sysinv/common/service_parameter.py

@@ -129,6 +129,25 @@ def _validate_read_only(name, value):
         "Parameter '%s' is readonly" % name))
 
 
+def _validate_SAN_list(name, value):
+    """
+    Validate list of Subject Alternative Name for x509 certificates. Each entry
+    must be an IP address or domain name
+    For example:
+      "localhost.localdomain,192.168.204.2,controller"
+    """
+    san_entries = value.split(',')
+    if len(san_entries) == 0:
+        raise wsme.exc.ClientSideError(_(
+            "No values provided for '%s'" % name))
+
+    for entry in san_entries:
+        if not cutils.is_valid_domain_or_ip(entry):
+            raise wsme.exc.ClientSideError(_(
+                "The value provided is not a domain name or IP address. (%s)"
+                % entry))
+
+
 def _get_network_pool_from_ip_address(ip, networks):
     for name in networks:
         try:
@@ -464,6 +483,23 @@ DOCKER_REGISTRY_PARAMETER_RESOURCE = {
         'platform::docker::params::insecure_registry',
 }
 
+KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL = [
+    constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST,
+]
+
+KUBERNETES_CERTIFICATES_PARAMETER_VALIDATOR = {
+    constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST: _validate_SAN_list,
+}
+
+KUBERNETES_CERTIFICATES_PARAMETER_RESOURCE = {
+    constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST:
+        'platform::kubernetes::params::apiserver_cert_san',
+}
+
+KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT = {
+    constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST: SERVICE_PARAMETER_DATA_FORMAT_ARRAY,
+}
+
 HTTPD_PORT_PARAMETER_OPTIONAL = [
     constants.SERVICE_PARAM_HTTP_PORT_HTTP,
     constants.SERVICE_PARAM_HTTP_PORT_HTTPS,
@@ -548,6 +584,14 @@ SERVICE_PARAMETER_SCHEMA = {
             SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE,
         },
     },
+    constants.SERVICE_TYPE_KUBERNETES: {
+        constants.SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES: {
+            SERVICE_PARAM_OPTIONAL: KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL,
+            SERVICE_PARAM_VALIDATOR: KUBERNETES_CERTIFICATES_PARAMETER_VALIDATOR,
+            SERVICE_PARAM_RESOURCE: KUBERNETES_CERTIFICATES_PARAMETER_RESOURCE,
+            SERVICE_PARAM_DATA_FORMAT: KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT,
+        },
+    },
     constants.SERVICE_TYPE_HTTP: {
         constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: {
             SERVICE_PARAM_OPTIONAL: HTTPD_PORT_PARAMETER_OPTIONAL,

sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py

@@ -53,19 +53,21 @@ class KubernetesPuppet(base.BasePuppet):
 
     def get_secure_system_config(self):
         config = {}
-        # This is retrieving the certificates that 'kubeadm init'
-        # generated. We will want to change this to generate the
-        # certificates ourselves, store in hiera and then feed those
-        # back into 'kubeadm init'.
-        if os.path.exists('/etc/kubernetes/pki/ca.crt'):
+        # This retrieves the certificates that were used during the bootstrap
+        # ansible playbook.
+        if os.path.exists(constants.KUBERNETES_PKI_SHARED_DIR):
             # Store required certificates in configuration.
-            with open('/etc/kubernetes/pki/ca.crt', 'r') as f:
+            with open(os.path.join(
+                    constants.KUBERNETES_PKI_SHARED_DIR, 'ca.crt'), 'r') as f:
                 ca_crt = f.read()
-            with open('/etc/kubernetes/pki/ca.key', 'r') as f:
+            with open(os.path.join(
+                    constants.KUBERNETES_PKI_SHARED_DIR, 'ca.key'), 'r') as f:
                 ca_key = f.read()
-            with open('/etc/kubernetes/pki/sa.key', 'r') as f:
+            with open(os.path.join(
+                    constants.KUBERNETES_PKI_SHARED_DIR, 'sa.key'), 'r') as f:
                 sa_key = f.read()
-            with open('/etc/kubernetes/pki/sa.pub', 'r') as f:
+            with open(os.path.join(
+                    constants.KUBERNETES_PKI_SHARED_DIR, 'sa.pub'), 'r') as f:
                 sa_pub = f.read()
             config.update(
                 {'platform::kubernetes::params::ca_crt': ca_crt,