diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index 471ec8c69a..950f25113f 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -17,6 +17,7 @@ class platform::kubernetes::params ( $k8s_nodeset = undef, $k8s_reserved_cpus = undef, $k8s_reserved_mem = undef, + $apiserver_cert_san = [] ) { } @@ -189,6 +190,8 @@ class platform::kubernetes::master::init 6 => '::1', } + $apiserver_certsans = concat($apiserver_cert_san, $apiserver_loopback_address, $apiserver_advertise_address) + # This is used for imageRepository in template kubeadm.yaml.erb if $::platform::docker::params::k8s_registry { $k8s_registry = $::platform::docker::params::k8s_registry diff --git a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb index 84c4033abe..00d6a2f4d1 100644 --- a/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb +++ b/puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb @@ -3,28 +3,32 @@ kind: InitConfiguration apiEndpoint: advertiseAddress: <%= @apiserver_advertise_address %> --- -apiVersion: kubeadm.k8s.io/v1alpha3 +apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration kubernetesVersion: 1.13.5 + +apiServer: + certSANs: +<% @apiserver_certsans.each do |item| -%> + - <%= item %> +<% end -%> + extraArgs: + default-not-ready-toleration-seconds: "30" + default-unreachable-toleration-seconds: "30" +controllerManager: + extraArgs: + node-monitor-period: "2s" + node-monitor-grace-period: "20s" + pod-eviction-timeout: "30s" etcd: external: endpoints: - <%= @etcd_endpoint %> -apiServerExtraArgs: - default-not-ready-toleration-seconds: "30" - default-unreachable-toleration-seconds: "30" -apiServerCertSANs: -- "<%= @apiserver_advertise_address %>" -- "<%= @apiserver_loopback_address %>" +imageRepository: "<%= @k8s_registry %>" networking: dnsDomain: <%= @service_domain %> podSubnet: <%= @pod_network_cidr %> serviceSubnet: <%= @service_network_cidr %> -controllerManagerExtraArgs: - node-monitor-period: "2s" - node-monitor-grace-period: "20s" - pod-eviction-timeout: "30s" -imageRepository: "<%= @k8s_registry %>" --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index f87c24501f..579dd48065 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -910,6 +910,7 @@ SERVICE_TYPE_BARBICAN = 'barbican' SERVICE_TYPE_DOCKER = 'docker' SERVICE_TYPE_HTTP = 'http' SERVICE_TYPE_OPENSTACK = 'openstack' +SERVICE_TYPE_KUBERNETES = 'kubernetes' SERVICE_PARAM_SECTION_IDENTITY_CONFIG = 'config' @@ -968,6 +969,10 @@ SERVICE_PARAM_NAME_DOCKER_DOCKER_REGISTRY = 'docker' SERVICE_PARAM_NAME_DOCKER_REGISTRIES = 'registries' SERVICE_PARAM_NAME_DOCKER_INSECURE_REGISTRY = 'insecure_registry' +# kubernetes parameters +SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates' +SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan' + # default filesystem size to 25 MB SERVICE_PARAM_SWIFT_FS_SIZE_MB_DEFAULT = 25 @@ -1199,6 +1204,8 @@ SSL_CERT_CA_DIR = "/etc/pki/ca-trust/source/anchors/" SSL_CERT_CA_FILE = os.path.join(SSL_CERT_CA_DIR, CERT_CA_FILE) SSL_CERT_CA_FILE_SHARED = os.path.join(tsc.CONFIG_PATH, CERT_CA_FILE) +KUBERNETES_PKI_SHARED_DIR = os.path.join(tsc.CONFIG_PATH, "kubernetes/pki") + CERT_OPENSTACK_DIR = "/etc/ssl/private/openstack" CERT_OPENSTACK_SHARED_DIR = os.path.join(tsc.CONFIG_PATH, 'openstack') OPENSTACK_CERT_FILE = os.path.join(CERT_OPENSTACK_DIR, CERT_FILE) diff --git a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py index 6ff77d519e..5046b28a19 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py +++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py @@ -129,6 +129,25 @@ def _validate_read_only(name, value): "Parameter '%s' is readonly" % name)) +def _validate_SAN_list(name, value): + """ + Validate list of Subject Alternative Name for x509 certificates. Each entry + must be an IP address or domain name + For example: + "localhost.localdomain,192.168.204.2,controller" + """ + san_entries = value.split(',') + if len(san_entries) == 0: + raise wsme.exc.ClientSideError(_( + "No values provided for '%s'" % name)) + + for entry in san_entries: + if not cutils.is_valid_domain_or_ip(entry): + raise wsme.exc.ClientSideError(_( + "The value provided is not a domain name or IP address. (%s)" + % entry)) + + def _get_network_pool_from_ip_address(ip, networks): for name in networks: try: @@ -464,6 +483,23 @@ DOCKER_REGISTRY_PARAMETER_RESOURCE = { 'platform::docker::params::insecure_registry', } +KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL = [ + constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST, +] + +KUBERNETES_CERTIFICATES_PARAMETER_VALIDATOR = { + constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST: _validate_SAN_list, +} + +KUBERNETES_CERTIFICATES_PARAMETER_RESOURCE = { + constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST: + 'platform::kubernetes::params::apiserver_cert_san', +} + +KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT = { + constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST: SERVICE_PARAMETER_DATA_FORMAT_ARRAY, +} + HTTPD_PORT_PARAMETER_OPTIONAL = [ constants.SERVICE_PARAM_HTTP_PORT_HTTP, constants.SERVICE_PARAM_HTTP_PORT_HTTPS, @@ -548,6 +584,14 @@ SERVICE_PARAMETER_SCHEMA = { SERVICE_PARAM_RESOURCE: DOCKER_REGISTRY_PARAMETER_RESOURCE, }, }, + constants.SERVICE_TYPE_KUBERNETES: { + constants.SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES: { + SERVICE_PARAM_OPTIONAL: KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL, + SERVICE_PARAM_VALIDATOR: KUBERNETES_CERTIFICATES_PARAMETER_VALIDATOR, + SERVICE_PARAM_RESOURCE: KUBERNETES_CERTIFICATES_PARAMETER_RESOURCE, + SERVICE_PARAM_DATA_FORMAT: KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT, + }, + }, constants.SERVICE_TYPE_HTTP: { constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: { SERVICE_PARAM_OPTIONAL: HTTPD_PORT_PARAMETER_OPTIONAL, diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index 729a78ab01..c126f1ba69 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -53,19 +53,21 @@ class KubernetesPuppet(base.BasePuppet): def get_secure_system_config(self): config = {} - # This is retrieving the certificates that 'kubeadm init' - # generated. We will want to change this to generate the - # certificates ourselves, store in hiera and then feed those - # back into 'kubeadm init'. - if os.path.exists('/etc/kubernetes/pki/ca.crt'): + # This retrieves the certificates that were used during the bootstrap + # ansible playbook. + if os.path.exists(constants.KUBERNETES_PKI_SHARED_DIR): # Store required certificates in configuration. - with open('/etc/kubernetes/pki/ca.crt', 'r') as f: + with open(os.path.join( + constants.KUBERNETES_PKI_SHARED_DIR, 'ca.crt'), 'r') as f: ca_crt = f.read() - with open('/etc/kubernetes/pki/ca.key', 'r') as f: + with open(os.path.join( + constants.KUBERNETES_PKI_SHARED_DIR, 'ca.key'), 'r') as f: ca_key = f.read() - with open('/etc/kubernetes/pki/sa.key', 'r') as f: + with open(os.path.join( + constants.KUBERNETES_PKI_SHARED_DIR, 'sa.key'), 'r') as f: sa_key = f.read() - with open('/etc/kubernetes/pki/sa.pub', 'r') as f: + with open(os.path.join( + constants.KUBERNETES_PKI_SHARED_DIR, 'sa.pub'), 'r') as f: sa_pub = f.read() config.update( {'platform::kubernetes::params::ca_crt': ca_crt,