Puppet support for authenticated registries

This commit supports to pull images from alternative authenticated registries that configured at Ansible bootstrap to bring up k8s pods at puppet time. At bootstrap time, barbican secrets are created to store credentials for accessing registry and alternative registries info are stored in service parameter. At puppet time, the barbican sercret is retrieved to get the credentials in order to pre-pull k8s images that required by kubeadm to bring up static pods(ie..kube-controller-manager, kube-apiserver, kube-scheduler..). The images for dynamic pods(kube-multus, kube-sriov-cni, calico..) and tiller are not needed to pre-pull, imagePullSecrets is added in their pod spec to pass credentials to kubelet. This is done in Ansible bootstrap https://review.opendev.org/#/c/679136/ This commit also updates to pull Armada image before creating Armada container if Armada image is not available in docker cache. Tests(AIO-SX, AIO-DX, Standard): - All types of system are installed successfully - Verified all k8s/gcr/docker images are downloaded from authenticated registry on controller-1 and worker nodes - Verified images from authenticated registries are used by k8s static/dynamic pods on controller-1 and worker nodes - Swact to controller-1, lock/unlock controller-0. Verified that tiller image is downloaded from authenticated registry and tiller pod is created on controller-1 - Swact to controller-1, apply application. Verified that Armada image is downloaded from authenticated registry and Armada container is created. Change-Id: Iaabef0f5d8a6a4640dcfde93a8c0449948f4a59f Depends-On: https://review.opendev.org/679335 Story: 2006274 Task: 36379 Signed-off-by: Angie Wang <angie.wang@windriver.com>
2019-08-26 12:53:50 -04:00 · 2019-08-26 12:53:50 -04:00 · 38c9eb3a4a
parent 6123f0fa9c
commit 38c9eb3a4a
7 changed files with 154 additions and 9 deletions
--- a/puppet-manifests/src/modules/platform/manifests/client.pp
+++ b/puppet-manifests/src/modules/platform/manifests/client.pp
@ -6,6 +6,7 @@ class platform::client::params (
  $admin_user_domain = 'Default',
  $admin_project_domain = 'Default',
  $admin_project_name = 'admin',
+  $admin_password = undef,
  $keystone_identity_region = 'RegionOne',
 ) { }

--- a/puppet-manifests/src/modules/platform/manifests/docker.pp
+++ b/puppet-manifests/src/modules/platform/manifests/docker.pp
@ -7,6 +7,10 @@ class platform::docker::params (
  $gcr_registry    = undef,
  $quay_registry   = undef,
  $docker_registry = undef,
+  $k8s_registry_secret    = undef,
+  $gcr_registry_secret    = undef,
+  $quay_registry_secret   = undef,
+  $docker_registry_secret = undef,
  $insecure_registry    = undef,
 ) { }

@ -85,3 +89,42 @@ class platform::docker::bootstrap
  include ::platform::docker::install
  include ::platform::docker::config::bootstrap
 }
+
+define platform::docker::login_registry (
+  $registry_url,
+  $registry_secret,
+) {
+  include ::platform::client::params
+
+  $auth_url = $::platform::client::params::identity_auth_url
+  $username = $::platform::client::params::admin_username
+  $user_domain = $::platform::client::params::admin_user_domain
+  $project_name = $::platform::client::params::admin_project_name
+  $project_domain = $::platform::client::params::admin_project_domain
+  $region_name = $::platform::client::params::keystone_identity_region
+  $password = $::platform::client::params::admin_password
+  $interface = 'internal'
+
+  # Registry credentials have been stored in Barbican secret at Ansible
+  # bootstrap time, retrieve Barbican secret to get the payload
+  notice("Get payload of Barbican secret ${registry_secret}")
+  $secret_payload = generate(
+    '/bin/sh', '-c', template('platform/get-secret-payload.erb'))
+
+  if $secret_payload {
+    # Parse Barbican secret payload to get the registry username and password
+    $secret_payload_array = split($secret_payload, ' ')
+    $registry_username = split($secret_payload_array[0], 'username:')[1]
+    $registry_password = split($secret_payload_array[1], 'password:')[1]
+
+    # Login to authenticated registry
+    if $registry_username and $registry_password {
+      exec { 'Login registry':
+        command   => "docker login ${registry_url} -u ${registry_username} -p ${registry_password}",
+        logoutput => true,
+      }
+    } else {
+      notice('Registry username or/and password NOT FOUND')
+    }
+  }
+}
--- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
+++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp
@ -109,12 +109,6 @@ class platform::kubernetes::kubeadm {
  $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
    net.bridge.bridge-nf-call-iptables = 1"

-  if $::platform::docker::params::k8s_registry {
-    $k8s_registry = $::platform::docker::params::k8s_registry
-  } else {
-    $k8s_registry = 'k8s.gcr.io'
-  }
-
  # Configure kubelet cpumanager options
  if str2bool($::is_worker_subfunction)
    and !('openstack-compute-node'
@ -335,6 +329,27 @@ class platform::kubernetes::master::init
      #   This flag is created by Ansible on controller-0;
      # - Ansible replay is not impacted by flag creation.

+      # If alternative k8s registry requires the authentication,
+      # kubeadm required images need to be pre-pulled on controller
+      if $k8s_registry != 'k8s.gcr.io' and $::platform::docker::params::k8s_registry_secret != undef {
+        File['/etc/kubernetes/kubeadm.yaml']
+        -> platform::docker::login_registry { 'login k8s registry':
+          registry_url    => $k8s_registry,
+          registry_secret => $::platform::docker::params::k8s_registry_secret
+        }
+
+        -> exec { 'kubeadm to pre pull images':
+          command   => 'kubeadm config images pull --config /etc/kubernetes/kubeadm.yaml',
+          logoutput => true,
+          before    => Exec['configure master node']
+        }
+
+        -> exec { 'logout k8s registry':
+          command   => "docker logout ${k8s_registry}",
+          logoutput => true,
+        }
+      }
+
      # Create necessary certificate files
      file { '/etc/kubernetes/pki':
        ensure => directory,
@ -467,6 +482,44 @@ class platform::kubernetes::worker::init

  Class['::platform::docker::config'] -> Class[$name]

+  if str2bool($::is_initial_config) {
+    include ::platform::params
+
+    if $::platform::docker::params::k8s_registry {
+      $k8s_registry = $::platform::docker::params::k8s_registry
+    } else {
+      $k8s_registry = 'k8s.gcr.io'
+    }
+
+    # If alternative k8s registry requires the authentication,
+    # k8s pause image needs to be pre-pulled on worker nodes
+    if $k8s_registry != 'k8s.gcr.io' and $::platform::docker::params::k8s_registry_secret != undef {
+      # Get the pause image tag from kubeadm required images
+      # list and replace with alternative k8s registry
+      $get_k8s_pause_img = "kubeadm config images list 2>/dev/null |\
+        awk '/^k8s.gcr.io\\/pause:/{print \$1}' | sed 's/k8s.gcr.io/${k8s_registry}/'"
+      $k8s_pause_img = generate('/bin/sh', '-c', $get_k8s_pause_img)
+
+      if k8s_pause_img {
+        platform::docker::login_registry { 'login k8s registry':
+          registry_url    => $k8s_registry,
+          registry_secret => $::platform::docker::params::k8s_registry_secret
+        }
+
+        -> exec { 'load k8s pause image':
+          command   => "docker image pull ${k8s_pause_img}",
+          logoutput => true,
+          before    => Exec['configure worker node']
+        }
+
+        -> exec { 'logout k8s registry':
+          command   => "docker logout ${k8s_registry}",
+          logoutput => true,
+        }
+      }
+    }
+  }
+
  # Configure the worker node. Only do this once, so check whether the
  # kubelet.conf file has already been created (by the join).
  exec { 'configure worker node':
--- a/puppet-manifests/src/modules/platform/templates/get-secret-payload.erb
+++ b/puppet-manifests/src/modules/platform/templates/get-secret-payload.erb
@ -0,0 +1,11 @@
+# Retrieve barbican secret payload
+openstack secret get <%=@registry_secret %> \
+  --os-auth-url <%=@auth_url %> \
+  --os-username <%=@username %> \
+  --os-user-domain-name <%=@user_domain %> \
+  --os-project-name <%=@project_name %> \
+  --os-project-domain-name <%=@project_domain %> \
+  --os-region-name <%=@region_name %> \
+  --os-interface <%=@interface %> \
+  --os-password <%=@password %> \
+  -p -f value -c Payload
--- a/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
+++ b/sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
@ -291,6 +291,13 @@ def _validate_docker_registry_address(name, value):
            "Parameter '%s' must be a valid address." % name))


+def _validate_docker_registry_auth_secret(name, value):
+    """Check if registry auth secret is a valid UUID"""
+    if not cutils.is_uuid_like(value):
+        raise wsme.exc.ClientSideError(_(
+            "Parameter '%s' must be a valid UUID." % name))
+
+
 def _validate_docker_insecure_registry_bool(name, value):
    """Check if insecure registry is a valid bool"""
    if not cutils.is_valid_boolstr(value):
@ -462,26 +469,35 @@ DOCKER_REGISTRIES_PARAMETER_OPTIONAL = [

 DOCKER_REGISTRIES_PARAMETER_VALIDATOR = {
    constants.SERVICE_PARAM_NAME_DOCKER_URL: _validate_docker_registry_address,
+    constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET: _validate_docker_registry_auth_secret
 }

 DOCKER_DOCKER_REGISTRY_PARAMETER_RESOURCE = {
    constants.SERVICE_PARAM_NAME_DOCKER_URL:
        'platform::docker::params::docker_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET:
+        'platform::docker::params::docker_registry_secret'
 }

 DOCKER_GCR_REGISTRY_PARAMETER_RESOURCE = {
    constants.SERVICE_PARAM_NAME_DOCKER_URL:
-        'platform::docker::params::gcr_registry'
+        'platform::docker::params::gcr_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET:
+        'platform::docker::params::gcr_registry_secret'
 }

 DOCKER_K8S_REGISTRY_PARAMETER_RESOURCE = {
    constants.SERVICE_PARAM_NAME_DOCKER_URL:
-        'platform::docker::params::k8s_registry'
+        'platform::docker::params::k8s_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET:
+        'platform::docker::params::k8s_registry_secret'
 }

 DOCKER_QUAY_REGISTRY_PARAMETER_RESOURCE = {
    constants.SERVICE_PARAM_NAME_DOCKER_URL:
-        'platform::docker::params::quay_registry'
+        'platform::docker::params::quay_registry',
+    constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET:
+        'platform::docker::params::quay_registry_secret'
 }

 KUBERNETES_CERTIFICATES_PARAMETER_OPTIONAL = [
--- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
+++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
@ -2513,6 +2513,25 @@ class DockerHelper(object):
                    overrides_dir: {'bind': '/overrides', 'mode': 'ro'},
                    logs_dir: {'bind': ARMADA_CONTAINER_LOG_LOCATION, 'mode': 'rw'}}

+                armada_image = client.images.list(CONF.armada_image_tag)
+                # Pull Armada image if it's not available
+                if not armada_image:
+                    LOG.info("Downloading Armada image %s ..." % CONF.armada_image_tag)
+
+                    quay_registry_secret = self._dbapi.service_parameter_get_all(
+                        service=constants.SERVICE_TYPE_DOCKER,
+                        section=constants.SERVICE_PARAM_SECTION_DOCKER_QUAY_REGISTRY,
+                        name=constants.SERVICE_PARAM_NAME_DOCKER_AUTH_SECRET)
+                    if quay_registry_secret:
+                        quay_registry_auth = self._parse_barbican_secret(
+                            quay_registry_secret[0].value)
+                    else:
+                        quay_registry_auth = None
+
+                    client.images.pull(CONF.armada_image_tag,
+                                       auth_config=quay_registry_auth)
+                    LOG.info("Armada image %s downloaded!" % CONF.armada_image_tag)
+
                container = client.containers.run(
                    CONF.armada_image_tag,
                    name=ARMADA_CONTAINER_NAME,
--- a/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py
+++ b/sysinv/sysinv/sysinv/sysinv/puppet/keystone.py
@ -77,6 +77,7 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
            'keystone::db::postgresql::password': dbpass,

            'keystone::roles::admin::password': admin_password,
+            'platform::client::params::admin_password': admin_password,
        }

    def get_system_config(self):
@ -145,6 +146,7 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
            'keystone::admin_password': admin_password,
            'keystone::roles::admin::password': admin_password,
            'keystone::database_connection': db_connection,
+            'platform::client::params::admin_password': admin_password,
        }
        return config