Local Docker Registry HA Integration

Add new 1g DRBD filesystem for docker local registry Make SM manage docker local registry Story: 2002840 Task: 22782 Change-Id: I55fb1369c7141e2f40c35d01379925439264eed9 Signed-off-by: Jack Ding <jack.ding@windriver.com>
2018-06-27 14:55:41 -04:00 · 2018-06-27 14:55:41 -04:00 · 959b018784
parent d2da0ddd1e
commit 959b018784
9 changed files with 258 additions and 36 deletions
--- a/puppet-manifests/src/modules/platform/files/docker-distribution
+++ b/puppet-manifests/src/modules/platform/files/docker-distribution
@ -0,0 +1,92 @@
+#!/bin/bash
+#
+# Startup script for docker-distribution
+#
+
+
+DESC="Docker Distribution aka Docker Registry"
+SERVICE="docker-distribution.service"
+PIDFILE="/var/run/docker-distribution.pid"
+
+
+status()
+{
+    if [ "`systemctl is-active docker-distribution.service`" = "active" ]; then
+        RETVAL=0
+        echo "$DESC is running"
+        return
+    else    
+        echo "$DESC is Not running"
+        RETVAL=1
+    fi
+}
+
+start()
+{
+    if [ -e $PIDFILE ]; then
+        PIDDIR=/proc/$(cat $PIDFILE)
+        if [ -d $PIDDIR ]; then
+            echo "$DESC already running."
+            return
+        else
+            echo "Removing stale PID file $PIDFILE"
+            rm -f $PIDFILE
+        fi
+    fi
+    
+    echo "Starting $SERVICE..."
+
+    systemctl start $SERVICE
+
+    if [ $? -eq 0 ]; then
+        echo "Started $SERVICE successfully"
+        RETVAL=0
+    else
+        echo "$SERVICE failed!"
+        RETVAL=1
+    fi
+    
+}
+
+stop()
+{
+    echo -n "Stopping $SERVICE..."
+    systemctl stop $SERVICE
+    if [ $? -eq 0 ]; then
+        echo "$SERVICE stopped."
+    else
+        echo "failed to stop $SERVICE!"
+    fi
+
+    if [ -e $PIDFILE ]; then
+        echo "Removing stale PID file $PIDFILE"
+        rm -f $PIDFILE
+    fi
+}
+
+
+case "$1" in
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    status)
+        status
+        ;;
+    restart)
+        stop
+        start
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|status|restart}"
+        exit 1
+        ;;
+esac
+
+exit $RETVAL
+
+
+
+
--- a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp
+++ b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp
@ -7,28 +7,8 @@ class platform::dockerdistribution::config

  if $enabled {
    include ::platform::network::mgmt::params
-#    This stuff will be needed for SM integration in the future.
-#    I wrote it as part of trying to get it to work, but it's not useable right now
-#    $controller_0_hostname         = $::platform::params::controller_0_hostname
-#    $controller_1_hostname         = $::platform::params::controller_1_hostname
-#    $system_mode                   = $::platform::params::system_mode
-#    if $system_mode == 'simplex' {
-#      $docker_registry_ip   = $::platform::network::mgmt::params::controller0_address
-#    } else {
-#      case $::hostname {
-#        $controller_0_hostname: {
-#          $docker_registry_ip   = $::platform::network::mgmt::params::controller0_address
-#        }
-#        $controller_1_hostname: {
-#          $docker_registry_ip   = $::platform::network::mgmt::params::controller1_address
-#        }
-#      }
-#    }

-    # insecure workaround will be removed along with the template when proper authentication is implemented
-    $insecure_docker_registry_workaround = $::platform::network::mgmt::params::controller_address
-    # proper docker registry ip will be set with SM integration
-    $docker_registry_ip = '0.0.0.0'
+    $docker_registry_ip = $::platform::network::mgmt::params::controller_address

    # currently docker registry is running insecure mode
    # when proper authentication is implemented, this would go away
@ -52,16 +32,14 @@ class platform::dockerdistribution::config
      group   => 'root',
      mode    => '0644',
      content => template('platform/dockerdistribution.conf.erb'),
-    } ->
+    }

-    # for now, start with systemd and not sm
-    service { 'docker-distribution':
-      ensure  => 'running',
-      name    => 'docker-distribution',
-      enable  => true,
-    } ->
-    exec { 'systemctl enable docker-distribution':
-      command => "/usr/bin/systemctl enable docker-distribution.service",
+    # copy the startup script to where it is supposed to be
+    file {'docker_distribution_initd_script':
+      path    => '/etc/init.d/docker-distribution',
+      ensure  => 'present',
+      mode    => '0755',
+      source  => "puppet:///modules/${module_name}/docker-distribution"
    }
  }
 }
--- a/puppet-manifests/src/modules/platform/manifests/drbd.pp
+++ b/puppet-manifests/src/modules/platform/manifests/drbd.pp
@ -357,6 +357,50 @@ class platform::drbd::etcd (
  }
 }

+class platform::drbd::dockerdistribution::params (
+  $device = '/dev/drbd8',
+  $lv_name = 'dockerdistribution-lv',
+  $lv_size = '1',
+  $mountpoint = '/var/lib/docker-distribution',
+  $port = '7798',
+  $resource_name = 'drbd-dockerdistribution',
+  $vg_name = 'cgts-vg',
+) {}
+
+class platform::drbd::dockerdistribution ()
+  inherits ::platform::drbd::dockerdistribution::params {
+
+  include ::platform::kubernetes::params
+
+  if str2bool($::is_initial_config_primary) {
+    $drbd_primary = true
+    $drbd_initial = true
+    $drbd_automount = true
+    $drbd_manage = true
+  } else {
+    $drbd_primary = undef
+    $drbd_initial = undef
+    $drbd_automount = undef
+    $drbd_manage = undef
+  }
+
+  if $::platform::kubernetes::params::enabled {
+    platform::drbd::filesystem { $resource_name:
+      vg_name    => $vg_name,
+      lv_name    => $lv_name,
+      lv_size    => $lv_size,
+      port       => $port,
+      device     => $device,
+      mountpoint => $mountpoint,
+      resync_after => undef,
+      manage_override => $drbd_manage,
+      ha_primary_override => $drbd_primary,
+      initial_setup_override => $drbd_initial,
+      automount_override => $drbd_automount,
+    }
+  }
+}
+
 class platform::drbd(
  $service_enable = false,
  $service_ensure = 'stopped',
@ -389,6 +433,7 @@ class platform::drbd(
  include ::platform::drbd::extension
  include ::platform::drbd::patch_vault
  include ::platform::drbd::etcd
+  include ::platform::drbd::dockerdistribution

  # network changes need to be applied prior to DRBD resources
  Anchor['platform::networking'] ->
@ -455,3 +500,8 @@ class platform::drbd::etcd::runtime {
  include ::platform::drbd::params
  include ::platform::drbd::etcd
 }
+
+class platform::drbd::dockerdistribution::runtime {
+  include ::platform::drbd::params
+  include ::platform::drbd::dockerdistribution
+}
--- a/puppet-manifests/src/modules/platform/manifests/sm.pp
+++ b/puppet-manifests/src/modules/platform/manifests/sm.pp
@ -74,6 +74,11 @@ class platform::sm
  $etcd_fs_device               = $::platform::drbd::etcd::params::device
  $etcd_fs_directory            = $::platform::drbd::etcd::params::mountpoint

+  include ::platform::drbd::dockerdistribution::params
+  $dockerdistribution_drbd_resource          = $::platform::drbd::dockerdistribution::params::resource_name
+  $dockerdistribution_fs_device              = $::platform::drbd::dockerdistribution::params::device
+  $dockerdistribution_fs_directory           = $::platform::drbd::dockerdistribution::params::mountpoint
+
  include ::openstack::keystone::params
  $keystone_api_version          = $::openstack::keystone::params::api_version
  $keystone_identity_uri         = $::openstack::keystone::params::identity_uri
@ -342,6 +347,40 @@ class platform::sm
    command => "sm-configure service_instance rabbit rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},pid_file=${rabbit_pid},nodename=${rabbit_node_name},mnesia_base=${rabbit_mnesia_base},ip=${mgmt_ip_param_ip}\"",
  }

+  if $kubernetes_enabled {
+    exec { 'Provision Docker Distribution FS in SM (service-group-member dockerdistribution-fs)':
+      command => "sm-provision service-group-member controller-services dockerdistribution-fs",
+    } ->
+    exec { 'Provision Docker Distribution FS in SM (service dockerdistribution-fs)':
+      command => "sm-provision service dockerdistribution-fs",
+    } ->
+    exec { 'Provision Docker Distribution DRBD in SM (service-group-member drbd-dockerdistribution)':
+      command => "sm-provision service-group-member controller-services drbd-dockerdistribution",
+    } ->
+    exec { 'Provision Docker Distribution DRBD in SM (service drbd-dockerdistribution)':
+      command => "sm-provision service drbd-dockerdistribution",
+    } ->
+    exec { 'Configure Docker Distribution DRBD':
+      command => "sm-configure service_instance drbd-dockerdistribution drbd-dockerdistribution:${hostunit} \"drbd_resource=${dockerdistribution_drbd_resource}\"",
+    }->
+    exec { 'Configure Docker Distribution FileSystem':
+      command => "sm-configure service_instance dockerdistribution-fs dockerdistribution-fs \"device=${dockerdistribution_fs_device},directory=${dockerdistribution_fs_directory},options=noatime,nodiratime,fstype=ext4,check_level=20\"",
+    }
+  } else {
+    exec { 'Deprovision Docker Distribution FS in SM (service-group-member dockerdistribution-fs)':
+      command => "sm-deprovision service-group-member controller-services dockerdistribution-fs",
+    } ->
+    exec { 'Deprovision Docker Distribution FS in SM (service dockerdistribution-fs)':
+      command => "sm-deprovision service dockerdistribution-fs",
+    } ->
+    exec { 'Deprovision Docker Distribution DRBD in SM (service-group-member drbd-dockerdistribution)':
+      command => "sm-deprovision service-group-member controller-services drbd-dockerdistribution",
+    } ->
+    exec { 'Deprovision Docker Distribution DRBD in SM (service drbd-dockerdistribution)':
+      command => "sm-deprovision service drbd-dockerdistribution",
+    }
+  }
+
  exec { 'Configure CGCS DRBD':
    command => "sm-configure service_instance drbd-cgcs drbd-cgcs:${hostunit} drbd_resource=${cgcs_drbd_resource}",
  }
@ -885,6 +924,10 @@ class platform::sm
    command => "sm-configure service_instance etcd etcd \"config=/etc/etcd/etcd.conf,user=root\"",
  }

+  # Docker Distribution
+  exec { 'Configure Docker Distribution':
+    command => "sm-configure service_instance docker-distribution docker-distribution \"\"",
+  }

  if $system_mode == 'duplex-direct' or $system_mode == 'simplex' {
      exec { 'Configure Platform NFS':
@ -1032,6 +1075,16 @@ class platform::sm
    }
  }

+  # Configure Docker Distribution
+  if $kubernetes_enabled {
+    exec { 'Provision Docker Distribution (service-group-member)':
+        command => "sm-provision service-group-member controller-services docker-distribution",
+    } ->
+    exec { 'Provision Docker Distribution (service)':
+      command => "sm-provision service docker-distribution",
+    }
+  }
+
  exec { 'Configure Murano Rabbit':
    command => "sm-configure service_instance murano-rabbit murano-rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},nodename=${murano_rabbit_node_name},mnesia_base=${murano_rabbit_mnesia_base},ip=${oam_ip_param_ip},config_file=${murano_rabbit_config_file},env_config_file=${murano_rabbit_env_config_file},pid_file=${murano_rabbit_pid},dist_port=${murano_rabbit_dist_port}\"",
  }
--- a/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb
+++ b/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb
@ -6,12 +6,12 @@ storage:
  cache:
    blobdescriptor: inmemory
  filesystem:
-    rootdirectory: /var/lib/registry
+    rootdirectory: /var/lib/docker-distribution
 http:
  addr: <%= @docker_registry_ip %>:9001
  tls:
-    certificate: /etc/ssl/private/server-cert.pem
-    key: /etc/ssl/private/server-cert.pem
+    certificate: /etc/ssl/private/self-signed-server-cert.pem
+    key: /etc/ssl/private/self-signed-server-cert.pem
  headers:
    X-Content-Type-Options: [nosniff]
 health:
--- a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb
+++ b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb
@ -1,3 +1,3 @@
 {
-    "insecure-registries" : [ "<%= @insecure_docker_registry_workaround %>:9001" ]
+    "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ]
 }
--- a/sysinv/sysinv/sysinv/sysinv/common/constants.py
+++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py
@ -251,6 +251,7 @@ VSWITCH_TYPE_NUAGE_VRS = "nuage_vrs"
 # Partition default sizes
 DEFAULT_IMAGE_STOR_SIZE = 10
 DEFAULT_DOCKER_STOR_SIZE = 1
+DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE = 1
 DEFAULT_DATABASE_STOR_SIZE = 20
 DEFAULT_IMG_CONVERSION_STOR_SIZE = 20
 DEFAULT_SMALL_IMAGE_STOR_SIZE = 10
@ -422,6 +423,7 @@ DRBD_CGCS = 'cgcs'
 DRBD_EXTENSION = 'extension'
 DRBD_PATCH_VAULT = 'patch-vault'
 DRBD_ETCD = 'etcd'
+DRBD_DOCKER_DISTRIBUTION = 'docker-distribution'

 # File system names
 FILESYSTEM_NAME_BACKUP = 'backup'
@ -432,6 +434,7 @@ FILESYSTEM_NAME_DATABASE = 'database'
 FILESYSTEM_NAME_IMG_CONVERSIONS = 'img-conversions'
 FILESYSTEM_NAME_SCRATCH = 'scratch'
 FILESYSTEM_NAME_DOCKER = 'docker'
+FILESYSTEM_NAME_DOCKER_DISTRIBUTION = 'docker-distribution'
 FILESYSTEM_NAME_EXTENSION = 'extension'
 FILESYSTEM_NAME_ETCD = 'etcd'
 FILESYSTEM_NAME_PATCH_VAULT = 'patch-vault'
@ -441,6 +444,7 @@ FILESYSTEM_LV_DICT = {
    FILESYSTEM_NAME_BACKUP: 'backup-lv',
    FILESYSTEM_NAME_SCRATCH: 'scratch-lv',
    FILESYSTEM_NAME_DOCKER: 'docker-lv',
+    FILESYSTEM_NAME_DOCKER_DISTRIBUTION: 'dockerdistribution-lv',
    FILESYSTEM_NAME_IMG_CONVERSIONS: 'img-conversions-lv',
    FILESYSTEM_NAME_DATABASE: 'pgsql-lv',
    FILESYSTEM_NAME_EXTENSION: 'extension-lv',
@ -459,6 +463,7 @@ SUPPORTED_FILEYSTEM_LIST = [
    FILESYSTEM_NAME_IMG_CONVERSIONS,
    FILESYSTEM_NAME_SCRATCH,
    FILESYSTEM_NAME_DOCKER,
+    FILESYSTEM_NAME_DOCKER_DISTRIBUTION,
    FILESYSTEM_NAME_PATCH_VAULT,
    FILESYSTEM_NAME_ETCD,
 ]
@ -469,6 +474,7 @@ SUPPORTED_REPLICATED_FILEYSTEM_LIST = [
    FILESYSTEM_NAME_EXTENSION,
    FILESYSTEM_NAME_PATCH_VAULT,
    FILESYSTEM_NAME_ETCD,
+    FILESYSTEM_NAME_DOCKER_DISTRIBUTION,
 ]

 # Storage: Volume Group Types
--- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py
+++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py
@ -5290,6 +5290,8 @@ class ConductorManager(service.PeriodicService):
                        'platform::filesystem::scratch::runtime',
                    constants.FILESYSTEM_NAME_DOCKER:
                        'platform::filesystem::docker::runtime',
+                    constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION:
+                        'platform::drbd::dockerdistribution::runtime',
                    constants.FILESYSTEM_NAME_DATABASE:
                        'platform::drbd::pgsql::runtime',
                    constants.FILESYSTEM_NAME_CGCS:
@ -6634,6 +6636,17 @@ class ConductorManager(service.PeriodicService):
                data_etcd['name'], data_etcd['logical_volume'], data_etcd['size']))
            self.dbapi.controller_fs_create(data_etcd)

+            data = {
+                'name': constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION,
+                'size': constants.DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE,
+                'logical_volume': constants.FILESYSTEM_LV_DICT[
+                    constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION],
+                'replicated': True,
+            }
+            LOG.info("Creating FS:%s:%s %d" % (
+                data['name'], data['logical_volume'], data['size']))
+            self.dbapi.controller_fs_create(data)
+
        if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and
                tsc.system_type != constants.TIS_AIO_BUILD):
            data = {
@ -7125,6 +7138,8 @@ class ConductorManager(service.PeriodicService):
                fs.append(constants.DRBD_PATCH_VAULT)
            if "drbd-etcd" in row and ("SyncSource" in row or "PausedSyncS" in row):
                fs.append(constants.DRBD_ETCD)
+            if "drbd-dockerdistribution" in row and ("SyncSource" in row or "PausedSyncS" in row):
+                fs.append(constants.DRBD_DOCKER_DISTRIBUTION)
        return fs

    def _drbd_fs_updated(self, context):
@ -7134,6 +7149,8 @@ class ConductorManager(service.PeriodicService):

        drbd_patch_size = 0
        patch_lv_size = 0
+        dockerdistribution_size = 0
+        dockerdistribution_lv_size = 0
        drbd_etcd_size = 0
        etcd_lv_size = 0

@ -7163,6 +7180,8 @@ class ConductorManager(service.PeriodicService):
                    drbd_patch_size = size
                if 'drbd-etcd' in row:
                    drbd_etcd_size = size
+                if 'drbd-dockerdistribution' in row:
+                    dockerdistribution_size = size

        lvdisplay_dict = self.get_controllerfs_lv_sizes(context)
        if lvdisplay_dict.get('pgsql-lv', None):
@ -7175,9 +7194,11 @@ class ConductorManager(service.PeriodicService):
            patch_lv_size = round(float(lvdisplay_dict['patch-vault-lv']))
        if lvdisplay_dict.get('etcd-lv', None):
            etcd_lv_size = round(float(lvdisplay_dict['etcd-lv']))
+        if lvdisplay_dict.get('dockerdistribution-lv', None):
+            dockerdistribution_lv_size = round(float(lvdisplay_dict['dockerdistribution-lv']))

-        LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size)
-        LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size)
+        LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s, dockerdistribution-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size, dockerdistribution_size)
+        LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s, dockerdistribution-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size, dockerdistribution_lv_size)

        drbd_fs_updated = []
        if drbd_pgsql_size < pgsql_lv_size:
@ -7190,6 +7211,8 @@ class ConductorManager(service.PeriodicService):
            drbd_fs_updated.append(constants.DRBD_PATCH_VAULT)
        if drbd_etcd_size < etcd_lv_size:
            drbd_fs_updated.append(constants.DRBD_ETCD)
+        if dockerdistribution_size < dockerdistribution_lv_size:
+            drbd_fs_updated.append(constants.DRBD_DOCKER_DISTRIBUTION)

        return drbd_fs_updated

@ -7226,6 +7249,7 @@ class ConductorManager(service.PeriodicService):
                extension_resized = False
                patch_resized = False
                etcd_resized = False
+                dockerdistribution_resized = False
                loop_timeout = 0
                drbd_fs_updated = self._drbd_fs_updated(context)
                if drbd_fs_updated:
@ -7282,6 +7306,17 @@ class ConductorManager(service.PeriodicService):
                                LOG.info("Performed %s" % progress)
                                etcd_resized = True

+                        if constants.DRBD_DOCKER_DISTRIBUTION in drbd_fs_updated:
+                            if (not dockerdistribution_resized and
+                                (not standby_host or (standby_host and
+                                 constants.DRBD_DOCKER_DISTRIBUTION in self._drbd_fs_sync()))):
+                                # patch_gib /var/lib/docker-distribution
+                                progress = "resize2fs drbd8"
+                                cmd = ["resize2fs", "/dev/drbd8"]
+                                stdout, __ = cutils.execute(*cmd, attempts=retry_attempts, run_as_root=True)
+                                LOG.info("Performed %s" % progress)
+                                dockerdistribution_resized = True
+
                        if not standby_host:
                            break

@ -7297,6 +7332,8 @@ class ConductorManager(service.PeriodicService):
                                all_resized = False
                            elif drbd == constants.DRBD_ETCD and not etcd_resized:
                                all_resized = False
+                            elif drbd == constants.DRBD_DOCKER_DISTRIBUTION and not dockerdistribution_resized:
+                                all_resized = False

                        if all_resized:
                            break
@ -9009,6 +9046,7 @@ class ConductorManager(service.PeriodicService):
        if kubernetes_config:
            lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/docker-lv '
            lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/etcd-lv'
+            lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/dockerdistribution-lv '

        if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and
                tsc.system_type != constants.TIS_AIO_BUILD):
--- a/sysinv/sysinv/sysinv/sysinv/puppet/storage.py
+++ b/sysinv/sysinv/sysinv/sysinv/puppet/storage.py
@ -76,6 +76,11 @@ class StoragePuppet(base.BasePuppet):
                    'platform::drbd::etcd::params::lv_size':
                        controller_fs.size
                })
+            elif controller_fs.name == constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION:
+                config.update({
+                    'platform::drbd::dockerdistribution::params::lv_size':
+                        controller_fs.size
+                })

        return config