diff --git a/puppet-manifests/src/modules/platform/files/docker-distribution b/puppet-manifests/src/modules/platform/files/docker-distribution new file mode 100644 index 0000000000..f99ee690aa --- /dev/null +++ b/puppet-manifests/src/modules/platform/files/docker-distribution @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Startup script for docker-distribution +# + + +DESC="Docker Distribution aka Docker Registry" +SERVICE="docker-distribution.service" +PIDFILE="/var/run/docker-distribution.pid" + + +status() +{ + if [ "`systemctl is-active docker-distribution.service`" = "active" ]; then + RETVAL=0 + echo "$DESC is running" + return + else + echo "$DESC is Not running" + RETVAL=1 + fi +} + +start() +{ + if [ -e $PIDFILE ]; then + PIDDIR=/proc/$(cat $PIDFILE) + if [ -d $PIDDIR ]; then + echo "$DESC already running." + return + else + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi + fi + + echo "Starting $SERVICE..." + + systemctl start $SERVICE + + if [ $? -eq 0 ]; then + echo "Started $SERVICE successfully" + RETVAL=0 + else + echo "$SERVICE failed!" + RETVAL=1 + fi + +} + +stop() +{ + echo -n "Stopping $SERVICE..." + systemctl stop $SERVICE + if [ $? -eq 0 ]; then + echo "$SERVICE stopped." + else + echo "failed to stop $SERVICE!" + fi + + if [ -e $PIDFILE ]; then + echo "Removing stale PID file $PIDFILE" + rm -f $PIDFILE + fi +} + + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + restart) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|status|restart}" + exit 1 + ;; +esac + +exit $RETVAL + + + + diff --git a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp index 84f4ca0154..b330c37bff 100644 --- a/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp +++ b/puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp @@ -7,28 +7,8 @@ class platform::dockerdistribution::config if $enabled { include ::platform::network::mgmt::params -# This stuff will be needed for SM integration in the future. -# I wrote it as part of trying to get it to work, but it's not useable right now -# $controller_0_hostname = $::platform::params::controller_0_hostname -# $controller_1_hostname = $::platform::params::controller_1_hostname -# $system_mode = $::platform::params::system_mode -# if $system_mode == 'simplex' { -# $docker_registry_ip = $::platform::network::mgmt::params::controller0_address -# } else { -# case $::hostname { -# $controller_0_hostname: { -# $docker_registry_ip = $::platform::network::mgmt::params::controller0_address -# } -# $controller_1_hostname: { -# $docker_registry_ip = $::platform::network::mgmt::params::controller1_address -# } -# } -# } - # insecure workaround will be removed along with the template when proper authentication is implemented - $insecure_docker_registry_workaround = $::platform::network::mgmt::params::controller_address - # proper docker registry ip will be set with SM integration - $docker_registry_ip = '0.0.0.0' + $docker_registry_ip = $::platform::network::mgmt::params::controller_address # currently docker registry is running insecure mode # when proper authentication is implemented, this would go away @@ -52,16 +32,14 @@ class platform::dockerdistribution::config group => 'root', mode => '0644', content => template('platform/dockerdistribution.conf.erb'), - } -> + } - # for now, start with systemd and not sm - service { 'docker-distribution': - ensure => 'running', - name => 'docker-distribution', - enable => true, - } -> - exec { 'systemctl enable docker-distribution': - command => "/usr/bin/systemctl enable docker-distribution.service", + # copy the startup script to where it is supposed to be + file {'docker_distribution_initd_script': + path => '/etc/init.d/docker-distribution', + ensure => 'present', + mode => '0755', + source => "puppet:///modules/${module_name}/docker-distribution" } } } diff --git a/puppet-manifests/src/modules/platform/manifests/drbd.pp b/puppet-manifests/src/modules/platform/manifests/drbd.pp index 5dc16a5f2d..c7f653f0e2 100644 --- a/puppet-manifests/src/modules/platform/manifests/drbd.pp +++ b/puppet-manifests/src/modules/platform/manifests/drbd.pp @@ -357,6 +357,50 @@ class platform::drbd::etcd ( } } +class platform::drbd::dockerdistribution::params ( + $device = '/dev/drbd8', + $lv_name = 'dockerdistribution-lv', + $lv_size = '1', + $mountpoint = '/var/lib/docker-distribution', + $port = '7798', + $resource_name = 'drbd-dockerdistribution', + $vg_name = 'cgts-vg', +) {} + +class platform::drbd::dockerdistribution () + inherits ::platform::drbd::dockerdistribution::params { + + include ::platform::kubernetes::params + + if str2bool($::is_initial_config_primary) { + $drbd_primary = true + $drbd_initial = true + $drbd_automount = true + $drbd_manage = true + } else { + $drbd_primary = undef + $drbd_initial = undef + $drbd_automount = undef + $drbd_manage = undef + } + + if $::platform::kubernetes::params::enabled { + platform::drbd::filesystem { $resource_name: + vg_name => $vg_name, + lv_name => $lv_name, + lv_size => $lv_size, + port => $port, + device => $device, + mountpoint => $mountpoint, + resync_after => undef, + manage_override => $drbd_manage, + ha_primary_override => $drbd_primary, + initial_setup_override => $drbd_initial, + automount_override => $drbd_automount, + } + } +} + class platform::drbd( $service_enable = false, $service_ensure = 'stopped', @@ -389,6 +433,7 @@ class platform::drbd( include ::platform::drbd::extension include ::platform::drbd::patch_vault include ::platform::drbd::etcd + include ::platform::drbd::dockerdistribution # network changes need to be applied prior to DRBD resources Anchor['platform::networking'] -> @@ -455,3 +500,8 @@ class platform::drbd::etcd::runtime { include ::platform::drbd::params include ::platform::drbd::etcd } + +class platform::drbd::dockerdistribution::runtime { + include ::platform::drbd::params + include ::platform::drbd::dockerdistribution +} diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index f469e91578..786979bb71 100755 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -74,6 +74,11 @@ class platform::sm $etcd_fs_device = $::platform::drbd::etcd::params::device $etcd_fs_directory = $::platform::drbd::etcd::params::mountpoint + include ::platform::drbd::dockerdistribution::params + $dockerdistribution_drbd_resource = $::platform::drbd::dockerdistribution::params::resource_name + $dockerdistribution_fs_device = $::platform::drbd::dockerdistribution::params::device + $dockerdistribution_fs_directory = $::platform::drbd::dockerdistribution::params::mountpoint + include ::openstack::keystone::params $keystone_api_version = $::openstack::keystone::params::api_version $keystone_identity_uri = $::openstack::keystone::params::identity_uri @@ -342,6 +347,40 @@ class platform::sm command => "sm-configure service_instance rabbit rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},pid_file=${rabbit_pid},nodename=${rabbit_node_name},mnesia_base=${rabbit_mnesia_base},ip=${mgmt_ip_param_ip}\"", } + if $kubernetes_enabled { + exec { 'Provision Docker Distribution FS in SM (service-group-member dockerdistribution-fs)': + command => "sm-provision service-group-member controller-services dockerdistribution-fs", + } -> + exec { 'Provision Docker Distribution FS in SM (service dockerdistribution-fs)': + command => "sm-provision service dockerdistribution-fs", + } -> + exec { 'Provision Docker Distribution DRBD in SM (service-group-member drbd-dockerdistribution)': + command => "sm-provision service-group-member controller-services drbd-dockerdistribution", + } -> + exec { 'Provision Docker Distribution DRBD in SM (service drbd-dockerdistribution)': + command => "sm-provision service drbd-dockerdistribution", + } -> + exec { 'Configure Docker Distribution DRBD': + command => "sm-configure service_instance drbd-dockerdistribution drbd-dockerdistribution:${hostunit} \"drbd_resource=${dockerdistribution_drbd_resource}\"", + }-> + exec { 'Configure Docker Distribution FileSystem': + command => "sm-configure service_instance dockerdistribution-fs dockerdistribution-fs \"device=${dockerdistribution_fs_device},directory=${dockerdistribution_fs_directory},options=noatime,nodiratime,fstype=ext4,check_level=20\"", + } + } else { + exec { 'Deprovision Docker Distribution FS in SM (service-group-member dockerdistribution-fs)': + command => "sm-deprovision service-group-member controller-services dockerdistribution-fs", + } -> + exec { 'Deprovision Docker Distribution FS in SM (service dockerdistribution-fs)': + command => "sm-deprovision service dockerdistribution-fs", + } -> + exec { 'Deprovision Docker Distribution DRBD in SM (service-group-member drbd-dockerdistribution)': + command => "sm-deprovision service-group-member controller-services drbd-dockerdistribution", + } -> + exec { 'Deprovision Docker Distribution DRBD in SM (service drbd-dockerdistribution)': + command => "sm-deprovision service drbd-dockerdistribution", + } + } + exec { 'Configure CGCS DRBD': command => "sm-configure service_instance drbd-cgcs drbd-cgcs:${hostunit} drbd_resource=${cgcs_drbd_resource}", } @@ -885,6 +924,10 @@ class platform::sm command => "sm-configure service_instance etcd etcd \"config=/etc/etcd/etcd.conf,user=root\"", } + # Docker Distribution + exec { 'Configure Docker Distribution': + command => "sm-configure service_instance docker-distribution docker-distribution \"\"", + } if $system_mode == 'duplex-direct' or $system_mode == 'simplex' { exec { 'Configure Platform NFS': @@ -1032,6 +1075,16 @@ class platform::sm } } + # Configure Docker Distribution + if $kubernetes_enabled { + exec { 'Provision Docker Distribution (service-group-member)': + command => "sm-provision service-group-member controller-services docker-distribution", + } -> + exec { 'Provision Docker Distribution (service)': + command => "sm-provision service docker-distribution", + } + } + exec { 'Configure Murano Rabbit': command => "sm-configure service_instance murano-rabbit murano-rabbit \"server=${rabbitmq_server},ctl=${rabbitmqctl},nodename=${murano_rabbit_node_name},mnesia_base=${murano_rabbit_mnesia_base},ip=${oam_ip_param_ip},config_file=${murano_rabbit_config_file},env_config_file=${murano_rabbit_env_config_file},pid_file=${murano_rabbit_pid},dist_port=${murano_rabbit_dist_port}\"", } diff --git a/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb b/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb index cb9f5334be..6fd11227af 100644 --- a/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/dockerdistribution.conf.erb @@ -6,12 +6,12 @@ storage: cache: blobdescriptor: inmemory filesystem: - rootdirectory: /var/lib/registry + rootdirectory: /var/lib/docker-distribution http: addr: <%= @docker_registry_ip %>:9001 tls: - certificate: /etc/ssl/private/server-cert.pem - key: /etc/ssl/private/server-cert.pem + certificate: /etc/ssl/private/self-signed-server-cert.pem + key: /etc/ssl/private/self-signed-server-cert.pem headers: X-Content-Type-Options: [nosniff] health: diff --git a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb index 7acc158954..0998bc827f 100644 --- a/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb +++ b/puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb @@ -1,3 +1,3 @@ { - "insecure-registries" : [ "<%= @insecure_docker_registry_workaround %>:9001" ] + "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ] } diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 21c55dbc9f..0cc644987f 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -251,6 +251,7 @@ VSWITCH_TYPE_NUAGE_VRS = "nuage_vrs" # Partition default sizes DEFAULT_IMAGE_STOR_SIZE = 10 DEFAULT_DOCKER_STOR_SIZE = 1 +DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE = 1 DEFAULT_DATABASE_STOR_SIZE = 20 DEFAULT_IMG_CONVERSION_STOR_SIZE = 20 DEFAULT_SMALL_IMAGE_STOR_SIZE = 10 @@ -422,6 +423,7 @@ DRBD_CGCS = 'cgcs' DRBD_EXTENSION = 'extension' DRBD_PATCH_VAULT = 'patch-vault' DRBD_ETCD = 'etcd' +DRBD_DOCKER_DISTRIBUTION = 'docker-distribution' # File system names FILESYSTEM_NAME_BACKUP = 'backup' @@ -432,6 +434,7 @@ FILESYSTEM_NAME_DATABASE = 'database' FILESYSTEM_NAME_IMG_CONVERSIONS = 'img-conversions' FILESYSTEM_NAME_SCRATCH = 'scratch' FILESYSTEM_NAME_DOCKER = 'docker' +FILESYSTEM_NAME_DOCKER_DISTRIBUTION = 'docker-distribution' FILESYSTEM_NAME_EXTENSION = 'extension' FILESYSTEM_NAME_ETCD = 'etcd' FILESYSTEM_NAME_PATCH_VAULT = 'patch-vault' @@ -441,6 +444,7 @@ FILESYSTEM_LV_DICT = { FILESYSTEM_NAME_BACKUP: 'backup-lv', FILESYSTEM_NAME_SCRATCH: 'scratch-lv', FILESYSTEM_NAME_DOCKER: 'docker-lv', + FILESYSTEM_NAME_DOCKER_DISTRIBUTION: 'dockerdistribution-lv', FILESYSTEM_NAME_IMG_CONVERSIONS: 'img-conversions-lv', FILESYSTEM_NAME_DATABASE: 'pgsql-lv', FILESYSTEM_NAME_EXTENSION: 'extension-lv', @@ -459,6 +463,7 @@ SUPPORTED_FILEYSTEM_LIST = [ FILESYSTEM_NAME_IMG_CONVERSIONS, FILESYSTEM_NAME_SCRATCH, FILESYSTEM_NAME_DOCKER, + FILESYSTEM_NAME_DOCKER_DISTRIBUTION, FILESYSTEM_NAME_PATCH_VAULT, FILESYSTEM_NAME_ETCD, ] @@ -469,6 +474,7 @@ SUPPORTED_REPLICATED_FILEYSTEM_LIST = [ FILESYSTEM_NAME_EXTENSION, FILESYSTEM_NAME_PATCH_VAULT, FILESYSTEM_NAME_ETCD, + FILESYSTEM_NAME_DOCKER_DISTRIBUTION, ] # Storage: Volume Group Types diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index cb119680c1..aa11e144d6 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -5290,6 +5290,8 @@ class ConductorManager(service.PeriodicService): 'platform::filesystem::scratch::runtime', constants.FILESYSTEM_NAME_DOCKER: 'platform::filesystem::docker::runtime', + constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION: + 'platform::drbd::dockerdistribution::runtime', constants.FILESYSTEM_NAME_DATABASE: 'platform::drbd::pgsql::runtime', constants.FILESYSTEM_NAME_CGCS: @@ -6634,6 +6636,17 @@ class ConductorManager(service.PeriodicService): data_etcd['name'], data_etcd['logical_volume'], data_etcd['size'])) self.dbapi.controller_fs_create(data_etcd) + data = { + 'name': constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION, + 'size': constants.DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE, + 'logical_volume': constants.FILESYSTEM_LV_DICT[ + constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION], + 'replicated': True, + } + LOG.info("Creating FS:%s:%s %d" % ( + data['name'], data['logical_volume'], data['size'])) + self.dbapi.controller_fs_create(data) + if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and tsc.system_type != constants.TIS_AIO_BUILD): data = { @@ -7125,6 +7138,8 @@ class ConductorManager(service.PeriodicService): fs.append(constants.DRBD_PATCH_VAULT) if "drbd-etcd" in row and ("SyncSource" in row or "PausedSyncS" in row): fs.append(constants.DRBD_ETCD) + if "drbd-dockerdistribution" in row and ("SyncSource" in row or "PausedSyncS" in row): + fs.append(constants.DRBD_DOCKER_DISTRIBUTION) return fs def _drbd_fs_updated(self, context): @@ -7134,6 +7149,8 @@ class ConductorManager(service.PeriodicService): drbd_patch_size = 0 patch_lv_size = 0 + dockerdistribution_size = 0 + dockerdistribution_lv_size = 0 drbd_etcd_size = 0 etcd_lv_size = 0 @@ -7163,6 +7180,8 @@ class ConductorManager(service.PeriodicService): drbd_patch_size = size if 'drbd-etcd' in row: drbd_etcd_size = size + if 'drbd-dockerdistribution' in row: + dockerdistribution_size = size lvdisplay_dict = self.get_controllerfs_lv_sizes(context) if lvdisplay_dict.get('pgsql-lv', None): @@ -7175,9 +7194,11 @@ class ConductorManager(service.PeriodicService): patch_lv_size = round(float(lvdisplay_dict['patch-vault-lv'])) if lvdisplay_dict.get('etcd-lv', None): etcd_lv_size = round(float(lvdisplay_dict['etcd-lv'])) + if lvdisplay_dict.get('dockerdistribution-lv', None): + dockerdistribution_lv_size = round(float(lvdisplay_dict['dockerdistribution-lv'])) - LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size) - LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size) + LOG.info("drbd-overview: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s, dockerdistribution-%s", drbd_pgsql_size, drbd_cgcs_size, drbd_extension_size, drbd_patch_size, drbd_etcd_size, dockerdistribution_size) + LOG.info("lvdisplay: pgsql-%s, cgcs-%s, extension-%s, patch-vault-%s, etcd-%s, dockerdistribution-%s", pgsql_lv_size, cgcs_lv_size, extension_lv_size, patch_lv_size, etcd_lv_size, dockerdistribution_lv_size) drbd_fs_updated = [] if drbd_pgsql_size < pgsql_lv_size: @@ -7190,6 +7211,8 @@ class ConductorManager(service.PeriodicService): drbd_fs_updated.append(constants.DRBD_PATCH_VAULT) if drbd_etcd_size < etcd_lv_size: drbd_fs_updated.append(constants.DRBD_ETCD) + if dockerdistribution_size < dockerdistribution_lv_size: + drbd_fs_updated.append(constants.DRBD_DOCKER_DISTRIBUTION) return drbd_fs_updated @@ -7226,6 +7249,7 @@ class ConductorManager(service.PeriodicService): extension_resized = False patch_resized = False etcd_resized = False + dockerdistribution_resized = False loop_timeout = 0 drbd_fs_updated = self._drbd_fs_updated(context) if drbd_fs_updated: @@ -7282,6 +7306,17 @@ class ConductorManager(service.PeriodicService): LOG.info("Performed %s" % progress) etcd_resized = True + if constants.DRBD_DOCKER_DISTRIBUTION in drbd_fs_updated: + if (not dockerdistribution_resized and + (not standby_host or (standby_host and + constants.DRBD_DOCKER_DISTRIBUTION in self._drbd_fs_sync()))): + # patch_gib /var/lib/docker-distribution + progress = "resize2fs drbd8" + cmd = ["resize2fs", "/dev/drbd8"] + stdout, __ = cutils.execute(*cmd, attempts=retry_attempts, run_as_root=True) + LOG.info("Performed %s" % progress) + dockerdistribution_resized = True + if not standby_host: break @@ -7297,6 +7332,8 @@ class ConductorManager(service.PeriodicService): all_resized = False elif drbd == constants.DRBD_ETCD and not etcd_resized: all_resized = False + elif drbd == constants.DRBD_DOCKER_DISTRIBUTION and not dockerdistribution_resized: + all_resized = False if all_resized: break @@ -9009,6 +9046,7 @@ class ConductorManager(service.PeriodicService): if kubernetes_config: lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/docker-lv ' lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/etcd-lv' + lvdisplay_command = lvdisplay_command + '/dev/cgts-vg/dockerdistribution-lv ' if (system_dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER and tsc.system_type != constants.TIS_AIO_BUILD): diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/storage.py b/sysinv/sysinv/sysinv/sysinv/puppet/storage.py index 6e4dcae41d..68d228c42d 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/storage.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/storage.py @@ -76,6 +76,11 @@ class StoragePuppet(base.BasePuppet): 'platform::drbd::etcd::params::lv_size': controller_fs.size }) + elif controller_fs.name == constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION: + config.update({ + 'platform::drbd::dockerdistribution::params::lv_size': + controller_fs.size + }) return config