From ad14905521126ce9e99c3e3a2661297ba9518eba Mon Sep 17 00:00:00 2001 From: Angie Wang Date: Mon, 14 Jan 2019 11:10:14 -0500 Subject: [PATCH] Helm repository replication Currently, starlingx helm local repository is only configured on controller-0, that causes stx-openstack application apply failure when controller-1 is active. This commit is to replicate helm repository on both controllers. To ensure charts synchronized on controllers, charts are placed in drbd fs(/opt/cgcs/helm_charts). However, lighttpd is configured in chroot environment(/www). So the directory /www/pages/helm_charts is served by lighttpd as helm repository which means the request looks for charts under /www/pages/helm_charts. In order to access charts outside of the chroot /www, a bind mount is created to mount /opt/cgcs/helm_charts on /www/pages/helm_charts. This mount resource is managed by SM. Validated: on AIO-DX, Standard - system application-upload on controller-0 system host-swact controller-0 system application-apply on controller-1 - system application-upload on controller-0 system host-swact controller-0 reboot controller-0 system application-delete on controller-1 system application-upload with new tarball on controller-1 system application-apply on active controller - system application-upload on controller-0 reboot controller-0 reboot controller-1 system application-apply on active controller on AIO-SX, - system application-upload on controller-0 - system application-apply Story: 2004520 Task: 28343 Depends-On: https://review.openstack.org/#/c/630764/ Change-Id: I6c70e0f1866589888a826d8a890a21888c55633e Signed-off-by: Angie Wang --- .../src/modules/platform/manifests/helm.pp | 120 +++++++++++------- .../src/modules/platform/manifests/sm.pp | 24 ++++ 2 files changed, 98 insertions(+), 46 deletions(-) diff --git a/puppet-manifests/src/modules/platform/manifests/helm.pp b/puppet-manifests/src/modules/platform/manifests/helm.pp index 1e5d931369..632cc2521f 100644 --- a/puppet-manifests/src/modules/platform/manifests/helm.pp +++ b/puppet-manifests/src/modules/platform/manifests/helm.pp @@ -1,60 +1,89 @@ +class platform::helm::repository::params( + $source_helm_repo_dir = '/opt/cgcs/helm_charts', + $target_helm_repo_dir = '/www/pages/helm_charts', +) {} + class platform::helm -{ + inherits ::platform::helm::repository::params { + include ::platform::kubernetes::params - include ::platform::kubernetes::master if $::platform::kubernetes::params::enabled { - if str2bool($::is_initial_config_primary) { + file {$source_helm_repo_dir: + ensure => directory, + path => $source_helm_repo_dir, + owner => 'www', + require => User['www'] + } - Class['::platform::kubernetes::master'] + -> file {$target_helm_repo_dir: + ensure => directory, + path => $target_helm_repo_dir, + owner => 'www', + require => User['www'] + } - # TODO(jrichard): Upversion tiller image to v2.11.1 once released. - -> exec { 'load tiller docker image': - command => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1', - logoutput => true, + if (str2bool($::is_initial_config) and $::personality == 'controller') { + + if str2bool($::is_initial_config_primary) { + + Class['::platform::kubernetes::master'] + + # TODO(jrichard): Upversion tiller image to v2.11.1 once released. + -> exec { 'load tiller docker image': + command => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1', + logoutput => true, + } + + # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test + -> exec { 'load armada docker image': + command => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9', + logoutput => true, + } + + -> exec { 'create service account for tiller': + command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf create serviceaccount --namespace kube-system tiller', + logoutput => true, + } + + -> exec { 'create cluster role binding for tiller service account': + command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller', # lint:ignore:140chars + logoutput => true, + } + + # TODO(jrichard): Upversion tiller image to v2.11.1 once released. + -> exec { 'initialize helm': + environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ], + command => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller@sha256:022ce9d4a99603be1d30a4ca96a7fa57a45e6f2ef11172f4333c18aaae407f5b', # lint:ignore:140chars + logoutput => true, + user => 'wrsroot', + group => 'wrs', + require => User['wrsroot'] + } + + exec { "bind mount ${target_helm_repo_dir}": + command => "mount -o bind -t ext4 ${source_helm_repo_dir} ${target_helm_repo_dir}", + require => Exec['add local starlingx helm repo'] + } + } else { + exec { 'initialize helm': + environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ], + command => 'helm init --client-only', + logoutput => true, + user => 'wrsroot', + group => 'wrs', + require => User['wrsroot'] + } } - # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test - -> exec { 'load armada docker image': - command => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9', - logoutput => true, - } - - -> exec { 'create service account for tiller': - command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf create serviceaccount --namespace kube-system tiller', - logoutput => true, - } - - -> exec { 'create cluster role binding for tiller service account': - command => 'kubectl --kubeconfig=/etc/kubernetes/admin.conf create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller', # lint:ignore:140chars - logoutput => true, - } - - # TODO(jrichard): Upversion tiller image to v2.11.1 once released. - -> exec { 'initialize helm': - environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ], - command => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars - logoutput => true, - user => 'wrsroot', - group => 'wrs', - require => User['wrsroot'] - } - - -> file {'/www/pages/helm_charts': - ensure => directory, - path => '/www/pages/helm_charts', - owner => 'www', - require => User['www'] - } - - -> exec { 'restart lighttpd for helm': - require => File['/etc/lighttpd/lighttpd.conf'], + exec { 'restart lighttpd for helm': + require => [File['/etc/lighttpd/lighttpd.conf', $target_helm_repo_dir], Exec['initialize helm']], command => 'systemctl restart lighttpd.service', logoutput => true, } - -> exec { 'generate helm repo index': - command => 'helm repo index /www/pages/helm_charts', + -> exec { 'generate helm repo index on target': + command => "helm repo index ${target_helm_repo_dir}", logoutput => true, user => 'www', group => 'www', @@ -73,4 +102,3 @@ class platform::helm } } } - diff --git a/puppet-manifests/src/modules/platform/manifests/sm.pp b/puppet-manifests/src/modules/platform/manifests/sm.pp index 90242d6e72..60e5ec95c5 100644 --- a/puppet-manifests/src/modules/platform/manifests/sm.pp +++ b/puppet-manifests/src/modules/platform/manifests/sm.pp @@ -88,6 +88,10 @@ class platform::sm $dockerdistribution_fs_device = $::platform::drbd::dockerdistribution::params::device $dockerdistribution_fs_directory = $::platform::drbd::dockerdistribution::params::mountpoint + include ::platform::helm::repository::params + $helmrepo_fs_source_dir = $::platform::helm::repository::params::source_helm_repo_dir + $helmrepo_fs_target_dir = $::platform::helm::repository::params::target_helm_repo_dir + include ::platform::drbd::cephmon::params $cephmon_drbd_resource = $::platform::drbd::cephmon::params::resource_name $cephmon_fs_device = $::platform::drbd::cephmon::params::device @@ -498,6 +502,26 @@ class platform::sm } } + # Configure helm chart repository + if $kubernetes_enabled { + exec { 'Provision Helm Chart Repository FS in SM (service-group-member helmrepository-fs)': + command => 'sm-provision service-group-member controller-services helmrepository-fs', + } + -> exec { 'Provision Helm Chart Repository FS in SM (service helmrepository-fs)': + command => 'sm-provision service helmrepository-fs', + } + -> exec { 'Configure Helm Chart Repository FileSystem': + command => "sm-configure service_instance helmrepository-fs helmrepository-fs \"rmon_rsc_name=helm-charts-storage,device=${helmrepo_fs_source_dir},directory=${helmrepo_fs_target_dir},options=bind,noatime,nodiratime,fstype=ext4,check_level=20\"", + } + } else { + exec { 'Deprovision Helm Chart Repository FS in SM (service-group-member helmrepository-fs)': + command => 'sm-deprovision service-group-member controller-services helmrepository-fs', + } + -> exec { 'Deprovision Helm Chart Repository FS in SM (service helmrepository-fs)': + command => 'sm-deprovision service helmrepository-fs', + } + } + if $kubernetes_enabled { exec { 'Configure ETCD DRBD': command => "sm-configure service_instance drbd-etcd drbd-etcd:${hostunit} drbd_resource=${etcd_drbd_resource}",