starlingx
/
config
Code Issues Proposed changes

Add Ceph pools management chart 

- new helm chart to set replication and min replication for
each Ceph pool:
  -> new helm chart name: ceph-pools-audit
  -> the ceph-pools-audit chart creates a CronJob that runs
     every 5 minutes; the CronJob checks the replication for
     each existing pool and sets it right if needed, to reflect
     the attributes of the Ceph backends
  -> the CronJob is needed for: charts that may not manage pool
     configuration, pools created dynamically by services that
     may not have the current pool configuration uploaded
     (ex: swift), updating replication without reinstalling the
     charts that created the pools
  -> the ceph-pools-audit chart is installed after the
     rbd-provisioner in the application-apply
- new overrides for the ceph-pools-audit chart that provide
the replication values from the attributes of the present
Ceph backends
- enable rados-gw by default when a Ceph backend is enabled

Change-Id: I1565268bac3ddc77e8368d2d6ab8600b3e4ed893
Story: 2004520
Task: 29034
Signed-off-by: Irina Mihai <irina.mihai@windriver.com>
This commit is contained in:
Irina Mihai 2019-01-22 19:32:37 +00:00 committed by Al Bailey
parent 1e52662765
commit 754f49a357
17 changed files with 415 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
kubernetes/applications/stx-openstack/stx-openstack-helm/centos/build_srpm.data
View File

@ -1,3 +1,3 @@
SRC_DIR="stx-openstack-helm"
COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner $PKG_BASE/../../../helm-charts/garbd"
TIS_PATCH_VER=3
COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner $PKG_BASE/../../../helm-charts/garbd $PKG_BASE/../../../helm-charts/ceph-pools-audit"
TIS_PATCH_VER=4

1
kubernetes/applications/stx-openstack/stx-openstack-helm/centos/stx-openstack-helm.spec
View File

@ -58,6 +58,7 @@ helm repo add local http://localhost:8879/charts
make nova-api-proxy
make rbd-provisioner
make garbd
make ceph-pools-audit
# terminate helm server (the last backgrounded task)
kill %1

40
kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest-no-tests.yaml
View File

@ -127,6 +127,35 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-ceph-pools-audit
data:
chart_name: ceph-pools-audit
release: openstack-ceph-pools-audit
namespace: openstack
wait:
timeout: 1800
labels:
app: ceph-pools-audit
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
app: ceph-pools-audit
source:
type: tar
location: http://172.17.0.1/helm_charts/ceph-pools-audit-0.1.0.tgz
subpath: ceph-pools-audit
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-mariadb
@ -2340,6 +2369,16 @@ data:
- openstack-rbd-provisioner
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: ceph-pools-audit
data:
description: "Ceph pools audit"
sequenced: false
chart_group:
- openstack-ceph-pools-audit
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: openstack-mariadb
@ -2456,6 +2495,7 @@ data:
- kube-system-ingress
- openstack-ingress
- provisioner
- ceph-pools-audit
- openstack-mariadb
- openstack-memcached
- openstack-rabbitmq

40
kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml
View File

@ -127,6 +127,35 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-ceph-pools-audit
data:
chart_name: ceph-pools-audit
release: openstack-ceph-pools-audit
namespace: openstack
wait:
timeout: 1800
labels:
app: ceph-pools-audit
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- type: job
labels:
app: ceph-pools-audit
source:
type: tar
location: http://172.17.0.1/helm_charts/ceph-pools-audit-0.1.0.tgz
subpath: ceph-pools-audit
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: openstack-mariadb
@ -2340,6 +2369,16 @@ data:
- openstack-rbd-provisioner
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: ceph-pools-audit
data:
description: "Ceph pools audit"
sequenced: false
chart_group:
- openstack-ceph-pools-audit
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: openstack-mariadb
@ -2456,6 +2495,7 @@ data:
- kube-system-ingress
- openstack-ingress
- provisioner
- ceph-pools-audit
- openstack-mariadb
- openstack-memcached
- openstack-rabbitmq

10
kubernetes/helm-charts/ceph-pools-audit/Chart.yaml Normal file
View File

@ -0,0 +1,10 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
apiVersion: v1
appVersion: "1.0"
description: Ceph RBD pool replication monitor chart
name: ceph-pools-audit
version: 0.1.0

9
kubernetes/helm-charts/ceph-pools-audit/requirements.yaml Normal file
View File

@ -0,0 +1,9 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
version: 0.1.0

64
kubernetes/helm-charts/ceph-pools-audit/templates/bin/_ceph-pools-audit.sh.tpl Executable file
View File

@ -0,0 +1,64 @@
#!/bin/bash
{{/*
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
ceph -s
ret=$?
if [ $ret -ne 0 ]; then
msg="Error: Ceph cluster is not accessible, check Pod logs for details."
echo "$msg"
exit $ret
fi
touch /etc/ceph/ceph.client.admin.keyring
echo "RBD_POOL_CRUSH_RULESET: $RBD_POOL_CRUSH_RULESET"
if [ -z $RBD_POOL_CRUSH_RULESET ]; then
msg="No Ceph crush ruleset specified"
echo "$msg"
exit 1
fi
ruleset=$(ceph osd crush rule dump $RBD_POOL_CRUSH_RULESET | grep \"ruleset\" | awk '{print $2}' | grep -Eo '[0-9]+')
ret=$?
if [ $ret -ne 0 ]; then
msg="Ceph crush ruleset $RBD_POOL_CRUSH_RULESET not found, exit"
echo "$msg"
exit $ret
fi
echo "ruleset: $ruleset"
set -ex
POOLS=( $(ceph osd pool ls) )
for pool_name in "${POOLS[@]}"
do
echo "Check for pool name: $pool_name"
pool_crush_ruleset=$(ceph osd pool get $pool_name crush_ruleset | awk '{print $2}')
echo "pool_crush_ruleset: $pool_crush_ruleset"
if [ "$pool_crush_ruleset" != "$ruleset" ]; then
continue
fi
pool_size=$(ceph osd pool get $pool_name size | awk '{print $2}')
pool_min_size=$(ceph osd pool get $pool_name min_size | awk '{print $2}')
echo "===> pool_size: $pool_size pool_min_size: $pool_min_size"
if [ $pool_size != $RBD_POOL_REPLICATION ]; then
echo "set replication for pool $pool_name at $RBD_POOL_REPLICATION"
ceph osd pool set $pool_name size $RBD_POOL_REPLICATION
fi
if [ $pool_min_size != $RBD_POOL_MIN_REPLICATION ]; then
echo "set min replication for pool $pool_name at $RBD_POOL_MIN_REPLICATION"
ceph osd pool set $pool_name min_size $RBD_POOL_MIN_REPLICATION
fi
done

19
kubernetes/helm-charts/ceph-pools-audit/templates/confimap-bin.yaml Normal file
View File

@ -0,0 +1,19 @@
{{/*
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.manifests.configmap_bin }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ceph-pools-bin
data:
ceph-pools-audit.sh: |
{{ tuple "bin/_ceph-pools-audit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

82
kubernetes/helm-charts/ceph-pools-audit/templates/job-ceph-pools-audit.yaml Normal file
View File

@ -0,0 +1,82 @@
{{/*
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.manifests.job_ceph_pools_audit }}
{{- $envAll := . }}
---
#
# The CronJob makes sure all the Ceph pools have the right replication,
# as present in the attributes of the Ceph backends.
# This is needed for:
# - charts that don't manage pool configuration
# - pools created dynamically by services that may not have the current
# pool configuration uploaded (ex: swift)
# - when replication is changed and we don't want to reinstall all the
# charts that created Ceph pools
#
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: ceph-pools-audit
spec:
schedule: {{ .Values.jobs.job_ceph_pools_audit.cron | quote }}
successfulJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.success }}
failedJobsHistoryLimit: {{ .Values.jobs.job_ceph_pools_audit.history.failed }}
concurrencyPolicy: Forbid
jobTemplate:
metadata:
name: "{{$envAll.Release.Name}}"
namespace: {{ $envAll.Release.namespace }}
labels:
app: ceph-pools-audit
spec:
template:
metadata:
labels:
app: ceph-pools-audit
spec:
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
volumes:
- name: ceph-pools-bin
configMap:
name: ceph-pools-bin
defaultMode: 0555
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: {{ $envAll.Values.ceph_client.configmap }}
defaultMode: 0444
containers:
{{- range $tierConfig := $envAll.Values.conf.ceph.storage_tiers }}
- name: ceph-pools-audit-{{- $tierConfig.name }}
image: {{ $envAll.Values.images.tags.ceph_config_helper | quote }}
env:
- name: RBD_POOL_REPLICATION
value: {{ $tierConfig.replication | quote }}
- name: RBD_POOL_MIN_REPLICATION
value: {{ $tierConfig.min_replication | quote }}
- name: RBD_POOL_CRUSH_RULESET
value: {{ $tierConfig.crush_ruleset | quote }}
command:
- /tmp/ceph-pools-audit.sh
volumeMounts:
- name: ceph-pools-bin
mountPath: /tmp/ceph-pools-audit.sh
subPath: ceph-pools-audit.sh
readOnly: true
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
{{- end }}
{{- end }}

49
kubernetes/helm-charts/ceph-pools-audit/values.yaml Normal file
View File

@ -0,0 +1,49 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
replicaCount: 1
labels:
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
name: ceph-pools-audit
ceph_client:
configmap: ceph-etc
conf:
ceph:
storage_tiers:
- name: ceph-store
replication: 2
min_replication: 1
crush_ruleset: storage_tier_ruleset
monitors: []
images:
tags:
ceph_config_helper: docker.io/port/ceph-config-helper:v1.10.3
pullPolicy: "IfNotPresent"
jobs:
job_ceph_pools_audit:
cron: "*/5 * * * *"
history:
success: 3
failed: 1
resources: {}
nodeSelector: { node-role.kubernetes.io/master: "" }
tolerations: []
affinity: {}
manifests:
job_ceph_pools_audit: true
configmap_bin: true

2
puppet-manifests/src/modules/openstack/manifests/keystone.pp
View File

@ -386,7 +386,7 @@ class openstack::keystone::endpoint::runtime {
}
include ::platform::ceph::params
if $::platform::ceph::params::rgw_enabled {
if $::platform::ceph::params::service_enabled {
include ::platform::ceph::rgw::keystone::auth
}

7
puppet-manifests/src/modules/platform/manifests/ceph.pp
View File

@ -21,7 +21,6 @@ class platform::ceph::params(
$mon_2_host = undef,
$mon_2_ip = undef,
$mon_2_addr = undef,
$rgw_enabled = false,
$rgw_client_name = 'radosgw.gateway',
$rgw_user_name = 'root',
$rgw_frontend_type = 'civetweb',
@ -374,7 +373,7 @@ class platform::ceph::storage(
class platform::ceph::firewall
inherits ::platform::ceph::params {
if $rgw_enabled {
if $service_enabled {
platform::firewall::rule { 'ceph-radosgw':
service_name => 'ceph-radosgw',
ports => $rgw_port,
@ -386,7 +385,7 @@ class platform::ceph::firewall
class platform::ceph::haproxy
inherits ::platform::ceph::params {
if $rgw_enabled {
if $service_enabled {
platform::haproxy::proxy { 'ceph-radosgw-restapi':
server_name => 's-ceph-radosgw',
public_port => $rgw_port,
@ -398,7 +397,7 @@ class platform::ceph::haproxy
class platform::ceph::rgw
inherits ::platform::ceph::params {
if $rgw_enabled {
if $service_enabled {
include ::platform::params
include ::openstack::keystone::params

3
puppet-manifests/src/modules/platform/manifests/sm.pp
View File

@ -255,7 +255,6 @@ class platform::sm
# Ceph-Rados-Gateway
include ::platform::ceph::params
$ceph_configured = $::platform::ceph::params::service_enabled
$rgw_configured = $::platform::ceph::params::rgw_enabled
# Gnocchi
include ::openstack::gnocchi::params
@ -1568,7 +1567,7 @@ class platform::sm
}
# Ceph-Rados-Gateway
if $rgw_configured {
if $ceph_configured {
exec {'Provision Ceph-Rados-Gateway (service-group-member ceph-radosgw)':
command => 'sm-provision service-group-member storage-monitoring-services ceph-radosgw'
}

1
sysinv/sysinv/sysinv/setup.cfg
View File

@ -97,6 +97,7 @@ systemconfig.helm_plugins =
panko = sysinv.helm.panko:PankoHelm
rabbitmq = sysinv.helm.rabbitmq:RabbitmqHelm
rbd-provisioner = sysinv.helm.rbd_provisioner:RbdProvisionerHelm
ceph-pools-audit = sysinv.helm.ceph_pools_audit:CephPoolsAuditHelm
sysinv.agent.lldp.drivers =
lldpd = sysinv.agent.lldp.drivers.lldpd.driver:SysinvLldpdAgentDriver

3
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -1414,6 +1414,7 @@ HELM_CHART_OPENVSWITCH = 'openvswitch'
HELM_CHART_PANKO = 'panko'
HELM_CHART_RABBITMQ = 'rabbitmq'
HELM_CHART_RBD_PROVISIONER = 'rbd-provisioner'
HELM_CHART_CEPH_POOLS_AUDIT = 'ceph-pools-audit'
SUPPORTED_HELM_CHARTS = [
HELM_CHART_AODH,
@ -1439,6 +1440,7 @@ SUPPORTED_HELM_CHARTS = [
HELM_CHART_PANKO,
HELM_CHART_RABBITMQ,
HELM_CHART_RBD_PROVISIONER,
HELM_CHART_CEPH_POOLS_AUDIT,
]
# Helm: Supported application (aka chart bundles)
@ -1452,6 +1454,7 @@ SUPPORTED_HELM_APP_CHARTS = {
HELM_APP_OPENSTACK: [
HELM_CHART_INGRESS,
HELM_CHART_RBD_PROVISIONER,
HELM_CHART_CEPH_POOLS_AUDIT,
HELM_CHART_MARIADB,
HELM_CHART_GARBD,
HELM_CHART_RABBITMQ,

90
sysinv/sysinv/sysinv/sysinv/helm/ceph_pools_audit.py Normal file
View File

@ -0,0 +1,90 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from sysinv.common import constants
from sysinv.common import exception
from sysinv.openstack.common import log as logging
from sysinv.helm import base
from sysinv.helm import common
LOG = logging.getLogger(__name__)
class CephPoolsAuditHelm(base.BaseHelm):
"""Class to encapsulate helm operations for the ceph-pools-audit chart"""
CHART = constants.HELM_CHART_CEPH_POOLS_AUDIT
SUPPORTED_NAMESPACES = [
common.HELM_NS_OPENSTACK
]
SERVICE_NAME = 'ceph-pools'
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
@property
def docker_repo_tag(self):
return common.DOCKER_SRCS[self.docker_repo_source][common.IMG_TAG_KEY]
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES
def get_overrides(self, namespace=None):
backends = self.dbapi.storage_backend_get_list()
ceph_bks = [bk for bk in backends if bk.backend == constants.SB_TYPE_CEPH]
if not ceph_bks:
return {} # ceph is not configured
monitors = self._get_formatted_ceph_monitor_ips()
# Get tier info.
tiers = self.dbapi.storage_tier_get_list()
tiers_cfg = []
for bk in ceph_bks:
# Get the tier associated to the Ceph backend.
tier = next((t for t in tiers if t.forbackendid == bk.id), None)
if not tier:
raise Exception("No tier present for backend %s" % bk.name)
# Get the ruleset name.
rule_name = "{0}{1}{2}".format(
tier.name,
constants.CEPH_CRUSH_TIER_SUFFIX,
"-ruleset").replace('-', '_')
# Tier config needed for the overrides.
tier_cfg = {
"name": bk.name.encode('utf8', 'strict'),
"replication": int(bk.capabilities.get("replication")),
"min_replication": int(bk.capabilities.get("min_replication")),
"crush_ruleset": rule_name.encode('utf8', 'strict'),
}
tiers_cfg.append(tier_cfg)
overrides = {
common.HELM_NS_OPENSTACK: {
'conf': {
'ceph': {
'monitors': monitors,
'storage_tiers': tiers_cfg
}
}
}
}
if namespace in self.SUPPORTED_NAMESPACES:
return overrides[namespace]
elif namespace:
raise exception.InvalidHelmNamespace(chart=self.CHART,
namespace=namespace)
else:
return overrides

2
sysinv/sysinv/sysinv/sysinv/puppet/ceph.py
View File

@ -103,8 +103,6 @@ class CephPuppet(openstack.OpenstackBasePuppet):
'platform::ceph::params::mon_1_addr': mon_1_addr,
'platform::ceph::params::mon_2_addr': mon_2_addr,
'platform::ceph::params::rgw_enabled':
ceph_backend.object_gateway,
'platform::ceph::params::rgw_admin_user':
ksuser,
'platform::ceph::params::rgw_admin_domain':