Add rbd-provisioner to common StarlingX charts

Add the custom StarlingX rbd-provisioner chart to the common custom
chart area of the stx-config repo. Add the chart to the stx-openstack
application RPM.

This chart is based on the content of the ceph rbd storage code
(kubernetes-incubator/external-storage):
 - c463bd18 -> kubernetes-1.12.0-beta.1

Change-Id: Ib5fe40ed82bf5ffdd2eea4ebc1cd534e1b5ceacd
Story: 2004005
Task: 27802
Signed-off-by: Robert Church <robert.church@windriver.com>
This commit is contained in:
Ovidiu Poncea 2018-11-05 04:03:15 -05:00 committed by Robert Church
parent cb6b0fbde2
commit 7e1d73f95a
16 changed files with 567 additions and 1 deletions

View File

@ -1,2 +1,3 @@
SRC_DIR="stx-openstack-helm"
COPY_LIST_TO_TAR="$PKG_BASE/../../../helm-charts/rbd-provisioner"
TIS_PATCH_VER=1

View File

@ -36,6 +36,7 @@ helm repo add local http://localhost:8879/charts
# Make the charts. These produce a tgz file
make nova-api-proxy
make rbd-provisioner
# remove helm-toolkit. This will be packaged with openstack-helm-infra
rm ./helm-toolkit-%{toolkit_version}.tgz

View File

@ -1 +0,0 @@
This is a placeholder, remove when there are actual helm charts.

View File

@ -0,0 +1,10 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
apiVersion: v1
description: rbd provisioner chart
name: rbd-provisioner
version: 0.1.0

View File

@ -0,0 +1,22 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
classdefaults:
adminId: admin
adminSecretName: ceph-admin
monitors:
- 192.168.204.4:6789
- 192.168.204.3:6789
- 192.168.204.22:6789
classes:
- name: rbd
pool: kube-rbd
userId: ceph-pool-kube-rbd
userSecretName: ceph-pool-kube-rbd
- name: gold-rbd
pool: kube-rbd-gold
userId: ceph-pool-gold-kube-rbd-gold
userSecretName: ceph-pool-gold-kube-rbd-gold

View File

@ -0,0 +1,17 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
classes:
- name: slow-rbd
monitors:
- 192.168.204.3:6789
- 192.168.204.150:6789
- 192.168.204.4:6789
adminId: admin
adminSecretName: ceph-secret
pool: kube
userId: kube
userSecretName: ceph-secret-kube

View File

@ -0,0 +1,34 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.rbac }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ .Values.rbac.clusterRole }}
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns"]
verbs: ["list", "get"]
{{- end}}

View File

@ -0,0 +1,22 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.rbac }}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ .Values.rbac.clusterRoleBinding }}
subjects:
- kind: ServiceAccount
name: {{ .Values.rbac.serviceAccount }}
namespace: {{ .Values.global.namespace }}
roleRef:
kind: ClusterRole
name: {{ .Values.rbac.clusterRole }}
apiGroup: rbac.authorization.k8s.io
{{- end}}

View File

@ -0,0 +1,48 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if eq .Values.global.deployAs "DaemonSet" }}
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: {{ .Values.global.name }}
namespace: {{ .Values.global.namespace }}
labels:
app: {{ .Values.global.name }}
spec:
selector:
matchLabels:
app: {{ .Values.global.name }}
template:
metadata:
labels:
app: {{ .Values.global.name }}
spec:
{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
serviceAccountName: {{.Values.rbac.serviceAccount}}
{{- end}}
{{- if .Values.global.tolerations }}
tolerations:
{{ .Values.global.tolerations | toYaml | trim | indent 8 }}
{{- end }}
{{- if .Values.global.nodeSelector }}
nodeSelector:
{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
{{- end }}
containers:
- image: {{ .Values.global.image | quote }}
name: {{ .Values.global.name }}
{{- if .Values.global.resources }}
resources:
{{ .Values.global.resources | toYaml | trim | indent 12 }}
{{- end }}
env:
- name: PROVISIONER_NAME
value: ceph.com/rbd
{{- end}}

View File

@ -0,0 +1,45 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if eq .Values.global.deployAs "Deployment" }}
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: {{ .Values.global.name }}
namespace: {{ .Values.global.namespace }}
spec:
replicas: {{ .Values.global.replicas }}
strategy:
type: Recreate
template:
metadata:
labels:
app: {{ .Values.global.name }}
spec:
containers:
- name: {{ .Values.global.name }}
image: {{ .Values.global.image | quote }}
env:
- name: PROVISIONER_NAME
value: ceph.com/rbd
{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
serviceAccount: {{ .Values.rbac.serviceAccount }}
{{- end }}
{{- if .Values.global.nodeSelector }}
nodeSelector:
{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
{{- end }}
{{- if .Values.global.tolerations }}
tolerations:
{{ .Values.global.tolerations | toYaml | trim | indent 8 }}
{{- end}}
{{- if .Values.global.resources }}
resources:
{{ .Values.global.resources | toYaml | trim | indent 8 }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,138 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.doPreInstallVerification }}
{{ $root := . }}
{{ $defaults := .Values.classdefaults}}
{{ $mount := "/tmp/mount" }}
{{- range $classConfig := .Values.classes }}
kind: ConfigMap
apiVersion: v1
metadata:
creationTimestamp: 2016-02-18T19:14:38Z
name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
namespace: {{ $root.Values.global.namespace }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-weight": "-6"
"helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
data:
ceph.conf: |
{{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ range $index, $element := $monitors}}
[mon.{{- $index }}]
mon_addr = {{ $element }}
{{- end }}
check_ceph.sh: |-
#!/bin/bash
# Copy from read only mount to Ceph config folder
cp {{ $mount -}}/ceph.conf /etc/ceph/
# Set up admin key in Ceph format
CEPH_ADMIN_KEY="/etc/ceph/ceph.client.admin.keyring"
if [ ! -z "$CEPH_ADMIN_SECRET" ]; then
cat <<EOF > $CEPH_ADMIN_KEY
[client.admin]
key = $CEPH_ADMIN_SECRET
EOF
else
touch $CEPH_ADMIN_KEY
fi
# Set up pool key in Ceph format
CEPH_USER_KEY=/etc/ceph/ceph.client.{{- $classConfig.userId -}}.keyring
echo $CEPH_USER_SECRET > $CEPH_USER_KEY
# Check if ceph is accessible
echo "===================================="
ceph -s
ret=$?
if [ $ret -ne 0 ]; then
msg="Error: Ceph cluster is not accessible, check Pod logs for details."
echo "$msg"
echo "$msg" > /dev/termination-log
exit $ret
fi
# Check if pool exists
echo "===================================="
ceph osd lspools | grep {{ $classConfig.pool }}
ret=$?
if [ $ret -ne 0 ]; then
msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible, check Pod logs for details."
echo "$msg"
echo "$msg" > /dev/termination-log
exit $ret
fi
# Check if pool is accessible using provided credentials
echo "===================================="
rbd -p {{ $classConfig.pool }} --user {{ $classConfig.userId }} ls -K $CEPH_USER_KEY
ret=$?
if [ $ret -ne 0 ]; then
msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible using \
credentials for user {{ $classConfig.userId }}, check Pod logs for details."
echo "$msg"
echo "$msg" > /dev/termination-log
exit $ret
fi
---
apiVersion: batch/v1
kind: Job
metadata:
name: check-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
namespace: {{ $root.Values.global.namespace }}
labels:
heritage: {{$root.Release.Service | quote }}
release: {{$root.Release.Name | quote }}
chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
spec:
# Note due to https://github.com/kubernetes/kubernetes/issues/62382
# backoffLimit doesn't work in 1.10.x
backoffLimit: 1 # Limit the number of job restart in case of failure
activeDeadlineSeconds: 60
template:
metadata:
name: "{{$root.Release.Name}}"
namespace: {{ $root.Values.global.namespace }}
labels:
heritage: {{$root.Release.Service | quote }}
release: {{$root.Release.Name | quote }}
chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
spec:
restartPolicy: Never
volumes:
- name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
configMap:
name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
containers:
- name: pre-install-job-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
image: {{ $root.Values.global.image | quote }}
command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
volumeMounts:
- name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
mountPath: {{ $mount }}
env:
- name: CEPH_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
key: key
- name: CEPH_USER_SECRET
valueFrom:
secretKeyRef:
name: {{ or $classConfig.userSecretName }}
key: key
---
{{- end }}
{{- end }}

View File

@ -0,0 +1,19 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.rbac }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ .Values.rbac.role }}
namespace: {{ .Values.global.namespace }}
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
{{- end}}

View File

@ -0,0 +1,23 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.rbac }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ .Values.rbac.roleBinding }}
namespace: {{ .Values.global.namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ .Values.rbac.role }}
subjects:
- kind: ServiceAccount
name: {{ .Values.rbac.serviceAccount }}
namespace: {{ .Values.global.namespace }}
{{- end}}

View File

@ -0,0 +1,15 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.rbac }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.rbac.serviceAccount }}
namespace: {{ .Values.global.namespace }}
{{- end }}

View File

@ -0,0 +1,30 @@
{{/*
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
*/}}
{{- if .Values.global.provisionStorageClass }}
{{ $namespace := .Values.global.namespace }}
{{ $defaults := .Values.classdefaults}}
{{- range $classConfig := .Values.classes }}
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: {{ $classConfig.name }}
provisioner: ceph.com/rbd
parameters:
monitors: {{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ join "," $monitors}}
adminId: {{ or $classConfig.adminId $defaults.adminId}}
adminSecretName: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
adminSecretNamespace: {{ $namespace }}
pool: {{ or $classConfig.pool $defaults.pool }}
userId: {{ or $classConfig.userId $defaults.userId }}
userSecretName: {{ $classConfig.userSecretName }}
imageFormat: {{ or $classConfig.imageFormat $defaults.imageFormat | quote }}
imageFeatures: {{ or $classConfig.imageFeatures $defaults.imageFeatures}}
---
{{- end }}
{{- end }}

View File

@ -0,0 +1,142 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Global options.
# Defaults should be fine in most cases.
global:
#
# Defines the name of a Provisioner.
#
name: "rbd-provisioner"
#
# Defines the namespace where provisioner runs.
#
namespace: kube-system
#
# Run pre-install verifications or skip them.
# Skipping them is not recommended
#
doPreInstallVerification: True
#
# Defines Provisioner's image name including container registry.
#
image: quay.io/external_storage/rbd-provisioner:latest
#
# Defines whether to reuse an already defined RBAC policy.
# Make sure that the serviceAccount defined in the RBAC section matches the one
# in the policy you reuse.
#
reuseRbac: false
#
# Defines whether to generate service account and role bindings.
#
rbac: true
#
# Provision storage class. If false you have to provision storage classes by hand.
#
provisionStorageClass: true
#
# Choose if rbd-provisioner pod should be deployed as deplyment or DaemonSet
# Values: none, Deployment, DaemonSet
#
deployAs: Deployment
#
# If configured, tolerations will add a toleration field to the Pod.
#
# Node tolerations for rbd-volume-provisioner scheduling to nodes with taints.
# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
# Example:
# [
# {
# "key": "node-role.kubernetes.io/master",
# "operator": "Exists"
# }
# ]
#
tolerations: []
# If configured, resources will set the requests/limits field to the Pod.
# Ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
# Example:
# {
# "limits": {
# "memory": "200Mi"
# },
# "requests": {
# "cpu": "100m",
# "memory": "200Mi"
# }
# }
resources: {}
#
# Number of replicas to start when configured as deployment
#
replicas: 1
#
# Node Selector
#
nodeSelector: { node-role.kubernetes.io/master: "" }
#
# RBAC options.
# Defaults should be fine in most cases.
rbac:
#
# Cluster Role name
#
clusterRole: rbd-provisioner
#
# Cluster Role Binding name
#
clusterRoleBinding: rbd-provisioner
#
# Role name
#
role: rbd-provisioner
#
# Role Binding name
#
roleBinding: rbd-provisioner
#
# Defines a name of the service account which Provisioner will use to communicate with API server.
#
serviceAccount: rbd-provisioner
#
# Configure storage classes.
# Defaults for storage classes. Update this if you have a single Ceph storage cluster.
# No need to add them to each class.
#
classdefaults:
# Define ip addresses of Ceph Monitors
monitors:
- 192.168.204.3:6789
- 192.168.204.150:6789
- 192.168.204.4:6789
# Ceph admin account
adminId: admin
# K8 secret name for the admin context
adminSecretName: ceph-secret
# Ceph RBD image format version
imageFormat: 2
# Ceph RBD image features.
imageFeatures: layering
#
# Configure storage classes.
# This section should be tailored to your setup. It allows you to define multiple storage
# classes for the same cluster (e.g. if you have tiers of drives with different speeds).
# If you have multiple Ceph clusters take attributes from classdefaults and add them here.
classes:
- name: fast-rbd # Name of storage class.
# Ceph pool name
pool: kube
# Ceph user name to access this pool
userId: kube
# K8 secret name with key for accessing the Ceph pool
userSecretName: ceph-secret-kube