Merge "Decouple Ceph pools creation from sysinv"
This commit is contained in:
commit
a49a2c3710
|
@ -31,4 +31,7 @@ rules:
|
|||
resources: ["services"]
|
||||
resourceNames: ["kube-dns"]
|
||||
verbs: ["list", "get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "create", "list", "update"]
|
||||
{{- end}}
|
||||
|
|
|
@ -21,15 +21,15 @@ spec:
|
|||
labels:
|
||||
app: {{ .Values.global.name }}
|
||||
spec:
|
||||
{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
|
||||
serviceAccount: {{ .Values.rbac.serviceAccount }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ .Values.global.name }}
|
||||
image: {{ .Values.global.image | quote }}
|
||||
image: {{ .Values.images.tags.rbd_provisioner | quote }}
|
||||
env:
|
||||
- name: PROVISIONER_NAME
|
||||
value: ceph.com/rbd
|
||||
{{- if (.Values.global.rbac) or (.Values.global.reuseRbac)}}
|
||||
serviceAccount: {{ .Values.rbac.serviceAccount }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{ .Values.global.nodeSelector | toYaml | trim | indent 8 }}
|
||||
|
|
|
@ -6,47 +6,47 @@
|
|||
#
|
||||
*/}}
|
||||
|
||||
{{- if .Values.global.doPreInstallVerification }}
|
||||
{{- if .Values.global.job_storage_init }}
|
||||
{{ $root := . }}
|
||||
{{ $defaults := .Values.classdefaults}}
|
||||
{{ $mount := "/tmp/mount" }}
|
||||
{{- range $classConfig := .Values.classes }}
|
||||
kind: ConfigMap
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
creationTimestamp: 2016-02-18T19:14:38Z
|
||||
name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
name: config-{{- $root.Values.global.name }}
|
||||
namespace: {{ $root.Values.global.namespace }}
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install
|
||||
"helm.sh/hook-weight": "-6"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed
|
||||
data:
|
||||
ceph.conf: |
|
||||
{{ $monitors := or $classConfig.monitors $defaults.monitors }}{{ range $index, $element := $monitors}}
|
||||
{{ $monitors := $defaults.monitors }}{{ range $index, $element := $monitors}}
|
||||
[mon.{{- $index }}]
|
||||
mon_addr = {{ $element }}
|
||||
{{- end }}
|
||||
|
||||
check_ceph.sh: |-
|
||||
#!/bin/bash
|
||||
|
||||
# Copy from read only mount to Ceph config folder
|
||||
cp {{ $mount -}}/ceph.conf /etc/ceph/
|
||||
|
||||
# Set up admin key in Ceph format
|
||||
CEPH_ADMIN_KEY="/etc/ceph/ceph.client.admin.keyring"
|
||||
if [ ! -z "$CEPH_ADMIN_SECRET" ]; then
|
||||
cat <<EOF > $CEPH_ADMIN_KEY
|
||||
[client.admin]
|
||||
key = $CEPH_ADMIN_SECRET
|
||||
EOF
|
||||
else
|
||||
touch $CEPH_ADMIN_KEY
|
||||
if [ ! -z $CEPH_ADMIN_SECRET ]; then
|
||||
kubectl get secret -n kube-system | grep $CEPH_ADMIN_SECRET
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Create $CEPH_ADMIN_SECRET secret"
|
||||
echo "$msg"
|
||||
kubectl create secret generic $CEPH_ADMIN_SECRET --type="kubernetes.io/rbd" --from-literal=key= --namespace=$NAMESPACE
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Error creating secret $CEPH_ADMIN_SECRET, exit"
|
||||
echo "$msg"
|
||||
exit $ret
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Set up pool key in Ceph format
|
||||
CEPH_USER_KEY=/etc/ceph/ceph.client.{{- $classConfig.userId -}}.keyring
|
||||
echo $CEPH_USER_SECRET > $CEPH_USER_KEY
|
||||
touch /etc/ceph/ceph.client.admin.keyring
|
||||
|
||||
# Check if ceph is accessible
|
||||
echo "===================================="
|
||||
|
@ -55,52 +55,67 @@ data:
|
|||
if [ $ret -ne 0 ]; then
|
||||
msg="Error: Ceph cluster is not accessible, check Pod logs for details."
|
||||
echo "$msg"
|
||||
echo "$msg" > /dev/termination-log
|
||||
exit $ret
|
||||
fi
|
||||
|
||||
# Check if pool exists
|
||||
echo "===================================="
|
||||
ceph osd lspools | grep {{ $classConfig.pool }}
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible, check Pod logs for details."
|
||||
set -ex
|
||||
|
||||
# Get the ruleset from the rule name.
|
||||
ruleset=$(ceph osd crush rule dump $POOL_CRUSH_RULE_NAME | grep "\"ruleset\":" | grep -Eo '[0-9]*')
|
||||
# Make sure the pool exists.
|
||||
ceph osd pool stats $POOL_NAME || ceph osd pool create $POOL_NAME $POOL_CHUNK_SIZE
|
||||
# Set pool configuration.
|
||||
ceph osd pool set $POOL_NAME size $POOL_REPLICATION
|
||||
ceph osd pool set $POOL_NAME crush_rule $ruleset
|
||||
|
||||
if [[ -z $USER_ID && -z $CEPH_USER_SECRET ]]; then
|
||||
msg="No need to create secrets for pool $POOL_NAME"
|
||||
echo "$msg"
|
||||
echo "$msg" > /dev/termination-log
|
||||
exit $ret
|
||||
exit 0
|
||||
fi
|
||||
|
||||
KEYRING=$(ceph auth get-or-create client.$USER_ID mon "allow r" osd "allow rwx pool=${POOL_NAME}" | sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p')
|
||||
# Set up pool key in Ceph format
|
||||
CEPH_USER_KEYRING=/etc/ceph/ceph.client.$USER_ID.keyring
|
||||
echo $KEYRING > $CEPH_USER_KEYRING
|
||||
IFS=',' read -a POOL_SECRET_NAMESPACES_ARR <<< "${POOL_SECRET_NAMESPACES}"
|
||||
|
||||
for pool_secret_namespace in "${POOL_SECRET_NAMESPACES_ARR[@]}"
|
||||
do
|
||||
kubectl create secret generic $CEPH_USER_SECRET --type="kubernetes.io/rbd" --from-literal=key=$KEYRING --namespace=$pool_secret_namespace
|
||||
done
|
||||
|
||||
set +ex
|
||||
|
||||
# Check if pool is accessible using provided credentials
|
||||
echo "===================================="
|
||||
rbd -p {{ $classConfig.pool }} --user {{ $classConfig.userId }} ls -K $CEPH_USER_KEY
|
||||
echo "====================================="
|
||||
rbd -p $POOL_NAME --user $USER_ID ls -K $CEPH_USER_KEYRING
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
msg="Error: Ceph pool {{ $classConfig.pool }} is not accessible using \
|
||||
credentials for user {{ $classConfig.userId }}, check Pod logs for details."
|
||||
msg="Error: Ceph pool $POOL_NAME is not accessible using \
|
||||
credentials for user $USER_ID, check Pod logs for details."
|
||||
echo "$msg"
|
||||
echo "$msg" > /dev/termination-log
|
||||
exit $ret
|
||||
else
|
||||
msg="Pool $POOL_NAME accessible"
|
||||
echo "$msg"
|
||||
fi
|
||||
|
||||
ceph -s
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: check-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
name: rbd-provisioner-storage-init
|
||||
namespace: {{ $root.Values.global.namespace }}
|
||||
labels:
|
||||
heritage: {{$root.Release.Service | quote }}
|
||||
release: {{$root.Release.Name | quote }}
|
||||
chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
|
||||
annotations:
|
||||
"helm.sh/hook": pre-install
|
||||
"helm.sh/hook-weight": "-5"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
|
||||
spec:
|
||||
# Note due to https://github.com/kubernetes/kubernetes/issues/62382
|
||||
# backoffLimit doesn't work in 1.10.x
|
||||
backoffLimit: 1 # Limit the number of job restart in case of failure
|
||||
activeDeadlineSeconds: 60
|
||||
backoffLimit: 3 # Limit the number of job restart in case of failure
|
||||
activeDeadlineSeconds: 180
|
||||
template:
|
||||
metadata:
|
||||
name: "{{$root.Release.Name}}"
|
||||
|
@ -110,29 +125,57 @@ spec:
|
|||
release: {{$root.Release.Name | quote }}
|
||||
chart: "{{$root.Chart.Name}}-{{$root.Chart.Version}}"
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: {{ $root.Values.rbac.serviceAccount }}
|
||||
restartPolicy: OnFailure
|
||||
volumes:
|
||||
- name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
- name: config-volume-{{- $root.Values.global.name }}
|
||||
configMap:
|
||||
name: config-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
name: config-{{- $root.Values.global.name }}
|
||||
containers:
|
||||
- name: pre-install-job-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
image: {{ $root.Values.global.image | quote }}
|
||||
{{- range $classConfig := $root.Values.classes }}
|
||||
- name: storage-init-{{- $classConfig.name }}
|
||||
image: {{ $root.Values.images.tags.rbd_provisioner_storage_init | quote }}
|
||||
command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
|
||||
volumeMounts:
|
||||
- name: config-volume-{{- $root.Values.global.name -}}-{{- $classConfig.name }}
|
||||
mountPath: {{ $mount }}
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
value: {{ $root.Values.global.namespace }}
|
||||
- name: POOL_SECRET_NAMESPACES
|
||||
value: {{ $classConfig.pool_secrets_namespaces }}
|
||||
- name: CEPH_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
|
||||
key: key
|
||||
value: {{ $defaults.adminSecretName }}
|
||||
- name: CEPH_USER_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ or $classConfig.userSecretName }}
|
||||
key: key
|
||||
---
|
||||
{{- end }}
|
||||
value: {{ $classConfig.userSecretName }}
|
||||
- name: USER_ID
|
||||
value: {{ $classConfig.userId }}
|
||||
- name: POOL_NAME
|
||||
value: {{ $classConfig.pool_name }}
|
||||
- name: POOL_REPLICATION
|
||||
value: {{ $classConfig.replication | quote }}
|
||||
- name: POOL_CRUSH_RULE_NAME
|
||||
value: {{ $classConfig.crush_rule_name | quote }}
|
||||
- name: POOL_CHUNK_SIZE
|
||||
value: {{ $classConfig.chunk_size | quote }}
|
||||
volumeMounts:
|
||||
- name: config-volume-{{- $root.Values.global.name }}
|
||||
mountPath: {{ $mount }}
|
||||
{{- end }}
|
||||
{{- range $ephemeralPool := $root.Values.ephemeral_pools }}
|
||||
- name: storage-init-{{- $ephemeralPool.pool_name }}
|
||||
image: {{ $root.Values.images.tags.rbd_provisioner_storage_init | quote }}
|
||||
command: [ "/bin/bash", "{{ $mount }}/check_ceph.sh" ]
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
value: {{ $root.Values.global.namespace }}
|
||||
- name: POOL_NAME
|
||||
value: {{ $ephemeralPool.pool_name }}
|
||||
- name: POOL_REPLICATION
|
||||
value: {{ $ephemeralPool.replication | quote }}
|
||||
- name: POOL_CRUSH_RULE_NAME
|
||||
value: {{ $ephemeralPool.crush_rule_name | quote }}
|
||||
- name: POOL_CHUNK_SIZE
|
||||
value: {{ $ephemeralPool.chunk_size | quote }}
|
||||
volumeMounts:
|
||||
- name: config-volume-{{- $root.Values.global.name }}
|
||||
mountPath: {{ $mount }}
|
||||
{{- end }}
|
||||
{{- end }}
|
|
@ -15,5 +15,5 @@ metadata:
|
|||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get"]
|
||||
verbs: ["get", "create", "list", "update"]
|
||||
{{- end}}
|
|
@ -20,7 +20,7 @@ parameters:
|
|||
adminId: {{ or $classConfig.adminId $defaults.adminId}}
|
||||
adminSecretName: {{ or $classConfig.adminSecretName $defaults.adminSecretName }}
|
||||
adminSecretNamespace: {{ $namespace }}
|
||||
pool: {{ or $classConfig.pool $defaults.pool }}
|
||||
pool: {{ or $classConfig.pool_name $defaults.pool_name }}
|
||||
userId: {{ or $classConfig.userId $defaults.userId }}
|
||||
userSecretName: {{ $classConfig.userSecretName }}
|
||||
imageFormat: {{ or $classConfig.imageFormat $defaults.imageFormat | quote }}
|
||||
|
|
|
@ -17,14 +17,12 @@ global:
|
|||
#
|
||||
namespace: kube-system
|
||||
#
|
||||
# Run pre-install verifications or skip them.
|
||||
# Skipping them is not recommended
|
||||
# Execute initialization job to verify external Ceph cluster access
|
||||
# and setup additional dependencies assumed by dependent helm charts
|
||||
# (i.e. configmap and secrets).
|
||||
# Skipping is not recommended.
|
||||
#
|
||||
doPreInstallVerification: True
|
||||
#
|
||||
# Defines Provisioner's image name including container registry.
|
||||
#
|
||||
image: quay.io/external_storage/rbd-provisioner:latest
|
||||
job_storage_init: true
|
||||
#
|
||||
# Defines whether to reuse an already defined RBAC policy.
|
||||
# Make sure that the serviceAccount defined in the RBAC section matches the one
|
||||
|
@ -132,11 +130,40 @@ classdefaults:
|
|||
classes:
|
||||
- name: fast-rbd # Name of storage class.
|
||||
# Ceph pool name
|
||||
pool: kube
|
||||
pool_name: kube
|
||||
# Ceph user name to access this pool
|
||||
userId: kube
|
||||
# K8 secret name with key for accessing the Ceph pool
|
||||
userSecretName: ceph-secret-kube
|
||||
# Namespaces for creating the k8s secrets for accessing the Ceph pools
|
||||
pool_secrets_namespaces: kube-system
|
||||
# Name of pool to configure
|
||||
pool_name: kube-rbd
|
||||
# Pool replication
|
||||
replication: 1
|
||||
# Pool crush rule name
|
||||
crush_rule_name: storage_tier_ruleset
|
||||
# Pool chunk size / PG_NUM
|
||||
chunk_size: 8
|
||||
# Configuration data for the ephemeral pool(s)
|
||||
ephemeral_pools:
|
||||
- chunk_size: 8
|
||||
crush_rule_name: storage_tier_ruleset
|
||||
pool_name: ephemeral
|
||||
replication: 1
|
||||
|
||||
|
||||
|
||||
#
|
||||
# Defines:
|
||||
# - Provisioner's image name including container registry.
|
||||
# - CEPH helper image
|
||||
#
|
||||
images:
|
||||
tags:
|
||||
rbd_provisioner: quay.io/external_storage/rbd-provisioner:latest
|
||||
rbd_provisioner_storage_init: docker.io/port/ceph-config-helper:v1.10.3
|
||||
pull_policy: "IfNotPresent"
|
||||
local_registry:
|
||||
active: false
|
||||
exclude:
|
||||
- dep_check
|
||||
- image_repo_sync
|
||||
|
|
|
@ -2293,12 +2293,16 @@ class HostController(rest.RestController):
|
|||
storage_nodes = pecan.request.dbapi.ihost_get_by_personality(
|
||||
constants.STORAGE)
|
||||
if len(storage_nodes) == 1:
|
||||
# TODO(CephPoolsDecouple): rework
|
||||
# delete osd pools
|
||||
# It would be nice if we have a ceph API that can delete
|
||||
# all osd pools at once.
|
||||
pools = pecan.request.rpcapi.list_osd_pools(pecan.request.context)
|
||||
for ceph_pool in pools:
|
||||
pecan.request.rpcapi.delete_osd_pool(pecan.request.context, ceph_pool)
|
||||
if not utils.is_kubernetes_config():
|
||||
pools = pecan.request.rpcapi.list_osd_pools(
|
||||
pecan.request.context)
|
||||
for ceph_pool in pools:
|
||||
pecan.request.rpcapi.delete_osd_pool(
|
||||
pecan.request.context, ceph_pool)
|
||||
|
||||
# update tier status
|
||||
tier_list = pecan.request.dbapi.storage_tier_get_list()
|
||||
|
@ -5198,7 +5202,7 @@ class HostController(rest.RestController):
|
|||
StorageBackendConfig.get_ceph_pool_replication(pecan.request.dbapi, bk)
|
||||
stors = pecan.request.dbapi.istor_get_by_tier(tier.id)
|
||||
if len(stors) < replication:
|
||||
word = 'is' if len(replication) == 1 else 'are'
|
||||
word = 'is' if replication == 1 else 'are'
|
||||
msg = ("Can not unlock node until at least %(replication)s osd stor %(word)s "
|
||||
"configured for tier '%(tier)s'."
|
||||
% {'replication': str(replication), 'word': word, 'tier': tier['name']})
|
||||
|
@ -5534,7 +5538,7 @@ class HostController(rest.RestController):
|
|||
)
|
||||
if not backend:
|
||||
raise wsme.exc.ClientSideError(
|
||||
_("Ceph must be configured as a backend."))
|
||||
_("Ceph must be configured as a backend."))
|
||||
|
||||
if (backend.task == constants.SB_TASK_RESTORE and force):
|
||||
LOG.info("%s Allow force-locking as ceph backend is in "
|
||||
|
@ -5589,19 +5593,18 @@ class HostController(rest.RestController):
|
|||
if not pools_usage:
|
||||
raise wsme.exc.ClientSideError(
|
||||
_("Cannot lock a storage node when ceph pool usage is undetermined."))
|
||||
for ceph_pool in pools_usage:
|
||||
# We only need to check data pools
|
||||
if ([pool for pool in constants.ALL_CEPH_POOLS
|
||||
if ceph_pool['name'].startswith(pool)] and
|
||||
int(ceph_pool['stats']['bytes_used']) > 0):
|
||||
# Ceph pool is not empty and no other enabled storage
|
||||
# in set, so locking this storage node is not allowed.
|
||||
msg = _("Cannot lock a storage node when ceph pools are"
|
||||
" not empty and replication is lost. This may"
|
||||
" result in data loss. ")
|
||||
raise wsme.exc.ClientSideError(msg)
|
||||
|
||||
ceph_pools_empty = True
|
||||
ceph_pools_empty = self._ceph.ceph_pools_empty(
|
||||
pecan.request.dbapi, pools_usage)
|
||||
|
||||
if not ceph_pools_empty:
|
||||
msg = _(
|
||||
"Cannot lock a storage node when ceph pools are"
|
||||
" not empty and replication is lost. This may"
|
||||
" result in data loss. ")
|
||||
# Ceph pool is not empty and no other enabled storage
|
||||
# in set, so locking this storage node is not allowed.
|
||||
raise wsme.exc.ClientSideError(msg)
|
||||
|
||||
# Perform checks on storage regardless of operational state
|
||||
# as a minimum number of monitor is required.
|
||||
|
|
|
@ -413,11 +413,9 @@ def _discover_and_validate_backend_config_data(caps_dict, confirmed):
|
|||
elif k == constants.CEPH_BACKEND_MIN_REPLICATION_CAP:
|
||||
rep = int(caps_dict[constants.CEPH_BACKEND_REPLICATION_CAP])
|
||||
v_supported = constants.CEPH_REPLICATION_MAP_SUPPORTED[rep]
|
||||
msg = _("Missing or invalid value for "
|
||||
"backend parameter \'%s\', when "
|
||||
"replication is set as \'%s\'. "
|
||||
"Supported values are %s." %
|
||||
(k, rep, str(v_supported)))
|
||||
msg = _("Missing or invalid value for backend parameter \'%s\', "
|
||||
"when replication is set as \'%s\'. Supported values are "
|
||||
"%s." % (k, rep, str(v_supported)))
|
||||
try:
|
||||
v = int(v)
|
||||
except ValueError:
|
||||
|
@ -691,16 +689,6 @@ def _check_and_update_rbd_provisioner(new_storceph, remove=False):
|
|||
LOG.info("Ceph not configured, delaying rbd-provisioner configuration.")
|
||||
return new_storceph
|
||||
|
||||
# Cluster is configured, run live.
|
||||
try:
|
||||
new_storceph = \
|
||||
pecan.request.rpcapi.check_and_update_rbd_provisioner(pecan.request.context,
|
||||
new_storceph)
|
||||
except Exception as e:
|
||||
msg = _("Error configuring rbd-provisioner service. Please "
|
||||
"investigate and try again: %s." % str(e))
|
||||
raise wsme.exc.ClientSideError(msg)
|
||||
|
||||
return new_storceph
|
||||
|
||||
|
||||
|
@ -1015,6 +1003,7 @@ def _check_replication_number(new_cap, orig_cap):
|
|||
new_cap[constants.CEPH_BACKEND_REPLICATION_CAP])))
|
||||
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _is_quotaconfig_changed(ostorceph, storceph):
|
||||
if storceph and ostorceph:
|
||||
if (storceph.cinder_pool_gib != ostorceph.cinder_pool_gib or
|
||||
|
@ -1026,6 +1015,7 @@ def _is_quotaconfig_changed(ostorceph, storceph):
|
|||
return False
|
||||
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _check_pool_quotas_data(ostorceph, storceph):
|
||||
# Only relevant for ceph backend
|
||||
if not StorageBackendConfig.has_backend_configured(
|
||||
|
@ -1161,6 +1151,7 @@ def _check_pool_quotas_data(ostorceph, storceph):
|
|||
% (total_quota_gib, int(tier_size)))
|
||||
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _update_pool_quotas(storceph):
|
||||
# In R4, the object data pool name could be either
|
||||
# CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER or CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL
|
||||
|
@ -1241,6 +1232,7 @@ def _patch(storceph_uuid, patch):
|
|||
|
||||
# Obtain the fields that have changed.
|
||||
delta = rpc_storceph.obj_what_changed()
|
||||
# TODO(CephPoolsDecouple): remove quota values
|
||||
allowed_attributes = ['services', 'capabilities', 'task',
|
||||
'cinder_pool_gib',
|
||||
'glance_pool_gib',
|
||||
|
@ -1248,6 +1240,7 @@ def _patch(storceph_uuid, patch):
|
|||
'object_pool_gib',
|
||||
'kube_pool_gib',
|
||||
'object_gateway']
|
||||
# TODO(CephPoolsDecouple): remove variable
|
||||
quota_attributes = ['cinder_pool_gib', 'glance_pool_gib',
|
||||
'ephemeral_pool_gib', 'object_pool_gib',
|
||||
'kube_pool_gib']
|
||||
|
@ -1296,6 +1289,7 @@ def _patch(storceph_uuid, patch):
|
|||
# We only have changes to fast configurable services and/or to their capabilities
|
||||
fast_config = True
|
||||
|
||||
# TODO(CephPoolsDecouple): remove variable
|
||||
quota_only_update = True
|
||||
replication_only_update = False
|
||||
for d in delta:
|
||||
|
@ -1303,12 +1297,13 @@ def _patch(storceph_uuid, patch):
|
|||
raise wsme.exc.ClientSideError(
|
||||
_("Can not modify '%s' with this operation." % d))
|
||||
|
||||
# TODO(CephPoolsDecouple): remove condition
|
||||
if d not in quota_attributes:
|
||||
quota_only_update = False
|
||||
|
||||
# TODO (rchurch): In R6, refactor and remove object_gateway attribute
|
||||
# and DB column. This should be driven by if the service is added to the
|
||||
# services list
|
||||
# and DB column. This should be driven by if the service is added to
|
||||
# the services list
|
||||
if d == 'object_gateway':
|
||||
if ostorceph[d]:
|
||||
raise wsme.exc.ClientSideError(
|
||||
|
@ -1364,15 +1359,20 @@ def _patch(storceph_uuid, patch):
|
|||
LOG.info("SYS_I orig storage_ceph: %s " % ostorceph.as_dict())
|
||||
LOG.info("SYS_I patched storage_ceph: %s " % storceph_config.as_dict())
|
||||
|
||||
if _is_quotaconfig_changed(ostorceph, storceph_config):
|
||||
_check_pool_quotas_data(ostorceph, storceph_config.as_dict())
|
||||
_update_pool_quotas(storceph_config.as_dict())
|
||||
# check again after update
|
||||
_check_pool_quotas_data(ostorceph, storceph_config.as_dict())
|
||||
# TODO(CephPoolsDecouple): remove block
|
||||
if not utils.is_kubernetes_config():
|
||||
if _is_quotaconfig_changed(ostorceph, storceph_config):
|
||||
_check_pool_quotas_data(ostorceph, storceph_config.as_dict())
|
||||
_update_pool_quotas(storceph_config.as_dict())
|
||||
# check again after update
|
||||
_check_pool_quotas_data(ostorceph, storceph_config.as_dict())
|
||||
else:
|
||||
LOG.info("Don't check quotas")
|
||||
|
||||
if not quota_only_update:
|
||||
# Execute the common semantic checks for all backends, if backend is
|
||||
# not present this will not return.
|
||||
# TODO(CephPoolsDecouple): remove condition
|
||||
if not quota_only_update or utils.is_kubernetes_config():
|
||||
# Execute the common semantic checks for all backends, if backend
|
||||
# is not present this will not return.
|
||||
api_helper.common_checks(constants.SB_API_OP_MODIFY,
|
||||
rpc_storceph.as_dict())
|
||||
|
||||
|
@ -1392,20 +1392,18 @@ def _patch(storceph_uuid, patch):
|
|||
rpc_storceph[field] != storceph_config.as_dict()[field]):
|
||||
rpc_storceph[field] = storceph_config.as_dict()[field]
|
||||
|
||||
# TODO(CephPoolsDecouple): remove - on a containerized deployment,
|
||||
# replication is updated through the helm charts.
|
||||
# Update replication on the fly on a single node install.
|
||||
if (replication_only_update and
|
||||
utils.is_aio_simplex_system(pecan.request.dbapi)):
|
||||
# For single node setups update replication number on the fly.
|
||||
min_replication = new_cap.get(constants.CEPH_BACKEND_MIN_REPLICATION_CAP, None)
|
||||
replication = new_cap.get(constants.CEPH_BACKEND_REPLICATION_CAP, None)
|
||||
pecan.request.rpcapi.configure_osd_pools(pecan.request.context,
|
||||
rpc_storceph,
|
||||
replication, min_replication)
|
||||
|
||||
# Perform changes to the RBD Provisioner service
|
||||
remove_rbd_provisioner = constants.SB_SVC_RBD_PROVISIONER in services_removed
|
||||
ret = _check_and_update_rbd_provisioner(rpc_storceph.as_dict(), remove_rbd_provisioner)
|
||||
rpc_storceph['capabilities'] = ret['capabilities']
|
||||
if not utils.is_kubernetes_config():
|
||||
if (replication_only_update and
|
||||
utils.is_aio_simplex_system(pecan.request.dbapi)):
|
||||
# For single node setups update replication number on the fly.
|
||||
min_replication = new_cap.get(constants.CEPH_BACKEND_MIN_REPLICATION_CAP, None)
|
||||
replication = new_cap.get(constants.CEPH_BACKEND_REPLICATION_CAP, None)
|
||||
pecan.request.rpcapi.configure_osd_pools(
|
||||
pecan.request.context, rpc_storceph, replication,
|
||||
min_replication)
|
||||
|
||||
LOG.info("SYS_I new storage_ceph: %s " % rpc_storceph.as_dict())
|
||||
try:
|
||||
|
@ -1413,10 +1411,11 @@ def _patch(storceph_uuid, patch):
|
|||
|
||||
rpc_storceph.save()
|
||||
|
||||
# TODO(CephPoolsDecouple): rework - remove quota_only_update
|
||||
if ((not quota_only_update and
|
||||
not fast_config and
|
||||
not replication_only_update) or
|
||||
(storceph_config.state == constants.SB_STATE_CONFIG_ERR)):
|
||||
(storceph_config.state == constants.SB_STATE_CONFIG_ERR)):
|
||||
# Enable the backend changes:
|
||||
_apply_backend_changes(constants.SB_API_OP_MODIFY,
|
||||
rpc_storceph)
|
||||
|
|
|
@ -395,6 +395,13 @@ def is_aio_duplex_system():
|
|||
SystemHelper.get_product_build() == constants.TIS_AIO_BUILD
|
||||
|
||||
|
||||
def is_aio_kubernetes(dbapi=None):
|
||||
if not dbapi:
|
||||
dbapi = pecan.request.dbapi
|
||||
return SystemHelper.get_product_build() == constants.TIS_AIO_BUILD and \
|
||||
is_kubernetes_config(dbapi)
|
||||
|
||||
|
||||
def get_compute_count(dbapi=None):
|
||||
if not dbapi:
|
||||
dbapi = pecan.request.dbapi
|
||||
|
|
|
@ -605,6 +605,41 @@ class CephApiOperator(object):
|
|||
osd_list.append(osd_id)
|
||||
return constants.CEPH_HEALTH_OK
|
||||
|
||||
def ceph_pools_empty(self, db_api, pools_usage):
|
||||
""" Determine if data CEPH pools are empty.
|
||||
:return True if the data CEPH pools are empty
|
||||
:return False if the data CEPH pools are not empty
|
||||
"""
|
||||
# TODO(CephPoolsDecouple): rework
|
||||
if utils.is_kubernetes_config(db_api):
|
||||
for ceph_pool in pools_usage:
|
||||
# We only need to check data pools.
|
||||
if (constants.CEPH_POOL_OBJECT_GATEWAY_NAME_PART in
|
||||
ceph_pool['name']):
|
||||
if not (
|
||||
ceph_pool['name'].startswith(
|
||||
constants.CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL) or
|
||||
ceph_pool['name'].startswith(
|
||||
constants.CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER)):
|
||||
continue
|
||||
|
||||
# Ceph pool is not empty.
|
||||
if int(ceph_pool['stats']['bytes_used']) > 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# TODO(CephPoolsDecouple): remove iteration below
|
||||
for ceph_pool in pools_usage:
|
||||
# We only need to check data pools.
|
||||
if ([pool for pool in constants.ALL_CEPH_POOLS
|
||||
if ceph_pool['name'].startswith(pool)] and
|
||||
int(ceph_pool['stats']['bytes_used']) > 0):
|
||||
# Ceph pool is not empty.
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def get_monitors_status(self, db_api):
|
||||
# first check that the monitors are available in sysinv
|
||||
num_active_monitors = 0
|
||||
|
|
|
@ -704,6 +704,7 @@ CEPH_POOL_KUBE_QUOTA_GIB = 20
|
|||
# Ceph RADOS Gateway default data pool
|
||||
# Hammer version pool name will be kept if upgrade from R3 and
|
||||
# Swift/Radosgw was configured/enabled in R3.
|
||||
CEPH_POOL_OBJECT_GATEWAY_NAME_PART = 'rgw'
|
||||
CEPH_POOL_OBJECT_GATEWAY_NAME_JEWEL = 'default.rgw.buckets.data'
|
||||
CEPH_POOL_OBJECT_GATEWAY_NAME_HAMMER = '.rgw.buckets'
|
||||
CEPH_POOL_OBJECT_GATEWAY_ROOT_NAME = '.rgw.root'
|
||||
|
@ -1461,8 +1462,6 @@ SUPPORTED_HELM_APP_CHARTS = {
|
|||
K8S_RBD_PROV_STORAGECLASS_NAME = 'rbd_storageclass_name' # Customer
|
||||
K8S_RBD_PROV_NAMESPACES = 'rbd_provisioner_namespaces' # Customer
|
||||
K8S_RBD_PROV_NAMESPACES_READY = '.rbd_provisioner_namespaces_ready' # Hidden
|
||||
K8S_RBD_PROV_ADMIN_SECRET_READY = '.k8s_admin_secret_ready' # Hidden
|
||||
K8S_RBD_PROV_CEPH_POOL_KEY_READY = '.k8s_pool_secret_ready' # Hidden
|
||||
|
||||
# RBD Provisioner defaults and constants
|
||||
K8S_RBD_PROV_NAMESPACE_DEFAULT = "kube-system"
|
||||
|
|
|
@ -554,104 +554,3 @@ class K8RbdProvisioner(object):
|
|||
"Details: %s" % (secret_name, namespace, str(e)))
|
||||
|
||||
return stdout
|
||||
|
||||
@staticmethod
|
||||
def create_k8s_pool_secret(bk, key=None, namespace=None, force=False):
|
||||
user_secret_name = K8RbdProvisioner.get_user_secret_name(bk)
|
||||
|
||||
if K8RbdProvisioner.get_k8s_secret(user_secret_name,
|
||||
namespace=namespace):
|
||||
if not force:
|
||||
return
|
||||
# Key already exists
|
||||
LOG.warning("K8S Secret for backend: %s and namespace: %s exists and "
|
||||
"should not be present! Removing existing and creating "
|
||||
"a new one." % (bk['name'], namespace))
|
||||
K8RbdProvisioner.remove_k8s_pool_secret(bk, namespace)
|
||||
|
||||
LOG.info("Creating Kubernetes RBD Provisioner Ceph pool secret "
|
||||
"for namespace: %s." % namespace)
|
||||
try:
|
||||
# Create the k8s secret for the given Ceph pool and namespace.
|
||||
cmd = ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
|
||||
'create', 'secret', 'generic',
|
||||
user_secret_name,
|
||||
'--type=kubernetes.io/rbd']
|
||||
if key:
|
||||
cmd.append('--from-literal=key=%s' % key)
|
||||
if namespace:
|
||||
cmd.append('--namespace=%s' % namespace)
|
||||
_, _ = cutils.execute(*cmd, run_as_root=False)
|
||||
except exception.ProcessExecutionError as e:
|
||||
raise exception.SysinvException(
|
||||
"Could not create Kubernetes secret: %s for backend: %s, "
|
||||
"namespace: %s, Details: %s." %
|
||||
(user_secret_name, bk['name'], namespace, str(e)))
|
||||
|
||||
@staticmethod
|
||||
def remove_k8s_pool_secret(bk, namespace):
|
||||
user_secret_name = K8RbdProvisioner.get_user_secret_name(bk)
|
||||
if not K8RbdProvisioner.get_k8s_secret(user_secret_name,
|
||||
namespace=namespace):
|
||||
LOG.warning("K8S secret for backend: %s and namespace: %s "
|
||||
"does not exists. Skipping removal." % (bk['name'], namespace))
|
||||
return
|
||||
try:
|
||||
# Remove the k8s secret from given namepsace.
|
||||
cmd = ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
|
||||
'delete', 'secret', user_secret_name,
|
||||
'--namespace=%s' % namespace]
|
||||
_, _ = cutils.execute(*cmd, run_as_root=False)
|
||||
except exception.ProcessExecutionError as e:
|
||||
raise exception.SysinvException(
|
||||
"Could not remove Kubernetes secret: %s for backend: %s, "
|
||||
"namespace: %s, Details: %s." %
|
||||
(user_secret_name, bk['name'], namespace, str(e)))
|
||||
|
||||
@staticmethod
|
||||
def create_k8s_admin_secret():
|
||||
admin_secret_name = constants.K8S_RBD_PROV_ADMIN_SECRET_NAME
|
||||
namespace = constants.K8S_RBD_PROV_NAMESPACE_DEFAULT
|
||||
|
||||
if K8RbdProvisioner.get_k8s_secret(
|
||||
admin_secret_name, namespace=namespace):
|
||||
# Key already exists
|
||||
return
|
||||
|
||||
LOG.info("Creating Kubernetes RBD Provisioner Ceph admin secret.")
|
||||
try:
|
||||
# TODO(oponcea): Get admin key on Ceph clusters with
|
||||
# enabled authentication. For now feed an empty key
|
||||
# to satisfy RBD Provisioner requirements.
|
||||
cmd = ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
|
||||
'create', 'secret', 'generic',
|
||||
admin_secret_name,
|
||||
'--type=kubernetes.io/rbd',
|
||||
'--from-literal=key=']
|
||||
cmd.append('--namespace=%s' % namespace)
|
||||
_, _ = cutils.execute(*cmd, run_as_root=False)
|
||||
except exception.ProcessExecutionError as e:
|
||||
raise exception.SysinvException(
|
||||
"Could not create Kubernetes secret: %s, namespace: %s,"
|
||||
"Details: %s" % (admin_secret_name, namespace, str(e)))
|
||||
|
||||
@staticmethod
|
||||
def remove_k8s_admin_secret():
|
||||
admin_secret_name = constants.K8S_RBD_PROV_ADMIN_SECRET_NAME
|
||||
namespace = constants.K8S_RBD_PROV_NAMESPACE_DEFAULT
|
||||
|
||||
if not K8RbdProvisioner.get_k8s_secret(
|
||||
admin_secret_name, namespace=namespace):
|
||||
# Secret does not exist.
|
||||
return
|
||||
|
||||
LOG.info("Removing Kubernetes RBD Provisioner Ceph admin secret.")
|
||||
try:
|
||||
cmd = ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
|
||||
'delete', 'secret', admin_secret_name,
|
||||
'--namespace=%s' % namespace]
|
||||
_, _ = cutils.execute(*cmd, run_as_root=False)
|
||||
except exception.ProcessExecutionError as e:
|
||||
raise exception.SysinvException(
|
||||
"Could not delete Kubernetes secret: %s, namespace: %s,"
|
||||
"Details: %s." % (admin_secret_name, namespace, str(e)))
|
||||
|
|
|
@ -15,6 +15,7 @@ from __future__ import absolute_import
|
|||
import os
|
||||
import uuid
|
||||
import copy
|
||||
import tsconfig.tsconfig as tsc
|
||||
from requests.exceptions import RequestException, ReadTimeout
|
||||
|
||||
from cephclient import wrapper as ceph
|
||||
|
@ -25,7 +26,6 @@ from sysinv.common import utils as cutils
|
|||
from sysinv.openstack.common import log as logging
|
||||
from sysinv.openstack.common import uuidutils
|
||||
from sysinv.common.storage_backend_conf import StorageBackendConfig
|
||||
from sysinv.common.storage_backend_conf import K8RbdProvisioner
|
||||
|
||||
from sysinv.api.controllers.v1 import utils
|
||||
|
||||
|
@ -273,6 +273,7 @@ class CephOperator(object):
|
|||
reason=response.reason)
|
||||
return response, body
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def osd_set_pool_param(self, pool_name, param, value):
|
||||
response, body = self._ceph_api.osd_set_pool_param(
|
||||
pool_name, param, value,
|
||||
|
@ -391,6 +392,7 @@ class CephOperator(object):
|
|||
LOG.info("osdmap is empty, restoring Ceph config...")
|
||||
return self.rebuild_osdmap()
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _pool_create(self, name, pg_num, pgp_num, ruleset,
|
||||
size, min_size):
|
||||
"""Create Ceph pool and ruleset.
|
||||
|
@ -447,7 +449,7 @@ class CephOperator(object):
|
|||
"pool_name={}, ruleset={}").format(
|
||||
name, ruleset))
|
||||
else:
|
||||
msg = _("Failed to to complete parameter assignment on OSD pool"
|
||||
msg = _("Failed to complete parameter assignment on OSD pool"
|
||||
": {0}. reason: {1}").format(name, response.reason)
|
||||
e = exception.CephFailure(reason=msg)
|
||||
LOG.error(e)
|
||||
|
@ -472,13 +474,14 @@ class CephOperator(object):
|
|||
"pool_name={}, min_size={}").format(name,
|
||||
min_size))
|
||||
else:
|
||||
msg = _("Failed to to complete parameter assignment on existing"
|
||||
msg = _("Failed to complete parameter assignment on existing "
|
||||
"OSD pool: {0}. reason: {1}").format(name,
|
||||
response.reason)
|
||||
e = exception.CephFailure(reason=msg)
|
||||
LOG.error(e)
|
||||
raise e
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def create_or_resize_osd_pool(self, pool_name, pg_num, pgp_num,
|
||||
size, min_size):
|
||||
"""Create or resize an osd pool as needed
|
||||
|
@ -588,6 +591,7 @@ class CephOperator(object):
|
|||
return {"max_objects": quota["output"]["quota_max_objects"],
|
||||
"max_bytes": quota["output"]["quota_max_bytes"]}
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def set_osd_pool_quota(self, pool, max_bytes=0, max_objects=0):
|
||||
"""Set the quota for an OSD pool
|
||||
Setting max_bytes or max_objects to 0 will disable that quota param
|
||||
|
@ -622,6 +626,8 @@ class CephOperator(object):
|
|||
LOG.error(e)
|
||||
raise e
|
||||
|
||||
# TODO(CephPoolsDecouple): rework if needed: we determine the existing
|
||||
# pools on the spot
|
||||
def get_pools_values(self):
|
||||
"""Create or resize all of the osd pools as needed
|
||||
"""
|
||||
|
@ -652,6 +658,7 @@ class CephOperator(object):
|
|||
LOG.debug("Pool Quotas: %s" % quotas)
|
||||
return tuple(quotas)
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def set_quota_gib(self, pool_name):
|
||||
quota_gib_value = None
|
||||
cinder_pool_gib, glance_pool_gib, ephemeral_pool_gib, \
|
||||
|
@ -754,6 +761,7 @@ class CephOperator(object):
|
|||
|
||||
return rc
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _configure_primary_tier_pool(self, pool, size, min_size):
|
||||
"""Configure the default Ceph tier pools."""
|
||||
|
||||
|
@ -769,6 +777,7 @@ class CephOperator(object):
|
|||
except exception.CephFailure:
|
||||
pass
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def _configure_secondary_tier_pools(self, tier_obj, size, min_size):
|
||||
"""Configure the service pools that are allowed for additional ceph tiers.
|
||||
"""
|
||||
|
@ -821,6 +830,7 @@ class CephOperator(object):
|
|||
name=rule_name, reason=body['status'])
|
||||
raise e
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def configure_osd_pools(self, ceph_backend=None, new_pool_size=None, new_pool_min_size=None):
|
||||
"""Create or resize all of the osd pools as needed
|
||||
ceph backend could be 2nd backend which is in configuring state
|
||||
|
@ -881,74 +891,6 @@ class CephOperator(object):
|
|||
except exception.CephFailure as e:
|
||||
LOG.info("Cannot add pools: %s" % e)
|
||||
|
||||
def _update_k8s_ceph_pool_secrets(self, ceph_backend):
|
||||
"""Create CEPH pool secrets for k8s namespaces.
|
||||
:param ceph_backend input/output storage backend data
|
||||
"""
|
||||
|
||||
pool_name = K8RbdProvisioner.get_pool(ceph_backend)
|
||||
|
||||
namespaces_to_add, namespaces_to_rm = \
|
||||
K8RbdProvisioner.getNamespacesDelta(ceph_backend)
|
||||
|
||||
# Get or create Ceph pool key. One per pool.
|
||||
# This key will be used by the K8S secrets from the rbd-provisioner.
|
||||
if namespaces_to_add:
|
||||
key = self._configure_pool_key(pool_name)
|
||||
|
||||
# Get the capabilities of the backend directly from DB to avoid
|
||||
# committing changes unrelated to ceph pool keys.
|
||||
try:
|
||||
orig_ceph_backend = self._db_api.storage_backend_get(ceph_backend['id'])
|
||||
orig_capab = orig_ceph_backend['capabilities']
|
||||
except exception.InvalidParameterValue:
|
||||
# This is a new backend, not yet stored in DB.
|
||||
orig_ceph_backend = None
|
||||
|
||||
configured_namespaces = \
|
||||
K8RbdProvisioner.getListFromNamespaces(orig_ceph_backend,
|
||||
get_configured=True)
|
||||
|
||||
# Adding secrets to namespaces
|
||||
for namespace in namespaces_to_add:
|
||||
K8RbdProvisioner.create_k8s_pool_secret(
|
||||
ceph_backend, key=key,
|
||||
namespace=namespace, force=(True if not ceph_backend else False))
|
||||
|
||||
# Update the backend's capabilities to reflect that a secret
|
||||
# has been created for the k8s pool in the given namespace.
|
||||
# Update DB for each item to reflect reality in case of error.
|
||||
configured_namespaces.append(namespace)
|
||||
if orig_ceph_backend:
|
||||
orig_capab[constants.K8S_RBD_PROV_NAMESPACES_READY] = \
|
||||
','.join(configured_namespaces)
|
||||
self._db_api.storage_backend_update(ceph_backend['id'],
|
||||
{'capabilities': orig_capab})
|
||||
|
||||
# Removing secrets from namespaces
|
||||
for namespace in namespaces_to_rm:
|
||||
K8RbdProvisioner.remove_k8s_pool_secret(ceph_backend,
|
||||
namespace)
|
||||
configured_namespaces.remove(namespace)
|
||||
if orig_ceph_backend:
|
||||
if configured_namespaces:
|
||||
orig_capab[constants.K8S_RBD_PROV_NAMESPACES_READY] = \
|
||||
','.join(configured_namespaces)
|
||||
elif constants.K8S_RBD_PROV_NAMESPACES_READY in orig_capab:
|
||||
# No RBD Provisioner configured, cleanup
|
||||
del orig_capab[constants.K8S_RBD_PROV_NAMESPACES_READY]
|
||||
self._db_api.storage_backend_update(ceph_backend['id'],
|
||||
{'capabilities': orig_capab})
|
||||
|
||||
# Done, store the updated capabilities in the ceph_backend reference
|
||||
capab = ceph_backend['capabilities']
|
||||
if configured_namespaces:
|
||||
capab[constants.K8S_RBD_PROV_NAMESPACES_READY] = \
|
||||
','.join(configured_namespaces)
|
||||
elif constants.K8S_RBD_PROV_NAMESPACES_READY in capab:
|
||||
# No RBD Provisioner configured, cleanup
|
||||
del capab[constants.K8S_RBD_PROV_NAMESPACES_READY]
|
||||
|
||||
def _update_db_capabilities(self, bk, new_storceph):
|
||||
# Avoid updating DB for all capabilities in new_storceph as we
|
||||
# don't manage them. Leave the callers deal with it.
|
||||
|
@ -959,96 +901,6 @@ class CephOperator(object):
|
|||
{'capabilities': bk['capabilities']}
|
||||
)
|
||||
|
||||
def check_and_update_rbd_provisioner(self, new_storceph=None):
|
||||
""" Check and/or update RBD Provisioner configuration for all Ceph
|
||||
internal backends.
|
||||
|
||||
This function should be called when:
|
||||
1. Making any changes to rbd-provisioner service
|
||||
(adding a new, removing or updating an existing provisioner)
|
||||
2. Synchronizing changes with the DB.
|
||||
|
||||
To speed up synchronization, DB entries are used to determine when
|
||||
changes are needed and only then proceed with more time consuming
|
||||
operations.
|
||||
|
||||
Note: This function assumes a functional Ceph cluster
|
||||
|
||||
:param new_storceph a storage backend object as_dict() with updated
|
||||
data. This is required as database updates can happen later.
|
||||
:returns an updated version of new_storceph or None
|
||||
"""
|
||||
# Get an updated list of backends
|
||||
if new_storceph:
|
||||
ceph_backends = [b.as_dict() for b in
|
||||
self._db_api.storage_backend_get_list()
|
||||
if b['backend'] == constants.SB_TYPE_CEPH and
|
||||
b['name'] != new_storceph['name']]
|
||||
ceph_backends.append(new_storceph)
|
||||
else:
|
||||
ceph_backends = [b.as_dict() for b in
|
||||
self._db_api.storage_backend_get_list()
|
||||
if b['backend'] == constants.SB_TYPE_CEPH]
|
||||
|
||||
# Nothing to do if rbd-provisioner is not configured and was never
|
||||
# configured on any backend.
|
||||
for bk in ceph_backends:
|
||||
svcs = utils.SBApiHelper.getListFromServices(bk)
|
||||
if (constants.SB_SVC_RBD_PROVISIONER in svcs or
|
||||
bk['capabilities'].get(constants.K8S_RBD_PROV_NAMESPACES_READY) or
|
||||
bk['capabilities'].get(constants.K8S_RBD_PROV_ADMIN_SECRET_READY)):
|
||||
break
|
||||
else:
|
||||
return new_storceph
|
||||
|
||||
# In order for an RBD provisioner to work we need:
|
||||
# - A couple of Ceph keys:
|
||||
# 1. A cluster wide admin key (e.g. the one in
|
||||
# /etc/ceph/ceph.client.admin.keyring)
|
||||
# 2. A key for accessing the pool (e.g. client.kube-rbd)
|
||||
# - The Ceph keys above passed into Kubernetes secrets:
|
||||
# 1. An admin secret in the RBD Provisioner POD namespace with the
|
||||
# Ceph cluster wide admin key.
|
||||
# 2. One or more K8S keys with the Ceph pool key for each namespace
|
||||
# we allow RBD PV and PVC creations.
|
||||
|
||||
LOG.info("Updating rbd-provisioner configuration.")
|
||||
# Manage Ceph cluster wide admin key and associated secret - we create
|
||||
# it if needed or remove it if no longer needed.
|
||||
admin_secret_exists = False
|
||||
remove_admin_secret = True
|
||||
for bk in ceph_backends:
|
||||
svcs = utils.SBApiHelper.getListFromServices(bk)
|
||||
|
||||
# Create secret
|
||||
# Check to see if we need the admin Ceph key. This key is created
|
||||
# once per cluster and references to it are kept in all Ceph tiers
|
||||
# of that cluster. So make sure they are up to date.
|
||||
if constants.SB_SVC_RBD_PROVISIONER in svcs:
|
||||
remove_admin_secret = False
|
||||
if bk['capabilities'].get(constants.K8S_RBD_PROV_ADMIN_SECRET_READY):
|
||||
admin_secret_exists = True
|
||||
else:
|
||||
if not admin_secret_exists:
|
||||
K8RbdProvisioner.create_k8s_admin_secret()
|
||||
admin_secret_exists = True
|
||||
bk['capabilities'][constants.K8S_RBD_PROV_ADMIN_SECRET_READY] = True
|
||||
self._update_db_capabilities(bk, new_storceph)
|
||||
# Remove admin secret and any references to it if RBD Provisioner is
|
||||
# unconfigured.
|
||||
if remove_admin_secret:
|
||||
K8RbdProvisioner.remove_k8s_admin_secret()
|
||||
for bk in ceph_backends:
|
||||
if bk['capabilities'].get(constants.K8S_RBD_PROV_ADMIN_SECRET_READY):
|
||||
del bk['capabilities'][constants.K8S_RBD_PROV_ADMIN_SECRET_READY]
|
||||
self._update_db_capabilities(bk, new_storceph)
|
||||
|
||||
for bk in ceph_backends:
|
||||
self._update_k8s_ceph_pool_secrets(bk)
|
||||
|
||||
# Return updated new_storceph reference
|
||||
return new_storceph
|
||||
|
||||
def get_osd_tree(self):
|
||||
"""Get OSD tree info
|
||||
return: list of nodes and a list of stray osds e.g.:
|
||||
|
@ -1174,13 +1026,24 @@ class CephOperator(object):
|
|||
return body["output"]
|
||||
|
||||
def get_ceph_cluster_info_availability(self):
|
||||
# TODO(CephPoolsDecouple): rework
|
||||
# Check if the ceph cluster is ready to return statistics
|
||||
storage_hosts = self._db_api.ihost_get_by_personality(
|
||||
constants.STORAGE)
|
||||
|
||||
is_aio_kubernetes = (
|
||||
tsc.system_type == constants.TIS_AIO_BUILD and
|
||||
utils.is_kubernetes_config(self._db_api))
|
||||
|
||||
if not storage_hosts and is_aio_kubernetes:
|
||||
storage_hosts = self._db_api.ihost_get_by_personality(
|
||||
constants.CONTROLLER)
|
||||
|
||||
# If there is no storage node present, ceph usage
|
||||
# information is not relevant
|
||||
if not storage_hosts:
|
||||
return False
|
||||
|
||||
# At least one storage node must be in available state
|
||||
for host in storage_hosts:
|
||||
if host['availability'] == constants.AVAILABILITY_AVAILABLE:
|
||||
|
@ -1190,6 +1053,7 @@ class CephOperator(object):
|
|||
return False
|
||||
return True
|
||||
|
||||
# TODO(CephPoolsDecouple): rework - determine the existing pools
|
||||
def get_pools_config(self):
|
||||
for pool in CEPH_POOLS:
|
||||
# Here it is okay for object pool name is either
|
||||
|
@ -1249,6 +1113,7 @@ class CephOperator(object):
|
|||
|
||||
return storage_hosts_upgraded
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
# TIER SUPPORT
|
||||
def _calculate_target_pg_num_for_tier_pool(self, tiers_obj, pool_name,
|
||||
storage_hosts):
|
||||
|
@ -1364,6 +1229,7 @@ class CephOperator(object):
|
|||
|
||||
return target_pg_num, osds_raw
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def audit_osd_pool_on_tier(self, tier_obj, storage_hosts, pool_name):
|
||||
""" Audit an osd pool and update pg_num, pgp_num accordingly.
|
||||
storage_hosts; list of known storage host objects
|
||||
|
@ -1453,6 +1319,7 @@ class CephOperator(object):
|
|||
# we attempt to increase the pgp number. We will wait for the
|
||||
# audit to call us and increase the pgp number at that point.
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def audit_osd_quotas_for_tier(self, tier_obj):
|
||||
|
||||
# TODO(rchurch): Make this smarter.Just look at the OSD for the tier to
|
||||
|
@ -1639,6 +1506,7 @@ class CephOperator(object):
|
|||
# Special case: For now with one pool allow no quota
|
||||
self.executed_default_quota_check_by_tier[tier_obj.name] = True
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def audit_osd_pools_by_tier(self):
|
||||
""" Check osd pool pg_num vs calculated target pg_num.
|
||||
Set pool quotas default values dynamically depending
|
||||
|
|
|
@ -1512,13 +1512,12 @@ class ConductorManager(service.PeriodicService):
|
|||
host.action == constants.FORCE_UNLOCK_ACTION or
|
||||
host.action == constants.UNLOCK_ACTION):
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
# Ensure the OSD pools exists. In the case of a system restore,
|
||||
# the pools must be re-created when the first storage node is
|
||||
# unlocked.
|
||||
self._ceph.configure_osd_pools()
|
||||
|
||||
# Generate CEPH keys for k8s pools.
|
||||
self.check_and_update_rbd_provisioner(context)
|
||||
if not utils.is_kubernetes_config(self.dbapi):
|
||||
self._ceph.configure_osd_pools()
|
||||
|
||||
# Generate host configuration files
|
||||
self._puppet.update_host_config(host)
|
||||
|
@ -1530,6 +1529,7 @@ class ConductorManager(service.PeriodicService):
|
|||
# Set up the PXE config file for this host so it can run the installer
|
||||
self._update_pxe_config(host)
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def configure_osd_pools(self, context, ceph_backend=None, new_pool_size=None, new_pool_min_size=None):
|
||||
"""Configure or update configuration of the OSD pools.
|
||||
If none of the optionals are provided then all pools are updated based on DB configuration.
|
||||
|
@ -4941,8 +4941,13 @@ class ConductorManager(service.PeriodicService):
|
|||
continue
|
||||
self._audit_ihost_action(host)
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
@periodic_task.periodic_task(spacing=60)
|
||||
def _osd_pool_audit(self, context):
|
||||
if utils.is_kubernetes_config(self.dbapi):
|
||||
LOG.debug("_osd_pool_audit skip")
|
||||
return
|
||||
|
||||
# Only do the audit if ceph is configured.
|
||||
if not StorageBackendConfig.has_backend(
|
||||
self.dbapi,
|
||||
|
@ -4950,6 +4955,8 @@ class ConductorManager(service.PeriodicService):
|
|||
):
|
||||
return
|
||||
|
||||
LOG.debug("_osd_pool_audit")
|
||||
|
||||
# Only run the pool audit task if we have at least one storage node
|
||||
# available. Pools are created with initial PG num values and quotas
|
||||
# when the first OSD is added. This is done with only controller-0
|
||||
|
@ -5002,32 +5009,13 @@ class ConductorManager(service.PeriodicService):
|
|||
cmd = ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
|
||||
'get', 'namespaces', '-o',
|
||||
'go-template=\'{{range .items}}{{.metadata.name}}\'{{end}}\'']
|
||||
stdout, _ = cutils.execute(*cmd, run_as_root=False)
|
||||
stdout, stderr = cutils.execute(*cmd, run_as_root=False)
|
||||
namespaces = [n for n in stdout.split("\'") if n]
|
||||
return namespaces
|
||||
except exception.ProcessExecutionError as e:
|
||||
raise exception.SysinvException(
|
||||
_("Error getting Kubernetes list of namespaces, "
|
||||
"Details: %s" % str(e)))
|
||||
|
||||
def check_and_update_rbd_provisioner(self, context, new_storceph=None):
|
||||
""" Check and/or update RBD Provisioner configuration for all Ceph
|
||||
internal backends.
|
||||
|
||||
This function should be called in two cases:
|
||||
1. When making any changes to the rbd-provisioner service.
|
||||
2. When delaying changes due to Ceph not being up.
|
||||
|
||||
To allow delayed executions we check DB entries for changes and only
|
||||
then proceed with time consuming modifications.
|
||||
|
||||
Note: This function assumes a functional Ceph cluster
|
||||
|
||||
:param new_storceph a storage backend object as_dict() with updated
|
||||
data. This is required as database updates can happen later.
|
||||
:returns an updated version of new_storceph or None
|
||||
"""
|
||||
return self._ceph.check_and_update_rbd_provisioner(new_storceph)
|
||||
"Details: %s") % str(e))
|
||||
|
||||
def configure_isystemname(self, context, systemname):
|
||||
"""Configure the systemname with the supplied data.
|
||||
|
@ -5146,7 +5134,9 @@ class ConductorManager(service.PeriodicService):
|
|||
# Update the osdid in the stor object
|
||||
istor_obj['osdid'] = body['output']['osdid']
|
||||
|
||||
self._ceph.configure_osd_pools()
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
if not utils.is_kubernetes_config(self.dbapi):
|
||||
self._ceph.configure_osd_pools()
|
||||
|
||||
return istor_obj
|
||||
|
||||
|
@ -6345,8 +6335,6 @@ class ConductorManager(service.PeriodicService):
|
|||
if utils.is_aio_simplex_system(self.dbapi):
|
||||
task = None
|
||||
cceph.fix_crushmap(self.dbapi)
|
||||
# Ceph is up, update rbd-provisioner config if needed.
|
||||
self.check_and_update_rbd_provisioner(context)
|
||||
else:
|
||||
task = constants.SB_TASK_PROVISION_STORAGE
|
||||
values = {'state': state,
|
||||
|
|
|
@ -117,6 +117,7 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
|
|||
host=host,
|
||||
do_compute_apply=do_compute_apply))
|
||||
|
||||
# TODO(CephPoolsDecouple): remove
|
||||
def configure_osd_pools(self, context, ceph_backend=None, new_pool_size=None, new_pool_min_size=None):
|
||||
"""Configure or update configuration of the OSD pools.
|
||||
If none of the optionals are provided then all pools are updated based on DB configuration.
|
||||
|
@ -912,27 +913,6 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
|
|||
return self.call(context,
|
||||
self.make_msg('get_k8s_namespaces'))
|
||||
|
||||
def check_and_update_rbd_provisioner(self, context, new_storceph=None):
|
||||
""" Check and/or update RBD Provisioner is correctly configured
|
||||
for all Ceph internal backends.
|
||||
|
||||
This function should be called in two cases:
|
||||
1. When making any change to rbd-provisioner
|
||||
2. When delaying changes due to Ceph not being up
|
||||
|
||||
To allow delayed executions we check DB entries for changes and only
|
||||
then proceed with time consuming modifications.
|
||||
|
||||
Note: This function assumes a fully functional Ceph cluster
|
||||
|
||||
:param new_storceph a storage backend object as_dict() with updated
|
||||
data. This is needed as database updates can happen later.
|
||||
:returns an updated version of new_storceph
|
||||
"""
|
||||
return self.call(context,
|
||||
self.make_msg('check_and_update_rbd_provisioner',
|
||||
new_storceph=new_storceph))
|
||||
|
||||
def report_config_status(self, context, iconfig,
|
||||
status, error=None):
|
||||
""" Callback from Sysinv Agent on manifest apply success or failure
|
||||
|
|
|
@ -90,14 +90,12 @@ class CinderHelm(openstack.OpenstackBaseHelm):
|
|||
# tier pools.
|
||||
'replication': replication,
|
||||
'crush_rule': ruleset,
|
||||
'chunk_size': 8
|
||||
},
|
||||
'volume': {
|
||||
# We don't use the chart to configure the cinder-volumes
|
||||
# pool, so these values don't have any impact right now.
|
||||
# The cinder chart doesn't currently support specifying
|
||||
# the config for multiple volume/backup pools.
|
||||
'replication': replication,
|
||||
'crush_rule': ruleset,
|
||||
'chunk_size': 8
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -23,8 +23,17 @@ class RbdProvisionerHelm(base.BaseHelm):
|
|||
common.HELM_NS_KUBE_SYSTEM
|
||||
]
|
||||
|
||||
SERVICE_NAME = 'rbd-provisioner'
|
||||
SERVICE_PORT_MON = 6789
|
||||
|
||||
@property
|
||||
def docker_repo_source(self):
|
||||
return common.DOCKER_SRC_STX
|
||||
|
||||
@property
|
||||
def docker_repo_tag(self):
|
||||
return common.DOCKER_SRCS[self.docker_repo_source][common.IMG_TAG_KEY]
|
||||
|
||||
def get_namespaces(self):
|
||||
return self.SUPPORTED_NAMESPACES
|
||||
|
||||
|
@ -52,20 +61,84 @@ class RbdProvisionerHelm(base.BaseHelm):
|
|||
"adminSecretName": constants.K8S_RBD_PROV_ADMIN_SECRET_NAME
|
||||
}
|
||||
|
||||
# Get tier info.
|
||||
tiers = self.dbapi.storage_tier_get_list()
|
||||
primary_tier_name = \
|
||||
constants.SB_TIER_DEFAULT_NAMES[constants.SB_TIER_TYPE_CEPH]
|
||||
|
||||
classes = []
|
||||
for bk in rbd_provisioner_bks:
|
||||
# Get the ruleset for the new kube-rbd pool.
|
||||
tier = next((t for t in tiers if t.forbackendid == bk.id), None)
|
||||
if not tier:
|
||||
raise Exception("No tier present for backend %s" % bk.name)
|
||||
|
||||
rule_name = "{0}{1}{2}".format(
|
||||
tier.name,
|
||||
constants.CEPH_CRUSH_TIER_SUFFIX,
|
||||
"-ruleset").replace('-', '_')
|
||||
|
||||
# Check namespaces. We need to know on what namespaces to create
|
||||
# the secrets for the kube-rbd pools.
|
||||
pool_secrets_namespaces = bk.capabilities.get(
|
||||
constants.K8S_RBD_PROV_NAMESPACES)
|
||||
if not pool_secrets_namespaces:
|
||||
raise Exception("Please specify the rbd_provisioner_namespaces"
|
||||
" for the %s backend." % bk.name)
|
||||
|
||||
cls = {
|
||||
"name": K8RbdProvisioner.get_storage_class_name(bk),
|
||||
"pool": K8RbdProvisioner.get_pool(bk),
|
||||
"pool_name": K8RbdProvisioner.get_pool(bk),
|
||||
"pool_secrets_namespaces": pool_secrets_namespaces.encode(
|
||||
'utf8', 'strict'),
|
||||
"replication": int(bk.capabilities.get("replication")),
|
||||
"crush_rule_name": rule_name,
|
||||
"chunk_size": 64,
|
||||
"userId": K8RbdProvisioner.get_user_id(bk),
|
||||
"userSecretName": K8RbdProvisioner.get_user_secret_name(bk)
|
||||
}
|
||||
classes.append(cls)
|
||||
|
||||
# Get all the info for creating the ephemeral pool.
|
||||
ephemeral_pools = []
|
||||
# Right now the ephemeral pool will only use the primary tier.
|
||||
rule_name = "{0}{1}{2}".format(
|
||||
primary_tier_name,
|
||||
constants.CEPH_CRUSH_TIER_SUFFIX,
|
||||
"-ruleset").replace('-', '_')
|
||||
|
||||
sb_list_ext = self.dbapi.storage_backend_get_list_by_type(
|
||||
backend_type=constants.SB_TYPE_CEPH_EXTERNAL)
|
||||
sb_list = self.dbapi.storage_backend_get_list_by_type(
|
||||
backend_type=constants.SB_TYPE_CEPH)
|
||||
|
||||
if sb_list_ext:
|
||||
for sb in sb_list_ext:
|
||||
if constants.SB_SVC_NOVA in sb.services:
|
||||
rbd_pool = sb.capabilities.get('ephemeral_pool')
|
||||
ephemeral_pool = {
|
||||
"pool_name": rbd_pool,
|
||||
"replication": int(sb.capabilities.get("replication")),
|
||||
"crush_rule_name": rule_name,
|
||||
"chunk_size": 64,
|
||||
}
|
||||
ephemeral_pools.append(ephemeral_pool)
|
||||
# Treat internal CEPH.
|
||||
if sb_list:
|
||||
ephemeral_pool = {
|
||||
"pool_name": constants.CEPH_POOL_EPHEMERAL_NAME,
|
||||
"replication": int(sb_list[0].capabilities.get("replication")),
|
||||
"crush_rule_name": rule_name,
|
||||
"chunk_size": 64,
|
||||
}
|
||||
ephemeral_pools.append(ephemeral_pool)
|
||||
|
||||
overrides = {
|
||||
common.HELM_NS_KUBE_SYSTEM: {
|
||||
"classdefaults": classdefaults,
|
||||
"classes": classes
|
||||
"classes": classes,
|
||||
"ephemeral_pools": ephemeral_pools,
|
||||
"images": self._get_images_overrides(),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -76,3 +149,21 @@ class RbdProvisionerHelm(base.BaseHelm):
|
|||
namespace=namespace)
|
||||
else:
|
||||
return overrides
|
||||
|
||||
def _get_images_overrides(self):
|
||||
# TODO: Remove after ceph upgrade
|
||||
# Format the name of the stx specific ceph config helper
|
||||
ceph_config_helper_image = "{}/{}{}:{}".format(
|
||||
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY],
|
||||
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
|
||||
'ceph-config-helper', self.docker_repo_tag)
|
||||
|
||||
rbd_provisioner_image = \
|
||||
'quay.io/external_storage/rbd-provisioner:latest'
|
||||
|
||||
return {
|
||||
'tags': {
|
||||
'rbd_provisioner': rbd_provisioner_image,
|
||||
'rbd_provisioner_storage_init': ceph_config_helper_image,
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue