From f73b181a08dec1d5acc95a5bed5f6bdcf3e88ec1 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 19 Sep 2018 11:45:48 -0400 Subject: [PATCH] Add OCF file for cinder-backup As part of switching to the upstream implementation of cinder B&R, we need an OCF script to manage the cinder-backup process. Depends-On: I6bec51c7401339f4c71f9558d73389d0c793093d Change-Id: I63a6fd3d112a98449ea22524bb2a83b5db8ce6d1 Story: 2003715 Task: 26375 Signed-off-by: Scott Little --- .../stx-ocf-scripts/src/ocf/cinder-backup | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 openstack/stx-ocf-scripts/src/ocf/cinder-backup diff --git a/openstack/stx-ocf-scripts/src/ocf/cinder-backup b/openstack/stx-ocf-scripts/src/ocf/cinder-backup new file mode 100644 index 00000000..c740eef1 --- /dev/null +++ b/openstack/stx-ocf-scripts/src/ocf/cinder-backup @@ -0,0 +1,413 @@ +#!/bin/sh +# +# +# OpenStack Cinder Backup (cinder-backup) +# +# Description: Manages an OpenStack Backup (cinder-backup) process as an HA resource +# +# Authors: +# Mainly inspired by the cinder volume resource agent written by Sébastien Han +# +# Support: openstack@lists.launchpad.net +# License: Apache Software License (ASL) 2.0 +# +# (c) 2012 hastexo Professional Services GmbH +# +# Copyright (c) 2014 Wind River Systems, Inc. +# +# The right to copy, distribute, modify, or otherwise make use +# of this software may be licensed only pursuant to the terms +# of an applicable Wind River license agreement. +# +# +# See usage() function below for more details ... +# +# OCF instance parameters: +# OCF_RESKEY_binary +# OCF_RESKEY_config +# OCF_RESKEY_user +# OCF_RESKEY_pid +# OCF_RESKEY_amqp_server_port +# OCF_RESKEY_additional_parameters +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs +. /usr/bin/tsconfig + +####################################################################### + +# Fill in some defaults if no values are specified + +OCF_RESKEY_binary_default="cinder-backup" +OCF_RESKEY_config_default="/etc/cinder/cinder.conf" +OCF_RESKEY_user_default="cinder" +OCF_RESKEY_pid_default="$HA_RSCTMP/$OCF_RESOURCE_INSTANCE.pid" +OCF_RESKEY_amqp_server_port_default="5672" + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}} +: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} +: ${OCF_RESKEY_amqp_server_port=${OCF_RESKEY_amqp_server_port_default}} + +####################################################################### + +usage() { + cat < + + +1.0 + + +Resource agent for the OpenStack Cinder Backup (cinder-backup) +May manage a cinder-backup instance or a clone set that +creates a distributed cinder-backup cluster. + +Manages the OpenStack Cinder Backup (cinder-backup) + + + + +Location of the OpenStack Cinder Backup server binary (cinder-backup) + +OpenStack Cinder Backup server binary (cinder-backup) + + + + + +Location of the OpenStack Cinder Backup (cinder-backup) configuration file + +OpenStack Cinder Backup (cinder-backup) config file + + + + + +User running OpenStack Cinder Backup (cinder-backup) + +OpenStack Cinder Backup (cinder-backup) user + + + + + +The pid file to use for this OpenStack Cinder Backup (cinder-backup) instance + +OpenStack Cinder Backup (cinder-backup) pid file + + + + + +The listening port number of the AMQP server. Mandatory to perform a monitor check + +AMQP listening port + + + + + +Additional parameters to pass on to the OpenStack Cinder Backup (cinder-backup) + +Additional parameters for cinder-backup + + + + + + + + + + + + + + +END +} + +####################################################################### +# Functions invoked by resource manager actions + +cinder_backup_validate() { + local rc + + check_binary $OCF_RESKEY_binary + check_binary netstat + + # A config file on shared storage that is not available + # during probes is OK. + if [ ! -f $OCF_RESKEY_config ]; then + if ! ocf_is_probe; then + ocf_log err "Config $OCF_RESKEY_config doesn't exist" + return $OCF_ERR_INSTALLED + fi + ocf_log_warn "Config $OCF_RESKEY_config not available during a probe" + fi + + getent passwd $OCF_RESKEY_user >/dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "User $OCF_RESKEY_user doesn't exist" + return $OCF_ERR_INSTALLED + fi + + true +} + +cinder_backup_status() { + local pid + local rc + + if [ ! -f $OCF_RESKEY_pid ]; then + ocf_log info "OpenStack Cinder Backup (cinder-backup) is not running" + return $OCF_NOT_RUNNING + else + pid=`cat $OCF_RESKEY_pid` + fi + + ocf_run -warn kill -s 0 $pid + rc=$? + if [ $rc -eq 0 ]; then + return $OCF_SUCCESS + else + ocf_log info "Old PID file found, but OpenStack Cinder Backup (cinder-backup) is not running" + rm -f $OCF_RESKEY_pid + return $OCF_NOT_RUNNING + fi +} + +cinder_backup_get_service_status() { + local status_file=${VOLATILE_PATH}/cinder-backup.down-after + if [[ -f "${status_file}" ]]; then + echo "exists=True" + if [[ "$(date -u +'%Y-%m-%d %H:%M:%S.%N')" < "$(cat ${status_file})" ]]; then + echo "is_up=True" + else + echo "is_up=False" + fi + return 0 + else + echo "exists=False" + echo "is_up=False" + return 1 + fi +} + +cinder_backup_monitor() { + local rc + local pid + local backup_amqp_check + local check_service_status=$1; shift + + cinder_backup_status + rc=$? + + # If status returned anything but success, return that immediately + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + + # Grab cinder-backup PID + pid=`cat $OCF_RESKEY_pid` + + # Check the connections according to the PID + # We are sure to hit the backup process and not other cinder process with the same connection + # behavior (for example nova-cert) + backup_amqp_check=`netstat -punt | grep -s "$OCF_RESKEY_amqp_server_port" | grep -s "$pid" | grep -qs "ESTABLISHED"` + rc=$? + if [ $rc -ne 0 ]; then + ocf_log info "Cinder Backup is not connected to the AMQP server: $rc" + return $OCF_NOT_RUNNING + fi + + # Is this needed? + if [ $check_service_status == "check-service-status" ]; then + local retries_left + local retry_interval + + retries_left=3 + retry_interval=3 + while [ $retries_left -gt 0 ]; do + retries_left=`expr $retries_left - 1` + status=$(cinder_backup_get_service_status) + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "Unable to get Cinder Backup status" + if [ $retries_left -gt 0 ]; then + sleep $retry_interval + continue + else + return $OCF_ERR_GENERIC + fi + fi + + local exists + local is_up + eval $status + + if [ "$exists" == "True" ] && [ "$is_up" == "False" ]; then + ocf_log err "Cinder Backup service status is down" + if [ $retries_left -gt 0 ]; then + sleep $retry_interval + continue + else + ocf_log info "Trigger Cinder Backup guru meditation report" + ocf_run kill -s USR2 $pid + return $OCF_ERR_GENERIC + fi + fi + + break + done + fi + + ocf_log debug "OpenStack Cinder Backup (cinder-backup) monitor succeeded" + return $OCF_SUCCESS +} + +cinder_backup_start() { + local rc + + cinder_backup_status + rc=$? + if [ $rc -eq $OCF_SUCCESS ]; then + ocf_log info "OpenStack Cinder Backup (cinder-backup) already running" + return $OCF_SUCCESS + fi + + # run the actual cinder-backup daemon. Don't use ocf_run as we're sending the tool's output + # straight to /dev/null anyway and using ocf_run would break stdout-redirection here. + su ${OCF_RESKEY_user} -s /bin/sh -c "${OCF_RESKEY_binary} --config-file=$OCF_RESKEY_config \ + $OCF_RESKEY_additional_parameters"' >> /dev/null 2>&1 & echo $!' > $OCF_RESKEY_pid + + # Spin waiting for the server to come up. + # Let the CRM/LRM time us out if required + while true; do + cinder_backup_monitor + rc=$? + [ $rc -eq $OCF_SUCCESS ] && break + if [ $rc -ne $OCF_NOT_RUNNING ]; then + ocf_log err "OpenStack Cinder Backup (cinder-backup) start failed" + exit $OCF_ERR_GENERIC + fi + sleep 1 + done + + ocf_log info "OpenStack Cinder Backup (cinder-backup) started" + return $OCF_SUCCESS +} + +cinder_backup_confirm_stop() { + local my_bin + local my_processes + + my_binary=`which ${OCF_RESKEY_binary}` + my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"` + + if [ -n "${my_processes}" ] + then + ocf_log info "About to SIGKILL the following: ${my_processes}" + pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)" + fi +} + +cinder_backup_stop() { + local rc + local pid + + cinder_backup_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + ocf_log info "OpenStack Cinder Backup (cinder-backup) already stopped" + cinder_backup_confirm_stop + return $OCF_SUCCESS + fi + + # Try SIGTERM + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s TERM $pid + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "OpenStack Cinder Backup (cinder-backup) couldn't be stopped" + cinder_backup_confirm_stop + exit $OCF_ERR_GENERIC + fi + + # stop waiting + shutdown_timeout=15 + if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then + shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5)) + fi + count=0 + while [ $count -lt $shutdown_timeout ]; do + cinder_backup_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + break + fi + count=`expr $count + 1` + sleep 1 + ocf_log debug "OpenStack Cinder backup (cinder-backup) still hasn't stopped yet. Waiting ..." + done + + cinder_backup_status + rc=$? + if [ $rc -ne $OCF_NOT_RUNNING ]; then + # SIGTERM didn't help either, try SIGKILL + ocf_log info "OpenStack Cinder Backup (cinder-backup) failed to stop after ${shutdown_timeout}s \ + using SIGTERM. Trying SIGKILL ..." + ocf_run kill -s KILL -$pid + fi + cinder_backup_confirm_stop + + ocf_log info "OpenStack Cinder Backup (cinder-backup) stopped" + + rm -f $OCF_RESKEY_pid + + return $OCF_SUCCESS +} + +####################################################################### + +case "$1" in + meta-data) meta_data + exit $OCF_SUCCESS;; + usage|help) usage + exit $OCF_SUCCESS;; +esac + +# Anything except meta-data and help must pass validation +cinder_backup_validate || exit $? + +# What kind of method was invoked? +case "$1" in + start) cinder_backup_start;; + stop) cinder_backup_stop;; + status) cinder_backup_status;; + monitor) cinder_backup_monitor "check-service-status";; + validate-all) ;; + *) usage + exit $OCF_ERR_UNIMPLEMENTED;; +esac