Merge "Include cleanup mechanism for deploy start"
This commit is contained in:
Zuul
Gerrit Code Review
commit
18192b93d7
|
|
@ -82,4 +82,6 @@ override_dh_install:
|
|||
${ROOT}/usr/sbin/software-deploy/usm_load_import
|
||||
install -m 755 scripts/sync-controllers-feed \
|
||||
${ROOT}/usr/sbin/software-deploy/sync-controllers-feed
|
||||
install -m 755 scripts/deploy-cleanup \
|
||||
${ROOT}/usr/sbin/software-deploy/deploy-cleanup
|
||||
dh_install
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
# Copyright (c) 2023-2024 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -16,17 +16,20 @@ ETC_PATH=/etc
|
|||
PROC_PATH=/proc
|
||||
LOG_PATH=/var/log
|
||||
|
||||
mount_points=(
|
||||
"$DEV_PATH"
|
||||
"$PLATFORM_PATH"
|
||||
"$RABBIT_PATH"
|
||||
"$POSTGRES_PATH"
|
||||
"$PROC_PATH"
|
||||
"$LOG_PATH"
|
||||
)
|
||||
|
||||
OSTREE_DEPLOYMENT_BRANCH="$1"
|
||||
|
||||
# src:dst
|
||||
mount_points=(
|
||||
"${DEV_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${DEV_PATH}"
|
||||
"${PLATFORM_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${PLATFORM_PATH}"
|
||||
"${RABBIT_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${RABBIT_PATH}"
|
||||
"${POSTGRES_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${POSTGRES_PATH}"
|
||||
"${PROC_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${PROC_PATH}"
|
||||
"${LOG_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${LOG_PATH}"
|
||||
"${OSTREE_DEPLOYMENT_BRANCH}/${USR_PATH}/${ETC_PATH}:${OSTREE_DEPLOYMENT_BRANCH}/${ETC_PATH}"
|
||||
"${PLATFORM_CONF_PATH}:${TMP_PATH}/${PLATFORM_CONF_PATH}"
|
||||
)
|
||||
|
||||
handle_error() {
|
||||
local exit_code="$1"
|
||||
local error_message="$2"
|
||||
|
|
@ -34,55 +37,87 @@ handle_error() {
|
|||
echo "Error: $error_message" >&2
|
||||
echo "Please check the error details and take appropriate action for recovery." >&2
|
||||
|
||||
# attempt to unmount if there were successful mounts before the error
|
||||
umount_all
|
||||
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
mount_all() {
|
||||
for dir in "${mount_points[@]}"; do
|
||||
target_dir=$OSTREE_DEPLOYMENT_BRANCH$dir
|
||||
if [ ! -d "$target_dir" ]; then
|
||||
sudo mkdir -p "$target_dir"
|
||||
fi
|
||||
echo "mount --bind $dir $target_dir"
|
||||
sudo mount --bind "$dir" "$target_dir" ||
|
||||
handle_error 1 "Failed to bind mount $dir to $target_dir"
|
||||
for mnt in ${mount_points[@]}; do
|
||||
# split source and destination, squeeze multiple '/'
|
||||
src_dst=($(echo ${mnt//:/ } | tr -s "/"))
|
||||
src=${src_dst[0]}
|
||||
dst=${src_dst[1]}
|
||||
|
||||
echo "mount --bind ${src} ${dst}"
|
||||
sudo mkdir -p ${dst}
|
||||
sudo mount --bind ${src} ${dst} || handle_error 1 "Failed to bind mount ${src} to ${dst}"
|
||||
done
|
||||
|
||||
if [ ! -d "$TMP_PATH$PLATFORM_CONF_PATH" ]; then
|
||||
mkdir -p "$TMP_PATH$PLATFORM_CONF_PATH"
|
||||
fi
|
||||
|
||||
echo "mkdir $OSTREE_DEPLOYMENT_BRANCH/etc -p"
|
||||
sudo mkdir $OSTREE_DEPLOYMENT_BRANCH/etc -p
|
||||
echo "mount --bind $OSTREE_DEPLOYMENT_BRANCH$USR_PATH$ETC_PATH $OSTREE_DEPLOYMENT_BRANCH$ETC_PATH"
|
||||
sudo mount --bind "$OSTREE_DEPLOYMENT_BRANCH$USR_PATH$ETC_PATH" "$OSTREE_DEPLOYMENT_BRANCH$ETC_PATH" ||
|
||||
handle_error 1 "Failed to bind mount $OSTREE_DEPLOYMENT_BRANCH$USR_PATH$ETC_PATH to
|
||||
$OSTREE_DEPLOYMENT_BRANCH$ETC_PATH"
|
||||
|
||||
echo "mount --bind $PLATFORM_CONF_PATH $TMP_PATH$PLATFORM_CONF_PATH"
|
||||
sudo mount --bind "$PLATFORM_CONF_PATH" "$TMP_PATH$PLATFORM_CONF_PATH" ||
|
||||
handle_error 1 "Failed to bind mount $PLATFORM_CONF_PATH to $TMP_PATH$PLATFORM_CONF_PATH"
|
||||
return 0
|
||||
}
|
||||
|
||||
umount_all() {
|
||||
for dir in "${mount_points[@]}"; do
|
||||
target_dir=$OSTREE_DEPLOYMENT_BRANCH$dir
|
||||
echo "sudo umount $target_dir"
|
||||
sudo umount $target_dir
|
||||
done
|
||||
local rc=0
|
||||
for mnt in ${mount_points[@]}; do
|
||||
# split source and destination, squeeze multiple '/'
|
||||
src_dst=($(echo ${mnt//:/ } | tr -s "/"))
|
||||
src=${src_dst[0]}
|
||||
dst=${src_dst[1]}
|
||||
|
||||
sudo umount $OSTREE_DEPLOYMENT_BRANCH$ETC_PATH
|
||||
sudo umount $TMP_PATH$PLATFORM_CONF_PATH
|
||||
echo "sudo umount $dst"
|
||||
umount_output=$(sudo umount $dst 2>&1)
|
||||
if [ $? -ne 0 ]; then
|
||||
# ignore messages that are not harmful
|
||||
if [[ ! $umount_output =~ ("not mounted"|"no mount point specified") ]]; then
|
||||
echo $umount_output
|
||||
rc=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
check_all() {
|
||||
local rc=0
|
||||
local mounted=()
|
||||
for mnt in ${mount_points[@]}; do
|
||||
# split source and destination, squeeze multiple '/'
|
||||
src_dst=($(echo ${mnt//:/ } | tr -s "/"))
|
||||
src=${src_dst[0]}
|
||||
dst=${src_dst[1]}
|
||||
|
||||
mount | grep -w $dst 2>&1 > /dev/null
|
||||
if [[ $? -eq 0 ]]; then
|
||||
rc=1
|
||||
mounted+=(${dst})
|
||||
fi
|
||||
done
|
||||
if [[ ${#mounted[@]} -gt 0 ]]; then
|
||||
echo "Mounted mount points:"
|
||||
for mnt in ${mounted[@]}; do
|
||||
echo $mnt
|
||||
done
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Error: Ostree deployment branch parameter is missing."
|
||||
echo "Error: OSTree deployment branch parameter is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $# -eq 2 && $2 == "-u" ]]; then
|
||||
umount_all
|
||||
if [[ $# -eq 2 ]]; then
|
||||
if [[ $2 == "-u" ]]; then
|
||||
umount_all
|
||||
rc=$?
|
||||
elif [[ $2 == "-c" ]]; then
|
||||
check_all
|
||||
rc=$?
|
||||
fi
|
||||
else
|
||||
mount_all
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
exit $rc
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2024 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This script removes artifacts created by the deploy start script:
|
||||
# 1. Stop temporary database created for data migrations
|
||||
# 2. Unmount the bind mounts used by the data migration process
|
||||
# 3. Remove the staging deployment directory created by checking out TO release ostree branch
|
||||
#
|
||||
# It can be used either by another script (e.g. software-deploy-start) to automatically
|
||||
# cleanup the environment after both success/failure paths of the data migration process,
|
||||
# or can be used by a system administrator to manually cleanup the environment if the
|
||||
# automatic cleanup process fails.
|
||||
#
|
||||
|
||||
log() {
|
||||
type=$1
|
||||
echo "[$(date --iso-8601=seconds)] $type: $2"
|
||||
}
|
||||
|
||||
stop_database() {
|
||||
local rootdir=$1
|
||||
local to_ver=$2
|
||||
local rc=0
|
||||
|
||||
# attempt to stop temporary database if still up
|
||||
tmp_db_dir=${rootdir}/var/lib/postgresql/${to_ver}
|
||||
log "INFO" "Attempting to stop the temporary database in ${tmp_db_dir}..."
|
||||
if [ -d $tmp_db_dir ]; then
|
||||
lsof $tmp_db_dir
|
||||
if [ $? -eq 0 ]; then
|
||||
tmp_db_bin_dir=$(${rootdir}/usr/bin/pg_config --bindir)
|
||||
sudo -u postgres ${tmp_db_bin_dir}/pg_ctl -D ${tmp_db_dir} stop
|
||||
if [ $? -ne 0 ]; then
|
||||
rc=1
|
||||
log "ERROR" "Error stopping database."
|
||||
else
|
||||
log "INFO" "Success stopping database."
|
||||
fi
|
||||
else
|
||||
log "INFO" "Database is not running."
|
||||
fi
|
||||
else
|
||||
log "WARN" "No database found in the specified directory."
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
unmount_filesystems() {
|
||||
local rootdir=$1
|
||||
local rc=0
|
||||
|
||||
log "INFO" "Attempting to unmount filesystems under ${rootdir}..."
|
||||
k8s_umount_output=$(sudo umount ${rootdir}/usr/local/kubernetes/current 2>&1)
|
||||
if [ $? -ne 0 ]; then
|
||||
if [[ ! $k8s_umount_output =~ "not mounted" ]]; then
|
||||
rc=1
|
||||
log "ERROR" $k8s_umount_output
|
||||
fi
|
||||
fi
|
||||
sudo ${rootdir}/usr/sbin/software-deploy/chroot_mounts.sh ${rootdir} -u
|
||||
if [ $? -ne 0 ]; then
|
||||
rc=1
|
||||
log "ERROR" "Error unmounting filesystems."
|
||||
else
|
||||
log "INFO" "Success unmounting filesystems."
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
remove_temp_directories() {
|
||||
local repo=$1
|
||||
local rootdir=$2
|
||||
local rc=0
|
||||
|
||||
log "INFO" "Attempting to remove temporary deployment directories [${repo}, ${rootdir}]..."
|
||||
sudo ${rootdir}/usr/sbin/software-deploy/chroot_mounts.sh ${rootdir} -c
|
||||
if [ $? -ne 0 ]; then
|
||||
rc=1
|
||||
log "ERROR" "Some mount points are still mounted, cannot proceed with the cleanup."
|
||||
else
|
||||
rm -rf $repo $rootdir
|
||||
log "INFO" "Temporary deployment directories removed successfully."
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# script usage
|
||||
if [ $# -ne 3 ]; then
|
||||
echo
|
||||
echo "usage: deploy-start-cleanup <tmp_ostree_repo_dir> <tmp_root_dir> <action>"
|
||||
echo -e "\nParameters"
|
||||
echo "=========="
|
||||
echo "tmp_ostree_repo_dir: temporary ostree repo directory (example: /sysroot/upgrade/ostree_repo)"
|
||||
echo "tmp_root_dir: temporary ostree checkout root directory (example: /sysroot/upgrade/sysroot)"
|
||||
echo "action: action executed by the script (db|umount|remove|all)"
|
||||
echo "- db: stop temporary database created for target release"
|
||||
echo "- umount: unmount the bind mounts"
|
||||
echo "- remove: delete the target release temporary directory"
|
||||
echo "- all: all of the above, in top-down order"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
repo=$1
|
||||
rootdir=$2
|
||||
action=$3
|
||||
|
||||
# basic checks
|
||||
for dir in $repo $rootdir; do
|
||||
if [ ! -d $dir ]; then
|
||||
log "ERROR" "Specified directory $dir does not exist, cannot proceed with cleanup."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
target_build_info=${rootdir}/usr/etc/build.info
|
||||
if [ ! -f $target_build_info ]; then
|
||||
log "ERROR" "Cannot get target release build-info information, cannot proceed with cleanup."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# set to_ver based on build-info inside rootdir
|
||||
to_ver=$(cat $target_build_info | grep SW_VERSION | cut -d'"' -f2)
|
||||
|
||||
# main
|
||||
log "INFO" "Starting cleanup for staging directories [${repo}, ${rootdir}]..."
|
||||
case $action in
|
||||
"db")
|
||||
stop_database $rootdir $to_ver
|
||||
;;
|
||||
"umount")
|
||||
unmount_filesystems $rootdir
|
||||
;;
|
||||
"remove")
|
||||
remove_temp_directories $repo $rootdir
|
||||
;;
|
||||
"all")
|
||||
stop_database $rootdir $to_ver && \
|
||||
unmount_filesystems $rootdir && \
|
||||
remove_temp_directories $repo $rootdir
|
||||
;;
|
||||
*)
|
||||
log "ERROR" "Invalid action specified: ${action}"
|
||||
;;
|
||||
esac
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
log "ERROR" "Error cleaning up [${repo}, ${rootdir}], please check the logs, take manual actions and retry the script."
|
||||
fi
|
||||
log "INFO" "Cleanup script ended."
|
||||
|
||||
exit $rc
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# !/bin/bash
|
||||
#
|
||||
# Copyright (c) 2023 Wind River Systems, Inc.
|
||||
# Copyright (c) 2023-2024 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -40,10 +40,15 @@ else
|
|||
fi
|
||||
|
||||
k8s_ver=$(echo ${k8s_ver} | sed 's/^v//')
|
||||
rootdir="/sysroot/upgrade/sysroot"
|
||||
repo="/sysroot/upgrade/ostree_repo"
|
||||
staging_dir="/sysroot/upgrade"
|
||||
rootdir=${staging_dir}"/sysroot"
|
||||
repo=${staging_dir}"/ostree_repo"
|
||||
instbr="starlingx"
|
||||
|
||||
deploy_cleanup() {
|
||||
sudo ${rootdir}/usr/sbin/software-deploy/deploy-cleanup ${repo} ${rootdir} all
|
||||
}
|
||||
|
||||
handle_error() {
|
||||
local exit_code="$1"
|
||||
local error_message="$2"
|
||||
|
|
@ -51,18 +56,18 @@ handle_error() {
|
|||
echo "Error: ${error_message}" >&2
|
||||
echo "Please check the error details and take appropriate action for recovery." >&2
|
||||
|
||||
# cleanup before exiting
|
||||
deploy_cleanup
|
||||
|
||||
exit ${exit_code}
|
||||
}
|
||||
|
||||
if [ -e ${rootdir} ]; then
|
||||
echo "${rootdir} already exists. Please ensure to clean up environment to continue." >&2
|
||||
for dir in $rootdir $repo; do
|
||||
if [ -e ${dir} ]; then
|
||||
echo "${dir} already exists. Please ensure to clean up environment to continue." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -e ${repo} ]; then
|
||||
echo "${repo} already exists. Please ensure to clean up environment to continue." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# TODO(bqian) below ostree operations will be replaced by apt-ostree
|
||||
sudo mkdir ${repo} -p
|
||||
|
|
@ -110,3 +115,6 @@ sudo chroot ${rootdir} /usr/bin/software-migrate ${from_ver} ${to_ver} ${port} |
|
|||
SYNC_CONTROLLERS_SCRIPT="/usr/sbin/software-deploy/sync-controllers-feed"
|
||||
sync_controllers_cmd="${SYNC_CONTROLLERS_SCRIPT} ${cmd_line} --feed=${feed}"
|
||||
${sync_controllers_cmd} || handle_error $? "Failed to sync feeds"
|
||||
|
||||
# cleanup after successful data migration
|
||||
deploy_cleanup
|
||||
|
|
|
|||
Loading…
Reference in New Issue