From af0a86e357bd08ecbc533dece24601afcf774581 Mon Sep 17 00:00:00 2001 From: Heitor Matsui Date: Thu, 22 Feb 2024 11:13:59 -0300 Subject: [PATCH] Improve logging for deploy start shell scripts This commit improves logging during deploy start by: 1. Creating a common module to be sourced by shell scripts to load general-use functions, thus reducing code duplication between the scripts 2. Replacing plain "echo" commands on the scripts by logging functions present on the common module 3. Adding timestamps to the log messages 4. Centralizing all scripts logs into software.log, favouring the troubleshooting, now that log lines contain timestamps and the process/script that generated them This commit also deletes the ostree_mounts.yaml file since it would be used by apt-ostree integration, which was dropped. Test Plan PASS: run deploy start successfully and verify that deploy start log messages are logged with the expected format Story: 2010676 Task: 49607 Change-Id: I0bdebde8147faa5b29a642e35bfaf26e9862ed0a Signed-off-by: Heitor Matsui --- software/debian/deb_folder/rules | 4 +- software/scripts/chroot_mounts.sh | 25 +++++++--- .../scripts/create_postgresql_database.sh | 19 +++++-- software/scripts/deploy-cleanup | 50 ++++++++++--------- software/scripts/ostree_mounts.yaml | 10 ---- software/scripts/shell-utils | 29 +++++++++++ software/scripts/software-deploy-start | 46 ++++++++++++----- 7 files changed, 123 insertions(+), 60 deletions(-) delete mode 100644 software/scripts/ostree_mounts.yaml create mode 100644 software/scripts/shell-utils diff --git a/software/debian/deb_folder/rules b/software/debian/deb_folder/rules index f16f93f0..158dfa70 100755 --- a/software/debian/deb_folder/rules +++ b/software/debian/deb_folder/rules @@ -74,8 +74,6 @@ override_dh_install: ${ROOT}/etc/software/${METADATA_FILE} install -m 755 scripts/chroot_mounts.sh \ ${ROOT}/usr/sbin/software-deploy/chroot_mounts.sh - install -m 444 scripts/ostree_mounts.yaml \ - ${ROOT}/etc/software/ostree_mounts.yaml install -m 755 scripts/create_postgresql_database.sh \ ${ROOT}/usr/sbin/software-deploy/create_postgresql_database.sh install -m 755 scripts/usm_load_import \ @@ -84,4 +82,6 @@ override_dh_install: ${ROOT}/usr/sbin/software-deploy/sync-controllers-feed install -m 755 scripts/deploy-cleanup \ ${ROOT}/usr/sbin/software-deploy/deploy-cleanup + install -m 755 scripts/shell-utils \ + ${ROOT}/usr/sbin/software-deploy/shell-utils dh_install diff --git a/software/scripts/chroot_mounts.sh b/software/scripts/chroot_mounts.sh index 74144b00..b4544204 100644 --- a/software/scripts/chroot_mounts.sh +++ b/software/scripts/chroot_mounts.sh @@ -5,6 +5,15 @@ # SPDX-License-Identifier: Apache-2.0 # +script_dir=$(dirname $0) +shell_utils=${script_dir}/shell-utils +if [ -f $shell_utils ]; then + source $shell_utils +else + echo "ERROR: ${shell_utils} module not found." + exit 1 +fi + DEV_PATH=/dev PLATFORM_PATH=/opt/platform RABBIT_PATH=/var/lib/rabbitmq @@ -34,8 +43,8 @@ handle_error() { local exit_code="$1" local error_message="$2" - echo "Error: $error_message" >&2 - echo "Please check the error details and take appropriate action for recovery." >&2 + log "$error_message" >&2 + log "Please check the error details and take appropriate action for recovery." >&2 # attempt to unmount if there were successful mounts before the error umount_all @@ -50,7 +59,7 @@ mount_all() { src=${src_dst[0]} dst=${src_dst[1]} - echo "mount --bind ${src} ${dst}" + info "Bind mounting ${src} -> ${dst}" sudo mkdir -p ${dst} sudo mount --bind ${src} ${dst} || handle_error 1 "Failed to bind mount ${src} to ${dst}" done @@ -65,12 +74,12 @@ umount_all() { src=${src_dst[0]} dst=${src_dst[1]} - echo "sudo umount $dst" + info "Unmounting $dst" umount_output=$(sudo umount $dst 2>&1) if [ $? -ne 0 ]; then # ignore messages that are not harmful if [[ ! $umount_output =~ ("not mounted"|"no mount point specified") ]]; then - echo $umount_output + error "$umount_output" rc=1 fi fi @@ -94,16 +103,16 @@ check_all() { fi done if [[ ${#mounted[@]} -gt 0 ]]; then - echo "Mounted mount points:" + info "Mounted mount points:" for mnt in ${mounted[@]}; do - echo $mnt + info $mnt done fi return $rc } if [ -z "$1" ]; then - echo "Error: OSTree deployment branch parameter is missing." + error "OSTree deployment branch parameter is missing." exit 1 fi diff --git a/software/scripts/create_postgresql_database.sh b/software/scripts/create_postgresql_database.sh index 1b1ae13c..a4d833e8 100644 --- a/software/scripts/create_postgresql_database.sh +++ b/software/scripts/create_postgresql_database.sh @@ -1,13 +1,22 @@ #!/bin/bash # -# Copyright (c) 2023 Wind River Systems, Inc. +# Copyright (c) 2023-2024 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # # # This script is used create a 2nd instance of postgres on a DX upgrade. # It needs the port number as parameter and it should be different from the default. +# +script_dir=$(dirname $0) +shell_utils=${script_dir}/shell-utils +if [ -f $shell_utils ]; then + source $shell_utils +else + echo "ERROR: ${shell_utils} module not found." + exit 1 +fi DEFAULT_POSTGRESQL_PORT=5432 POSTGRESQL_PATH=/var/lib/postgresql @@ -16,7 +25,7 @@ POSTGRESQL_RUNTIME=/var/run/postgresql INFO_FILE=/etc/build.info if [ -z "$1" ]; then - echo "Error: Port parameter is missing." + error "'Port' parameter is missing." exit 1 fi @@ -24,7 +33,7 @@ PORT="$1" # Prevent issues with the default postgres port if [ "$PORT" -eq "$DEFAULT_POSTGRESQL_PORT" ]; then - echo "Error: Port number should be different from the default." + error "Port number should be different from the default." exit 1 fi @@ -32,8 +41,8 @@ cleanup_and_exit() { local exit_code="$1" local error_message="$2" - echo "Error: $error_message" >&2 - echo "Please check the error details and take appropriate action for recovery." >&2 + error "$error_message" >&2 + error "Please check the error details and take appropriate action for recovery." >&2 exit "$exit_code" } diff --git a/software/scripts/deploy-cleanup b/software/scripts/deploy-cleanup index a8511bc1..3a4d5912 100644 --- a/software/scripts/deploy-cleanup +++ b/software/scripts/deploy-cleanup @@ -15,10 +15,14 @@ # automatic cleanup process fails. # -log() { - type=$1 - echo "[$(date --iso-8601=seconds)] $type: $2" -} +script_dir=$(dirname $0) +shell_utils=${script_dir}/shell-utils +if [ -f $shell_utils ]; then + source $shell_utils +else + echo "ERROR: ${shell_utils} module not found." + exit 1 +fi stop_database() { local rootdir=$1 @@ -27,7 +31,7 @@ stop_database() { # attempt to stop temporary database if still up tmp_db_dir=${rootdir}/var/lib/postgresql/${to_ver} - log "INFO" "Attempting to stop the temporary database in ${tmp_db_dir}..." + info "Attempting to stop the temporary database in ${tmp_db_dir}..." if [ -d $tmp_db_dir ]; then lsof $tmp_db_dir if [ $? -eq 0 ]; then @@ -35,15 +39,15 @@ stop_database() { sudo -u postgres ${tmp_db_bin_dir}/pg_ctl -D ${tmp_db_dir} stop if [ $? -ne 0 ]; then rc=1 - log "ERROR" "Error stopping database." + error "Error stopping database." else - log "INFO" "Success stopping database." + info "Success stopping database." fi else - log "INFO" "Database is not running." + info "Database is not running." fi else - log "WARN" "No database found in the specified directory." + warning "No database found in the specified directory." fi return $rc } @@ -52,20 +56,20 @@ unmount_filesystems() { local rootdir=$1 local rc=0 - log "INFO" "Attempting to unmount filesystems under ${rootdir}..." + info "Attempting to unmount filesystems under ${rootdir}..." k8s_umount_output=$(sudo umount ${rootdir}/usr/local/kubernetes/current 2>&1) if [ $? -ne 0 ]; then if [[ ! $k8s_umount_output =~ "not mounted" ]]; then rc=1 - log "ERROR" $k8s_umount_output + error $k8s_umount_output fi fi sudo ${rootdir}/usr/sbin/software-deploy/chroot_mounts.sh ${rootdir} -u if [ $? -ne 0 ]; then rc=1 - log "ERROR" "Error unmounting filesystems." + error "Error unmounting filesystems." else - log "INFO" "Success unmounting filesystems." + info "Success unmounting filesystems." fi return $rc } @@ -75,14 +79,14 @@ remove_temp_directories() { local rootdir=$2 local rc=0 - log "INFO" "Attempting to remove temporary deployment directories [${repo}, ${rootdir}]..." + info "Attempting to remove temporary deployment directories [${repo}, ${rootdir}]..." sudo ${rootdir}/usr/sbin/software-deploy/chroot_mounts.sh ${rootdir} -c if [ $? -ne 0 ]; then rc=1 - log "ERROR" "Some mount points are still mounted, cannot proceed with the cleanup." + error "Some mount points are still mounted, cannot proceed with the cleanup." else rm -rf $repo $rootdir - log "INFO" "Temporary deployment directories removed successfully." + info "Temporary deployment directories removed successfully." fi return $rc } @@ -90,7 +94,7 @@ remove_temp_directories() { # script usage if [ $# -ne 3 ]; then echo - echo "usage: deploy-start-cleanup " + echo "usage: deploy-cleanup " echo -e "\nParameters" echo "==========" echo "tmp_ostree_repo_dir: temporary ostree repo directory (example: /sysroot/upgrade/ostree_repo)" @@ -111,14 +115,14 @@ action=$3 # basic checks for dir in $repo $rootdir; do if [ ! -d $dir ]; then - log "ERROR" "Specified directory $dir does not exist, cannot proceed with cleanup." + error "Specified directory $dir does not exist, cannot proceed with cleanup." exit 1 fi done target_build_info=${rootdir}/usr/etc/build.info if [ ! -f $target_build_info ]; then - log "ERROR" "Cannot get target release build-info information, cannot proceed with cleanup." + error "Cannot get target release build-info information, cannot proceed with cleanup." exit 1 fi @@ -126,7 +130,7 @@ fi to_ver=$(cat $target_build_info | grep SW_VERSION | cut -d'"' -f2) # main -log "INFO" "Starting cleanup for staging directories [${repo}, ${rootdir}]..." +info "Starting cleanup for staging directories [${repo}, ${rootdir}]..." case $action in "db") stop_database $rootdir $to_ver @@ -143,13 +147,13 @@ case $action in remove_temp_directories $repo $rootdir ;; *) - log "ERROR" "Invalid action specified: ${action}" + error "Invalid action specified: ${action}" ;; esac rc=$? if [ $rc -ne 0 ]; then - log "ERROR" "Error cleaning up [${repo}, ${rootdir}], please check the logs, take manual actions and retry the script." + error "Error cleaning up [${repo}, ${rootdir}], please check the logs, take manual actions and retry the script." fi -log "INFO" "Cleanup script ended." +info "Cleanup script ended." exit $rc diff --git a/software/scripts/ostree_mounts.yaml b/software/scripts/ostree_mounts.yaml deleted file mode 100644 index 52b801e9..00000000 --- a/software/scripts/ostree_mounts.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- "{{ OSTREE_DEPLOYMENT_BRANCH }}/usr/etc:{{ OSTREE_DEPLOYMENT_BRANCH }}/etc" -- "/dev:{{ OSTREE_DEPLOYMENT_BRANCH }}/dev" -- "/proc:{{ OSTREE_DEPLOYMENT_BRANCH }}/proc" -- "/opt/platform:{{ OSTREE_DEPLOYMENT_BRANCH }}/opt/platform" -- "/var/lib/rabbitmq:{{ OSTREE_DEPLOYMENT_BRANCH }}/var/lib/rabbitmq" -- "/var/log:{{ OSTREE_DEPLOYMENT_BRANCH }}/var/log" -- > - "/var/lib/postgresql/upgrade: - {{ OSTREE_DEPLOYMENT_BRANCH }}/var/lib/postgresql/upgrade" diff --git a/software/scripts/shell-utils b/software/scripts/shell-utils new file mode 100644 index 00000000..fd564723 --- /dev/null +++ b/software/scripts/shell-utils @@ -0,0 +1,29 @@ +# !/bin/bash +# +# Copyright (c) 2024 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# File containing common shell functions that +# can be sourced and used by other shell scripts. +# + +log() +{ + script_name=$(basename $0) + log_type=$1 + shift + echo "$(date -Iseconds | cut -d'+' -f1): ${script_name}[${$}]: ${log_type}: ${@}" +} + +info() { + log "INFO" $@ +} + +warning() { + log "WARNING" $@ +} + +error() { + log "ERROR" $@ >&2 +} diff --git a/software/scripts/software-deploy-start b/software/scripts/software-deploy-start index 827bf11b..20deec4d 100755 --- a/software/scripts/software-deploy-start +++ b/software/scripts/software-deploy-start @@ -13,10 +13,17 @@ # 5. perform data migration # -# TODO: centralize USM upgrade scripts output into one single log -exec > /var/log/deploy_start.log 2>&1 +exec 2>&1 >> /var/log/software.log + +script_dir=$(dirname $0) +shell_utils=${script_dir}/shell-utils +if [ -f $shell_utils ]; then + source $shell_utils +else + echo "ERROR: ${shell_utils} module not found." + exit 1 +fi -exec_path=$(dirname $0) usage() { echo "usage: $0 from_ver to_ver k8s_ver postgresql_port feed [commit_id|latest_commit]" @@ -63,10 +70,10 @@ handle_error() { local error_message="$2" local state="start-failed" - echo "Error: ${error_message}" >&2 - echo "Please check the error details and take appropriate action for recovery." >&2 + error "${error_message}" >&2 + error "Please check the error details and take appropriate action for recovery." >&2 - echo "Update deploy state ${state}." >&2 + error "Update deploy state ${state}." >&2 deploy_update_state ${state} # cleanup before exiting @@ -77,13 +84,14 @@ handle_error() { for dir in $rootdir $repo; do if [ -e ${dir} ]; then - echo "${dir} already exists. Please ensure to clean up environment to continue." >&2 + error "${dir} already exists. Please ensure to clean up environment to continue." >&2 exit 1 fi done sudo mkdir ${repo} -p +info "Initializing ostree repo..." sudo ostree --repo=${repo} init --mode=archive || handle_error $? "Failed to init repo" sudo ostree --repo=${repo} remote add ${instbr} ${feed_url} --no-gpg-verify || handle_error $? "Failed to remote add repo" sudo ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr} || handle_error $? "Failed to pull repo" @@ -92,23 +100,27 @@ sudo ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr} || handl if [ -z ${commit_id} ]; then # get commit id, only latest for now commit_id=$(ostree rev-parse --repo=${repo} ${instbr}) - echo "latest commit id ${commit_id}" + info "Latest commit id ${commit_id}" if [ -z "${commit_id}" ]; then handle_error 1 "Failed to retrieve commit id" fi fi -echo "checkout commit id ${commit_id}" +info "Checkout commit id ${commit_id}" +info "Checking out ostree repo..." sudo ostree --repo=${repo} checkout ${commit_id} ${rootdir} || handle_error $? "Failed to checkout ${commit_id}" # create proper mounts on deploy file system +info "Creating mount points..." sudo ${rootdir}/usr/sbin/software-deploy/chroot_mounts.sh ${rootdir} || handle_error $? "Failed to mount required mount points" sudo mount --bind ${rootdir}/usr/local/kubernetes/${k8s_ver} ${rootdir}/usr/local/kubernetes/current sudo cp /etc/kubernetes/admin.conf ${rootdir}/etc/kubernetes/ +info "Mount points created successfully" -DATA_PREP_SCRIPT="${exec_path}/prep-data-migration" +info "Preparing for data migration..." +DATA_PREP_SCRIPT="${script_dir}/prep-data-migration" # OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_PROJECT_NAME, OS_USER_DOMAIN_NAME, # OS_PROJECT_DOMAIN_NAME, OS_REGION_NAME are in env variables. cmd_line=" --rootdir=${rootdir} --from_release=${from_ver} --to_release=${to_ver}" @@ -118,17 +130,27 @@ cmd_line+=" --project_domain_name=${OS_PROJECT_DOMAIN_NAME} --region_name=${OS_R prep_cmd="${DATA_PREP_SCRIPT} ${cmd_line}" ${prep_cmd} || handle_error $? "Failed to extract data for migration" +info "Data migration preparations complete." +info "Creating temporary database..." sudo chroot ${rootdir} /usr/sbin/software-deploy/create_postgresql_database.sh ${port} || handle_error $? "Failed to start 2nd instance of postgresql" -sudo chroot ${rootdir} /usr/bin/software-migrate ${from_ver} ${to_ver} ${port} || handle_error $? "Failed to migrate data" +info "Database creation complete." +info "Starting data migration..." +sudo chroot ${rootdir} /usr/bin/software-migrate ${from_ver} ${to_ver} ${port} || handle_error $? "Failed to migrate data" +info "Data migration completed." + +info "Syncing feed between controllers..." SYNC_CONTROLLERS_SCRIPT="/usr/sbin/software-deploy/sync-controllers-feed" sync_controllers_cmd="${SYNC_CONTROLLERS_SCRIPT} ${cmd_line} --feed=${feed}" ${sync_controllers_cmd} || handle_error $? "Failed to sync feeds" +info "Feed sync complete." state="start-done" deploy_update_state $state -echo "Update deploy state ${state}." +info "Update deploy state ${state}." # cleanup after successful data migration +info "Starting cleanup..." deploy_cleanup +info "Cleanup complete."