From 7cd1b65d89ab7bece8f296bf6f4131a61aca0f2f Mon Sep 17 00:00:00 2001
From: Scott Little <scott.little@windriver.com>
Date: Wed, 27 Sep 2023 11:03:41 -0400
Subject: [PATCH] Add reposync retries

The reposync stage of the download_mirror.sh fails
on a regulare basis due to network issues.

Add a retry loop to the reposync step.

Change-Id: I174046f4875882f1bd1ad074bfde6607480032a9
Signed-off-by: Scott Little <scott.little@windriver.com>
---
 centos-mirror-tools/dl_lower_layer_rpms.sh |  4 +-
 centos-mirror-tools/utils.sh               | 97 ++++++++++++++++++++++
 doc/source/conf.py                         |  2 +-
 3 files changed, 100 insertions(+), 3 deletions(-)

diff --git a/centos-mirror-tools/dl_lower_layer_rpms.sh b/centos-mirror-tools/dl_lower_layer_rpms.sh
index af353011..c8ddeb57 100755
--- a/centos-mirror-tools/dl_lower_layer_rpms.sh
+++ b/centos-mirror-tools/dl_lower_layer_rpms.sh
@@ -404,9 +404,9 @@ dl_repo () {
     fi
 
     # Sync the repo's rpms
-    CMD="reposync --tempcache --norepopath -l --config='${YUM_CONF_TMP}' --repoid=$REPOID --download_path='${DOWNLOAD_PATH_NEW}'"
+    CMD="reposync --tempcache --norepopath -l --config=${YUM_CONF_TMP} --repoid=$REPOID --download_path=${DOWNLOAD_PATH_NEW}"
     echo "$CMD"
-    eval $CMD
+    with_retries --delay 60 3 $CMD
     if [ $? -ne 0 ]; then
         echo "Error: $CMD"
         return 1
diff --git a/centos-mirror-tools/utils.sh b/centos-mirror-tools/utils.sh
index 20cff687..e24e6e4f 100644
--- a/centos-mirror-tools/utils.sh
+++ b/centos-mirror-tools/utils.sh
@@ -286,3 +286,100 @@ check_sha256sum() {
 
     sha256sum "${file}" | cut -d' ' -f1 | grep -q -F -x "${sha256sum}"
 }
+
+#
+# Echo to stderr
+#    echo_stderr [any text you want]
+#
+echo_stderr ()
+{
+    echo "$@" >&2
+}
+
+#
+# Function to call a command, with support for retries
+#
+#   with_retries [<options>] <retries> <cmd> [<cmd_args>...]
+#
+#   Options:
+#       -d <secs> | --delay <secs>
+#            Wait given number of seconds between retries
+#       -t <secs> | --timeout <secs>
+#            Each iteration of the command runs under a timeout
+#       -k <secs> | --kill-timeout <secs>
+#            Each iteration of the command is killed violently
+#            if it doesn't exit voluntarily within the set time
+#            after the initial timeout signal.
+#
+function with_retries {
+    local delay=5
+    local max_time=0
+    local kill_time=0
+    local to_cmd=""
+
+    while [ $1 != "" ]; do
+        case "$1" in
+            -d | --delay)
+                delay=$2
+                shift 2
+                ;;
+            -t | --timeout)
+                max_time=$2
+                shift 2
+                ;;
+            -k | --kill-timeout)
+                kill_time=$2
+                shift 2
+                ;;
+            *)
+                break
+                ;;
+        esac
+    done
+
+    local max_attempts=$1
+    local cmd=$2
+    shift 2
+
+    if [ ${max_time} -gt 0 ]; then
+        to_cmd="timeout "
+        if [ ${kill_time} -gt 0 ]; then
+            to_cmd+="--kill-after=${kill_time} "
+        fi
+        to_cmd+="${max_time} "
+    fi
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+
+    local -i attempt=0
+    local rc=0
+
+    while :; do
+        let attempt++
+
+        echo_stderr "Running: ${cmd} $@"
+        ${to_cmd} ${cmd} "$@"
+        rc=$?
+        if [ $rc -eq 0 ]; then
+            return 0
+        fi
+
+        if [ $rc -eq 124 ]; then
+            echo_stderr "Command (${cmd}) timed out, attempt ${attempt} of ${max_attempts}."
+        elif [ $rc -eq 137 ]; then
+            echo_stderr "Command (${cmd}) timed out and killed, attempt ${attempt} of ${max_attempts}."
+        else
+            echo_stderr "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        fi
+
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            echo_stderr "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo_stderr "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 16c03516..4c87774f 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -62,7 +62,7 @@ openstackdocs_auto_name = False
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.