Add reposync retries

The reposync stage of the download_mirror.sh fails
on a regulare basis due to network issues.

Add a retry loop to the reposync step.

Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I164c0f38d3ba261766a119f68c60c620b32a434e

centos-mirror-tools/dl_lower_layer_rpms.sh

@@ -404,9 +404,9 @@ dl_repo () {
     fi
 
     # Sync the repo's rpms
-    CMD="reposync --tempcache --norepopath -l --config='${YUM_CONF_TMP}' --repoid=$REPOID --download_path='${DOWNLOAD_PATH_NEW}'"
+    CMD="reposync --tempcache --norepopath -l --config=${YUM_CONF_TMP} --repoid=$REPOID --download_path=${DOWNLOAD_PATH_NEW}"
     echo "$CMD"
-    eval $CMD
+    with_retries --delay 60 3 $CMD
     if [ $? -ne 0 ]; then
         echo "Error: $CMD"
         return 1

centos-mirror-tools/utils.sh

@@ -280,3 +280,107 @@ get_from() {
     from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
     echo $from
 }
+
+check_sha256sum() {
+    local file="${1}"
+    local sha256sum="${2}"
+
+    sha256sum "${file}" | cut -d' ' -f1 | grep -q -F -x "${sha256sum}"
+}
+
+#
+# Echo to stderr
+#    echo_stderr [any text you want]
+#
+echo_stderr ()
+{
+    echo "$@" >&2
+}
+
+#
+# Function to call a command, with support for retries
+#
+#   with_retries [<options>] <retries> <cmd> [<cmd_args>...]
+#
+#   Options:
+#       -d <secs> | --delay <secs>
+#            Wait given number of seconds between retries
+#       -t <secs> | --timeout <secs>
+#            Each iteration of the command runs under a timeout
+#       -k <secs> | --kill-timeout <secs>
+#            Each iteration of the command is killed violently
+#            if it doesn't exit voluntarily within the set time
+#            after the initial timeout signal.
+#
+function with_retries {
+    local delay=5
+    local max_time=0
+    local kill_time=0
+    local to_cmd=""
+
+    while [ $1 != "" ]; do
+        case "$1" in
+            -d | --delay)
+                delay=$2
+                shift 2
+                ;;
+            -t | --timeout)
+                max_time=$2
+                shift 2
+                ;;
+            -k | --kill-timeout)
+                kill_time=$2
+                shift 2
+                ;;
+            *)
+                break
+                ;;
+        esac
+    done
+
+    local max_attempts=$1
+    local cmd=$2
+    shift 2
+
+    if [ ${max_time} -gt 0 ]; then
+        to_cmd="timeout "
+        if [ ${kill_time} -gt 0 ]; then
+            to_cmd+="--kill-after=${kill_time} "
+        fi
+        to_cmd+="${max_time} "
+    fi
+
+    # Pop the first two arguments off the list,
+    # so we can pass additional args to the command safely
+
+    local -i attempt=0
+    local rc=0
+
+    while :; do
+        let attempt++
+
+        echo_stderr "Running: ${cmd} $@"
+        ${to_cmd} ${cmd} "$@"
+        rc=$?
+        if [ $rc -eq 0 ]; then
+            return 0
+        fi
+
+        if [ $rc -eq 124 ]; then
+            echo_stderr "Command (${cmd}) timed out, attempt ${attempt} of ${max_attempts}."
+        elif [ $rc -eq 137 ]; then
+            echo_stderr "Command (${cmd}) timed out and killed, attempt ${attempt} of ${max_attempts}."
+        else
+            echo_stderr "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}."
+        fi
+
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            echo_stderr "Waiting ${delay} seconds before retrying..."
+            sleep ${delay}
+            continue
+        else
+            echo_stderr "Max command attempts reached. Aborting..."
+            return 1
+        fi
+    done
+}

doc/source/conf.py

@@ -62,7 +62,7 @@ openstackdocs_auto_name = False
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.