From 492b5b218d0a01263714d94ded931b03fa608477 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Tue, 9 Apr 2019 16:17:22 -0400 Subject: [PATCH] Retries for certain wheel and image build commands The wheel and image build tools hit intermittent issues such as: * Network or server issues that cause file download failures * Repo access failures due to repo updates or other issues To avoid formal build failures on such issues, this update adds retries around specific commands, like wget and docker build. Change-Id: Ifafdc6201872f43b2dd77efd4dcb033456477c2e Closes-Bug: 1823986 Signed-off-by: Don Penney --- .../build-docker-images/build-stx-base.sh | 19 ++++++-- .../build-docker-images/build-stx-images.sh | 23 ++++++--- build-tools/build-wheels/build-base-wheels.sh | 18 +++++-- .../build-wheels/build-wheel-tarball.sh | 18 +++++-- .../build-wheels/docker/docker-build-wheel.sh | 47 ++++++++++++++++--- build-tools/build-wheels/utils.sh | 44 +++++++++++++++++ 6 files changed, 143 insertions(+), 26 deletions(-) create mode 100644 build-tools/build-wheels/utils.sh diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 39a1abaf..ac53e17f 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -1,14 +1,16 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# This utility builds the StarlingX wheel tarball +# This utility builds the StarlingX base image # MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -31,6 +33,7 @@ CLEAN=no TAG_LATEST=no LATEST_TAG=latest HOST=${HOSTNAME} +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 <&2 exit 1 diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index edb39701..7c5dec06 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -1,14 +1,16 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# This utility builds the StarlingX wheel tarball +# This utility builds the StarlingX container images # MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -35,6 +37,7 @@ TAG_LIST_FILE= TAG_LIST_LATEST_FILE= declare -a ONLY declare -a SKIP +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 < : Skip building the specified image(s). Multiple images can be specified with a comma-separated list, or with multiple --skip arguments. + --attempts: Max attempts, in case of failure (default: 1) EOF @@ -276,7 +280,7 @@ function build_image_loci { local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - docker build ${WORKDIR}/loci --no-cache \ + with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \ "${BUILD_ARGS[@]}" \ --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then @@ -361,7 +365,7 @@ function build_image_docker { BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY) fi BASE_BUILD_ARGS+=(--tag ${build_image_name}) - docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log + with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" @@ -395,7 +399,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -477,6 +481,10 @@ while true; do TAG_LATEST=yes shift ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -555,6 +563,9 @@ fi docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$" BASE_IMAGE_PRESENT=$? +# Pull the image anyway, to ensure it's up to date +docker pull ${BASE} + # Download loci, if needed. get_loci if [ $? -ne 0 ]; then @@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_ done done -if [ "${CLEAN}" = "yes" ]; then +if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then # Delete the images echo "Deleting images" docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]} diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index c9d5b076..b8425fe6 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -8,6 +8,10 @@ # for a set of upstream python modules. # +MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) + +source ${MY_SCRIPT_DIR}/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -21,6 +25,7 @@ OS=centos OS_VERSION=7.5.1804 BUILD_STREAM=stable PROXY="" +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 <: --stream: Build stream, stable or dev (default: stable) + --attempts: Max attempts, in case of failure (default: 1) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -81,6 +87,10 @@ while true; do PROXY=$2 shift 2 ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}) # Build image -docker build "${BUILD_ARGS[@]}" +with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -230,7 +240,7 @@ fi RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) # Run container to build wheels -docker run ${RUN_ARGS[@]} +with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]} if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 89c6bafb..a3ae94e3 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -9,6 +9,8 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -25,6 +27,7 @@ PUSH=no PROXY="" CLEAN=no DOCKER_USER=${USER} +declare -i MAX_ATTEMPTS=1 # List of top-level services for images, which should not be listed in upper-constraints.txt SKIP_CONSTRAINTS=( @@ -56,11 +59,12 @@ Options: --proxy: Set proxy : --user: Docker repo userid --version: Version for pushed image (if used with --push) + --attempts: Max attempts, in case of failure (default: 1) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -111,6 +115,10 @@ while true; do VERSION=$2 shift 2 ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then BUILD_BASE_WL_ARGS+=(--proxy ${PROXY}) fi -${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} +${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 @@ -177,13 +185,13 @@ else OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK} fi -wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt +with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt if [ $? -ne 0 ]; then echo "Failed to download global-requirements.txt" >&2 exit 1 fi -wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt +with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 diff --git a/build-tools/build-wheels/docker/docker-build-wheel.sh b/build-tools/build-wheels/docker/docker-build-wheel.sh index 5fcb1d39..60281036 100755 --- a/build-tools/build-wheels/docker/docker-build-wheel.sh +++ b/build-tools/build-wheels/docker/docker-build-wheel.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -11,6 +11,7 @@ CFGFILE=/wheels.cfg OUTPUTDIR=/wheels FAILED_LOG=$OUTPUTDIR/failed.lst +declare -i MAX_ATTEMPTS=5 # # Function to log the start of a build @@ -102,6 +103,41 @@ function fix_setup { } +# +# Function to call a command, with support for retries +# +function with_retries { + local max_attempts=$1 + local cmd=$2 + + # Pop the first two arguments off the list, + # so we can pass additional args to the command safely + shift 2 + + local -i attempt=0 + + while :; do + let -i attempt++ + + echo "Running: ${cmd} $@" + ${cmd} "$@" + if [ $? -eq 0 ]; then + return 0 + fi + + echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." + if [ ${attempt} -lt ${max_attempts} ]; then + local delay=5 + echo "Waiting ${delay} seconds before retrying..." + sleep ${delay} + continue + else + echo "Max command attempts reached. Aborting..." + return 1 + fi + done +} + # # Function to use git to clone the module source and build a wheel. # @@ -176,9 +212,8 @@ function from_tar { taropts="xf" fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi @@ -228,9 +263,8 @@ function from_zip { continue fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi @@ -280,9 +314,8 @@ function from_pypi { continue fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi diff --git a/build-tools/build-wheels/utils.sh b/build-tools/build-wheels/utils.sh new file mode 100644 index 00000000..d00b3f15 --- /dev/null +++ b/build-tools/build-wheels/utils.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Image and wheel build utility functions +# + +# +# Function to call a command, with support for retries +# +function with_retries { + local max_attempts=$1 + local cmd=$2 + + # Pop the first two arguments off the list, + # so we can pass additional args to the command safely + shift 2 + + local -i attempt=0 + + while :; do + let -i attempt++ + + echo "Running: ${cmd} $@" + ${cmd} "$@" + if [ $? -eq 0 ]; then + return 0 + fi + + echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." + if [ ${attempt} -lt ${max_attempts} ]; then + local delay=5 + echo "Waiting ${delay} seconds before retrying..." + sleep ${delay} + continue + else + echo "Max command attempts reached. Aborting..." + return 1 + fi + done +} +