diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 39a1abaf..ac53e17f 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -1,14 +1,16 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# This utility builds the StarlingX wheel tarball +# This utility builds the StarlingX base image # MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -31,6 +33,7 @@ CLEAN=no TAG_LATEST=no LATEST_TAG=latest HOST=${HOSTNAME} +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 <&2 exit 1 diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index edb39701..7c5dec06 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -1,14 +1,16 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# This utility builds the StarlingX wheel tarball +# This utility builds the StarlingX container images # MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -35,6 +37,7 @@ TAG_LIST_FILE= TAG_LIST_LATEST_FILE= declare -a ONLY declare -a SKIP +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 < : Skip building the specified image(s). Multiple images can be specified with a comma-separated list, or with multiple --skip arguments. + --attempts: Max attempts, in case of failure (default: 1) EOF @@ -276,7 +280,7 @@ function build_image_loci { local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - docker build ${WORKDIR}/loci --no-cache \ + with_retries ${MAX_ATTEMPTS} docker build ${WORKDIR}/loci --no-cache \ "${BUILD_ARGS[@]}" \ --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then @@ -361,7 +365,7 @@ function build_image_docker { BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY) fi BASE_BUILD_ARGS+=(--tag ${build_image_name}) - docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log + with_retries ${MAX_ATTEMPTS} docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${BUILD_STREAM}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" @@ -395,7 +399,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -477,6 +481,10 @@ while true; do TAG_LATEST=yes shift ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -555,6 +563,9 @@ fi docker images --format '{{.Repository}}:{{.Tag}}' ${BASE} | grep -q "^${BASE}$" BASE_IMAGE_PRESENT=$? +# Pull the image anyway, to ensure it's up to date +docker pull ${BASE} + # Download loci, if needed. get_loci if [ $? -ne 0 ]; then @@ -573,7 +584,7 @@ for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_ done done -if [ "${CLEAN}" = "yes" ]; then +if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then # Delete the images echo "Deleting images" docker image rm ${RESULTS_BUILT[@]} ${RESULTS_PUSHED[@]} diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index c9d5b076..b8425fe6 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -8,6 +8,10 @@ # for a set of upstream python modules. # +MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) + +source ${MY_SCRIPT_DIR}/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -21,6 +25,7 @@ OS=centos OS_VERSION=7.5.1804 BUILD_STREAM=stable PROXY="" +declare -i MAX_ATTEMPTS=1 function usage { cat >&2 <: --stream: Build stream, stable or dev (default: stable) + --attempts: Max attempts, in case of failure (default: 1) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,proxy:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -81,6 +87,10 @@ while true; do PROXY=$2 shift 2 ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -210,7 +220,7 @@ BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}) # Build image -docker build "${BUILD_ARGS[@]}" +with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -230,7 +240,7 @@ fi RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) # Run container to build wheels -docker run ${RUN_ARGS[@]} +with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]} if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 89c6bafb..a3ae94e3 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -9,6 +9,8 @@ MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) +source ${MY_SCRIPT_DIR}/utils.sh + # Required env vars if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then echo "Environment not setup for builds" >&2 @@ -25,6 +27,7 @@ PUSH=no PROXY="" CLEAN=no DOCKER_USER=${USER} +declare -i MAX_ATTEMPTS=1 # List of top-level services for images, which should not be listed in upper-constraints.txt SKIP_CONSTRAINTS=( @@ -56,11 +59,12 @@ Options: --proxy: Set proxy : --user: Docker repo userid --version: Version for pushed image (if used with --push) + --attempts: Max attempts, in case of failure (default: 1) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,proxy:,version:,attempts: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -111,6 +115,10 @@ while true; do VERSION=$2 shift 2 ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -143,7 +151,7 @@ if [ ! -z "$PROXY" ]; then BUILD_BASE_WL_ARGS+=(--proxy ${PROXY}) fi -${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} +${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 @@ -177,13 +185,13 @@ else OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK} fi -wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt +with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt if [ $? -ne 0 ]; then echo "Failed to download global-requirements.txt" >&2 exit 1 fi -wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt +with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 diff --git a/build-tools/build-wheels/docker/docker-build-wheel.sh b/build-tools/build-wheels/docker/docker-build-wheel.sh index 5fcb1d39..60281036 100755 --- a/build-tools/build-wheels/docker/docker-build-wheel.sh +++ b/build-tools/build-wheels/docker/docker-build-wheel.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2019 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -11,6 +11,7 @@ CFGFILE=/wheels.cfg OUTPUTDIR=/wheels FAILED_LOG=$OUTPUTDIR/failed.lst +declare -i MAX_ATTEMPTS=5 # # Function to log the start of a build @@ -102,6 +103,41 @@ function fix_setup { } +# +# Function to call a command, with support for retries +# +function with_retries { + local max_attempts=$1 + local cmd=$2 + + # Pop the first two arguments off the list, + # so we can pass additional args to the command safely + shift 2 + + local -i attempt=0 + + while :; do + let -i attempt++ + + echo "Running: ${cmd} $@" + ${cmd} "$@" + if [ $? -eq 0 ]; then + return 0 + fi + + echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." + if [ ${attempt} -lt ${max_attempts} ]; then + local delay=5 + echo "Waiting ${delay} seconds before retrying..." + sleep ${delay} + continue + else + echo "Max command attempts reached. Aborting..." + return 1 + fi + done +} + # # Function to use git to clone the module source and build a wheel. # @@ -176,9 +212,8 @@ function from_tar { taropts="xf" fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi @@ -228,9 +263,8 @@ function from_zip { continue fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi @@ -280,9 +314,8 @@ function from_pypi { continue fi - wget $wgetsrc + with_retries ${MAX_ATTEMPTS} wget $wgetsrc if [ $? -ne 0 ]; then - echo "Failure running: wget $wgetsrc" echo $wheelname >> $FAILED_LOG continue fi diff --git a/build-tools/build-wheels/utils.sh b/build-tools/build-wheels/utils.sh new file mode 100644 index 00000000..d00b3f15 --- /dev/null +++ b/build-tools/build-wheels/utils.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Image and wheel build utility functions +# + +# +# Function to call a command, with support for retries +# +function with_retries { + local max_attempts=$1 + local cmd=$2 + + # Pop the first two arguments off the list, + # so we can pass additional args to the command safely + shift 2 + + local -i attempt=0 + + while :; do + let -i attempt++ + + echo "Running: ${cmd} $@" + ${cmd} "$@" + if [ $? -eq 0 ]; then + return 0 + fi + + echo "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." + if [ ${attempt} -lt ${max_attempts} ]; then + local delay=5 + echo "Waiting ${delay} seconds before retrying..." + sleep ${delay} + continue + else + echo "Max command attempts reached. Aborting..." + return 1 + fi + done +} +