From ae68691bf0c6d1fbb74f0dd12c0278ca28ae1098 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 31 Oct 2018 11:18:17 -0400 Subject: [PATCH] Enable starlingx mirror StarlingX needs to download a variety of rpms and tarballs from various upstream sources. Unfortunately the upstream sources are not always dependable. Either servers go down, are unreachable, or drop older content that we still depend on. Our proposed solution is to run our own mirror to capture an independent copy of the content needed by StarlingX. For this purpose, a server has been set up at http://mirror.starlingx.cengn.ca/mirror/centos The mirror will use deterministic paths derived from the upstream urls. Scripts will be used to convert an upstream url to the mirror's equivalent url (see function url_to_stx_mirror_url in url_utils.sh) The mirror will be refreshed daily. New .lst entries will be processed at that time. Processing of changes under yum.repos.d is not automated by this update. Expect a follow-up update to address this issue soon. These scripts are found under the 'stx_mirror_scripts' subdirectory. Changes are made to the download_mirror.sh script, and it's supporting scripts. New arguments have been added to each script to select the download source. -s StarlingX mirror only -S StarlingX mirror, with upstream source as backup -u Upstream source only -U Upstream source, with StarlingX mirror as backup You do not need to provide any of these flags. Continue to us download_mirror.sh as you always have. The default behavior is currently set to '-S', i.e. first try the StarlingX mirror, with upstream source as backup. If this proves to place to heavy a load on the existing server, we might switch the default to '-U', i.e. first try the upstream source, with StarlingX mirror as backup. If poor download performance is seen, you might want to try explicitly adding -U as an argument. The remaining two options are not recommended for regular use. Upstream only, i.e. '-u', restores original behaviour, but you may once again encounter rpms that have aged out, and been removed from their original repos. StarlingX only, i.e. '-s', is vulnerable if a .lst file has been updated, but the mirror has not yet processed it. Change-Id: I7e0f3d9fb99253662f9f4bf12457d39250408c0b Story: 2003906 Task: 26785 Signed-off-by: Scott Little --- .../dl_other_from_centos_repo.sh | 160 +++++++- centos-mirror-tools/dl_rpms.sh | 111 ++++- centos-mirror-tools/dl_tarball.sh | 308 +++++++++++++- centos-mirror-tools/download_mirror.sh | 148 ++++++- .../make_stx_mirror_yum_conf.sh | 226 +++++++++++ .../stx_mirror_scripts/daily_dl_sync.sh | 383 ++++++++++++++++++ .../stx_mirror_scripts/daily_repo_sync.sh | 129 ++++++ centos-mirror-tools/tarball-dl.lst | 100 ++--- centos-mirror-tools/url_utils.sh | 266 ++++++++++++ centos-mirror-tools/utils.sh | 11 +- .../yum.repos.d/StarlingX-Centos-7.2.repo | 7 + .../yum.repos.d/StarlingX-Centos-7.3.repo | 6 + .../yum.repos.d/StarlingX-Centos-7.4.repo | 6 + .../yum.repos.d/StarlingX-EPEL-7.repo | 6 +- 14 files changed, 1756 insertions(+), 111 deletions(-) create mode 100755 centos-mirror-tools/make_stx_mirror_yum_conf.sh create mode 100755 centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh create mode 100755 centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh create mode 100755 centos-mirror-tools/url_utils.sh diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 1c5a6060..7ec42320 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -1,8 +1,93 @@ #!/bin/bash -e -# download non-RPM files from http://vault.centos.org/7.4.1708/os/x86_64/ + +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Download non-RPM files from http://vault.centos.org/7.4.1708/os/x86_64/ +# + +DL_OTHER_FROM_CENTOS_REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_OTHER_FROM_CENTOS_REPO_DIR/url_utils.sh + +usage () { + echo "$0 [-D ] [-s|-S|-u|-U] [-h] []" +} + +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "D:hsSuU" o; do + case "${o}" in + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) if [ $# -lt 2 ]; then - echo "$0 []" + usage exit -1 fi @@ -13,12 +98,15 @@ if [ ! -e $download_list ];then fi save_path=$2 -url_prefix="http://vault.centos.org/7.4.1708/os/x86_64/" -echo "NOTE: please assure Internet access to $url_prefix !!" +upstream_url_prefix="http://vault.centos.org/7.4.1708/os/x86_64/" +stx_mirror_url_prefix="$(url_to_stx_mirror_url "$upstream_url_prefix" "$distro")" + +echo "NOTE: please assure Internet access to $upstream_url_prefix !!" force_update=$3 i=0 +error_count=0 all=`cat $download_list` for ff in $all; do ## skip commented_out item which starts with '#' @@ -30,28 +118,68 @@ for ff in $all; do _name=`echo $ff | cut -d":" -f2-2` if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name + if [ $? -ne 0 ]; then + echo "Error: mkdir -p '$save_path/$_name'" + error_count=$((error_count + 1)) + fi else if [ -e "$save_path/$_name" ]; then echo "Already have $save_path/$_name" continue fi - echo "remote path: $url_prefix/$_name" - echo "local path: $save_path/$_name" - if wget $url_prefix/$_name; then - file_name=`basename $_name` - sub_path=`dirname $_name` - if [ -e "./$file_name" ]; then - let i+=1 - echo "$file_name is downloaded successfully" - mv -f ./$file_name $save_path/$_name - ls -l $save_path/$_name + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url_prefix="$stx_mirror_url_prefix" + ;; + $dl_from_upstream) + url_prefix="$upstream_url_prefix" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + echo "remote path: $url_prefix/$_name" + echo "local path: $save_path/$_name" + if wget $url_prefix/$_name; then + file_name=`basename $_name` + sub_path=`dirname $_name` + if [ -e "./$file_name" ]; then + let i+=1 + echo "$file_name is downloaded successfully" + + \mv -f ./$file_name $save_path/$_name + if [ $? -ne 0 ]; then + echo "Error: mv -f './$file_name' '$save_path/$_name'" + error_count=$((error_count + 1)) + fi + + ls -l $save_path/$_name + fi + break + else + echo "Warning: failed to download $url_prefix/$_name" fi - else - echo "ERROR: failed to download $url_prefix/$_name" + done + + if [ ! -e "$save_path/$_name" ]; then + echo "Error: failed to download '$url_prefix/$_name'" + error_count=$((error_count + 1)) + continue fi fi done +echo "" echo "totally $i files are downloaded!" +if [ $error_count -ne 0 ]; then + echo "" + echo "Encountered $error_count errors" + exit 1 +fi + +exit 0 diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 5b290741..582b3e0f 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -15,7 +15,9 @@ SUDOCMD="sudo -E" RELEASEVER="--releasever=7" YUMCONFOPT="" -source utils.sh +DL_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_RPMS_DIR/utils.sh usage() { echo "$0 [-n] [-c ] " @@ -45,8 +47,34 @@ usage() { CLEAN_LOGS_ONLY=0 dl_rc=0 +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + # Parse option flags -while getopts "c:nxh" o; do +while getopts "c:nxD:sSuUh" o; do case "${o}" in n) # No-sudo @@ -61,6 +89,35 @@ while getopts "c:nxh" o; do YUMCONFOPT="-c $OPTARG" grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})" ;; + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) # Help usage @@ -124,8 +181,8 @@ fi download () { local _file=$1 local _level=$2 - local _list=$(cat $_file) - local _from=$(get_from $_file) + local _list="" + local _from="" local _arch="" @@ -134,23 +191,51 @@ download () { local download_url="" local rpm_name="" local SFILE="" + local lvl + local dl_result + + _list=$(cat $_file) + _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do _arch=$(get_arch_from_rpm $ff) rpm_name="$(get_rpm_name $ff)" - download_cmd="$(get_download_cmd $ff $_level)" dest_dir="$(get_dest_directory $_arch)" if [ ! -e $dest_dir/$rpm_name ]; then - echo "Looking for $rpm_name" - echo "--> run: $download_cmd" - if $download_cmd ; then - download_url="$(get_url $ff $_level)" - SFILE="$(get_rpm_level_name $rpm_name $_level)" - process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" - else - echo "Warning: $rpm_name not found" + dl_result=1 + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + lvl=$dl_from_stx_mirror + ;; + $dl_from_upstream) + lvl=$_level + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + download_cmd="$(get_download_cmd $ff $lvl)" + + echo "Looking for $rpm_name" + echo "--> run: $download_cmd" + if $download_cmd ; then + download_url="$(get_url $ff $lvl)" + SFILE="$(get_rpm_level_name $rpm_name $lvl)" + process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" + dl_result=0 + break + else + echo "Warning: $rpm_name not found" + fi + done + + if [ $dl_result -eq 1 ]; then + echo "Error: $rpm_name not found" echo "missing_srpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_SRPMS rc=1 diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index 0658a99b..0c198802 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# +# SPDX-License-Identifier: Apache-2.0 +# + # The build of StarlingX relies, besides RPM Binaries and Sources, in this # repository which is a collection of packages in the form of Tar Compressed # files and 3 RPMs obtained from a Tar Compressed file. This script and a text @@ -14,6 +18,85 @@ script_path="$(dirname $(readlink -f $0))" tarball_file="$script_path/tarball-dl.lst" mvn_artf_file="$script_path/mvn-artifacts.lst" +DL_TARBALL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_TARBALL_DIR/url_utils.sh + +usage () { + echo "$0 [-D ] [-s|-S|-u|-U] [-h] []" +} + +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "D:hsSuU" o; do + case "${o}" in + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + + if [ ! -e $tarball_file -o ! -e $mvn_artf_file ];then echo "$download_list does not exist, please have a check!" exit -1 @@ -40,20 +123,55 @@ fi # Download function using wget command download_package() { - wget --spider $1 - if [ $? != 0 ]; then - echo "$1 is broken" - else - wget -t 5 --wait=15 $1 + local upstream_url="$1" + local stx_url="" + local url="" + local rc=1 + + stx_url="$(url_to_stx_mirror_url "$upstream_url" "$distro")" + + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$stx_url" + ;; + $dl_from_upstream) + url="$upstream_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + wget --spider "$url" if [ $? != 0 ]; then - echo "$1" > "$output_log" + echo "Warning: '$url' is broken" + else + wget -t 5 --wait=15 "$url" + if [ $? -eq 0 ]; then + rc=0 + break + else + echo "Warning: failed to download '$url'" + continue + fi fi + done + + if [ $rc != 0 ]; then + echo "Error: failed to download '$upstream_url'" + echo "$upstream_url" > "$output_log" fi + + return $rc } # This script will iterate over the tarball.lst text file and execute specific # tasks based on the name of the package: +error_count=0; + for line in $(cat $tarball_file); do # A line from the text file starting with "#" character is ignored @@ -69,11 +187,31 @@ for line in $(cat $tarball_file); do # denotes special handling is required tarball_name=`echo $line | cut -d"#" -f1-1` # - Column 2, name of the directory path after it is decompressed as it is # referenced in the build system recipe. - # - Column 3, the URL for the package download + # - Column 3, the URL for the file or git to download + # - Column 4, download method, one of + # http - download a simple file + # http_filelist - download multiple files by appending a list of subpaths + # to the base url. Tar up the lot. + # http_script - download a simple file, run script whos output is a tarball + # git - download a git, checkout branch and tar it up + # git_script - download a git, checkout branch, run script whos output is a tarball + # + # - Column 5, utility field + # If method is git or git_script, this is a branch,tag,sha we need to checkout + # If method is http_filelist, this is the path to a file containing subpaths. + # Subpaths are appended to the urls and downloaded. + # Otherwise unused + # - Column 6, Path to script. + # Not yet supported. + # Intent is to run this script to produce the final tarball, replacing + # all the special case code currently embedded in this script. tarball_name=$(echo $line | cut -d"#" -f1-1) directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + method=$(echo $line | cut -d"#" -f4-4) + util=$(echo $line | cut -d"#" -f5-5) + script=$(echo $line | cut -d"#" -f6-6) # Remove leading '!' if present tarball_name="${tarball_name//!/}" @@ -104,6 +242,12 @@ for line in $(cat $tarball_file); do pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz mv linux-tpmdd-e6aef06/security/integrity/ $directory_name tar czvf $tarball_name $directory_name @@ -111,6 +255,12 @@ for line in $(cat $tarball_file); do rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + mkdir $directory_name tar xf $tarball_name --strip-components 1 -C $directory_name rm $tarball_name @@ -122,6 +272,7 @@ for line in $(cat $tarball_file); do popd tar czvf $tarball_name $directory_name rm -rf $directory_name + popd # pushd $directory_name # The mvn.repo.tgz tarball will be created downloading a serie of # of maven artifacts described in mvn-artifacts file. elif [ "$tarball_name" = "mvn.repo.tgz" ]; then @@ -130,11 +281,49 @@ for line in $(cat $tarball_file); do echo "$mvn_artf_file no found" 1>&2 exit 1 fi + + success_all=0 while read -r artf; do echo "download: $(basename $artf)" - wget "$tarball_url/$artf" -P "$directory_name/$(dirname $artf)" + success=1 + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url/$artf" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url/$artf" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + wget "$url" -P "$directory_name/$(dirname $artf)" + if [ $? -eq 0 ]; then + success=0 + break + else + echo "Warning: Failed to download from $url" + continue; + fi + done + + if [ $success -ne 0 ]; then + success_all=1 + echo "Error: Failed to download from '$tarball_url/$artf'" + fi done < "$mvn_artf_file" + if [ $success_all -ne 0 ]; then + echo "Error: Failed to download one or more components from '$tarball_url'" + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + # Create tarball tar -zcvf "$tarball_name" -C "$directory_name"/ . rm -rf "$directory_name" @@ -142,7 +331,14 @@ for line in $(cat $tarball_file); do elif [[ "$tarball_name" =~ ^'MLNX_OFED_LINUX' ]]; then pkg_version=$(echo "$tarball_name" | cut -d "-" -f2-3) srpm_path="MLNX_OFED_SRC-${pkg_version}/SRPMS/" + download_package "$tarball_url" + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar -xf "$tarball_name" tar -xf "$directory_name/src/MLNX_OFED_SRC-${pkg_version}.tgz" # This section of code gets specific SRPMs versions according @@ -168,8 +364,20 @@ for line in $(cat $tarball_file); do rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name tar czvf $tarball_name $directory_name @@ -177,22 +385,74 @@ for line in $(cat $tarball_file); do rm -rf $directory_name rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then - git clone https://git.code.sf.net/p/ibmtpm20tss/tss ibmtpm20tss-tss - pushd ibmtpm20tss-tss - git checkout v930 + dest_dir=ibmtpm20tss-tss + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + git clone $url $dest_dir + if [ $? -eq 0 ]; then + # Success + break + else + echo "Warning: Failed to git clone from '$url'" + continue + fi + done + + if [ ! -d $dest_dir ]; then + echo "Error: Failed to git clone from '$tarball_url'" + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + + pushd $dest_dir + branch=$util + git checkout $branch rm -rf .git popd mv ibmtpm20tss-tss $directory_name tar czvf $tarball_name $directory_name rm -rf $directory_name + popd # pushd $dest_dir fi - popd + popd # pushd $output_tarball continue fi - download_cmd="wget -t 5 --wait=15 $tarball_url -O $download_path" + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + download_cmd="wget -t 5 --wait=15 $url -O $download_path" - if [ ! -e $download_path ]; then if $download_cmd ; then echo "Ok: $download_path" pushd $download_directory @@ -204,16 +464,26 @@ for line in $(cat $tarball_file); do rm -r $directory_name fi popd + break else - echo "Error: Failed to download $tarball_url" 1>&2 - echo "$tarball_url" > "$output_log" + echo "Warning: Failed to download $url" 1>&2 + continue fi + done - else - echo "Already have $download_path" + if [ ! -e $download_path ]; then + echo "Error: Failed to download $tarball_url" 1>&2 + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) fi - done # End of file +if [ $error_count -ne 0 ]; then + echo "" + echo "Encountered $error_count errors" + exit 1 +fi + +exit 0 diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 6b6731e1..e55e3d8b 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -4,7 +4,7 @@ # usage() { - echo "$0 [-n] [-c ] [-g]" + echo "$0 [-n] [-c ] [-g] [-s|-S|-u|-U]" echo "" echo "Options:" echo " -n: Do not use sudo when performing operations (option passed on to" @@ -12,6 +12,10 @@ usage() { echo " -c: Use an alternate yum.conf rather than the system file (option passed" echo " on to subscripts when appropriate)" echo " -g: do not change group IDs of downloaded artifacts" + echo " -s: Download from StarlingX mirror only" + echo " -S: Download from StarlingX mirror, upstream as backup (default)" + echo " -u: Download from original upstream sources only" + echo " -U: Download from original upstream sources, StarlingX mirror as backup" echo "" } @@ -24,8 +28,17 @@ generate_log_name() { need_file(){ for f in $*; do - if [ ! -e $f ]; then - echo "ERROR: $f does not exist." + if [ ! -f $f ]; then + echo "ERROR: File $f does not exist." + exit 1 + fi + done +} + +need_dir(){ + for d in $*; do + if [ ! -d $d ]; then + echo "ERROR: Directory $d does not exist." exit 1 fi done @@ -35,12 +48,16 @@ need_file(){ rpm_downloader="./dl_rpms.sh" tarball_downloader="./dl_tarball.sh" other_downloader="./dl_other_from_centos_repo.sh" +make_stx_mirror_yum_conf="./make_stx_mirror_yum_conf.sh" # track optional arguments change_group_ids=1 use_system_yum_conf=1 +alternate_yum_conf="" +alternate_repo_dir="" rpm_downloader_extra_args="" tarball_downloader_extra_args="" +distro="centos" # lst files to use as input rpms_from_3rd_parties="./rpms_3rdparties.lst" @@ -51,8 +68,43 @@ other_downloads="./other_downloads.lst" # Overall success success=1 -# Parse out optional -c or -n arguments -while getopts "c:ngh" o; do +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +dl_from_stx () { + local re="\\b$dl_from_stx_mirror\\b" + [[ "$dl_source" =~ $re ]] +} + +dl_from_upstream () { + local re="\\b$dl_from_upstream\\b" + [[ "$dl_source" =~ $re ]] +} + + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "c:nghsSuU" o; do case "${o}" in n) # Pass -n ("no-sudo") to rpm downloader @@ -61,12 +113,36 @@ while getopts "c:ngh" o; do c) # Pass -c ("use alternate yum.conf") to rpm downloader use_system_yum_conf=0 - rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}" + alternate_yum_conf="${OPTARG}" ;; g) # Do not attempt to change group IDs on downloaded packages change_group_ids=0 ;; + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; h) # Help usage @@ -98,7 +174,6 @@ need_file ${rpms_from_centos_repo} need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst - #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." @@ -111,6 +186,51 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi fi +if [ $use_system_yum_conf -eq 0 ]; then + need_file "${alternate_yum_conf}" + if [ "$alternate_repo_dir" == "" ]; then + alternate_repo_dir=$(grep '^repodir=' "${alternate_yum_conf}" | cut -d '=' -f 2) + need_dir "${alternate_repo_dir}" + fi +fi + +TEMP_DIR="" +rpm_downloader_extra_args="${rpm_downloader_extra_args} -D $distro" + +if [ "$dl_flag" == "" ]; then + if [ $use_system_yum_conf -eq 0 ]; then + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${alternate_yum_conf}" + fi +else + rpm_downloader_extra_args="${rpm_downloader_extra_args} $dl_flag" + + if ! dl_from_stx; then + if [ $use_system_yum_conf -eq 0 ]; then + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${alternate_yum_conf}" + fi + else + TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX) + TEMP_CONF="$TEMP_DIR/yum.conf" + need_file ${make_stx_mirror_yum_conf} + need_dir ${TEMP_DIR} + if [ $use_system_yum_conf -eq 0 ]; then + if dl_from_upstream; then + ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro + else + ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro + fi + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c $TEMP_CONF" + else + if dl_from_upstream; then + ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro + else + ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro + fi + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c $TEMP_CONF" + fi + fi +fi + list=${rpms_from_3rd_parties} level=L1 logfile=$(generate_log_name $list $level) @@ -139,6 +259,9 @@ fi if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo + if [ "$TEMP_DIR" != "" ]; then + \rm -f $TEMP_DIR/yum.repos.d/StarlingX_3rd*.repo + fi fi @@ -232,7 +355,7 @@ fi echo "step #3: start downloading other files ..." logfile=$LOGSDIR"/otherfiles_centos_download.log" -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile +${other_downloader} ${dl_flag} -D "$distro" ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" @@ -249,7 +372,7 @@ fi # they will be downloaded. echo "step #4: start downloading tarball compressed files" logfile=$LOGSDIR"/tarballs_download.log" -${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile +${tarball_downloader} ${dl_flag} -D "$distro" ${tarball_downloader_extra_args} |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #4: done successfully" @@ -261,6 +384,13 @@ else success=0 fi +# +# Clean up the mktemp directory, if required. +# +if [ "$TEMP_DIR" != "" ]; then + \rm -rf "$TEMP_DIR" +fi + echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" echo "running \"update-pxe-network-installer\" command after \"build-iso\"" diff --git a/centos-mirror-tools/make_stx_mirror_yum_conf.sh b/centos-mirror-tools/make_stx_mirror_yum_conf.sh new file mode 100755 index 00000000..90f5d387 --- /dev/null +++ b/centos-mirror-tools/make_stx_mirror_yum_conf.sh @@ -0,0 +1,226 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Replicate a yum.conf and yum.repo.d under a temporary directory and +# then modify the files to point to equivalent repos in the StarlingX mirror. +# This script was originated by Scott Little +# + +MAKE_STX_MIRROR_YUM_CONF_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "$MAKE_STX_MIRROR_YUM_CONF_DIR/url_utils.sh" + +DISTRO="centos" + +TEMP_DIR="" +SRC_REPO_DIR="$MAKE_STX_MIRROR_YUM_CONF_DIR/yum.repos.d" +SRC_YUM_CONF="$MAKE_STX_MIRROR_YUM_CONF_DIR/yum.conf.sample" + +RETAIN_REPODIR=0 + +usage () { + echo "" + echo "$0 -d [-D ] [-y ] [-r ] [-R]" + echo "" + echo "Replicate a yum.conf and yum.repo.d under a new directory and" + echo "then modify the files to point to equivalent repos in the StarlingX" + echo "mirror." + echo "" + echo "-d = Place modified yum.conf and yum.repo.d into this directory" + echo "-D = Target distro on StarlingX mirror. Default is 'centos'" + echo "-y = Path to yum.conf file that we will modify. Default is" + echo " 'yum.conf.sample' in same directory as this script" + echo "-r = Path to yum.repos.d that we will modify. Default is" + echo " 'yum.repos.d' in same directory as this script" +} + + +# +# option processing +# +while getopts "D:d:Rr:y:" o; do + case "${o}" in + D) + DISTRO="${OPTARG}" + ;; + d) + TEMP_DIR="${OPTARG}" + ;; + r) + SRC_REPO_DIR="${OPTARG}" + ;; + R) + RETAIN_REPODIR=1 + ;; + y) + SRC_YUM_CONF="${OPTARG}" + ;; + *) + usage + exit 1 + ;; + esac +done + +# +# option validation +# +if [ ! -f $SRC_YUM_CONF ]; then + echo "Error: yum.conf not found at '$SRC_YUM_CONF'" + exit 1 +fi + +if [ ! -d $SRC_REPO_DIR ]; then + echo "Error: repo dir not found at '$SRC_REPO_DIR'" + exit 1 +fi + +if [ "$TEMP_DIR" == "" ]; then + echo "Error: working dir not provided" + usage + exit 1 +fi + +if [ ! -d $TEMP_DIR ]; then + echo "Error: working dir not found at '$TEMP_DIR'" + exit 1 +fi + +# +# Get the value of the $releasever variable. +# +# If the source yum.conf has a releasever= setting, we will honor +# that, even though yum will not. +# +# Otherwise use yum to query the host environment (Docker). +# This assumes the host environmnet has the same releasever +# as that which will be used inside StarlingX. +# +# NOTE: In other scripts we will read releasever= out of yum.conf +# and push it back into yum via --releasever=<#>. +# +get_releasever () { + if [ -f $SRC_YUM_CONF ] && grep -q '^releasever=' $SRC_YUM_CONF; then + grep '^releasever=' $SRC_YUM_CONF | cut -d '=' -f 2 + else + yum version nogroups | grep Installed | cut -d ' ' -f 2 | cut -d '/' -f 1 + fi +} + +# +# Get the value of the $basearch variable. +# +# Just use yum to query the host environment (Docker) as we don't support +# cross compiling. +# +get_arch () { + yum version nogroups | grep Installed | cut -d ' ' -f 2 | cut -d '/' -f 2 +} + + +# +# Global variables we will use later. +# +CENGN_REPOS_DIR="$TEMP_DIR/yum.repos.d" +CENGN_YUM_CONF="$TEMP_DIR/yum.conf" +CENGN_YUM_LOG="$TEMP_DIR/yum.log" +CENGN_YUM_CACHDIR="$TEMP_DIR/cache/yum/\$basearch/\$releasever" + +RELEASEVER=$(get_releasever) +ARCH=$(get_arch) + +# +# Copy as yet unmodified yum.conf and yum.repos.d from source to dest. +# +echo "\cp -r '$SRC_REPO_DIR' '$CENGN_REPOS_DIR'" +\cp -r "$SRC_REPO_DIR" "$CENGN_REPOS_DIR" +echo "\cp '$SRC_YUM_CONF' '$CENGN_YUM_CONF'" +\cp "$SRC_YUM_CONF" "$CENGN_YUM_CONF" + +# +# Add or modify reposdir= value in our new yum.conf +# +if grep -q '^reposdir=' $CENGN_YUM_CONF; then + # reposdir= already exists, modify it + if [ $RETAIN_REPODIR -eq 1 ]; then + # Append CENGN_REPOS_DIR + sed "s#^reposdir=.*\$#reposdir=\1 $CENGN_REPOS_DIR#" -i $CENGN_YUM_CONF + else + # replace with CENGN_REPOS_DIR + sed "s#^reposdir=.*\$#reposdir=$CENGN_REPOS_DIR#" -i $CENGN_YUM_CONF + fi +else + # reposdir= doeas not yet exist, add it + if [ $RETAIN_REPODIR -eq 1 ]; then + # Add both SRC_REPO_DIR and CENGN_REPOS_DIR + echo "reposdir=$SRC_REPO_DIR $CENGN_REPOS_DIR" >> $CENGN_YUM_CONF + else + # Add CENGN_REPOS_DIR only + echo "reposdir=$CENGN_REPOS_DIR" >> $CENGN_YUM_CONF + fi +fi + +# +# modify or add logfile= value in our new yum.conf +# +if grep -q '^logfile=' $CENGN_YUM_CONF; then + sed "s#^logfile=.*\$#logfile=$CENGN_YUM_LOG#" -i $CENGN_YUM_CONF +else + echo "logfile=$CENGN_YUM_LOG" >> $CENGN_YUM_CONF +fi + +# +# modify or add cachedir= value in our new yum.conf +# +if grep -q '^cachedir=' $CENGN_YUM_CONF; then + sed "s#^cachedir=.*\$#cachedir=$CENGN_YUM_CACHDIR#" -i $CENGN_YUM_CONF +else + echo "cachedir=$CENGN_YUM_CACHDIR" >> $CENGN_YUM_CONF +fi + + +# +# Modify all the repo files in our new yum.repos.d +# +for REPO in $(find "$CENGN_REPOS_DIR" -type f -name '*repo'); do + # + # Replace mirrorlist with baseurl if required + # + if grep -q '^mirrorlist=' "$REPO" ; then + sed '/^mirrorlist=/d' -i "$REPO" + sed 's%^#baseurl%baseurl%' -i "$REPO" + fi + + # + # Substitute any $releasever or $basearch variables + # + sed "s#/[$]releasever/#/$RELEASEVER/#g" -i "$REPO" + sed "s#/[$]basearch/#/$ARCH/#g" -i "$REPO" + + # + # Turn off gpgcheck for now. + # Must revisit this at a later date! + # + sed 's#^gpgcheck=1#gpgcheck=0#' -i "$REPO" + sed '/^gpgkey=/d' -i "$REPO" + + # + # Convert baseurl(s) to cengn equivalent + # + for URL in $(grep '^baseurl=' "$REPO" | sed 's#^baseurl=##'); do + CENGN_URL="$(url_to_stx_mirror_url "$URL" "$DISTRO")" + sed "s#^baseurl=$URL\$#baseurl=$CENGN_URL#" -i "$REPO" + done + + # + # Prefix repoid and name with CENGN + # + sed "s#^name=\(.*\)#name=CENGN_\1#" -i "$REPO" + sed "s#^\[\([^]]*\)\]#[CENGN_\1]#" -i "$REPO" +done + +echo $TEMP_DIR diff --git a/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh b/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh new file mode 100755 index 00000000..cfae2577 --- /dev/null +++ b/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh @@ -0,0 +1,383 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# +# Daily update script for mirror.starlingx.cengn.ca covering +# tarballs and other files not downloaded from a yum repository. +# This script was originated by Scott Little. +# +# IMPORTANT: This script is only to be run on the StarlingX mirror. +# It is not for use by the general StarlinX developer. +# +# This script was originated by Scott Little. +# + +DAILY_DL_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +if [ -f "$DAILY_DL_SYNC_DIR/url_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/url_utils.sh" +elif [ -f "$DAILY_DL_SYNC_DIR/../url_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/../url_utils.sh" +else + echo "Error: Can't find 'url_utils.sh'" + exit 1 +fi + + +LOGFILE=/export/log/daily_dl_sync.log +DOWNLOAD_PATH_ROOT=/export/mirror/centos + +STX_TOOLS_BRANCH="master" +STX_TOOLS_BRANCH_ROOT_DIR="$HOME" +STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git" +STX_TOOLS_OS_SUBDIR="centos-mirror-tools" + +usage () { + echo "$0 [-b ] [-d ]" + echo "" + echo "Options:" + echo " -b: Use an alternate branch of stx-tools. Default is 'master'." + echo " -d: Directory where we will clone stx-tools. Default is \$HOME." + echo "" +} + +while getopts "b:d:h" opt; do + case "${opt}" in + b) + # branch + STX_TOOLS_BRANCH="${OPTARG}" + if [ $"STX_TOOLS_BRANCH" == "" ]; then + usage + exit 1 + fi + ;; + d) + # download directory for stx-tools + STX_TOOLS_BRANCH_ROOT_DIR="${OPTARG}" + if [ "$STX_TOOLS_BRANCH_ROOT_DIR" == "" ]; then + usage + exit 1 + fi + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH" +STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools" +LST_FILE_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR" + +dl_git_from_url () { + local GIT_URL="$1" + local BRANCH="$2" + local DL_DIR="$3" + local DL_ROOT_DIR="" + local SAVE_DIR + local CMD="" + + SAVE_DIR="$(pwd)" + + if [ "$DL_DIR" == "" ]; then + DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')" + fi + + echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'" + DL_ROOT_DIR=$(dirname "$DL_DIR") + + if [ ! -d "$DL_DIR" ]; then + if [ ! -d "$DL_ROOT_DIR" ]; then + CMD="mkdir -p '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + + CMD="cd '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git clone --bare '$GIT_URL' '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git --bare update-server-info" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + if [ -f hooks/post-update.sample ]; then + CMD="mv -f hooks/post-update.sample hooks/post-update" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git fetch" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git checkout '$BRANCH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + cd "$SAVE_DIR" + return 0 +} + + +dl_file_from_url () { + local URL="$1" + local DOWNLOAD_PATH="" + local DOWNLOAD_DIR="" + local PROTOCOL="" + local CMD="" + + DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")" + DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")" + PROTOCOL=$(url_protocol $URL) + echo "$PROTOCOL $URL $DOWNLOAD_PATH" + + if [ -f "$DOWNLOAD_PATH" ]; then + echo "Already have '$DOWNLOAD_PATH'" + return 0 + fi + + case "$PROTOCOL" in + https|http) + if [ ! -d "$DOWNLOAD_DIR" ]; then + CMD="mkdir -p '$DOWNLOAD_DIR'" + echo "$CMD" + eval "$CMD" + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + fi + + CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + ;; + *) + echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'" + ;; + esac + + return 0 +} + + +raw_dl_from_rpm_lst () { + local FILE="$1" + local RPM="" + local URL="" + local ERROR_COUNT=0 + + # Expected format # + grep -v '^#' $FILE | while IFS='#' read -r RPM URL; do + echo "Processing: RPM=$RPM URL=$URL" + dl_file_from_url "$URL" + ERR_COUNT=$((ERR_COUNT+$?)) + done + + return $ERR_COUNT +} + +raw_dl_from_non_rpm_lst () { + local FILE="$1" + local TAR="" + local URL="" + local METHOD="" + local UTIL="" + local SCRIPT="" + local BRANCH="" + local SUBDIRS_FILE="" + local TARBALL_NAME="" + local ERROR_COUNT=0 + + # Expected format ## + # or !####[]#[