From fa995d69812c43b0bc908f96648e3170a92bfad5 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Thu, 9 Aug 2018 14:22:40 -0400 Subject: [PATCH] Use kojipkgs.fedoraproject.org as a backup rpm source. EPEL7 rpms builds are visible at kojipkgs.fedoraproject.org and often remain available there long after they age out of the official repo. Use kojipkgs.fedoraproject.org as a backup source for packages when not found in the repo. Log seperately so that these dropouts can be identified and corrected in the lst files. Also improved the tests for already downloaded packages in several places to accelerate the test cycle. Change-Id: Iaaa9fe05ac605a3604acf0048c571c6e88303692 Story: 2003157 Task: 23292 Signed-off-by: Scott Little --- .../dl_other_from_centos_repo.sh | 5 + centos-mirror-tools/dl_rpms.sh | 145 +++++++++++++++--- centos-mirror-tools/dl_tarball.sh | 18 ++- centos-mirror-tools/download_mirror.sh | 107 +++++++++++-- 4 files changed, 229 insertions(+), 46 deletions(-) diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 83d040b1..1c5a6060 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -31,6 +31,11 @@ for ff in $all; do if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name else + if [ -e "$save_path/$_name" ]; then + echo "Already have $save_path/$_name" + continue + fi + echo "remote path: $url_prefix/$_name" echo "local path: $save_path/$_name" if wget $url_prefix/$_name; then diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 303bf94e..327f2143 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -11,6 +11,7 @@ usage() { echo "Options:" echo " -n: Do not use sudo when performing operations" echo " -c: Use an alternate yum.conf rather than the system file" + echo " -x: Clean log files only, do not run." echo " rpm_list: a list of RPM files to be downloaded." echo " match_level: value could be L1, L2 or L3:" echo " L1: use name, major version and minor version:" @@ -19,6 +20,12 @@ usage() { echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" echo " L3: use name:" echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " K1: Use Koji rather than yum repos as a source." + echo " Koji has a longer retention period than epel mirrors." + echo "" + echo "Returns: 0 = All files downloaded successfully" + echo " 1 = Some files could not be downloaded" + echo " 2 = Bad arguements or other error" echo "" } @@ -34,13 +41,20 @@ get_from() { SUDOCMD="sudo -E" YUMCONFOPT="" +CLEAN_LOGS_ONLY=0 +dl_rc=0 + # Parse option flags -while getopts "c:nh" o; do +while getopts "c:nxh" o; do case "${o}" in n) # No-sudo SUDOCMD="" ;; + x) + # Clean only + CLEAN_LOGS_ONLY=1 + ;; c) # Use an alternate yum.conf YUMCONFOPT="-c $OPTARG" @@ -52,7 +66,7 @@ while getopts "c:nh" o; do ;; *) usage - exit 1 + exit 2 ;; esac done @@ -60,12 +74,12 @@ shift $((OPTIND-1)) if [ $# -lt 2 ]; then usage - exit -1 + exit 2 fi if [ "$1" == "" ]; then echo "Need to supply the rpm file list" - exit -1; + exit 2; else rpms_list=$1 echo "using $rpms_list as the download name lists" @@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS cat /dev/null > $URL_RPMS +if [ $CLEAN_LOGS_ONLY -eq 1 ];then + exit 0 +fi + +# Function to split an rpm filename into parts. +# +# Returns a space seperated list containing: +# +# +split_filename () { + local rpm_filename=$1 + + local RPM="" + local SFILE="" + local ARCH="" + local RELEASE="" + local VERSION="" + local NAME="" + + RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) + ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) + RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + + if [[ $NAME = *":"* ]]; then + EPOCH=$(echo $NAME | cut -d':' -f-1) + NAME=$(echo $NAME | cut -d':' -f2-) + fi + + echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" +} + +# Function to predict the URL where a rpm might be found. +# Assumes the rpm was compile for EPEL by fedora's koji. +koji_url () { + local rpm_filename=$1 + + local arr=( $(split_filename $rpm_filename) ) + + local n=${arr[0]} + local v=${arr[1]} + local r=${arr[2]} + local a=${arr[3]} + local e=${arr[4]} + + echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" +} + # Function to download different types of RPMs in different ways download () { - _file=$1 - _level=$2 - _list=$(cat $_file) - _from=$(get_from $_file) + local _file=$1 + local _level=$2 + + local _list=$(cat $_file) + local _from=$(get_from $_file) + local _type="" + + local rc=0 + local download_cmd="" + local download_url_cmd="" + local rpm_name="" + local rpm_url="" + local SFILE="" + echo "now the rpm will come from: $_from" for ff in $_list; do + download_cmd="" + download_url_cmd="" _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) # Decide if the list will be downloaded using yumdownloader or wget if [[ $ff != *"#"* ]]; then rpm_name=$ff - # Cut the rpm name for the specified level (L1, L2 or L3) - if [ $_level == "L1" ]; then + if [ $_level == "K1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - elif [ $match_level == "L2" ];then - SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + rpm_url=$(koji_url $rpm_name) + download_cmd="wget $rpm_url)" + download_url_cmd="echo $rpm_url)" else - SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` - fi - echo " ------ using $SFILE to search $rpm_name ------" - # Yumdownloader with the appropriate flag for src, noarch or x86_64 - if [ "$_type" == "src" ];then - download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" - else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + if [ $_level == "L1" ]; then + SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` + elif [ $match_level == "L2" ];then + SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + else + SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` + fi + echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 + if [ "$_type" == "src" ];then + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" + else + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + fi fi else # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" + download_url_cmd="echo $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -165,15 +250,18 @@ download () { fi echo $rpm_name >> $FOUND_SRPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_SRPMS + rc=1 fi else - echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" + echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" echo $rpm_name >> $FOUND_SRPMS fi else ## noarch or x86_64 if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then echo "Looking for $rpm_name..." + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -187,7 +275,9 @@ download () { fi echo $rpm_name >> $FOUND_RPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_RPMS + rc=1 fi else echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" @@ -195,6 +285,8 @@ download () { fi fi done + + return $rc } # Prime the cache @@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache if [ -s "$rpms_list" ];then echo "--> start searching "$rpms_list download $rpms_list $match_level + if [ $? -ne 0 ]; then + dl_rc=1 + fi fi echo "done!!" -exit 0 +exit $dl_rc diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index e0a64331..0658a99b 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + # Remove leading '!' if present + tarball_name="${tarball_name//!/}" + # - For the General category and the Puppet category: # - Packages have a common process: download, decompressed, # change the directory path and compressed. @@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do download_directory=$output_tarball fi + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + # We have 6 packages from the text file starting with the character "!": # they require special handling besides the common process: remove directory, # remove text from some files, clone a git repository, etc. if [[ "$line" =~ ^'!' ]]; then - tarball_name="${tarball_name//!/}" - if [ -e "$output_tarball/$tarball_name" ]; then - echo "Already have $tarball_name" - continue - fi echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then @@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do else echo "$pkg_version : unknown version" fi - rm -f "$tarball_name" + # Don't delete the original MLNX_OFED_LINUX tarball. + # We don't use it, but it will prevent re-downloading this file. + # rm -f "$tarball_name" + rm -rf "MLNX_OFED_SRC-${pkg_version}" rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 963f7661..e74d71fd 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst" rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" other_downloads="./other_downloads.lst" +# Overall success +success=1 + # Parse out optional -c or -n arguments while getopts "c:ngh" o; do case "${o}" in @@ -71,7 +74,7 @@ need_file(){ for f in $*; do if [ ! -e $f ]; then echo "ERROR: $f does not exist." - exit -1 + exit 1 fi done } @@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo} need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst + #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." @@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi logfile="log_download_3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then - echo "ERROR: something wrong with downloading, please check the log!!" +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" logfile="log_download_centos3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 +fi if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo fi + echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" logfile="log_download_centos_L1.txt" -$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile +$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then + +K1_logfile="log_download_rpms_from_centos_K1.txt" +if [ $retcode -ne 1 ]; then + # K1 step not needed. Clear any K1 logs from previous download attempts. + $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile +fi + +if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" +elif [ $retcode -eq 1 ]; then + echo "finish 1st round of RPM downloading with missing files!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` + + echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." + $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile + retcode=${PIPESTATUS[0]} + if [ $retcode -eq 0 ]; then + echo "finish 2nd round of RPM downloading successfully!" + elif [ $retcode -eq 1 ]; then + echo "finish 2nd round of RPM downloading with missing files!" + if [ -e "./output/rpms_missing_K1.txt" ]; then + echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !" + fi + fi + + # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent + # false reporting of missing files. + grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp + mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt + + + missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: -------RPMs missing: $missing_num ---------------" + retcode=1 fi fi if [ -e "./output/centos_srpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: --------- SRPMs missing: $missing_num ---------------" + retcode=1 fi fi -else - echo "finish 1st round with failures!" +fi + +if [ $retcode -ne 0 ]; then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." + echo " Please check the logs at $(pwd)/logs/$logfile" + echo " and $(pwd)/logs/$K1_logfile !" + echo "" + success=0 fi ## verify all RPMs SRPMs we download for the GPG keys @@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` let total_line=$line1+$line2+$line3 -echo "We expect to download $total_line RPMs." +echo "We expected to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` echo "There are $num_of_downloaded_rpms RPMs in output directory." if [ "$total_line" != "$num_of_downloaded_rpms" ]; then @@ -162,16 +213,33 @@ fi echo "step #3: start downloading other files ..." -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" +else + echo "step #3: finished with errors" + echo "ERROR: Something wrong with downloading from ${other_downloads}." + echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !" + echo "" + success=0 fi + # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -${tarball_downloader} ${tarball_downloader_extra_args} +${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then + echo "step #4: done successfully" +else + echo "step #4: finished with errors" + echo "ERROR: Something wrong with downloading tarballs." + echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !" + echo "" + success=0 +fi echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" @@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\"" echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" + +echo "" +if [ $success -ne 1 ]; then + echo "Warning: Not all download steps succeeded. You are likely missing files." + exit 1 +fi + +echo "Success" +exit 0