diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 83d040b1..1c5a6060 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -31,6 +31,11 @@ for ff in $all; do if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name else + if [ -e "$save_path/$_name" ]; then + echo "Already have $save_path/$_name" + continue + fi + echo "remote path: $url_prefix/$_name" echo "local path: $save_path/$_name" if wget $url_prefix/$_name; then diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 303bf94e..327f2143 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -11,6 +11,7 @@ usage() { echo "Options:" echo " -n: Do not use sudo when performing operations" echo " -c: Use an alternate yum.conf rather than the system file" + echo " -x: Clean log files only, do not run." echo " rpm_list: a list of RPM files to be downloaded." echo " match_level: value could be L1, L2 or L3:" echo " L1: use name, major version and minor version:" @@ -19,6 +20,12 @@ usage() { echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" echo " L3: use name:" echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " K1: Use Koji rather than yum repos as a source." + echo " Koji has a longer retention period than epel mirrors." + echo "" + echo "Returns: 0 = All files downloaded successfully" + echo " 1 = Some files could not be downloaded" + echo " 2 = Bad arguements or other error" echo "" } @@ -34,13 +41,20 @@ get_from() { SUDOCMD="sudo -E" YUMCONFOPT="" +CLEAN_LOGS_ONLY=0 +dl_rc=0 + # Parse option flags -while getopts "c:nh" o; do +while getopts "c:nxh" o; do case "${o}" in n) # No-sudo SUDOCMD="" ;; + x) + # Clean only + CLEAN_LOGS_ONLY=1 + ;; c) # Use an alternate yum.conf YUMCONFOPT="-c $OPTARG" @@ -52,7 +66,7 @@ while getopts "c:nh" o; do ;; *) usage - exit 1 + exit 2 ;; esac done @@ -60,12 +74,12 @@ shift $((OPTIND-1)) if [ $# -lt 2 ]; then usage - exit -1 + exit 2 fi if [ "$1" == "" ]; then echo "Need to supply the rpm file list" - exit -1; + exit 2; else rpms_list=$1 echo "using $rpms_list as the download name lists" @@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS cat /dev/null > $URL_RPMS +if [ $CLEAN_LOGS_ONLY -eq 1 ];then + exit 0 +fi + +# Function to split an rpm filename into parts. +# +# Returns a space seperated list containing: +# +# +split_filename () { + local rpm_filename=$1 + + local RPM="" + local SFILE="" + local ARCH="" + local RELEASE="" + local VERSION="" + local NAME="" + + RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) + ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) + RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + + if [[ $NAME = *":"* ]]; then + EPOCH=$(echo $NAME | cut -d':' -f-1) + NAME=$(echo $NAME | cut -d':' -f2-) + fi + + echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" +} + +# Function to predict the URL where a rpm might be found. +# Assumes the rpm was compile for EPEL by fedora's koji. +koji_url () { + local rpm_filename=$1 + + local arr=( $(split_filename $rpm_filename) ) + + local n=${arr[0]} + local v=${arr[1]} + local r=${arr[2]} + local a=${arr[3]} + local e=${arr[4]} + + echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" +} + # Function to download different types of RPMs in different ways download () { - _file=$1 - _level=$2 - _list=$(cat $_file) - _from=$(get_from $_file) + local _file=$1 + local _level=$2 + + local _list=$(cat $_file) + local _from=$(get_from $_file) + local _type="" + + local rc=0 + local download_cmd="" + local download_url_cmd="" + local rpm_name="" + local rpm_url="" + local SFILE="" + echo "now the rpm will come from: $_from" for ff in $_list; do + download_cmd="" + download_url_cmd="" _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) # Decide if the list will be downloaded using yumdownloader or wget if [[ $ff != *"#"* ]]; then rpm_name=$ff - # Cut the rpm name for the specified level (L1, L2 or L3) - if [ $_level == "L1" ]; then + if [ $_level == "K1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - elif [ $match_level == "L2" ];then - SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + rpm_url=$(koji_url $rpm_name) + download_cmd="wget $rpm_url)" + download_url_cmd="echo $rpm_url)" else - SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` - fi - echo " ------ using $SFILE to search $rpm_name ------" - # Yumdownloader with the appropriate flag for src, noarch or x86_64 - if [ "$_type" == "src" ];then - download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" - else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + if [ $_level == "L1" ]; then + SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` + elif [ $match_level == "L2" ];then + SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + else + SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` + fi + echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 + if [ "$_type" == "src" ];then + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" + else + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + fi fi else # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" + download_url_cmd="echo $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -165,15 +250,18 @@ download () { fi echo $rpm_name >> $FOUND_SRPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_SRPMS + rc=1 fi else - echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" + echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" echo $rpm_name >> $FOUND_SRPMS fi else ## noarch or x86_64 if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then echo "Looking for $rpm_name..." + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -187,7 +275,9 @@ download () { fi echo $rpm_name >> $FOUND_RPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_RPMS + rc=1 fi else echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" @@ -195,6 +285,8 @@ download () { fi fi done + + return $rc } # Prime the cache @@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache if [ -s "$rpms_list" ];then echo "--> start searching "$rpms_list download $rpms_list $match_level + if [ $? -ne 0 ]; then + dl_rc=1 + fi fi echo "done!!" -exit 0 +exit $dl_rc diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index e0a64331..0658a99b 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + # Remove leading '!' if present + tarball_name="${tarball_name//!/}" + # - For the General category and the Puppet category: # - Packages have a common process: download, decompressed, # change the directory path and compressed. @@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do download_directory=$output_tarball fi + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + # We have 6 packages from the text file starting with the character "!": # they require special handling besides the common process: remove directory, # remove text from some files, clone a git repository, etc. if [[ "$line" =~ ^'!' ]]; then - tarball_name="${tarball_name//!/}" - if [ -e "$output_tarball/$tarball_name" ]; then - echo "Already have $tarball_name" - continue - fi echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then @@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do else echo "$pkg_version : unknown version" fi - rm -f "$tarball_name" + # Don't delete the original MLNX_OFED_LINUX tarball. + # We don't use it, but it will prevent re-downloading this file. + # rm -f "$tarball_name" + rm -rf "MLNX_OFED_SRC-${pkg_version}" rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 963f7661..e74d71fd 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst" rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" other_downloads="./other_downloads.lst" +# Overall success +success=1 + # Parse out optional -c or -n arguments while getopts "c:ngh" o; do case "${o}" in @@ -71,7 +74,7 @@ need_file(){ for f in $*; do if [ ! -e $f ]; then echo "ERROR: $f does not exist." - exit -1 + exit 1 fi done } @@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo} need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst + #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." @@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi logfile="log_download_3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then - echo "ERROR: something wrong with downloading, please check the log!!" +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" logfile="log_download_centos3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 +fi if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo fi + echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" logfile="log_download_centos_L1.txt" -$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile +$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then + +K1_logfile="log_download_rpms_from_centos_K1.txt" +if [ $retcode -ne 1 ]; then + # K1 step not needed. Clear any K1 logs from previous download attempts. + $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile +fi + +if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" +elif [ $retcode -eq 1 ]; then + echo "finish 1st round of RPM downloading with missing files!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` + + echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." + $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile + retcode=${PIPESTATUS[0]} + if [ $retcode -eq 0 ]; then + echo "finish 2nd round of RPM downloading successfully!" + elif [ $retcode -eq 1 ]; then + echo "finish 2nd round of RPM downloading with missing files!" + if [ -e "./output/rpms_missing_K1.txt" ]; then + echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !" + fi + fi + + # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent + # false reporting of missing files. + grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp + mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt + + + missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: -------RPMs missing: $missing_num ---------------" + retcode=1 fi fi if [ -e "./output/centos_srpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: --------- SRPMs missing: $missing_num ---------------" + retcode=1 fi fi -else - echo "finish 1st round with failures!" +fi + +if [ $retcode -ne 0 ]; then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." + echo " Please check the logs at $(pwd)/logs/$logfile" + echo " and $(pwd)/logs/$K1_logfile !" + echo "" + success=0 fi ## verify all RPMs SRPMs we download for the GPG keys @@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` let total_line=$line1+$line2+$line3 -echo "We expect to download $total_line RPMs." +echo "We expected to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` echo "There are $num_of_downloaded_rpms RPMs in output directory." if [ "$total_line" != "$num_of_downloaded_rpms" ]; then @@ -162,16 +213,33 @@ fi echo "step #3: start downloading other files ..." -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" +else + echo "step #3: finished with errors" + echo "ERROR: Something wrong with downloading from ${other_downloads}." + echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !" + echo "" + success=0 fi + # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -${tarball_downloader} ${tarball_downloader_extra_args} +${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then + echo "step #4: done successfully" +else + echo "step #4: finished with errors" + echo "ERROR: Something wrong with downloading tarballs." + echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !" + echo "" + success=0 +fi echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" @@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\"" echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" + +echo "" +if [ $success -ne 1 ]; then + echo "Warning: Not all download steps succeeded. You are likely missing files." + exit 1 +fi + +echo "Success" +exit 0