Merge "Use kojipkgs.fedoraproject.org as a backup rpm source."

centos-mirror-tools/dl_other_from_centos_repo.sh

@@ -31,6 +31,11 @@ for ff in $all; do
     if [ "$_type" == "folder" ];then
         mkdir -p $save_path/$_name
     else
+        if [ -e "$save_path/$_name" ]; then
+            echo "Already have $save_path/$_name"
+            continue
+        fi
+
         echo "remote path: $url_prefix/$_name"
         echo "local path: $save_path/$_name"
         if wget $url_prefix/$_name; then

centos-mirror-tools/dl_rpms.sh

@@ -11,6 +11,7 @@ usage() {
     echo "Options:"
     echo "  -n: Do not use sudo when performing operations"
     echo "  -c: Use an alternate yum.conf rather than the system file"
+    echo "  -x: Clean log files only, do not run."
     echo "  rpm_list: a list of RPM files to be downloaded."
     echo "  match_level: value could be L1, L2 or L3:"
     echo "    L1: use name, major version and minor version:"
@@ -19,6 +20,12 @@ usage() {
     echo "        using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
     echo "    L3: use name:"
     echo "        using vim to search vim-7.4.160-2.el7.src.rpm"
+    echo "    K1: Use Koji rather than yum repos as a source."
+    echo "        Koji has a longer retention period than epel mirrors."
+    echo ""
+    echo "Returns: 0 = All files downloaded successfully"
+    echo "         1 = Some files could not be downloaded"
+    echo "         2 = Bad arguements or other error"
     echo ""
 }
 
@@ -34,13 +41,20 @@ get_from() {
 SUDOCMD="sudo -E"
 YUMCONFOPT=""
 
+CLEAN_LOGS_ONLY=0
+dl_rc=0
+
 # Parse option flags
-while getopts "c:nh" o; do
+while getopts "c:nxh" o; do
     case "${o}" in
         n)
             # No-sudo
             SUDOCMD=""
             ;;
+        x)
+            # Clean only
+            CLEAN_LOGS_ONLY=1
+            ;;
         c)
             # Use an alternate yum.conf
             YUMCONFOPT="-c $OPTARG"
@@ -52,7 +66,7 @@ while getopts "c:nh" o; do
             ;;
         *)
             usage
-            exit 1
+            exit 2
             ;;
     esac
 done
@@ -60,12 +74,12 @@ shift $((OPTIND-1))
 
 if [ $# -lt 2 ]; then
     usage
-    exit -1
+    exit 2
 fi
 
 if [ "$1" == "" ]; then
     echo "Need to supply the rpm file list"
-    exit -1;
+    exit 2;
 else
     rpms_list=$1
     echo "using $rpms_list as the download name lists"
@@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS
 cat /dev/null > $MISSING_RPMS
 cat /dev/null > $URL_RPMS
 
+if [ $CLEAN_LOGS_ONLY -eq 1 ];then
+    exit 0
+fi
+
+# Function to split an rpm filename into parts.
+#
+# Returns a space seperated list containing:
+#    <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
+#
+split_filename () {
+    local rpm_filename=$1
+
+    local RPM=""
+    local SFILE=""
+    local ARCH=""
+    local RELEASE=""
+    local VERSION=""
+    local NAME=""
+
+    RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
+    ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
+    RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+    VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+
+    if [[ $NAME = *":"* ]]; then
+        EPOCH=$(echo $NAME | cut -d':' -f-1)
+        NAME=$(echo $NAME | cut -d':' -f2-)
+    fi
+
+    echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
+}
+
+# Function to predict the URL where a rpm might be found.
+# Assumes the rpm was compile for EPEL by fedora's koji.
+koji_url () {
+    local rpm_filename=$1
+
+    local arr=( $(split_filename $rpm_filename) )
+
+    local n=${arr[0]}
+    local v=${arr[1]}
+    local r=${arr[2]}
+    local a=${arr[3]}
+    local e=${arr[4]}
+
+    echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
+}
+
 # Function to download different types of RPMs in different ways
 download () {
-    _file=$1
+    local _file=$1
-    _level=$2
+    local _level=$2
-    _list=$(cat $_file)
+
-    _from=$(get_from $_file)
+    local _list=$(cat $_file)
+    local _from=$(get_from $_file)
+    local _type=""
+
+    local rc=0
+    local download_cmd=""
+    local download_url_cmd=""
+    local rpm_name=""
+    local rpm_url=""
+    local SFILE=""
+
     echo "now the rpm will come from: $_from"
     for ff in $_list; do
+        download_cmd=""
+        download_url_cmd=""
         _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
 
         # Decide if the list will be downloaded using yumdownloader or wget
         if [[ $ff != *"#"* ]]; then
             rpm_name=$ff
 
-            # Cut the rpm name for the specified level (L1, L2 or L3)
+            if [ $_level == "K1" ]; then
-            if [ $_level == "L1" ]; then
                 SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-            elif [ $match_level == "L2" ];then
+                rpm_url=$(koji_url $rpm_name)
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
+                download_cmd="wget $rpm_url)"
+                download_url_cmd="echo $rpm_url)"
             else
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
+                if [ $_level == "L1" ]; then
-            fi
+                    SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-            echo " ------ using $SFILE to search $rpm_name ------"
+                elif [ $match_level == "L2" ];then
-            # Yumdownloader with the appropriate flag for src, noarch or x86_64
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
-            if [ "$_type" == "src" ];then
+                else
-                download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
-                download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
+                fi
-            else
+                echo " ------ using $SFILE to search $rpm_name ------"
-                download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                # Yumdownloader with the appropriate flag for src, noarch or x86_64
-                download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                if [ "$_type" == "src" ];then
+                    download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
+                else
+                    download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                fi
             fi
         else
             # Buid wget command
             rpm_name=`echo $ff | cut -d"#" -f1-1`
             rpm_url=`echo $ff | cut -d"#" -f2-2`
             download_cmd="wget $rpm_url"
+            download_url_cmd="echo $rpm_url"
             SFILE=$rpm_name
         fi
 
-        echo "--> run: $download_cmd"
         # Put the RPM in the Binary or Source directory
         if [ "$_type" == "src" ]; then
             if [ ! -e $MDIR_SRC/$rpm_name ]; then
                 echo "Looking for $rpm_name"
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
                     # Use 'sort --unique' because sometimes
@@ -165,15 +250,18 @@ download () {
                     fi
                     echo $rpm_name >> $FOUND_SRPMS
                 else
+                    echo "Warning: $rpm_name not found"
                     echo $rpm_name >> $MISSING_SRPMS
+                    rc=1
                 fi
             else
-                echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
+                echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
                 echo $rpm_name >> $FOUND_SRPMS
             fi
         else  ## noarch or x86_64
             if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
                 echo "Looking for $rpm_name..."
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
                     # Use 'sort --unique' because sometimes
@@ -187,7 +275,9 @@ download () {
                     fi
                     echo $rpm_name >> $FOUND_RPMS
                 else
+                    echo "Warning: $rpm_name not found"
                     echo $rpm_name >> $MISSING_RPMS
+                    rc=1
                 fi
             else
                 echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
@@ -195,6 +285,8 @@ download () {
             fi
         fi
     done
+
+    return $rc
 }
 
 # Prime the cache
@@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
 if [ -s "$rpms_list" ];then
     echo "--> start searching "$rpms_list
     download $rpms_list $match_level
+    if [ $? -ne 0 ]; then
+        dl_rc=1
+    fi
 fi
 
 echo "done!!"
 
-exit 0
+exit $dl_rc

centos-mirror-tools/dl_tarball.sh

@@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do
     directory_name=$(echo $line | cut -d"#" -f2-2)
     tarball_url=$(echo $line | cut -d"#" -f3-3)
 
+    # Remove leading '!' if present
+    tarball_name="${tarball_name//!/}"
+
     # - For the General category and the Puppet category:
     #   - Packages have a common process: download, decompressed,
     #     change the directory path and compressed.
@@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do
         download_directory=$output_tarball
     fi
 
+    if [ -e $download_path ]; then
+        echo "Already have $download_path"
+        continue
+    fi
+
     # We have 6 packages from the text file starting with the character "!":
     # they require special handling besides the common process: remove directory,
     # remove text from some files, clone a git repository, etc.
 
     if [[ "$line" =~ ^'!' ]]; then
-        tarball_name="${tarball_name//!/}"
-        if [ -e "$output_tarball/$tarball_name" ]; then
-            echo "Already have $tarball_name"
-            continue
-        fi
         echo $tarball_name
         pushd $output_tarball
         if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
@@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do
             else
                 echo "$pkg_version : unknown version"
             fi
-            rm -f "$tarball_name"
+            # Don't delete the original MLNX_OFED_LINUX tarball.
+            # We don't use it, but it will prevent re-downloading this file.
+            #   rm -f "$tarball_name"
+
             rm -rf "MLNX_OFED_SRC-${pkg_version}"
             rm -rf "$directory_name"
         elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then

centos-mirror-tools/download_mirror.sh

@@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst"
 rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
 other_downloads="./other_downloads.lst"
 
+# Overall success
+success=1
+
 # Parse out optional -c or -n arguments
 while getopts "c:ngh" o; do
     case "${o}" in
@@ -71,7 +74,7 @@ need_file(){
     for f in $*; do
         if [ ! -e $f ]; then
             echo "ERROR: $f does not exist."
-            exit -1
+            exit 1
         fi
     done
 }
@@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo}
 need_file ${other_downloads}
 need_file tarball-dl.lst mvn-artifacts.lst
 
+
 #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
 echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
 
@@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
 fi
 
 logfile="log_download_3rdparties_L1.txt"
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
-if [ $retcode -ne 0 ]; then
+if [ $retcode -ne 0 ];then
-    echo "ERROR: something wrong with downloading, please check the log!!"
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
+    echo "   Please check the log at $(pwd)/logs/$logfile !"
+    echo ""
+    success=0
 fi
 
 # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
 logfile="log_download_centos3rdparties_L1.txt"
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -ne 0 ];then
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
+    echo "   Please check the log at $(pwd)/logs/$logfile !"
+    echo ""
+    success=0
+fi
 
 if [ ${use_system_yum_conf} -eq 1 ]; then
     # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
     \rm -f $REPO_DIR/StarlingX_3rd*.repo
 fi
 
+
 echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
 #download RPMs/SRPMs from CentOS repos by "yumdownloader"
 logfile="log_download_centos_L1.txt"
-$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
+$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
-if [ $retcode -ne 0 ]; then
+
+K1_logfile="log_download_rpms_from_centos_K1.txt"
+if [ $retcode -ne 1 ]; then
+    # K1 step not needed. Clear any K1 logs from previous download attempts.
+    $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
+fi
+
+if [ $retcode -eq 0 ]; then
     echo "finish 1st round of RPM downloading successfully!"
+elif [ $retcode -eq 1 ]; then
+    echo "finish 1st round of RPM downloading with missing files!"
     if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
-        missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
+
+        echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
+        $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
+        retcode=${PIPESTATUS[0]}
+        if [ $retcode -eq 0 ]; then
+            echo "finish 2nd round of RPM downloading successfully!"
+        elif [ $retcode -eq 1 ]; then
+            echo "finish 2nd round of RPM downloading with missing files!"
+            if [ -e "./output/rpms_missing_K1.txt" ]; then
+                echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
+            fi
+        fi
+
+        # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
+        # false reporting of missing files.
+        grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt  > ./output/centos_rpms_missing_L1.tmp
+        mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
+
+
+        missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
         if [ "$missing_num" != "0" ];then
-            echo "ERROR:  -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
+            echo "ERROR:  -------RPMs missing: $missing_num ---------------"
+            retcode=1
         fi
     fi
 
     if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
         missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
         if [ "$missing_num" != "0" ];then
-            echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
+            echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
+            retcode=1
         fi
     fi
-else
+fi
-    echo "finish 1st round with failures!"
+
+if [ $retcode -ne 0 ]; then
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
+    echo "   Please check the logs at $(pwd)/logs/$logfile"
+    echo "   and $(pwd)/logs/$K1_logfile !"
+    echo ""
+    success=0
 fi
 
 ## verify all RPMs SRPMs we download for the GPG keys
@@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
 line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
 line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
 let total_line=$line1+$line2+$line3
-echo "We expect to download $total_line RPMs."
+echo "We expected to download $total_line RPMs."
 num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
 echo "There are $num_of_downloaded_rpms RPMs in output directory."
 if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
@@ -162,16 +213,33 @@ fi
 
 echo "step #3: start downloading other files ..."
 
-${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
+${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
 retcode=${PIPESTATUS[0]}
 if [ $retcode -eq 0 ];then
     echo "step #3: done successfully"
+else
+    echo "step #3: finished with errors"
+    echo "ERROR: Something wrong with downloading from ${other_downloads}."
+    echo "   Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
+    echo ""
+    success=0
 fi
 
+
 # StarlingX requires a group of source code pakages, in this section
 # they will be downloaded.
 echo "step #4: start downloading tarball compressed files"
-${tarball_downloader} ${tarball_downloader_extra_args}
+${tarball_downloader} ${tarball_downloader_extra_args}  |& tee ./logs/log_download_tarballs.txt
+retcode=${PIPESTATUS[0]}
+if [ $retcode -eq 0 ];then
+    echo "step #4: done successfully"
+else
+    echo "step #4: finished with errors"
+    echo "ERROR: Something wrong with downloading tarballs."
+    echo "   Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
+    echo ""
+    success=0
+fi
 
 echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
 echo "on them, the workable images for StarlingX could be generated by"
@@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
 echo "    - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
+
+echo ""
+if [ $success -ne 1 ]; then
+    echo "Warning: Not all download steps succeeded.  You are likely missing files."
+    exit 1
+fi
+
+echo "Success"
+exit 0