From 7f3ca9a776cb2d8c29211b7851f9f2b324e06a3e Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Wed, 5 Sep 2018 15:35:59 -0500 Subject: [PATCH] Refactor on dl_rpms.sh A refactor was made in the dl_rpms.sh script to simplify the maintenance and development. A new file utils.sh was created to store all the common functions that could be used by another scripts. Also, some unit testing was added to the util.sh file. Story: 2002736 Task: 26194 Change-Id: Iaac2e74a84abbfc9f40ef51b1da819702d040655 Signed-off-by: Marcela Rosales Signed-off-by: Erich Cordoba --- centos-mirror-tools/dl_rpms.sh | 204 +++++------------------------ centos-mirror-tools/utils.sh | 196 +++++++++++++++++++++++++++ centos-mirror-tools/utils_tests.sh | 90 +++++++++++++ 3 files changed, 322 insertions(+), 168 deletions(-) create mode 100644 centos-mirror-tools/utils.sh create mode 100644 centos-mirror-tools/utils_tests.sh diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index d6753db0..5b290741 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -5,6 +5,18 @@ # download RPMs/SRPMs from different sources. # this script was originated by Brian Avery, and later updated by Yong Hu +set -o errexit +set -o nounset + +# By default, we use "sudo" and we don't use a local yum.conf. These can +# be overridden via flags. + +SUDOCMD="sudo -E" +RELEASEVER="--releasever=7" +YUMCONFOPT="" + +source utils.sh + usage() { echo "$0 [-n] [-c ] " echo "" @@ -29,19 +41,6 @@ usage() { echo "" } -get_from() { - list=$1 - base=$(basename $list .lst) # removing lst extension - base=$(basename $base .log) # removing log extension - from=$(echo $base | rev | cut -d'_' -f1-1 | rev) - echo $from -} - -# By default, we use "sudo" and we don't use a local yum.conf. These can -# be overridden via flags. -SUDOCMD="sudo -E" -RELEASEVER="--releasever=7" -YUMCONFOPT="" CLEAN_LOGS_ONLY=0 dl_rc=0 @@ -94,12 +93,9 @@ if [ ! -z "$2" -a "$2" != " " ];then match_level=$2 fi - timestamp=$(date +%F_%H%M) echo $timestamp - - DESTDIR="output" MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source mkdir -p $MDIR_SRC @@ -124,179 +120,52 @@ if [ $CLEAN_LOGS_ONLY -eq 1 ];then exit 0 fi -# Function to split an rpm filename into parts. -# -# Returns a space seperated list containing: -# -# -split_filename () { - local rpm_filename=$1 - - local RPM="" - local SFILE="" - local ARCH="" - local RELEASE="" - local VERSION="" - local NAME="" - - RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) - SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) - ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) - SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) - RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) - SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) - VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) - NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) - - if [[ $NAME = *":"* ]]; then - EPOCH=$(echo $NAME | cut -d':' -f-1) - NAME=$(echo $NAME | cut -d':' -f2-) - fi - - echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" -} - -# Function to predict the URL where a rpm might be found. -# Assumes the rpm was compile for EPEL by fedora's koji. -koji_url () { - local rpm_filename=$1 - - local arr=( $(split_filename $rpm_filename) ) - - local n=${arr[0]} - local v=${arr[1]} - local r=${arr[2]} - local a=${arr[3]} - local e=${arr[4]} - - echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" -} - # Function to download different types of RPMs in different ways download () { local _file=$1 local _level=$2 + local _list=$(cat $_file) + local _from=$(get_from $_file) - local _list - local _from - local _type="" + local _arch="" local rc=0 local download_cmd="" - local download_url_cmd="" + local download_url="" local rpm_name="" - local rpm_url="" local SFILE="" - _list=$(cat $_file) - _from=$(get_from $_file) - echo "now the rpm will come from: $_from" for ff in $_list; do - download_cmd="" - download_url_cmd="" - _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) + _arch=$(get_arch_from_rpm $ff) + rpm_name="$(get_rpm_name $ff)" + download_cmd="$(get_download_cmd $ff $_level)" + dest_dir="$(get_dest_directory $_arch)" - # Decide if the list will be downloaded using yumdownloader or wget - if [[ $ff != *"#"* ]]; then - rpm_name=$ff - - if [ $_level == "K1" ]; then - SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - rpm_url=$(koji_url $rpm_name) - download_cmd="wget $rpm_url)" - download_url_cmd="echo $rpm_url)" + if [ ! -e $dest_dir/$rpm_name ]; then + echo "Looking for $rpm_name" + echo "--> run: $download_cmd" + if $download_cmd ; then + download_url="$(get_url $ff $_level)" + SFILE="$(get_rpm_level_name $rpm_name $_level)" + process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" else - if [ $_level == "L1" ]; then - SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - elif [ $match_level == "L2" ];then - SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` - else - SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` - fi - echo " ------ using $SFILE to search $rpm_name ------" - # Yumdownloader with the appropriate flag for src, noarch or x86_64 - if [ "$_type" == "src" ];then - download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" - else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64" - fi + echo "Warning: $rpm_name not found" + echo "missing_srpm:$rpm_name" >> $LOG + echo $rpm_name >> $MISSING_SRPMS + rc=1 fi else - # Buid wget command - rpm_name=`echo $ff | cut -d"#" -f1-1` - rpm_url=`echo $ff | cut -d"#" -f2-2` - download_cmd="wget $rpm_url" - download_url_cmd="echo $rpm_url" - SFILE=$rpm_name - fi - - # Put the RPM in the Binary or Source directory - if [ "$_type" == "src" ]; then - if [ ! -e $MDIR_SRC/$rpm_name ]; then - echo "Looking for $rpm_name" - echo "--> run: $download_cmd" - if $download_cmd ; then - # Success! Record download URL. - # Use 'sort --unique' because sometimes - # yumdownloader reports the url twice - URL=$($download_url_cmd | sort --unique) - echo "The url is: $URL" - echo "url_srpm:$URL" >> $LOG - - if ! mv -f $SFILE* $MDIR_SRC ; then - echo "FAILED to move $rpm_name" - echo "fail_move_srpm:$rpm_name" >> $LOG - fi - echo "found_srpm:$rpm_name" >> $LOG - echo $rpm_name >> $FOUND_SRPMS - else - echo "Warning: $rpm_name not found" - echo "missing_srpm:$rpm_name" >> $LOG - echo $rpm_name >> $MISSING_SRPMS - rc=1 - fi - else - echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" - echo "already_there_srpm:$rpm_name" >> $LOG - fi - else ## noarch or x86_64 - if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then - echo "Looking for $rpm_name..." - echo "--> run: $download_cmd" - if $download_cmd ; then - # Success! Record download URL. - # Use 'sort --unique' because sometimes - # yumdownloader reports the url twice - URL=$($download_url_cmd | sort --unique) - echo "The url is: $URL" - echo "url_rpm:$URL" >> $LOG - - mkdir -p $MDIR_BIN/${_type} - if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then - echo "FAILED to move $rpm_name" - echo "fail_move_rpm:$rpm_name" >> $LOG - fi - echo "found_rpm:$rpm_name" >> $LOG - echo $rpm_name >> $FOUND_RPMS - else - echo "Warning: $rpm_name not found" - echo "missing_rpm:$rpm_name" >> $LOG - echo $rpm_name >> $MISSING_RPMS - rc=1 - fi - else - echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" - echo "already_there_rpm:$rpm_name" >> $LOG - fi + echo "Already have $dest_dir/$rpm_name" + echo "already_there_srpm:$rpm_name" >> $LOG fi + echo done return $rc } + # Prime the cache ${SUDOCMD} yum ${YUMCONFOPT} ${RELEASEVER} makecache @@ -309,7 +178,6 @@ if [ -s "$rpms_list" ];then fi fi -echo "done!!" +echo "Done!" exit $dl_rc - diff --git a/centos-mirror-tools/utils.sh b/centos-mirror-tools/utils.sh new file mode 100644 index 00000000..3fa8970d --- /dev/null +++ b/centos-mirror-tools/utils.sh @@ -0,0 +1,196 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +get_yum_command() { + local _file=$1 + local _level=$2 + local rpm_name="" + local arr=( $(split_filename $_file) ) + local arch=${arr[3]} + local yumdownloader_extra_opts="" + rpm_name="$(get_rpm_level_name $_file $_level)" + + if [ "$arch" == "src" ]; then + yumdownloader_extra_opts="--source" + else + yumdownloader_extra_opts="--archlist=noarch,x86_64" + fi + + echo "yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $yumdownloader_extra_opts $rpm_name" +} + +get_wget_command() { + local _name="$1" + local _ret="" + if [[ "$_name" == http?(s)://* ]]; then + _ret="wget -q $_name" + else + _ret="wget -q $(koji_url $_name)" + fi + echo "$_ret" +} + +get_rpm_level_name() { + local _rpm_name=$1 + local _level=$2 + if [ $_level == "L1" ]; then + SFILE=`echo $_rpm_name | rev | cut -d'.' -f3- | rev` + elif [ $_level == "L2" ];then + SFILE=`echo $_rpm_name | rev | cut -d'-' -f2- | rev` + else + SFILE=`echo $_rpm_name | rev | cut -d'-' -f3- | rev` + fi + echo "$SFILE" +} + +get_url() { + local _name="$1" + local _level="$2" + local _ret="" + + if [ "$_level" == "K1" ]; then + _ret="$(koji_url $_name)" + elif [[ "$_name" == *"#"* ]]; then + _ret="$(echo $_name | cut -d'#' -f2-2)" + else + _url_cmd="$(get_yum_command $_name $_level)" + _ret="$($_url_cmd --url)" + fi + echo "$_ret" +} + +# Function to split an rpm filename into parts. +# +# Returns a space seperated list containing: +# +# +split_filename () { + local rpm_filename=$1 + + local RPM="" + local SFILE="" + local ARCH="" + local RELEASE="" + local VERSION="" + local NAME="" + local EPOCH="" + + RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) + ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) + RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + + if [[ $NAME = *":"* ]]; then + EPOCH=$(echo $NAME | cut -d':' -f-1) + NAME=$(echo $NAME | cut -d':' -f2-) + fi + + echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" +} + +# Function to predict the URL where a rpm might be found. +# Assumes the rpm was compile for EPEL by fedora's koji. +koji_url () { + local rpm_filename=$1 + + local arr=( $(split_filename $rpm_filename) ) + + local n=${arr[0]} + local v=${arr[1]} + local r=${arr[2]} + local a=${arr[3]} + + echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" +} + +get_dest_directory() { + local _type=$1 + local _dest="" + if [ "$_type" == "src" ]; then + _dest="$MDIR_SRC" + else + _dest="$MDIR_BIN/$_type" + fi + echo "$_dest" +} + +process_result() { + local _type="$1" + local dest_dir="$2" + local url="$3" + local sfile="$4" + + if [ "$_type" != "src" ] && [ ! -d $dest_dir ]; then + mkdir -p $dest_dir + fi + + echo "url_srpm:$url" >> $LOG + + if ! mv -f $sfile* $dest_dir ; then + echo "FAILED to move $rpm_name" + echo "fail_move_srpm:$rpm_name" >> $LOG + fi + echo "found_srpm:$rpm_name" >> $LOG + echo $rpm_name >> $FOUND_SRPMS +} + + +get_download_cmd() { + local ff="$1" + local _level="$2" + + # Decide if the list will be downloaded using yumdownloader or wget + if [[ $ff != *"#"* ]]; then + rpm_name=$ff + if [ $_level == "K1" ]; then + download_cmd="$(get_wget_command $rpm_name)" + else + # yumdownloader with the appropriate flag for src, noarch or x86_64 + download_cmd="${SUDOCMD} $(get_yum_command $rpm_name $_level)" + fi + else + # Build wget command + rpm_url=$(echo $ff | cut -d"#" -f2-2) + download_cmd="$(get_wget_command $rpm_url)" + fi + + echo "$download_cmd" +} + +get_rpm_name() { + local ret="" + + if [[ "$1" != *"#"* ]]; then + ret="$1" + else + ret="$(echo $1 | cut -d"#" -f1-1)" + fi + echo "$ret" +} + +get_arch_from_rpm() { + local _file=$1 + local _split=() + local _arch="" + if [[ "$1" == *"#"* ]]; then + _file=$(echo $_file | cut -d"#" -f1-1) + fi + + _split=( $(split_filename $_file) ) + _arch=${_split[3]} + + echo "$_arch" +} + +get_from() { + list=$1 + base=$(basename $list .lst) # removing lst extension + base=$(basename $base .log) # removing log extension + from=$(echo $base | rev | cut -d'_' -f1-1 | rev) + echo $from +} diff --git a/centos-mirror-tools/utils_tests.sh b/centos-mirror-tools/utils_tests.sh new file mode 100644 index 00000000..de598811 --- /dev/null +++ b/centos-mirror-tools/utils_tests.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Set of unit tests for dl_rpms.sh + +set -o errexit +set -o nounset + +YUMCONFOPT="" +RELEASEVER="--releasever=7" + +source utils.sh + +check_result() { + local _res="$1" + local _expect="$2" + if [ "$_res" != "$_expect" ]; then + echo "Fail" + echo "expected $_expect" + echo "returned $_res" + exit 1 + fi + echo "Success" +} + +# get_wget_command + +res=$(get_wget_command "https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm") +expect="wget -q https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm" +check_result "$res" "$expect" + +res=$(get_wget_command "python2-httpbin-0.5.0-6.el7.noarch.rpm") +expect="wget -q https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" +check_result "$res" "$expect" + +# get_url + +res=$(get_url "acpid-2.0.19-9.el7.x86_64.rpm" "L1") +expect="http://vault.centos.org/centos/7.4.1708/cr/x86_64/Packages/acpid-2.0.19-9.el7.x86_64.rpm" +check_result "$res" "$expect" + +res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" "L1") +expect="http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" +check_result "$res" "$expect" + +res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm" "K1") +expect="https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" +check_result "$res" "$expect" + +# get_yum_command + +res=$(get_yum_command "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L1") +expect="yumdownloader -q -C --releasever=7 --source anaconda-21.48.22.121-1.el7.centos" +check_result "$res" "$expect" + +res=$(get_yum_command "acpid-2.0.19-9.el7.x86_64.rpm" "L1") +expect="yumdownloader -q -C --releasever=7 --archlist=noarch,x86_64 acpid-2.0.19-9.el7" +check_result "$res" "$expect" + +# get_rpm_level_name + +res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L1") +expect="acl-2.2.51-12.el7" +check_result "$res" "$expect" + +res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L3") +expect="acl" +check_result "$res" "$expect" + +res=$(get_rpm_level_name "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L2") +expect="anaconda-21.48.22.121" +check_result "$res" "$expect" + +res=$(get_arch_from_rpm "acl-2.2.51-12.el7.x86_64.rpm") +expect="x86_64" +check_result "$res" "$expect" + +res=$(get_arch_from_rpm "acl-2.2.51-12.el7.noarch.rpm") +expect="noarch" +check_result "$res" "$expect" + +res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm") +expect="src" +check_result "$res" "$expect" + +res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm#https://someurl.com/acl-2.2.51-12.el7.src.rpm") +expect="src" +check_result "$res" "$expect"