Refactor on dl_rpms.sh

A refactor was made in the dl_rpms.sh script to simplify the
maintenance and development. A new file utils.sh was created to
store all the common functions that could be used by another scripts.

Also, some unit testing was added to the util.sh file.

Story: 2002736
Task: 26194

Change-Id: Iaac2e74a84abbfc9f40ef51b1da819702d040655
Signed-off-by: Marcela Rosales <marcela.a.rosales.jimenez@intel.com>
Signed-off-by: Erich Cordoba <erich.cordoba.malibran@intel.com>
This commit is contained in:
Erich Cordoba 2018-09-05 15:35:59 -05:00
parent 8ff2f0e8ba
commit 7f3ca9a776
3 changed files with 322 additions and 168 deletions

View File

@ -5,6 +5,18 @@
# download RPMs/SRPMs from different sources.
# this script was originated by Brian Avery, and later updated by Yong Hu
set -o errexit
set -o nounset
# By default, we use "sudo" and we don't use a local yum.conf. These can
# be overridden via flags.
SUDOCMD="sudo -E"
RELEASEVER="--releasever=7"
YUMCONFOPT=""
source utils.sh
usage() {
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
echo ""
@ -29,19 +41,6 @@ usage() {
echo ""
}
get_from() {
list=$1
base=$(basename $list .lst) # removing lst extension
base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from
}
# By default, we use "sudo" and we don't use a local yum.conf. These can
# be overridden via flags.
SUDOCMD="sudo -E"
RELEASEVER="--releasever=7"
YUMCONFOPT=""
CLEAN_LOGS_ONLY=0
dl_rc=0
@ -94,12 +93,9 @@ if [ ! -z "$2" -a "$2" != " " ];then
match_level=$2
fi
timestamp=$(date +%F_%H%M)
echo $timestamp
DESTDIR="output"
MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
mkdir -p $MDIR_SRC
@ -124,179 +120,52 @@ if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0
fi
# Function to split an rpm filename into parts.
#
# Returns a space seperated list containing:
# <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
#
split_filename () {
local rpm_filename=$1
local RPM=""
local SFILE=""
local ARCH=""
local RELEASE=""
local VERSION=""
local NAME=""
RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
if [[ $NAME = *":"* ]]; then
EPOCH=$(echo $NAME | cut -d':' -f-1)
NAME=$(echo $NAME | cut -d':' -f2-)
fi
echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
}
# Function to predict the URL where a rpm might be found.
# Assumes the rpm was compile for EPEL by fedora's koji.
koji_url () {
local rpm_filename=$1
local arr=( $(split_filename $rpm_filename) )
local n=${arr[0]}
local v=${arr[1]}
local r=${arr[2]}
local a=${arr[3]}
local e=${arr[4]}
echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
}
# Function to download different types of RPMs in different ways
download () {
local _file=$1
local _level=$2
local _list=$(cat $_file)
local _from=$(get_from $_file)
local _list
local _from
local _type=""
local _arch=""
local rc=0
local download_cmd=""
local download_url_cmd=""
local download_url=""
local rpm_name=""
local rpm_url=""
local SFILE=""
_list=$(cat $_file)
_from=$(get_from $_file)
echo "now the rpm will come from: $_from"
for ff in $_list; do
download_cmd=""
download_url_cmd=""
_type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
_arch=$(get_arch_from_rpm $ff)
rpm_name="$(get_rpm_name $ff)"
download_cmd="$(get_download_cmd $ff $_level)"
dest_dir="$(get_dest_directory $_arch)"
# Decide if the list will be downloaded using yumdownloader or wget
if [[ $ff != *"#"* ]]; then
rpm_name=$ff
if [ $_level == "K1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
rpm_url=$(koji_url $rpm_name)
download_cmd="wget $rpm_url)"
download_url_cmd="echo $rpm_url)"
if [ ! -e $dest_dir/$rpm_name ]; then
echo "Looking for $rpm_name"
echo "--> run: $download_cmd"
if $download_cmd ; then
download_url="$(get_url $ff $_level)"
SFILE="$(get_rpm_level_name $rpm_name $_level)"
process_result "$_arch" "$dest_dir" "$download_url" "$SFILE"
else
if [ $_level == "L1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $match_level == "L2" ];then
SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
else
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo " ------ using $SFILE to search $rpm_name ------"
# Yumdownloader with the appropriate flag for src, noarch or x86_64
if [ "$_type" == "src" ];then
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE"
else
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64"
fi
echo "Warning: $rpm_name not found"
echo "missing_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
# Buid wget command
rpm_name=`echo $ff | cut -d"#" -f1-1`
rpm_url=`echo $ff | cut -d"#" -f2-2`
download_cmd="wget $rpm_url"
download_url_cmd="echo $rpm_url"
SFILE=$rpm_name
fi
# Put the RPM in the Binary or Source directory
if [ "$_type" == "src" ]; then
if [ ! -e $MDIR_SRC/$rpm_name ]; then
echo "Looking for $rpm_name"
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_srpm:$URL" >> $LOG
if ! mv -f $SFILE* $MDIR_SRC ; then
echo "FAILED to move $rpm_name"
echo "fail_move_srpm:$rpm_name" >> $LOG
fi
echo "found_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_SRPMS
else
echo "Warning: $rpm_name not found"
echo "missing_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo "already_there_srpm:$rpm_name" >> $LOG
fi
else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
echo "Looking for $rpm_name..."
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_rpm:$URL" >> $LOG
mkdir -p $MDIR_BIN/${_type}
if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
echo "FAILED to move $rpm_name"
echo "fail_move_rpm:$rpm_name" >> $LOG
fi
echo "found_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_RPMS
else
echo "Warning: $rpm_name not found"
echo "missing_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_RPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo "already_there_rpm:$rpm_name" >> $LOG
fi
echo "Already have $dest_dir/$rpm_name"
echo "already_there_srpm:$rpm_name" >> $LOG
fi
echo
done
return $rc
}
# Prime the cache
${SUDOCMD} yum ${YUMCONFOPT} ${RELEASEVER} makecache
@ -309,7 +178,6 @@ if [ -s "$rpms_list" ];then
fi
fi
echo "done!!"
echo "Done!"
exit $dl_rc

View File

@ -0,0 +1,196 @@
#
# SPDX-License-Identifier: Apache-2.0
#
get_yum_command() {
local _file=$1
local _level=$2
local rpm_name=""
local arr=( $(split_filename $_file) )
local arch=${arr[3]}
local yumdownloader_extra_opts=""
rpm_name="$(get_rpm_level_name $_file $_level)"
if [ "$arch" == "src" ]; then
yumdownloader_extra_opts="--source"
else
yumdownloader_extra_opts="--archlist=noarch,x86_64"
fi
echo "yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $yumdownloader_extra_opts $rpm_name"
}
get_wget_command() {
local _name="$1"
local _ret=""
if [[ "$_name" == http?(s)://* ]]; then
_ret="wget -q $_name"
else
_ret="wget -q $(koji_url $_name)"
fi
echo "$_ret"
}
get_rpm_level_name() {
local _rpm_name=$1
local _level=$2
if [ $_level == "L1" ]; then
SFILE=`echo $_rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $_level == "L2" ];then
SFILE=`echo $_rpm_name | rev | cut -d'-' -f2- | rev`
else
SFILE=`echo $_rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo "$SFILE"
}
get_url() {
local _name="$1"
local _level="$2"
local _ret=""
if [ "$_level" == "K1" ]; then
_ret="$(koji_url $_name)"
elif [[ "$_name" == *"#"* ]]; then
_ret="$(echo $_name | cut -d'#' -f2-2)"
else
_url_cmd="$(get_yum_command $_name $_level)"
_ret="$($_url_cmd --url)"
fi
echo "$_ret"
}
# Function to split an rpm filename into parts.
#
# Returns a space seperated list containing:
# <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
#
split_filename () {
local rpm_filename=$1
local RPM=""
local SFILE=""
local ARCH=""
local RELEASE=""
local VERSION=""
local NAME=""
local EPOCH=""
RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
if [[ $NAME = *":"* ]]; then
EPOCH=$(echo $NAME | cut -d':' -f-1)
NAME=$(echo $NAME | cut -d':' -f2-)
fi
echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
}
# Function to predict the URL where a rpm might be found.
# Assumes the rpm was compile for EPEL by fedora's koji.
koji_url () {
local rpm_filename=$1
local arr=( $(split_filename $rpm_filename) )
local n=${arr[0]}
local v=${arr[1]}
local r=${arr[2]}
local a=${arr[3]}
echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
}
get_dest_directory() {
local _type=$1
local _dest=""
if [ "$_type" == "src" ]; then
_dest="$MDIR_SRC"
else
_dest="$MDIR_BIN/$_type"
fi
echo "$_dest"
}
process_result() {
local _type="$1"
local dest_dir="$2"
local url="$3"
local sfile="$4"
if [ "$_type" != "src" ] && [ ! -d $dest_dir ]; then
mkdir -p $dest_dir
fi
echo "url_srpm:$url" >> $LOG
if ! mv -f $sfile* $dest_dir ; then
echo "FAILED to move $rpm_name"
echo "fail_move_srpm:$rpm_name" >> $LOG
fi
echo "found_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_SRPMS
}
get_download_cmd() {
local ff="$1"
local _level="$2"
# Decide if the list will be downloaded using yumdownloader or wget
if [[ $ff != *"#"* ]]; then
rpm_name=$ff
if [ $_level == "K1" ]; then
download_cmd="$(get_wget_command $rpm_name)"
else
# yumdownloader with the appropriate flag for src, noarch or x86_64
download_cmd="${SUDOCMD} $(get_yum_command $rpm_name $_level)"
fi
else
# Build wget command
rpm_url=$(echo $ff | cut -d"#" -f2-2)
download_cmd="$(get_wget_command $rpm_url)"
fi
echo "$download_cmd"
}
get_rpm_name() {
local ret=""
if [[ "$1" != *"#"* ]]; then
ret="$1"
else
ret="$(echo $1 | cut -d"#" -f1-1)"
fi
echo "$ret"
}
get_arch_from_rpm() {
local _file=$1
local _split=()
local _arch=""
if [[ "$1" == *"#"* ]]; then
_file=$(echo $_file | cut -d"#" -f1-1)
fi
_split=( $(split_filename $_file) )
_arch=${_split[3]}
echo "$_arch"
}
get_from() {
list=$1
base=$(basename $list .lst) # removing lst extension
base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from
}

View File

@ -0,0 +1,90 @@
#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
# Set of unit tests for dl_rpms.sh
set -o errexit
set -o nounset
YUMCONFOPT=""
RELEASEVER="--releasever=7"
source utils.sh
check_result() {
local _res="$1"
local _expect="$2"
if [ "$_res" != "$_expect" ]; then
echo "Fail"
echo "expected $_expect"
echo "returned $_res"
exit 1
fi
echo "Success"
}
# get_wget_command
res=$(get_wget_command "https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm")
expect="wget -q https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm"
check_result "$res" "$expect"
res=$(get_wget_command "python2-httpbin-0.5.0-6.el7.noarch.rpm")
expect="wget -q https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
check_result "$res" "$expect"
# get_url
res=$(get_url "acpid-2.0.19-9.el7.x86_64.rpm" "L1")
expect="http://vault.centos.org/centos/7.4.1708/cr/x86_64/Packages/acpid-2.0.19-9.el7.x86_64.rpm"
check_result "$res" "$expect"
res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm" "L1")
expect="http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
check_result "$res" "$expect"
res=$(get_url "python2-httpbin-0.5.0-6.el7.noarch.rpm" "K1")
expect="https://kojipkgs.fedoraproject.org/packages/python2-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm"
check_result "$res" "$expect"
# get_yum_command
res=$(get_yum_command "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L1")
expect="yumdownloader -q -C --releasever=7 --source anaconda-21.48.22.121-1.el7.centos"
check_result "$res" "$expect"
res=$(get_yum_command "acpid-2.0.19-9.el7.x86_64.rpm" "L1")
expect="yumdownloader -q -C --releasever=7 --archlist=noarch,x86_64 acpid-2.0.19-9.el7"
check_result "$res" "$expect"
# get_rpm_level_name
res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L1")
expect="acl-2.2.51-12.el7"
check_result "$res" "$expect"
res=$(get_rpm_level_name "acl-2.2.51-12.el7.x86_64.rpm" "L3")
expect="acl"
check_result "$res" "$expect"
res=$(get_rpm_level_name "anaconda-21.48.22.121-1.el7.centos.src.rpm" "L2")
expect="anaconda-21.48.22.121"
check_result "$res" "$expect"
res=$(get_arch_from_rpm "acl-2.2.51-12.el7.x86_64.rpm")
expect="x86_64"
check_result "$res" "$expect"
res=$(get_arch_from_rpm "acl-2.2.51-12.el7.noarch.rpm")
expect="noarch"
check_result "$res" "$expect"
res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm")
expect="src"
check_result "$res" "$expect"
res=$(get_arch_from_rpm "acl-2.2.51-12.el7.src.rpm#https://someurl.com/acl-2.2.51-12.el7.src.rpm")
expect="src"
check_result "$res" "$expect"