Change lists names and downloading scripts
-By standardizing the list files name it's possible to eliminate one parameter in the download function in dl_rpms and reduce complexity. -Now the download function does not receive it as a parameter anymore, and instead it gets it from the RPM name. -Also the download function now decides if build a wget command or a yumdownloader commnad based on the content of the list (by identifying the # character) instead of using the name of the list. I adapted download_mirror.sh. Change-Id: I041fc9c704156215f06149e5b4c16cd92990e17c Signed-off-by: Marcela Rosales <marcela.a.rosales.jimenez@intel.com>
This commit is contained in:
Marcela Rosales
parent
fdd64ad1c6
commit
1a539ac2a5
|
|
@ -6,7 +6,7 @@
|
|||
# this script was originated by Brian Avery, and later updated by Yong Hu
|
||||
|
||||
usage() {
|
||||
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> <from_where>"
|
||||
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -n: Do not use sudo when performing operations"
|
||||
|
|
@ -19,11 +19,16 @@ usage() {
|
|||
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
|
||||
echo " L3: use name:"
|
||||
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
|
||||
echo " from_where: where to download the RPMs: 'centos'from CentOS Repos,"
|
||||
echo " otherwise from 3rd-party websets"
|
||||
echo ""
|
||||
}
|
||||
|
||||
get_from() {
|
||||
list=$1
|
||||
base=$(basename $list .lst)
|
||||
from=$(echo $base | cut -d'_' -f2-2)
|
||||
echo $from
|
||||
}
|
||||
|
||||
# By default, we use "sudo" and we don't use a local yum.conf. These can
|
||||
# be overridden via flags.
|
||||
SUDOCMD="sudo -E"
|
||||
|
|
@ -53,7 +58,7 @@ while getopts "c:nh" o; do
|
|||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
if [ $# -lt 2 ]; then
|
||||
usage
|
||||
exit -1
|
||||
fi
|
||||
|
|
@ -72,7 +77,6 @@ if [ ! -z "$2" -a "$2" != " " ];then
|
|||
match_level=$2
|
||||
fi
|
||||
|
||||
from=$3
|
||||
|
||||
timestamp=$(date +%F_%H%M)
|
||||
echo $timestamp
|
||||
|
|
@ -83,6 +87,8 @@ mkdir -p $MDIR_SRC
|
|||
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
|
||||
mkdir -p $MDIR_BIN
|
||||
|
||||
|
||||
from=$(get_from $rpms_list)
|
||||
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
|
||||
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
|
||||
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
|
||||
|
|
@ -103,18 +109,21 @@ cat /dev/null > $FOUND_RPMS
|
|||
cat /dev/null > $MISSING_RPMS
|
||||
cat /dev/null > $URL_RPMS
|
||||
|
||||
#function to download different type of RPMs in different ways
|
||||
# Function to download different types of RPMs in different ways
|
||||
download () {
|
||||
_list=$1
|
||||
_file=$1
|
||||
_level=$2
|
||||
_from=$3
|
||||
_type=$4
|
||||
|
||||
_list=$(cat $_file)
|
||||
_from=$(get_from $_file)
|
||||
echo "now the rpm will come from: $_from"
|
||||
for ff in $_list; do
|
||||
## download RPM from CentOS repos
|
||||
if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then
|
||||
_type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
|
||||
|
||||
# Decide if the list will be downloaded using yumdownloader or wget
|
||||
if [[ $ff != *"#"* ]]; then
|
||||
rpm_name=$ff
|
||||
|
||||
# Cut the rpm name for the specified level (L1, L2 or L3)
|
||||
if [ $_level == "L1" ]; then
|
||||
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
|
||||
elif [ $match_level == "L2" ];then
|
||||
|
|
@ -123,6 +132,7 @@ download () {
|
|||
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
|
||||
fi
|
||||
echo " ------ using $SFILE to search $rpm_name ------"
|
||||
# Yumdownloader with the appropriate flag for src, noarch or x86_64
|
||||
if [ "$_type" == "src" ];then
|
||||
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
|
||||
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
|
||||
|
|
@ -131,12 +141,15 @@ download () {
|
|||
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
|
||||
fi
|
||||
else
|
||||
# Buid wget command
|
||||
rpm_name=`echo $ff | cut -d"#" -f1-1`
|
||||
rpm_url=`echo $ff | cut -d"#" -f2-2`
|
||||
download_cmd="wget $rpm_url"
|
||||
SFILE=$rpm_name
|
||||
fi
|
||||
|
||||
echo "--> run: $download_cmd"
|
||||
# Put the RPM in the Binary or Source directory
|
||||
if [ "$_type" == "src" ]; then
|
||||
if [ ! -e $MDIR_SRC/$rpm_name ]; then
|
||||
echo "Looking for $rpm_name"
|
||||
|
|
@ -163,7 +176,7 @@ download () {
|
|||
echo "Looking for $rpm_name..."
|
||||
if $download_cmd ; then
|
||||
# Success! Record download URL.
|
||||
# Use 'sort --unique' because sometimes
|
||||
# Use 'sort --unique' because sometimes
|
||||
# yumdownloader reports the url twice
|
||||
$download_url_cmd | sort --unique >> $URL_RPMS
|
||||
|
||||
|
|
@ -184,28 +197,13 @@ download () {
|
|||
done
|
||||
}
|
||||
|
||||
# prime the cache
|
||||
# Prime the cache
|
||||
${SUDOCMD} yum ${YUMCONFOPT} makecache
|
||||
|
||||
#go to download *.noarch.rpm files
|
||||
noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"`
|
||||
if [ ! -z "$noarch_rpms" ];then
|
||||
echo "--> start searching noarch RPMs ....."
|
||||
download "$noarch_rpms" $match_level $from "noarch"
|
||||
fi
|
||||
|
||||
#go to download *.x86_64.rpm files
|
||||
x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"`
|
||||
if [ ! -z "$x86_64_rpms" ];then
|
||||
echo "--> start searching x86_64 RPMs ....."
|
||||
download "$x86_64_rpms" $match_level $from "x86_64"
|
||||
fi
|
||||
|
||||
#go to download *.src.rpm files
|
||||
src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"`
|
||||
if [ ! -z "$src_rpms" ];then
|
||||
echo "--> start searching source RPMs ....."
|
||||
download "$src_rpms" $match_level $from "src"
|
||||
# Download files
|
||||
if [ -s "$rpms_list" ];then
|
||||
echo "--> start searching "$rpms_list
|
||||
download $rpms_list $match_level
|
||||
fi
|
||||
|
||||
echo "done!!"
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ rpm_downloader_extra_args=""
|
|||
tarball_downloader_extra_args=""
|
||||
|
||||
# lst files to use as input
|
||||
rpms_from_3rd_parties="./rpms_from_3rd_parties.lst"
|
||||
rpms_from_centos_repo="./rpms_from_centos_repo.lst"
|
||||
rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst"
|
||||
rpms_from_3rd_parties="./rpms_3rdparties.lst"
|
||||
rpms_from_centos_repo="./rpms_centos.lst"
|
||||
rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
|
||||
other_downloads="./other_downloads.lst"
|
||||
|
||||
# Parse out optional -c or -n arguments
|
||||
|
|
@ -96,14 +96,17 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
|
||||
logfile="log_download_3rdparties_L1.txt"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -ne 0 ];then
|
||||
if [ $retcode -ne 0 ]; then
|
||||
echo "ERROR: something wrong with downloading, please check the log!!"
|
||||
fi
|
||||
|
||||
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
|
||||
logfile="log_download_centos3rdparties_L1.txt"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
|
||||
|
||||
if [ ${use_system_yum_conf} -eq 1 ]; then
|
||||
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
|
||||
\rm -f $REPO_DIR/StarlingX_3rd*.repo
|
||||
|
|
@ -111,9 +114,8 @@ fi
|
|||
|
||||
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
|
||||
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
|
||||
|
||||
$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt
|
||||
|
||||
logfile="log_download_centos_L1.txt"
|
||||
$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -ne 0 ]; then
|
||||
echo "finish 1st round of RPM downloading successfully!"
|
||||
|
|
@ -144,7 +146,6 @@ find ./output -name "*.i686.rpm" | xargs rm -f
|
|||
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
|
||||
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
|
||||
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
|
||||
|
||||
let total_line=$line1+$line2+$line3
|
||||
echo "We expect to download $total_line RPMs."
|
||||
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
|
||||
|
|
@ -178,4 +179,3 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
|
|||
echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
|
||||
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
|
||||
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue