starlingx
/
tools
Code Issues Proposed changes

Merge "Use kojipkgs.fedoraproject.org as a backup rpm source."

This commit is contained in:
Zuul 2018-08-20 20:54:02 +00:00 committed by Gerrit Code Review
parent 26679e6ed5 fa995d6981
commit 19acf7dc9c
4 changed files with 229 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
centos-mirror-tools/dl_other_from_centos_repo.sh
View File

@ -31,6 +31,11 @@ for ff in $all; do
if [ "$_type" == "folder" ];then
mkdir -p $save_path/$_name
else
if [ -e "$save_path/$_name" ]; then
echo "Already have $save_path/$_name"
continue
fi
echo "remote path: $url_prefix/$_name"
echo "local path: $save_path/$_name"
if wget $url_prefix/$_name; then

145
centos-mirror-tools/dl_rpms.sh
View File

@ -11,6 +11,7 @@ usage() {
echo "Options:"
echo " -n: Do not use sudo when performing operations"
echo " -c: Use an alternate yum.conf rather than the system file"
echo " -x: Clean log files only, do not run."
echo " rpm_list: a list of RPM files to be downloaded."
echo " match_level: value could be L1, L2 or L3:"
echo " L1: use name, major version and minor version:"
@ -19,6 +20,12 @@ usage() {
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
echo " L3: use name:"
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
echo " K1: Use Koji rather than yum repos as a source."
echo " Koji has a longer retention period than epel mirrors."
echo ""
echo "Returns: 0 = All files downloaded successfully"
echo " 1 = Some files could not be downloaded"
echo " 2 = Bad arguements or other error"
echo ""
}
@ -34,13 +41,20 @@ get_from() {
SUDOCMD="sudo -E"
YUMCONFOPT=""
CLEAN_LOGS_ONLY=0
dl_rc=0
# Parse option flags
while getopts "c:nh" o; do
while getopts "c:nxh" o; do
case "${o}" in
n)
# No-sudo
SUDOCMD=""
;;
x)
# Clean only
CLEAN_LOGS_ONLY=1
;;
c)
# Use an alternate yum.conf
YUMCONFOPT="-c $OPTARG"
@ -52,7 +66,7 @@ while getopts "c:nh" o; do
;;
*)
usage
exit 1
exit 2
;;
esac
done
@ -60,12 +74,12 @@ shift $((OPTIND-1))
if [ $# -lt 2 ]; then
usage
exit -1
exit 2
fi
if [ "$1" == "" ]; then
echo "Need to supply the rpm file list"
exit -1;
exit 2;
else
rpms_list=$1
echo "using $rpms_list as the download name lists"
@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS
cat /dev/null > $MISSING_RPMS
cat /dev/null > $URL_RPMS
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0
fi
# Function to split an rpm filename into parts.
#
# Returns a space seperated list containing:
# <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
#
split_filename () {
local rpm_filename=$1
local RPM=""
local SFILE=""
local ARCH=""
local RELEASE=""
local VERSION=""
local NAME=""
RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
if [[ $NAME = *":"* ]]; then
EPOCH=$(echo $NAME | cut -d':' -f-1)
NAME=$(echo $NAME | cut -d':' -f2-)
fi
echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
}
# Function to predict the URL where a rpm might be found.
# Assumes the rpm was compile for EPEL by fedora's koji.
koji_url () {
local rpm_filename=$1
local arr=( $(split_filename $rpm_filename) )
local n=${arr[0]}
local v=${arr[1]}
local r=${arr[2]}
local a=${arr[3]}
local e=${arr[4]}
echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
}
# Function to download different types of RPMs in different ways
download () {
_file=$1
_level=$2
_list=$(cat $_file)
_from=$(get_from $_file)
local _file=$1
local _level=$2
local _list=$(cat $_file)
local _from=$(get_from $_file)
local _type=""
local rc=0
local download_cmd=""
local download_url_cmd=""
local rpm_name=""
local rpm_url=""
local SFILE=""
echo "now the rpm will come from: $_from"
for ff in $_list; do
download_cmd=""
download_url_cmd=""
_type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
# Decide if the list will be downloaded using yumdownloader or wget
if [[ $ff != *"#"* ]]; then
rpm_name=$ff
# Cut the rpm name for the specified level (L1, L2 or L3)
if [ $_level == "L1" ]; then
if [ $_level == "K1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $match_level == "L2" ];then
SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
rpm_url=$(koji_url $rpm_name)
download_cmd="wget $rpm_url)"
download_url_cmd="echo $rpm_url)"
else
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo " ------ using $SFILE to search $rpm_name ------"
# Yumdownloader with the appropriate flag for src, noarch or x86_64
if [ "$_type" == "src" ];then
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
else
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
if [ $_level == "L1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $match_level == "L2" ];then
SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
else
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo " ------ using $SFILE to search $rpm_name ------"
# Yumdownloader with the appropriate flag for src, noarch or x86_64
if [ "$_type" == "src" ];then
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
else
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
fi
fi
else
# Buid wget command
rpm_name=`echo $ff | cut -d"#" -f1-1`
rpm_url=`echo $ff | cut -d"#" -f2-2`
download_cmd="wget $rpm_url"
download_url_cmd="echo $rpm_url"
SFILE=$rpm_name
fi
echo "--> run: $download_cmd"
# Put the RPM in the Binary or Source directory
if [ "$_type" == "src" ]; then
if [ ! -e $MDIR_SRC/$rpm_name ]; then
echo "Looking for $rpm_name"
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
@ -165,15 +250,18 @@ download () {
fi
echo $rpm_name >> $FOUND_SRPMS
else
echo "Warning: $rpm_name not found"
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_SRPMS
fi
else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
echo "Looking for $rpm_name..."
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
@ -187,7 +275,9 @@ download () {
fi
echo $rpm_name >> $FOUND_RPMS
else
echo "Warning: $rpm_name not found"
echo $rpm_name >> $MISSING_RPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
@ -195,6 +285,8 @@ download () {
fi
fi
done
return $rc
}
# Prime the cache
@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
if [ -s "$rpms_list" ];then
echo "--> start searching "$rpms_list
download $rpms_list $match_level
if [ $? -ne 0 ]; then
dl_rc=1
fi
fi
echo "done!!"
exit 0
exit $dl_rc

18
centos-mirror-tools/dl_tarball.sh
View File

@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do
directory_name=$(echo $line | cut -d"#" -f2-2)
tarball_url=$(echo $line | cut -d"#" -f3-3)
# Remove leading '!' if present
tarball_name="${tarball_name//!/}"
# - For the General category and the Puppet category:
# - Packages have a common process: download, decompressed,
# change the directory path and compressed.
@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do
download_directory=$output_tarball
fi
if [ -e $download_path ]; then
echo "Already have $download_path"
continue
fi
# We have 6 packages from the text file starting with the character "!":
# they require special handling besides the common process: remove directory,
# remove text from some files, clone a git repository, etc.
if [[ "$line" =~ ^'!' ]]; then
tarball_name="${tarball_name//!/}"
if [ -e "$output_tarball/$tarball_name" ]; then
echo "Already have $tarball_name"
continue
fi
echo $tarball_name
pushd $output_tarball
if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do
else
echo "$pkg_version : unknown version"
fi
rm -f "$tarball_name"
# Don't delete the original MLNX_OFED_LINUX tarball.
# We don't use it, but it will prevent re-downloading this file.
# rm -f "$tarball_name"
rm -rf "MLNX_OFED_SRC-${pkg_version}"
rm -rf "$directory_name"
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then

107
centos-mirror-tools/download_mirror.sh
View File

@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst"
rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
other_downloads="./other_downloads.lst"
# Overall success
success=1
# Parse out optional -c or -n arguments
while getopts "c:ngh" o; do
case "${o}" in
@ -71,7 +74,7 @@ need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit -1
exit 1
fi
done
}
@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo}
need_file ${other_downloads}
need_file tarball-dl.lst mvn-artifacts.lst
#download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
fi
logfile="log_download_3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ]; then
echo "ERROR: something wrong with downloading, please check the log!!"
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo ""
success=0
fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
logfile="log_download_centos3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo ""
success=0
fi
if [ ${use_system_yum_conf} -eq 1 ]; then
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
\rm -f $REPO_DIR/StarlingX_3rd*.repo
fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
logfile="log_download_centos_L1.txt"
$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ]; then
K1_logfile="log_download_rpms_from_centos_K1.txt"
if [ $retcode -ne 1 ]; then
# K1 step not needed. Clear any K1 logs from previous download attempts.
$rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
fi
if [ $retcode -eq 0 ]; then
echo "finish 1st round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 1st round of RPM downloading with missing files!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
$rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then
echo "finish 2nd round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 2nd round of RPM downloading with missing files!"
if [ -e "./output/rpms_missing_K1.txt" ]; then
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
fi
fi
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
# false reporting of missing files.
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
echo "ERROR: -------RPMs missing: $missing_num ---------------"
retcode=1
fi
fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
retcode=1
fi
fi
else
echo "finish 1st round with failures!"
fi
if [ $retcode -ne 0 ]; then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
echo " Please check the logs at $(pwd)/logs/$logfile"
echo " and $(pwd)/logs/$K1_logfile !"
echo ""
success=0
fi
## verify all RPMs SRPMs we download for the GPG keys
@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
let total_line=$line1+$line2+$line3
echo "We expect to download $total_line RPMs."
echo "We expected to download $total_line RPMs."
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
echo "There are $num_of_downloaded_rpms RPMs in output directory."
if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
@ -162,16 +213,33 @@ fi
echo "step #3: start downloading other files ..."
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #3: done successfully"
else
echo "step #3: finished with errors"
echo "ERROR: Something wrong with downloading from ${other_downloads}."
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
echo ""
success=0
fi
# StarlingX requires a group of source code pakages, in this section
# they will be downloaded.
echo "step #4: start downloading tarball compressed files"
${tarball_downloader} ${tarball_downloader_extra_args}
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #4: done successfully"
else
echo "step #4: finished with errors"
echo "ERROR: Something wrong with downloading tarballs."
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
echo ""
success=0
fi
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
echo "on them, the workable images for StarlingX could be generated by"
@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
echo ""
if [ $success -ne 1 ]; then
echo "Warning: Not all download steps succeeded. You are likely missing files."
exit 1
fi
echo "Success"
exit 0