starlingx
/
tools
Code Issues Proposed changes

Merge "Reduce number of log files in dl_rpms.sh and keep all logs in logs dir"

This commit is contained in:
Zuul 2018-08-30 16:37:25 +00:00 committed by Gerrit Code Review
parent bd4b59dde2 cceb3ee9d0
commit bd39de7fb9
2 changed files with 89 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
centos-mirror-tools/dl_rpms.sh
View File

@ -31,8 +31,9 @@ usage() {
get_from() { get_from() {
list=$1 list=$1
base=$(basename $list .lst) base=$(basename $list .lst) # removing lst extension
from=$(echo $base | cut -d'_' -f2-2) base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from echo $from
} }
@ -95,33 +96,27 @@ fi
timestamp=$(date +%F_%H%M) timestamp=$(date +%F_%H%M)
echo $timestamp echo $timestamp
DESTDIR="output" DESTDIR="output"
MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
mkdir -p $MDIR_SRC mkdir -p $MDIR_SRC
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
mkdir -p $MDIR_BIN mkdir -p $MDIR_BIN
LOGSDIR="logs"
from=$(get_from $rpms_list) from=$(get_from $rpms_list)
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt" LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log"
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt" MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log"
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt" MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log"
URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt" FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log"
FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log"
cat /dev/null > $FAIL_MOVE_SRPMS cat /dev/null > $LOG
cat /dev/null > $FOUND_SRPMS
cat /dev/null > $MISSING_SRPMS cat /dev/null > $MISSING_SRPMS
cat /dev/null > $URL_SRPMS
FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
cat /dev/null > $FAIL_MOVE_RPMS
cat /dev/null > $FOUND_RPMS
cat /dev/null > $MISSING_RPMS cat /dev/null > $MISSING_RPMS
cat /dev/null > $URL_RPMS cat /dev/null > $FOUND_SRPMS
cat /dev/null > $FOUND_RPMS
if [ $CLEAN_LOGS_ONLY -eq 1 ];then if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0 exit 0
@ -245,21 +240,25 @@ download () {
# Success! Record download URL. # Success! Record download URL.
# Use 'sort --unique' because sometimes # Use 'sort --unique' because sometimes
# yumdownloader reports the url twice # yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_SRPMS URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_srpm:$URL" >> $LOG
if ! mv -f $SFILE* $MDIR_SRC ; then if ! mv -f $SFILE* $MDIR_SRC ; then
echo "FAILED to move $rpm_name" echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_SRPMS echo "fail_move_srpm:$rpm_name" >> $LOG
fi fi
echo "found_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_SRPMS echo $rpm_name >> $FOUND_SRPMS
else else
echo "Warning: $rpm_name not found" echo "Warning: $rpm_name not found"
echo "missing_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_SRPMS echo $rpm_name >> $MISSING_SRPMS
rc=1 rc=1
fi fi
else else
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_SRPMS echo "already_there_srpm:$rpm_name" >> $LOG
fi fi
else ## noarch or x86_64 else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
@ -269,22 +268,26 @@ download () {
# Success! Record download URL. # Success! Record download URL.
# Use 'sort --unique' because sometimes # Use 'sort --unique' because sometimes
# yumdownloader reports the url twice # yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_RPMS URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_rpm:$URL" >> $LOG
mkdir -p $MDIR_BIN/${_type} mkdir -p $MDIR_BIN/${_type}
if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
echo "FAILED to move $rpm_name" echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_RPMS echo "fail_move_rpm:$rpm_name" >> $LOG
fi fi
echo "found_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_RPMS echo $rpm_name >> $FOUND_RPMS
else else
echo "Warning: $rpm_name not found" echo "Warning: $rpm_name not found"
echo "missing_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_RPMS echo $rpm_name >> $MISSING_RPMS
rc=1 rc=1
fi fi
else else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_RPMS echo "already_there_rpm:$rpm_name" >> $LOG
fi fi
fi fi
done done
@ -297,7 +300,7 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
# Download files # Download files
if [ -s "$rpms_list" ];then if [ -s "$rpms_list" ];then
echo "--> start searching "$rpms_list echo "--> start searching $rpms_list"
download $rpms_list $match_level download $rpms_list $match_level
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
dl_rc=1 dl_rc=1
@ -307,3 +310,4 @@ fi
echo "done!!" echo "done!!"
exit $dl_rc exit $dl_rc

96
centos-mirror-tools/download_mirror.sh
View File

@ -15,6 +15,23 @@ usage() {
echo "" echo ""
} }
generate_log_name() {
filename=$1
level=$2
base=$(basename $filename .lst)
echo $LOGSDIR"/"$base"_download_"$level".log"
}
need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
# Downloader scripts
rpm_downloader="./dl_rpms.sh" rpm_downloader="./dl_rpms.sh"
tarball_downloader="./dl_tarball.sh" tarball_downloader="./dl_tarball.sh"
other_downloader="./dl_other_from_centos_repo.sh" other_downloader="./dl_other_from_centos_repo.sh"
@ -68,16 +85,10 @@ echo "--------------------------------------------------------------"
echo "WARNING: this script HAS TO access internet (http/https/ftp)," echo "WARNING: this script HAS TO access internet (http/https/ftp),"
echo "so please make sure your network working properly!!" echo "so please make sure your network working properly!!"
mkdir -p ./logs
need_file(){ LOGSDIR="logs"
for f in $*; do mkdir -p $LOGSDIR
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
# Check extistence of prerequisites files # Check extistence of prerequisites files
need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader} need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
@ -100,23 +111,27 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
fi fi
fi fi
logfile="log_download_3rdparties_L1.txt" list=${rpms_from_3rd_parties}
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}." echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/logs/$logfile !" echo " Please check the log at $(pwd)/$logfile !"
echo "" echo ""
success=0 success=0
fi fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader" # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
logfile="log_download_centos3rdparties_L1.txt" list=${rpms_from_centos_3rd_parties}
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}." echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/logs/$logfile !" echo " Please check the log at $(pwd)/$logfile !"
echo "" echo ""
success=0 success=0
fi fi
@ -129,49 +144,52 @@ fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader" #download RPMs/SRPMs from CentOS repos by "yumdownloader"
logfile="log_download_centos_L1.txt" list=${rpms_from_centos_repo}
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
K1_logfile="log_download_rpms_from_centos_K1.txt"
K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
if [ $retcode -ne 1 ]; then if [ $retcode -ne 1 ]; then
# K1 step not needed. Clear any K1 logs from previous download attempts. # K1 step not needed. Clear any K1 logs from previous download attempts.
$rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile $rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
fi fi
if [ $retcode -eq 0 ]; then if [ $retcode -eq 0 ]; then
echo "finish 1st round of RPM downloading successfully!" echo "finish 1st round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then elif [ $retcode -eq 1 ]; then
echo "finish 1st round of RPM downloading with missing files!" echo "finish 1st round of RPM downloading with missing files!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
$rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile $rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then if [ $retcode -eq 0 ]; then
echo "finish 2nd round of RPM downloading successfully!" echo "finish 2nd round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then elif [ $retcode -eq 1 ]; then
echo "finish 2nd round of RPM downloading with missing files!" echo "finish 2nd round of RPM downloading with missing files!"
if [ -e "./output/rpms_missing_K1.txt" ]; then if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !" echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
fi fi
fi fi
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent # Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
# false reporting of missing files. # false reporting of missing files.
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1` missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing: $missing_num ---------------" echo "ERROR: -------RPMs missing: $missing_num ---------------"
retcode=1 retcode=1
fi fi
fi fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing: $missing_num ---------------" echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
retcode=1 retcode=1
@ -181,17 +199,17 @@ fi
if [ $retcode -ne 0 ]; then if [ $retcode -ne 0 ]; then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
echo " Please check the logs at $(pwd)/logs/$logfile" echo " Please check the logs at $(pwd)/$logfile"
echo " and $(pwd)/logs/$K1_logfile !" echo " and $(pwd)/logs/$K1_logfile !"
echo "" echo ""
success=0 success=0
fi fi
## verify all RPMs SRPMs we download for the GPG keys ## verify all RPMs SRPMs we download for the GPG keys
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
# remove all i686.rpms to avoid pollute the chroot dep chain # remove all i686.rpms to avoid pollute the chroot dep chain
find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
find ./output -name "*.i686.rpm" | xargs rm -f find ./output -name "*.i686.rpm" | xargs rm -f
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
@ -213,14 +231,15 @@ fi
echo "step #3: start downloading other files ..." echo "step #3: start downloading other files ..."
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt logfile=$LOGSDIR"/otherfiles_centos_download.log"
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then if [ $retcode -eq 0 ];then
echo "step #3: done successfully" echo "step #3: done successfully"
else else
echo "step #3: finished with errors" echo "step #3: finished with errors"
echo "ERROR: Something wrong with downloading from ${other_downloads}." echo "ERROR: Something wrong with downloading from ${other_downloads}."
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !" echo " Please check the log at $(pwd)/$logfile!"
echo "" echo ""
success=0 success=0
fi fi
@ -229,14 +248,15 @@ fi
# StarlingX requires a group of source code pakages, in this section # StarlingX requires a group of source code pakages, in this section
# they will be downloaded. # they will be downloaded.
echo "step #4: start downloading tarball compressed files" echo "step #4: start downloading tarball compressed files"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt logfile=$LOGSDIR"/tarballs_download.log"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile
retcode=${PIPESTATUS[0]} retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then if [ $retcode -eq 0 ];then
echo "step #4: done successfully" echo "step #4: done successfully"
else else
echo "step #4: finished with errors" echo "step #4: finished with errors"
echo "ERROR: Something wrong with downloading tarballs." echo "ERROR: Something wrong with downloading tarballs."
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !" echo " Please check the log at $(pwd)/$logfile !"
echo "" echo ""
success=0 success=0
fi fi