From cceb3ee9d02c614232a69801e6f56d018fc484d8 Mon Sep 17 00:00:00 2001 From: Marcela Rosales Date: Tue, 21 Aug 2018 10:11:23 -0500 Subject: [PATCH] Reduce number of log files in dl_rpms.sh and keep all logs in logs dir Eight log files per RPM list is complicated for error detection and managing. This commmit is changing that to three logs. One for general information about errors, other for missing RPMs and another for missing SRPMs. These could be used by download_mirror.sh script to look for packages with K1. All logs are now kept in logs directory. And now the download_mirror.sh has a function for generating logs' names. Change-Id: I901657bc56209d54a3fe5be492d76716249f2755 Signed-off-by: Marcela Rosales --- centos-mirror-tools/dl_rpms.sh | 58 ++++++++-------- centos-mirror-tools/download_mirror.sh | 96 ++++++++++++++++---------- 2 files changed, 89 insertions(+), 65 deletions(-) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 327f2143..91cde147 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -31,8 +31,9 @@ usage() { get_from() { list=$1 - base=$(basename $list .lst) - from=$(echo $base | cut -d'_' -f2-2) + base=$(basename $list .lst) # removing lst extension + base=$(basename $base .log) # removing log extension + from=$(echo $base | rev | cut -d'_' -f1-1 | rev) echo $from } @@ -95,33 +96,27 @@ fi timestamp=$(date +%F_%H%M) echo $timestamp + + DESTDIR="output" MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source mkdir -p $MDIR_SRC MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary mkdir -p $MDIR_BIN - +LOGSDIR="logs" from=$(get_from $rpms_list) -FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt" -FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt" -MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt" -URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt" - -cat /dev/null > $FAIL_MOVE_SRPMS -cat /dev/null > $FOUND_SRPMS +LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log" +MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log" +MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log" +FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log" +FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log" +cat /dev/null > $LOG cat /dev/null > $MISSING_SRPMS -cat /dev/null > $URL_SRPMS - -FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt" -FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt" -MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt" -URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt" - -cat /dev/null > $FAIL_MOVE_RPMS -cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS -cat /dev/null > $URL_RPMS +cat /dev/null > $FOUND_SRPMS +cat /dev/null > $FOUND_RPMS + if [ $CLEAN_LOGS_ONLY -eq 1 ];then exit 0 @@ -242,21 +237,25 @@ download () { # Success! Record download URL. # Use 'sort --unique' because sometimes # yumdownloader reports the url twice - $download_url_cmd | sort --unique >> $URL_SRPMS + URL=$($download_url_cmd | sort --unique) + echo "The url is: $URL" + echo "url_srpm:$URL" >> $LOG if ! mv -f $SFILE* $MDIR_SRC ; then echo "FAILED to move $rpm_name" - echo $rpm_name >> $FAIL_MOVE_SRPMS + echo "fail_move_srpm:$rpm_name" >> $LOG fi + echo "found_srpm:$rpm_name" >> $LOG echo $rpm_name >> $FOUND_SRPMS else echo "Warning: $rpm_name not found" + echo "missing_srpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_SRPMS rc=1 fi else echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" - echo $rpm_name >> $FOUND_SRPMS + echo "already_there_srpm:$rpm_name" >> $LOG fi else ## noarch or x86_64 if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then @@ -266,22 +265,26 @@ download () { # Success! Record download URL. # Use 'sort --unique' because sometimes # yumdownloader reports the url twice - $download_url_cmd | sort --unique >> $URL_RPMS + URL=$($download_url_cmd | sort --unique) + echo "The url is: $URL" + echo "url_rpm:$URL" >> $LOG mkdir -p $MDIR_BIN/${_type} if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then echo "FAILED to move $rpm_name" - echo $rpm_name >> $FAIL_MOVE_RPMS + echo "fail_move_rpm:$rpm_name" >> $LOG fi + echo "found_rpm:$rpm_name" >> $LOG echo $rpm_name >> $FOUND_RPMS else echo "Warning: $rpm_name not found" + echo "missing_rpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_RPMS rc=1 fi else echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" - echo $rpm_name >> $FOUND_RPMS + echo "already_there_rpm:$rpm_name" >> $LOG fi fi done @@ -294,7 +297,7 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache # Download files if [ -s "$rpms_list" ];then - echo "--> start searching "$rpms_list + echo "--> start searching $rpms_list" download $rpms_list $match_level if [ $? -ne 0 ]; then dl_rc=1 @@ -304,3 +307,4 @@ fi echo "done!!" exit $dl_rc + diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 3e739bab..6b6731e1 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -15,6 +15,23 @@ usage() { echo "" } +generate_log_name() { + filename=$1 + level=$2 + base=$(basename $filename .lst) + echo $LOGSDIR"/"$base"_download_"$level".log" +} + +need_file(){ + for f in $*; do + if [ ! -e $f ]; then + echo "ERROR: $f does not exist." + exit 1 + fi + done +} + +# Downloader scripts rpm_downloader="./dl_rpms.sh" tarball_downloader="./dl_tarball.sh" other_downloader="./dl_other_from_centos_repo.sh" @@ -68,16 +85,10 @@ echo "--------------------------------------------------------------" echo "WARNING: this script HAS TO access internet (http/https/ftp)," echo "so please make sure your network working properly!!" -mkdir -p ./logs -need_file(){ - for f in $*; do - if [ ! -e $f ]; then - echo "ERROR: $f does not exist." - exit 1 - fi - done -} +LOGSDIR="logs" +mkdir -p $LOGSDIR + # Check extistence of prerequisites files need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader} @@ -100,23 +111,27 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi fi -logfile="log_download_3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile +list=${rpms_from_3rd_parties} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ];then - echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}." - echo " Please check the log at $(pwd)/logs/$logfile !" + echo "ERROR: Something wrong with downloading files listed in $list." + echo " Please check the log at $(pwd)/$logfile !" echo "" success=0 fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -logfile="log_download_centos3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile +list=${rpms_from_centos_3rd_parties} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ];then - echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}." - echo " Please check the log at $(pwd)/logs/$logfile !" + echo "ERROR: Something wrong with downloading files listed in $list." + echo " Please check the log at $(pwd)/$logfile !" echo "" success=0 fi @@ -129,49 +144,52 @@ fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" -logfile="log_download_centos_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile +list=${rpms_from_centos_repo} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile retcode=${PIPESTATUS[0]} -K1_logfile="log_download_rpms_from_centos_K1.txt" + +K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1) if [ $retcode -ne 1 ]; then # K1 step not needed. Clear any K1 logs from previous download attempts. - $rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile + $rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile fi if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" elif [ $retcode -eq 1 ]; then echo "finish 1st round of RPM downloading with missing files!" - if [ -e "./output/centos_rpms_missing_L1.txt" ]; then + if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." - $rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile + $rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ]; then echo "finish 2nd round of RPM downloading successfully!" elif [ $retcode -eq 1 ]; then echo "finish 2nd round of RPM downloading with missing files!" - if [ -e "./output/rpms_missing_K1.txt" ]; then - echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !" + if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then + echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !" fi fi - # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent + # Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent # false reporting of missing files. - grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp - mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt + grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp + mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log - missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1` + missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then echo "ERROR: -------RPMs missing: $missing_num ---------------" retcode=1 fi fi - if [ -e "./output/centos_srpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` + if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then + missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then echo "ERROR: --------- SRPMs missing: $missing_num ---------------" retcode=1 @@ -181,17 +199,17 @@ fi if [ $retcode -ne 0 ]; then echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." - echo " Please check the logs at $(pwd)/logs/$logfile" + echo " Please check the logs at $(pwd)/$logfile" echo " and $(pwd)/logs/$K1_logfile !" echo "" success=0 fi ## verify all RPMs SRPMs we download for the GPG keys -find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt +find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt # remove all i686.rpms to avoid pollute the chroot dep chain -find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt +find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt find ./output -name "*.i686.rpm" | xargs rm -f line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` @@ -213,14 +231,15 @@ fi echo "step #3: start downloading other files ..." -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt +logfile=$LOGSDIR"/otherfiles_centos_download.log" +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" else echo "step #3: finished with errors" echo "ERROR: Something wrong with downloading from ${other_downloads}." - echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !" + echo " Please check the log at $(pwd)/$logfile!" echo "" success=0 fi @@ -229,14 +248,15 @@ fi # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt +logfile=$LOGSDIR"/tarballs_download.log" +${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #4: done successfully" else echo "step #4: finished with errors" echo "ERROR: Something wrong with downloading tarballs." - echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !" + echo " Please check the log at $(pwd)/$logfile !" echo "" success=0 fi