Reduce number of log files in dl_rpms.sh and keep all logs in logs dir

Eight log files per RPM list is complicated for error detection and
managing. This commmit is changing that to three logs.
One for general information about errors, other for missing RPMs and
another for missing SRPMs. These could be used by download_mirror.sh
script to look for packages with K1.

All logs are now kept in logs directory.

And now the download_mirror.sh has a function for generating logs'
names.

Change-Id: I901657bc56209d54a3fe5be492d76716249f2755
Signed-off-by: Marcela Rosales <marcela.a.rosales.jimenez@intel.com>
This commit is contained in:
Marcela Rosales 2018-08-21 10:11:23 -05:00
parent d1c216f327
commit cceb3ee9d0
2 changed files with 89 additions and 65 deletions

View File

@ -31,8 +31,9 @@ usage() {
get_from() {
list=$1
base=$(basename $list .lst)
from=$(echo $base | cut -d'_' -f2-2)
base=$(basename $list .lst) # removing lst extension
base=$(basename $base .log) # removing log extension
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
echo $from
}
@ -95,33 +96,27 @@ fi
timestamp=$(date +%F_%H%M)
echo $timestamp
DESTDIR="output"
MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
mkdir -p $MDIR_SRC
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
mkdir -p $MDIR_BIN
LOGSDIR="logs"
from=$(get_from $rpms_list)
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt"
cat /dev/null > $FAIL_MOVE_SRPMS
cat /dev/null > $FOUND_SRPMS
LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log"
MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log"
MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log"
FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log"
FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log"
cat /dev/null > $LOG
cat /dev/null > $MISSING_SRPMS
cat /dev/null > $URL_SRPMS
FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
cat /dev/null > $FAIL_MOVE_RPMS
cat /dev/null > $FOUND_RPMS
cat /dev/null > $MISSING_RPMS
cat /dev/null > $URL_RPMS
cat /dev/null > $FOUND_SRPMS
cat /dev/null > $FOUND_RPMS
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0
@ -242,21 +237,25 @@ download () {
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_SRPMS
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_srpm:$URL" >> $LOG
if ! mv -f $SFILE* $MDIR_SRC ; then
echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_SRPMS
echo "fail_move_srpm:$rpm_name" >> $LOG
fi
echo "found_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_SRPMS
else
echo "Warning: $rpm_name not found"
echo "missing_srpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_SRPMS
echo "already_there_srpm:$rpm_name" >> $LOG
fi
else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
@ -266,22 +265,26 @@ download () {
# Success! Record download URL.
# Use 'sort --unique' because sometimes
# yumdownloader reports the url twice
$download_url_cmd | sort --unique >> $URL_RPMS
URL=$($download_url_cmd | sort --unique)
echo "The url is: $URL"
echo "url_rpm:$URL" >> $LOG
mkdir -p $MDIR_BIN/${_type}
if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
echo "FAILED to move $rpm_name"
echo $rpm_name >> $FAIL_MOVE_RPMS
echo "fail_move_rpm:$rpm_name" >> $LOG
fi
echo "found_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $FOUND_RPMS
else
echo "Warning: $rpm_name not found"
echo "missing_rpm:$rpm_name" >> $LOG
echo $rpm_name >> $MISSING_RPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_RPMS
echo "already_there_rpm:$rpm_name" >> $LOG
fi
fi
done
@ -294,7 +297,7 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
# Download files
if [ -s "$rpms_list" ];then
echo "--> start searching "$rpms_list
echo "--> start searching $rpms_list"
download $rpms_list $match_level
if [ $? -ne 0 ]; then
dl_rc=1
@ -304,3 +307,4 @@ fi
echo "done!!"
exit $dl_rc

View File

@ -15,6 +15,23 @@ usage() {
echo ""
}
generate_log_name() {
filename=$1
level=$2
base=$(basename $filename .lst)
echo $LOGSDIR"/"$base"_download_"$level".log"
}
need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
# Downloader scripts
rpm_downloader="./dl_rpms.sh"
tarball_downloader="./dl_tarball.sh"
other_downloader="./dl_other_from_centos_repo.sh"
@ -68,16 +85,10 @@ echo "--------------------------------------------------------------"
echo "WARNING: this script HAS TO access internet (http/https/ftp),"
echo "so please make sure your network working properly!!"
mkdir -p ./logs
need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit 1
fi
done
}
LOGSDIR="logs"
mkdir -p $LOGSDIR
# Check extistence of prerequisites files
need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
@ -100,23 +111,27 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
fi
fi
logfile="log_download_3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
list=${rpms_from_3rd_parties}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
logfile="log_download_centos3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
list=${rpms_from_centos_3rd_parties}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo "ERROR: Something wrong with downloading files listed in $list."
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi
@ -129,49 +144,52 @@ fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
logfile="log_download_centos_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
list=${rpms_from_centos_repo}
level=L1
logfile=$(generate_log_name $list $level)
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
retcode=${PIPESTATUS[0]}
K1_logfile="log_download_rpms_from_centos_K1.txt"
K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
if [ $retcode -ne 1 ]; then
# K1 step not needed. Clear any K1 logs from previous download attempts.
$rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
$rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
fi
if [ $retcode -eq 0 ]; then
echo "finish 1st round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 1st round of RPM downloading with missing files!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
$rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
$rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then
echo "finish 2nd round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 2nd round of RPM downloading with missing files!"
if [ -e "./output/rpms_missing_K1.txt" ]; then
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
fi
fi
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
# Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
# false reporting of missing files.
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp
mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing: $missing_num ---------------"
retcode=1
fi
fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
retcode=1
@ -181,17 +199,17 @@ fi
if [ $retcode -ne 0 ]; then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
echo " Please check the logs at $(pwd)/logs/$logfile"
echo " Please check the logs at $(pwd)/$logfile"
echo " and $(pwd)/logs/$K1_logfile !"
echo ""
success=0
fi
## verify all RPMs SRPMs we download for the GPG keys
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
# remove all i686.rpms to avoid pollute the chroot dep chain
find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt
find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
find ./output -name "*.i686.rpm" | xargs rm -f
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
@ -213,14 +231,15 @@ fi
echo "step #3: start downloading other files ..."
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
logfile=$LOGSDIR"/otherfiles_centos_download.log"
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #3: done successfully"
else
echo "step #3: finished with errors"
echo "ERROR: Something wrong with downloading from ${other_downloads}."
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
echo " Please check the log at $(pwd)/$logfile!"
echo ""
success=0
fi
@ -229,14 +248,15 @@ fi
# StarlingX requires a group of source code pakages, in this section
# they will be downloaded.
echo "step #4: start downloading tarball compressed files"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt
logfile=$LOGSDIR"/tarballs_download.log"
${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #4: done successfully"
else
echo "step #4: finished with errors"
echo "ERROR: Something wrong with downloading tarballs."
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
echo " Please check the log at $(pwd)/$logfile !"
echo ""
success=0
fi