Reduce number of log files in dl_rpms.sh and keep all logs in logs dir
Eight log files per RPM list is complicated for error detection and managing. This commmit is changing that to three logs. One for general information about errors, other for missing RPMs and another for missing SRPMs. These could be used by download_mirror.sh script to look for packages with K1. All logs are now kept in logs directory. And now the download_mirror.sh has a function for generating logs' names. Change-Id: I901657bc56209d54a3fe5be492d76716249f2755 Signed-off-by: Marcela Rosales <marcela.a.rosales.jimenez@intel.com>
This commit is contained in:
parent
d1c216f327
commit
cceb3ee9d0
|
@ -31,8 +31,9 @@ usage() {
|
|||
|
||||
get_from() {
|
||||
list=$1
|
||||
base=$(basename $list .lst)
|
||||
from=$(echo $base | cut -d'_' -f2-2)
|
||||
base=$(basename $list .lst) # removing lst extension
|
||||
base=$(basename $base .log) # removing log extension
|
||||
from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
|
||||
echo $from
|
||||
}
|
||||
|
||||
|
@ -95,33 +96,27 @@ fi
|
|||
timestamp=$(date +%F_%H%M)
|
||||
echo $timestamp
|
||||
|
||||
|
||||
|
||||
DESTDIR="output"
|
||||
MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
|
||||
mkdir -p $MDIR_SRC
|
||||
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
|
||||
mkdir -p $MDIR_BIN
|
||||
|
||||
|
||||
LOGSDIR="logs"
|
||||
from=$(get_from $rpms_list)
|
||||
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
|
||||
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
|
||||
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
|
||||
URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt"
|
||||
|
||||
cat /dev/null > $FAIL_MOVE_SRPMS
|
||||
cat /dev/null > $FOUND_SRPMS
|
||||
LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log"
|
||||
MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log"
|
||||
MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log"
|
||||
FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log"
|
||||
FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log"
|
||||
cat /dev/null > $LOG
|
||||
cat /dev/null > $MISSING_SRPMS
|
||||
cat /dev/null > $URL_SRPMS
|
||||
|
||||
FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
|
||||
FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
|
||||
MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
|
||||
URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
|
||||
|
||||
cat /dev/null > $FAIL_MOVE_RPMS
|
||||
cat /dev/null > $FOUND_RPMS
|
||||
cat /dev/null > $MISSING_RPMS
|
||||
cat /dev/null > $URL_RPMS
|
||||
cat /dev/null > $FOUND_SRPMS
|
||||
cat /dev/null > $FOUND_RPMS
|
||||
|
||||
|
||||
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
|
||||
exit 0
|
||||
|
@ -242,21 +237,25 @@ download () {
|
|||
# Success! Record download URL.
|
||||
# Use 'sort --unique' because sometimes
|
||||
# yumdownloader reports the url twice
|
||||
$download_url_cmd | sort --unique >> $URL_SRPMS
|
||||
URL=$($download_url_cmd | sort --unique)
|
||||
echo "The url is: $URL"
|
||||
echo "url_srpm:$URL" >> $LOG
|
||||
|
||||
if ! mv -f $SFILE* $MDIR_SRC ; then
|
||||
echo "FAILED to move $rpm_name"
|
||||
echo $rpm_name >> $FAIL_MOVE_SRPMS
|
||||
echo "fail_move_srpm:$rpm_name" >> $LOG
|
||||
fi
|
||||
echo "found_srpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $FOUND_SRPMS
|
||||
else
|
||||
echo "Warning: $rpm_name not found"
|
||||
echo "missing_srpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $MISSING_SRPMS
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
|
||||
echo $rpm_name >> $FOUND_SRPMS
|
||||
echo "already_there_srpm:$rpm_name" >> $LOG
|
||||
fi
|
||||
else ## noarch or x86_64
|
||||
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
|
||||
|
@ -266,22 +265,26 @@ download () {
|
|||
# Success! Record download URL.
|
||||
# Use 'sort --unique' because sometimes
|
||||
# yumdownloader reports the url twice
|
||||
$download_url_cmd | sort --unique >> $URL_RPMS
|
||||
URL=$($download_url_cmd | sort --unique)
|
||||
echo "The url is: $URL"
|
||||
echo "url_rpm:$URL" >> $LOG
|
||||
|
||||
mkdir -p $MDIR_BIN/${_type}
|
||||
if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
|
||||
echo "FAILED to move $rpm_name"
|
||||
echo $rpm_name >> $FAIL_MOVE_RPMS
|
||||
echo "fail_move_rpm:$rpm_name" >> $LOG
|
||||
fi
|
||||
echo "found_rpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $FOUND_RPMS
|
||||
else
|
||||
echo "Warning: $rpm_name not found"
|
||||
echo "missing_rpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $MISSING_RPMS
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
|
||||
echo $rpm_name >> $FOUND_RPMS
|
||||
echo "already_there_rpm:$rpm_name" >> $LOG
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
@ -294,7 +297,7 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
|
|||
|
||||
# Download files
|
||||
if [ -s "$rpms_list" ];then
|
||||
echo "--> start searching "$rpms_list
|
||||
echo "--> start searching $rpms_list"
|
||||
download $rpms_list $match_level
|
||||
if [ $? -ne 0 ]; then
|
||||
dl_rc=1
|
||||
|
@ -304,3 +307,4 @@ fi
|
|||
echo "done!!"
|
||||
|
||||
exit $dl_rc
|
||||
|
||||
|
|
|
@ -15,6 +15,23 @@ usage() {
|
|||
echo ""
|
||||
}
|
||||
|
||||
generate_log_name() {
|
||||
filename=$1
|
||||
level=$2
|
||||
base=$(basename $filename .lst)
|
||||
echo $LOGSDIR"/"$base"_download_"$level".log"
|
||||
}
|
||||
|
||||
need_file(){
|
||||
for f in $*; do
|
||||
if [ ! -e $f ]; then
|
||||
echo "ERROR: $f does not exist."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Downloader scripts
|
||||
rpm_downloader="./dl_rpms.sh"
|
||||
tarball_downloader="./dl_tarball.sh"
|
||||
other_downloader="./dl_other_from_centos_repo.sh"
|
||||
|
@ -68,16 +85,10 @@ echo "--------------------------------------------------------------"
|
|||
echo "WARNING: this script HAS TO access internet (http/https/ftp),"
|
||||
echo "so please make sure your network working properly!!"
|
||||
|
||||
mkdir -p ./logs
|
||||
|
||||
need_file(){
|
||||
for f in $*; do
|
||||
if [ ! -e $f ]; then
|
||||
echo "ERROR: $f does not exist."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
LOGSDIR="logs"
|
||||
mkdir -p $LOGSDIR
|
||||
|
||||
|
||||
# Check extistence of prerequisites files
|
||||
need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
|
||||
|
@ -100,23 +111,27 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
logfile="log_download_3rdparties_L1.txt"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
|
||||
list=${rpms_from_3rd_parties}
|
||||
level=L1
|
||||
logfile=$(generate_log_name $list $level)
|
||||
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -ne 0 ];then
|
||||
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
|
||||
echo " Please check the log at $(pwd)/logs/$logfile !"
|
||||
echo "ERROR: Something wrong with downloading files listed in $list."
|
||||
echo " Please check the log at $(pwd)/$logfile !"
|
||||
echo ""
|
||||
success=0
|
||||
fi
|
||||
|
||||
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
|
||||
logfile="log_download_centos3rdparties_L1.txt"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
|
||||
list=${rpms_from_centos_3rd_parties}
|
||||
level=L1
|
||||
logfile=$(generate_log_name $list $level)
|
||||
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -ne 0 ];then
|
||||
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
|
||||
echo " Please check the log at $(pwd)/logs/$logfile !"
|
||||
echo "ERROR: Something wrong with downloading files listed in $list."
|
||||
echo " Please check the log at $(pwd)/$logfile !"
|
||||
echo ""
|
||||
success=0
|
||||
fi
|
||||
|
@ -129,49 +144,52 @@ fi
|
|||
|
||||
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
|
||||
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
|
||||
logfile="log_download_centos_L1.txt"
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
|
||||
list=${rpms_from_centos_repo}
|
||||
level=L1
|
||||
logfile=$(generate_log_name $list $level)
|
||||
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
|
||||
K1_logfile="log_download_rpms_from_centos_K1.txt"
|
||||
|
||||
K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
|
||||
if [ $retcode -ne 1 ]; then
|
||||
# K1 step not needed. Clear any K1 logs from previous download attempts.
|
||||
$rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
|
||||
$rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
|
||||
fi
|
||||
|
||||
if [ $retcode -eq 0 ]; then
|
||||
echo "finish 1st round of RPM downloading successfully!"
|
||||
elif [ $retcode -eq 1 ]; then
|
||||
echo "finish 1st round of RPM downloading with missing files!"
|
||||
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
|
||||
if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
|
||||
|
||||
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
|
||||
$rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
|
||||
$rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -eq 0 ]; then
|
||||
echo "finish 2nd round of RPM downloading successfully!"
|
||||
elif [ $retcode -eq 1 ]; then
|
||||
echo "finish 2nd round of RPM downloading with missing files!"
|
||||
if [ -e "./output/rpms_missing_K1.txt" ]; then
|
||||
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
|
||||
if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
|
||||
echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
|
||||
# Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
|
||||
# false reporting of missing files.
|
||||
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp
|
||||
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
|
||||
grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp
|
||||
mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
|
||||
|
||||
|
||||
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
|
||||
missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
|
||||
if [ "$missing_num" != "0" ];then
|
||||
echo "ERROR: -------RPMs missing: $missing_num ---------------"
|
||||
retcode=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
|
||||
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
|
||||
if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
|
||||
missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
|
||||
if [ "$missing_num" != "0" ];then
|
||||
echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
|
||||
retcode=1
|
||||
|
@ -181,17 +199,17 @@ fi
|
|||
|
||||
if [ $retcode -ne 0 ]; then
|
||||
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
|
||||
echo " Please check the logs at $(pwd)/logs/$logfile"
|
||||
echo " Please check the logs at $(pwd)/$logfile"
|
||||
echo " and $(pwd)/logs/$K1_logfile !"
|
||||
echo ""
|
||||
success=0
|
||||
fi
|
||||
|
||||
## verify all RPMs SRPMs we download for the GPG keys
|
||||
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt
|
||||
find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
|
||||
|
||||
# remove all i686.rpms to avoid pollute the chroot dep chain
|
||||
find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt
|
||||
find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
|
||||
find ./output -name "*.i686.rpm" | xargs rm -f
|
||||
|
||||
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
|
||||
|
@ -213,14 +231,15 @@ fi
|
|||
|
||||
echo "step #3: start downloading other files ..."
|
||||
|
||||
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
|
||||
logfile=$LOGSDIR"/otherfiles_centos_download.log"
|
||||
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -eq 0 ];then
|
||||
echo "step #3: done successfully"
|
||||
else
|
||||
echo "step #3: finished with errors"
|
||||
echo "ERROR: Something wrong with downloading from ${other_downloads}."
|
||||
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
|
||||
echo " Please check the log at $(pwd)/$logfile!"
|
||||
echo ""
|
||||
success=0
|
||||
fi
|
||||
|
@ -229,14 +248,15 @@ fi
|
|||
# StarlingX requires a group of source code pakages, in this section
|
||||
# they will be downloaded.
|
||||
echo "step #4: start downloading tarball compressed files"
|
||||
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt
|
||||
logfile=$LOGSDIR"/tarballs_download.log"
|
||||
${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile
|
||||
retcode=${PIPESTATUS[0]}
|
||||
if [ $retcode -eq 0 ];then
|
||||
echo "step #4: done successfully"
|
||||
else
|
||||
echo "step #4: finished with errors"
|
||||
echo "ERROR: Something wrong with downloading tarballs."
|
||||
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
|
||||
echo " Please check the log at $(pwd)/$logfile !"
|
||||
echo ""
|
||||
success=0
|
||||
fi
|
||||
|
|
Loading…
Reference in New Issue