From 42f7c228d3752534032170803d846121fbc5e889 Mon Sep 17 00:00:00 2001
From: Scott Little <scott.little@windriver.com>
Date: Thu, 26 Jul 2018 16:37:21 -0400
Subject: [PATCH] Log the urls of downloaded rpms

Benefit:
- Confirm repos are working as expected.
- Discover origin of a package for security/risk assesment purposes.
- Determine if a repo is no longer providing packages and can be retired.

URL's will be logged in:
  output/3rd-centos_rpms_urls_L1.txt
  output/3rd-centos_srpms_urls_L1.txt
  output/3rd_rpms_urls_L1.txt
  output/3rd_srpms_urls_L1.txt
  output/centos_rpms_urls_L1.txt
  output/centos_srpms_urls_L1.txt

URL's for packages that are already available and do not require
re-download are not logged.

Story: 2003155
Task: 23290
Change-Id: I069f688241296b71e32054b00d5d1d34af22f202
Signed-off-by: Scott Little <scott.little@windriver.com>
---
 centos-mirror-tools/dl_rpms.sh | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh
index c413437f..767761a6 100755
--- a/centos-mirror-tools/dl_rpms.sh
+++ b/centos-mirror-tools/dl_rpms.sh
@@ -45,18 +45,22 @@ mkdir -p $MDIR_BIN
 FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
 FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
 MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
+URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt"
 
 cat /dev/null > $FAIL_MOVE_SRPMS
 cat /dev/null > $FOUND_SRPMS
 cat /dev/null > $MISSING_SRPMS
+cat /dev/null > $URL_SRPMS
 
 FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
 FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
 MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
+URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
 
 cat /dev/null > $FAIL_MOVE_RPMS
 cat /dev/null > $FOUND_RPMS
 cat /dev/null > $MISSING_RPMS
+cat /dev/null > $URL_RPMS
 
 #function to download different type of RPMs in different ways
 download () {
@@ -80,8 +84,10 @@ download () {
             echo " ------ using $SFILE to search $rpm_name ------"
             if [ "$_type" == "src" ];then
                 download_cmd="sudo -E yumdownloader -q -C --source $SFILE"
+                download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE"
             else
                 download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64"
+                download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64"
             fi
         else
             rpm_name=`echo $ff | cut -d"#" -f1-1`
@@ -94,6 +100,11 @@ download () {
             if [ ! -e $MDIR_SRC/$rpm_name ]; then
                 echo "Looking for $rpm_name"
                 if $download_cmd ; then
+                    # Success!   Record download URL.
+                    # Use 'sort --unique' because sometimes 
+                    # yumdownloader reports the url twice
+                    $download_url_cmd | sort --unique >> $URL_SRPMS
+
                     if ! mv -f $SFILE* $MDIR_SRC ; then
                         echo "FAILED to move $rpm_name"
                         echo $rpm_name >> $FAIL_MOVE_SRPMS
@@ -110,6 +121,11 @@ download () {
             if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
                 echo "Looking for $rpm_name..."
                 if $download_cmd ; then
+                    # Success!   Record download URL.
+                    # Use 'sort --unique' because sometimes 
+                    # yumdownloader reports the url twice
+                    $download_url_cmd | sort --unique >> $URL_RPMS
+
                     mkdir -p $MDIR_BIN/${_type}
                     if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
                         echo "FAILED to move $rpm_name"