Fast download of lower layer rpms
The default yumdownloader approach to downloading files does so one at a time, and is very slow. Too slow for layered build which is built around the assumption that files can be downloaded faster than they can be built. This update will switch from yumdownloader to reposync for the download of lower layer rpms. It exploits the fact that each layer and build type publishes it's own lst file of rpms to download. The lst file is transformed into an' includepkgs directive in a custom yum.conf which is passed to reposync, so we only sync the desited rpms. Reposync won't redownload rpms that it already has, even if the upstream repodata indicates that the files checksum has changed. Forceing the redownload of these rpms requires that we manually download the upstream repodata and that we use verifytree to identify and delete the new obsolete rpms. Also including two small bug fixes found while investivgating an alternative solution to launchpad 1864221. - incorrect userid in a chown - Bug in get_url, --url pass being passed in the wrong place Story: 2006166 Task: 39307 Closes-Bug: 1864221 Change-Id: If12b98ff4f5f24d9318250356920f397419f0f80 Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little
parent
6279b3f773
commit
bbd79fcd38
|
|
@ -0,0 +1,529 @@
|
|||
#!/bin/bash
|
||||
|
||||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
#
|
||||
# Fast download of StarlingX built rpms using verifytree and repsync
|
||||
#
|
||||
|
||||
DL_LOWER_LAYER_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
||||
|
||||
source $DL_LOWER_LAYER_RPMS_DIR/utils.sh
|
||||
|
||||
usage() {
|
||||
echo "$0 -l <layer> -b <build_type> -r <url> <rpms_list> <match_level> [ -c <yum.conf> ] [-D <distro> ] [-s|-S|-u|-U] [-x]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -b: <build type>: e.g. std, rt, installer."
|
||||
echo " -c: Use an alternate yum.conf rather than the system file"
|
||||
echo " -l: <layer>: e.g. compiler, distro, flock."
|
||||
echo " -r: <url>: Url of the root of the repo. Expect to find a repodata dir there."
|
||||
echo " -x: Clean log files only, do not run."
|
||||
echo " rpm_list: a list of RPM files to be downloaded."
|
||||
echo " match_level: value could be L1, L2 or L3:"
|
||||
echo " L1: use name, major version and minor version:"
|
||||
echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
|
||||
echo " L2: use name and major version:"
|
||||
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
|
||||
echo " L3: use name:"
|
||||
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
|
||||
echo " K1: Use Koji rather than yum repos as a source."
|
||||
echo " Koji has a longer retention period than epel mirrors."
|
||||
echo ""
|
||||
echo " Download Source Options: Only select one of these."
|
||||
echo " -s: Download from StarlingX mirror only"
|
||||
echo " -S: Download from StarlingX mirror, upstream as backup (default)"
|
||||
echo " -u: Download from original upstream sources only"
|
||||
echo " -U: Download from original upstream sources, StarlingX mirror as backup"
|
||||
echo ""
|
||||
echo "Returns: 0 = All files downloaded successfully"
|
||||
echo " 1 = Some files could not be downloaded"
|
||||
echo " 2 = Bad arguements or other error"
|
||||
echo ""
|
||||
}
|
||||
|
||||
|
||||
CLEAN_LOGS_ONLY=0
|
||||
dl_rc=0
|
||||
|
||||
# Permitted values of dl_source
|
||||
dl_from_stx_mirror="stx_mirror"
|
||||
dl_from_upstream="upstream"
|
||||
dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream"
|
||||
dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror"
|
||||
|
||||
# Download from what source?
|
||||
# dl_from_stx_mirror = StarlingX mirror only
|
||||
# dl_from_upstream = Original upstream source only
|
||||
# dl_from_stx_then_upstream = Either source, STX prefered (default)"
|
||||
# dl_from_upstream_then_stx = Either source, UPSTREAM prefered"
|
||||
dl_source="$dl_from_stx_then_upstream"
|
||||
dl_flag=""
|
||||
build_type=""
|
||||
lower_layer=""
|
||||
url_root=""
|
||||
distro="centos"
|
||||
|
||||
# Set a default yum.conf which can be overridden by use of '-c' option.
|
||||
# I assume we are called from download_mirror.sh and are already in
|
||||
# stx-tools/centos-mirror-tools directory.
|
||||
YUM_CONF=yum.conf.sample
|
||||
|
||||
MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U"
|
||||
|
||||
multiple_dl_flag_check () {
|
||||
if [ "$dl_flag" != "" ]; then
|
||||
echo "$MULTIPLE_DL_FLAG_ERROR_MSG"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Parse option flags
|
||||
while getopts "b:c:l:D:hr:sSuUx" o; do
|
||||
case "${o}" in
|
||||
b)
|
||||
build_type="$OPTARG"
|
||||
;;
|
||||
c)
|
||||
# Use an alternate yum.conf
|
||||
YUM_CONF="$OPTARG"
|
||||
;;
|
||||
D)
|
||||
distro="${OPTARG}"
|
||||
;;
|
||||
l)
|
||||
lower_layer="$OPTARG"
|
||||
;;
|
||||
r)
|
||||
# URL
|
||||
url_root="${OPTARG}"
|
||||
;;
|
||||
s)
|
||||
# Download from StarlingX mirror only. Do not use upstream sources.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_stx_mirror"
|
||||
dl_flag="-s"
|
||||
;;
|
||||
S)
|
||||
# Download from StarlingX mirror first, only use upstream source as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_stx_then_upstream"
|
||||
dl_flag="-S"
|
||||
;;
|
||||
u)
|
||||
# Download from upstream only. Do not use StarlingX mirror.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_upstream"
|
||||
dl_flag="-u"
|
||||
;;
|
||||
U)
|
||||
# Download from upstream first, only use StarlingX mirror as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_upstream_then_stx"
|
||||
dl_flag="-U"
|
||||
;;
|
||||
x)
|
||||
# Clean only
|
||||
CLEAN_LOGS_ONLY=1
|
||||
;;
|
||||
h)
|
||||
# Help
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
if [ -z "${build_type}" ] || [ -z "${lower_layer}" ] || [ -z "${url_root}" ]; then
|
||||
usage
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ $# -lt 2 ]; then
|
||||
usage
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ "$1" == "" ]; then
|
||||
echo "Need to supply the rpm file list"
|
||||
exit 2;
|
||||
else
|
||||
rpms_list=$1
|
||||
echo "using $rpms_list as the download name lists"
|
||||
fi
|
||||
|
||||
match_level="L1"
|
||||
|
||||
if [ ! -z "$2" -a "$2" != " " ];then
|
||||
match_level=$2
|
||||
fi
|
||||
|
||||
if [ ! -f ${YUM_CONF} ]; then
|
||||
echo "ERROR: failed to find ${YUM_CONF}"
|
||||
usage
|
||||
exit 2
|
||||
fi
|
||||
|
||||
timestamp=$(date +%F_%H%M)
|
||||
echo $timestamp
|
||||
|
||||
export DL_MIRROR_LOG_DIR="${DL_MIRROR_LOG_DIR:-./logs}"
|
||||
export DL_MIRROR_OUTPUT_DIR="${DL_MIRROR_OUTPUT_DIR:-./output/stx-r1/CentOS/pike}"
|
||||
|
||||
dl_dir="$(readlink -f ${DL_MIRROR_OUTPUT_DIR})/layer_repos/${lower_layer}/${build_type}"
|
||||
|
||||
LOGSDIR="${DL_MIRROR_LOG_DIR}"
|
||||
MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${lower_layer}_${build_type}.log"
|
||||
FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${lower_layer}_${build_type}.log"
|
||||
cat /dev/null > $MISSING_RPMS
|
||||
cat /dev/null > $FOUND_RPMS
|
||||
|
||||
|
||||
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CREATEREPO=$(which createrepo_c)
|
||||
if [ $? -ne 0 ]; then
|
||||
CREATEREPO="createrepo"
|
||||
fi
|
||||
|
||||
number_of_cpus () {
|
||||
/usr/bin/nproc
|
||||
}
|
||||
|
||||
get_remote_dir () {
|
||||
local url="${1}"
|
||||
local dest_dir="${2}"
|
||||
mkdir -p "${dest_dir}" || return 1
|
||||
\rm "${dest_dir}/"index.html*
|
||||
wget -c -N --recursive --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}/"
|
||||
}
|
||||
|
||||
get_remote_file () {
|
||||
local url="${1}"
|
||||
local dest_dir="${2}"
|
||||
mkdir -p "${dest_dir}" || return 1
|
||||
wget -c -N --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}"
|
||||
}
|
||||
|
||||
get_remote_file_overwrite () {
|
||||
local url="${1}"
|
||||
local dest_dir="${2}"
|
||||
local dest_file="${dest_dir}/$(basename ${url})"
|
||||
mkdir -p "${dest_dir}" || return 1
|
||||
|
||||
if [ -f "${dest_file}" ]; then
|
||||
\rm "${dest_file}"
|
||||
fi
|
||||
wget -c -N --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}"
|
||||
}
|
||||
|
||||
clean_repodata () {
|
||||
local repodata="${1}"
|
||||
local f=""
|
||||
local f2=""
|
||||
|
||||
if [ ! -f "${repodata}/repomd.xml" ]; then
|
||||
echo "Error: clean_repodata: file not found: ${repodata}/repomd.xml"
|
||||
return 1
|
||||
fi
|
||||
|
||||
for f in $(find "${repodata}" -name '[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]*'); do
|
||||
f2=$(basename "${f}")
|
||||
if ! grep -q "${f2}" "${repodata}/repomd.xml"; then
|
||||
\rm "${f}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
dl_repo () {
|
||||
local base_url="${1}"
|
||||
local dl_dir="${2}"
|
||||
local rpms_list="${3}"
|
||||
|
||||
if [ -z "${base_url}" ] || [ -z "${dl_dir}" ] || [ -z "${rpms_list}" ]; then
|
||||
echo "ERROR: dl_repo: missing arguement"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${rpms_list}" ]; then
|
||||
echo "ERROR: dl_repo: no such file '${rpms_list}'"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local REPO_URL="${base_url}"
|
||||
local DOWNLOAD_PATH="${dl_dir}"
|
||||
local DOWNLOAD_PATH_NEW="${DOWNLOAD_PATH}.new"
|
||||
local DOWNLOAD_PATH_OLD="${DOWNLOAD_PATH}.old"
|
||||
|
||||
mkdir -p "${DOWNLOAD_PATH}"
|
||||
|
||||
local YUM_CONF_TMP
|
||||
local TMP
|
||||
local YUM_CONF_DIR_TMP
|
||||
local MUNGED_LIST
|
||||
YUM_CONF_TMP="$(mktemp "/tmp/${YUM_CONF}.XXXXXX")"
|
||||
TMP=$(echo "${YUM_CONF_TMP}" | sed "s#^.*${YUM_CONF}.##")
|
||||
YUM_CONF_DIR=$(dirname "${YUM_CONF_TMP}")
|
||||
YUM_REPOS_DIR_TMP="${YUM_CONF_DIR}/yum.repos.d.${TMP}"
|
||||
MUNGED_LIST="${YUM_CONF_DIR}/yum.lst.${TMP}"
|
||||
|
||||
grep -v '^$' "${rpms_list}" | grep -v '^#' | sed 's#^\(.*\)[.]rpm#\t\1#' | sort --unique > ${MUNGED_LIST}
|
||||
\cp "${YUM_CONF}" "${YUM_CONF_TMP}"
|
||||
sed -i "s#^reposdir=.*#reposdir=${YUM_REPOS_DIR_TMP}#" "${YUM_CONF_TMP}"
|
||||
mkdir -p "${YUM_REPOS_DIR_TMP}"
|
||||
|
||||
REPOID=${lower_layer}_${build_type}
|
||||
REPO_FILE="${YUM_REPOS_DIR_TMP}/${REPOID}.repo"
|
||||
echo "[${REPOID}]" > "${REPO_FILE}"
|
||||
echo "name=${REPOID}" >> "${REPO_FILE}"
|
||||
echo "baseurl=${REPO_URL}" >> "${REPO_FILE}"
|
||||
echo "includepkgs=" >> "${REPO_FILE}"
|
||||
echo "include=file://${MUNGED_LIST}" >> "${REPO_FILE}"
|
||||
echo "enabled=0" >> "${REPO_FILE}"
|
||||
|
||||
# copy repo to a temp location
|
||||
if [ -d "${DOWNLOAD_PATH_NEW}" ]; then
|
||||
\rm -rf "${DOWNLOAD_PATH_NEW}"
|
||||
fi
|
||||
|
||||
if [ -d "${DOWNLOAD_PATH}" ]; then
|
||||
CMD="\cp --archive --link '${DOWNLOAD_PATH}' '${DOWNLOAD_PATH_NEW}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Download latest repodata
|
||||
get_remote_dir "${REPO_URL}/repodata" "${DOWNLOAD_PATH_NEW}/repodata.upstream"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: get_remote_dir ${REPO_URL}/repodata ${DOWNLOAD_PATH_NEW}/repodata.upstream"
|
||||
return 1
|
||||
fi
|
||||
|
||||
get_remote_file_overwrite "${REPO_URL}/repodata/repomd.xml" "${DOWNLOAD_PATH_NEW}/repodata.upstream/"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: get_remote_file_overwrite ${REPO_URL}/repodata/repomd.xml ${DOWNLOAD_PATH_NEW}/repodata.upstream/"
|
||||
return 1
|
||||
fi
|
||||
|
||||
clean_repodata "${DOWNLOAD_PATH_NEW}/repodata.upstream/"
|
||||
|
||||
# Download latest rpm.lst
|
||||
get_remote_file_overwrite "${REPO_URL}/rpm.lst" "${DOWNLOAD_PATH_NEW}/"
|
||||
|
||||
#
|
||||
# Delete rpms that are no longer valid
|
||||
#
|
||||
|
||||
# Save active repodata as local
|
||||
if [ -d "${DOWNLOAD_PATH_NEW}/repodata" ]; then
|
||||
CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata' '${DOWNLOAD_PATH_NEW}/repodata.local'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Make upstream repodata the active copy
|
||||
CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata.upstream' '${DOWNLOAD_PATH_NEW}/repodata'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Do the audit, delete anything broken
|
||||
for f in $(verifytree -a "file://${DOWNLOAD_PATH_NEW}" | \
|
||||
sed '1,/Checking all packages/d' | \
|
||||
grep -v ' FAILED$' | \
|
||||
awk '{ print $2 }' | \
|
||||
sed 's/^[0-9]*://'); do
|
||||
echo "Already have $f"
|
||||
done
|
||||
for f in $(verifytree -a "file://${DOWNLOAD_PATH_NEW}" | \
|
||||
sed '1,/Checking all packages/d' | \
|
||||
grep ' FAILED$' | \
|
||||
awk '{ print $2 }' | \
|
||||
sed 's/^[0-9]*://'); do
|
||||
echo "Downloading $f"
|
||||
for f_path in $(find "${DOWNLOAD_PATH_NEW}" -name ${f}.rpm); do
|
||||
CMD="\rm '${f_path}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# deactivate and restore upstream repo data
|
||||
CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata' '${DOWNLOAD_PATH_NEW}/repodata.upstream'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restore our active repodata
|
||||
if [ -d "${DOWNLOAD_PATH_NEW}/repodata.local" ]; then
|
||||
CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata.local' '${DOWNLOAD_PATH_NEW}/repodata'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Sync the repo's rpms
|
||||
CMD="reposync --tempcache --norepopath -l --config='${YUM_CONF_TMP}' --repoid=$REPOID --download_path='${DOWNLOAD_PATH_NEW}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
|
||||
CMD="pushd '${DOWNLOAD_PATH_NEW}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Update the repodata
|
||||
OPTIONS="--workers $(number_of_cpus)"
|
||||
if [ -f comps.xml ]; then
|
||||
OPTIONS="$OPTIONS -g comps.xml"
|
||||
fi
|
||||
if [ -d repodata ]; then
|
||||
OPTIONS="$OPTIONS --update"
|
||||
fi
|
||||
|
||||
CMD="$CREATEREPO $OPTIONS ."
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
popd
|
||||
return 1
|
||||
fi
|
||||
|
||||
popd
|
||||
|
||||
# Swap out the old copy of our repo
|
||||
if [ -d "${DOWNLOAD_PATH}" ]; then
|
||||
CMD="\mv '${DOWNLOAD_PATH}' '${DOWNLOAD_PATH_OLD}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
\rm -rf "${DOWNLOAD_PATH_NEW}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Swap in the updated repo
|
||||
CMD="\mv '${DOWNLOAD_PATH_NEW}' '${DOWNLOAD_PATH}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
\mv "${DOWNLOAD_PATH_NEW}" "${DOWNLOAD_PATH}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Delete the old repo
|
||||
if [ -d "${DOWNLOAD_PATH_OLD}" ]; then
|
||||
CMD="\rm -rf '${DOWNLOAD_PATH_OLD}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
fi
|
||||
fi
|
||||
|
||||
CMD="\rm '${YUM_CONF_TMP}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
fi
|
||||
|
||||
CMD="\rm '${MUNGED_LIST}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
fi
|
||||
|
||||
CMD="\rm -rf '${YUM_REPOS_DIR_TMP}'"
|
||||
echo "$CMD"
|
||||
eval $CMD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: $CMD"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
# Loop over download sources... typically the STX mirror folowed by upstream
|
||||
# ... until we have all the rpms.
|
||||
#
|
||||
|
||||
RC=1
|
||||
for dl_src in $dl_source; do
|
||||
url_root_to_use="${url_root}"
|
||||
case $dl_src in
|
||||
$dl_from_stx_mirror)
|
||||
url_root_to_use="$(url_to_stx_mirror_url "${url_root}" ${distro})"
|
||||
;;
|
||||
$dl_from_upstream)
|
||||
url_root_to_use="${url_root}"
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown dl_source '$dl_src'"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
dl_repo "${url_root_to_use}" "${dl_dir}" "${rpms_list}"
|
||||
if [ $? -eq 0 ]; then
|
||||
RC=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
for rpm_name in $(grep -v '^$' "${rpms_list}" | grep -v '^#' ); do
|
||||
if [ ! -f "${dl_dir}/${rpm_name}" ]; then
|
||||
echo "${rpm_name}" >> $MISSING_RPMS
|
||||
echo "Failed to download: ${rpm_name}"
|
||||
RC=1
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
exit $RC
|
||||
|
|
@ -20,7 +20,7 @@ DL_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
|||
source $DL_RPMS_DIR/utils.sh
|
||||
|
||||
usage() {
|
||||
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
|
||||
echo "$0 [-n] [-c <yum.conf>] [-s|-S|-u|-U] [-x] <rpms_list> <match_level> "
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -n: Do not use sudo when performing operations"
|
||||
|
|
@ -37,6 +37,12 @@ usage() {
|
|||
echo " K1: Use Koji rather than yum repos as a source."
|
||||
echo " Koji has a longer retention period than epel mirrors."
|
||||
echo ""
|
||||
echo " Download Source Options: Only select one of these."
|
||||
echo " -s: Download from StarlingX mirror only"
|
||||
echo " -S: Download from StarlingX mirror, upstream as backup (default)"
|
||||
echo " -u: Download from original upstream sources only"
|
||||
echo " -U: Download from original upstream sources, StarlingX mirror as backup"
|
||||
echo ""
|
||||
echo "Returns: 0 = All files downloaded successfully"
|
||||
echo " 1 = Some files could not be downloaded"
|
||||
echo " 2 = Bad arguements or other error"
|
||||
|
|
@ -100,7 +106,7 @@ while getopts "c:nxD:sSuUh" o; do
|
|||
dl_flag="-s"
|
||||
;;
|
||||
S)
|
||||
# Download from StarlingX mirror only. Do not use upstream sources.
|
||||
# Download from StarlingX mirror first, only use upstream source as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_stx_then_upstream"
|
||||
dl_flag="-S"
|
||||
|
|
@ -112,7 +118,7 @@ while getopts "c:nxD:sSuUh" o; do
|
|||
dl_flag="-u"
|
||||
;;
|
||||
U)
|
||||
# Download from upstream only. Do not use StarlingX mirror.
|
||||
# Download from upstream first, only use StarlingX mirror as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_upstream_then_stx"
|
||||
dl_flag="-U"
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ need_dir(){
|
|||
|
||||
# Downloader scripts
|
||||
rpm_downloader="${DOWNLOAD_MIRROR_DIR}/dl_rpms.sh"
|
||||
lower_layer_rpm_downloader="${DOWNLOAD_MIRROR_DIR}/dl_lower_layer_rpms.sh"
|
||||
rpm_from_url_downloader="${DOWNLOAD_MIRROR_DIR}/dl_rpms_from_url.sh"
|
||||
tarball_downloader="${DOWNLOAD_MIRROR_DIR}/dl_tarball.sh"
|
||||
other_downloader="${DOWNLOAD_MIRROR_DIR}/dl_other_from_centos_repo.sh"
|
||||
|
|
@ -102,6 +103,7 @@ rpms_from_3rd_parties_template="rpms_3rdparties.lst"
|
|||
rpms_from_centos_repo_template="rpms_centos.lst"
|
||||
rpms_from_centos_3rd_parties_template="rpms_centos3rdparties.lst"
|
||||
rpms_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_pkg_lists
|
||||
rpms_from_layer_repos_dir=${DL_MIRROR_OUTPUT_DIR}/layer_repos
|
||||
image_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_image_inc
|
||||
build_info_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_build_info
|
||||
tarball_downloads_template="tarball-dl.lst"
|
||||
|
|
@ -190,7 +192,7 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
|
|||
dl_flag="-s"
|
||||
;;
|
||||
S)
|
||||
# Download from StarlingX mirror only. Do not use upstream sources.
|
||||
# Download from StarlingX mirror first, only use upstream source as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_stx_then_upstream"
|
||||
dl_flag="-S"
|
||||
|
|
@ -202,7 +204,7 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
|
|||
dl_flag="-u"
|
||||
;;
|
||||
U)
|
||||
# Download from upstream only. Do not use StarlingX mirror.
|
||||
# Download from upstream first, only use StarlingX mirror as a fallback.
|
||||
multiple_dl_flag_check
|
||||
dl_source="$dl_from_upstream_then_stx"
|
||||
dl_flag="-U"
|
||||
|
|
@ -324,7 +326,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
|
|||
url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' )
|
||||
name_from_url=$(url_to_file_name $url)
|
||||
dest="${build_info_from_layer_build_dir}/${name_from_url}"
|
||||
curl --silent --fail ${url} > ${dest} ||
|
||||
curl --silent --fail ${url} > ${dest}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "ERROR: Failed to download from url: ${url}"
|
||||
exit 1
|
||||
|
|
@ -432,7 +434,12 @@ for key in "${!layer_pkg_urls[@]}"; do
|
|||
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
|
||||
level=L1
|
||||
logfile=$(generate_log_name $list $level)
|
||||
$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
|
||||
llrd_extra_args=""
|
||||
if [ $use_system_yum_conf -eq 0 ]; then
|
||||
llrd_extra_args="-c '${alternate_yum_conf}'"
|
||||
fi
|
||||
echo "$lower_layer_rpm_downloader -l ${lower_layer} -b ${build_type} -r $(dirname $url) ${llrd_extra_args} ${list} ${level}"
|
||||
$lower_layer_rpm_downloader -l ${lower_layer} -b ${build_type} -r $(dirname $url) ${llrd_extra_args} ${list} ${level} |& tee $logfile
|
||||
local_retcode=${PIPESTATUS[0]}
|
||||
fi
|
||||
|
||||
|
|
@ -583,7 +590,9 @@ fi
|
|||
|
||||
if [ $change_group_ids -eq 1 ]; then
|
||||
# change "./output" and sub-folders to 751 (cgcs) group
|
||||
${SUDO} chown 751:751 -R ./output
|
||||
NEW_UID=$(id -u)
|
||||
NEW_GID=751
|
||||
${SUDO} chown ${NEW_UID}:${NEW_GID} -R ./output
|
||||
fi
|
||||
|
||||
echo "step #4: start downloading other files ..."
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ get_url() {
|
|||
_ret="$(url_to_stx_mirror_url $_ret $distro)"
|
||||
fi
|
||||
else
|
||||
_url_cmd="$(get_yum_command $_name $_level)"
|
||||
_url_cmd="$(get_yum_command $_name $_level) --url"
|
||||
|
||||
# When we add --url to the yum download command,
|
||||
# --archlist is no longer enforced. Multiple
|
||||
|
|
@ -74,7 +74,7 @@ get_url() {
|
|||
# filter urls for the desitered arch.
|
||||
local arr=( $(split_filename $_name) )
|
||||
local arch=${arr[3]}
|
||||
_ret="$($_url_cmd --url | grep "[.]$arch[.]rpm$")"
|
||||
_ret="$($_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)"
|
||||
fi
|
||||
echo "$_ret"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -142,7 +142,7 @@ missing_rpms_file=missing.txt
|
|||
\rm -f ${missing_rpms_file}
|
||||
|
||||
# Strip trailing / from mirror_dir if it was specified...
|
||||
mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%")
|
||||
mirror_dir=$(readlink -f ${mirror_dir} | sed "s%/$%%")
|
||||
|
||||
if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then
|
||||
echo "The mirror ${mirror_dir} doesn't has the Binary and Source"
|
||||
|
|
|
|||
Loading…
Reference in New Issue