starlingx
/
tools
Code Issues Proposed changes

downloader fails to follow redirects 

A redirect is used to aid in the transition from
mirror.starlingx.cengn.ca to mirror.starlingx.windriver.com.

Curl does not support redirects, unless the -L/--location options
are specified.

Partial-Bug: 2028678
Signed-off-by: Scott Little <scott.little@windriver.com>
Change-Id: I963c4cdbf4239b853ac4240d76b7f75e6e9c1173
This commit is contained in:
Scott Little 2023-08-02 12:36:18 -04:00
parent b89f1a40d5
commit 700987b216
4 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
centos-mirror-tools/dl_rpms_from_url.sh
View File

@ -138,7 +138,7 @@ download () {
dl_result=1 dl_result=1
download_url="$_url/$rpm_name" download_url="$_url/$rpm_name"
download_cmd="curl --silent --output $rpm_name ${download_url}" download_cmd="curl -L --silent --output $rpm_name ${download_url}"
echo "Looking for $rpm_name" echo "Looking for $rpm_name"
echo "--> run: $download_cmd" echo "--> run: $download_cmd"

2
centos-mirror-tools/download_mirror.sh
View File

@ -315,7 +315,7 @@ for key in "${!layer_pkg_urls[@]}"; do
url="${layer_pkg_urls[${key}]}" url="${layer_pkg_urls[${key}]}"
name_from_url=$(url_to_file_name $url) name_from_url=$(url_to_file_name $url)
list="${rpms_from_layer_build_dir}/${name_from_url}" list="${rpms_from_layer_build_dir}/${name_from_url}"
curl --silent --fail ${url} > ${list} || curl -L --silent --fail ${url} > ${list} ||
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url: ${url}" echo "ERROR: Failed to download from url: ${url}"
exit 1 exit 1

6
centos-mirror-tools/utils.sh
View File

@ -53,9 +53,9 @@ get_download_file_command() {
_opts+=" -O" _opts+=" -O"
fi fi
if [[ "$_name" == http?(s)://* ]]; then if [[ "$_name" == http?(s)://* ]]; then
_ret="curl $_opts $_name" _ret="curl -L $_opts $_name"
else else
_ret="curl $_opts $(koji_url $_name)" _ret="curl -L $_opts $(koji_url $_name)"
fi fi
echo "$_ret" echo "$_ret"
} }
@ -81,7 +81,7 @@ download_file() {
else else
local _dest_file="$(basename "$1")" local _dest_file="$(basename "$1")"
fi fi
if curl $_opts -o "${_dest_file}.dl_part" "$1" ; then if curl -L $_opts -o "${_dest_file}.dl_part" "$1" ; then
\mv -fT "${_dest_file}.dl_part" "${_dest_file}" \mv -fT "${_dest_file}.dl_part" "${_dest_file}"
return 0 return 0
fi fi

9
toCOPY/generate-centos-repo.sh
View File

@ -221,7 +221,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
done done
if [ ! -f ${inc_file_dir}/${list} ]; then if [ ! -f ${inc_file_dir}/${list} ]; then
curl --silent --fail ${url} > ${inc_file_dir}/${list} curl -L --silent --fail ${url} > ${inc_file_dir}/${list}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url '${url}'" echo "ERROR: Failed to download from url '${url}'"
exit 1 exit 1
@ -252,7 +252,7 @@ for key in "${!layer_wheels_inc_urls[@]}"; do
done done
if [ ! -f ${wheels_file_dir}/${list} ]; then if [ ! -f ${wheels_file_dir}/${list} ]; then
curl --silent --fail ${url} > ${wheels_file_dir}/${list} curl -L --silent --fail ${url} > ${wheels_file_dir}/${list}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url '${url}'" echo "ERROR: Failed to download from url '${url}'"
exit 1 exit 1
@ -292,7 +292,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
done done
if [ ! -f ${build_info_file_dir}/${list} ]; then if [ ! -f ${build_info_file_dir}/${list} ]; then
curl --silent --fail ${url} > ${build_info_file_dir}/${list} curl -L --silent --fail ${url} > ${build_info_file_dir}/${list}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "WARNING: Failed to download from url '${url}'" echo "WARNING: Failed to download from url '${url}'"
fi fi
@ -319,7 +319,7 @@ for key in "${!layer_pkg_urls[@]}"; do
done done
if [ ! -f ${lst_file_dir}/${list} ]; then if [ ! -f ${lst_file_dir}/${list} ]; then
curl --silent --fail ${url} > ${lst_file_dir}/${list} curl -L --silent --fail ${url} > ${lst_file_dir}/${list}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "ERROR: Failed to download from url '${url}'" echo "ERROR: Failed to download from url '${url}'"
exit 1 exit 1
@ -360,6 +360,7 @@ process_lst_file () {
local lst_file="${1}" local lst_file="${1}"
local dest_dir="${2}" local dest_dir="${2}"
echo "process_lst_file: ${1} ${2}"
grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do
if [ -z "${rpmname}" ]; then if [ -z "${rpmname}" ]; then
continue continue