Fix starlingx_add_pkgs.sh for use in a layered environment.
1) Make starlingx_add_pkgs.sh aware of the extra rpm repositories required for a layered build. 2) Don't spew errors if we come accross a dependency on rpmlib(*). These aren't dependencies on packages, but rather internal features of rpm. 3) Don't get baffled by epochs. feature_to_pkg returns a package version that might included an epoch. The queries within pkg_in_cgcsrepo were baffled by this. I add a second query stripping the epoch from the query. 4) Favor packages we already have over newer content from upstream. pkg_to_dependencies favors upstream packages if they look to be a higher version, so I dropped it's use. Instead I do a recursive search, with a modified feature_to_pkg that prefers existing sources. Story: 2006166 Task: 37103 Change-Id: I2b199c809267904fef08d9e93383dd2475adcb94 Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little
parent
451088b757
commit
faea86cf16
|
|
@ -84,16 +84,11 @@ if [ -z "$MY_REPO_ROOT_DIR" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
STARLINGX_ADD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
||||
|
||||
source $STARLINGX_ADD_PKGS_DIR/../toCOPY/lst_utils.sh
|
||||
|
||||
STXTOOLS=${MY_REPO_ROOT_DIR}/stx-tools
|
||||
REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d
|
||||
REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd)
|
||||
REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo
|
||||
REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX)
|
||||
cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED
|
||||
REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX)
|
||||
cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED
|
||||
REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX)
|
||||
cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES > $REPOCFG_ALL_MERGED
|
||||
|
||||
CGCSREPO_PATH=$MY_REPO/cgcs-centos-repo/Binary
|
||||
TISREPO_PATH=$MY_WORKSPACE/std/rpmbuild/RPMS
|
||||
|
|
@ -110,18 +105,9 @@ FAILED_LOG=failed.log
|
|||
RPMLIST=
|
||||
DOWNLOAD_LIST=
|
||||
|
||||
# It seems we have to manually disable the repos from /etc/yum.repos.d,
|
||||
# even though we're specifying a config file
|
||||
REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/')
|
||||
|
||||
REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS"
|
||||
REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED"
|
||||
REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED"
|
||||
REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED"
|
||||
REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS"
|
||||
|
||||
function cleanup {
|
||||
rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_ALL_MERGED
|
||||
rm -f $REPOCFG_STD_MERGED $REPOCFG_3RD_MERGED $REPOCFG_LOWER_LAYER_MERGED $REPOCFG_ALL_MERGED
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
|
@ -129,7 +115,7 @@ trap cleanup EXIT
|
|||
function show_usage {
|
||||
cat >&2 <<EOF
|
||||
Usage:
|
||||
$(basename $0) [ -d <pkgname> ] ... [ <rpmfile> ] ...
|
||||
$(basename $0) [ -C <config_dir> ] [ -l <layer> ] [ -d <pkgname> ] ... [ <rpmfile> ] ...
|
||||
|
||||
This utility uses the cgcs-centos-repo repo, and optionally the rpmbuild/RPMS
|
||||
repo from \$MY_WORKSPACE/std, as a baseline, downloading packages required
|
||||
|
|
@ -145,7 +131,7 @@ In addition, this utility will record a list of downloaded RPMs in the $RESULTS_
|
|||
or $RESULTS_3RD_LOG files, with failures recorded in $FAILED_LOG or $NOTFOUND_LOG.
|
||||
|
||||
The resulting download list can then be added to the appropriate .lst file in
|
||||
\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools
|
||||
\$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools/config/centos/flock/
|
||||
|
||||
Example:
|
||||
$(basename $0) -d linuxptp -d zlib puppet-gnocchi-11.3.0-1.el7.src.rpm
|
||||
|
|
@ -155,11 +141,19 @@ EOF
|
|||
exit 1
|
||||
}
|
||||
|
||||
while getopts "d:h" opt; do
|
||||
while getopts "C:d:l:h" opt; do
|
||||
case $opt in
|
||||
C)
|
||||
# Alternate config directory
|
||||
set_and_validate_config_dir "${OPTARG}"
|
||||
;;
|
||||
d)
|
||||
DOWNLOAD_LIST="$DOWNLOAD_LIST $OPTARG"
|
||||
;;
|
||||
l)
|
||||
# Set layer
|
||||
set_and_validate_layer "${OPTARG}"
|
||||
;;
|
||||
h)
|
||||
show_usage
|
||||
;;
|
||||
|
|
@ -173,31 +167,82 @@ done
|
|||
shift $((OPTIND-1))
|
||||
RPMLIST="${RPMLIST} $@"
|
||||
|
||||
REPO_DIR=${STXTOOLS}/centos-mirror-tools/yum.repos.d
|
||||
LAYER_REPO_DIR=${config_dir}/centos/${layer}/yum.repos.d
|
||||
|
||||
REPOCFG_STD_FILES=$(ls ${REPO_DIR}/StarlingX*.repo | grep -v StarlingX_3rd)
|
||||
REPOCFG_3RD_FILES=${REPO_DIR}/StarlingX_3rd*.repo
|
||||
REPOCFG_LOWER_LAYER_FILES=""
|
||||
if [ -d ${LAYER_REPO_DIR} ]; then
|
||||
REPOCFG_LOWER_LAYER_FILES=${LAYER_REPO_DIR}/StarlingX*.repo
|
||||
fi
|
||||
|
||||
REPOCFG_STD_MERGED=$(mktemp /tmp/REPOCFG_STD_MERGED_XXXXXX)
|
||||
cat $REPOCFG_STD_FILES > $REPOCFG_STD_MERGED
|
||||
|
||||
REPOCFG_3RD_MERGED=$(mktemp /tmp/REPOCFG_3RD_MERGED_XXXXXX)
|
||||
cat $REPOCFG_3RD_FILES > $REPOCFG_3RD_MERGED
|
||||
|
||||
REPOCFG_LOWER_LAYER_MERGED=$(mktemp /tmp/REPOCFG_LOWER_LAYER_MERGED_XXXXXX)
|
||||
if [ "$REPOCFG_LOWER_LAYER_FILES" != "" ]; then
|
||||
cat $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_LOWER_LAYER_MERGED
|
||||
fi
|
||||
|
||||
REPOCFG_ALL_MERGED=$(mktemp /tmp/REPOCFG_ALL_MERGED_XXXXXX)
|
||||
cat $REPOCFG_STD_FILES $REPOCFG_3RD_FILES $REPOCFG_LOWER_LAYER_FILES > $REPOCFG_ALL_MERGED
|
||||
|
||||
# It seems we have to manually disable the repos from /etc/yum.repos.d,
|
||||
# even though we're specifying a config file
|
||||
REPOQUERY_ARGS=$(grep -h '^\[' /etc/yum.repos.d/* | sed 's/[][]//g' | sed 's/^/--disablerepo=/')
|
||||
|
||||
REPOQUERY_CMD="repoquery --archlist=x86_64,noarch $REPOQUERY_ARGS"
|
||||
REPOQUERY_STD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_STD_MERGED"
|
||||
REPOQUERY_3RD_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_3RD_MERGED"
|
||||
REPOQUERY_LOWER_LAYER_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_LOWER_LAYER_MERGED"
|
||||
REPOQUERY_ALL_CMD="$REPOQUERY_CMD --quiet -c $REPOCFG_ALL_MERGED"
|
||||
REPOQUERY_LOCAL_CMD="$REPOQUERY_CMD --quiet --repofrompath cgcs,$CGCSREPO_PATH $TISREPO_PATH_ARGS"
|
||||
|
||||
|
||||
function rpmfile_requires {
|
||||
#
|
||||
# Map a specified rpm file to its dependency list
|
||||
#
|
||||
local rpmfile=$1
|
||||
|
||||
rpm -qp --requires $rpmfile | awk '{print $1}'
|
||||
# Never process rpmlib(*) requirements.
|
||||
# They are not regular requirements, but rather internal features of rpm.
|
||||
rpm -qp --requires $rpmfile | grep -v '^rpmlib[(]'
|
||||
}
|
||||
|
||||
function feature_to_pkg {
|
||||
#
|
||||
# Map a feature/capability to the package that provides it
|
||||
#
|
||||
local feature=$1
|
||||
local feature="$1"
|
||||
local pkg=
|
||||
pkg=$($REPOQUERY_STD_CMD $feature | head -1)
|
||||
|
||||
pkg=$($REPOQUERY_LOCAL_CMD "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides $feature | head -1)
|
||||
pkg=$($REPOQUERY_LOCAL_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_3RD_CMD $feature | head -1)
|
||||
pkg=$($REPOQUERY_LOWER_LAYER_CMD "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides $feature | head -1)
|
||||
if [ -z "$pkg" ]; then
|
||||
echo "Could not find in repo: $feature" >&2
|
||||
echo "Could not find in repo: $feature" >> $NOTFOUND_LOG
|
||||
pkg=$($REPOQUERY_LOWER_LAYER_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_STD_CMD "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_STD_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_3RD_CMD "$feature" | head -1)
|
||||
if [ -z $pkg ]; then
|
||||
pkg=$($REPOQUERY_3RD_CMD --qf='%{name}' --whatprovides "$feature" | head -1)
|
||||
if [ -z "$pkg" ]; then
|
||||
echo "Could not find in repo: $feature" >&2
|
||||
echo "Could not find in repo: $feature" >> $NOTFOUND_LOG
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -214,6 +259,15 @@ function pkg_to_dependencies {
|
|||
$REPOQUERY_ALL_CMD --resolve --requires --qf='%{name}' $pkg
|
||||
}
|
||||
|
||||
function simplified_pkg {
|
||||
#
|
||||
# drop the epoch
|
||||
#
|
||||
local pkg=$1
|
||||
|
||||
echo $pkg | sed 's/-[0-9]*:/-/'
|
||||
}
|
||||
|
||||
function pkg_in_cgcsrepo {
|
||||
#
|
||||
# Check whether the specified package is already in the downloaded (or built) repo
|
||||
|
|
@ -223,14 +277,20 @@ function pkg_in_cgcsrepo {
|
|||
local results=
|
||||
results=$($REPOQUERY_LOCAL_CMD --whatprovides $pkg)
|
||||
if [ -n "$results" ]; then
|
||||
return 0
|
||||
results=$($REPOQUERY_LOCAL_CMD $(simplified_pkg $pkg))
|
||||
if [ -n "$results" ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
local pkgname=
|
||||
pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' --whatprovides $pkg | head -1)
|
||||
if [ -z "$pkgname" ]; then
|
||||
echo "Failed to find a package providing $pkg" >&2
|
||||
return 1
|
||||
pkgname=$($REPOQUERY_ALL_CMD --quiet $REPOCFG_ARGS --qf='%{name}' $(simplified_pkg $pkg) | head -1)
|
||||
if [ -z "$pkgname" ]; then
|
||||
echo "Failed to find a package providing $pkg" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
results=$($REPOQUERY_LOCAL_CMD $pkgname)
|
||||
|
||||
|
|
@ -241,9 +301,9 @@ function download_pkg {
|
|||
#
|
||||
# Download the specified package and its dependencies
|
||||
#
|
||||
local feature=$1
|
||||
local feature="$1"
|
||||
local pkg=
|
||||
pkg=$(feature_to_pkg $feature)
|
||||
pkg=$(feature_to_pkg "$feature")
|
||||
if [ -z "$pkg" ]; then
|
||||
# Error should already be to stderr
|
||||
return 1
|
||||
|
|
@ -254,25 +314,25 @@ function download_pkg {
|
|||
local relativepath=
|
||||
local arch=
|
||||
local deps=
|
||||
local rpm_path=
|
||||
|
||||
repoid=$($REPOQUERY_STD_CMD --qf='%{repoid}' $pkg | head -1)
|
||||
if [ -n "$repoid" ]; then
|
||||
url=$($REPOQUERY_STD_CMD --location $pkg | head -1)
|
||||
relativepath=$($REPOQUERY_STD_CMD --qf='%{relativepath}' $pkg | head -1)
|
||||
arch=$($REPOQUERY_STD_CMD --qf='%{arch}' $pkg | head -1)
|
||||
deps=$($REPOQUERY_STD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}')
|
||||
LOG=$RESULTS_LOG
|
||||
else
|
||||
repoid=$($REPOQUERY_3RD_CMD --qf='%{repoid}' $pkg | head -1)
|
||||
url=$($REPOQUERY_3RD_CMD --location $pkg | head -1)
|
||||
relativepath=$($REPOQUERY_3RD_CMD --qf='%{relativepath}' $pkg | head -1)
|
||||
arch=$($REPOQUERY_3RD_CMD --qf='%{arch}' $pkg | head -1)
|
||||
deps=$($REPOQUERY_3RD_CMD --requires --qf='%{name}' $pkg | awk '{print $1}')
|
||||
LOG=$RESULTS_3RD_LOG
|
||||
fi
|
||||
|
||||
echo "Downloading $url"
|
||||
wget -q -O $CGCSREPO_PATH/$arch/$(basename $relativepath) $url
|
||||
rpm_path=$CGCSREPO_PATH/$arch/$(basename $relativepath)
|
||||
wget -q -O $rpm_path $url
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download $url" >&2
|
||||
|
|
@ -293,29 +353,39 @@ function download_pkg {
|
|||
echo "${pkg},${repoid},$url" >> $LOG
|
||||
|
||||
# Now check its dependencies
|
||||
local dep=
|
||||
for dep in $deps; do
|
||||
pkg_in_cgcsrepo $dep && continue
|
||||
download_pkg $dep
|
||||
download_rpm_dependencies $rpm_path
|
||||
}
|
||||
|
||||
|
||||
function download_rpm_dependencies {
|
||||
local rf=$1
|
||||
|
||||
rpmfile_requires $rf | while read feature; do
|
||||
|
||||
local pkg=
|
||||
pkg=$(feature_to_pkg "$feature")
|
||||
if [ -z "$pkg" ]; then
|
||||
# Already msged to stderr
|
||||
continue
|
||||
fi
|
||||
|
||||
pkg_in_cgcsrepo $pkg && continue
|
||||
download_pkg $pkg
|
||||
done || exit $?
|
||||
}
|
||||
|
||||
if [ -n "$RPMLIST" ]; then
|
||||
for rf in $RPMLIST; do
|
||||
rpmfile_requires $rf | while read feature; do
|
||||
pkg=$(feature_to_pkg $feature)
|
||||
pkg=$(feature_to_pkg "$feature")
|
||||
if [ -z "$pkg" ]; then
|
||||
# Already msged to stderr
|
||||
continue
|
||||
fi
|
||||
|
||||
dependencies=$(pkg_to_dependencies $pkg)
|
||||
pkg_in_cgcsrepo $pkg && continue
|
||||
download_pkg $pkg
|
||||
|
||||
for dep in $feature $dependencies; do
|
||||
pkg_in_cgcsrepo $dep && continue
|
||||
|
||||
download_pkg $dep
|
||||
done
|
||||
done || exit $?
|
||||
done
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in New Issue