353 lines
12 KiB
Bash
Executable File
353 lines
12 KiB
Bash
Executable File
#!/bin/env bash
|
|
|
|
#
|
|
# Copyright (c) 2018-2020 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
# Here's the score, kids. There are a few different places from which we can
|
|
# get packages. In priority order, they are:
|
|
#
|
|
# The CGTS packages we've built ourselves
|
|
# The CGTS packages that Jenkins has built (coming soon to a script near you)
|
|
# The CentOS packages in various repos
|
|
# - Base OS
|
|
# - OpenStack Repos
|
|
# EPEL (Extra Packages for Enterprise Linux)
|
|
#
|
|
# This script can function in two ways:
|
|
# If you specify a filename, it assumes the file is a list of packages you
|
|
# want to install, or dependencies you want to meet. It installs whatever
|
|
# is in the list into current directory. Failure to find a dependency
|
|
# results in a return code of 1
|
|
#
|
|
# If no file is specified, we generate a file ($DEPLISTFILE) of dependencies
|
|
# based on current directory
|
|
#
|
|
# We then continuously loop through generating new dependencies and installing
|
|
# them until either all dependencies are met, or we cannot install anymore
|
|
#
|
|
# We also log where dependencies were installed from into
|
|
# export/dist/report_deps.txt
|
|
#
|
|
|
|
CGTS_DEPS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
|
|
|
# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and
|
|
# REPOQUERY_WHATPROVIDES_DELIM for our build environment.
|
|
source ${CGTS_DEPS_DIR}/../pkg-manager-utils.sh
|
|
|
|
# This function generates a simple file of dependencies we're trying to resolve
|
|
function generate_dep_list {
|
|
TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX)
|
|
mkdir -p $TMP_RPM_DB
|
|
rpm --initdb --dbpath $TMP_RPM_DB
|
|
rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' > $DEPLISTFILE_NEW 2>&1
|
|
cat $DEPLISTFILE_NEW >> $DEPDETAILLISTFILE
|
|
cat $DEPLISTFILE_NEW \
|
|
| grep -v -e "error:" -e "warning:" -e "Preparing..." \
|
|
-e "Verifying..." -e "installing package" \
|
|
| sed -e "s/ is needed by.*$//" -e "s/ [<=>].*$//" \
|
|
| sort -u > $DEPLISTFILE
|
|
\rm -rf $TMP_RPM_DB
|
|
}
|
|
|
|
join_array() {
|
|
local IFS="$1"
|
|
shift
|
|
echo "$*"
|
|
}
|
|
|
|
# Takes a list of requirements (either explcit package name, or capabilities
|
|
# to provide) and install packages to meet those dependancies
|
|
#
|
|
# We take the list of requirements and first try to look them up based on
|
|
# package name. If we can't find a package with the name of the requirement,
|
|
# we use --whatprovides to complete the lookup.
|
|
#
|
|
# The reason for this initial name-based attempt is that a couple of funky
|
|
# packages (notably -devel packages) have "Provides:" capabilities which
|
|
# conflict with named packages. So if explictly say we want "xyz" then we'll
|
|
# install the "xyz" package, rather than "something-devel" which has "xyz"
|
|
# capabilities.
|
|
function install_deps {
|
|
local DEP_LIST=""
|
|
local DEP_LIST_ARRAY=()
|
|
local DEP_LIST_FILE="$1"
|
|
|
|
# Temporary files are used in a few different ways
|
|
# Here we essenitally create variable aliases to make it easier to read
|
|
# the script
|
|
local UNSORTED_PACKAGES=$TMPFILE
|
|
local SORTED_PACKAGES=$TMPFILE1
|
|
local UNRESOLVED_PACKAGES=$TMPFILE2
|
|
|
|
rm -f $UNSORTED_PACKAGES
|
|
|
|
while read DEP
|
|
do
|
|
DEP_LIST+=" '${DEP}'"
|
|
done < $DEP_LIST_FILE
|
|
|
|
echo "Debug: List of deps to resolve: ${DEP_LIST}"
|
|
|
|
if [ -z "${DEP_LIST}" ]; then
|
|
return 0
|
|
fi
|
|
|
|
# go through each repo and convert deps to packages based on package name
|
|
for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do
|
|
echo "TMPDIR=${TMP_DIR}"\
|
|
"${REPOQUERY} --config=${YUM} --repoid=$REPOID"\
|
|
"${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\
|
|
"${DEP_LIST} --qf='%{name}'"
|
|
|
|
TMPDIR=${TMP_DIR} \
|
|
${REPOQUERY} --config=${YUM} --repoid=$REPOID \
|
|
${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \
|
|
--qf='%{name}' ${DEP_LIST} \
|
|
| sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES
|
|
|
|
\rm -rf $TMP_DIR/yum-$USER-*
|
|
done
|
|
sort $UNSORTED_PACKAGES -u > $SORTED_PACKAGES
|
|
|
|
# figure out any dependancies which could not be resolved based on
|
|
# package name. We use --whatpovides to deal with this
|
|
#
|
|
# First, we build a new DEP_LIST based on what was NOT found in
|
|
# search-by-name attempt
|
|
sort $DEP_LIST_FILE -u > $TMPFILE
|
|
comm -2 -3 $TMPFILE $SORTED_PACKAGES > $UNRESOLVED_PACKAGES
|
|
|
|
# If there are any requirements not resolved, look up the packages with
|
|
# --whatprovides
|
|
if [ -s $UNRESOLVED_PACKAGES ]; then
|
|
DEP_LIST_ARRAY=()
|
|
\cp $SORTED_PACKAGES $UNSORTED_PACKAGES
|
|
while read DEP
|
|
do
|
|
DEP_LIST_ARRAY+=( "${DEP}" )
|
|
done < $UNRESOLVED_PACKAGES
|
|
|
|
if [ "${REPOQUERY_WHATPROVIDES_DELIM}" != " " ]; then
|
|
DEP_LIST_ARRAY=( "$(join_array "${REPOQUERY_WHATPROVIDES_DELIM}" "${DEP_LIST_ARRAY[@]}" )" )
|
|
fi
|
|
|
|
if [ ${#DEP_LIST_ARRAY[@]} -gt 0 ]; then
|
|
|
|
for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do
|
|
echo "TMPDIR=${TMP_DIR}"\
|
|
"${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\
|
|
"${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\
|
|
"--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]}"
|
|
|
|
TMPDIR=${TMP_DIR} \
|
|
${REPOQUERY} --config=${YUM} --repoid=${REPOID} \
|
|
${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \
|
|
--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]} \
|
|
| sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES
|
|
|
|
\rm -rf $TMP_DIR/yum-$USER-*
|
|
done
|
|
fi
|
|
|
|
sort -u $UNSORTED_PACKAGES > $SORTED_PACKAGES
|
|
fi
|
|
|
|
# clean up
|
|
\rm -f $UNSORTED_PACKAGES $UNRESOLVED_PACKAGES
|
|
|
|
# We now have, in SORTED_PACKAGES, a list of all packages that we need to install
|
|
# to meet our dependancies
|
|
DEP_LIST=" "
|
|
while read DEP
|
|
do
|
|
DEP_LIST+="${DEP} "
|
|
done < $SORTED_PACKAGES
|
|
rm $SORTED_PACKAGES
|
|
|
|
# go through each repo and install packages
|
|
local TARGETS="${DEP_LIST}"
|
|
echo "Debug: Resolved list of deps to install: ${TARGETS}"
|
|
local UNRESOLVED
|
|
for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do
|
|
UNRESOLVED="$TARGETS"
|
|
|
|
if [[ ! -z "${TARGETS// }" ]]; then
|
|
REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::')
|
|
|
|
>&2 echo "TMPDIR=${TMP_DIR}"\
|
|
"${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\
|
|
"${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\
|
|
"--qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'"\
|
|
"${REPOQUERY_RESOLVE} ${TARGETS}"
|
|
|
|
TMPDIR=${TMP_DIR} \
|
|
${REPOQUERY} --config=${YUM} --repoid=${REPOID} \
|
|
${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \
|
|
--qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" \
|
|
${REPOQUERY_RESOLVE} ${TARGETS} \
|
|
| sort -r -V > $TMPFILE
|
|
|
|
\rm -rf $TMP_DIR/yum-$USER-*
|
|
|
|
while read STR
|
|
do
|
|
>&2 echo "STR=$STR"
|
|
if [ "x$STR" == "x" ]; then
|
|
continue
|
|
fi
|
|
|
|
PKG=`echo $STR | cut -d " " -f 1`
|
|
PKG_FILE=`echo $STR | cut -d " " -f 2`
|
|
PKG_REL_PATH=`echo $STR | cut -d " " -f 3`
|
|
PKG_PATH="${REPO_PATH}/${PKG_REL_PATH}"
|
|
|
|
>&2 echo "Installing PKG=$PKG PKG_FILE=$PKG_FILE PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH from repo $REPOID"
|
|
cp $PKG_PATH .
|
|
if [ $? -ne 0 ]; then
|
|
>&2 echo " Here's what I have to work with..."
|
|
>&2 echo " TMPDIR=${TMP_DIR}"\
|
|
"${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\
|
|
"${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\
|
|
"--qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\""\
|
|
"${REPOQUERY_RESOLVE} ${PKG}"
|
|
>&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH"
|
|
fi
|
|
|
|
echo $UNRESOLVED | grep $PKG >> /dev/null
|
|
if [ $? -eq 0 ]; then
|
|
echo "$PKG found in $REPOID as $PKG" >> $BUILT_REPORT
|
|
echo "$PKG_PATH" >> $BUILT_REPORT
|
|
UNRESOLVED=$(echo "$UNRESOLVED" | sed "s# $PKG # #g")
|
|
else
|
|
echo "$PKG satisfies unknown target in $REPOID" >> $BUILT_REPORT
|
|
echo " but it doesn't match targets, $UNRESOLVED" >> $BUILT_REPORT
|
|
echo " path $PKG_PATH" >> $BUILT_REPORT
|
|
FOUND_UNKNOWN=1
|
|
fi
|
|
done < $TMPFILE
|
|
\rm -rf $TMP_DIR/yum-$USER-*
|
|
TARGETS="$UNRESOLVED"
|
|
fi
|
|
done
|
|
>&2 echo "Debug: Packages still unresolved: $UNRESOLVED"
|
|
echo "Debug: Packages still unresolved: $UNRESOLVED" >> $WARNINGS_REPORT
|
|
echo "Debug: Packages still unresolved: $UNRESOLVED" >> $BUILT_REPORT
|
|
>&2 echo ""
|
|
}
|
|
|
|
function check_all_explicit_deps_installed {
|
|
|
|
PKGS_TO_CHECK=" "
|
|
while read PKG_TO_ADD
|
|
do
|
|
PKGS_TO_CHECK="$PKGS_TO_CHECK ${PKG_TO_ADD}"
|
|
done < $DEPLISTFILE
|
|
rpm -qp $MY_WORKSPACE/export/dist/isolinux/Packages/*.rpm --qf="%{name}\n" --nosignature > $TMPFILE
|
|
|
|
while read INSTALLED_PACKAGE
|
|
do
|
|
echo $PKGS_TO_CHECK | grep -q "${INSTALLED_PACKAGE}"
|
|
if [ $? -eq 0 ]; then
|
|
PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE} //"`
|
|
PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE} / /"`
|
|
PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE}\$//"`
|
|
PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE}\$//"`
|
|
fi
|
|
done < $TMPFILE
|
|
|
|
# Strip leading spaces. Don't want isomething like ' ' to trigger a failure
|
|
PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^[ ]*//"`
|
|
if [ -z "$PKGS_TO_CHECK" ]; then
|
|
>&2 echo "All explicitly specified packages resolved!"
|
|
else
|
|
>&2 echo "Could not resolve packages: $PKGS_TO_CHECK"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
ATTEMPTED=0
|
|
DISCOVERED=0
|
|
OUTPUT_DIR=$MY_WORKSPACE/export
|
|
TMP_DIR=$MY_WORKSPACE/tmp
|
|
YUM=$OUTPUT_DIR/yum.conf
|
|
DEPLISTFILE=$OUTPUT_DIR/deps.txt
|
|
DEPLISTFILE_NEW=$OUTPUT_DIR/deps_new.txt
|
|
DEPDETAILLISTFILE=$OUTPUT_DIR/deps_detail.txt
|
|
|
|
BUILT_REPORT=$OUTPUT_DIR/local.txt
|
|
WARNINGS_REPORT=$OUTPUT_DIR/warnings.txt
|
|
LAST_TEST=$OUTPUT_DIR/last_test.txt
|
|
TMPFILE=$OUTPUT_DIR/cgts_deps_tmp.txt
|
|
TMPFILE1=$OUTPUT_DIR/cgts_deps_tmp1.txt
|
|
TMPFILE2=$OUTPUT_DIR/cgts_deps_tmp2.txt
|
|
|
|
touch "$BUILT_REPORT"
|
|
touch "$WARNINGS_REPORT"
|
|
|
|
for i in "$@"
|
|
do
|
|
case $i in
|
|
-d=*|--deps=*)
|
|
DEPS="${i#*=}"
|
|
shift # past argument=value
|
|
;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p $TMP_DIR
|
|
|
|
rm -f "$DEPDETAILLISTFILE"
|
|
# FIRST PASS we are being given a list of REQUIRED dependencies
|
|
if [ "${DEPS}x" != "x" ]; then
|
|
cat $DEPS | grep -v "^#" | sed '/^\s*$/d' > $DEPLISTFILE
|
|
install_deps $DEPLISTFILE
|
|
if [ $? -ne 0 ]; then
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# check that we resolved them all
|
|
check_all_explicit_deps_installed
|
|
if [ $? -ne 0 ]; then
|
|
>&2 echo "Error -- could not install all explicitly listed packages"
|
|
exit 1
|
|
fi
|
|
|
|
ALL_RESOLVED=0
|
|
|
|
while [ $ALL_RESOLVED -eq 0 ]; do
|
|
cp $DEPLISTFILE $DEPLISTFILE.old
|
|
generate_dep_list
|
|
if [ ! -s $DEPLISTFILE ]; then
|
|
# no more dependencies!
|
|
ALL_RESOLVED=1
|
|
else
|
|
DIFFLINES=`diff $DEPLISTFILE.old $DEPLISTFILE | wc -l`
|
|
if [ $DIFFLINES -eq 0 ]; then
|
|
>&2 echo "Warning: Infinite loop detected in dependency resolution. See $DEPLISTFILE for details -- exiting"
|
|
>&2 echo "These RPMS had problems (likely version conflicts)"
|
|
>&2 cat $DEPLISTFILE
|
|
|
|
echo "Warning: Infinite loop detected in dependency resolution See $DEPLISTFILE for details -- exiting" >> $WARNINGS_REPORT
|
|
echo "These RPMS had problems (likely version conflicts)" >> $WARNINGS_REPORT
|
|
cat $DEPLISTFILE >> $WARNINGS_REPORT
|
|
|
|
date > $LAST_TEST
|
|
|
|
rm -f $DEPLISTFILE.old
|
|
exit 1 # nothing fixed
|
|
fi
|
|
install_deps $DEPLISTFILE
|
|
if [ $? -ne 0 ]; then
|
|
exit 1
|
|
fi
|
|
fi
|
|
done
|
|
|
|
exit 0
|