Prepare the build tools for code restucturing

Build tools need to be made ready for the code
restructuring:

1) new repos
2) relocation of release-info.inc and bsp files
3) Removal of the stx- prefix from sub-repos.

Changes:
1) Create new functions to find the release-info.inc and
bsp file under both pre and post restructure paths.

2) Update .gitignore to ignore all the new repos that
are to be created.

3) Remove an argument sanity check from build-helm-charts.sh.
It is making invalid assumptions about where applications
can be found in the directory structure.

4) build-iso will need to look to additional repos to find
packages from lower layers.

Change-Id: If62444390d0a5a363974c6ff8cdaceca35978bda
Story: 2006166
Task: 35687
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2019-08-30 13:01:04 -04:00
parent aee9d5689f
commit 83d4446d74
19 changed files with 198 additions and 75 deletions

View File

@ -112,8 +112,14 @@ function check_vars {
exit 1
fi
RELEASE_INFO=$STX_DIR/stx-integ/utilities/build-info/release-info.inc
export PLATFORM_RELEASE=$(source $RELEASE_INFO && echo $PLATFORM_RELEASE)
RELEASE_INFO="$(get_release_info)"
if [ $? -ne 0 ]; then
echo "ERROR: failed to find a release info file."
exit 1
fi
export PLATFORM_RELEASE=$(source "$RELEASE_INFO" && echo $PLATFORM_RELEASE)
}

View File

@ -373,21 +373,24 @@ if [ ${VALID_OS} -ne 0 ]; then
exit 1
fi
# Validate application
APP_REPO=${MY_REPO}/stx/stx-config/kubernetes/applications/
if [ ! -d ${APP_REPO} ];then
echo "Unable to find the applications directory: ${APP_REPO}" >&2
exit 1
fi
AVALIABLE_APPS=($(ls ${APP_REPO}))
if [ ${#AVALIABLE_APPS[@]} -eq 0 ]; then
echo "No application found" >&2
exit 1
fi
if ! is_in ${APP_NAME} ${AVALIABLE_APPS[@]}; then
echo "Invalid application: ${APP_NAME}" >&2
exit 1
fi
# Commenting out this code that attempts to validate the APP_NAME.
# It makes too many assumptions about the location and naming of apps.
#
# # Validate application
# APP_REPO=${MY_REPO}/stx/stx-config/kubernetes/applications/
# if [ ! -d ${APP_REPO} ];then
# echo "Unable to find the applications directory: ${APP_REPO}" >&2
# exit 1
# fi
# AVAILABLE_APPS=($(ls ${APP_REPO}))
# if [ ${#AVAILABLE_APPS[@]} -eq 0 ]; then
# echo "No application found" >&2
# exit 1
# fi
# if ! is_in ${APP_NAME} ${AVAILABLE_APPS[@]}; then
# echo "Invalid application: ${APP_NAME}" >&2
# exit 1
# fi
# Cleanup the previous chart build workspace
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-helm/stx

View File

@ -16,6 +16,7 @@
BUILD_ISO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_ISO_DIR}/image-utils.sh"
source "${BUILD_ISO_DIR}/git-utils.sh"
usage () {
echo ""
@ -39,6 +40,7 @@ usage () {
MY_YUM_CONF=""
STD_REPO_ID="local-std"
RT_REPO_ID="local-rt"
DISTRO_LAYER_REPO_ID="StxCentos7Distro"
NPROCS=$(nproc)
@ -263,12 +265,16 @@ function check_vars {
exit 1
fi
RELEASE_INFO=$STX_DIR/stx-integ/utilities/build-info/release-info.inc
export PLATFORM_RELEASE=$(source $RELEASE_INFO && echo $PLATFORM_RELEASE)
RELEASE_INFO="$(get_release_info)"
if [ $? -ne 0 ]; then
echo "ERROR: failed to find a release info file."
exit 1
fi
export PLATFORM_RELEASE=$(source "$RELEASE_INFO" && echo $PLATFORM_RELEASE)
# Where BSP files live
export BSP_FILES_PATH="$STX_DIR/stx-metal/bsp-files"
export BSP_FILES_PATH="$(get_bsp_dir)"
echo " Done"
echo ""
}
@ -467,13 +473,20 @@ function final_touches {
}
function extract_pkg_from_local_repo {
local yum_conf=$1
local repoid=$2
local pkgname=$3
local pkgname=$1
local yum_conf=$2
shift 2
local pkgfile=$(repoquery --config=${yum_conf} --repoid=${repoid} --location -q ${pkgname})
local repoid=""
local repoid_arg=""
for repoid in $@; do
repoid_arg+=" --repoid=${repoid}"
done
local pkgfile=$(repoquery --config=${yum_conf} ${repoid_arg} --location -q ${pkgname})
if [ -z "${pkgfile}" ]; then
echo "Could not find package $pkgname in $repodir"
echo "Could not find package $pkgname in $@"
exit 1
fi
@ -498,9 +511,9 @@ function extract_installer_files {
\cd kickstart.work
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} platform-kickstarts
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} platform-kickstarts-pxeboot
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} platform-kickstarts-extracfgs
extract_pkg_from_local_repo platform-kickstarts ${MY_YUM_CONF} ${STD_REPO_ID}
extract_pkg_from_local_repo platform-kickstarts-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID}
extract_pkg_from_local_repo platform-kickstarts-extracfgs ${MY_YUM_CONF} ${STD_REPO_ID}
\cp --preserve=all www/pages/feed/rel-*/*.cfg pxeboot/*.cfg ../kickstarts/ &&
\cp --preserve=all extra_cfgs/*.cfg ../extra_cfgs/
@ -556,9 +569,9 @@ EOM
\mkdir $WORKDIR
\cd $WORKDIR
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} pxe-network-installer
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} grub2-efi-x64-pxeboot
extract_pkg_from_local_repo ${MY_YUM_CONF} ${STD_REPO_ID} grub2-efi-x64-modules
extract_pkg_from_local_repo pxe-network-installer ${MY_YUM_CONF} ${STD_REPO_ID}
extract_pkg_from_local_repo grub2-efi-x64-pxeboot ${MY_YUM_CONF} ${STD_REPO_ID} ${DISTRO_LAYER_REPO_ID}
extract_pkg_from_local_repo grub2-efi-x64-modules ${MY_YUM_CONF} ${STD_REPO_ID} ${DISTRO_LAYER_REPO_ID}
\mkdir -p $OUTPUT_DIST_DIR/isolinux/pxeboot/EFI/centos/x86_64-efi
@ -690,6 +703,7 @@ check_vars
PKGLIST_MINIMAL="${INTERNAL_REPO_ROOT}/build-tools/build_iso/minimal_rpm_list.txt"
PKGLIST_STX="${OUTPUT_DIR}/image.inc"
PKGLIST_DEV="${OUTPUT_DIR}/image-dev.inc"
DISTRO="centos"
# Create skeleton build dir
init_output_dir
@ -697,22 +711,49 @@ init_output_dir
# Create the vanilla DVD
echo "Copying vanilla CentOS RPMs"
install_pkg_list "${PKGLIST_MINIMAL}"
if [ $? -eq 2 ]; then
echo "Error: Failed to install packages from ${PKGLIST_MINIMAL}"
exit 1
fi
# Find all CGCS packages
# SAL exit 0
# Find all StarlingX packages built locally
echo "Installing StarlingX packages"
install_pkg_list "${PKGLIST_STX}"
if [ $? -eq 2 ]; then
exit 1
echo "Error: Failed to install packages from ${PKGLIST_STX}"
exit 1
fi
if [ "x${RELEASE_BUILD}" == "x" ]; then
echo "Installing StarlingX developer packages"
install_pkg_list "${PKGLIST_DEV}"
if [ $? -eq 2 ]; then
echo "Error: Failed to install packages from ${PKGLIST_DEV}"
exit 1
fi
fi
required_layer_cfg_name="${DISTRO}_required_build_layer.cfg"
layer_cfg_name="${DISTRO}_build_layer.cfg"
layer_cfgs=$(find $(for x in $GIT_LIST; do echo $x/; done) -maxdepth 1 -name ${layer_cfg_name})
for line in $(grep -v '^#' ${BUILD_ISO_DIR}/build_iso/${required_layer_cfg_name}); do
layer=${line%%,*}
url=${line##*,}
cat $layer_cfgs | grep -q "^${layer}$"
if [ $? -ne 0 ]; then
echo "need ${layer} from ${url}"
# layer_inc=${MY_REPO}/cgcs-centos-repo/$(basename ${url})
layer_inc=${MY_WORKSPACE}/export/$(basename ${url})
curl -s ${url} > ${layer_inc}
install_pkg_list ${layer_inc}
if [ $? -eq 2 ]; then
echo "Error: Failed to install packages from ${layer_inc}"
exit 1
fi
fi
done
\cd $OUTPUT_DIST_DIR
chmod -R 644 isolinux/Packages/*

View File

@ -80,7 +80,7 @@ if [ ! -d $DEP_CACHE ]; then
else
DEP_TMP=$(mktemp)
make_cache_current_rpms $DEP_TMP
if diff $DEP_DELTAS $DEP_TMP; then
if diff $DEP_DELTAS $DEP_TMP > /dev/null; then
echo "No changes for stx projects"
rm $DEP_TMP
else

View File

@ -1515,17 +1515,17 @@ else
RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild
fi
RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc
if [ -f $RELEASE_INFO_FILE ]; then
source $RELEASE_INFO_FILE
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "ERROR: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
exit 1
echo "Warning: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
fi
if [ "x$PLATFORM_RELEASE" == "x" ]; then
echo "ERROR: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
exit 1
echo "Warning: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
PLATFORM_RELEASE="00.00"
fi
export RPM_BUILD_BASE="$RPM_BUILD_ROOT"

View File

@ -1283,9 +1283,10 @@ else
RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild
fi
RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc
if [ -f $RELEASE_INFO_FILE ]; then
source $RELEASE_INFO_FILE
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "ERROR: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
exit 1

View File

@ -15,7 +15,7 @@
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. stx/stx-integ/centos_pkg_dirs).
# any git tree (e.g. stx/integ/centos_pkg_dirs).
#
# The build of an individul package is driven by it's build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
@ -30,6 +30,7 @@ source $BUILD_SRPMS_PARALLEL_DIR/spec-utils
source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils
source $BUILD_SRPMS_PARALLEL_DIR/classify
source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh
source $BUILD_SRPMS_PARALLEL_DIR/image-utils.sh
INITIAL_DIR=$(pwd)
@ -61,7 +62,7 @@ fi
if [ $MAX_WORKERS -gt $ABSOLUTE_MAX_WORKERS ]; then
MAX_WORKERS=$ABSOLUTE_MAX_WORKERS
fi
fi
echo "MAX_WORKERS=$MAX_WORKERS"
@ -295,17 +296,17 @@ if [ ! -d $BUILD_BASE ]; then
exit 1
fi
RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc
if [ -f $RELEASE_INFO_FILE ]; then
source $RELEASE_INFO_FILE
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "ERROR: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
exit 1
echo "Warning: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
fi
if [ "x$PLATFORM_RELEASE" == "x" ]; then
echo "ERROR: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
exit 1
echo "Warning: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE"
PLATFORM_RELEASE=00.00
fi
export PLATFORM_RELEASE
@ -416,7 +417,7 @@ build_dir () {
echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'"
ORIG_SRPM_PATH=""
return 3
fi
fi
done
fi

View File

@ -15,7 +15,7 @@
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. stx/stx-integ/centos_pkg_dirs).
# any git tree (e.g. stx/integ/centos_pkg_dirs).
#
# The build of an individul package is driven by it's build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
@ -31,6 +31,7 @@ source $BUILD_SRPMS_SERIAL_DIR/spec-utils
source $BUILD_SRPMS_SERIAL_DIR/srpm-utils
source $BUILD_SRPMS_SERIAL_DIR/classify
source $BUILD_SRPMS_SERIAL_DIR/build-srpms-common.sh
source $BUILD_SRPMS_SERIAL_DIR/image-utils.sh
INITIAL_DIR=$(pwd)
@ -281,9 +282,10 @@ if [ ! -d $BUILD_BASE ]; then
exit 1
fi
RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc
if [ -f $RELEASE_INFO_FILE ]; then
source $RELEASE_INFO_FILE
RELEASE_INFO_FILE="$(get_release_info)"
if [ -f "$RELEASE_INFO_FILE" ]; then
source "$RELEASE_INFO_FILE"
else
echo "ERROR: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE"
exit 1
@ -401,7 +403,7 @@ build_dir () {
echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'"
ORIG_SRPM_PATH=""
return 3
fi
fi
done
fi

View File

@ -20,7 +20,7 @@ git_ctx_root_dir () {
git_list () {
local DIR=${1}
find "${DIR}" -type d -name '.git' -exec dirname {} \; | sort -V
find -L "${DIR}" -maxdepth 5 -type d -name '.git' -exec dirname {} \; | sort -V
}

View File

@ -12,6 +12,38 @@ IMAGE_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${IMAGE_UTILS_DIR}/git-utils.sh"
get_release_info () {
local dir=""
local path=""
for dir in utilities integ stx-utilities stx-integ; do
path="$MY_REPO/stx/$dir/utilities/build-info/release-info.inc"
if [ -f "$path" ]; then
echo "$path"
return 0
fi
done
echo "/invalid-path-to-release-info.inc"
return 1
}
get_bsp_dir () {
local dir=""
local path=""
for dir in stx-metal metal; do
path="$MY_REPO/stx/$dir/bsp-files"
if [ -d "$path" ]; then
echo "$path"
return 0
fi
done
echo "/invalid-path-to-bsp-files"
return 1
}
#
# image_inc_list <build_target> <build_type> <distro>
#
@ -48,7 +80,7 @@ image_inc_list () {
grep '^[^#]' ${root_file}
fi
for d in $GIT_LIST; do
find $d -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} +
find $d/ -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} +
done
) | sort --unique
}

View File

@ -3,6 +3,8 @@
# Utility for adding patches to an unpatched ISO
#
source "${BUILD_ISO_DIR}/image-utils.sh"
if [ -z "${MY_REPO}" ]; then
echo "Required environment variable MY_REPO is not set"
exit 1
@ -16,7 +18,13 @@ if [ ! -x ${SETUP_PATCH_REPO} ]; then
fi
REPO_UPGRADES_DIR=${STX_DIR}/common-bsp/files/upgrades
RELEASE_INFO=${STX_DIR}/stx-integ/utilities/build-info/release-info.inc
RELEASE_INFO="$(get_release_info)"
if [ $? -ne 0 ]; then
echo "ERROR: failed to find a release info file."
exit 1
fi
PLATFORM_RELEASE=$(source $RELEASE_INFO && echo $PLATFORM_RELEASE)
function usage() {

View File

@ -18,7 +18,7 @@ usage () {
echo " PKG=lighttpd"
echo " OLD_SRC_RPM=lighttpd-1.4.41-1.el7.src.rpm"
echo " NEW_SRC_RPM=lighttpd-1.4.41-2.el7.src.rpm"
echo " SRPM_PATH=$MY_REPO/stx/stx-integ/extended/lighttpd/centos/srpm_path"
echo " SRPM_PATH=$MY_REPO/stx/integ/extended/lighttpd/centos/srpm_path"
echo " echo \"\$PKG#\$SRPM_PATH##\$OLD_SRC_RPM#\$NEW_SRC_RPM\" > UPVERSION_DATA"
echo ""
}

View File

@ -18,7 +18,7 @@ usage () {
echo " PKG=lighttpd"
echo " OLD_SRC_RPM=lighttpd-1.4.41-1.el7.src.rpm"
echo " NEW_SRC_RPM=lighttpd-1.4.41-2.el7.src.rpm"
echo " SRPM_PATH=$MY_REPO/stx/stx-integ/extended/lighttpd/centos/srpm_path"
echo " SRPM_PATH=$MY_REPO/stx/integ/extended/lighttpd/centos/srpm_path"
echo " echo \"\$PKG#\$SRPM_PATH##\$OLD_SRC_RPM#\$NEW_SRC_RPM\" > UPVERSION_DATA"
echo ""
}

View File

@ -18,7 +18,7 @@ usage () {
echo " PKG=lighttpd"
echo " OLD_SRC_RPM=lighttpd-1.4.41-1.el7.src.rpm"
echo " NEW_SRC_RPM=lighttpd-1.4.41-2.el7.src.rpm"
echo " SRPM_PATH=$MY_REPO/stx/stx-integ/extended/lighttpd/centos/srpm_path"
echo " SRPM_PATH=$MY_REPO/stx/integ/extended/lighttpd/centos/srpm_path"
echo " echo \"\$PKG#\$SRPM_PATH##\$OLD_SRC_RPM#\$NEW_SRC_RPM\" > UPVERSION_DATA"
echo ""
}

View File

@ -18,7 +18,7 @@ usage () {
echo " PKG=lighttpd"
echo " OLD_SRC_RPM=lighttpd-1.4.41-1.el7.src.rpm"
echo " NEW_SRC_RPM=lighttpd-1.4.41-2.el7.src.rpm"
echo " SRPM_PATH=$MY_REPO/stx/stx-integ/extended/lighttpd/centos/srpm_path"
echo " SRPM_PATH=$MY_REPO/stx/integ/extended/lighttpd/centos/srpm_path"
echo " echo \"\$PKG#\$SRPM_PATH##\$OLD_SRC_RPM#\$NEW_SRC_RPM\" > UPVERSION_DATA"
echo ""
}

View File

@ -103,7 +103,9 @@ function sign_packages {
_MOCK_CFG=$MY_BUILD_DIR/${MY_BUILD_ENVIRONMENT}-sign.cfg
# recreate configuration file
rm $_MOCK_CFG
if [ -f $_MOCK_CFG ]; then
rm $_MOCK_CFG
fi
export BUILD_TYPE=std
export MY_BUILD_DIR_TOP=$MY_BUILD_DIR
modify-build-cfg $_MOCK_CFG

View File

@ -3081,7 +3081,7 @@ srpm_match_target_list () {
fi
fi
done
fi
fi
SERVICE=$(srpm_find_tag Service "$SRPM_FILE")
if [ $? -eq 0 ]; then

View File

@ -138,18 +138,18 @@ ${SUDOPREFIX} cp "$BSP_FILES_PATH/grub.cfg" "$EFI_MOUNT/EFI/BOOT/grub.cfg"
# To do this, we extract the RPMS, grab the two executables we need, and replace
# the ones in the current filesystem
TMPDIR=`mktemp -d`
SHIMPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS/shim-x64-[0-9]*.x86_64.rpm`
SHIMPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS $MY_REPO/cgcs-centos-repo/Binary -name 'shim-x64-[0-9]*.x86_64.rpm'`
if [ -z "$SHIMPKG" ]; then
SHIMPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS/shim-[0-9]*.x86_64.rpm`
SHIMPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS $MY_REPO/cgcs-centos-repo/Binary -name 'shim-[0-9]*.x86_64.rpm'`
fi
if [ -z "$SHIMPKG" ]; then
printf " Error -- could not locate shim binary package"
exit 1
fi
GRUBPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS/grub2-efi-x64-[0-9]*.x86_64.rpm`
GRUBPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS $MY_REPO/cgcs-centos-repo/Binary -name 'grub2-efi-x64-[0-9]*.x86_64.rpm'`
if [ -z "$GRUBPKG" ]; then
GRUBPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS/grub2-efi-[0-9]*.x86_64.rpm`
GRUBPKG=`find $MY_WORKSPACE/std/rpmbuild/RPMS $MY_REPO/cgcs-centos-repo/Binary -name 'grub2-efi-[0-9]*.x86_64.rpm'`
fi
if [ -z "$GRUBPKG" ]; then
printf " Error -- could not locate grub binary package"

29
stx/.gitignore vendored
View File

@ -10,12 +10,39 @@
installer-prebuilt
/stx-ansible-playbooks
/stx-clients
/stx-compile
/stx-config
/stx-config-files
/stx-fault
/stx-gui
/stx-ha
/stx-integ
/stx-kubernetes
/stx-metal
/stx-monitoring
/stx-nfv
/stx-puppet
/stx-update
/stx-upstream
/stx-gui
/stx-utilities
/ansible-playbooks
/clients
/compile
/config
/config-files
/fault
/gui
/ha
/helm-charts
/integ
/kubernetes
/metal
/monitor-armada-app
/monitoring
/nfv
/openstack-armada-app
/platform-armada-app
/puppet
/update
/upstream
/utilities