From fa455c0b9428a81820ed5ab4980457e0548e3657 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 15 Aug 2018 14:04:19 -0400 Subject: [PATCH] Tool changes to allow image.inc to be split across git repos. Currently compiling a new package and adding it to the iso still requires a multi-git update because image.inc is a single centralized file in the root git. It would be better to allow a single git update to add a package. Too allow this, image.inc must be split across the git repos and the build tools must be changed to read/merge those files to arrive at the final package list. Current scheme is to name the image.inc files using this schema. ${distro}_${build_target}_image_${build_type}.inc distro = centos, ... build_target = iso, guest ... build_type = std, rt ... Traditionally build_type=std is omitted from config files, so we instread use ${distro}_${build_target}_image.inc. Story: 2003447 Task: 24650 Change-Id: Ib39b8063e7759842ba15330c68503bfe2dea6e20 Signed-off-by: Scott Little --- build-tools/build-guest | 22 ++++++++++---- build-tools/build-iso | 27 ++++++++++++----- build-tools/build-rpms-parallel | 29 ++++++++++++------ build-tools/build-rpms-serial | 23 ++++++++------ build-tools/git-utils.sh | 14 +++++++++ build-tools/image-utils.sh | 54 +++++++++++++++++++++++++++++++++ 6 files changed, 136 insertions(+), 33 deletions(-) create mode 100755 build-tools/git-utils.sh create mode 100755 build-tools/image-utils.sh diff --git a/build-tools/build-guest b/build-tools/build-guest index ca6f14d5..2d507e85 100755 --- a/build-tools/build-guest +++ b/build-tools/build-guest @@ -1,10 +1,17 @@ #!/bin/env bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # # Build the tis-centos-image.img or tis-centos-image-rt.img file # -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BUILD_GUEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_GUEST_DIR}/image-utils.sh" # NOTE: TMP_DIR must end in '/' # NOTE: /tmp/ is now tmpfs like. Can't be trusted across multiple mock commands @@ -35,7 +42,7 @@ function init_vars { fi # Path to guest configuration - GUEST_BUILD_DIR=$DIR/build_guest + GUEST_BUILD_DIR="${BUILD_GUEST_DIR}/build_guest" GUEST_BUILD_CMD=$GUEST_BUILD_DIR/build-guest-image.py if [ $VERBOSE -eq 1 ]; then GUEST_BUILD_CMD="$GUEST_BUILD_CMD -x" @@ -108,7 +115,7 @@ function create_rootfs { GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest" GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE - MY_BUILD_ENVIRONMENT=$GUEST_ENV ${DIR}/modify-build-cfg $GUEST_CFG + MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG if [ $? -ne 0 ]; then printf " Error -- Could not update $GUEST_CFG\n"; exit 1 @@ -144,10 +151,10 @@ function create_rootfs { if [ $BUILD_MODE == 'std' ]; then INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list.txt) - TIS_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/image.inc) + TIS_RPM_LIST=$(image_inc_list guest std centos) elif [ $BUILD_MODE == 'rt' ]; then INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list-rt.txt) - TIS_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/image-rt.inc) + TIS_RPM_LIST=$(image_inc_list guest rt centos) EXTRA_REPOS="--enablerepo local-rt" else printf " Error -- unknown BUILD_MODE '$BUILD_MODE'\n"; @@ -156,6 +163,9 @@ function create_rootfs { $MOCK -r $GUEST_CFG --install ${INC_RPM_LIST} ${TIS_RPM_LIST} ${EXTRA_REPOS} if [ $? -ne 0 ]; then + printf "=====\n" + cat $GUEST_DIR/mock/result/root.log | sed -n '/Error:/,$p' | sed '/Child return code was:/q' + printf "=====\n" printf " Error -- Failed to install RPM packages\n"; exit 1 fi @@ -260,7 +270,7 @@ function clean_guest { GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE if [ ! -e $GUEST_CFG ]; then - MY_BUILD_ENVIRONMENT=$GUEST_ENV ${DIR}/modify-build-cfg $GUEST_CFG + MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG if [ $? -ne 0 ]; then printf " Error -- Could not update $GUEST_CFG\n"; exit 1 diff --git a/build-tools/build-iso b/build-tools/build-iso index 71a77e3f..e99f66f3 100755 --- a/build-tools/build-iso +++ b/build-tools/build-iso @@ -1,5 +1,11 @@ #!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # Build the export/bootimage.iso file # # This script uses environment variables to determine the source of @@ -8,6 +14,9 @@ # It starts by building a basic "vanilla CentOS" ISO, and then adds our # packages to it. +BUILD_ISO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_ISO_DIR}/image-utils.sh" + usage () { echo "" echo "Usage: " @@ -274,6 +283,10 @@ function init_output_dir { # This directory will contains files required for the PXE network installer \mkdir -p $OUTPUT_DIST_DIR/isolinux/pxeboot + echo " Preparing package lists" + image_inc_list iso std centos > "${PKGLIST_TI}" + image_inc_list iso dev centos > "${PKGLIST_DEV}" + echo " Copying base files" # Generate .discinfo file @@ -664,27 +677,27 @@ printf "*************************\n\n" # Init variables init_vars check_vars -PKGLIST_MINIMAL=$INTERNAL_REPO_ROOT/build-tools/build_iso/minimal_rpm_list.txt -PKGLIST_TI=$INTERNAL_REPO_ROOT/build-tools/build_iso/image.inc -PKGLIST_DEV=$INTERNAL_REPO_ROOT/build-tools/build_iso/image-dev.inc +PKGLIST_MINIMAL="${INTERNAL_REPO_ROOT}/build-tools/build_iso/minimal_rpm_list.txt" +PKGLIST_TI="${OUTPUT_DIR}/image.inc" +PKGLIST_DEV="${OUTPUT_DIR}/image-dev.inc" # Create skeleton build dir init_output_dir # Create the vanilla DVD echo "Copying vanilla CentOS RPMs" -install_pkg_list $PKGLIST_MINIMAL +install_pkg_list "${PKGLIST_MINIMAL}" # Find all CGCS packages # SAL exit 0 echo "Installing Titanium Cloud packages" -install_pkg_list $PKGLIST_TI +install_pkg_list "${PKGLIST_TI}" if [ $? -eq 2 ]; then exit 1 fi if [ "x${RELEASE_BUILD}" == "x" ]; then echo "Installing Titanium Cloud developer packages" - install_pkg_list ${PKGLIST_DEV} + install_pkg_list "${PKGLIST_DEV}" if [ $? -eq 2 ]; then exit 1 fi @@ -711,7 +724,7 @@ final_touches # Sign the ISO sign_iso -make_report $PKGLIST_MINIMAL $PKGLIST_TI +make_report "${PKGLIST_MINIMAL}" "${PKGLIST_TI}" # Check sanity FILESIZE=$(wc -c <"$OUTPUT_FILE") diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index f9aadc3e..275d0054 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -1,10 +1,20 @@ #!/bin/bash -# set -x + +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + export ME=$(basename "$0") CMDLINE="$ME $@" + +# Build for distribution. Currently 'centos' is only supported value. +export DISTRO="centos" + # Maximum number of parallel build environments ABSOLUTE_MAX_WORKERS=4 @@ -52,16 +62,16 @@ RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm" SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm" UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns" -IMAGE_INC_FILE="$MY_REPO/build-tools/build_iso/image.inc" SIGN_SECURE_BOOT="sign-secure-boot" SIGN_SECURE_BOOT_LOG="sign-secure-boot.log" export MOCK=/usr/bin/mock -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $DIR/spec-utils -source $DIR/srpm-utils +BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh" +source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils" +source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils" HOME=$(pwd) @@ -1450,11 +1460,10 @@ export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root -LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release" +IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc" +image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}" -if [ "$BUILD_TYPE" != "std" ]; then - PKG_DIRS_FILE=centos_pkg_dirs_$BUILD_TYPE -fi +LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release" TARGETS=$@ @@ -1548,7 +1557,7 @@ export BUILD_CFG="$MY_BUILD_CFG" # Place build-time environement variables in mock environment echo "FORMAL_BUILD=$FORMAL_BUILD" echo "modify-build-cfg $BUILD_CFG" -${DIR}/modify-build-cfg $BUILD_CFG +${BUILD_RPMS_PARALLEL_DIR}/modify-build-cfg $BUILD_CFG if [ $? -ne 0 ]; then echo "Could not modifiy $BUILD_CFG"; exit 1 diff --git a/build-tools/build-rpms-serial b/build-tools/build-rpms-serial index d9a7afe4..eef0424f 100755 --- a/build-tools/build-rpms-serial +++ b/build-tools/build-rpms-serial @@ -1,5 +1,10 @@ #!/bin/bash -# set -x + +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# export ME=$(basename "$0") CMDLINE="$ME $@" @@ -21,16 +26,16 @@ RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm" SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm" UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns" -IMAGE_INC_FILE="$MY_REPO/build-tools/build_iso/image.inc" SIGN_SECURE_BOOT="sign-secure-boot" SIGN_SECURE_BOOT_LOG="sign-secure-boot.log" export MOCK=/usr/bin/mock -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $DIR/spec-utils -source $DIR/srpm-utils +BUILD_RPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_RPMS_SERIAL_DIR}/image-utils.sh" +source "${BUILD_RPMS_SERIAL_DIR}/spec-utils" +source "${BUILD_RPMS_SERIAL_DIR}/srpm-utils" HOME=$(pwd) @@ -548,10 +553,8 @@ export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root - -if [ "$BUILD_TYPE" != "std" ]; then - PKG_DIRS_FILE=centos_pkg_dirs_$BUILD_TYPE -fi +IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc" +image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}" TARGETS=$@ @@ -645,7 +648,7 @@ export BUILD_CFG="$MY_BUILD_CFG" # Place build-time environement variables in mock environment echo "FORMAL_BUILD=$FORMAL_BUILD" echo "modify-build-cfg $BUILD_CFG" -${DIR}/modify-build-cfg $BUILD_CFG +${BUILD_RPMS_SERIAL_DIR}/modify-build-cfg $BUILD_CFG if [ $? -ne 0 ]; then echo "Could not modifiy $BUILD_CFG"; exit 1 diff --git a/build-tools/git-utils.sh b/build-tools/git-utils.sh new file mode 100755 index 00000000..b4c99c2c --- /dev/null +++ b/build-tools/git-utils.sh @@ -0,0 +1,14 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# A place for any functions relating to git, or the git hierarchy created +# by repo manifests. +# + +# GIT_LIST: A list of root directories for all the gits under $MY_REPO +export GIT_LIST=$(find $MY_REPO -type d -name '.git' -exec dirname {} \;) + diff --git a/build-tools/image-utils.sh b/build-tools/image-utils.sh new file mode 100755 index 00000000..f0a06908 --- /dev/null +++ b/build-tools/image-utils.sh @@ -0,0 +1,54 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# A place for any functions related to image.inc files +# + +IMAGE_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "${IMAGE_UTILS_DIR}/git-utils.sh" + +# +# image_inc_list +# +# Parameters: +# build_target: One of 'iso', 'guest' ... +# build_type: One of 'std', 'rt', 'dev' ... +# distro: One of 'centos', ... +# +# Returns: A list of unique package that must be included for +# the desired distro's build target and build type. +# This is the union of the global and per git +# image.inc files. + +image_inc_list () { + local build_target=$1 + local build_type=$2 + local distro=$3 + + local root_file="" + local build_type_extension="" + local search_target="" + + if [ "${build_type}" != "std" ]; then + build_type_extension="_${build_type}" + build_type_extension_bt="-${build_type}" + fi + + root_dir="${MY_REPO}/build-tools/build_${build_target}" + root_file="${root_dir}/image${build_type_extension_bt}.inc" + search_target=${distro}_${build_target}_image${build_type_extension}.inc + + ( + if [ -f ${root_file} ]; then + grep '^[^#]' ${root_file} + fi + for d in $GIT_LIST; do + find $d -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} + + done + ) | sort --unique +}