From b0bcb59c15c48456c41b5bffe917b6a4b7854f2b Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 29 Aug 2018 16:04:16 -0400 Subject: [PATCH] Build: Improved logging, error handling, and other small fixes Add function and line number to error logs. Log git commands that are issured when using --edit. Replace global DIR variable with a more unique variable name. DIR is used in many scripts that get sourced, possibly overwriting the value in the calling script. Make sure scripts won't fail if a redundant --parallel or --serial argurement is seen. Source files once at top of script, rather than in multiple places. Source build_srpm.data once per package, and handle case where build_srpm.data file is missing. Fix handling of ' --installer '. Add handling of 'BUILD_IS_BIG' and 'UILD_IS_SLOW' to serial build. Fix handling of packages that are built twice, once with a -rt extension, during srpm audit. Fix some exit code handling. Prioritize Name over Service when searching for a package. Delete some commented out code paths. Change-Id: Ib5153cecf7b586d68aa382d382bc5a1a03a6b326 Story: 2002835 Task: 24519 Signed-off-by: Scott Little --- build-tools/build-pkgs | 4 + build-tools/build-pkgs-parallel | 61 +++++++--- build-tools/build-pkgs-serial | 95 ++++++--------- build-tools/build-pkgs4 | 1 - build-tools/build-rpms4 | 1 - build-tools/build-srpms-parallel | 161 +++++++++++++------------ build-tools/build-srpms-serial | 200 ++++++++++++++++++++----------- build-tools/build-srpms4 | 1 - build-tools/classify | 12 -- build-tools/mockchain-parallel | 8 +- build-tools/srpm-utils | 24 ++-- 11 files changed, 320 insertions(+), 248 deletions(-) delete mode 120000 build-tools/build-pkgs4 delete mode 120000 build-tools/build-rpms4 delete mode 120000 build-tools/build-srpms4 diff --git a/build-tools/build-pkgs b/build-tools/build-pkgs index d1372d84..9def3b01 100755 --- a/build-tools/build-pkgs +++ b/build-tools/build-pkgs @@ -20,6 +20,7 @@ usage () { } SERIAL_FLAG=0 +RC=0 for arg in "$@"; do case "$1" in @@ -44,8 +45,11 @@ fi if [ $SERIAL_FLAG -eq 1 ]; then echo "build-pkgs-serial $@" build-pkgs-serial "$@" + RC=$? else echo "build-pkgs-parallel $@" build-pkgs-parallel "$@" + RC=$? fi +exit $RC diff --git a/build-tools/build-pkgs-parallel b/build-tools/build-pkgs-parallel index af9e5f94..f64ac031 100755 --- a/build-tools/build-pkgs-parallel +++ b/build-tools/build-pkgs-parallel @@ -2,7 +2,10 @@ # This program is a wrapper around build-srpms-parallel and build-rpms-parallel -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BUILD_PKGS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "${BUILD_PKGS_PARALLEL_DIR}/spec-utils" + usage () { echo "" @@ -40,7 +43,7 @@ RT_BUILD=1 INSTALLER_BUILD=0 # read the options -TEMP=$(getopt -o h --long rt,std,installer,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs-parallel' -- "$@") +TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs-parallel' -- "$@") if [ $? -ne 0 ]; then usage exit 0 @@ -66,6 +69,7 @@ while true ; do --rt) STD_BUILD=0 ; shift ;; --std) RT_BUILD=0 ; shift ;; --installer) INSTALLER_BUILD=1 ; STD_BUILD=0 ; RT_BUILD=0 ; shift ;; + --parallel) shift ;; --) shift ; break ;; *) usage; exit 1 ;; esac @@ -203,16 +207,18 @@ echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" if [ "x$TARGETS" != "x " ]; then - source $MY_REPO/build-tools/spec-utils TARGETS_STD="$(find_targets centos_pkg_dirs)" BUILD_TYPE_SAVE="$BUILD_TYPE" BUILD_TYPE="rt" TARGETS_RT="$(find_targets centos_pkg_dirs_rt)" + BUILD_TYPE="installer" + TARGETS_INSTALLER="$(find_targets centos_pkg_dirs_installer)" BUILD_TYPE="$BUILD_TYPE_SAVE" echo "TARGETS_STD=$TARGETS_STD" echo "TARGETS_RT=$TARGETS_RT" + echo "TARGETS_INSTALLER=$TARGETS_INSTALLER" for x in $TARGETS; do if [[ $TARGETS_STD == *" $x "* ]] @@ -223,7 +229,13 @@ if [ "x$TARGETS" != "x " ]; then then echo "found $x" >> /dev/null; else - TARGETS_MISC+="$x " + if [[ $TARGETS_INSTALLER == *" $x "* ]] + then + echo "found $x" >> /dev/null; + INSTALLER_BUILD=1 + else + TARGETS_MISC+="$x " + fi fi fi done @@ -243,25 +255,43 @@ if [ $CLEAN_FLAG -eq 1 ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "$DIR/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" $DIR/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" + ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 + fi fi if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then - echo "$DIR/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" - $DIR/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" + ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi + if [ $RT_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then if [ $EDIT_FLAG -ne 1 ]; then - echo "$DIR/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" - $DIR/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" + ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then - echo "$DIR/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" - $DIR/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" + ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi + + if [ $INSTALLER_BUILD -eq 1 ]; then + if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ $EDIT_FLAG -ne 1 ]; then + echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER" + ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1 + fi + fi + if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then + echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER" + ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1 + fi + fi + exit $? fi @@ -293,8 +323,9 @@ function launch_build() echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile # No clean flag, call build-srpms-parallel followed by build-rpms-parallel - echo "$DIR/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile - $DIR/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile + echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile + ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile + rc=${PIPESTATUS[0]} if [ $rc -eq 0 ]; then echo -e "\n######## $(date): build-srpm-parallel --$build_type was successful" | tee --append $logfile @@ -306,8 +337,8 @@ function launch_build() if [ $EDIT_FLAG -ne 1 ]; then echo -e "\n######## $(date): Launching build-rpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile - echo "$DIR/build-rpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets" | tee --append $logfile - $DIR/build-rpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets 2>&1 | tee --append $logfile + echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets" | tee --append $logfile + ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets 2>&1 | tee --append $logfile rc=${PIPESTATUS[0]} if [ $rc -eq 0 ]; then echo -e "\n######## $(date): build-rpm-parallel --$build_type was successful" | tee --append $logfile @@ -344,6 +375,7 @@ if [ $STD_BUILD -eq 1 ]; then else echo "Skipping 'std' build" fi + if [ $RT_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then launch_build rt @@ -353,6 +385,7 @@ if [ $RT_BUILD -eq 1 ]; then else echo "Skipping 'rt' build" fi + if [ $INSTALLER_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then launch_build installer diff --git a/build-tools/build-pkgs-serial b/build-tools/build-pkgs-serial index 6935d106..4d0ed24e 100755 --- a/build-tools/build-pkgs-serial +++ b/build-tools/build-pkgs-serial @@ -2,7 +2,10 @@ # This program is a wrapper around build-srpms and build-rpms -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BUILD_PKGS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "${BUILD_PKGS_SERIAL_DIR}/spec-utils" + usage () { echo "" @@ -12,22 +15,22 @@ usage () { echo " --no-autoclean) are not recommended for the first build after a clone/pull," echo " nor the final build prior to creating an iso or patch, but can be used" echo " for intermediate builds. i.e. while debugging compilation failures." - echo " build-pkgs [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" + echo " build-pkgs-serial [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" echo "" echo " Delete source rpms, and the directories associated with it's creation:" echo " Note: does not clean an edit environment" - echo " build-pkgs --clean [ list of package names ]" + echo " build-pkgs-serial --clean [ list of package names ]" echo "" echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents," echo " one for source code and one for metadata such as the spec file." echo " If --no-meta-patch is specified, then WRS patches are omitted." - echo " build-pkgs --edit [--no-meta-patch] [ list of package names ]" + echo " build-pkgs-serial --edit [--no-meta-patch] [ list of package names ]" echo "" echo " Delete an edit environment" - echo " build-pkgs --edit --clean [ list of package names ]" + echo " build-pkgs-serial --edit --clean [ list of package names ]" echo "" echo " This help page" - echo " build-pkgs [--help]" + echo " build-pkgs-serial [--help]" echo "" } @@ -43,7 +46,7 @@ INSTALLER_BUILD=0 TEMP=`getopt -o h --long serial,rt,std,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs' -- "$@"` if [ $? -ne 0 ]; then usage - exit 0 + exit 1 fi eval set -- "$TEMP" @@ -160,7 +163,6 @@ echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" if [ "x$TARGETS" != "x " ]; then - source $MY_REPO/build-tools/spec-utils TARGETS_STD="$(find_targets centos_pkg_dirs)" BUILD_TYPE_SAVE="$BUILD_TYPE" @@ -208,37 +210,40 @@ if [ $CLEAN_FLAG -eq 1 ]; then if [ $STD_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then if [ $EDIT_FLAG -ne 1 ]; then - echo "$DIR/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" - $DIR/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" + ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 + fi fi if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then - echo "$DIR/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" - $DIR/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" + ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi + if [ $RT_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x" ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then if [ $EDIT_FLAG -ne 1 ]; then - echo "$DIR/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" - $DIR/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" + ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then - echo "$DIR/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" - $DIR/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" + ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi + if [ $INSTALLER_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then if [ $EDIT_FLAG -ne 1 ]; then - echo "$DIR/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER" - $DIR/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER" + ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1 fi fi if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then - echo "$DIR/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER" - $DIR/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1 + echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER" + ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1 fi fi exit $? @@ -260,7 +265,7 @@ function launch_build() targets="$TARGETS_RT $TARGETS_MISC" else if [ "$build_type" == "installer" ]; then - targets="$TARGETS_INSTALLER" + targets="$TARGETS_INSTALLER $TARGETS_MISC" else targets="$TARGETS" fi @@ -271,8 +276,8 @@ function launch_build() echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile # No clean flag, call build-srpms-serial followed by build-rpms-serial - echo "$DIR/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile - $DIR/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile + echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile + ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile rc=${PIPESTATUS[0]} if [ $rc -eq 0 ]; then echo -e "\n######## $(date): build-srpm-serial --$build_type was successful" | tee --append $logfile @@ -284,8 +289,8 @@ function launch_build() if [ $EDIT_FLAG -ne 1 ]; then echo -e "\n######## $(date): Launching build-rpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile - echo "$DIR/build-rpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile - $DIR/build-rpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets 2>&1 | tee --append $logfile + echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile + ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $targets 2>&1 | tee --append $logfile rc=${PIPESTATUS[0]} if [ $rc -eq 0 ]; then echo -e "\n######## $(date): build-rpm-serial --$build_type was successful" | tee --append $logfile @@ -322,6 +327,7 @@ if [ $STD_BUILD -eq 1 ]; then else echo "Skipping 'std' build" fi + if [ $RT_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then launch_build rt @@ -331,48 +337,17 @@ if [ $RT_BUILD -eq 1 ]; then else echo "Skipping 'rt' build" fi + if [ $INSTALLER_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then launch_build installer else echo "Skipping 'installer' build, no valid targets in list: $TARGETS" fi -# else - # echo "Skipping 'installer' build" +else + echo "Skipping 'installer' build" fi - -#progbar & - -#function killemall() { -# for pid in $(jobs -p) -# do -# pgid=$(echo -n $(ps -o pgid= $pid)) -# if [ -n "$pgid" ]; then -# echo "Killing child progress group: kill -TERM -$pgid" -# kill -TERM -$pgid -# fi -# done -# echo "Bad mojo when you ctrl-C, so don't" -#} - -#trap killemall INT - -## Wait for builds to complete -#wait %1 -#STD_BUILD_RC=$? - -#wait %2 -#RT_BUILD_RC=$? - -# Kill the progress bar -#kill %3 - -#if [ $STD_BUILD_RC -ne 0 -o $RT_BUILD_RC -ne 0 ]; then -# echo "One or more builds failed" -# exit 1 -#fi - echo "All builds were successful" exit 0 diff --git a/build-tools/build-pkgs4 b/build-tools/build-pkgs4 deleted file mode 120000 index a8849bfe..00000000 --- a/build-tools/build-pkgs4 +++ /dev/null @@ -1 +0,0 @@ -build-pkgs-parallel \ No newline at end of file diff --git a/build-tools/build-rpms4 b/build-tools/build-rpms4 deleted file mode 120000 index caca9cc7..00000000 --- a/build-tools/build-rpms4 +++ /dev/null @@ -1 +0,0 @@ -build-rpms-parallel \ No newline at end of file diff --git a/build-tools/build-srpms-parallel b/build-tools/build-srpms-parallel index ed381a14..83e68ce2 100755 --- a/build-tools/build-srpms-parallel +++ b/build-tools/build-srpms-parallel @@ -4,10 +4,11 @@ export ME=$(basename "$0") CMDLINE="$ME $@" -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $DIR/spec-utils -source $DIR/srpm-utils -source $DIR/classify +BUILD_SRPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source $BUILD_SRPMS_PARALLEL_DIR/spec-utils +source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils +source $BUILD_SRPMS_PARALLEL_DIR/classify + INITIAL_DIR=$(pwd) export DISTRO="centos" @@ -15,7 +16,7 @@ SRPM_SCRIPT="build_srpm" SRPM_DATA="build_srpm.data" PKG_DIRS_FILE="${DISTRO}_pkg_dirs" -DEFAULT_SRPM_SCRIPT="$DIR/default_$SRPM_SCRIPT" +DEFAULT_SRPM_SCRIPT="$BUILD_SRPMS_PARALLEL_DIR/default_$SRPM_SCRIPT" SCRIPT_PATH="$DISTRO" DATA_PATH="$DISTRO" FILES_PATH="$DISTRO/files" @@ -123,7 +124,7 @@ EDIT_FLAG=0 NO_META_PATCH_FLAG=0 # read the options -TEMP=$(getopt -o ha::bc: --long std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@") +TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@") if [ $? -ne 0 ]; then usage @@ -158,6 +159,7 @@ while true ; do --installer) BUILD_TYPE=installer; shift ;; --edit) EDIT_FLAG=1 ; shift ;; --no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;; + --parallel) shift ;; --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac @@ -200,7 +202,7 @@ if [ $FORMAL_FLAG -eq 1 ]; then fi if [ "x$TARGETS" == "x" ] && [ $EDIT_FLAG -eq 1 ]; then - echo "ERROR: a package name is required when --edit is specified" + echo "ERROR: $FUNCNAME (${LINENO}): a package name is required when --edit is specified" usage exit 0 fi @@ -215,7 +217,7 @@ if [ "x$MY_WORKSPACE" == "x" ]; then BUILD_ROOT="$MY_PATCH_WORKSPACE" if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then - echo "ERROR: require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined" + echo "ERROR: $FUNCNAME (${LINENO}): require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined" exit 1 fi fi @@ -247,12 +249,12 @@ export SOURCE_OUT="$RPM_BUILD_BASE/SOURCES" export RPM_DIR="$RPM_BUILD_BASE/RPMS" if [ ! -d $CGCS_BASE ]; then - echo "ERROR: expected to find directory at '$CGCS_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$CGCS_BASE'" exit 1 fi if [ ! -d $BUILD_BASE ]; then - echo "ERROR: expected to find directory at '$BUILD_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'" exit 1 fi @@ -260,12 +262,12 @@ RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc if [ -f $RELEASE_INFO_FILE ]; then source $RELEASE_INFO_FILE else - echo "ERROR: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE" + echo "ERROR: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE" exit 1 fi if [ "x$PLATFORM_RELEASE" == "x" ]; then - echo "ERROR: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE" + echo "ERROR: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE" exit 1 fi @@ -273,19 +275,19 @@ export PLATFORM_RELEASE mkdir -p $RPM_BUILD_BASE if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$RPM_BUILD_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_BUILD_BASE'" exit 1 fi mkdir -p $SRPM_OUT if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$SRPM_OUT'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_OUT'" exit 1 fi mkdir -p $RPM_DIR if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$RPM_DIR'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_DIR'" exit 1 fi @@ -303,7 +305,7 @@ build_dir () { cd "$PKG_BASE" if [ $? -ne 0 ]; then - echo "ERROR: failed to cd into '$PKG_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to cd into '$PKG_BASE'" return 1 fi @@ -374,7 +376,7 @@ build_dir () { if [ -f $ORIG_SRPM_PATH ]; then SRPM_COUNT=$((SRPM_COUNT + 1)) else - echo "ERROR: Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'" ORIG_SRPM_PATH="" return 3 fi @@ -396,7 +398,7 @@ build_dir () { if [ $SPEC_COUNT -eq 0 ] && [ $SRPM_COUNT -eq 0 ]; then # nothing to do - echo "ERROR: Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'" cd "$ORIG_DIR" return 0 fi @@ -404,7 +406,7 @@ build_dir () { if [ $SPEC_COUNT -gt 0 ] && [ $SRPM_COUNT -gt 0 ]; then # nothing to do - echo "ERROR: Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'" cd $ORIG_DIR return 0 fi @@ -542,7 +544,6 @@ build_dir_srpm () { local BUILD_DIR="$PKG_DIR/rpmbuild" local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" - # local SOURCES_DIR="$FULL_BUILD_DIR/SOURCES" local SOURCES_DIR="$SOURCE_OUT" if [ $CLEAN_FLAG -eq 1 ]; then @@ -567,23 +568,28 @@ build_dir_srpm () { if [ ! -d $ROOT_DIR ]; then mkdir -p "$ROOT_DIR" if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: mkdir '$ROOT_DIR' failed" + echo "ERROR: $FUNCNAME (${LINENO}): mkdir '$ROOT_DIR' failed" return 1 fi fi + # + # Load data from build_srpm.data + # export DATA="$DATA_PATH/$SRPM_DATA" local COPY_LIST local COPY_LIST_TO_TAR local SRC_DIR local TIS_PATCH_VER + local BUILD_IS_BIG=0 + local BUILD_IS_SLOW=0 - BUILD_IS_BIG=0 - BUILD_IS_SLOW=0 - srpm_source_build_data $DATA - if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to source $DATA" - return 1 + if [ -f $DATA ]; then + srpm_source_build_data $DATA + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" + return 1 + fi fi local BUILD_NEEDED=2 @@ -666,7 +672,7 @@ build_dir_srpm () { RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then - echo "ERROR: srpm_extract_to_git: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi @@ -678,6 +684,10 @@ build_dir_srpm () { return 0 fi + # + # Find age of youngest input file. + # We will apply this as the creation/modification timestamp of the src.rpm we produce. + # AGE=$(find $PKG_BASE $ORIG_SRPM_PATH -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) if [ -f $PKG_BASE/$DATA ]; then AGE2=$( @@ -708,7 +718,7 @@ build_dir_srpm () { srpm_extract $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" return 1 fi @@ -718,7 +728,7 @@ build_dir_srpm () { # echo "COPY_LIST: $p" \cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES if [ $? -ne 0 ]; then - echo "ERROR: COPY_LIST: file not found: '$p'" + echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'" exit 1 fi done @@ -726,9 +736,9 @@ build_dir_srpm () { srpm_assemble $FULL_BUILD_DIR $TIS_PATCH_VER if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to assemble srpm for '$PKG_NAME_VER'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to assemble srpm for '$PKG_NAME_VER'" + echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx} - # SRPM_FAILED_REBUILD_LIST="$SRPM_FAILED_REBUILD_LIST $TARGET_FOUND" return 1 fi @@ -741,8 +751,6 @@ build_dir_srpm () { mkdir -p $SOURCES_DIR/$ss BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" -echo "BIG_FLAG_FILE=$BIG_FLAG_FILE" -echo "SLOW_FLAG_FILE=$SLOW_FLAG_FILE" if [ $BUILD_IS_BIG -gt 0 ]; then echo "$BUILD_IS_BIG" > $BIG_FLAG_FILE @@ -763,10 +771,10 @@ echo "SLOW_FLAG_FILE=$SLOW_FLAG_FILE" done echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx} - # SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND" echo "SRPM build successful for '$PKG_NAME_VER'" echo "===== Build complete for '$TARGET_FOUND' =====" echo + fi return 0 @@ -782,9 +790,9 @@ build_dir_spec () { if [ "x$TARGETS" == "x" ]; then NEED_BUILD=1 for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec'); do - TARGET_FOUND=$(spec_find_global service "$f" 2>> /dev/null) + TARGET_FOUND=$(spec_find_tag Name "$f" 2>> /dev/null) if [ $? -ne 0 ]; then - TARGET_FOUND=$(spec_find_tag Name "$f" 2>> /dev/null) + TARGET_FOUND=$(spec_find_global service "$f" 2>> /dev/null) if [ $? -ne 0 ]; then TARGET_FOUND="" fi @@ -818,9 +826,8 @@ build_dir_spec () { local PKG_NAME_VER=$(spec_name_ver_rel $SPEC) local PKG_DIR="$NAME" local BUILD_DIR="$PKG_DIR/rpmbuild" - local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR" + local FULL_BUILD_DIR="$ROOT_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" - # local SOURCES_DIR="$FULL_BUILD_DIR/SOURCES" local SOURCES_DIR="$SOURCE_OUT" if [ $EDIT_FLAG -eq 1 ]; then @@ -859,10 +866,22 @@ build_dir_spec () { local BUILD_NEEDED=2 local SRPM_OUT_PATH2 - BUILD_IS_BIG=0 - BUILD_IS_SLOW=0 -# SAL source + # + # Load data from build_srpm.data + # + local COPY_LIST + local COPY_LIST_TO_TAR + local SRC_DIR + local OPT_DEP_LIST + local TIS_PATCH_VER + local BUILD_IS_BIG=0 + local BUILD_IS_SLOW=0 + srpm_source_build_data $DATA + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" + return 1 + fi for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do if [ $BUILD_NEEDED -eq 2 ]; then @@ -917,10 +936,6 @@ build_dir_spec () { done if [ $EDIT_FLAG -eq 1 ]; then - local COPY_LIST - local COPY_LIST_TO_TAR - local SRC_DIR - local TIS_PATCH_VER PKG_CLASSIFICATION=$(classify $PKG_BASE) echo "$PKG_CLASSIFICATION = classify $PKG_BASE" @@ -931,18 +946,12 @@ build_dir_spec () { return 1 fi - srpm_source_build_data $DATA - if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to source $DATA" - return 1 - fi - echo "tar_and_spec_extract_to_git '$SPEC' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER'" tar_and_spec_extract_to_git "$SPEC" "$PKG_BASE" "$ROOT_DIR" "$BUILD_DIR" "$PKG_NAME_VER" "$NO_META_PATCH_FLAG" "$TIS_PATCH_VER" RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then - echo "ERROR: srpm_extract_to_git: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi @@ -979,24 +988,28 @@ build_dir_spec () { mkdir -p "$SPEC_PATH" && \ mkdir -p "$SOURCES_PATH" if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directories under: $WORK_BASE" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directories under: $WORK_BASE" fi \cp -L -f -v $SPECS_BASE/*.spec $SPEC_PATH/ if [ $? -ne 0 ]; then - echo "ERROR: Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'" fi + # # build + # $MAKE_SRPM if [ $? -ne 0 ]; then - echo "ERROR: script failed '$MAKE_SRPM'" + echo "ERROR: $FUNCNAME (${LINENO}): script failed '$MAKE_SRPM'" echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx} - # SRPM_FAILED_REBUILD_LIST="$SRPM_FAILED_REBUILD_LIST $TARGET_FOUND" exit 1 fi - + # + # Find age of youngest input file. + # We will apply this as the creation/modification timestamp of the src.rpm we produce. + # AGE=$(find $PKG_BASE -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) if [ -f $PKG_BASE/$DATA ]; then AGE2=$( @@ -1038,8 +1051,6 @@ build_dir_spec () { mkdir -p $SOURCES_DIR/$ss BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" -echo "BIG_FLAG_FILE=$BIG_FLAG_FILE" -echo "SLOW_FLAG_FILE=$SLOW_FLAG_FILE" if [ $BUILD_IS_BIG -gt 0 ]; then echo $BUILD_IS_BIG > $BIG_FLAG_FILE @@ -1059,7 +1070,6 @@ echo "SLOW_FLAG_FILE=$SLOW_FLAG_FILE" done echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx} - # SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND" echo "===== Build complete for '$TARGET_FOUND' =====" echo fi @@ -1171,16 +1181,16 @@ reaper () { if [ $CLEAN_FLAG -eq 1 ]; then VERB="edit clean" fi - fi - - if [ $CLEAN_FLAG -eq 1 ]; then - VERB="clean" + else + if [ $CLEAN_FLAG -eq 1 ]; then + VERB="clean" + fi fi sleep 1 - echo "ERROR: Failed to $VERB src.rpm from source at 'b$i'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'" cat "$LOG_DIR/$i" >> $LOG_DIR/errors - echo "ERROR: Failed to $VERB src.rpm from source at 'b$i'" >> $LOG_DIR/errors + echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'" >> $LOG_DIR/errors echo "" >> $LOG_DIR/errors STOP_SCHEDULING=1 fi @@ -1262,10 +1272,10 @@ for g in $(find "$SRC_BASE" -type d -name .git | sort -V); do pp=$! set_build_env_pid $b $pp else - echo "ERROR: Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi else - echo "ERROR: Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'" + echo "ERROR: $FUNCNAME (${LINENO}): Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi done done @@ -1318,7 +1328,6 @@ UNRESOLVED_TARGETS=$(for i in $(seq 0 $((max_workers-1))); do if [ $ALL -eq 1 ]; then echo echo "Auditing for obsolete srpms" - # PACKAGES_CONSIDERED="" AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX) if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then for g in $(find $SRC_BASE -type d -name .git | sort -V); do @@ -1332,7 +1341,6 @@ if [ $ALL -eq 1 ]; then for f in $(find $src_dir/${DISTRO} -name '*.spec' | sort -V); do NAME=$(spec_find_tag Name "$f" 2>> /dev/null) if [ $? -eq 0 ]; then - # PACKAGES_CONSIDERED="$PACKAGES_CONSIDERED $NAME" touch "$AUDIT_DIR/$NAME" fi done @@ -1367,7 +1375,6 @@ if [ $ALL -eq 1 ]; then if [ -f $ORIG_SRPM_PATH ]; then NAME=$(rpm -q --queryformat '%{NAME}\n' -p $ORIG_SRPM_PATH 2>> /dev/null) if [ $? -eq 0 ]; then - # PACKAGES_CONSIDERED="$PACKAGES_CONSIDERED $NAME" touch "$AUDIT_DIR/$NAME" fi fi @@ -1387,15 +1394,11 @@ if [ $ALL -eq 1 ]; then NAME=$(rpm -q --queryformat '%{NAME}\n' -p $r 2>> /dev/null) ALT_NAME=$(echo $NAME | sed "s#-$BUILD_TYPE\$##") FOUND=0 - # for p in $PACKAGES_CONSIDERED; do - # if [[ "$NAME" == "$p" || ( "$BUILD_TYPE" != "std" && "$NAME" == "$p-$BUILD_TYPE" ) ]]; then - # FOUND=1 - # break - # fi - # done + if [[ -f "$AUDIT_DIR/$NAME" || ( "$BUILD_TYPE" != "std" && -f "$AUDIT_DIR/$ALT_NAME" ) ]]; then FOUND=1 fi + if [ $FOUND -eq 0 ]; then for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then @@ -1462,7 +1465,7 @@ fi if [ "$UNRESOLVED_TARGETS" != "" ]; then echo "" - echo "ERROR: failed to resolve build targets: $UNRESOLVED_TARGETS" + echo "ERROR: $FUNCNAME (${LINENO}): failed to resolve build targets: $UNRESOLVED_TARGETS" FINAL_RC=1 fi diff --git a/build-tools/build-srpms-serial b/build-tools/build-srpms-serial index 41ba7f9e..b1c05cab 100755 --- a/build-tools/build-srpms-serial +++ b/build-tools/build-srpms-serial @@ -1,12 +1,15 @@ #!/bin/bash # set -x -CMDLINE="build-srpms-serial $@" +export ME=$(basename "$0") +CMDLINE="$ME $@" + + +BUILD_SRPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source $BUILD_SRPMS_SERIAL_DIR/spec-utils +source $BUILD_SRPMS_SERIAL_DIR/srpm-utils +source $BUILD_SRPMS_SERIAL_DIR/classify -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $DIR/spec-utils -source $DIR/srpm-utils -source $DIR/classify INITIAL_DIR=`pwd` export DISTRO="centos" @@ -14,7 +17,7 @@ SRPM_SCRIPT="build_srpm" SRPM_DATA="build_srpm.data" PKG_DIRS_FILE="${DISTRO}_pkg_dirs" -DEFAULT_SRPM_SCRIPT="$DIR/default_$SRPM_SCRIPT" +DEFAULT_SRPM_SCRIPT="$BUILD_SRPMS_SERIAL_DIR/default_$SRPM_SCRIPT" SCRIPT_PATH="$DISTRO" DATA_PATH="$DISTRO" FILES_PATH="$DISTRO/files" @@ -36,22 +39,22 @@ usage () { echo "" echo "Usage: " echo " Create source rpms:" - echo " build-srpms [--rt | --std | --installer] [--no-descendants] [--formal] [ list of package names ]" + echo " $ME [--rt | --std | --installer] [--no-descendants] [--formal] [ list of package names ]" echo "" echo " Delete source rpms, and the directories associated with it's creation:" echo " Note: does not clean an edit environment" - echo " build-srpms --clean [--rt | --std | --installer] [optional list of package names]" + echo " $ME --clean [--rt | --std | --installer] [optional list of package names]" echo "" echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents," echo " one for source code and one for metadata such as the spec file." echo " If --no-meta-patch is specified, then WRS patches are omitted." - echo " build-srpms --edit [--rt | --std | --installer] [--no-meta-patch] [list of package names]" + echo " $ME --edit [--rt | --std | --installer] [--no-meta-patch] [list of package names]" echo "" echo " Delete an edit environment" - echo " build-srpms --edit --clean [--rt | --std | --installer] [list of package names]" + echo " $ME --edit --clean [--rt | --std | --installer] [list of package names]" echo "" echo " This help page" - echo " build-srpms --help" + echo " $ME --help" echo "" } @@ -108,7 +111,7 @@ EDIT_FLAG=0 NO_META_PATCH_FLAG=0 # read the options -TEMP=`getopt -o ha::bc: --long serial,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n 'build-srpms' -- "$@"` +TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@") if [ $? -ne 0 ]; then usage @@ -186,7 +189,7 @@ if [ $FORMAL_FLAG -eq 1 ]; then fi if [ "x$TARGETS" == "x" ] && [ $EDIT_FLAG -eq 1 ]; then - echo "ERROR: a package name is required when --edit is specified" + echo "ERROR: $FUNCNAME (${LINENO}): a package name is required when --edit is specified" usage exit 0 fi @@ -201,7 +204,7 @@ if [ "x$MY_WORKSPACE" == "x" ]; then BUILD_ROOT="$MY_PATCH_WORKSPACE" if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then - echo "ERROR: require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined" + echo "ERROR: $FUNCNAME (${LINENO}): require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined" exit 1 fi fi @@ -229,15 +232,16 @@ create_output_dirs export RPM_BUILD_BASE="$RPM_BUILD_ROOT" export SRPM_OUT="$RPM_BUILD_BASE/SRPMS" +export SOURCE_OUT="$RPM_BUILD_BASE/SOURCES" export RPM_DIR="$RPM_BUILD_BASE/RPMS" if [ ! -d $CGCS_BASE ]; then - echo "ERROR: expected to find directory at '$CGCS_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$CGCS_BASE'" exit 1 fi if [ ! -d $BUILD_BASE ]; then - echo "ERROR: expected to find directory at '$BUILD_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'" exit 1 fi @@ -245,12 +249,12 @@ RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc if [ -f $RELEASE_INFO_FILE ]; then source $RELEASE_INFO_FILE else - echo "ERROR: failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE" + echo "ERROR: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE" exit 1 fi if [ "x$PLATFORM_RELEASE" == "x" ]; then - echo "ERROR: PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE" + echo "ERROR: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE" exit 1 fi @@ -258,19 +262,19 @@ export PLATFORM_RELEASE mkdir -p $RPM_BUILD_BASE if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$RPM_BUILD_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_BUILD_BASE'" exit 1 fi mkdir -p $SRPM_OUT if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$SRPM_OUT'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_OUT'" exit 1 fi mkdir -p $RPM_DIR if [ $? -ne 0 ]; then - echo "ERROR: Failed to create directory '$RPM_DIR'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_DIR'" exit 1 fi @@ -287,7 +291,7 @@ build_dir () { cd "$PKG_BASE" if [ $? -ne 0 ]; then - echo "ERROR: failed to cd into '$PKG_BASE'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to cd into '$PKG_BASE'" return 1 fi @@ -358,7 +362,7 @@ build_dir () { if [ -f $ORIG_SRPM_PATH ]; then SRPM_COUNT=$((SRPM_COUNT + 1)) else - echo "ERROR: Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$SRPM_LIST_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'" ORIG_SRPM_PATH="" exit 1 fi @@ -380,7 +384,7 @@ build_dir () { if [ $SPEC_COUNT -eq 0 ] && [ $SRPM_COUNT -eq 0 ]; then # nothing to do - echo "ERROR: Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'" cd "$ORIG_DIR" return 0 fi @@ -388,7 +392,7 @@ build_dir () { if [ $SPEC_COUNT -gt 0 ] && [ $SRPM_COUNT -gt 0 ]; then # nothing to do - echo "ERROR: Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'" cd $ORIG_DIR return 0 fi @@ -512,6 +516,7 @@ build_dir_srpm () { local BUILD_DIR="$PKG_DIR/rpmbuild" local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" + local SOURCES_DIR="$SOURCE_OUT" if [ $CLEAN_FLAG -eq 1 ]; then # clean @@ -535,21 +540,28 @@ build_dir_srpm () { if [ ! -d $ROOT_DIR ]; then mkdir -p "$ROOT_DIR" if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: mkdir '$ROOT_DIR' failed" + echo "ERROR: $FUNCNAME (${LINENO}): mkdir '$ROOT_DIR' failed" return 1 fi fi + # + # Load data from build_srpm.data + # export DATA="$DATA_PATH/$SRPM_DATA" local COPY_LIST local COPY_LIST_TO_TAR local SRC_DIR local TIS_PATCH_VER + local BUILD_IS_BIG=0 + local BUILD_IS_SLOW=0 - srpm_source_build_data $DATA - if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to source $DATA" - return 1 + if [ -f $DATA ]; then + srpm_source_build_data $DATA + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" + return 1 + fi fi local BUILD_NEEDED=2 @@ -580,7 +592,6 @@ build_dir_srpm () { ( cd $PKG_BASE BUILD_NEEDED=0 - srpm_source_build_data $DATA # NOTE: SRC_DIR is not honored in this build path @@ -632,7 +643,7 @@ build_dir_srpm () { RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then - echo "ERROR: srpm_extract_to_git: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi @@ -648,7 +659,6 @@ build_dir_srpm () { if [ -f $PKG_BASE/$DATA ]; then AGE2=$( cd $PKG_BASE - srpm_source_build_data $DATA PATH_LIST="" # NOTE: SRC_DIR is not honored in this build path @@ -674,7 +684,7 @@ build_dir_srpm () { srpm_extract $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" return 1 fi @@ -684,7 +694,7 @@ build_dir_srpm () { # echo "COPY_LIST: $p" \cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES if [ $? -ne 0 ]; then - echo "ERROR: COPY_LIST: file not found: '$p'" + echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'" exit 1 fi done @@ -692,7 +702,7 @@ build_dir_srpm () { srpm_assemble $FULL_BUILD_DIR $TIS_PATCH_VER if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to assemble srpm for '$PKG_NAME_VER'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to assemble srpm for '$PKG_NAME_VER'" SRPM_FAILED_REBUILD_LIST="$SRPM_FAILED_REBUILD_LIST $TARGET_FOUND" return 1 fi @@ -702,12 +712,33 @@ build_dir_srpm () { \cp -L -f -v $s $SRPM_OUT/ ss=$(basename $s) touch $SRPM_OUT/$ss --date=$TS + + mkdir -p $SOURCES_DIR/$ss + BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" + SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" + + if [ $BUILD_IS_BIG -gt 0 ]; then + echo "$BUILD_IS_BIG" > $BIG_FLAG_FILE + else + if [ -f $BIG_FLAG_FILE ]; then + \rm -f $BIG_FLAG_FILE + fi + fi + + if [ $BUILD_IS_SLOW -gt 0 ]; then + echo "$BUILD_IS_SLOW" > $SLOW_FLAG_FILE + else + if [ -f $SLOW_FLAG_FILE ]; then + \rm -f $SLOW_FLAG_FILE + fi + fi done SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND" echo "SRPM build successful for '$PKG_NAME_VER'" echo "===== Build complete for '$TARGET_FOUND' =====" echo + fi return 0 @@ -720,10 +751,10 @@ build_dir_spec () { if [ "x$TARGETS" == "x" ]; then NEED_BUILD=1 - for f in `find $SPECS_BASE -maxdepth 1 -name '*.spec'`; do - TARGET_FOUND=`spec_find_global service "$f" 2>> /dev/null` + for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec'); do + TARGET_FOUND=$(spec_find_tag Name "$f" 2>> /dev/null) if [ $? -ne 0 ]; then - TARGET_FOUND=`spec_find_tag Name "$f" 2>> /dev/null` + TARGET_FOUND=$(spec_find_global service "$f" 2>> /dev/null) if [ $? -ne 0 ]; then TARGET_FOUND="" fi @@ -757,8 +788,9 @@ build_dir_spec () { local PKG_NAME_VER=$(spec_name_ver_rel $SPEC) local PKG_DIR="$NAME" local BUILD_DIR="$PKG_DIR/rpmbuild" - local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR" + local FULL_BUILD_DIR="$ROOT_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" + local SOURCES_DIR="$SOURCE_OUT" if [ $EDIT_FLAG -eq 1 ]; then mkdir -p $SRPM_WORK @@ -796,6 +828,25 @@ build_dir_spec () { local BUILD_NEEDED=2 local SRPM_OUT_PATH2 + # + # Load data from build_srpm.data + # + local COPY_LIST + local COPY_LIST_TO_TAR + local SRC_DIR + local OPT_DEP_LIST + local TIS_PATCH_VER + local BUILD_IS_BIG=0 + local BUILD_IS_SLOW=0 + + if [ -f $DATA ]; then + srpm_source_build_data $DATA + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" + return 1 + fi + fi + for SRPM_PATH2 in `find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V`; do if [ $BUILD_NEEDED -eq 2 ]; then BUILD_NEEDED=0 @@ -816,7 +867,6 @@ build_dir_spec () { ( cd $PKG_BASE BUILD_NEEDED=0 - srpm_source_build_data $DATA if [ "x$SRC_DIR" != "x" ]; then if [ -d "$SRC_DIR" ]; then n=`find $SRC_DIR -type f -cnewer $SRPM_PATH2 | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | wc -l` @@ -849,10 +899,6 @@ build_dir_spec () { done if [ $EDIT_FLAG -eq 1 ]; then - local COPY_LIST - local COPY_LIST_TO_TAR - local SRC_DIR - local TIS_PATCH_VER PKG_CLASSIFICATION=$(classify $PKG_BASE) echo "$PKG_CLASSIFICATION = classify $PKG_BASE" @@ -863,18 +909,12 @@ build_dir_spec () { return 1 fi - srpm_source_build_data $DATA - if [ $? -ne 0 ]; then - echo "ERROR: build_dir_srpm: failed to source $DATA" - return 1 - fi - echo "tar_and_spec_extract_to_git '$SPEC' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER'" tar_and_spec_extract_to_git "$SPEC" "$PKG_BASE" "$ROOT_DIR" "$BUILD_DIR" "$PKG_NAME_VER" "$NO_META_PATCH_FLAG" "$TIS_PATCH_VER" RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then - echo "ERROR: srpm_extract_to_git: failed to extract srpm '$ORIG_SRPM_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi @@ -905,6 +945,7 @@ build_dir_spec () { rm -rf "$RPMBUILD_BASE" fi + echo "mkdir -p $WORK_BASE $SRPM_PATH $SPEC_PATH $SOURCES_PATH" mkdir -p "$WORK_BASE" && \ mkdir -p "$SRPM_PATH" && \ mkdir -p "$SPEC_PATH" && \ @@ -915,13 +956,15 @@ build_dir_spec () { \cp -L -f -v $SPECS_BASE/*.spec $SPEC_PATH/ if [ $? -ne 0 ]; then - echo "ERROR: Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'" fi + # # build + # $MAKE_SRPM if [ $? -ne 0 ]; then - echo "ERROR: script failed '$MAKE_SRPM'" + echo "ERROR: $FUNCNAME (${LINENO}): script failed '$MAKE_SRPM'" SRPM_FAILED_REBUILD_LIST="$SRPM_FAILED_REBUILD_LIST $TARGET_FOUND" exit 1 fi @@ -931,7 +974,6 @@ build_dir_spec () { if [ -f $PKG_BASE/$DATA ]; then AGE2=$( cd $PKG_BASE - srpm_source_build_data $DATA PATH_LIST="" if [ "x$SRC_DIR" != "x" ]; then if [ -d "$SRC_DIR" ]; then @@ -964,6 +1006,27 @@ build_dir_spec () { \cp -L -f $s $SRPM_OUT/ ss=$(basename $s) touch $SRPM_OUT/$ss --date=$TS + + mkdir -p $SOURCES_DIR/$ss + BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" + SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" + + if [ $BUILD_IS_BIG -gt 0 ]; then + echo $BUILD_IS_BIG > $BIG_FLAG_FILE + else + if [ -f $BIG_FLAG_FILE ]; then + \rm -f $BIG_FLAG_FILE + fi + fi + + if [ $BUILD_IS_SLOW -gt 0 ]; then + echo $BUILD_IS_SLOW > $SLOW_FLAG_FILE + else + if [ -f $SLOW_FLAG_FILE ]; then + \rm -f $SLOW_FLAG_FILE + fi + fi + done SRPM_REBUILT_LIST="$SRPM_REBUILT_LIST $TARGET_FOUND" @@ -984,7 +1047,7 @@ if [ -L $BUILD_ROOT/repo ]; then echo "Error: MY_REPO changed since last build" echo " old path: $REPO_DEST" echo " new path: $SRC_ROOT" - echo "Please run 'build-srpms --clean' if you want to compile from a new source tree" + echo "Please run '$ME --clean' if you want to compile from a new source tree" exit 1 fi fi @@ -1054,10 +1117,10 @@ for g in `find "$SRC_BASE" -type d -name .git | sort -V`; do exit 1 fi else - echo "ERROR: Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi else - echo "ERROR: Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'" + echo "ERROR: $FUNCNAME (${LINENO}): Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi done done @@ -1121,15 +1184,16 @@ if [ $ALL -eq 1 ]; then done done + echo "Auditing for obsolete srpms Phase 2" for r in $(find $SRPM_OUT -name '*.src.rpm' | sort -V); do NAME=$(rpm -q --queryformat '%{NAME}\n' -p $r 2>> /dev/null) + ALT_NAME=$(echo $NAME | sed "s#-$BUILD_TYPE\$##") FOUND=0 - for p in $PACKAGES_CONSIDERED; do - if [[ "$NAME" == "$p" || ( "$BUILD_TYPE" != "std" && "$NAME" == "$p-$BUILD_TYPE" ) ]]; then - FOUND=1 - break - fi - done + + if [[ -f "$AUDIT_DIR/$NAME" || ( "$BUILD_TYPE" != "std" && -f "$AUDIT_DIR/$ALT_NAME" ) ]]; then + FOUND=1 + fi + if [ $FOUND -eq 0 ]; then for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then @@ -1165,12 +1229,14 @@ if [ $EDIT_FLAG -ne 1 ]; then echo "==== Update repodata complete =====" fi +FINAL_RC=0 if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ]; then echo "" if [ "$SRPM_FAILED_REBUILD_LIST" != "" ]; then N=`echo "$SRPM_FAILED_REBUILD_LIST" | wc -w` echo "Failed to build $N packages:" echo " $SRPM_FAILED_REBUILD_LIST" + FINAL_RC=1 fi if [ "$SRPM_REBUILT_LIST" != "" ]; then N=`echo "$SRPM_REBUILT_LIST" | wc -w` @@ -1187,9 +1253,9 @@ fi if [ "$UNRESOLVED_TARGETS" != " " ]; then echo "" - echo "ERROR: failed to resolve build targets: $UNRESOLVED_TARGETS" - exit 1 + echo "ERROR: $FUNCNAME (${LINENO}): failed to resolve build targets: $UNRESOLVED_TARGETS" + FINAL_RC=1 fi -exit 0 +exit $FINAL_RC ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-srpms-serial_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build-srpms4 b/build-tools/build-srpms4 deleted file mode 120000 index 5301273b..00000000 --- a/build-tools/build-srpms4 +++ /dev/null @@ -1 +0,0 @@ -build-srpms-parallel \ No newline at end of file diff --git a/build-tools/classify b/build-tools/classify index 22457474..9594db38 100644 --- a/build-tools/classify +++ b/build-tools/classify @@ -53,15 +53,3 @@ classify () { echo "unknown" fi } - -# for g in $(find $MY_REPO -type d -name .git); do -# d=$(dirname $g) -# c="$d/centos_pkg_dirs" -# if [ -f $c ]; then -# for d2 in $(cat $c); do -# pkg_dir="$d/$d2" -# echo "$(classify $pkg_dir): $pkg_dir" -# done -# fi -# done - diff --git a/build-tools/mockchain-parallel b/build-tools/mockchain-parallel index 0e1fd407..194b631c 100755 --- a/build-tools/mockchain-parallel +++ b/build-tools/mockchain-parallel @@ -651,7 +651,7 @@ def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts) # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: if can_build_at_idx(build_idx, name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s'" % name) + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) return name else: # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) @@ -666,7 +666,7 @@ def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts) if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: if can_build_at_idx(build_idx, name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep" % name) + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) return name else: # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) @@ -690,7 +690,7 @@ def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts) # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: if can_build_at_idx(build_idx, dep_name, opts): - log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s'" % (dep_name, name)) + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) return dep_name if not dep_name in unbuilt_deps[0]: @@ -702,7 +702,7 @@ def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts) if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: if can_build_at_idx(build_idx, dep_name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s'" % (dep_name, name)) + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) return dep_name if name in unbuilt_deps[0][dep_name]: diff --git a/build-tools/srpm-utils b/build-tools/srpm-utils index 8e704bf0..42e558f6 100644 --- a/build-tools/srpm-utils +++ b/build-tools/srpm-utils @@ -1694,7 +1694,7 @@ tarball_extract () { RAW_SCRIPT=$ROOT_DIR/$PKG_DIR/raw_script EXTRACT_SCRIPT=$ROOT_DIR/$PKG_DIR/extract_script - echo "srpm_create_raw_extract_script '$SPEC_DIR/$SPEC' '$ROOT_DIR/$PKG_DIR' '$ROOT_DIR/$BUILD_DIR' '$TARGET_ARCH' '$TIS_PATCH_VER' '$RAW_SCRIPT'" + echo "srpm_create_raw_extract_script '$SPEC_DIR/$SPEC' '$ROOT_DIR/$PKG_DIR' '$ROOT_DIR/$BUILD_DIR' '$TARGET_ARCH' '$TIS_PATCH_VER' '$RAW_SCRIPT' '$TAR_DIR'" srpm_create_raw_extract_script "$SPEC_DIR/$SPEC" "$ROOT_DIR/$PKG_DIR" "$ROOT_DIR/$BUILD_DIR" "$TARGET_ARCH" "$TIS_PATCH_VER" "$RAW_SCRIPT" "$TAR_DIR" if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): srpm_create_raw_extract_script failed" @@ -1909,9 +1909,12 @@ tarball_extract () { pwd # Delete old branch if it exists + echo "git checkout $BRANCH @ $(pwd)" git checkout $BRANCH &>> /dev/null if [ $? -eq 0 ]; then + echo "git checkout master @ $(pwd)" git checkout master + echo "git branch -D $BRANCH @ $(pwd)" git branch -D $BRANCH if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to delete branch '$BRANCH'" @@ -1922,8 +1925,10 @@ tarball_extract () { # Determine origin of our branch if [ $TAR_HAS_CHANGED -gt 0 ]; then + echo "git checkout master @ $(pwd)" git checkout master else + echo "git checkout $ORIG_PRE_WRS_TAG @ $(pwd)" git checkout $ORIG_PRE_WRS_TAG if [ $? -eq 0 ]; then NEED_TAR_EXTRACT=0 @@ -2526,7 +2531,7 @@ srpm_extract_to_git () { echo "COPY_LIST: $p" \cp -L -f -r -v $p $META_PATCH_TARGET_DIR/SOURCES if [ $? -ne 0 ]; then - echo "ERROR: COPY_LIST: file not found: '$p'" + echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'" cd $ORIG_DIR return 1 fi @@ -2563,6 +2568,7 @@ srpm_extract_to_git () { fi mkdir -p $RPMBUILD_BUILD_DIR2 + echo "srpm_create_raw_extract_script '$SPEC_DIR/$SPEC' '$ROOT_DIR/$PKG_DIR' '$ROOT_DIR/$BUILD_DIR' '$TARGET_ARCH' '$TIS_PATCH_VER' '$RAW_SCRIPT' '$TAR_DIR'" srpm_create_raw_extract_script "$SPEC_DIR/$SPEC" "$ROOT_DIR/$PKG_DIR" "$ROOT_DIR/$BUILD_DIR" "$TARGET_ARCH" "$TIS_PATCH_VER" "$RAW_SCRIPT" "$TAR_DIR" if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): srpm_create_raw_extract_script post meta-patches failed" @@ -2592,7 +2598,7 @@ srpm_extract_to_git () { exit $RC ) | tee $EXTRACT_SCRIPT.post.log if [ ${PIPESTATUS[0]} -ne 0 ] ; then - echo "ERROR: Failed in script '$EXTRACT_SCRIPT'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed in script '$EXTRACT_SCRIPT'" cd $ORIG_DIR return 1 fi @@ -2843,7 +2849,7 @@ srpm_apply_patch() { source $PATCH_SCRIPT ) if [ $? -ne 0 ]; then - echo "ERROR: Failed to apply patch '$PATCH' using script '$PATCH_SCRIPT'" + echo "ERROR: $FUNCNAME (${LINENO}): Failed to apply patch '$PATCH' using script '$PATCH_SCRIPT'" return 1 fi else @@ -3151,31 +3157,31 @@ srpm_git_revision_count () { srpm_source_build_data () { local DATA_FILE=$1 if [ ! -f $DATA_FILE ]; then - >&2 echo "ERROR: $DATA_FILE not found" + >&2 echo "ERROR: $FUNCNAME (${LINENO}): $DATA_FILE not found" return 1 fi source $DATA_FILE # TIS_PATCH_VER is mandatory if [ -z "$TIS_PATCH_VER" ]; then - >&2 echo "ERROR: srpm_source_srpm_data: TIS_PATCH_VER must be set in $DATA_FILE" + >&2 echo "ERROR: $FUNCNAME (${LINENO}): TIS_PATCH_VER must be set in $DATA_FILE" return 1 elif [[ "$TIS_PATCH_VER" == GITREVCOUNT* ]]; then # Calculate the patch version dynamically based on the number of commits # in the subgit. This also supports adding a "+N" at the end to force # an additional increment (e.g., TIS_PATCH_VER=GITREVCOUNT+1) if [ -z "$TIS_BASE_SRCREV" ]; then - >&2 echo "ERROR: srpm_source_srpm_data: TIS_BASE_SRCREV must be set in $DATA_FILE" + >&2 echo "ERROR: $FUNCNAME (${LINENO}): TIS_BASE_SRCREV must be set in $DATA_FILE" return 1 fi if [ ! -d "$SRC_DIR" ]; then - >&2 echo "ERROR: srpm_source_srpm_data: SRC_DIR must specify a subgit root path" + >&2 echo "ERROR: $FUNCNAME (${LINENO}): SRC_DIR must specify a subgit root path" return 1 fi TIS_PATCH_INC=${TIS_PATCH_VER//[A-Z \+]/} TIS_PATCH_VER=$(srpm_git_revision_count $SRC_DIR $TIS_BASE_SRCREV) if [ $? -ne 0 ] || [ "$TIS_PATCH_VER" == "" ]; then - >&2 echo "ERROR: srpm_source_srpm_data: Invalid TIS_BASE_SRCREV '$TIS_BASE_SRCREV'" + >&2 echo "ERROR: $FUNCNAME (${LINENO}): Invalid TIS_BASE_SRCREV '$TIS_BASE_SRCREV'" return 1 fi if [[ "$TIS_PATCH_INC" =~ ^-?[0-9]+$ ]]; then