diff --git a/.gitignore b/.gitignore index 7770dd90..df27462f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /cgcs-centos-repo /cgcs-tis-repo /cgcs-3rd-party-repo +/local-build-data diff --git a/build-data/build_avoidance_source b/build-data/build_avoidance_source new file mode 100644 index 00000000..71135f20 --- /dev/null +++ b/build-data/build_avoidance_source @@ -0,0 +1,23 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# +# What files and directories need to be copied +# +BUILD_AVOIDANCE_SRPM_DIRECTORIES="inputs srpm_assemble rpmbuild/SRPMS rpmbuild/SOURCES" +BUILD_AVOIDANCE_SRPM_FILES="" +BUILD_AVOIDANCE_RPM_DIRECTORIES="results rpmbuild/RPMS rpmbuild/SPECS" +BUILD_AVOIDANCE_RPM_FILES=".platform_release" + + +# +# Copy the lines below to $MY_REPO/local-build-data/build_avoidance_source, +# then uncomment and fill in the values giving the location of your local reference build. +# +# BUILD_AVOIDANCE_USR="jenkins" +# BUILD_AVOIDANCE_HOST="machine.corp.com" +# BUILD_AVOIDANCE_DIR="/localdisk/loadbuild/jenkins/StarlingX_Build" diff --git a/build-tools/build-avoidance-utils.sh b/build-tools/build-avoidance-utils.sh new file mode 100644 index 00000000..d728c760 --- /dev/null +++ b/build-tools/build-avoidance-utils.sh @@ -0,0 +1,842 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Functions related to build avoidance. +# +# Do not call directly. Used by build-pkgs. +# +# Build avoidance downloads rpm, src.rpm and other artifacts of +# build-pkgs for a local reference build. The reference would +# typically be an automated build run atleast daily. +# The MY_WORKSPACE directory for the reference build shall have +# a common root directory, and a leaf directory that is a time stamp +# in a sortable parsable format. Default YYYYMMDDThhmmssZ. +# e.g. /localdisk/loadbuild/jenkins/StarlingX/20180719T113021Z +# +# Other formats can be used by setting the following variables +# in $MY_REPO/local-build-data/build_avoidance_source. +# e.g. to allow format YYYY-MM-DD_hh-mm-ss +# BUILD_AVOIDANCE_DATE_FORMAT="%Y-%m-%d" +# BUILD_AVOIDANCE_TIME_FORMAT="%H-%M-%S" +# BUILD_AVOIDANCE_DATE_TIME_DELIM="_" +# BUILD_AVOIDANCE_DATE_TIME_POSTFIX="" +# +# Note: Must be able to rsync and ssh to the machine that holds the +# reference builds. +# +# In future alternative transfer protocols may be supported. +# Select the alternate protocol by setting the following variables +# in $MY_REPO/local-build-data/build_avoidance_source. +# e.g. +# BUILD_AVOIDANCE_FILE_TRANSFER="my-supported-prototcol" +# + +BUILD_AVOIDANCE_USR="" +BUILD_AVOIDANCE_HOST="" +BUILD_AVOIDANCE_DIR="" +BUILD_AVOIDANCE_URL="" + +# Default date/time format, iso-8601 compact, 20180912T143913Z +# Syntax is a subset of that use by the unix 'date' command. +BUILD_AVOIDANCE_DATE_FORMAT="%Y%m%d" +BUILD_AVOIDANCE_TIME_FORMAT="%H%M%S" +BUILD_AVOIDANCE_DATE_TIME_DELIM="T" +BUILD_AVOIDANCE_DATE_TIME_POSTFIX="Z" + +# Default file transfer method +BUILD_AVOIDANCE_FILE_TRANSFER="rsync" + +# Default is to use timestamps and days in UTC +# +# If you prefer local time, then set 'BUILD_AVOIDANCE_DATE_UTC=0' +# in '$MY_REPO/local-build-data/build_avoidance_source' +BUILD_AVOIDANCE_DATE_UTC=1 + +BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data" +BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source" +BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source" +BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context" +BUILD_AVOIDANCE_LAST_SYNC_FILE="$BUILD_AVOIDANCE_DATA_DIR/last_sync_context" + +if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then + echo "Couldn't read $BUILD_AVOIDANCE_SOURCE" + exit 1 +fi + +echo "Reading: $BUILD_AVOIDANCE_SOURCE" +source $BUILD_AVOIDANCE_SOURCE + +if [ -f $BUILD_AVOIDANCE_LOCAL_SOURCE ]; then + echo "Reading: $BUILD_AVOIDANCE_LOCAL_SOURCE" + source $BUILD_AVOIDANCE_LOCAL_SOURCE +fi + +UTC="" + +if [ $BUILD_AVOIDANCE_DATE_UTC -eq 1 ]; then + UTC="--utc" +fi + + +if [ "x$BUILD_AVOIDANCE_OVERRIDE_DIR" != "x" ]; then + BUILD_AVOIDANCE_DIR="$BUILD_AVOIDANCE_OVERRIDE_DIR" +fi + +if [ "x$BUILD_AVOIDANCE_OVERRIDE_HOST" != "x" ]; then + BUILD_AVOIDANCE_HOST="$BUILD_AVOIDANCE_OVERRIDE_HOST" +fi + +if [ "x$BUILD_AVOIDANCE_OVERRIDE_USR" != "x" ]; then + BUILD_AVOIDANCE_USR="$BUILD_AVOIDANCE_OVERRIDE_USR" +fi + +echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR" +echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST" +echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR" + +build_avoidance_clean () { + if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then + \rm -f -v "$BUILD_AVOIDANCE_LAST_SYNC_FILE" + fi +} + + +date_to_iso_8601 () { + local DATE="$1" + local CENTURY="" + local YEAR_IN_CENTURY="00" + local MONTH="01" + local DAY="01" + local DAY_OF_YEAR="" + + CENTURY="$(date '+%C')" + + for x in $(echo "${BUILD_AVOIDANCE_DATE_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do + # Consume format case options + case ${x:0:1} in + ^) x=${x:1};; + \#) x=${x:1};; + *) ;; + esac + + # Process format + case $x in + Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:1};; + 0Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; + _Y*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};; + + y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; + 0y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; + _y*) YEAR_IN_CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; + + C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; + 0C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; + _C*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; + + m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; + 0m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; + _m*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; + e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:1};; + 0e*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; + _e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; + b*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; + h*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};; + + d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};; + 0d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};; + _d*) DAY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};; + + j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:1};; + 0j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:2};; + _j*) DAY_OF_YEAR=$(echo "${DATE:0:3}" | tr ' ' '0'); DATE=${DATE:3}; x=${x:2};; + + D*) MONTH=${DATE:0:2}; DAY=${DATE:3:2}; YEAR_IN_CENTURY=${DATE:6:2}; DATE=${DATE:8}; x=${x:1};; + F*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; MONTH=${DATE:5:2}; DAY=${DATE:8:2}; DATE=${DATE:10}; x=${x:1};; + *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported date format: ${BUILD_AVOIDANCE_DATE_FORMAT}"; return 1;; + esac + + # consume remaing non-interpreted content + if [ "$(echo "${DATE:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then + >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${DATE:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_DATE_FORMAT}'" + return 1 + fi + DATE=${DATE:${#x}} + done + + if [ "${DAY_OF_YEAR}" != "" ]; then + local YEAR_SEC + local DOY_SEC + YEAR_SEC="$(date -d "${CENTURY}${YEAR_IN_CENTURY}-01-01" '+%s')" + DOY_SEC=$((YEAR_SEC+(DAY_OF_YEAR-1)*24*60*60)) + MONTH="$(date "@$DOY_SEC" "+%m")" + DAY="$(date "@$DOY_SEC" "+%d")" + fi + + echo "${CENTURY}${YEAR_IN_CENTURY}-${MONTH}-${DAY}" + return 0 +} + +time_to_iso_8601 () { + TIME="$1" + local HOUR="00" + local H12="" + local AMPM="" + local MINUTE="00" + local SECOND="00" + + CENTURY="$(date '+%C')" + + for x in $(echo "${BUILD_AVOIDANCE_TIME_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do + # Consume format case options + case ${x:0:1} in + ^) x=${x:1};; + \#) x=${x:1};; + *) ;; + esac + + # Process format + case $x in + H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; + 0H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _H*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; + 0k*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + + I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; + 0I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _I*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};; + 0l*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + p*) AMPM=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; + + M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; + 0M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _M*) MINUTE="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + + S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};; + 0S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};; + _S*) SECOND="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};; + + R*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2} TIME=${TIME:5}; x=${x:1};; + r*) H12=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; AMPM=${TIME:9:2}; TIME=${TIME:11}; x=${x:1};; + T*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; TIME=${TIME:8}; x=${x:1};; + + *) >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}"; return 1;; + esac + + # consume remaing non-interpreted content + if [ "$(echo "${TIME:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then + >&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${TIME:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_TIME_FORMAT}'" + return 1 + fi + TIME=${TIME:${#x}} + done + + if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then + HOUR="$(date "$H12:01:01 $AMPM" '+%H')" + else + if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then + >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}" + return 1 + fi + fi + + echo "${HOUR}:${MINUTE}:${SECOND}" + return 0 +} + +date_time_to_iso_8601 () { + local DATE_TIME="$1" + local DATE + local TIME + local DECODED_DATE + local DECODED_TIME + DATE=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 1) + TIME=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 2 | sed "s#${BUILD_AVOIDANCE_DATE_TIME_POSTFIX}\$##") + DECODED_DATE=$(date_to_iso_8601 "${DATE}") + DECODED_TIME=$(time_to_iso_8601 "${TIME}") + echo "${DECODED_DATE}T${DECODED_TIME}$(date $UTC '+%:z')" +} + +# +# test_build_avoidance_context +# +# Is the provided context file compatible with the current +# state of all of our gits? A compatible context is one +# where every commit in the context file is visible in our +# current git history. +# +# Returns: Timestamp of context tested. +# Exit code: 0 = Compatible +# 1 = This context is older than the last applied +# build avoidance context. If you are searching +# newest to oldest, you might as well stop. +# 2 = Not compatible +# +test_build_avoidance_context () { + local context="$1" + local BA_LAST_SYNC_CONTEXT="$2" + local BA_CONTEXT="" + + BA_CONTEXT=$(basename $context | cut -d '.' -f 1) + >&2 echo "test: $BA_CONTEXT" + + if [ "$BA_CONTEXT" == "$BA_LAST_SYNC_CONTEXT" ]; then + # Stop the search. We've reached the last sync point + BA_CONTEXT="" + echo "$BA_CONTEXT" + return 1 + fi + + git_test_context "$context" + result=$? + if [ $result -eq 0 ]; then + # found a new context !!! + echo "$BA_CONTEXT" + return 0 + fi + + # Continue the search + BA_CONTEXT="" + echo "$BA_CONTEXT" + return 2 +} + + +# +# get_build_avoidance_context +# +# Return URL of the most recent jenkins build that is compatable with +# the current software context under $MY_REPO. +# +get_build_avoidance_context () { + ( + local context + local BA_CONTEXT="" + local BA_LAST_SYNC_CONTEXT="" + + # Load last synced context + if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then + BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE) + fi + + mkdir -p $BUILD_AVOIDANCE_DATA_DIR + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $BUILD_AVOIDANCE_DATA_DIR" + return 1 + fi + + local REMOTE_CTX_DIR="context" + local LOCAL_CTX_DIR="$BUILD_AVOIDANCE_DATA_DIR/context" + + # First copy the directory containing all the context files for + # the reference builds. + >&2 echo "Download latest reference build contexts" + + # Must set this prior to build_avoidance_copy_dir. + # The setting is not exported outside of the subshell. + BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR" + + build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$REMOTE_CTX_DIR' '$LOCAL_CTX_DIR'" + return 1 + fi + + # Search for a new context to sync + cd $MY_REPO + + if [ "$BUILD_AVOIDANCE_DAY" == "" ]; then + # Normal case: + # Search all contexts, newest to oldest, for a good context. + for context in $(ls -1rd $LOCAL_CTX_DIR/*context); do + >&2 echo "context=$context" + BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) + if [ $? -le 1 ]; then + # Stop search. Might or might not have found a good context. + break; + fi + done + else + # Special case when a target day is specified. Why would we do this? + # Reason is we might want the reference build to itself use build + # avoidance referencing prior builds of itself, except for one build + # a week when we use a full build rather than a build avoidance build. + # e.g. Sunday - full build + # Mon-Sat - avoidance builds that refernce Sunday build. + # + # Starting from last (e.g. "Sunday"), search newest to + # oldest for a good context. If none found, increment the target + # day (e.g. Monday) and search again. Keep incrementing until a + # good build is found, or target day + offset days would be a date + # in the furure. + # + local TARG_DAY=$BUILD_AVOIDANCE_DAY + local TODAY_DATE + local TODAY_DAY + local TARG_DATE="" + local TARG_TS + local TODAY_TS + + TODAY_DATE=$(date $UTC +%Y-%m-%d) + TODAY_DAY=$(date $UTC "+%A") + + for OFFSET_DAYS in 0 1 2 3 4 5 6; do + if [ "$TARG_DAY" != "" ]; then + # Convert TARG_DAY+OFFSET_DAYS to TARG_DATE + + if [ "$TODAY_DAY" == "$TARG_DAY" ]; then + TARG_DATE=$(date $UTC -d"$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) + else + TARG_DATE=$(date $UTC -d"last-$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d) + fi + >&2 echo "TARG_DATE=$TARG_DATE" + + TARG_TS=$(date $UTC -d "$TARG_DATE" +%s) + TODAY_TS=$(date $UTC -d "$TODAY_DATE" +%s) + if [ $TARG_TS -gt $TODAY_TS ]; then + # Skip if offset has pushed us into future dates + continue; + fi + + if [ "$TARG_DATE" == "$TODAY_DATE" ]; then + TARG_DATE="" + fi + fi + + # Search build, newest to oldest, satisfying TARG_DATE + for f in $(ls -1rd $LOCAL_CTX_DIR/*context); do + DATE=$(date_to_iso_8601 $(basename "$f")) + if [ $? -ne 0 ]; then + >&2 echo "Failed to extract date from filename '$(basename "$f")', ignoring file" + continue + fi + + >&2 echo " DATE=$DATE, TARG_DATE=$TARG_DATE" + + if [ "$DATE" == "$TARG_DATE" ] || [ "$TARG_DATE" == "" ] ; then + context=$f; + else + continue + fi + + >&2 echo "context=$context" + + BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT) + + if [ $? -le 1 ]; then + # Stop search. Might or might not have found a good context. + break; + fi + done + + if [ "$BA_CONTEXT" != "" ]; then + # Found a good context. + break + fi + done + fi + + if [ "$BA_CONTEXT" == "" ]; then + # No new context found + return 1 + fi + + # test that the reference build context hasn't been deleted + local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT" + >&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'" + if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then + return 1 + fi + + # Save the latest context + >&2 echo "BA_CONTEXT=$BA_CONTEXT" + >&2 echo "BUILD_AVOIDANCE_LAST_SYNC_FILE=$BUILD_AVOIDANCE_LAST_SYNC_FILE" + echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE + + # The location of the load with the most compatable new context + URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR + + # return URL to caller. + echo $URL + return 0 + ) +} + + +# +# build_avoidance_pre_clean +# +# A place for any cleanup actions that must preceed a build avoidance build. +# +build_avoidance_pre_clean () { + local BUILD_TYPE="$1" + + if [ "$BUILD_TYPE" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" + return 1 + fi + + # clean prior builds + if [ -d $MY_WORKSPACE/$BUILD_TYPE ]; then + build-pkgs --clean --$BUILD_TYPE --no-build-avoidance + if [ $? -ne 0 ]; then + return 1 + fi + fi + + for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do + if [ -f $MY_WORKSPACE/$BUILD_TYPE/$f ]; then + \rm -f $MY_WORKSPACE/$BUILD_TYPE/$f + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f $MY_WORKSPACE/$BUILD_TYPE/$f" + return 1 + fi + fi + done + + for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do + + if [ -d $MY_WORKSPACE/$BUILD_TYPE/$d ]; then + \rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d" + return 1 + fi + fi + done + + return 0 +} + + +# +# build_avoidance_copy_dir_rsync ['verbose'] +# +# Copy a file from $BUILD_AVOIDANCE_URL/ +# to using rsync. +# +build_avoidance_copy_dir_rsync () { + local FROM="$1" + local TO="$2" + local VERBOSE="$3" + local FLAGS="-a -u" + + if [ "$BUILD_AVOIDANCE_URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" + return 1 + fi + if [ "$VERBOSE" != "" ]; then + FLAGS="$FLAGS -v" + echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/*' '$TO/'" + fi + rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/*" "$TO/" + return $? +} + +# +# build_avoidance_copy_file_rsync ['verbose'] +# +# Copy a file from $BUILD_AVOIDANCE_URL/ +# to using rsync. +# +build_avoidance_copy_file_rsync () { + local FROM="$1" + local TO="$2" + local VERBOSE="$3" + local FLAGS="-a -u" + + if [ "$BUILD_AVOIDANCE_URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set" + return 1 + fi + if [ "$VERBOSE" != "" ]; then + FLAGS="$FLAGS -v" + echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'" + fi + rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM" "$TO" + return $? +} + +# +# build_avoidance_copy_dir ['verbose'] +# +# Copy a file from $BUILD_AVOIDANCE_URL/ +# to . The copy method will be determined by +# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. +# +# should be a directory, +# mkdir -p will be called on . +# +build_avoidance_copy_dir () { + local FROM="$1" + local TO="$2" + local VERBOSE="$3" + + if [ "$VERBOSE" != "" ]; then + echo "mkdir -p '$TO'" + fi + mkdir -p "$TO" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $TO" + return 1 + fi + + case ${BUILD_AVOIDANCE_FILE_TRANSFER} in + rsync) + build_avoidance_copy_dir_rsync "$FROM" "$TO" "$VERBOSE" + return $? + ;; + *) + >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" + return 1 + ;; + esac + return 1 +} + +# +# build_avoidance_copy_file ['verbose'] +# +# Copy a file from $BUILD_AVOIDANCE_URL/ +# to . The copy method will be determined by +# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present. +# +# should be a file, not a directory, +# mkdir -p will be called on $(dirname ) +# +build_avoidance_copy_file () { + local FROM="$1" + local TO="$2" + local VERBOSE="$3" + + if [ "$VERBOSE" != "" ]; then + echo "mkdir -p $(dirname '$TO')" + fi + mkdir -p "$(dirname "$TO")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $(dirname "$TO")" + return 1 + fi + + case ${BUILD_AVOIDANCE_FILE_TRANSFER} in + rsync) + build_avoidance_copy_file_rsync "$FROM" "$TO" "$VERBOSE" + return $? + ;; + *) + >&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'" + return 1 + ;; + esac + return 1 +} + +# +# build_avoidance_copy ['verbose'] +# +# Copy the needed build artifacts for from $BUILD_AVOIDANCE_URL. +# +build_avoidance_copy () { + local BUILD_TYPE="$1" + local VERBOSE="$2" + + if [ "$BUILD_TYPE" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" + return 1 + fi + + # Iterate through list of directories to copy + for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do + build_avoidance_copy_dir "$BUILD_TYPE/$d" "$MY_WORKSPACE/$BUILD_TYPE/$d" "$VERBOSE" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$BUILD_TYPE/$d' '$MY_WORKSPACE/$BUILD_TYPE/$d'" + return 1 + fi + done + + # Iterate through list of files to copy + for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do + build_avoidance_copy_file "$BUILD_TYPE/$f" "$MY_WORKSPACE/$BUILD_TYPE/$f" "$VERBOSE" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_file '$BUILD_TYPE/$f' '$MY_WORKSPACE/$BUILD_TYPE/$f'" + return 1 + fi + done + + return 0 +} + +# +# build_avoidance_fixups +# +# Fix paths in the build artifacts that we coppied that contain +# the user name. +# +# Also, our credentials may differ from the reference build, +# so substitute unsigned packages in place of signed packages. +# +build_avoidance_fixups () { + local BUILD_TYPE="$1" + + local BA_SOURCE_BUILD_ENVIRONMENT + BA_SOURCE_BUILD_ENVIRONMENT="${BUILD_AVOIDANCE_USR}-$(basename $(dirname $BUILD_AVOIDANCE_URL))-$(basename $BUILD_AVOIDANCE_URL)-${SRC_BUILD_ENVIRONMENT}" + local RESULT_DIR="" + local FROM_DIR="" + local TO_DIR="" + local rpm_path_post_signing + local rpm_path_pre_signing + local rpm_name + local md5sum_post_signing + local md5sum_pre_signing + + if [ "$BUILD_TYPE" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" + return 1 + fi + + RESULT_DIR="$MY_WORKSPACE/$BUILD_TYPE/results" + FROM_DIR="${RESULT_DIR}/${BA_SOURCE_BUILD_ENVIRONMENT}-${BUILD_TYPE}" + TO_DIR="${RESULT_DIR}/${MY_BUILD_ENVIRONMENT}-${BUILD_TYPE}" + echo "$FUNCNAME: FROM_DIR=$FROM_DIR" + echo "$FUNCNAME: TO_DIR=$TO_DIR" + echo "$FUNCNAME: MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT" + + # Fix patchs the use MY_BUILD_ENVIRONMENT + if [ ! -d "$FROM_DIR" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): Expected directory '$FROM_DIR' is missing." + return 1 + fi + + echo "$FUNCNAME: mv '$FROM_DIR' '$TO_DIR'" + \mv "$FROM_DIR" "$TO_DIR" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): mv '$FROM_DIR' '$TO_DIR'" + return 1 + fi + + local MY_WS_BT="$MY_WORKSPACE/$BUILD_TYPE" + + # Replace signed rpms with non-signed copies .... we aren't a formal build + for rpm_path_post_signing in $(find $MY_WS_BT/rpmbuild/RPMS -type f -name '*.rpm' | grep -v src.rpm); do + + rpm_name=$(basename $rpm_path_post_signing) + rpm_path_pre_signing=$(find $MY_WS_BT/results -name $b | head -n1) + if [ "$rpm_path_pre_signing" != "" ]; then + md5sum_post_signing=$(md5sum ${rpm_path_post_signing} | cut -d ' ' -f 1) + md5sum_pre_signing=$(md5sum ${rpm_path_pre_signing} | cut -d ' ' -f 1) + if [ "${md5sum_post_signing}" != "${md5sum_pre_signing}" ]; then + echo "$FUNCNAME: fixing $rpm_name" + \rm -f ${rpm_path_post_signing} + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): rm -f ${rpm_path_post_signing}" + return 1 + fi + + \cp ${rpm_path_pre_signing} ${rpm_path_post_signing} + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): cp ${rpm_path_pre_signing} ${rpm_path_post_signing}" + return 1 + fi + fi + fi; + done + + return 0 +} + + +# +# build_avoidance +# +# Look for a reference build that is applicable to our current git context. +# and copy it to our local workspace, if we haven't already done so. +# +build_avoidance () { + local BUILD_TYPE="$1" + + echo "==== Build Avoidance Start ====" + + if [ "$BUILD_TYPE" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required" + return 1 + fi + + if [ "$BUILD_TYPE" == "installer" ]; then + >&2 echo "build_avoidance: BUILD_TYPE==installer not supported" + return 1 + fi + + build_avoidance_pre_clean $BUILD_TYPE + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_pre_clean $BUILD_TYPE" + return 1 + fi + + build_avoidance_copy $BUILD_TYPE 'verbose' + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy $BUILD_TYPE" + return 1 + fi + + build_avoidance_fixups $BUILD_TYPE + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_fixups $BUILD_TYPE" + return 1 + fi + + echo "==== Build Avoidance Complete ====" + return 0 +} + +# +# build_avoidance_save_reference_context +# +# For use by a reference build. Copy the 'CONTEXT' file +# from the build into a central directory where we save +# the context of old builds. +# +# Individual reference builds use: +# MY_WORKSPACE=/ +# and context files are collected in dir: +# DEST_CTX_DIR=/context +# using name: +# DEST_CTX=.context + +build_avoidance_save_reference_context () { + local DIR + DIR=$(dirname "${MY_WORKSPACE}") + + # Note: SUB_DIR should be a timestamp + local SUB_DIR + SUB_DIR=$(basename "${MY_WORKSPACE}") + + local SRC_CTX="${MY_WORKSPACE}/CONTEXT" + local DEST_CTX_DIR="${DIR}/context" + local DEST_CTX="${DEST_CTX_DIR}/${SUB_DIR}.context" + + if [ ! -f "${SRC_CTX}" ]; then + echo "Context file not found at '${SRC_CTX}'" + return 1 + fi + + mkdir -p "${DEST_CTX_DIR}" + if [ $? -ne 0 ]; then + echo "Error: $FUNCNAME (${LINENO}): Failed to create directory '${DEST_CTX_DIR}'" + return 1 + fi + + cp "${SRC_CTX}" "${DEST_CTX}" + if [ $? -ne 0 ]; then + echo "Error: $FUNCNAME (${LINENO}): Failed to copy ${SRC_CTX} -> ${DEST_CTX}" + return 1 + fi + + return 0 +} diff --git a/build-tools/build-pkgs b/build-tools/build-pkgs index 9def3b01..15f7dd93 100755 --- a/build-tools/build-pkgs +++ b/build-tools/build-pkgs @@ -6,6 +6,9 @@ # SPDX-License-Identifier: Apache-2.0 # +# +# Build first src.rpms, then rpms, from source, or from a downloaded tarball +# or src.rpm plus our additional patches. # # This program is a wrapper around build-pkgs-parallel and build-pkgs-serial # diff --git a/build-tools/build-pkgs-parallel b/build-tools/build-pkgs-parallel index 02360ac0..7d7bb2cc 100755 --- a/build-tools/build-pkgs-parallel +++ b/build-tools/build-pkgs-parallel @@ -1,10 +1,22 @@ #!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Build first src.rpms, then rpms, from source, or from a downloaded tarball +# or src.rpm plus our additional patches. +# # This program is a wrapper around build-srpms-parallel and build-rpms-parallel +# BUILD_PKGS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_PKGS_PARALLEL_DIR}/git-utils.sh" +source "${BUILD_PKGS_PARALLEL_DIR}/build-avoidance-utils.sh" source "${BUILD_PKGS_PARALLEL_DIR}/spec-utils" @@ -13,14 +25,15 @@ usage () { echo "Usage: " echo " Create source and Binary rpms:" echo " Build optimizations (--no-descendants, --no-required, --no-build-info," - echo " --no-autoclean) are not recommended for the first build after a clone/pull," - echo " nor the final build prior to creating an iso or patch, but can be used" - echo " for intermediate builds. i.e. while debugging compilation failures." - echo " build-pkgs-parallel [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" + echo " --no-autoclean, --no-build-avoidance) are not recommended for the first build" + echo " after a clone/pull, nor the final build prior to creating an iso or patch," + echo " but can be used for intermediate builds." + echo " i.e. while debugging compilation failures." + echo " build-pkgs-parallel [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" echo "" echo " Delete source rpms, and the directories associated with it's creation:" echo " Note: does not clean an edit environment" - echo " build-pkgs-parallel --clean [ list of package names ]" + echo " build-pkgs-parallel --clean [--build-avoidance | --no-build-avoidance] [ list of package names ]" echo "" echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents," echo " one for source code and one for metadata such as the spec file." @@ -39,15 +52,20 @@ usage () { HELP=0 CLEAN_FLAG=0 EDIT_FLAG=0 +APPEND_LOG_FLAG=0 +BUILD_AVOIDANCE_FLAG=0 STD_BUILD=1 RT_BUILD=1 INSTALLER_BUILD=0 +export BUILD_AVOIDANCE_URL="" + + # read the options -TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs-parallel' -- "$@") +TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@") if [ $? -ne 0 ]; then usage - exit 0 + exit 1 fi eval set -- "$TEMP" @@ -55,8 +73,21 @@ eval set -- "$TEMP" EXTRA_ARGS_COMMON="" EXTRA_ARGS_SRPM="" EXTRA_ARGS_RPM="" + +export BUILD_AVOIDANCE_OVERRIDE_DIR="" +export BUILD_AVOIDANCE_OVERRIDE_HOST="" +export BUILD_AVOIDANCE_OVERRIDE_USR="" +export BUILD_AVOIDANCE_DAY="" + while true ; do case "$1" in + --append-log) APPEND_LOG_FLAG=1 ; shift ;; + --build-avoidance) BUILD_AVOIDANCE_FLAG=1 ; shift ;; + --no-build-avoidance) BUILD_AVOIDANCE_FLAG=0 ; shift ;; + --build-avoidance-dir) BUILD_AVOIDANCE_OVERRIDE_DIR=$2; shift 2 ;; + --build-avoidance-host) BUILD_AVOIDANCE_OVERRIDE_HOST=$2; shift 2 ;; + --build-avoidance-user) BUILD_AVOIDANCE_OVERRIDE_USR=$2; shift 2 ;; + --build-avoidance-day) BUILD_AVOIDANCE_DAY=$2; shift 2 ;; --no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;; --formal) EXTRA_ARGS_COMMON+=" --formal" ; shift ;; --careful) EXTRA_ARGS_RPM+=" --careful" ; shift ;; @@ -125,11 +156,15 @@ trap my_sighup HUP trap my_sigabrt ABRT trap my_sigterm TERM +# Note: For ease of parsing, a TARGETS list always begins and ends +# with a space. An empty target list consistes of two spaces. TARGETS=" $@ " -TARGETS_STD=" " -TARGETS_RT=" " -TARGETS_INSTALLER=" " -TARGETS_MISC=" " +EMPTY_TARGETS=" " + +TARGETS_STD="$EMPTY_TARGETS" +TARGETS_RT="$EMPTY_TARGETS" +TARGETS_INSTALLER="$EMPTY_TARGETS" +TARGETS_MISC="$EMPTY_TARGETS" find_targets () { local centos_pkg_dirs=$1 @@ -199,14 +234,28 @@ find_targets () { fi done - echo "$RESULT" + if [ "$RESULT" == " " ]; then + echo "$EMPTY_LIST" + else + echo "$RESULT" + fi return 0 } +if [ $EDIT_FLAG -eq 1 ] || [ "$TARGETS" != "$EMPTY_TARGETS" ]; then + BUILD_AVOIDANCE_FLAG=0 +fi + +echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG" echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" -if [ "x$TARGETS" != "x " ]; then +if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then + # Build Avoidance requested. Get URL of a usable context, if any. + export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context) +fi + +if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then TARGETS_STD="$(find_targets centos_pkg_dirs)" BUILD_TYPE_SAVE="$BUILD_TYPE" @@ -247,44 +296,50 @@ echo "EXTRA_ARGS_RPM='$EXTRA_ARGS_RPM'" echo "TARGETS='$TARGETS'" echo "TARGETS_STD='$TARGETS_STD'" echo "TARGETS_RT='$TARGETS_RT'" +echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'" echo "TARGETS_MISC='$TARGETS_MISC'" if [ $CLEAN_FLAG -eq 1 ]; then + + if [ "$TARGETS" == "$EMPTY_TARGETS" ] && [ $BUILD_AVOIDANCE_FLAG -eq 1 ] ; then + build_avoidance_clean + fi + if [ $STD_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi if [ $RT_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi if [ $INSTALLER_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER" ${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER" ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1 fi @@ -317,13 +372,22 @@ function launch_build() fi echo "Launching $build_type build, logging to $logfile" - \rm $logfile + if [ $APPEND_LOG_FLAG -eq 0 ] && [ -f $logfile ]; then + \rm $logfile + fi + echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile + + echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile + if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then + echo "build_avoidance $build_type" | tee --append $logfile + build_avoidance $build_type 2>&1 | tee --append $logfile + fi + # No clean flag, call build-srpms-parallel followed by build-rpms-parallel echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile ${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile - rc=${PIPESTATUS[0]} if [ $rc -eq 0 ]; then echo -e "\n######## $(date): build-srpm-parallel --$build_type was successful" | tee --append $logfile @@ -371,19 +435,18 @@ fi echo "Capture build context" git_context > "${MY_WORKSPACE}/CONTEXT" - if [ $STD_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build std else - echo "Skipping 'std' build, no valid targets in list: $TARGETS" + echo "Skipping 'std' build, no valid targets in list: '$TARGETS'" fi else echo "Skipping 'std' build" fi if [ $RT_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build rt else echo "Skipping 'rt' build, no valid targets in list: $TARGETS" @@ -393,7 +456,7 @@ else fi if [ $INSTALLER_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build installer else echo "Skipping 'installer' build, no valid targets in list: $TARGETS" @@ -402,6 +465,15 @@ else echo "Skipping 'installer' build" fi +# Make sure REFERENCE_BUILD is set to something +if [ -z $REFERENCE_BUILD ]; then + REFERENCE_BUILD=0 +fi + +if [ $REFERENCE_BUILD -eq 1 ]; then + echo "Saving reference context" + build_avoidance_save_reference_context +fi echo "All builds were successful" diff --git a/build-tools/build-pkgs-serial b/build-tools/build-pkgs-serial index a0061d59..103ef537 100755 --- a/build-tools/build-pkgs-serial +++ b/build-tools/build-pkgs-serial @@ -1,10 +1,22 @@ #!/bin/bash -# This program is a wrapper around build-srpms and build-rpms +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Build first src.rpms, then rpms, from source, or from a downloaded tarball +# or src.rpm plus our additional patches. +# +# This program is a wrapper around build-srpms-serial and build-rpms-serial +# BUILD_PKGS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_PKGS_SERIAL_DIR}/git-utils.sh" +source "${BUILD_PKGS_SERIAL_DIR}/build-avoidance-utils.sh" source "${BUILD_PKGS_SERIAL_DIR}/spec-utils" @@ -13,14 +25,15 @@ usage () { echo "Usage: " echo " Create source and Binary rpms:" echo " Build optimizations (--no-descendants, --no-required, --no-build-info," - echo " --no-autoclean) are not recommended for the first build after a clone/pull," - echo " nor the final build prior to creating an iso or patch, but can be used" - echo " for intermediate builds. i.e. while debugging compilation failures." - echo " build-pkgs-serial [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" + echo " --no-autoclean, --no-build-avoidance) are not recommended for the first build" + echo " after a clone/pull, nor the final build prior to creating an iso or patch," + echo " but can be used for intermediate builds." + echo " i.e. while debugging compilation failures." + echo " build-pkgs-serial [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]" echo "" echo " Delete source rpms, and the directories associated with it's creation:" echo " Note: does not clean an edit environment" - echo " build-pkgs-serial --clean [ list of package names ]" + echo " build-pkgs-serial --clean [--build-avoidance | --no-build-avoidance] [ list of package names ]" echo "" echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents," echo " one for source code and one for metadata such as the spec file." @@ -39,12 +52,17 @@ usage () { HELP=0 CLEAN_FLAG=0 EDIT_FLAG=0 +APPEND_LOG_FLAG=0 +BUILD_AVOIDANCE_FLAG=0 STD_BUILD=1 RT_BUILD=1 INSTALLER_BUILD=0 +export BUILD_AVOIDANCE_URL="" + + # read the options -TEMP=$(getopt -o h --long serial,rt,std,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs' -- "$@") +TEMP=$(getopt -o h --long serial,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -55,8 +73,21 @@ eval set -- "$TEMP" EXTRA_ARGS_COMMON="" EXTRA_ARGS_SRPM="" EXTRA_ARGS_RPM="" + +export BUILD_AVOIDANCE_OVERRIDE_DIR="" +export BUILD_AVOIDANCE_OVERRIDE_HOST="" +export BUILD_AVOIDANCE_OVERRIDE_USR="" +export BUILD_AVOIDANCE_DAY="" + while true ; do case "$1" in + --append-log) APPEND_LOG_FLAG=1 ; shift ;; + --build-avoidance) BUILD_AVOIDANCE_FLAG=1 ; shift ;; + --no-build-avoidance) BUILD_AVOIDANCE_FLAG=0 ; shift ;; + --build-avoidance-dir) BUILD_AVOIDANCE_OVERRIDE_DIR=$2; shift 2 ;; + --build-avoidance-host) BUILD_AVOIDANCE_OVERRIDE_HOST=$2; shift 2 ;; + --build-avoidance-user) BUILD_AVOIDANCE_OVERRIDE_USR=$2; shift 2 ;; + --build-avoidance-day) BUILD_AVOIDANCE_DAY=$2; shift 2 ;; --no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;; --formal) EXTRA_ARGS_COMMON+=" --formal" ; shift ;; --careful) EXTRA_ARGS_RPM+=" --careful" ; shift ;; @@ -81,11 +112,59 @@ if [ $HELP -eq 1 ]; then exit 0 fi +function my_exit() { + build-rpms-parallel --std --tmpfs-clean + build-rpms-parallel --rt --tmpfs-clean +} + +function my_sigint() { + echo "build-pkgs-parallel sigint" + pkill -SIGABRT -P $BASHPID &> /dev/null + echo "build-pkgs-parallel waiting" + wait + echo "build-pkgs-parallel wait complete" + +} + +function my_sighup() { + echo "build-pkgs-parallel sighup" + pkill -SIGABRT -P $BASHPID &> /dev/null + echo "build-pkgs-parallel waiting" + wait + echo "build-pkgs-parallel wait complete" +} + +function my_sigabrt() { + echo "build-pkgs-parallel sigabrt" + pkill -SIGABRT -P $BASHPID &> /dev/null + echo "build-pkgs-parallel waiting" + wait + echo "build-pkgs-parallel wait complete" +} + +function my_sigterm() { + echo "build-pkgs-parallel sigterm" + pkill -SIGABRT -P $BASHPID &> /dev/null + echo "build-pkgs-parallel waiting" + wait + echo "build-pkgs-parallel wait complete" +} + +trap my_exit EXIT +trap my_sigint INT +trap my_sighup HUP +trap my_sigabrt ABRT +trap my_sigterm TERM + +# Note: For ease of parsing, a TARGETS list always begins and ends +# with a space. An empty target list consistes of two spaces. TARGETS=" $@ " -TARGETS_STD=" " -TARGETS_RT=" " -TARGETS_INSTALLER=" " -TARGETS_MISC=" " +EMPTY_TARGETS=" " + +TARGETS_STD="$EMPTY_TARGETS" +TARGETS_RT="$EMPTY_TARGETS" +TARGETS_INSTALLER="$EMPTY_TARGETS" +TARGETS_MISC="$EMPTY_TARGETS" find_targets () { local centos_pkg_dirs=$1 @@ -155,14 +234,28 @@ find_targets () { fi done - echo "$RESULT" + if [ "$RESULT" == " " ]; then + echo "$EMPTY_LIST" + else + echo "$RESULT" + fi return 0 } +if [ $EDIT_FLAG -eq 1 ] || [ "$TARGETS" != "$EMPTY_TARGETS" ]; then + BUILD_AVOIDANCE_FLAG=0 +fi + +echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG" echo "CLEAN_FLAG=$CLEAN_FLAG" echo "EDIT_FLAG=$EDIT_FLAG" -if [ "x$TARGETS" != "x " ]; then +if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then + # Build Avoidance requested. Get URL of a usable context, if any. + export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context) +fi + +if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then TARGETS_STD="$(find_targets centos_pkg_dirs)" BUILD_TYPE_SAVE="$BUILD_TYPE" @@ -207,45 +300,51 @@ echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'" echo "TARGETS_MISC='$TARGETS_MISC'" if [ $CLEAN_FLAG -eq 1 ]; then + + if [ "$TARGETS" == "$EMPTY_TARGETS" ] && [ $BUILD_AVOIDANCE_FLAG -eq 1 ] ; then + build_avoidance_clean + fi + if [ $STD_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC" ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC" ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1 fi fi if [ $RT_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x" ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC" ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC" ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1 fi fi if [ $INSTALLER_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then if [ $EDIT_FLAG -ne 1 ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER" ${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1 fi fi - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER" ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1 fi fi + exit $? fi @@ -273,8 +372,18 @@ function launch_build() fi echo "Launching $build_type build, logging to $logfile" + if [ $APPEND_LOG_FLAG -eq 0 ] && [ -f $logfile ]; then + \rm $logfile + fi echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile + + echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile + if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then + echo "build_avoidance $build_type" | tee --append $logfile + build_avoidance $build_type 2>&1 | tee --append $logfile + fi + # No clean flag, call build-srpms-serial followed by build-rpms-serial echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile ${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile @@ -326,7 +435,7 @@ echo "Capture build context" git_context > "${MY_WORKSPACE}/CONTEXT" if [ $STD_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build std else echo "Skipping 'std' build, no valid targets in list: $TARGETS" @@ -336,7 +445,7 @@ else fi if [ $RT_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build rt else echo "Skipping 'rt' build, no valid targets in list: $TARGETS" @@ -346,7 +455,7 @@ else fi if [ $INSTALLER_BUILD -eq 1 ]; then - if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then + if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then launch_build installer else echo "Skipping 'installer' build, no valid targets in list: $TARGETS" @@ -355,6 +464,16 @@ else echo "Skipping 'installer' build" fi +# Make sure REFERENCE_BUILD is set to something +if [ -z $REFERENCE_BUILD ]; then + REFERENCE_BUILD=0 +fi + +if [ $REFERENCE_BUILD -eq 1 ]; then + echo "Saving reference context" + build_avoidance_save_reference_context +fi + echo "All builds were successful" exit 0 diff --git a/build-tools/build-rpms b/build-tools/build-rpms index 937c406a..1867743e 100755 --- a/build-tools/build-rpms +++ b/build-tools/build-rpms @@ -1,6 +1,16 @@ #!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Builds rpm files from src.rpm files. +# # This program is a wrapper around build-rpms-parallel and build-rpms-serial +# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index 17234913..f56e9782 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -6,6 +6,24 @@ # SPDX-License-Identifier: Apache-2.0 # +# +# Builds rpm files from src.rpm files. +# +# This version compiles packages in parrallel if sufficient resources +# (cpu's and memory) are available. +# +# The location of packages to be built is +# $MY_WORKSPACE//rpmbuild/SRPMS. +# +# The build order is a derived from the BuildRequires in the +# spec files in the src.rpms. Note that the BuildRequires sometimes +# create dependency loops, so no correct order can be computed. In these +# cases we add a retry loop. As long as one new package builds, we +# keep retrying the loop, until all are built, or no progress is made. +# So please don't panic and CTRL-C just because you see a few error +# messages go by! +# + export ME=$(basename "$0") CMDLINE="$ME $@" @@ -122,12 +140,10 @@ create-no-clean-list () { if [ ! -f $NO_CLEAN_LIST_FILE ]; then NEED_REBUILD=1 else - if [ -f MY_BUILD_CFG ]; then - if [ -f MY_BUILD_CFG ]; then - find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG) - if [ $? -eq 0 ]; then - NEED_REBUILD=1 - fi + if [ -f $MY_BUILD_CFG ]; then + find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG) + if [ $? -eq 0 ]; then + NEED_REBUILD=1 fi fi fi diff --git a/build-tools/build-rpms-serial b/build-tools/build-rpms-serial index 77cece6f..6358e586 100755 --- a/build-tools/build-rpms-serial +++ b/build-tools/build-rpms-serial @@ -6,6 +6,22 @@ # SPDX-License-Identifier: Apache-2.0 # +# +# Builds rpm files from src.rpm files. +# +# This version compiles one package at a time. +# +# The location of packages to be built is +# $MY_WORKSPACE//rpmbuild/SRPMS. +# +# The build order is a derived from the BuildRequires in the +# spec files in the src.rpms. Note that the BuildRequires sometimes +# create dependency loops, so no correct order can be computed. In these +# cases we add a retry loop. As long as one new package builds, we +# keep retrying the loop, until all are built, or no progress is made. +# So please don't panic and CTRL-C just because you see a few error +# messages go by! +# export ME=$(basename "$0") CMDLINE="$ME $@" @@ -63,12 +79,10 @@ create-no-clean-list () { if [ ! -f $NO_CLEAN_LIST_FILE ]; then NEED_REBUILD=1 else - if [ -f MY_BUILD_CFG ]; then - if [ -f MY_BUILD_CFG ]; then - find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG) - if [ $? -eq 0 ]; then - NEED_REBUILD=1 - fi + if [ -f $MY_BUILD_CFG ]; then + find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG) + if [ $? -eq 0 ]; then + NEED_REBUILD=1 fi fi fi diff --git a/build-tools/build-srpms b/build-tools/build-srpms index aef8a681..c0f1bafc 100755 --- a/build-tools/build-srpms +++ b/build-tools/build-srpms @@ -1,6 +1,24 @@ #!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Create src.rpm files from source, or from a downloaded tarball +# or src.rpm plus our additional patches. +# # This program is a wrapper around build-srpms-parallel and build-srpms-serial +# +# The location of packages to be build are identified by +# _pkg_dirs[_] files located at the root of +# any git tree (e.g. istx/stx-integ/centos_pkg_dirs). +# +# The build of an individual package is driven by its build_srpm.data +# file plus a .spec file or an srpm_path file. +# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/build-tools/build-srpms-common.sh b/build-tools/build-srpms-common.sh new file mode 100644 index 00000000..2afce635 --- /dev/null +++ b/build-tools/build-srpms-common.sh @@ -0,0 +1,187 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Functions common to build-srpm-serial and build-srpm-parallel. +# + +SRC_BUILD_TYPE_SRPM="srpm" +SRC_BUILD_TYPE_SPEC="spec" +SRC_BUILD_TYPES="$SRC_BUILD_TYPE_SRPM $SRC_BUILD_TYPE_SPEC" + + +str_lst_contains() { + TARGET="$1" + LST="$2" + + if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then + return 0 + else + return 1 + fi +} + + +# +# capture_md5sum_from_input_vars +# +# Returns md5 data for all input files of a src.rpm. +# Assumes PKG_BASE, ORIG_SRPM_PATH have been defined and the +# build_srpm.data file has already been sourced. +# +# Arguments: +# src-build-type: Any single value from $SRC_BUILD_TYPES. +# e.g. 'srpm' or 'spec' +# srpm-or-spec-path: Absolute path to an src.rpm, or to a +# spec file. +# work-dir: Optional working directory. If a path is +# specified but does not exist, it will be created. +# +# Returns: output of md5sum command with canonical path names +# +md5sums_from_input_vars () { + local SRC_BUILD_TYPE="$1" + local SRPM_OR_SPEC_PATH="$2" + local WORK_DIR="$3" + + local TMP_FLAG=0 + local LINK_FILTER='[/]stx[/]downloads[/]' + + if ! str_lst_contains "$SRC_BUILD_TYPE" "$SRC_BUILD_TYPES" ; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): invalid arg: SRC_BUILD_TYPE='$SRC_BUILD_TYPE'" + return 1 + fi + + if [ -z $WORK_DIR ]; then + WORK_DIR=$(mktemp -d /tmp/${FUNCNAME}_XXXXXX) + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): mktemp -d /tmp/${FUNCNAME}_XXXXXX" + return 1 + fi + TMP_FLAG=1 + else + mkdir -p "$WORK_DIR" + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): mkdir -p '$WORK_DIR'" + return 1 + fi + fi + + local INPUT_FILES="$WORK_DIR/srpm_input.files" + local INPUT_LINKS="$WORK_DIR/srpm_input.links" + local INPUT_FILES_SORTED="$WORK_DIR/srpm_sorted_input.files" + + if [ -f "$INPUT_LINKS" ]; then + \rm -f "$INPUT_LINKS" + fi + + # Create lists of input files (INPUT_FILES) and symlinks (INPUT_LINKS). + # First elements are absolute paths... + find "$PKG_BASE" -type f > $INPUT_FILES + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$PKG_BASE' -type f" + return 1 + fi + + if [ "$SRC_BUILD_TYPE" == "$SRC_BUILD_TYPE_SRPM" ]; then + find "$SRPM_OR_SPEC_PATH" -type f >> $INPUT_FILES + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$SRPM_OR_SPEC_PATH' -type f" + return 1 + fi + fi + + # ...additional elements are based on values already sourced from + # build_srpm.data (COPY_LIST, SRC_DIR, COPY_LIST_TO_TAR, OPT_DEP_LIST) + # and may be relative to $PKG_BASE + # + # Use a subshell so any directory changes have no lastin effect. + ( + cd $PKG_BASE + if [ "x$COPY_LIST" != "x" ]; then + ABS_COPY_LIST=$(readlink -f $COPY_LIST) + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): readlink -f '$COPY_LIST' -type f" + return 1 + fi + + find $ABS_COPY_LIST -type f >> $INPUT_FILES + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$ABS_COPY_LIST' -type f" + return 1 + fi + + # Treat most links normally + find $ABS_COPY_LIST -type l | grep -v "$LINK_FILTER" >> $INPUT_FILES + + # Links in the downloads directory likely point outside of $MY_REPO + # and might not be 'portable' from a build avoidance prespective. + # We'll treat these specially. + find $ABS_COPY_LIST -type l | grep "$LINK_FILTER" >> $INPUT_LINKS + fi + + if [ "$SRC_BUILD_TYPE" == "$SRC_BUILD_TYPE_SPEC" ]; then + if [ "x$SRC_DIR" != "x" ]; then + if [ -d "$SRC_DIR" ]; then + find $(readlink -f "$SRC_DIR") -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$SRC_DIR' -type f" + return 1 + fi + fi + fi + + if [ "x$COPY_LIST_TO_TAR" != "x" ]; then + find $(readlink -f "$COPY_LIST_TO_TAR") -type f >> $INPUT_FILES + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$COPY_LIST_TO_TAR' -type f" + return 1 + fi + fi + fi + + if [ "x$OPT_DEP_LIST" != "x" ]; then + find $(readlink -f "$OPT_DEP_LIST") -type f >> $INPUT_FILES 2> /dev/null + if [ $? -ne 0 ]; then + >&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$OPT_DEP_LIST' -type f" + return 1 + fi + fi + ) + if [ $? -eq 1 ]; then + return -1 + fi + + # Create sorted, unique list of cononical paths + ( + # Regular files, get canonical path + cat $INPUT_FILES | xargs readlink -f + + # A Symlink that likely points outside of $MY_REPO. + # get canonical path to the symlink itself, and not + # to what the symlink points to. + if [ -f $INPUT_LINKS ]; then + while IFS= read -r f; do + echo "$(readlink -f $(dirname $f))/$(basename $f)" + done < "$INPUT_LINKS" + fi + ) | sort --unique > $INPUT_FILES_SORTED + + # Remove $MY_REPO prefix from paths + cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" + + if [ $TMP_FLAG -eq 0 ]; then + \rm -f $INPUT_FILES_SORTED + \rm -f $INPUT_FILES + if [ -f $INPUT_LINKS ]; then + \rm -f $INPUT_LINKS + fi + else + \rm -rf $WORK_DIR + fi + + return 0 +} diff --git a/build-tools/build-srpms-parallel b/build-tools/build-srpms-parallel index 13fa4804..47e91e87 100755 --- a/build-tools/build-srpms-parallel +++ b/build-tools/build-srpms-parallel @@ -1,6 +1,26 @@ #!/bin/bash # set -x +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Create src.rpm files from source, or from a downloaded tarball +# or src.rpm plus our additional patches. +# +# This version tries to compile many packages in parallel. +# +# The location of packages to be build are identified by +# _pkg_dirs[_] files located at the root of +# any git tree (e.g. stx/stx-integ/centos_pkg_dirs). +# +# The build of an individul package is driven by it's build_srpm.data +# file plus a .spec file or an srpm_path file. +# + export ME=$(basename "$0") CMDLINE="$ME $@" @@ -9,6 +29,7 @@ source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh source $BUILD_SRPMS_PARALLEL_DIR/spec-utils source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils source $BUILD_SRPMS_PARALLEL_DIR/classify +source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh INITIAL_DIR=$(pwd) @@ -372,7 +393,7 @@ build_dir () { fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then - # handle mirror: definitions + # handle 3rd_party: definitions echo "$p" | grep "^3rd_party:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^3rd_party:%$THIRD_PARTY_ROOT/%") fi @@ -455,7 +476,7 @@ clean_srpm_dir () { local SRPM_OUT_NAME local INPUTS_TO_CLEAN="" - if [ $EXCLUDE_MD5 == "" ]; then + if [ "$EXCLUDE_MD5" == "" ]; then EXCLUDE_MD5=0 fi @@ -626,24 +647,15 @@ build_dir_srpm () { # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" - local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" - local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" - find $PKG_BASE -type f > $INPUT_FILES - find $ORIG_SRPM_PATH -type f >> $INPUT_FILES - ( - cd $PKG_BASE - if [ "x$COPY_LIST" != "x" ]; then - find $COPY_LIST -type f >> $INPUT_FILES - fi - ) - cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED - \rm -f $INPUT_FILES - cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 - \rm -f $INPUT_FILES_SORTED + md5sums_from_input_vars "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5" + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SRPM' '$ORIG_SRPM_PATH' '$TARGET_SOURCES_DIR'" + return 1 + fi echo "Wrote: $INPUT_FILES_MD5" # @@ -946,38 +958,15 @@ build_dir_spec () { # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" - local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" - local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" - find $PKG_BASE -type f > $INPUT_FILES - find $ORIG_SRPM_PATH -type f >> $INPUT_FILES - ( - cd $PKG_BASE - if [ "x$SRC_DIR" != "x" ]; then - if [ -d "$SRC_DIR" ]; then - find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES - fi - fi - - if [ "x$COPY_LIST" != "x" ]; then - find $COPY_LIST -type f >> $INPUT_FILES - fi - - if [ "x$COPY_LIST_TO_TAR" != "x" ]; then - find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES - fi - - if [ "x$OPT_DEP_LIST" != "x" ]; then - find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null - fi - ) - cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED - \rm -f $INPUT_FILES - cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 - \rm -f $INPUT_FILES_SORTED + md5sums_from_input_vars "$SRC_BUILD_TYPE_SPEC" "$SPEC" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5" + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SPEC' '$SPEC' '$TARGET_SOURCES_DIR'" + return 1 + fi echo "Wrote: $INPUT_FILES_MD5" # diff --git a/build-tools/build-srpms-serial b/build-tools/build-srpms-serial index 4398115b..fbdd33cb 100755 --- a/build-tools/build-srpms-serial +++ b/build-tools/build-srpms-serial @@ -1,6 +1,26 @@ #!/bin/bash # set -x +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Create src.rpm files from source, or from a downloaded tarball +# or src.rpm plus our additional patches. +# +# This version only tries to compile on package at a time. +# +# The location of packages to be build are identified by +# _pkg_dirs[_] files located at the root of +# any git tree (e.g. stx/stx-integ/centos_pkg_dirs). +# +# The build of an individul package is driven by it's build_srpm.data +# file plus a .spec file or an srpm_path file. +# + export ME=$(basename "$0") CMDLINE="$ME $@" @@ -10,6 +30,7 @@ source $BUILD_SRPMS_SERIAL_DIR/git-utils.sh source $BUILD_SRPMS_SERIAL_DIR/spec-utils source $BUILD_SRPMS_SERIAL_DIR/srpm-utils source $BUILD_SRPMS_SERIAL_DIR/classify +source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh INITIAL_DIR=$(pwd) @@ -438,7 +459,7 @@ clean_srpm_dir () { local SRPM_OUT_NAME local INPUTS_TO_CLEAN="" - if [ $EXCLUDE_MD5 == "" ]; then + if [ "$EXCLUDE_MD5" == "" ]; then EXCLUDE_MD5=0 fi @@ -603,24 +624,15 @@ build_dir_srpm () { # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" - local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" - local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" - find $PKG_BASE -type f > $INPUT_FILES - find $ORIG_SRPM_PATH -type f >> $INPUT_FILES - ( - cd $PKG_BASE - if [ "x$COPY_LIST" != "x" ]; then - find $COPY_LIST -type f >> $INPUT_FILES - fi - ) - cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED - \rm -f $INPUT_FILES - cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 - \rm -f $INPUT_FILES_SORTED + md5sums_from_input_vars "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5" + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SRPM' '$ORIG_SRPM_PATH' '$TARGET_SOURCES_DIR'" + return 1 + fi echo "Wrote: $INPUT_FILES_MD5" # @@ -922,38 +934,15 @@ build_dir_spec () { # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" - local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" - local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" - find $PKG_BASE -type f > $INPUT_FILES - find $ORIG_SRPM_PATH -type f >> $INPUT_FILES - ( - cd $PKG_BASE - if [ "x$SRC_DIR" != "x" ]; then - if [ -d "$SRC_DIR" ]; then - find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES - fi - fi - - if [ "x$COPY_LIST" != "x" ]; then - find $COPY_LIST -type f >> $INPUT_FILES - fi - - if [ "x$COPY_LIST_TO_TAR" != "x" ]; then - find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES - fi - - if [ "x$OPT_DEP_LIST" != "x" ]; then - find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null - fi - ) - cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED - \rm -f $INPUT_FILES - cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 - \rm -f $INPUT_FILES_SORTED + md5sums_from_input_vars "$SRC_BUILD_TYPE_SPEC" "$SPEC" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5" + if [ $? -ne 0 ]; then + echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SPEC' '$SPEC' '$TARGET_SOURCES_DIR'" + return 1 + fi echo "Wrote: $INPUT_FILES_MD5" # diff --git a/build-tools/git-utils.sh b/build-tools/git-utils.sh index 94490df1..de432464 100755 --- a/build-tools/git-utils.sh +++ b/build-tools/git-utils.sh @@ -9,6 +9,10 @@ # by repo manifests. # +git_ctx_root_dir () { + dirname "${MY_REPO}" +} + # # git_list : # Return a list of git root directories found under @@ -22,12 +26,12 @@ git_list () { # GIT_LIST: A list of root directories for all the gits under $MY_REPO/.. # as absolute paths. -export GIT_LIST=$(git_list "$(dirname "${MY_REPO}")") +export GIT_LIST=$(git_list "$(git_ctx_root_dir)") # GIT_LIST_REL: A list of root directories for all the gits under $MY_REPO/.. # as relative paths. -export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(dirname ${MY_REPO})}; done) +export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(git_ctx_root_dir)}; done) # @@ -85,7 +89,9 @@ git_list_containing_tag () { git_context () { ( - cd $MY_REPO + cd $(git_ctx_root_dir) + + local d for d in $GIT_LIST_REL; do ( cd ${d} @@ -95,3 +101,48 @@ git_context () { done ) } + +# +# git_test_context : +# +# Test if all commits referenced in the context are present +# in the history of the gits in their current checkout state. +# +# Returns: 0 = context is present in git history +# 1 = At least one element of context is not present +# 2 = error +# +git_test_context () { + local context="$1" + local query="" + local target_hits=0 + local actual_hits=0 + + if [ ! -f "$context" ]; then + return 2 + fi + + query=$(mktemp "/tmp/git_test_context_XXXXXX") + if [ "$query" == "" ]; then + return 2 + fi + + # Transform a checkout context into a query that prints + # all the commits that are found in the git history. + # + # Limit search to last 500 commits in the interest of speed. + # I don't expect to be using contexts more than a few weeks old. + cat "$context" | \ + sed "s#checkout -f \([a-e0-9]*\)#rev-list --max-count=500 HEAD | \ + grep \1#" > $query + + target_hits=$(cat "$context" | wc -l) + actual_hits=$(cd $(git_ctx_root_dir); source $query | wc -l) + \rm $query + + if [ $actual_hits -eq $target_hits ]; then + return 0 + fi + + return 1 +} diff --git a/build-tools/sign-secure-boot b/build-tools/sign-secure-boot index 4401e5cf..19a6c62f 100755 --- a/build-tools/sign-secure-boot +++ b/build-tools/sign-secure-boot @@ -1,5 +1,11 @@ #!/bin/bash +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # This script calls into an external signing server to perform signing of some # packages in the system. The old packages (which are typically generated by # the build system and signed by placeholder keys) are overwritten by the new @@ -264,7 +270,7 @@ function rebuild_pkgs fi echo "Performing rebuild of packages: $PKGS_TO_REBUILD" - FORMAL_BUILD=0 build-pkgs --no-descendants --no-build-info --no-required --careful $PKGS_TO_REBUILD > $LOGFILE 2>&1 + FORMAL_BUILD=0 build-pkgs --no-descendants --no-build-info --no-required --careful --append-log $PKGS_TO_REBUILD > $LOGFILE 2>&1 if [ $? -ne 0 ]; then echo "Could not rebuild packages: $PKGS_TO_REBUILD -- see $LOGFILE for details"