Merge "Build Avoidance"

This commit is contained in:
Zuul 2018-09-18 17:09:48 +00:00 committed by Gerrit Code Review
commit e8d8d774ec
15 changed files with 1491 additions and 151 deletions

1
.gitignore vendored
View File

@ -2,3 +2,4 @@
/cgcs-centos-repo
/cgcs-tis-repo
/cgcs-3rd-party-repo
/local-build-data

View File

@ -0,0 +1,23 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# What files and directories need to be copied
#
BUILD_AVOIDANCE_SRPM_DIRECTORIES="inputs srpm_assemble rpmbuild/SRPMS rpmbuild/SOURCES"
BUILD_AVOIDANCE_SRPM_FILES=""
BUILD_AVOIDANCE_RPM_DIRECTORIES="results rpmbuild/RPMS rpmbuild/SPECS"
BUILD_AVOIDANCE_RPM_FILES=".platform_release"
#
# Copy the lines below to $MY_REPO/local-build-data/build_avoidance_source,
# then uncomment and fill in the values giving the location of your local reference build.
#
# BUILD_AVOIDANCE_USR="jenkins"
# BUILD_AVOIDANCE_HOST="machine.corp.com"
# BUILD_AVOIDANCE_DIR="/localdisk/loadbuild/jenkins/StarlingX_Build"

View File

@ -0,0 +1,842 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Functions related to build avoidance.
#
# Do not call directly. Used by build-pkgs.
#
# Build avoidance downloads rpm, src.rpm and other artifacts of
# build-pkgs for a local reference build. The reference would
# typically be an automated build run atleast daily.
# The MY_WORKSPACE directory for the reference build shall have
# a common root directory, and a leaf directory that is a time stamp
# in a sortable parsable format. Default YYYYMMDDThhmmssZ.
# e.g. /localdisk/loadbuild/jenkins/StarlingX/20180719T113021Z
#
# Other formats can be used by setting the following variables
# in $MY_REPO/local-build-data/build_avoidance_source.
# e.g. to allow format YYYY-MM-DD_hh-mm-ss
# BUILD_AVOIDANCE_DATE_FORMAT="%Y-%m-%d"
# BUILD_AVOIDANCE_TIME_FORMAT="%H-%M-%S"
# BUILD_AVOIDANCE_DATE_TIME_DELIM="_"
# BUILD_AVOIDANCE_DATE_TIME_POSTFIX=""
#
# Note: Must be able to rsync and ssh to the machine that holds the
# reference builds.
#
# In future alternative transfer protocols may be supported.
# Select the alternate protocol by setting the following variables
# in $MY_REPO/local-build-data/build_avoidance_source.
# e.g.
# BUILD_AVOIDANCE_FILE_TRANSFER="my-supported-prototcol"
#
BUILD_AVOIDANCE_USR=""
BUILD_AVOIDANCE_HOST=""
BUILD_AVOIDANCE_DIR=""
BUILD_AVOIDANCE_URL=""
# Default date/time format, iso-8601 compact, 20180912T143913Z
# Syntax is a subset of that use by the unix 'date' command.
BUILD_AVOIDANCE_DATE_FORMAT="%Y%m%d"
BUILD_AVOIDANCE_TIME_FORMAT="%H%M%S"
BUILD_AVOIDANCE_DATE_TIME_DELIM="T"
BUILD_AVOIDANCE_DATE_TIME_POSTFIX="Z"
# Default file transfer method
BUILD_AVOIDANCE_FILE_TRANSFER="rsync"
# Default is to use timestamps and days in UTC
#
# If you prefer local time, then set 'BUILD_AVOIDANCE_DATE_UTC=0'
# in '$MY_REPO/local-build-data/build_avoidance_source'
BUILD_AVOIDANCE_DATE_UTC=1
BUILD_AVOIDANCE_DATA_DIR="$MY_WORKSPACE/build_avoidance_data"
BUILD_AVOIDANCE_SOURCE="$MY_REPO/build-data/build_avoidance_source"
BUILD_AVOIDANCE_LOCAL_SOURCE="$MY_REPO/local-build-data/build_avoidance_source"
BUILD_AVOIDANCE_TEST_CONTEXT="$BUILD_AVOIDANCE_DATA_DIR/test_context"
BUILD_AVOIDANCE_LAST_SYNC_FILE="$BUILD_AVOIDANCE_DATA_DIR/last_sync_context"
if [ ! -f $BUILD_AVOIDANCE_SOURCE ]; then
echo "Couldn't read $BUILD_AVOIDANCE_SOURCE"
exit 1
fi
echo "Reading: $BUILD_AVOIDANCE_SOURCE"
source $BUILD_AVOIDANCE_SOURCE
if [ -f $BUILD_AVOIDANCE_LOCAL_SOURCE ]; then
echo "Reading: $BUILD_AVOIDANCE_LOCAL_SOURCE"
source $BUILD_AVOIDANCE_LOCAL_SOURCE
fi
UTC=""
if [ $BUILD_AVOIDANCE_DATE_UTC -eq 1 ]; then
UTC="--utc"
fi
if [ "x$BUILD_AVOIDANCE_OVERRIDE_DIR" != "x" ]; then
BUILD_AVOIDANCE_DIR="$BUILD_AVOIDANCE_OVERRIDE_DIR"
fi
if [ "x$BUILD_AVOIDANCE_OVERRIDE_HOST" != "x" ]; then
BUILD_AVOIDANCE_HOST="$BUILD_AVOIDANCE_OVERRIDE_HOST"
fi
if [ "x$BUILD_AVOIDANCE_OVERRIDE_USR" != "x" ]; then
BUILD_AVOIDANCE_USR="$BUILD_AVOIDANCE_OVERRIDE_USR"
fi
echo "BUILD_AVOIDANCE_DIR=$BUILD_AVOIDANCE_DIR"
echo "BUILD_AVOIDANCE_HOST=$BUILD_AVOIDANCE_HOST"
echo "BUILD_AVOIDANCE_USR=$BUILD_AVOIDANCE_USR"
build_avoidance_clean () {
if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
\rm -f -v "$BUILD_AVOIDANCE_LAST_SYNC_FILE"
fi
}
date_to_iso_8601 () {
local DATE="$1"
local CENTURY=""
local YEAR_IN_CENTURY="00"
local MONTH="01"
local DAY="01"
local DAY_OF_YEAR=""
CENTURY="$(date '+%C')"
for x in $(echo "${BUILD_AVOIDANCE_DATE_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do
# Consume format case options
case ${x:0:1} in
^) x=${x:1};;
\#) x=${x:1};;
*) ;;
esac
# Process format
case $x in
Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:1};;
0Y*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};;
_Y*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); YEAR_IN_CENTURY=${DATE:2:2}; DATE=${DATE:4}; x=${x:2};;
y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};;
0y*) YEAR_IN_CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};;
_y*) YEAR_IN_CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};;
C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};;
0C*) CENTURY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};;
_C*) CENTURY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};;
m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};;
0m*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};;
_m*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};;
e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:1};;
0e*) MONTH=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};;
_e*) MONTH=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};;
b*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};;
h*) MONTH="$(date -d "${DATE:0:3} 1 2000" '+%m')"; DATE=${DATE:3}; x=${x:1};;
d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:1};;
0d*) DAY=${DATE:0:2}; DATE=${DATE:2}; x=${x:2};;
_d*) DAY=$(echo "${DATE:0:2}" | tr ' ' '0'); DATE=${DATE:2}; x=${x:2};;
j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:1};;
0j*) DAY_OF_YEAR=${DATE:0:3}; DATE=${DATE:3}; x=${x:2};;
_j*) DAY_OF_YEAR=$(echo "${DATE:0:3}" | tr ' ' '0'); DATE=${DATE:3}; x=${x:2};;
D*) MONTH=${DATE:0:2}; DAY=${DATE:3:2}; YEAR_IN_CENTURY=${DATE:6:2}; DATE=${DATE:8}; x=${x:1};;
F*) CENTURY=${DATE:0:2}; YEAR_IN_CENTURY=${DATE:2:2}; MONTH=${DATE:5:2}; DAY=${DATE:8:2}; DATE=${DATE:10}; x=${x:1};;
*) >&2 echo "$FUNCNAME (${LINENO}): Unsupported date format: ${BUILD_AVOIDANCE_DATE_FORMAT}"; return 1;;
esac
# consume remaing non-interpreted content
if [ "$(echo "${DATE:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then
>&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${DATE:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_DATE_FORMAT}'"
return 1
fi
DATE=${DATE:${#x}}
done
if [ "${DAY_OF_YEAR}" != "" ]; then
local YEAR_SEC
local DOY_SEC
YEAR_SEC="$(date -d "${CENTURY}${YEAR_IN_CENTURY}-01-01" '+%s')"
DOY_SEC=$((YEAR_SEC+(DAY_OF_YEAR-1)*24*60*60))
MONTH="$(date "@$DOY_SEC" "+%m")"
DAY="$(date "@$DOY_SEC" "+%d")"
fi
echo "${CENTURY}${YEAR_IN_CENTURY}-${MONTH}-${DAY}"
return 0
}
time_to_iso_8601 () {
TIME="$1"
local HOUR="00"
local H12=""
local AMPM=""
local MINUTE="00"
local SECOND="00"
CENTURY="$(date '+%C')"
for x in $(echo "${BUILD_AVOIDANCE_TIME_FORMAT}" | tr ' ' '#' | sed 's/%%/#/g' | tr '%' ' ' ); do
# Consume format case options
case ${x:0:1} in
^) x=${x:1};;
\#) x=${x:1};;
*) ;;
esac
# Process format
case $x in
H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};;
0H*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_H*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};;
0k*) HOUR=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_k*) HOUR="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};;
0I*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_I*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:1};;
0l*) H12=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_l*) H12="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
p*) AMPM=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};;
M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};;
0M*) MINUTE=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_M*) MINUTE="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:1};;
0S*) SECOND=${TIME:0:2}; TIME=${TIME:2}; x=${x:2};;
_S*) SECOND="$(echo "${TIME:0:2}" | tr ' ' '0')"; TIME=${TIME:2}; x=${x:2};;
R*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2} TIME=${TIME:5}; x=${x:1};;
r*) H12=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; AMPM=${TIME:9:2}; TIME=${TIME:11}; x=${x:1};;
T*) HOUR=${TIME:0:2}; MINUTE=${TIME:3:2}; SECOND=${TIME:6:2}; TIME=${TIME:8}; x=${x:1};;
*) >&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}"; return 1;;
esac
# consume remaing non-interpreted content
if [ "$(echo "${TIME:0:${#x}}" | tr ' ' '#')" != "${x}" ]; then
>&2 echo "$FUNCNAME (${LINENO}): Unexpected content '${TIME:0:${#x}}' does not match expected '${x}': '$1' being parsed vs '${BUILD_AVOIDANCE_TIME_FORMAT}'"
return 1
fi
TIME=${TIME:${#x}}
done
if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then
HOUR="$(date "$H12:01:01 $AMPM" '+%H')"
else
if [ "$H12" != "" ] && [ "$AMPM" != "" ]; then
>&2 echo "$FUNCNAME (${LINENO}): Unsupported time format: ${BUILD_AVOIDANCE_TIME_FORMAT}"
return 1
fi
fi
echo "${HOUR}:${MINUTE}:${SECOND}"
return 0
}
date_time_to_iso_8601 () {
local DATE_TIME="$1"
local DATE
local TIME
local DECODED_DATE
local DECODED_TIME
DATE=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 1)
TIME=$(echo "${DATE_TIME}" | cut -d ${BUILD_AVOIDANCE_DATE_TIME_DELIM} -f 2 | sed "s#${BUILD_AVOIDANCE_DATE_TIME_POSTFIX}\$##")
DECODED_DATE=$(date_to_iso_8601 "${DATE}")
DECODED_TIME=$(time_to_iso_8601 "${TIME}")
echo "${DECODED_DATE}T${DECODED_TIME}$(date $UTC '+%:z')"
}
#
# test_build_avoidance_context <path-to-context-file>
#
# Is the provided context file compatible with the current
# state of all of our gits? A compatible context is one
# where every commit in the context file is visible in our
# current git history.
#
# Returns: Timestamp of context tested.
# Exit code: 0 = Compatible
# 1 = This context is older than the last applied
# build avoidance context. If you are searching
# newest to oldest, you might as well stop.
# 2 = Not compatible
#
test_build_avoidance_context () {
local context="$1"
local BA_LAST_SYNC_CONTEXT="$2"
local BA_CONTEXT=""
BA_CONTEXT=$(basename $context | cut -d '.' -f 1)
>&2 echo "test: $BA_CONTEXT"
if [ "$BA_CONTEXT" == "$BA_LAST_SYNC_CONTEXT" ]; then
# Stop the search. We've reached the last sync point
BA_CONTEXT=""
echo "$BA_CONTEXT"
return 1
fi
git_test_context "$context"
result=$?
if [ $result -eq 0 ]; then
# found a new context !!!
echo "$BA_CONTEXT"
return 0
fi
# Continue the search
BA_CONTEXT=""
echo "$BA_CONTEXT"
return 2
}
#
# get_build_avoidance_context
#
# Return URL of the most recent jenkins build that is compatable with
# the current software context under $MY_REPO.
#
get_build_avoidance_context () {
(
local context
local BA_CONTEXT=""
local BA_LAST_SYNC_CONTEXT=""
# Load last synced context
if [ -f $BUILD_AVOIDANCE_LAST_SYNC_FILE ]; then
BA_LAST_SYNC_CONTEXT=$(head -n 1 $BUILD_AVOIDANCE_LAST_SYNC_FILE)
fi
mkdir -p $BUILD_AVOIDANCE_DATA_DIR
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $BUILD_AVOIDANCE_DATA_DIR"
return 1
fi
local REMOTE_CTX_DIR="context"
local LOCAL_CTX_DIR="$BUILD_AVOIDANCE_DATA_DIR/context"
# First copy the directory containing all the context files for
# the reference builds.
>&2 echo "Download latest reference build contexts"
# Must set this prior to build_avoidance_copy_dir.
# The setting is not exported outside of the subshell.
BUILD_AVOIDANCE_URL="$BUILD_AVOIDANCE_HOST:$BUILD_AVOIDANCE_DIR"
build_avoidance_copy_dir "$REMOTE_CTX_DIR" "$LOCAL_CTX_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$REMOTE_CTX_DIR' '$LOCAL_CTX_DIR'"
return 1
fi
# Search for a new context to sync
cd $MY_REPO
if [ "$BUILD_AVOIDANCE_DAY" == "" ]; then
# Normal case:
# Search all contexts, newest to oldest, for a good context.
for context in $(ls -1rd $LOCAL_CTX_DIR/*context); do
>&2 echo "context=$context"
BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT)
if [ $? -le 1 ]; then
# Stop search. Might or might not have found a good context.
break;
fi
done
else
# Special case when a target day is specified. Why would we do this?
# Reason is we might want the reference build to itself use build
# avoidance referencing prior builds of itself, except for one build
# a week when we use a full build rather than a build avoidance build.
# e.g. Sunday - full build
# Mon-Sat - avoidance builds that refernce Sunday build.
#
# Starting from last <TARG_DAY> (e.g. "Sunday"), search newest to
# oldest for a good context. If none found, increment the target
# day (e.g. Monday) and search again. Keep incrementing until a
# good build is found, or target day + offset days would be a date
# in the furure.
#
local TARG_DAY=$BUILD_AVOIDANCE_DAY
local TODAY_DATE
local TODAY_DAY
local TARG_DATE=""
local TARG_TS
local TODAY_TS
TODAY_DATE=$(date $UTC +%Y-%m-%d)
TODAY_DAY=$(date $UTC "+%A")
for OFFSET_DAYS in 0 1 2 3 4 5 6; do
if [ "$TARG_DAY" != "" ]; then
# Convert TARG_DAY+OFFSET_DAYS to TARG_DATE
if [ "$TODAY_DAY" == "$TARG_DAY" ]; then
TARG_DATE=$(date $UTC -d"$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d)
else
TARG_DATE=$(date $UTC -d"last-$TARG_DAY+$OFFSET_DAYS days" +%Y-%m-%d)
fi
>&2 echo "TARG_DATE=$TARG_DATE"
TARG_TS=$(date $UTC -d "$TARG_DATE" +%s)
TODAY_TS=$(date $UTC -d "$TODAY_DATE" +%s)
if [ $TARG_TS -gt $TODAY_TS ]; then
# Skip if offset has pushed us into future dates
continue;
fi
if [ "$TARG_DATE" == "$TODAY_DATE" ]; then
TARG_DATE=""
fi
fi
# Search build, newest to oldest, satisfying TARG_DATE
for f in $(ls -1rd $LOCAL_CTX_DIR/*context); do
DATE=$(date_to_iso_8601 $(basename "$f"))
if [ $? -ne 0 ]; then
>&2 echo "Failed to extract date from filename '$(basename "$f")', ignoring file"
continue
fi
>&2 echo " DATE=$DATE, TARG_DATE=$TARG_DATE"
if [ "$DATE" == "$TARG_DATE" ] || [ "$TARG_DATE" == "" ] ; then
context=$f;
else
continue
fi
>&2 echo "context=$context"
BA_CONTEXT=$(test_build_avoidance_context $context $BA_LAST_SYNC_CONTEXT)
if [ $? -le 1 ]; then
# Stop search. Might or might not have found a good context.
break;
fi
done
if [ "$BA_CONTEXT" != "" ]; then
# Found a good context.
break
fi
done
fi
if [ "$BA_CONTEXT" == "" ]; then
# No new context found
return 1
fi
# test that the reference build context hasn't been deleted
local BA_CONTEXT_DIR="$BUILD_AVOIDANCE_DIR/$BA_CONTEXT"
>&2 echo "ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]'"
if ! ssh $BUILD_AVOIDANCE_HOST '[ -d $BA_CONTEXT_DIR ]' ; then
return 1
fi
# Save the latest context
>&2 echo "BA_CONTEXT=$BA_CONTEXT"
>&2 echo "BUILD_AVOIDANCE_LAST_SYNC_FILE=$BUILD_AVOIDANCE_LAST_SYNC_FILE"
echo $BA_CONTEXT > $BUILD_AVOIDANCE_LAST_SYNC_FILE
# The location of the load with the most compatable new context
URL=$BUILD_AVOIDANCE_HOST:$BA_CONTEXT_DIR
# return URL to caller.
echo $URL
return 0
)
}
#
# build_avoidance_pre_clean <build-type>
#
# A place for any cleanup actions that must preceed a build avoidance build.
#
build_avoidance_pre_clean () {
local BUILD_TYPE="$1"
if [ "$BUILD_TYPE" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
return 1
fi
# clean prior builds
if [ -d $MY_WORKSPACE/$BUILD_TYPE ]; then
build-pkgs --clean --$BUILD_TYPE --no-build-avoidance
if [ $? -ne 0 ]; then
return 1
fi
fi
for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do
if [ -f $MY_WORKSPACE/$BUILD_TYPE/$f ]; then
\rm -f $MY_WORKSPACE/$BUILD_TYPE/$f
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): rm -f $MY_WORKSPACE/$BUILD_TYPE/$f"
return 1
fi
fi
done
for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do
if [ -d $MY_WORKSPACE/$BUILD_TYPE/$d ]; then
\rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): rm -rf $MY_WORKSPACE/$BUILD_TYPE/$d"
return 1
fi
fi
done
return 0
}
#
# build_avoidance_copy_dir_rsync <remote-dir-path-rel> <local-dir-path> ['verbose']
#
# Copy a file from $BUILD_AVOIDANCE_URL/<remote-dir-path-rel>
# to <local-dir-path> using rsync.
#
build_avoidance_copy_dir_rsync () {
local FROM="$1"
local TO="$2"
local VERBOSE="$3"
local FLAGS="-a -u"
if [ "$BUILD_AVOIDANCE_URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set"
return 1
fi
if [ "$VERBOSE" != "" ]; then
FLAGS="$FLAGS -v"
echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM/*' '$TO/'"
fi
rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM/*" "$TO/"
return $?
}
#
# build_avoidance_copy_file_rsync <remote-file-path-rel> <local-file-path> ['verbose']
#
# Copy a file from $BUILD_AVOIDANCE_URL/<remote-file-path-rel>
# to <local-file-path> using rsync.
#
build_avoidance_copy_file_rsync () {
local FROM="$1"
local TO="$2"
local VERBOSE="$3"
local FLAGS="-a -u"
if [ "$BUILD_AVOIDANCE_URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_AVOIDANCE_URL no set"
return 1
fi
if [ "$VERBOSE" != "" ]; then
FLAGS="$FLAGS -v"
echo "rsync $FLAGS '$BUILD_AVOIDANCE_URL/$FROM' '$TO'"
fi
rsync $FLAGS "$BUILD_AVOIDANCE_URL/$FROM" "$TO"
return $?
}
#
# build_avoidance_copy_dir <remote-dir-path-rel> <local-dir-path> ['verbose']
#
# Copy a file from $BUILD_AVOIDANCE_URL/<remote-dir-path-rel>
# to <local-dir-path>. The copy method will be determined by
# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present.
#
# <local-dir-path> should be a directory,
# mkdir -p will be called on <local-file-path>.
#
build_avoidance_copy_dir () {
local FROM="$1"
local TO="$2"
local VERBOSE="$3"
if [ "$VERBOSE" != "" ]; then
echo "mkdir -p '$TO'"
fi
mkdir -p "$TO"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $TO"
return 1
fi
case ${BUILD_AVOIDANCE_FILE_TRANSFER} in
rsync)
build_avoidance_copy_dir_rsync "$FROM" "$TO" "$VERBOSE"
return $?
;;
*)
>&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'"
return 1
;;
esac
return 1
}
#
# build_avoidance_copy_file <remote-file-path-rel> <local-file-path> ['verbose']
#
# Copy a file from $BUILD_AVOIDANCE_URL/<remote-file-path-rel>
# to <local-file-path>. The copy method will be determined by
# BUILD_AVOIDANCE_FILE_TRANSFER. Only 'rsync' is supported at present.
#
# <local-file-path> should be a file, not a directory,
# mkdir -p will be called on $(dirname <local-file-path>)
#
build_avoidance_copy_file () {
local FROM="$1"
local TO="$2"
local VERBOSE="$3"
if [ "$VERBOSE" != "" ]; then
echo "mkdir -p $(dirname '$TO')"
fi
mkdir -p "$(dirname "$TO")"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): mkdir -p $(dirname "$TO")"
return 1
fi
case ${BUILD_AVOIDANCE_FILE_TRANSFER} in
rsync)
build_avoidance_copy_file_rsync "$FROM" "$TO" "$VERBOSE"
return $?
;;
*)
>&2 echo "Error: $FUNCNAME (${LINENO}): Unknown BUILD_AVOIDANCE_FILE_TRANSFER '${BUILD_AVOIDANCE_FILE_TRANSFER}'"
return 1
;;
esac
return 1
}
#
# build_avoidance_copy <build-type> ['verbose']
#
# Copy the needed build artifacts for <build-type> from $BUILD_AVOIDANCE_URL.
#
build_avoidance_copy () {
local BUILD_TYPE="$1"
local VERBOSE="$2"
if [ "$BUILD_TYPE" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
return 1
fi
# Iterate through list of directories to copy
for d in $BUILD_AVOIDANCE_SRPM_DIRECTORIES $BUILD_AVOIDANCE_RPM_DIRECTORIES; do
build_avoidance_copy_dir "$BUILD_TYPE/$d" "$MY_WORKSPACE/$BUILD_TYPE/$d" "$VERBOSE"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_dir '$BUILD_TYPE/$d' '$MY_WORKSPACE/$BUILD_TYPE/$d'"
return 1
fi
done
# Iterate through list of files to copy
for f in $BUILD_AVOIDANCE_SRPM_FILES $BUILD_AVOIDANCE_RPM_FILES; do
build_avoidance_copy_file "$BUILD_TYPE/$f" "$MY_WORKSPACE/$BUILD_TYPE/$f" "$VERBOSE"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy_file '$BUILD_TYPE/$f' '$MY_WORKSPACE/$BUILD_TYPE/$f'"
return 1
fi
done
return 0
}
#
# build_avoidance_fixups <build-type>
#
# Fix paths in the build artifacts that we coppied that contain
# the user name.
#
# Also, our credentials may differ from the reference build,
# so substitute unsigned packages in place of signed packages.
#
build_avoidance_fixups () {
local BUILD_TYPE="$1"
local BA_SOURCE_BUILD_ENVIRONMENT
BA_SOURCE_BUILD_ENVIRONMENT="${BUILD_AVOIDANCE_USR}-$(basename $(dirname $BUILD_AVOIDANCE_URL))-$(basename $BUILD_AVOIDANCE_URL)-${SRC_BUILD_ENVIRONMENT}"
local RESULT_DIR=""
local FROM_DIR=""
local TO_DIR=""
local rpm_path_post_signing
local rpm_path_pre_signing
local rpm_name
local md5sum_post_signing
local md5sum_pre_signing
if [ "$BUILD_TYPE" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
return 1
fi
RESULT_DIR="$MY_WORKSPACE/$BUILD_TYPE/results"
FROM_DIR="${RESULT_DIR}/${BA_SOURCE_BUILD_ENVIRONMENT}-${BUILD_TYPE}"
TO_DIR="${RESULT_DIR}/${MY_BUILD_ENVIRONMENT}-${BUILD_TYPE}"
echo "$FUNCNAME: FROM_DIR=$FROM_DIR"
echo "$FUNCNAME: TO_DIR=$TO_DIR"
echo "$FUNCNAME: MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT"
# Fix patchs the use MY_BUILD_ENVIRONMENT
if [ ! -d "$FROM_DIR" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): Expected directory '$FROM_DIR' is missing."
return 1
fi
echo "$FUNCNAME: mv '$FROM_DIR' '$TO_DIR'"
\mv "$FROM_DIR" "$TO_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): mv '$FROM_DIR' '$TO_DIR'"
return 1
fi
local MY_WS_BT="$MY_WORKSPACE/$BUILD_TYPE"
# Replace signed rpms with non-signed copies .... we aren't a formal build
for rpm_path_post_signing in $(find $MY_WS_BT/rpmbuild/RPMS -type f -name '*.rpm' | grep -v src.rpm); do
rpm_name=$(basename $rpm_path_post_signing)
rpm_path_pre_signing=$(find $MY_WS_BT/results -name $b | head -n1)
if [ "$rpm_path_pre_signing" != "" ]; then
md5sum_post_signing=$(md5sum ${rpm_path_post_signing} | cut -d ' ' -f 1)
md5sum_pre_signing=$(md5sum ${rpm_path_pre_signing} | cut -d ' ' -f 1)
if [ "${md5sum_post_signing}" != "${md5sum_pre_signing}" ]; then
echo "$FUNCNAME: fixing $rpm_name"
\rm -f ${rpm_path_post_signing}
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): rm -f ${rpm_path_post_signing}"
return 1
fi
\cp ${rpm_path_pre_signing} ${rpm_path_post_signing}
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): cp ${rpm_path_pre_signing} ${rpm_path_post_signing}"
return 1
fi
fi
fi;
done
return 0
}
#
# build_avoidance <build-type>
#
# Look for a reference build that is applicable to our current git context.
# and copy it to our local workspace, if we haven't already done so.
#
build_avoidance () {
local BUILD_TYPE="$1"
echo "==== Build Avoidance Start ===="
if [ "$BUILD_TYPE" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): BUILD_TYPE required"
return 1
fi
if [ "$BUILD_TYPE" == "installer" ]; then
>&2 echo "build_avoidance: BUILD_TYPE==installer not supported"
return 1
fi
build_avoidance_pre_clean $BUILD_TYPE
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_pre_clean $BUILD_TYPE"
return 1
fi
build_avoidance_copy $BUILD_TYPE 'verbose'
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_copy $BUILD_TYPE"
return 1
fi
build_avoidance_fixups $BUILD_TYPE
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_fixups $BUILD_TYPE"
return 1
fi
echo "==== Build Avoidance Complete ===="
return 0
}
#
# build_avoidance_save_reference_context
#
# For use by a reference build. Copy the 'CONTEXT' file
# from the build into a central directory where we save
# the context of old builds.
#
# Individual reference builds use:
# MY_WORKSPACE=<common-dir>/<timestamp>
# and context files are collected in dir:
# DEST_CTX_DIR=<common-dir>/context
# using name:
# DEST_CTX=<timestamp>.context
build_avoidance_save_reference_context () {
local DIR
DIR=$(dirname "${MY_WORKSPACE}")
# Note: SUB_DIR should be a timestamp
local SUB_DIR
SUB_DIR=$(basename "${MY_WORKSPACE}")
local SRC_CTX="${MY_WORKSPACE}/CONTEXT"
local DEST_CTX_DIR="${DIR}/context"
local DEST_CTX="${DEST_CTX_DIR}/${SUB_DIR}.context"
if [ ! -f "${SRC_CTX}" ]; then
echo "Context file not found at '${SRC_CTX}'"
return 1
fi
mkdir -p "${DEST_CTX_DIR}"
if [ $? -ne 0 ]; then
echo "Error: $FUNCNAME (${LINENO}): Failed to create directory '${DEST_CTX_DIR}'"
return 1
fi
cp "${SRC_CTX}" "${DEST_CTX}"
if [ $? -ne 0 ]; then
echo "Error: $FUNCNAME (${LINENO}): Failed to copy ${SRC_CTX} -> ${DEST_CTX}"
return 1
fi
return 0
}

View File

@ -6,6 +6,9 @@
# SPDX-License-Identifier: Apache-2.0
#
#
# Build first src.rpms, then rpms, from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This program is a wrapper around build-pkgs-parallel and build-pkgs-serial
#

View File

@ -1,10 +1,22 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Build first src.rpms, then rpms, from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This program is a wrapper around build-srpms-parallel and build-rpms-parallel
#
BUILD_PKGS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_PKGS_PARALLEL_DIR}/git-utils.sh"
source "${BUILD_PKGS_PARALLEL_DIR}/build-avoidance-utils.sh"
source "${BUILD_PKGS_PARALLEL_DIR}/spec-utils"
@ -13,14 +25,15 @@ usage () {
echo "Usage: "
echo " Create source and Binary rpms:"
echo " Build optimizations (--no-descendants, --no-required, --no-build-info,"
echo " --no-autoclean) are not recommended for the first build after a clone/pull,"
echo " nor the final build prior to creating an iso or patch, but can be used"
echo " for intermediate builds. i.e. while debugging compilation failures."
echo " build-pkgs-parallel [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
echo " --no-autoclean, --no-build-avoidance) are not recommended for the first build"
echo " after a clone/pull, nor the final build prior to creating an iso or patch,"
echo " but can be used for intermediate builds."
echo " i.e. while debugging compilation failures."
echo " build-pkgs-parallel [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
echo ""
echo " Delete source rpms, and the directories associated with it's creation:"
echo " Note: does not clean an edit environment"
echo " build-pkgs-parallel --clean [ list of package names ]"
echo " build-pkgs-parallel --clean [--build-avoidance | --no-build-avoidance] [ list of package names ]"
echo ""
echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents,"
echo " one for source code and one for metadata such as the spec file."
@ -39,15 +52,20 @@ usage () {
HELP=0
CLEAN_FLAG=0
EDIT_FLAG=0
APPEND_LOG_FLAG=0
BUILD_AVOIDANCE_FLAG=0
STD_BUILD=1
RT_BUILD=1
INSTALLER_BUILD=0
export BUILD_AVOIDANCE_URL=""
# read the options
TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs-parallel' -- "$@")
TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 0
exit 1
fi
eval set -- "$TEMP"
@ -55,8 +73,21 @@ eval set -- "$TEMP"
EXTRA_ARGS_COMMON=""
EXTRA_ARGS_SRPM=""
EXTRA_ARGS_RPM=""
export BUILD_AVOIDANCE_OVERRIDE_DIR=""
export BUILD_AVOIDANCE_OVERRIDE_HOST=""
export BUILD_AVOIDANCE_OVERRIDE_USR=""
export BUILD_AVOIDANCE_DAY=""
while true ; do
case "$1" in
--append-log) APPEND_LOG_FLAG=1 ; shift ;;
--build-avoidance) BUILD_AVOIDANCE_FLAG=1 ; shift ;;
--no-build-avoidance) BUILD_AVOIDANCE_FLAG=0 ; shift ;;
--build-avoidance-dir) BUILD_AVOIDANCE_OVERRIDE_DIR=$2; shift 2 ;;
--build-avoidance-host) BUILD_AVOIDANCE_OVERRIDE_HOST=$2; shift 2 ;;
--build-avoidance-user) BUILD_AVOIDANCE_OVERRIDE_USR=$2; shift 2 ;;
--build-avoidance-day) BUILD_AVOIDANCE_DAY=$2; shift 2 ;;
--no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;;
--formal) EXTRA_ARGS_COMMON+=" --formal" ; shift ;;
--careful) EXTRA_ARGS_RPM+=" --careful" ; shift ;;
@ -125,11 +156,15 @@ trap my_sighup HUP
trap my_sigabrt ABRT
trap my_sigterm TERM
# Note: For ease of parsing, a TARGETS list always begins and ends
# with a space. An empty target list consistes of two spaces.
TARGETS=" $@ "
TARGETS_STD=" "
TARGETS_RT=" "
TARGETS_INSTALLER=" "
TARGETS_MISC=" "
EMPTY_TARGETS=" "
TARGETS_STD="$EMPTY_TARGETS"
TARGETS_RT="$EMPTY_TARGETS"
TARGETS_INSTALLER="$EMPTY_TARGETS"
TARGETS_MISC="$EMPTY_TARGETS"
find_targets () {
local centos_pkg_dirs=$1
@ -199,14 +234,28 @@ find_targets () {
fi
done
echo "$RESULT"
if [ "$RESULT" == " " ]; then
echo "$EMPTY_LIST"
else
echo "$RESULT"
fi
return 0
}
if [ $EDIT_FLAG -eq 1 ] || [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
BUILD_AVOIDANCE_FLAG=0
fi
echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
echo "CLEAN_FLAG=$CLEAN_FLAG"
echo "EDIT_FLAG=$EDIT_FLAG"
if [ "x$TARGETS" != "x " ]; then
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
fi
if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_STD="$(find_targets centos_pkg_dirs)"
BUILD_TYPE_SAVE="$BUILD_TYPE"
@ -247,44 +296,50 @@ echo "EXTRA_ARGS_RPM='$EXTRA_ARGS_RPM'"
echo "TARGETS='$TARGETS'"
echo "TARGETS_STD='$TARGETS_STD'"
echo "TARGETS_RT='$TARGETS_RT'"
echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'"
echo "TARGETS_MISC='$TARGETS_MISC'"
if [ $CLEAN_FLAG -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] && [ $BUILD_AVOIDANCE_FLAG -eq 1 ] ; then
build_avoidance_clean
fi
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1
fi
fi
if [ $RT_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1
fi
fi
if [ $INSTALLER_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1
fi
@ -317,13 +372,22 @@ function launch_build()
fi
echo "Launching $build_type build, logging to $logfile"
\rm $logfile
if [ $APPEND_LOG_FLAG -eq 0 ] && [ -f $logfile ]; then
\rm $logfile
fi
echo -e "\n######## $(date): Launching build-srpms-parallel --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
echo "build_avoidance $build_type" | tee --append $logfile
build_avoidance $build_type 2>&1 | tee --append $logfile
fi
# No clean flag, call build-srpms-parallel followed by build-rpms-parallel
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile
rc=${PIPESTATUS[0]}
if [ $rc -eq 0 ]; then
echo -e "\n######## $(date): build-srpm-parallel --$build_type was successful" | tee --append $logfile
@ -371,19 +435,18 @@ fi
echo "Capture build context"
git_context > "${MY_WORKSPACE}/CONTEXT"
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build std
else
echo "Skipping 'std' build, no valid targets in list: $TARGETS"
echo "Skipping 'std' build, no valid targets in list: '$TARGETS'"
fi
else
echo "Skipping 'std' build"
fi
if [ $RT_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build rt
else
echo "Skipping 'rt' build, no valid targets in list: $TARGETS"
@ -393,7 +456,7 @@ else
fi
if [ $INSTALLER_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build installer
else
echo "Skipping 'installer' build, no valid targets in list: $TARGETS"
@ -402,6 +465,15 @@ else
echo "Skipping 'installer' build"
fi
# Make sure REFERENCE_BUILD is set to something
if [ -z $REFERENCE_BUILD ]; then
REFERENCE_BUILD=0
fi
if [ $REFERENCE_BUILD -eq 1 ]; then
echo "Saving reference context"
build_avoidance_save_reference_context
fi
echo "All builds were successful"

View File

@ -1,10 +1,22 @@
#!/bin/bash
# This program is a wrapper around build-srpms and build-rpms
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Build first src.rpms, then rpms, from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This program is a wrapper around build-srpms-serial and build-rpms-serial
#
BUILD_PKGS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${BUILD_PKGS_SERIAL_DIR}/git-utils.sh"
source "${BUILD_PKGS_SERIAL_DIR}/build-avoidance-utils.sh"
source "${BUILD_PKGS_SERIAL_DIR}/spec-utils"
@ -13,14 +25,15 @@ usage () {
echo "Usage: "
echo " Create source and Binary rpms:"
echo " Build optimizations (--no-descendants, --no-required, --no-build-info,"
echo " --no-autoclean) are not recommended for the first build after a clone/pull,"
echo " nor the final build prior to creating an iso or patch, but can be used"
echo " for intermediate builds. i.e. while debugging compilation failures."
echo " build-pkgs-serial [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
echo " --no-autoclean, --no-build-avoidance) are not recommended for the first build"
echo " after a clone/pull, nor the final build prior to creating an iso or patch,"
echo " but can be used for intermediate builds."
echo " i.e. while debugging compilation failures."
echo " build-pkgs-serial [--build-avoidance | --no-build-avoidance] [--no-descendants] [--no-required] [--no-build-info] [--no-autoclean] [--careful] [--formal] [ list of package names ]"
echo ""
echo " Delete source rpms, and the directories associated with it's creation:"
echo " Note: does not clean an edit environment"
echo " build-pkgs-serial --clean [ list of package names ]"
echo " build-pkgs-serial --clean [--build-avoidance | --no-build-avoidance] [ list of package names ]"
echo ""
echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents,"
echo " one for source code and one for metadata such as the spec file."
@ -39,12 +52,17 @@ usage () {
HELP=0
CLEAN_FLAG=0
EDIT_FLAG=0
APPEND_LOG_FLAG=0
BUILD_AVOIDANCE_FLAG=0
STD_BUILD=1
RT_BUILD=1
INSTALLER_BUILD=0
export BUILD_AVOIDANCE_URL=""
# read the options
TEMP=$(getopt -o h --long serial,rt,std,edit,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean -n 'build-pkgs' -- "$@")
TEMP=$(getopt -o h --long serial,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -55,8 +73,21 @@ eval set -- "$TEMP"
EXTRA_ARGS_COMMON=""
EXTRA_ARGS_SRPM=""
EXTRA_ARGS_RPM=""
export BUILD_AVOIDANCE_OVERRIDE_DIR=""
export BUILD_AVOIDANCE_OVERRIDE_HOST=""
export BUILD_AVOIDANCE_OVERRIDE_USR=""
export BUILD_AVOIDANCE_DAY=""
while true ; do
case "$1" in
--append-log) APPEND_LOG_FLAG=1 ; shift ;;
--build-avoidance) BUILD_AVOIDANCE_FLAG=1 ; shift ;;
--no-build-avoidance) BUILD_AVOIDANCE_FLAG=0 ; shift ;;
--build-avoidance-dir) BUILD_AVOIDANCE_OVERRIDE_DIR=$2; shift 2 ;;
--build-avoidance-host) BUILD_AVOIDANCE_OVERRIDE_HOST=$2; shift 2 ;;
--build-avoidance-user) BUILD_AVOIDANCE_OVERRIDE_USR=$2; shift 2 ;;
--build-avoidance-day) BUILD_AVOIDANCE_DAY=$2; shift 2 ;;
--no-descendants) EXTRA_ARGS_COMMON+=" --no-descendants" ; shift ;;
--formal) EXTRA_ARGS_COMMON+=" --formal" ; shift ;;
--careful) EXTRA_ARGS_RPM+=" --careful" ; shift ;;
@ -81,11 +112,59 @@ if [ $HELP -eq 1 ]; then
exit 0
fi
function my_exit() {
build-rpms-parallel --std --tmpfs-clean
build-rpms-parallel --rt --tmpfs-clean
}
function my_sigint() {
echo "build-pkgs-parallel sigint"
pkill -SIGABRT -P $BASHPID &> /dev/null
echo "build-pkgs-parallel waiting"
wait
echo "build-pkgs-parallel wait complete"
}
function my_sighup() {
echo "build-pkgs-parallel sighup"
pkill -SIGABRT -P $BASHPID &> /dev/null
echo "build-pkgs-parallel waiting"
wait
echo "build-pkgs-parallel wait complete"
}
function my_sigabrt() {
echo "build-pkgs-parallel sigabrt"
pkill -SIGABRT -P $BASHPID &> /dev/null
echo "build-pkgs-parallel waiting"
wait
echo "build-pkgs-parallel wait complete"
}
function my_sigterm() {
echo "build-pkgs-parallel sigterm"
pkill -SIGABRT -P $BASHPID &> /dev/null
echo "build-pkgs-parallel waiting"
wait
echo "build-pkgs-parallel wait complete"
}
trap my_exit EXIT
trap my_sigint INT
trap my_sighup HUP
trap my_sigabrt ABRT
trap my_sigterm TERM
# Note: For ease of parsing, a TARGETS list always begins and ends
# with a space. An empty target list consistes of two spaces.
TARGETS=" $@ "
TARGETS_STD=" "
TARGETS_RT=" "
TARGETS_INSTALLER=" "
TARGETS_MISC=" "
EMPTY_TARGETS=" "
TARGETS_STD="$EMPTY_TARGETS"
TARGETS_RT="$EMPTY_TARGETS"
TARGETS_INSTALLER="$EMPTY_TARGETS"
TARGETS_MISC="$EMPTY_TARGETS"
find_targets () {
local centos_pkg_dirs=$1
@ -155,14 +234,28 @@ find_targets () {
fi
done
echo "$RESULT"
if [ "$RESULT" == " " ]; then
echo "$EMPTY_LIST"
else
echo "$RESULT"
fi
return 0
}
if [ $EDIT_FLAG -eq 1 ] || [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
BUILD_AVOIDANCE_FLAG=0
fi
echo "BUILD_AVOIDANCE_FLAG=$BUILD_AVOIDANCE_FLAG"
echo "CLEAN_FLAG=$CLEAN_FLAG"
echo "EDIT_FLAG=$EDIT_FLAG"
if [ "x$TARGETS" != "x " ]; then
if [ $BUILD_AVOIDANCE_FLAG -eq 1 ]; then
# Build Avoidance requested. Get URL of a usable context, if any.
export BUILD_AVOIDANCE_URL=$(get_build_avoidance_context)
fi
if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_STD="$(find_targets centos_pkg_dirs)"
BUILD_TYPE_SAVE="$BUILD_TYPE"
@ -207,45 +300,51 @@ echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'"
echo "TARGETS_MISC='$TARGETS_MISC'"
if [ $CLEAN_FLAG -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] && [ $BUILD_AVOIDANCE_FLAG -eq 1 ] ; then
build_avoidance_clean
fi
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC"
${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_STD $TARGETS_MISC || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC"
${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --std --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_STD $TARGETS_MISC || exit 1
fi
fi
if [ $RT_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x" ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC"
${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_RT $TARGETS_MISC || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC"
${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --rt --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_RT $TARGETS_MISC || exit 1
fi
fi
if [ $INSTALLER_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x" ] || [ "$TARGETS_INSTALLER" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER"
${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1
fi
fi
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER"
${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1
fi
fi
exit $?
fi
@ -273,8 +372,18 @@ function launch_build()
fi
echo "Launching $build_type build, logging to $logfile"
if [ $APPEND_LOG_FLAG -eq 0 ] && [ -f $logfile ]; then
\rm $logfile
fi
echo -e "\n######## $(date): Launching build-srpms-serial --$build_type $EXTRA_ARGS $@\n" | tee --append $logfile
echo "BUILD_AVOIDANCE_URL=$BUILD_AVOIDANCE_URL" | tee --append $logfile
if [ "x$BUILD_AVOIDANCE_URL" != "x" ]; then
echo "build_avoidance $build_type" | tee --append $logfile
build_avoidance $build_type 2>&1 | tee --append $logfile
fi
# No clean flag, call build-srpms-serial followed by build-rpms-serial
echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets" | tee --append $logfile
${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --$build_type $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $targets 2>&1 | tee --append $logfile
@ -326,7 +435,7 @@ echo "Capture build context"
git_context > "${MY_WORKSPACE}/CONTEXT"
if [ $STD_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_STD" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build std
else
echo "Skipping 'std' build, no valid targets in list: $TARGETS"
@ -336,7 +445,7 @@ else
fi
if [ $RT_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_RT" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_RT" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build rt
else
echo "Skipping 'rt' build, no valid targets in list: $TARGETS"
@ -346,7 +455,7 @@ else
fi
if [ $INSTALLER_BUILD -eq 1 ]; then
if [ "x$TARGETS" == "x " ] || [ "$TARGETS_INSTALLER" != " " ] || [ "$TARGETS_MISC" != " " ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build installer
else
echo "Skipping 'installer' build, no valid targets in list: $TARGETS"
@ -355,6 +464,16 @@ else
echo "Skipping 'installer' build"
fi
# Make sure REFERENCE_BUILD is set to something
if [ -z $REFERENCE_BUILD ]; then
REFERENCE_BUILD=0
fi
if [ $REFERENCE_BUILD -eq 1 ]; then
echo "Saving reference context"
build_avoidance_save_reference_context
fi
echo "All builds were successful"
exit 0

View File

@ -1,6 +1,16 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Builds rpm files from src.rpm files.
#
# This program is a wrapper around build-rpms-parallel and build-rpms-serial
#
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

View File

@ -6,6 +6,24 @@
# SPDX-License-Identifier: Apache-2.0
#
#
# Builds rpm files from src.rpm files.
#
# This version compiles packages in parrallel if sufficient resources
# (cpu's and memory) are available.
#
# The location of packages to be built is
# $MY_WORKSPACE/<build-type>/rpmbuild/SRPMS.
#
# The build order is a derived from the BuildRequires in the
# spec files in the src.rpms. Note that the BuildRequires sometimes
# create dependency loops, so no correct order can be computed. In these
# cases we add a retry loop. As long as one new package builds, we
# keep retrying the loop, until all are built, or no progress is made.
# So please don't panic and CTRL-C just because you see a few error
# messages go by!
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
@ -122,12 +140,10 @@ create-no-clean-list () {
if [ ! -f $NO_CLEAN_LIST_FILE ]; then
NEED_REBUILD=1
else
if [ -f MY_BUILD_CFG ]; then
if [ -f MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
if [ -f $MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
fi
fi

View File

@ -6,6 +6,22 @@
# SPDX-License-Identifier: Apache-2.0
#
#
# Builds rpm files from src.rpm files.
#
# This version compiles one package at a time.
#
# The location of packages to be built is
# $MY_WORKSPACE/<build-type>/rpmbuild/SRPMS.
#
# The build order is a derived from the BuildRequires in the
# spec files in the src.rpms. Note that the BuildRequires sometimes
# create dependency loops, so no correct order can be computed. In these
# cases we add a retry loop. As long as one new package builds, we
# keep retrying the loop, until all are built, or no progress is made.
# So please don't panic and CTRL-C just because you see a few error
# messages go by!
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
@ -63,12 +79,10 @@ create-no-clean-list () {
if [ ! -f $NO_CLEAN_LIST_FILE ]; then
NEED_REBUILD=1
else
if [ -f MY_BUILD_CFG ]; then
if [ -f MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
if [ -f $MY_BUILD_CFG ]; then
find "$MY_BUILD_CFG" -not -newermm "$NO_CLEAN_LIST_FILE" | grep -q $(basename $MY_BUILD_CFG)
if [ $? -eq 0 ]; then
NEED_REBUILD=1
fi
fi
fi

View File

@ -1,6 +1,24 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Create src.rpm files from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This program is a wrapper around build-srpms-parallel and build-srpms-serial
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. istx/stx-integ/centos_pkg_dirs).
#
# The build of an individual package is driven by its build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
#
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

View File

@ -0,0 +1,187 @@
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Functions common to build-srpm-serial and build-srpm-parallel.
#
SRC_BUILD_TYPE_SRPM="srpm"
SRC_BUILD_TYPE_SPEC="spec"
SRC_BUILD_TYPES="$SRC_BUILD_TYPE_SRPM $SRC_BUILD_TYPE_SPEC"
str_lst_contains() {
TARGET="$1"
LST="$2"
if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then
return 0
else
return 1
fi
}
#
# capture_md5sum_from_input_vars <src-build-type> <srpm-or-spec-path> <work-dir>
#
# Returns md5 data for all input files of a src.rpm.
# Assumes PKG_BASE, ORIG_SRPM_PATH have been defined and the
# build_srpm.data file has already been sourced.
#
# Arguments:
# src-build-type: Any single value from $SRC_BUILD_TYPES.
# e.g. 'srpm' or 'spec'
# srpm-or-spec-path: Absolute path to an src.rpm, or to a
# spec file.
# work-dir: Optional working directory. If a path is
# specified but does not exist, it will be created.
#
# Returns: output of md5sum command with canonical path names
#
md5sums_from_input_vars () {
local SRC_BUILD_TYPE="$1"
local SRPM_OR_SPEC_PATH="$2"
local WORK_DIR="$3"
local TMP_FLAG=0
local LINK_FILTER='[/]stx[/]downloads[/]'
if ! str_lst_contains "$SRC_BUILD_TYPE" "$SRC_BUILD_TYPES" ; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): invalid arg: SRC_BUILD_TYPE='$SRC_BUILD_TYPE'"
return 1
fi
if [ -z $WORK_DIR ]; then
WORK_DIR=$(mktemp -d /tmp/${FUNCNAME}_XXXXXX)
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): mktemp -d /tmp/${FUNCNAME}_XXXXXX"
return 1
fi
TMP_FLAG=1
else
mkdir -p "$WORK_DIR"
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): mkdir -p '$WORK_DIR'"
return 1
fi
fi
local INPUT_FILES="$WORK_DIR/srpm_input.files"
local INPUT_LINKS="$WORK_DIR/srpm_input.links"
local INPUT_FILES_SORTED="$WORK_DIR/srpm_sorted_input.files"
if [ -f "$INPUT_LINKS" ]; then
\rm -f "$INPUT_LINKS"
fi
# Create lists of input files (INPUT_FILES) and symlinks (INPUT_LINKS).
# First elements are absolute paths...
find "$PKG_BASE" -type f > $INPUT_FILES
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$PKG_BASE' -type f"
return 1
fi
if [ "$SRC_BUILD_TYPE" == "$SRC_BUILD_TYPE_SRPM" ]; then
find "$SRPM_OR_SPEC_PATH" -type f >> $INPUT_FILES
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$SRPM_OR_SPEC_PATH' -type f"
return 1
fi
fi
# ...additional elements are based on values already sourced from
# build_srpm.data (COPY_LIST, SRC_DIR, COPY_LIST_TO_TAR, OPT_DEP_LIST)
# and may be relative to $PKG_BASE
#
# Use a subshell so any directory changes have no lastin effect.
(
cd $PKG_BASE
if [ "x$COPY_LIST" != "x" ]; then
ABS_COPY_LIST=$(readlink -f $COPY_LIST)
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): readlink -f '$COPY_LIST' -type f"
return 1
fi
find $ABS_COPY_LIST -type f >> $INPUT_FILES
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$ABS_COPY_LIST' -type f"
return 1
fi
# Treat most links normally
find $ABS_COPY_LIST -type l | grep -v "$LINK_FILTER" >> $INPUT_FILES
# Links in the downloads directory likely point outside of $MY_REPO
# and might not be 'portable' from a build avoidance prespective.
# We'll treat these specially.
find $ABS_COPY_LIST -type l | grep "$LINK_FILTER" >> $INPUT_LINKS
fi
if [ "$SRC_BUILD_TYPE" == "$SRC_BUILD_TYPE_SPEC" ]; then
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
find $(readlink -f "$SRC_DIR") -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$SRC_DIR' -type f"
return 1
fi
fi
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
find $(readlink -f "$COPY_LIST_TO_TAR") -type f >> $INPUT_FILES
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$COPY_LIST_TO_TAR' -type f"
return 1
fi
fi
fi
if [ "x$OPT_DEP_LIST" != "x" ]; then
find $(readlink -f "$OPT_DEP_LIST") -type f >> $INPUT_FILES 2> /dev/null
if [ $? -ne 0 ]; then
>&2 echo "ERROR: $FUNCNAME (${LINENO}): find '$OPT_DEP_LIST' -type f"
return 1
fi
fi
)
if [ $? -eq 1 ]; then
return -1
fi
# Create sorted, unique list of cononical paths
(
# Regular files, get canonical path
cat $INPUT_FILES | xargs readlink -f
# A Symlink that likely points outside of $MY_REPO.
# get canonical path to the symlink itself, and not
# to what the symlink points to.
if [ -f $INPUT_LINKS ]; then
while IFS= read -r f; do
echo "$(readlink -f $(dirname $f))/$(basename $f)"
done < "$INPUT_LINKS"
fi
) | sort --unique > $INPUT_FILES_SORTED
# Remove $MY_REPO prefix from paths
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #"
if [ $TMP_FLAG -eq 0 ]; then
\rm -f $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
if [ -f $INPUT_LINKS ]; then
\rm -f $INPUT_LINKS
fi
else
\rm -rf $WORK_DIR
fi
return 0
}

View File

@ -1,6 +1,26 @@
#!/bin/bash
# set -x
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Create src.rpm files from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This version tries to compile many packages in parallel.
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. stx/stx-integ/centos_pkg_dirs).
#
# The build of an individul package is driven by it's build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
@ -9,6 +29,7 @@ source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh
source $BUILD_SRPMS_PARALLEL_DIR/spec-utils
source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils
source $BUILD_SRPMS_PARALLEL_DIR/classify
source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh
INITIAL_DIR=$(pwd)
@ -372,7 +393,7 @@ build_dir () {
fi
if [ "${ORIG_SRPM_PATH}x" == "x" ]; then
# handle mirror: definitions
# handle 3rd_party: definitions
echo "$p" | grep "^3rd_party:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^3rd_party:%$THIRD_PARTY_ROOT/%")
fi
@ -455,7 +476,7 @@ clean_srpm_dir () {
local SRPM_OUT_NAME
local INPUTS_TO_CLEAN=""
if [ $EXCLUDE_MD5 == "" ]; then
if [ "$EXCLUDE_MD5" == "" ]; then
EXCLUDE_MD5=0
fi
@ -626,24 +647,15 @@ build_dir_srpm () {
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
md5sums_from_input_vars "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SRPM' '$ORIG_SRPM_PATH' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#
@ -946,38 +958,15 @@ build_dir_spec () {
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES
fi
fi
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES
fi
if [ "x$OPT_DEP_LIST" != "x" ]; then
find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
md5sums_from_input_vars "$SRC_BUILD_TYPE_SPEC" "$SPEC" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SPEC' '$SPEC' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#

View File

@ -1,6 +1,26 @@
#!/bin/bash
# set -x
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
#
# Create src.rpm files from source, or from a downloaded tarball
# or src.rpm plus our additional patches.
#
# This version only tries to compile on package at a time.
#
# The location of packages to be build are identified by
# <distro>_pkg_dirs[_<opt-build-type>] files located at the root of
# any git tree (e.g. stx/stx-integ/centos_pkg_dirs).
#
# The build of an individul package is driven by it's build_srpm.data
# file plus a <pkg-name>.spec file or an srpm_path file.
#
export ME=$(basename "$0")
CMDLINE="$ME $@"
@ -10,6 +30,7 @@ source $BUILD_SRPMS_SERIAL_DIR/git-utils.sh
source $BUILD_SRPMS_SERIAL_DIR/spec-utils
source $BUILD_SRPMS_SERIAL_DIR/srpm-utils
source $BUILD_SRPMS_SERIAL_DIR/classify
source $BUILD_SRPMS_PARALLEL_DIR/build-srpms-common.sh
INITIAL_DIR=$(pwd)
@ -438,7 +459,7 @@ clean_srpm_dir () {
local SRPM_OUT_NAME
local INPUTS_TO_CLEAN=""
if [ $EXCLUDE_MD5 == "" ]; then
if [ "$EXCLUDE_MD5" == "" ]; then
EXCLUDE_MD5=0
fi
@ -603,24 +624,15 @@ build_dir_srpm () {
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
md5sums_from_input_vars "$SRC_BUILD_TYPE_SRPM" "$ORIG_SRPM_PATH" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SRPM' '$ORIG_SRPM_PATH' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#
@ -922,38 +934,15 @@ build_dir_spec () {
# Capture md5 data for all input files
#
local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND"
local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files"
local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files"
local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5"
local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5"
mkdir -p "$TARGET_SOURCES_DIR"
find $PKG_BASE -type f > $INPUT_FILES
find $ORIG_SRPM_PATH -type f >> $INPUT_FILES
(
cd $PKG_BASE
if [ "x$SRC_DIR" != "x" ]; then
if [ -d "$SRC_DIR" ]; then
find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES
fi
fi
if [ "x$COPY_LIST" != "x" ]; then
find $COPY_LIST -type f >> $INPUT_FILES
fi
if [ "x$COPY_LIST_TO_TAR" != "x" ]; then
find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES
fi
if [ "x$OPT_DEP_LIST" != "x" ]; then
find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null
fi
)
cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED
\rm -f $INPUT_FILES
cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5
\rm -f $INPUT_FILES_SORTED
md5sums_from_input_vars "$SRC_BUILD_TYPE_SPEC" "$SPEC" "$TARGET_SOURCES_DIR" > "$INPUT_FILES_MD5"
if [ $? -ne 0 ]; then
echo "ERROR: $FUNCNAME (${LINENO}): md5sums_from_input_vars '$SRC_BUILD_TYPE_SPEC' '$SPEC' '$TARGET_SOURCES_DIR'"
return 1
fi
echo "Wrote: $INPUT_FILES_MD5"
#

View File

@ -9,6 +9,10 @@
# by repo manifests.
#
git_ctx_root_dir () {
dirname "${MY_REPO}"
}
#
# git_list <dir>:
# Return a list of git root directories found under <dir>
@ -22,12 +26,12 @@ git_list () {
# GIT_LIST: A list of root directories for all the gits under $MY_REPO/..
# as absolute paths.
export GIT_LIST=$(git_list "$(dirname "${MY_REPO}")")
export GIT_LIST=$(git_list "$(git_ctx_root_dir)")
# GIT_LIST_REL: A list of root directories for all the gits under $MY_REPO/..
# as relative paths.
export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(dirname ${MY_REPO})}; done)
export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(git_ctx_root_dir)}; done)
#
@ -85,7 +89,9 @@ git_list_containing_tag () {
git_context () {
(
cd $MY_REPO
cd $(git_ctx_root_dir)
local d
for d in $GIT_LIST_REL; do
(
cd ${d}
@ -95,3 +101,48 @@ git_context () {
done
)
}
#
# git_test_context <context>:
#
# Test if all commits referenced in the context are present
# in the history of the gits in their current checkout state.
#
# Returns: 0 = context is present in git history
# 1 = At least one element of context is not present
# 2 = error
#
git_test_context () {
local context="$1"
local query=""
local target_hits=0
local actual_hits=0
if [ ! -f "$context" ]; then
return 2
fi
query=$(mktemp "/tmp/git_test_context_XXXXXX")
if [ "$query" == "" ]; then
return 2
fi
# Transform a checkout context into a query that prints
# all the commits that are found in the git history.
#
# Limit search to last 500 commits in the interest of speed.
# I don't expect to be using contexts more than a few weeks old.
cat "$context" | \
sed "s#checkout -f \([a-e0-9]*\)#rev-list --max-count=500 HEAD | \
grep \1#" > $query
target_hits=$(cat "$context" | wc -l)
actual_hits=$(cd $(git_ctx_root_dir); source $query | wc -l)
\rm $query
if [ $actual_hits -eq $target_hits ]; then
return 0
fi
return 1
}

View File

@ -1,5 +1,11 @@
#!/bin/bash
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This script calls into an external signing server to perform signing of some
# packages in the system. The old packages (which are typically generated by
# the build system and signed by placeholder keys) are overwritten by the new
@ -264,7 +270,7 @@ function rebuild_pkgs
fi
echo "Performing rebuild of packages: $PKGS_TO_REBUILD"
FORMAL_BUILD=0 build-pkgs --no-descendants --no-build-info --no-required --careful $PKGS_TO_REBUILD > $LOGFILE 2>&1
FORMAL_BUILD=0 build-pkgs --no-descendants --no-build-info --no-required --careful --append-log $PKGS_TO_REBUILD > $LOGFILE 2>&1
if [ $? -ne 0 ]; then
echo "Could not rebuild packages: $PKGS_TO_REBUILD -- see $LOGFILE for details"