#!/bin/bash # set -x export ME=$(basename "$0") CMDLINE="$ME $@" BUILD_SRPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh source $BUILD_SRPMS_PARALLEL_DIR/spec-utils source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils source $BUILD_SRPMS_PARALLEL_DIR/classify INITIAL_DIR=$(pwd) export DISTRO="centos" SRPM_SCRIPT="build_srpm" SRPM_DATA="build_srpm.data" PKG_DIRS_FILE="${DISTRO}_pkg_dirs" DEFAULT_SRPM_SCRIPT="$BUILD_SRPMS_PARALLEL_DIR/default_$SRPM_SCRIPT" SCRIPT_PATH="$DISTRO" DATA_PATH="$DISTRO" FILES_PATH="$DISTRO/files" PATCHES_PATH="$DISTRO/patches" ORIG_SPECS_PATH="$DISTRO" SRPM_LIST_PATH="$DISTRO/srpm_path" MIRROR_ROOT="$MY_REPO/cgcs-${DISTRO}-repo" THIRD_PARTY_ROOT="$MY_REPO/cgcs-3rd-party-repo" REPO_DOWNLOADS_ROOT="$MY_REPO" SRPM_REBUILT_LIST="" SRPM_FAILED_REBUILD_LIST="" STOP_SCHEDULING=0 ABSOLUTE_MAX_WORKERS=8 MAX_WORKERS=$(grep -c ^processor /proc/cpuinfo) if [ "$MAX_WORKERS" == "" ] || [ "$MAX_WORKERS" == "0" ]; then MAX_WORKERS=1 fi if [ $MAX_WORKERS -gt $ABSOLUTE_MAX_WORKERS ]; then MAX_WORKERS=$ABSOLUTE_MAX_WORKERS fi echo "MAX_WORKERS=$MAX_WORKERS" CREATEREPO=$(which createrepo_c) if [ $? -ne 0 ]; then CREATEREPO="createrepo" fi usage () { echo "" echo "Usage: " echo " Create source rpms:" echo " $ME [--rt | --std | --installer] [--no-descendants] [--formal] [ list of package names ]" echo "" echo " Delete source rpms, and the directories associated with it's creation:" echo " Note: does not clean an edit environment" echo " $ME --clean [--rt | --std | --installer] [optional list of package names]" echo "" echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents," echo " one for source code and one for metadata such as the spec file." echo " If --no-meta-patch is specified, then WRS patches are omitted." echo " $ME --edit [--rt | --std | --installer] [--no-meta-patch] [list of package names]" echo "" echo " Delete an edit environment" echo " $ME --edit --clean [--rt | --std | --installer] [list of package names]" echo "" echo " This help page" echo " $ME --help" echo "" } spec_cache_dir_from_srpm () { local SRPM=${1} local SPEC_DIR=$(echo $SRPM | sed 's#/SRPMS/#/SPECS/#') echo "$SPEC_DIR" } result_dir_from_srpm () { local SRPM=$(basename ${1} | sed 's#.src.rpm$##') local RESULT_DIR="$MY_WORKSPACE/results/$MY_BUILD_ENVIRONMENT/$SRPM" echo "$RESULT_DIR" } # This function creates a bunch of subdirs in $MY_WORKSPACE and makes sure # that a $MY_BUILD_CFG file exists. # # The goal of this is to have a script do as much of the annoying # grunt-work so that the "how to build it" instructions aren't 200 lines create_output_dirs () { # make sure variables are sane before continuing # Note that $BUILD_ROOT contains either $MY_WORKSPACE or $MY_PATCH_WORKSPACE if [ "x$BUILD_ROOT" == "x" ]; then return fi if [ "x$MY_BUILD_CFG" == "x" ]; then return fi if [ "x$MY_BUILD_DIR" == "x" ]; then return fi if [ "x$MY_SRC_RPM_BUILD_DIR" == "x" ]; then return fi # create output dirs mkdir -p $MY_BUILD_DIR mkdir -p $MY_SRC_RPM_BUILD_DIR mkdir -p $MY_SRC_RPM_BUILD_DIR/SOURCES mkdir -p $MY_SRC_RPM_BUILD_DIR/SPECS mkdir -p $MY_SRC_RPM_BUILD_DIR/BUILD mkdir -p $MY_SRC_RPM_BUILD_DIR/RPMS mkdir -p $MY_SRC_RPM_BUILD_DIR/SRPMS # create $MY_BUILD_CFG, if required if [ ! -f $MY_BUILD_CFG ]; then echo "FORMAL_BUILD=$FORMAL_BUILD" echo "modify-build-cfg $MY_BUILD_CFG" ${DIR}/modify-build-cfg $MY_BUILD_CFG if [ $? -ne 0 ]; then echo "Could not modifiy $MY_BUILD_CFG"; exit 1 fi fi } NO_DESCENDANTS=0 NO_BUILD_INFO=0 HELP=0 CLEAN_FLAG=0 FORMAL_FLAG=0 BUILD_TYPE_FLAG=0 EDIT_FLAG=0 NO_META_PATCH_FLAG=0 # read the options TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@") if [ $? -ne 0 ]; then usage exit 1 fi eval set -- "$TEMP" export BUILD_TYPE=std # extract options and their arguments into variables. while true ; do case "$1" in -a|--arga) case "$2" in "") ARG_A='some default value' ; shift 2 ;; *) ARG_A=$2 ; shift 2 ;; esac ;; -b|--argb) ARG_B=1 ; shift ;; --no-descendants) NO_DESCENDANTS=1 ; shift ;; --no-build-info) NO_BUILD_INFO=1 ; shift ;; -h|--help) HELP=1 ; shift ;; -c|--argc) case "$2" in "") shift 2 ;; *) ARG_C=$2 ; shift 2 ;; esac ;; --clean) CLEAN_FLAG=1 ; shift ;; --formal) FORMAL_FLAG=1 ; shift ;; --std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;; --rt) BUILD_TYPE_FLAG=1; BUILD_TYPE=rt; shift ;; --installer) BUILD_TYPE=installer; shift ;; --edit) EDIT_FLAG=1 ; shift ;; --no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;; --parallel) shift ;; --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac done # Reset variables if [ -n "$MY_WORKSPACE" ]; then export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE} export MY_WORKSPACE=$MY_WORKSPACE_TOP/$BUILD_TYPE else export MY_PATCH_WORKSPACE_TOP=${MY_PATCH_WORKSPACE_TOP:-$MY_PATCH_WORKSPACE} export MY_PATCH_WORKSPACE=$MY_PATCH_WORKSPACE_TOP/$BUILD_TYPE fi export MY_BUILD_DIR_TOP=${MY_BUILD_DIR_TOP:-$MY_BUILD_DIR} export MY_BUILD_DIR=$MY_BUILD_DIR_TOP/$BUILD_TYPE export MY_BUILD_ENVIRONMENT_TOP=${MY_BUILD_ENVIRONMENT_TOP:-$MY_BUILD_ENVIRONMENT} export MY_BUILD_ENVIRONMENT=$MY_BUILD_ENVIRONMENT_TOP-$BUILD_TYPE export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg export MY_SRC_RPM_BUILD_DIR=$MY_BUILD_DIR/rpmbuild export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root if [ "$BUILD_TYPE" != "std" ]; then PKG_DIRS_FILE="${DISTRO}_pkg_dirs_${BUILD_TYPE}" fi echo "CLEAN_FLAG=$CLEAN_FLAG" TARGETS=$@ if [ $HELP -eq 1 ]; then usage exit 0 fi if [ $FORMAL_FLAG -eq 1 ]; then export FORMAL_BUILD="yes" fi if [ "x$TARGETS" == "x" ] && [ $EDIT_FLAG -eq 1 ]; then echo "ERROR: $FUNCNAME (${LINENO}): a package name is required when --edit is specified" usage exit 0 fi SRC_ROOT="$MY_REPO" if [ "x$MY_REPO" == "x" ]; then SRC_ROOT=$INITIAL_DIR fi BUILD_ROOT="$MY_WORKSPACE" if [ "x$MY_WORKSPACE" == "x" ]; then BUILD_ROOT="$MY_PATCH_WORKSPACE" if [ "x$MY_PATCH_WORKSPACE" == "x" ]; then echo "ERROR: $FUNCNAME (${LINENO}): require one of MY_WORKSPACE or MY_PATCH_WORKSPACE be defined" exit 1 fi fi export CCACHE_DIR="$BUILD_ROOT/.ccache" export SRC_BASE="$SRC_ROOT" export STX_BASE="$SRC_BASE/stx" export CGCS_BASE="$STX_BASE" export SPECS_BASE="$ORIG_SPECS_PATH" export FILES_BASE="$FILES_PATH" export PATCHES_BASE="$PATCHES_PATH" export BUILD_BASE="$BUILD_ROOT" BUILD_INPUTS="$BUILD_BASE/inputs" SRPM_ASSEMBLE="$BUILD_BASE/srpm_assemble" SRPM_WORK="$BUILD_BASE/srpm_work" if [ "x$MY_SRC_RPM_BUILD_DIR" != "x" ]; then RPM_BUILD_ROOT=$MY_SRC_RPM_BUILD_DIR else RPM_BUILD_ROOT=$BUILD_BASE/rpmbuild fi create_output_dirs export RPM_BUILD_BASE="$RPM_BUILD_ROOT" export SRPM_OUT="$RPM_BUILD_BASE/SRPMS" export SOURCE_OUT="$RPM_BUILD_BASE/SOURCES" export RPM_DIR="$RPM_BUILD_BASE/RPMS" if [ ! -d $CGCS_BASE ]; then echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$CGCS_BASE'" exit 1 fi if [ ! -d $BUILD_BASE ]; then echo "ERROR: $FUNCNAME (${LINENO}): expected to find directory at '$BUILD_BASE'" exit 1 fi RELEASE_INFO_FILE=$STX_BASE/stx-integ/utilities/build-info/release-info.inc if [ -f $RELEASE_INFO_FILE ]; then source $RELEASE_INFO_FILE else echo "ERROR: $FUNCNAME (${LINENO}): failed to find RELEASE_INFO_FILE=$RELEASE_INFO_FILE" exit 1 fi if [ "x$PLATFORM_RELEASE" == "x" ]; then echo "ERROR: $FUNCNAME (${LINENO}): PLATFORM_RELEASE is not defined in $RELEASE_INFO_FILE" exit 1 fi export PLATFORM_RELEASE mkdir -p $RPM_BUILD_BASE if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_BUILD_BASE'" exit 1 fi mkdir -p $SRPM_OUT if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$SRPM_OUT'" exit 1 fi mkdir -p $RPM_DIR if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directory '$RPM_DIR'" exit 1 fi build_dir () { local build_idx=$1 local d=$2 local w=$3 export PKG_BASE=$d export WORK_BASE=$w export SPECS_BASE="$PKG_BASE/$ORIG_SPECS_PATH" local RC local ORIG_DIR=$(pwd) # echo "build_dir: PKG_BASE=$PKG_BASE" cd "$PKG_BASE" if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to cd into '$PKG_BASE'" return 1 fi if [ ! -d $ORIG_SPECS_PATH ]; then # nothing to do echo "WARNING: '$ORIG_SPECS_PATH' not found in '$PKG_BASE'" cd "$ORIG_DIR" return 0 fi SRPM_COUNT=0 ORIG_SRPM_PATH="" if [ -f $SRPM_LIST_PATH ]; then # we've found a file (ex centos/srpm_path) which lists a path to a source # RPM file # # The specified file can be of the form # # repo:path/to/file.src.rpm # mirror:path/to/file.src.rpm # /path/to/file.rpm # path/to/file.rpm # # If "repo:" is specified, then we search for the file relative to # $REPO_DOWNLOADS_ROOT (i.e. a path to the file in a "downloads subgit) # # If "mirror:" is specified, then we search for the file relateive to # $MIRROR_ROOT # # If "3rd_party:" is specified, then we search for the file relateive to # $THIRD_PARTY_ROOT # # An absolute path is parsed as an absolute path (mainly intended for # developer/experimental use without checking in files or messing with # your git repos) # # A lack of prefix (relative path name) is interpretted as "mirror:" # (legacy support for existing packages) # # Other prefixes (file:, http:, whatever:)are unsupported at this time for p in $(grep -v '^#' $SRPM_LIST_PATH | grep -v '^$'); do # absolute path source rpms echo "$p" | grep "^/" >/dev/null && ORIG_SRPM_PATH=$p if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle repo: definitions echo "$p" | grep "^repo:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^repo:%$REPO_DOWNLOADS_ROOT/%") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle mirror: definitions echo "$p" | grep "^3rd_party:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^3rd_party:%$THIRD_PARTY_ROOT/%") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle mirror: definitions echo "$p" | grep "^mirror:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^mirror:%$MIRROR_ROOT/%" | sed "s#CentOS/tis-r3-CentOS/kilo/##" | sed "s#CentOS/tis-r3-CentOS/mitaka/##") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # we haven't found a valid prefix yet, so assume it's a legacy # file (mirror: interpretation) ORIG_SRPM_PATH="$MIRROR_ROOT/$p" fi # echo "ORIG_SRPM_PATH=$ORIG_SRPM_PATH" if [ -f $ORIG_SRPM_PATH ]; then SRPM_COUNT=$((SRPM_COUNT + 1)) else echo "ERROR: $FUNCNAME (${LINENO}): Invalid srpm path '$p', evaluated as '$ORIG_SRPM_PATH', found in '$PKG_BASE/$SRPM_LIST_PATH'" ORIG_SRPM_PATH="" return 3 fi done fi # Clean up an tmp_spec_*.spec file left by a prior failed build for f in $(find $ORIG_SPECS_PATH -name 'tmp_spec_*.spec'); do \rm -f $f done SPEC_COUNT=$(find $ORIG_SPECS_PATH -name '*.spec' | wc -l) if [ $SPEC_COUNT -eq 0 ]; then if [ -f $ORIG_SPECS_PATH/spec_path ]; then SPECS_BASE=$SRC_BASE/$(cat $SPECS_BASE/spec_path) SPEC_COUNT=$(find $SPECS_BASE -maxdepth 1 -name '*.spec' | wc -l) fi fi if [ $SPEC_COUNT -eq 0 ] && [ $SRPM_COUNT -eq 0 ]; then # nothing to do echo "ERROR: $FUNCNAME (${LINENO}): Neither srpm_path nor .spec file not found in '$PKG_BASE/$ORIG_SPECS_PATH'" cd "$ORIG_DIR" return 0 fi if [ $SPEC_COUNT -gt 0 ] && [ $SRPM_COUNT -gt 0 ]; then # nothing to do echo "ERROR: $FUNCNAME (${LINENO}): Please provide only one of srpm_path or .spec files, not both, in '$PKG_BASE/$ORIG_SPECS_PATH'" cd $ORIG_DIR return 0 fi if [ $SPEC_COUNT -gt 0 ]; then build_dir_spec $build_idx RC=$? cd "$ORIG_DIR" return $RC else build_dir_srpm $build_idx $ORIG_SRPM_PATH RC=$? cd "$ORIG_DIR" return $RC fi cd "$ORIG_DIR" return 0 } clean_srpm_dir () { local build_idx=$1 local DIR=$2 local EXCLUDE_MD5=$3 local SRPM_PATH local SRPM_FILE local SRPM_OUT_PATH local SRPM_NAME local SRPM_OUT_NAME local INPUTS_TO_CLEAN="" if [ $EXCLUDE_MD5 == "" ]; then EXCLUDE_MD5=0 fi echo "clean_srpm_dir build_idx=$build_idx DIR=$DIR" INPUTS_TO_CLEAN=$(dirname $(dirname $DIR)) echo "$INPUTS_TO_CLEAN" | grep -q "^$BUILD_INPUTS/" if [ $? -ne 0 ] ; then INPUTS_TO_CLEAN="" fi for SRPM_PATH in $(find "$DIR" -name '*.src.rpm'); do SRPM_FILE=$(basename $SRPM_PATH) SRPM_NAME=$(rpm -q --queryformat '%{NAME}\n' --nosignature -p $SRPM_PATH 2>> /dev/null) if [ $CLEAN_FLAG -eq 1 ]; then sed -i "/^$SRPM_NAME$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx} fi \rm -fv $SRPM_PATH $SRPM_OUT/$SRPM_FILE if [ -d $SRPM_ASSEMBLE/$SRPM_NAME ]; then echo "rm -rf $SRPM_ASSEMBLE/$SRPM_NAME" \rm -rf $SRPM_ASSEMBLE/$SRPM_NAME fi if [ -d $SOURCE_OUT/$SRPM_FILE ]; then echo "rm -rf $SOURCE_OUT/$SRPM_FILE" \rm -rf $SOURCE_OUT/$SRPM_FILE fi if [ $EXCLUDE_MD5 -eq 0 ] && [ -d $SOURCE_OUT/$SRPM_NAME ]; then echo "rm -rf $SOURCE_OUT/$SRPM_NAME" \rm -rf $SOURCE_OUT/$SRPM_NAME fi local d local src_d local spec local spec_name for d in $(find $BUILD_INPUTS -type d -name "${SRPM_NAME}*") ;do src_d=$(echo $d | sed "s#^$BUILD_INPUTS/#$MY_REPO/#") for spec in $(find $src_d/${DISTRO} -name '*.spec'); do spec_name=$(spec_find_tag Name $spec) if [ "$spec_name" == "$SRPM_NAME" ]; then INPUTS_TO_CLEAN=$(if [ "x$INPUTS_TO_CLEAN" != "x" ]; then echo $INPUTS_TO_CLEAN; fi; echo "$d") fi done done # Look for older versions of the same src rpm that also need cleaning for SRPM_OUT_PATH in $(ls -1 $SRPM_OUT/$SRPM_NAME*.src.rpm 2>> /dev/null); do SRPM_OUT_FILE=$(basename $SRPM_OUT_PATH) SRPM_OUT_NAME=$(rpm -q --queryformat '%{NAME}\n' -p $SRPM_OUT_PATH 2>> /dev/null) if [ "$SRPM_NAME" == "$SRPM_OUT_NAME" ]; then \rm -fv $SRPM_OUT_PATH if [ -d $SOURCE_OUT/$SRPM_OUT_FILE ]; then echo "rm -rf $SOURCE_OUT/$SRPM_OUT_FILE" \rm -rf $SOURCE_OUT/$SRPM_OUT_FILE fi fi done done if [ "x$INPUTS_TO_CLEAN" != "x" ]; then for d in $INPUTS_TO_CLEAN; do if [ -d $d/rpmbuild ]; then echo "rm -rf $d" \rm -rf $d fi done fi } build_dir_srpm () { local build_idx=$1 local ORIG_SRPM_PATH=$2 local ORIG_SRPM=$(basename $ORIG_SRPM_PATH) local NAME=$(rpm -q --queryformat '%{NAME}\n' --nosignature -p $ORIG_SRPM_PATH) local PKG_NAME_VER=$(rpm -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}\n' --nosignature -p $ORIG_SRPM_PATH) local PKG_DIR="$NAME" local TARGET_FOUND="" local RC=0 export SRPM_EXPORT_NAME=$NAME export SRPM_EXPORT_VER=$VER local NEED_BUILD=0 if [ "x$TARGETS" == "x" ]; then NEED_BUILD=1 TARGET_FOUND=$NAME else TARGET_LIST=( $TARGETS ) TARGET_FOUND=$(srpm_match_target_list TARGET_LIST "$ORIG_SRPM_PATH" 2>> /dev/null) if [ $? -eq 0 ]; then echo "found target '$TARGET_FOUND' in '$ORIG_SRPM'" NEED_BUILD=1 sed -i "/^$TARGET_FOUND$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx} fi fi if [ $NEED_BUILD -eq 0 ]; then return 0 fi local ROOT_DIR="$SRPM_ASSEMBLE" if [ $EDIT_FLAG -eq 1 ]; then mkdir -p $SRPM_WORK ROOT_DIR="$SRPM_WORK" fi local PKG_ROOT_DIR="$ROOT_DIR/$PKG_DIR" local BUILD_DIR="$PKG_DIR/rpmbuild" local FULL_BUILD_DIR="$ROOT_DIR/$BUILD_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" local SOURCES_DIR="$SOURCE_OUT" if [ $CLEAN_FLAG -eq 1 ]; then # clean echo "===== Cleaning '$TARGET_FOUND' =====" if [ -d $SRPM_DIR ] && [ $EDIT_FLAG -eq 0 ]; then clean_srpm_dir $build_idx "$SRPM_DIR" 0 fi if [ -d $PKG_ROOT_DIR ]; then echo "rm -rf $PKG_ROOT_DIR" \rm -rf "$PKG_ROOT_DIR" fi else #build echo "===== Build SRPM for '$TARGET_FOUND' =====" echo "PKG_BASE=$PKG_BASE" echo "BUILD_DIR=$BUILD_DIR" echo "SRPM_DIR=$SRPM_DIR" if [ ! -d $ROOT_DIR ]; then mkdir -p "$ROOT_DIR" if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): mkdir '$ROOT_DIR' failed" return 1 fi fi # # Load data from build_srpm.data # export DATA="$DATA_PATH/$SRPM_DATA" local COPY_LIST local COPY_LIST_TO_TAR local SRC_DIR local TIS_PATCH_VER local BUILD_IS_BIG=0 local BUILD_IS_SLOW=0 if [ -f $DATA ]; then srpm_source_build_data $DATA if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" return 1 fi fi # # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" find $PKG_BASE -type f > $INPUT_FILES find $ORIG_SRPM_PATH -type f >> $INPUT_FILES ( cd $PKG_BASE if [ "x$COPY_LIST" != "x" ]; then find $COPY_LIST -type f >> $INPUT_FILES fi ) cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED \rm -f $INPUT_FILES cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 \rm -f $INPUT_FILES_SORTED echo "Wrote: $INPUT_FILES_MD5" # # Is a rebuild required? # Compare md5 of current inputs vs md5 of previous build? # local BUILD_NEEDED=0 local SRPM_OUT_PATH2 local DIFF_LINE local DIFF_FILE if [ -f $REFERENCE_MD5 ]; then DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]}) if [ $? -ne 0 ]; then DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-) BUILD_NEEDED=1 case ${DIFF_LINE:0:1} in '>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;; '<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;; *) echo "Rebuild required due to diff: $DIFF_LINE" ;; esac fi else echo "Rebuild required due to missing reference md5: $REFERENCE_MD5" BUILD_NEEDED=1 fi if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then for SRPM_PATH in $(find "$FULL_BUILD_DIR/SRPMS" -name '*.src.rpm' | sort -V); do b=$(basename $SRPM_PATH) SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b) if [ "x$SRPM_OUT_PATH2" == "x" ]; then echo "Rebuild required due to missing srpm: $b" BUILD_NEEDED=1 fi done else echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'" BUILD_NEEDED=1 fi if [ $BUILD_NEEDED -eq 0 ]; then echo "SRPM build not required for '$PKG_BASE'" echo "===== Build complete for '$TARGET_FOUND' =====" echo return 0 fi if [ $EDIT_FLAG -eq 0 ]; then clean_srpm_dir $build_idx "$FULL_BUILD_DIR/SRPMS" 1 if [ -d $PKG_ROOT_DIR ]; then echo "arf rm -rf $PKG_ROOT_DIR" \rm -rf $PKG_ROOT_DIR fi fi if [ $EDIT_FLAG -eq 1 ]; then PKG_CLASSIFICATION=$(classify $PKG_BASE) echo "$PKG_CLASSIFICATION = classify $PKG_BASE" if [ "$PKG_CLASSIFICATION" == "spec + tarball" ] || [ "$PKG_CLASSIFICATION" == "srpm + patches" ]; then echo "OK to edit $PKG_BASE" else echo "Can't edit this package, it is of type '$PKG_CLASSIFICATION', it is not derived from SRPM or tarball and patches" return 1 fi echo "srpm_extract_to_git '$ORIG_SRPM_PATH' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER'" srpm_extract_to_git $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER $NO_META_PATCH_FLAG $TIS_PATCH_VER RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi local LOC LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" return 0 fi # # Find age of youngest input file. # We will apply this as the creation/modification timestamp of the src.rpm we produce. # AGE=$(find $PKG_BASE $ORIG_SRPM_PATH -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) if [ -f $PKG_BASE/$DATA ]; then AGE2=$( cd $PKG_BASE srpm_source_build_data $DATA PATH_LIST="" # NOTE: SRC_DIR is not honored in this build path if [ "x$COPY_LIST" != "x" ]; then PATH_LIST="$PATH_LIST $COPY_LIST" fi # NOTE: COPY_LIST_TO_TAR is not honored in this build path if [ "x$PATH_LIST" == "x" ]; then echo "0" else AGE2=$(find $PATH_LIST -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) echo "$AGE2" fi ) if [ $AGE2 -gt $AGE ]; then AGE=$AGE2 fi fi srpm_extract $ORIG_SRPM_PATH $PKG_BASE $ROOT_DIR $BUILD_DIR $PKG_NAME_VER if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" return 1 fi if [ "x$COPY_LIST" != "x" ]; then echo "COPY_LIST: $COPY_LIST" for p in $COPY_LIST; do # echo "COPY_LIST: $p" \cp -L -r -f -v $p $FULL_BUILD_DIR/SOURCES if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): COPY_LIST: file not found: '$p'" return 1 fi done fi srpm_assemble $FULL_BUILD_DIR $TIS_PATCH_VER if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to assemble srpm for '$PKG_NAME_VER'" echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx} return 1 fi TS=$(date -d @$AGE +%Y-%m-%dT%H:%M:%S) for s in $(find $FULL_BUILD_DIR/SRPMS -name '*.src.rpm'); do \cp -L -f -v $s $SRPM_OUT/ ss=$(basename $s) touch $SRPM_OUT/$ss --date=$TS mkdir -p $SOURCES_DIR/$ss BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" if [ $BUILD_IS_BIG -gt 0 ]; then echo "$BUILD_IS_BIG" > $BIG_FLAG_FILE else if [ -f $BIG_FLAG_FILE ]; then \rm -f $BIG_FLAG_FILE fi fi if [ $BUILD_IS_SLOW -gt 0 ]; then echo "$BUILD_IS_SLOW" > $SLOW_FLAG_FILE else if [ -f $SLOW_FLAG_FILE ]; then \rm -f $SLOW_FLAG_FILE fi fi \rm -f -v "$REFERENCE_MD5" \mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5" local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss) if [ -d $SPEC_DIR/BUILDS_VR ]; then for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do \rm -f -v $r done done fi local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss) if [ -d $RESULT_DIR ]; then echo "rm -rf $RESULT_DIR" \rm -rf $RESULT_DIR fi done echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx} echo "SRPM build successful for '$PKG_NAME_VER'" echo "===== Build complete for '$TARGET_FOUND' =====" echo fi return 0 } build_dir_spec () { local build_idx=$1 local NEED_BUILD=0 local TARGET_FOUND="" if [ "x$TARGETS" == "x" ]; then NEED_BUILD=1 for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec'); do TARGET_FOUND=$(spec_find_tag Name "$f" 2>> /dev/null) if [ $? -ne 0 ]; then TARGET_FOUND=$(spec_find_global service "$f" 2>> /dev/null) if [ $? -ne 0 ]; then TARGET_FOUND="" fi fi done else TARGET_LIST=( $TARGETS ) for f in $(find $SPECS_BASE -maxdepth 1 -name '*.spec' 2>> /dev/null); do TARGET_FOUND=$(spec_match_target_list TARGET_LIST "$f" 2>> /dev/null) if [ $? -eq 0 ]; then echo "found target '$TARGET_FOUND' in '$f'" NEED_BUILD=1 sed -i "/^$TARGET_FOUND$/d" $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_${build_idx} break fi done fi if [ $NEED_BUILD -eq 1 ]; then MAKE_SRPM="$SCRIPT_PATH/$SRPM_SCRIPT" export DATA="$DATA_PATH/$SRPM_DATA" export RPMBUILD_BASE="$WORK_BASE/rpmbuild" SRPM_PATH="$RPMBUILD_BASE/SRPMS" SPEC_PATH="$RPMBUILD_BASE/SPECS" SOURCES_PATH="$RPMBUILD_BASE/SOURCES" local ROOT_DIR="$RPMBUILD_BASE" local PKG_ROOT_DIR="$RPMBUILD_BASE" local SPEC=$(find $SPECS_BASE -maxdepth 1 -name '*.spec' | head -n 1) local NAME=$(spec_find_tag Name $SPEC) local PKG_NAME_VER=$(spec_name_ver_rel $SPEC) local PKG_DIR="$NAME" local BUILD_DIR="$PKG_DIR/rpmbuild" local FULL_BUILD_DIR="$ROOT_DIR" local SRPM_DIR="$FULL_BUILD_DIR/SRPMS" local SOURCES_DIR="$SOURCE_OUT" if [ $EDIT_FLAG -eq 1 ]; then mkdir -p $SRPM_WORK ROOT_DIR="$SRPM_WORK" PKG_ROOT_DIR="$ROOT_DIR/$PKG_DIR" fi if [ $CLEAN_FLAG -eq 1 ]; then # clean echo "===== Cleaning '$TARGET_FOUND' =====" if [ -d $SRPM_PATH ] && [ $EDIT_FLAG -eq 0 ]; then clean_srpm_dir $build_idx $SRPM_PATH 0 fi if [ -d $PKG_ROOT_DIR ]; then echo "rm -rf $PKG_ROOT_DIR" \rm -rf "$PKG_ROOT_DIR" fi else # build echo "===== Build SRPM for '$TARGET_FOUND' =====" echo "PKG_BASE=$PKG_BASE" echo "WORK_BASE=$WORK_BASE" echo "RPMBUILD_BASE=$RPMBUILD_BASE" if [ ! -x $MAKE_SRPM ]; then if [ ! -f $DATA ]; then echo "expected to find an executable script at '$MAKE_SRPM' or data for the default script at '$DATA'" cd $INITIAL_DIR exit 1 else MAKE_SRPM="$DEFAULT_SRPM_SCRIPT" fi fi # # Load data from build_srpm.data # local COPY_LIST local COPY_LIST_TO_TAR local SRC_DIR local OPT_DEP_LIST local TIS_PATCH_VER local BUILD_IS_BIG=0 local BUILD_IS_SLOW=0 srpm_source_build_data $DATA if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to source $DATA" return 1 fi # # Capture md5 data for all input files # local TARGET_SOURCES_DIR="$SOURCES_DIR/$TARGET_FOUND" local INPUT_FILES="$TARGET_SOURCES_DIR/srpm_input.files" local INPUT_FILES_SORTED="$TARGET_SOURCES_DIR/srpm_sorted_input.files" local INPUT_FILES_MD5="$TARGET_SOURCES_DIR/srpm_input.md5" local REFERENCE_MD5="$TARGET_SOURCES_DIR/srpm_reference.md5" mkdir -p "$TARGET_SOURCES_DIR" find $PKG_BASE -type f > $INPUT_FILES find $ORIG_SRPM_PATH -type f >> $INPUT_FILES ( cd $PKG_BASE if [ "x$SRC_DIR" != "x" ]; then if [ -d "$SRC_DIR" ]; then find $SRC_DIR -type f | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' >> $INPUT_FILES fi fi if [ "x$COPY_LIST" != "x" ]; then find $COPY_LIST -type f >> $INPUT_FILES fi if [ "x$COPY_LIST_TO_TAR" != "x" ]; then find $COPY_LIST_TO_TAR -type f >> $INPUT_FILES fi if [ "x$OPT_DEP_LIST" != "x" ]; then find $OPT_DEP_LIST -type f >> $INPUT_FILES 2> /dev/null fi ) cat $INPUT_FILES | xargs readlink -f | sort --unique > $INPUT_FILES_SORTED \rm -f $INPUT_FILES cat $INPUT_FILES_SORTED | xargs md5sum | sed "s# $(readlink -f $MY_REPO)/# #" > $INPUT_FILES_MD5 \rm -f $INPUT_FILES_SORTED echo "Wrote: $INPUT_FILES_MD5" # # Is a rebuild required? # Compare md5 of current inputs vs md5 of previous build? # local BUILD_NEEDED=0 local SRPM_OUT_PATH2 local DIFF_LINE local DIFF_FILE if [ -f $REFERENCE_MD5 ]; then DIFF_LINE=$(diff "$INPUT_FILES_MD5" "$REFERENCE_MD5" | head -n 2 | tail -n 1; exit ${PIPESTATUS[0]}) if [ $? -ne 0 ]; then DIFF_FILE=$(echo "$DIFF_LINE" | cut -d ' ' -f4-) BUILD_NEEDED=1 case ${DIFF_LINE:0:1} in '>') echo "Rebuild required due to deleted file: $DIFF_FILE" ;; '<') echo "Rebuild required due to new or changed file: $DIFF_FILE" ;; *) echo "Rebuild required due to diff: $DIFF_LINE" ;; esac fi else echo "Rebuild required due to missing reference md5: $REFERENCE_MD5" BUILD_NEEDED=1 fi if [ -d "$FULL_BUILD_DIR/SRPMS" ]; then for SRPM_PATH2 in $(find "$RPMBUILD_BASE/SRPMS" -name '*.src.rpm' | sort -V); do b=$(basename $SRPM_PATH2) SRPM_OUT_PATH2=$(find $SRPM_OUT -name $b) if [ "x$SRPM_OUT_PATH2" == "x" ]; then echo "Rebuild required due to missing srpm: $b" BUILD_NEEDED=1 fi done else echo "Rebuild required due to missing directory: '$FULL_BUILD_DIR/SRPMS'" BUILD_NEEDED=1 fi if [ $EDIT_FLAG -eq 1 ]; then PKG_CLASSIFICATION=$(classify $PKG_BASE) echo "$PKG_CLASSIFICATION = classify $PKG_BASE" if [ "$PKG_CLASSIFICATION" == "spec + tarball" ] || [ "$PKG_CLASSIFICATION" == "srpm + patches" ]; then echo "OK to edit $PKG_BASE" else echo "Can't edit this package, it is of type '$PKG_CLASSIFICATION', it is not derived from SRPM or tarball and patches" return 1 fi echo "tar_and_spec_extract_to_git '$SPEC' '$PKG_BASE' '$ROOT_DIR' '$BUILD_DIR' '$PKG_NAME_VER' '$NO_META_PATCH_FLAG' '$TIS_PATCH_VER'" tar_and_spec_extract_to_git "$SPEC" "$PKG_BASE" "$ROOT_DIR" "$BUILD_DIR" "$PKG_NAME_VER" "$NO_META_PATCH_FLAG" "$TIS_PATCH_VER" RC=$? if [ $RC -ne 0 ]; then if [ $RC -eq 1 ]; then echo "ERROR: $FUNCNAME (${LINENO}): failed to extract srpm '$ORIG_SRPM_PATH'" fi return $RC fi local LOC LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" return 0 fi if [ $BUILD_NEEDED -eq 0 ]; then echo "SRPM build not required for '$PKG_BASE'" echo "===== Build complete for '$TARGET_FOUND' =====" echo return 0 fi echo "MAKE_SRPM=$MAKE_SRPM" echo "DATA=$DATA" if [ -d "$RPMBUILD_BASE/SRPMS" ]; then clean_srpm_dir $build_idx "$RPMBUILD_BASE/SRPMS" 1 fi if [ -d $RPMBUILD_BASE ]; then echo "rm -rf $RPMBUILD_BASE" \rm -rf "$RPMBUILD_BASE" fi echo "mkdir -p $WORK_BASE $SRPM_PATH $SPEC_PATH $SOURCES_PATH" mkdir -p "$WORK_BASE" && \ mkdir -p "$SRPM_PATH" && \ mkdir -p "$SPEC_PATH" && \ mkdir -p "$SOURCES_PATH" if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): Failed to create directories under: $WORK_BASE" fi \cp -L -f -v $SPECS_BASE/*.spec $SPEC_PATH/ if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): Failed to copy spec files from '$SPECS_BASE' to '$SPEC_PATH'" fi # # build # $MAKE_SRPM if [ $? -ne 0 ]; then echo "ERROR: $FUNCNAME (${LINENO}): script failed '$MAKE_SRPM'" echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_${build_idx} exit 1 fi # # Find age of youngest input file. # We will apply this as the creation/modification timestamp of the src.rpm we produce. # AGE=$(find $PKG_BASE -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) if [ -f $PKG_BASE/$DATA ]; then AGE2=$( cd $PKG_BASE srpm_source_build_data $DATA PATH_LIST="" if [ "x$SRC_DIR" != "x" ]; then if [ -d "$SRC_DIR" ]; then PATH_LIST="$PATH_LIST $SRC_DIR" fi fi if [ "x$COPY_LIST" != "x" ]; then PATH_LIST="$PATH_LIST $COPY_LIST" fi if [ "x$COPY_LIST_TO_TAR" != "x" ]; then PATH_LIST="$PATH_LIST $COPY_LIST_TO_TAR" fi if [ "x$PATH_LIST" == "x" ]; then echo "0" else AGE2=$(find $PATH_LIST -type f -exec stat --format '%Y' "{}" \; | grep -v '[/][.]git$' | grep -v '[/][.]git[/]' | sort -nr | head -n 1) echo "$AGE2" fi ) if [ $AGE2 -gt $AGE ]; then AGE=$AGE2 fi fi TS=$(date -d @$AGE +%Y-%m-%dT%H:%M:%S) for s in $(find $SRPM_PATH -name '*.src.rpm'); do \cp -L -f $s $SRPM_OUT/ ss=$(basename $s) touch $SRPM_OUT/$ss --date=$TS mkdir -p $SOURCES_DIR/$ss BIG_FLAG_FILE="$SOURCES_DIR/$ss/BIG" SLOW_FLAG_FILE="$SOURCES_DIR/$ss/SLOW" if [ $BUILD_IS_BIG -gt 0 ]; then echo $BUILD_IS_BIG > $BIG_FLAG_FILE else if [ -f $BIG_FLAG_FILE ]; then \rm -f $BIG_FLAG_FILE fi fi if [ $BUILD_IS_SLOW -gt 0 ]; then echo $BUILD_IS_SLOW > $SLOW_FLAG_FILE else if [ -f $SLOW_FLAG_FILE ]; then \rm -f $SLOW_FLAG_FILE fi fi \rm -f -v "$REFERENCE_MD5" \mv -v "$INPUT_FILES_MD5" "$REFERENCE_MD5" local SPEC_DIR=$(spec_cache_dir_from_srpm $SRPM_OUT/$ss) if [ -d $SPEC_DIR/BUILDS_VR ]; then for f in $(ls -1 $SPEC_DIR/BUILDS_VR); do for r in $(find $RPM_DIR -name "$f*rpm" 2>> /dev/null); do \rm -f -v $r done done fi local RESULT_DIR=$(result_dir_from_srpm $SRPM_OUT/$ss) if [ -d $RESULT_DIR ]; then echo "rm -rf $RESULT_DIR" \rm -rf $RESULT_DIR fi done echo "$TARGET_FOUND" >> $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_${build_idx} echo "===== Build complete for '$TARGET_FOUND' =====" echo fi fi return 0 } ( echo "$CMDLINE" if [ -L $BUILD_ROOT/repo ]; then REPO_DEST=$(readlink $BUILD_ROOT/repo) if [ "$REPO_DEST" != "$SRC_ROOT" ]; then echo "Error: MY_REPO changed since last build" echo " old path: $REPO_DEST" echo " new path: $SRC_ROOT" echo "Please run '$ME --clean' if you want to compile from a new source tree" exit 1 fi fi if [ ! -L $BUILD_ROOT/repo ]; then ln -s $SRC_ROOT $BUILD_ROOT/repo fi ALL=0 UNRESOLVED_TARGETS="" if [ "x$TARGETS" == "x" ]; then echo "make: all" ALL=1 else echo "make: $TARGETS" UNRESOLVED_TARGETS="$TARGETS" fi workers=0 max_workers=$MAX_WORKERS declare -A build_env init_build_env () { local i=0 local stop=$((max_workers-1)) for i in $(seq 0 $stop); do build_env[$i]='Idle' done } init_build_env get_idle_build_env () { local i=0 local stop=$((max_workers-1)) if [ $stop -ge 255 ]; then stop=254 fi for i in $(seq 0 $stop); do if [ ${build_env[$i]} == 'Idle' ]; then build_env[$i]='Busy' return $i fi done return 255 } set_build_env_pid () { local idx=$1 local val=$2 build_env[$idx]=$val } release_build_env () { local idx=$1 build_env[$idx]='Idle' } reaper () { local reaped=0 local last_reaped=-1 local i=0 local stop=$((max_workers-1)) local p=0 local ret=0 if [ $stop -ge 255 ]; then stop=254 fi while [ $reaped -gt $last_reaped ]; do last_reaped=$reaped for i in $(seq 0 $stop); do p=${build_env[$i]} if [ "$p" == "Idle" ] || [ "$p" == "Busy" ]; then continue fi # echo "test $i $p" kill -0 $p &> /dev/null if [ $? -ne 0 ]; then wait $p ret=$? workers=$((workers-1)) reaped=$((reaped+1)) release_build_env $i if [ $ret -ne 0 ]; then VERB="build" if [ $EDIT_FLAG -eq 1 ]; then VERB="edit" if [ $CLEAN_FLAG -eq 1 ]; then VERB="edit clean" fi else if [ $CLEAN_FLAG -eq 1 ]; then VERB="clean" fi fi sleep 1 echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'" cat "$LOG_DIR/$i" >> $LOG_DIR/errors echo "ERROR: $FUNCNAME (${LINENO}): Failed to $VERB src.rpm from source at 'b$i'" >> $LOG_DIR/errors echo "" >> $LOG_DIR/errors STOP_SCHEDULING=1 fi fi done done return $reaped } # Set up files to collect parallel build results ... mkdir -p $MY_WORKSPACE/tmp fn="$MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge" if [ -f $fn ]; then \rm -f $fn fi for n in $UNRESOLVED_TARGETS; do echo $n >> $fn; done if [ -f $fn ]; then sort $fn > $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS else \rm -f -v $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS touch $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS fi for i in $(seq 0 $((max_workers-1))); do for fn in $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_$i $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_$i $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i; do if [ -f $fn ]; then \rm -f -v $fn fi done \cp $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i done # Build src.rpm's LOG_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-log-XXXXXX) if [ "x$LOG_DIR" == "x" ]; then echo "failed to create temporary directory" exit 1; fi for GIT_ROOT in $GIT_LIST; do export GIT_BASE="$GIT_ROOT" if [ $STOP_SCHEDULING -eq 1 ]; then break; fi for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do if [ $STOP_SCHEDULING -eq 1 ]; then break; fi src_dir="$GIT_ROOT/$p" if [ -d $src_dir ]; then if [ -d $src_dir/${DISTRO} ]; then rel_dir=$(echo $src_dir | sed "s:^$SRC_BASE::") work_dir="$BUILD_INPUTS$rel_dir" # Free up a worker while [ $workers -ge $max_workers ]; do reaper reaped=$? if [ $reaped -eq 0 ]; then sleep 0.1 fi done workers=$((workers+1)) get_idle_build_env b=$? if [ $b -ge 255 ]; then echo "get_idle_build_env failed to find a free slot" exit 1 fi PREFIX="b$b" ( build_dir $b $src_dir $work_dir 2>&1 | sed "s#^#${PREFIX}: #" | tee $LOG_DIR/$b; exit ${PIPESTATUS[0]} ) & pp=$! set_build_env_pid $b $pp else echo "ERROR: $FUNCNAME (${LINENO}): Failed to find 'centos' in '$p', found in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi else echo "ERROR: $FUNCNAME (${LINENO}): Bad path '$p' in file '$GIT_ROOT/$PKG_DIRS_FILE'" fi done done # Wait for remaining workers to exit while [ $workers -gt 0 ]; do reaper reaped=$? if [ $reaped -eq 0 ]; then sleep 0.1 fi done if [ $STOP_SCHEDULING -eq 1 ]; then echo "============ Build failed =============" if [ -f $LOG_DIR/errors ]; then cat $LOG_DIR/errors fi \rm -rf $LOG_DIR exit 1 fi \rm -rf $LOG_DIR # Transfer results from files back into variables SRPM_REBUILT_LIST=$((for i in $(seq 0 $((max_workers-1))); do fn=$MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_$i if [ -f $fn ]; then cat $fn | tr '\n' ' ' fi done) | sed 's/ $//') SRPM_FAILED_REBUILD_LIST=$((for i in $(seq 0 $((max_workers-1))); do fn=$MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_$i if [ -f $fn ]; then cat $fn | tr '\n' ' ' fi done) | sed 's/ $//') UNRESOLVED_TARGETS=$(for i in $(seq 0 $((max_workers-1))); do if [ -f $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i ]; then comm -1 -2 $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_$i > $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge \mv $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS_merge $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS fi done cat $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS | tr '\n' ' ' | sed 's/ $//') \rm -rf $MY_WORKSPACE/tmp/SRPM_REBUILT_LIST_* $MY_WORKSPACE/tmp/SRPM_FAILED_REBUILD_LIST_* $MY_WORKSPACE/tmp/UNRESOLVED_TARGETS* 2>> /dev/null cleanup() { if [ -n "$MY_WORKSPACE/tmp" -a -d "${MY_WORKSPACE}/tmp" ]; then echo "clean up $MY_WORKSPACE/tmp" rm -rf $MY_WORKSPACE/tmp fi } # cleanup temp files when system exiting trap cleanup EXIT # Try to find and clean orphaned and discontinued .src.rpm's if [ $ALL -eq 1 ]; then echo echo "Auditing for obsolete srpms" AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX) if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then for GIT_ROOT in $GIT_LIST; do for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do ( src_dir="$GIT_ROOT/$p" if [ -d $src_dir ]; then if [ -d $src_dir/$DISTRO ]; then for f in $(find $src_dir/${DISTRO} -name '*.spec' | sort -V); do NAME=$(spec_find_tag Name "$f" 2>> /dev/null) if [ $? -eq 0 ]; then touch "$AUDIT_DIR/$NAME" fi done if [ -f $src_dir/$SRPM_LIST_PATH ]; then for p in $(grep -v '^#' $src_dir/$SRPM_LIST_PATH | grep -v '^$'); do ORIG_SRPM_PATH="" # absolute path source rpms echo "$p" | grep "^/" >/dev/null && ORIG_SRPM_PATH=$p if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle repo: definitions echo "$p" | grep "^repo:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^repo:%$REPO_DOWNLOADS_ROOT/%") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle repo: definitions echo "$p" | grep "^3rd_party:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^3rd_party:%$THIRD_PARTY_ROOT/%") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # handle mirror: definitions echo "$p" | grep "^mirror:" >/dev/null && ORIG_SRPM_PATH=$(echo $p | sed "s%^mirror:%$MIRROR_ROOT/%" | sed "s#CentOS/tis-r3-CentOS/kilo/##" | sed "s#CentOS/tis-r3-CentOS/mitaka/##") fi if [ "${ORIG_SRPM_PATH}x" == "x" ]; then # we haven't found a valid prefix yet, so assume it's a legacy # file (mirror: interpretation) ORIG_SRPM_PATH="$MIRROR_ROOT/$p" fi if [ -f $ORIG_SRPM_PATH ]; then NAME=$(rpm -q --queryformat '%{NAME}\n' -p $ORIG_SRPM_PATH 2>> /dev/null) if [ $? -eq 0 ]; then touch "$AUDIT_DIR/$NAME" fi fi done fi fi fi ) & done done echo "waiting" wait echo "Auditing for obsolete srpms Phase 2" for r in $(find $SRPM_OUT -name '*.src.rpm' | sort -V); do ( NAME=$(rpm -q --queryformat '%{NAME}\n' -p $r 2>> /dev/null) ALT_NAME=$(echo $NAME | sed "s#-$BUILD_TYPE\$##") FOUND=0 if [[ -f "$AUDIT_DIR/$NAME" || ( "$BUILD_TYPE" != "std" && -f "$AUDIT_DIR/$ALT_NAME" ) ]]; then FOUND=1 fi if [ $FOUND -eq 0 ]; then for INPUT_DIR in $(find $BUILD_INPUTS -name $NAME | sort -V); do if [ -d "$INPUT_DIR/rpmbuild/SRPMS" ]; then clean_srpm_dir $build_idx "$INPUT_DIR/rpmbuild/SRPMS" 0 fi if [ -d $INPUT_DIR ]; then echo "rm -rf $r" \rm -rf $r fi done if [ -f $r ]; then \rm -f -v $r fi fi ) & done echo "waiting" wait \rm -rf "$AUDIT_DIR" fi echo "Auditing for obsolete srpms done" fi if [ $CLEAN_FLAG -eq 1 ]; then if [ $ALL -eq 1 ]; then \rm -rf $BUILD_INPUTS \rm -rf $SOURCE_OUT/*.src.rpm fi fi if [ $EDIT_FLAG -ne 1 ]; then echo "==== Update repodata =====" mkdir -p $SRPM_OUT/repodata for d in $(find -L $SRPM_OUT -type d -name repodata); do (cd $d/.. \rm -rf repodata $CREATEREPO $(pwd) ) done echo "==== Update repodata complete =====" fi FINAL_RC=0 if [ $CLEAN_FLAG -eq 0 ] && [ $EDIT_FLAG -eq 0 ]; then echo "" if [ "$SRPM_FAILED_REBUILD_LIST" != "" ]; then N=$(echo "$SRPM_FAILED_REBUILD_LIST" | wc -w) echo "Failed to build $N packages:" echo " $SRPM_FAILED_REBUILD_LIST" FINAL_RC=1 fi if [ "$SRPM_REBUILT_LIST" != "" ]; then N=$(echo "$SRPM_REBUILT_LIST" | wc -w) echo "Successfully built $N packages:" echo " $SRPM_REBUILT_LIST" echo "" echo "Compiled src.rpm's can be found here: $SRPM_OUT" fi if [ "$SRPM_FAILED_REBUILD_LIST" == "" ] && [ "$SRPM_REBUILT_LIST" == "" ]; then echo "No packages required a rebuild" fi fi if [ "$UNRESOLVED_TARGETS" != "" ]; then echo "" echo "ERROR: $FUNCNAME (${LINENO}): failed to resolve build targets: $UNRESOLVED_TARGETS" FINAL_RC=1 fi exit $FINAL_RC ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-srpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]}