From 58b942baf160da02171b985071f0b8ef4531698f Mon Sep 17 00:00:00 2001 From: Scott Little Date: Thu, 30 Aug 2018 17:10:07 -0400 Subject: [PATCH] Capture the git context of the build First step in build avoidance is the creation of a reference build. We need to capture the state of all the git trees that contribute to the reference build. This update will save the git context to $MY_WORKSPACE/CONTEXT The context file format is actually a shell script that can be run to recreate the build context. repo init ... repo sync cd $MY_REPO/.. source .../CONTEXT Sample CONTEXT file .... (cd ./cgcs-root && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) (cd ./cgcs-root/stx/git/calico && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) (cd ./cgcs-root/stx/git/ceilometer && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) ... (cd ./stx-tools && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) (cd ./.repo/manifests && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) (cd ./.repo/repo && git checkout -f 12a159594130d75aedf40da3f3b1e6bd7f9ff375) This update also the git_list function, which populates the previously introduced GIT_LIST variable, and can be used in several other code paths where we want to find all gits in a directory tree. Tested build/clean/edit parallel/serial Story: 2002835 Task: 22754 Change-Id: I5009b8a360bdb4a03e5a4e83430b7f2ef135115e Signed-off-by: Scott Little --- build-tools/build-pkgs-parallel | 14 +++-- build-tools/build-pkgs-serial | 13 +++-- build-tools/build-srpms-parallel | 14 ++--- build-tools/build-srpms-serial | 13 ++--- build-tools/git-utils.sh | 91 ++++++++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 21 deletions(-) diff --git a/build-tools/build-pkgs-parallel b/build-tools/build-pkgs-parallel index f64ac031..d0b0b48c 100755 --- a/build-tools/build-pkgs-parallel +++ b/build-tools/build-pkgs-parallel @@ -4,6 +4,7 @@ BUILD_PKGS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_PKGS_PARALLEL_DIR}/git-utils.sh" source "${BUILD_PKGS_PARALLEL_DIR}/spec-utils" @@ -141,8 +142,7 @@ find_targets () { local RESULT=" " local FOUND=0 - for g in $(find $MY_REPO -type d -name .git); do - d=$(dirname $g) + for d in $GIT_LIST; do if [ -f $d/$centos_pkg_dirs ]; then for d2 in $(grep -v '^#' $d/$centos_pkg_dirs); do name="" @@ -364,7 +364,15 @@ function progbar() } # Create $MY_WORKSPACE if it doesn't exist already -mkdir -p $MY_WORKSPACE +mkdir -p "${MY_WORKSPACE}" +if [ $? -ne 0 ]; then + echo "Failed to create directory '${MY_WORKSPACE}'" + exit 1 +fi + +echo "Capture build context" +git_context > "${MY_WORKSPACE}/CONTEXT" + if [ $STD_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then diff --git a/build-tools/build-pkgs-serial b/build-tools/build-pkgs-serial index 2a1326f3..a0061d59 100755 --- a/build-tools/build-pkgs-serial +++ b/build-tools/build-pkgs-serial @@ -4,6 +4,7 @@ BUILD_PKGS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source "${BUILD_PKGS_SERIAL_DIR}/git-utils.sh" source "${BUILD_PKGS_SERIAL_DIR}/spec-utils" @@ -97,8 +98,7 @@ find_targets () { local RESULT=" " local FOUND=0 - for g in $(find $MY_REPO -type d -name .git); do - d=$(dirname $g) + for d in $GIT_LIST; do if [ -f $d/$centos_pkg_dirs ]; then for d2 in $(grep -v '^#' $d/$centos_pkg_dirs); do name="" @@ -316,7 +316,14 @@ function progbar() } # Create $MY_WORKSPACE if it doesn't exist already -mkdir -p $MY_WORKSPACE +mkdir -p "${MY_WORKSPACE}" +if [ $? -ne 0 ]; then + echo "Failed to create directory '${MY_WORKSPACE}'" + exit 1 +fi + +echo "Capture build context" +git_context > "${MY_WORKSPACE}/CONTEXT" if [ $STD_BUILD -eq 1 ]; then if [ "x$TARGETS" == "x " ] || [ "$TARGETS_STD" != " " ] || [ "$TARGETS_MISC" != " " ]; then diff --git a/build-tools/build-srpms-parallel b/build-tools/build-srpms-parallel index 83e68ce2..34f192b9 100755 --- a/build-tools/build-srpms-parallel +++ b/build-tools/build-srpms-parallel @@ -5,6 +5,7 @@ export ME=$(basename "$0") CMDLINE="$ME $@" BUILD_SRPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source $BUILD_SRPMS_PARALLEL_DIR/git-utils.sh source $BUILD_SRPMS_PARALLEL_DIR/spec-utils source $BUILD_SRPMS_PARALLEL_DIR/srpm-utils source $BUILD_SRPMS_PARALLEL_DIR/classify @@ -677,7 +678,8 @@ build_dir_srpm () { return $RC fi - local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git tag | grep "pre_wrs_$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 ) + local LOC + LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" @@ -956,7 +958,8 @@ build_dir_spec () { return $RC fi - local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git branch --all | grep "$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 ) + local LOC + LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" @@ -1235,8 +1238,8 @@ if [ "x$LOG_DIR" == "x" ]; then echo "failed to create temporary directory" exit 1; fi -for g in $(find "$SRC_BASE" -type d -name .git | sort -V); do - GIT_ROOT=$(dirname $g) + +for GIT_ROOT in $GIT_LIST; do export GIT_BASE="$GIT_ROOT" if [ $STOP_SCHEDULING -eq 1 ]; then break; @@ -1330,8 +1333,7 @@ if [ $ALL -eq 1 ]; then echo "Auditing for obsolete srpms" AUDIT_DIR=$(mktemp -d $MY_WORKSPACE/tmp/$USER-$ME-audit-XXXXXX) if [ $? -eq 0 ] && [ "x$AUDIT_DIR" != "x" ]; then - for g in $(find $SRC_BASE -type d -name .git | sort -V); do - GIT_ROOT=$(dirname $g) + for GIT_ROOT in $GIT_LIST; do for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do ( src_dir="$GIT_ROOT/$p" diff --git a/build-tools/build-srpms-serial b/build-tools/build-srpms-serial index baf75d27..b20fe791 100755 --- a/build-tools/build-srpms-serial +++ b/build-tools/build-srpms-serial @@ -6,6 +6,7 @@ CMDLINE="$ME $@" BUILD_SRPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +source $BUILD_SRPMS_SERIAL_DIR/git-utils.sh source $BUILD_SRPMS_SERIAL_DIR/spec-utils source $BUILD_SRPMS_SERIAL_DIR/srpm-utils source $BUILD_SRPMS_SERIAL_DIR/classify @@ -647,7 +648,8 @@ build_dir_srpm () { return $RC fi - local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git tag | grep "pre_wrs_$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 ) + local LOC + LOC=$(git_list_containing_tag "${PKG_ROOT_DIR}/gits" "pre_wrs_$PKG_NAME_VER" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" @@ -922,7 +924,8 @@ build_dir_spec () { return $RC fi - local LOC=$(for g in $(find $PKG_ROOT_DIR/gits -type d -name .git); do d=$(dirname $g); (cd $d; git branch --all | grep "$PKG_NAME_VER" >> /dev/null; if [ $? -eq 0 ]; then echo $d; fi); done | head -n 1 ) + local LOC + LOC=$(git_list_containing_branch "${PKG_ROOT_DIR}/gits" "${PKG_NAME_VER}" | head -n 1 ) echo "===== '$TARGET_FOUND' has been extracted for editing. =====" echo "===== Metadata can be found at: $PKG_ROOT_DIR/rpmbuild" echo "===== Source code can be found at: $LOC" @@ -1093,8 +1096,7 @@ if [ $EDIT_FLAG -eq 0 ]; then fi fi -for g in $(find "$SRC_BASE" -type d -name .git | sort -V); do - GIT_ROOT=$(dirname $g) +for GIT_ROOT in $GIT_LIST; do export GIT_BASE="$GIT_ROOT" for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do src_dir="$GIT_ROOT/$p" @@ -1136,8 +1138,7 @@ if [ $ALL -eq 1 ]; then echo echo "Auditing for obsolete srpms" PACKAGES_CONSIDERED="" - for g in $(find $SRC_BASE -type d -name .git | sort -V); do - GIT_ROOT=$(dirname $g) + for GIT_ROOT in $GIT_LIST; do for p in $(cat $GIT_ROOT/$PKG_DIRS_FILE 2>> /dev/null); do src_dir="$GIT_ROOT/$p" if [ -d $src_dir ]; then diff --git a/build-tools/git-utils.sh b/build-tools/git-utils.sh index b4c99c2c..645f2b82 100755 --- a/build-tools/git-utils.sh +++ b/build-tools/git-utils.sh @@ -5,10 +5,95 @@ # # -# A place for any functions relating to git, or the git hierarchy created +# A place for any functions relating to git, or the git hierarchy created # by repo manifests. # -# GIT_LIST: A list of root directories for all the gits under $MY_REPO -export GIT_LIST=$(find $MY_REPO -type d -name '.git' -exec dirname {} \;) +# +# git_list : +# Return a list of git root directories found under +# +git_list () { + local DIR=${1} + find "${DIR}" -type d -name '.git' -exec dirname {} \; | sort -V +} + + +# GIT_LIST: A list of root directories for all the gits under $MY_REPO/.. +# as absolute paths. +export GIT_LIST=$(git_list "$(dirname "${MY_REPO}")") + + +# GIT_LIST_REL: A list of root directories for all the gits under $MY_REPO/.. +# as relative paths. +export GIT_LIST_REL=$(for p in $GIT_LIST; do + echo .${p#$(dirname ${MY_REPO})}; + done) + + +# +# git_list_containing_branch : +# Return a list of git root directories found under and +# having branch . The branch need not be current branch. +# + +git_list_containing_branch () { + local DIR="${1}" + local BRANCH="${2}" + + local d + for d in $(git_list "${DIR}"); do + ( + cd "$d" + git branch --all | grep -q "$BRANCH" + if [ $? -eq 0 ]; then + echo "$d" + fi + ) + done +} + + +# +# git_list_containing_tag : +# Return a list of git root directories found under and +# having tag . +# + +git_list_containing_tag () { + local DIR="${1}" + local TAG="${2}" + + local d + for d in $(git_list "${DIR}"); do + ( + cd "$d" + git tag | grep -q "$TAG" + if [ $? -eq 0 ]; then + echo "$d" + fi + ) + done +} + + +# +# git_context: +# Returns a bash script that can be used to recreate the current git context, +# +# Note: all paths are relative to $MY_REPO/.. +# + +git_context () { + ( + cd $MY_REPO + for d in $GIT_LIST_REL; do + ( + cd ${d} + echo -n "(cd ${d} && git checkout -f " + echo "$(git rev-list HEAD -1))" + ) + done + ) +}