diff --git a/build-tools/build-iso b/build-tools/build-iso index d6c6ccc6..a830cae2 100755 --- a/build-tools/build-iso +++ b/build-tools/build-iso @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018-2019 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -18,6 +18,10 @@ BUILD_ISO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_ISO_DIR}/image-utils.sh" source "${BUILD_ISO_DIR}/git-utils.sh" +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source "${BUILD_ISO_DIR}/pkg-manager-utils.sh" + usage () { echo "" echo "Usage: " @@ -40,8 +44,8 @@ usage () { MY_YUM_CONF="" STD_REPO_ID="local-std" RT_REPO_ID="local-rt" -LOWER_LAYER_STD_REPO_ID="StxCentos7Distro" -LOWER_LAYER_RT_REPO_ID="StxCentos7Distro-rt" +LOWER_LAYER_STD_REPO_ID="" +LOWER_LAYER_RT_REPO_ID="" NPROCS=$(nproc) @@ -77,13 +81,13 @@ function install_pkg_list { \cd $OUTPUT_DIST_DIR/isolinux/Packages $INTERNAL_REPO_ROOT/build-tools/build_iso/cgts_deps.sh --deps=$PKGLIST - + if [ $? -ne 0 ] then echo "Could not install dependencies" exit 1 fi - + # clean up echo "Removing local-std yum repo $CGCS_REPO_DIR/repodata" echo "Removing local-rt yum repo $CGCS_RT_REPO_DIR/repodata" @@ -111,7 +115,7 @@ function make_report { for PKGLIST in $PKGLISTFILES; do while read PKG; do PKG=`echo $PKG | sed "s/#.*//"`; - if [ "${PKG}x" != "x" ]; then + if [ "${PKG}x" != "x" ]; then echo $PKG >> $REPORT_FILE fi done < $PKGLIST @@ -127,7 +131,7 @@ function make_report { echo "-----------------" >> $REPORT_FILE echo " WARNINGS " >> $REPORT_FILE echo "-----------------" >> $REPORT_FILE - + # Note that the warnings file may have multiple lines for the same # missing dependency. A sort | uniq solves this so we don't duplicate # warnings @@ -158,13 +162,17 @@ function init_vars { exit 1 fi + # LOWER_LAYER_STD_REPO_ID should be something like StxCentos7Distro or StxCentos8Distro + LOWER_LAYER_STD_REPO_ID=$(grep '\[StxCentos.*Distro\]' ${MY_YUM_CONF} | sed -e 's/^\[//' -e 's/\].*//') + LOWER_LAYER_RT_REPO_ID=$(grep '\[StxCentos.*Distro-rt\]' ${MY_YUM_CONF} | sed -e 's/^\[//' -e 's/\].*//') + DISTRO_REPO_DIR=$(for d in $(grep baseurl $MY_YUM_CONF | grep file: | awk -F : '{print $2}' | sed 's:///:/:g'); do if [ -d $d/images ]; then echo $d ;fi; done) ##################################### # Output definitons # where to put stuff (curent dir unless MY_WORKSPACE defined) - OUTPUT_DIR="$PWD/export" + OUTPUT_DIR="$PWD/export" if [ ! -z "$MY_WORKSPACE" ] && [ -d "$MY_WORKSPACE" ] ; then OUTPUT_DIR="$MY_WORKSPACE/export" CGCS_REPO_DIR="$MY_WORKSPACE/std/rpmbuild/RPMS" @@ -316,7 +324,7 @@ function init_output_dir { date +%s.%N > $OUTPUT_DIST_DIR/isolinux/.discinfo echo $PLATFORM_RELEASE >> $OUTPUT_DIST_DIR/isolinux/.discinfo echo "x86_64" >> $OUTPUT_DIST_DIR/isolinux/.discinfo - + \cp -L -ru $DISTRO_REPO_DIR/isolinux/* $OUTPUT_DIST_DIR/isolinux/ \cp -L -ru $DISTRO_REPO_DIR/images/pxeboot $OUTPUT_DIST_DIR/isolinux/images/ @@ -342,7 +350,7 @@ function init_output_dir { perl -p -i -e "s/device=sda/device=${DEVICE}/g" $OUTPUT_DIST_DIR/isolinux/isolinux.cfg fi - # Copy UEFI files + # Copy UEFI files \cp -L -ru $DISTRO_REPO_DIR/EFI/* $OUTPUT_DIST_DIR/isolinux/EFI/ \cp -L "$BSP_FILES_PATH/grub.cfg" "$OUTPUT_DIST_DIR/isolinux/EFI/BOOT/grub.cfg" \cp -L "$BSP_FILES_PATH/pxeboot_grub.cfg" "$OUTPUT_DIST_DIR/isolinux/pxeboot/pxeboot_grub.cfg" @@ -472,7 +480,7 @@ function final_touches { # create the repo \cd $OUTPUT_DIST_DIR/isolinux $CREATEREPO -q -g ../comps.xml . - + # build the ISO printf "Building image $OUTPUT_FILE\n" \cd $OUTPUT_DIST_DIR @@ -485,7 +493,7 @@ function final_touches { -eltorito-alt-boot \ -e images/efiboot.img \ -no-emul-boot \ - isolinux/ + isolinux/ isohybrid --uefi $OUTPUT_FILE implantisomd5 $OUTPUT_FILE @@ -495,7 +503,7 @@ function final_touches { function extract_pkg_from_local_repo { local pkgname=$1 - local yum_conf=$2 + local pkg_mgr_conf=$2 shift 2 local repoid="" @@ -505,7 +513,14 @@ function extract_pkg_from_local_repo { repoid_arg+=" --repoid=${repoid}" done - local pkgfile=$(TMPDIR=$TMP_DIR repoquery --config=${yum_conf} ${repoid_arg} --location -q ${pkgname}) + echo "TMPDIR=$TMP_DIR"\ + "${REPOQUERY} --config=${pkg_mgr_conf} ${repoid_arg}"\ + "${REPOQUERY_SUB_COMMAND} --location"\ + "--arch=noarch,x86_64 -q ${pkgname}" + local pkgfile=$(TMPDIR=$TMP_DIR \ + ${REPOQUERY} --config=${pkg_mgr_conf} ${repoid_arg} \ + ${REPOQUERY_SUB_COMMAND} --location \ + --arch=noarch,x86_64 -q ${pkgname}) if [ -z "${pkgfile}" ]; then echo "Could not find package $pkgname in $@" exit 1 @@ -605,7 +620,7 @@ EOM echo "Error: Could not copy all files from installer" exit 1 fi - + \cp --preserve=all www/pages/feed/rel-*/LiveOS/squashfs.img $OUTPUT_DIST_DIR/isolinux/LiveOS if [ $? -ne 0 ]; then echo "Error: Could not copy squashfs from LiveOS" diff --git a/build-tools/build-pkgs b/build-tools/build-pkgs index edd980f8..da9d65d0 100755 --- a/build-tools/build-pkgs +++ b/build-tools/build-pkgs @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,6 +15,10 @@ BUILD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +# Set REPOQUERY and REPOQUERY_SUB_COMMAND for our build environment. +source "${BUILD_PKGS_DIR}/pkg-manager-utils.sh" + + usage () { echo "" echo "Usage: " @@ -75,11 +79,15 @@ make_cache_current_rpms () { if [ -d $MY_WORKSPACE/$build_type/rpmbuild/$rpm_type/repodata ]; then current=$MY_WORKSPACE/$build_type/rpmbuild/$rpm_type/ - repoquery \ + ${REPOQUERY} \ --repofrompath=$build_type-$rpm_type,$current \ - --repoid=$build_type-$rpm_type --arch=noarch,src,x86_64 -a \ + --repoid=$build_type-$rpm_type --arch=noarch,src,x86_64 \ + ${REPOQUERY_SUB_COMMAND} \ + --all \ --qf "%-10{repoid} %-40{name} %-10{version} %-10{release}" \ >> ${FILE} + + \rm -rf $TMP_DIR/yum-$USER-* fi done; done; diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index 4ca2202e..e2830c24 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,7 +15,7 @@ # The location of packages to be built is # $MY_WORKSPACE//rpmbuild/SRPMS. # -# The build order is a derived from the BuildRequires in the +# The build order is a derived from the BuildRequires in the # spec files in the src.rpms. Note that the BuildRequires sometimes # create dependency loops, so no correct order can be computed. In these # cases we add a retry loop. As long as one new package builds, we @@ -24,9 +24,12 @@ # messages go by! # - export ME=$(basename "$0") CMDLINE="$ME $@" +BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set PKG_MANAGER for our build environment. +source "${BUILD_RPMS_PARALLEL_DIR}/pkg-manager-utils.sh" # Build for distribution. Currently 'centos' is only supported value. @@ -43,22 +46,22 @@ MAX_MEM_PER_WORKER=11 # Note: tmpfs is typically 2.5 gb when compiling many small jobs MIN_MEM_PER_WORKER=3 -# Maximum number of disk based parallel build environments +# Maximum number of disk based parallel build environments MAX_DISK_BASED_WORKERS=2 # Minimum space in gb for each disk based parallel build environment MIN_DISK_PER_WORKER=20 -# How many srpms to build before we add another parallel build environment +# How many srpms to build before we add another parallel build environment MIN_TASKS_PER_CORE=3 # Max number of new concurrent builds to allow for MAX_SHARE_FACTOR=4 -# Always leave at least MEMORY_RESERVE gb of available mem for the system +# Always leave at least MEMORY_RESERVE gb of available mem for the system MEMORY_RESERVE=1 -# These two values will be reassigned in the 'compute_resources' subroutine +# These two values will be reassigned in the 'compute_resources' subroutine MOCKCHAIN_RESOURCE_ALLOCATION="" MAX_WORKERS=$ABSOLUTE_MAX_WORKERS @@ -164,7 +167,7 @@ create-no-clean-list () { local install_groups="" local install_packages="" local p - + for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do if [[ $p == @* ]] ; then install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-)) @@ -172,37 +175,44 @@ create-no-clean-list () { install_packages=$(join_by ' ' $install_packages $p) fi done - + local noclean_last_list_len=0 local noclean_list="" local tmp_list="" local g - + for g in $install_groups; do - tmp_list=$(yum -c $MY_YUM_CONF groupinfo $g 2>> /dev/null | awk 'f;/Mandatory Packages:/{f=1}' | cut -c 5-) + # Find mandatory packages in the group. + # Discard anything before (and including) 'Mandatory Packages:' + # and anything after (and including) 'Optional Packages:'. + # Also discard leading spaces or '+' characters. + tmp_list=$(${PKG_MANAGER} -c $MY_YUM_CONF groupinfo $g 2>> /dev/null \ + | awk 'f;/Mandatory Packages:/{f=1}' \ + | sed -n '/Optional Packages:/q;p' \ + | sed 's#[ +]*##') noclean_list=$(join_by ' ' $noclean_list $tmp_list) done - + noclean_list=$(join_by ' ' $noclean_list $install_packages) noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) - + while [ $noclean_list_len -gt $noclean_last_list_len ]; do noclean_last_list_len=$noclean_list_len - noclean_list=$( (yum -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) + noclean_list=$( (${PKG_MANAGER} -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) done - + echo $noclean_list > $NO_CLEAN_LIST_FILE fi cat $NO_CLEAN_LIST_FILE } -str_lst_contains() { +str_lst_contains() { TARGET="$1" LST="$2" - if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then + if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then return 0 else return 1 @@ -214,7 +224,7 @@ compute_resources () { local b echo "" - for f in $@; do + for f in $@; do b=$(basename $f) if [ -f $SOURCES_DIR/$b/BIG ] || [ ]; then weight=$((weight+MIN_TASKS_PER_CORE)) @@ -232,7 +242,7 @@ compute_resources () { local cpus=$(number_of_cpus) local num_users=$(sqrt $users) local num_build=$(number_of_builds_in_progress) - num_build=$((num_build+1)) + num_build=$((num_build+1)) echo "compute_resources: total: cpus=$cpus, mem=$mem, disk=$disk, weight=$weight, num_build=$num_build" # What fraction of the machine will we use @@ -311,7 +321,9 @@ create_lst () { recreate_repodata () { local DIR=${1} - (cd $DIR + ( + mkdir -p $DIR + cd $DIR if [ -f repodata/*comps*xml ]; then \mv repodata/*comps*xml comps.xml fi @@ -402,11 +414,11 @@ union () { echo $b fi done -} +} # # returns 0 if element is in the array -# +# # e.g. contains ARRAY $SEEKING && echo "$SEEKING is in 'ARRAY'" # contains () { @@ -453,7 +465,7 @@ build_order_recursive () { needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') ) needs_list=( $(intersection needs remainder_list) ) for((idx=0;idx<${#needs_list[@]};idx++)); do - build_order_recursive ${needs_list[idx]} + build_order_recursive ${needs_list[idx]} done echo $target break @@ -503,7 +515,7 @@ build_order () { UNORDERED_LIST=( ${original_list[@]} ) while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do element=${UNORDERED_LIST[0]} - build_order_recursive $element + build_order_recursive $element done fi } @@ -621,7 +633,7 @@ umount_mock_root_as_tmpfs () { return 0 fi mock_tmpfs_umount $mount_dir &> /dev/null - + rc=$? if [ $rc -ne 0 ]; then echo "FAILED: mock_tmpfs_umount $mount_dir" @@ -652,7 +664,7 @@ kill_descendents () recursive_promote_children="" recursive_children=$(pgrep -P $kill_pid) fi - + if [ $iteration -eq 0 ]; then other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done) if [ "$other_children" != "" ]; then @@ -667,7 +679,7 @@ kill_descendents () fi for pid in $recursive_children; do - kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) + kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) done for pid in $recursive_promote_children; do kill_descendents "$pid" 1 1 $((iteration + 1)) @@ -946,11 +958,11 @@ mock_clean_cfg () { echo "==================================" mock_clean_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --clean" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --clean + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --clean" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --clean ### Note: this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock echo "==================================" mock_clean_cache_all_cfg $CFG @@ -1034,10 +1046,10 @@ mock_partial_clean_cfg () { local RPMS_CLEAN_LIST="" local NEED_FULL_MOCK_CLEAN=0 for r in $RPMS_LIST; do - if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then + if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r) - else - echo "Can't remove '$r' from mock environment without a wipe"; + else + echo "Can't remove '$r' from mock environment without a wipe"; NEED_FULL_MOCK_CLEAN=1 fi done @@ -1050,19 +1062,19 @@ mock_partial_clean_cfg () { # Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST" CMD='LST="'$RPMS_CLEAN_LIST'"; - DELETE_LIST=""; - for r in $LST; do - FOUND=$(rpm -q $r) ; - if [ $? -eq 0 ]; then - DELETE_LIST="$DELETE_LIST $FOUND"; - fi; - done; - echo "uninstalling these packages: $DELETE_LIST"; + DELETE_LIST=""; + for r in $LST; do + FOUND=$(rpm -q $r) ; + if [ $? -eq 0 ]; then + DELETE_LIST="$DELETE_LIST $FOUND"; + fi; + done; + echo "uninstalling these packages: $DELETE_LIST"; if [ "$DELETE_LIST" != "" ]; then rpm -e --nodeps $DELETE_LIST; fi' - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1112,8 +1124,8 @@ mock_clean_cache_cfg () { clean_yum_cache_cfg $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1141,8 +1153,8 @@ mock_clean_cache_all_cfg () { echo "==================================" clean_yum_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" } @@ -1169,9 +1181,16 @@ mock_clean_metadata_cfg () { return 1 fi - CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';') - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP + CMD=$((cat $CFG; \ + grep config_opts\\[\'yum.conf\'\\\] $CFG | \ + sed 's#\\n#\n#g') | \ + grep '^[[]' | \ + grep -v main | \ + sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --enablerepo=#" -e 's#$# clean metadata#' | \ + sort -u | \ + tr '\n' ';') + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1228,7 +1247,7 @@ update_cgcs_repo () { if [ "x$NEW_UNTRACKED" != "x" ]; then NEED_REBUILD=1 fi - + if [ $NEED_REBUILD -eq 1 ]; then NEED_MOCK_CLEAN=1 echo "" @@ -1274,7 +1293,7 @@ mock_clean_mounts_dir () { fi fi if [ $RC -eq 1 ]; then - echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." + echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." echo "Ask your system administrator to umount '$MOUNT'." exit 1 fi @@ -1319,7 +1338,7 @@ clean_yum_cache_cfg () { local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum") local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock" # echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'" - + if [ "$YUM_CACHE_MOUNT" != "" ]; then mock_clean_mounts_dir "$YUM_CACHE_MOUNT" fi @@ -1358,8 +1377,8 @@ mock_update_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update echo "==================================" } @@ -1368,8 +1387,8 @@ mock_init_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init echo "==================================" } @@ -1388,14 +1407,14 @@ mock_update_or_init_cfg () { if [ -d $ROOT_DIR/root ]; then echo "Updating the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update &> $TMP RC=$? else echo "Init the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init &> $TMP RC=$? fi if [ $RC -ne 0 ]; then @@ -1411,7 +1430,7 @@ mock_update_or_init () { local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev) ( mock_update_or_init_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) & done - wait + wait echo "${FUNCNAME[0]}: out" } @@ -1429,12 +1448,12 @@ NO_DESCENDANTS=0 NO_REQUIRED=0 NO_AUTOCLEAN=0 NO_BUILD_INFO=0 -HELP=0 +HELP=0 CLEAN_FLAG=0 FORMAL_FLAG=0 CAREFUL=0 DEP_TEST_FLAG=0 - + # read the options TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,dep-test,clean,tmpfs-clean,formal,careful,help,layer: -n "$ME" -- "$@") @@ -1470,7 +1489,7 @@ while true ; do --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac -done +done # Reset variables if [ -n "$MY_WORKSPACE" ]; then @@ -1729,7 +1748,7 @@ clean_list () { echo "rm -rf $d" \rm -rf "$d" 2>> /dev/null done - + else rs=$(rpm_get_srpm $r) if [[ "$rs" != "$sn"-[0-9]* ]]; then @@ -1742,7 +1761,7 @@ clean_list () { fi done done - + TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s") if [ $CLEAN_FLAG -eq 1 ]; then @@ -1782,7 +1801,7 @@ clean_list () { \rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null mock_clean - else + else # If dependency test if [ $DEP_TEST_FLAG -eq 1 ]; then mock_clean @@ -2045,7 +2064,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done # For non-std build, and if non specific build targets are named, then search all - # packages that we might build and check if they require a package that DID build + # packages that we might build and check if they require a package that DID build # in the std build. If so build the package as a secondary target, even though the # primary target was from a different build_type. if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then @@ -2105,7 +2124,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done fi - # If the kernel or kernel-rt packages where absent from the primary build targets, but + # If the kernel or kernel-rt packages where absent from the primary build targets, but # added as a secondary target, then make sure all out-of-tree kernel modules are also # added. for n in kernel kernel-rt; do @@ -2234,9 +2253,9 @@ echo "SRPMS_LIST = $SRPMS_LIST" echo "RPMS_LIST = $RPMS_LIST" -echo +echo if [ $CLEAN_FLAG -eq 0 ]; then - # pre-create these directories as $USER, + # pre-create these directories as $USER, # else mock will create them as root and fails to clean them. # Note: keep these in sync with mockchain-parallel! for i in $(seq 0 $((MAX_WORKERS-1))); do @@ -2299,15 +2318,28 @@ mock_clean_metadata echo echo "Building" +recreate_repodata $BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR + CMD_PREFIX="" if [ -x /bin/ionice ]; then CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice " fi -CMD_OPTIONS="-m --no-clean -m --no-cleanup-after" -if [ $CAREFUL -eq 1 ]; then - CMD_OPTIONS="-m --no-cleanup-after" +REAL_MOCKCHAIN=0 +MOCK_PASSTHROUGH="-m" +MOCKCHAIN="mockchain-parallel" +CHAIN_OPTION="" +if file $(which mockchain) | grep -q 'Python script'; then + REAL_MOCKCHAIN=1 fi + +CMD_OPTIONS="$MOCK_PASSTHROUGH --no-clean $MOCK_PASSTHROUGH --no-cleanup-after" +if [ $CAREFUL -eq 1 ]; then + CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after" +fi + +CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG" + echo "CAREFUL=$CAREFUL" # Sets WORKERS and MOCKCHAIN_RESOURCE_ALLOCATION @@ -2315,13 +2347,13 @@ compute_resources $SRPMS_LIST if [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then - CMD_OPTIONS="$CMD_OPTIONS --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE" + CMD_OPTIONS+=" --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE" fi if [ -f "$RPM_DIRECT_REQUIRES_FILE" ]; then - CMD_OPTIONS="$CMD_OPTIONS --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE" + CMD_OPTIONS+=" --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE" fi if [ -f "$RPM_TO_SRPM_MAP_FILE" ]; then - CMD_OPTIONS="$CMD_OPTIONS --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE" + CMD_OPTIONS+=" --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE" fi @@ -2342,12 +2374,12 @@ echo "MAX_WORKERS=$MAX_WORKERS" echo "MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION" -CMD="$CMD_PREFIX mockchain-parallel -r $BUILD_CFG -l $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS -m --rebuild $SRPMS_LIST" +CMD="$CMD_PREFIX $MOCKCHAIN --root $BUILD_CFG --localrepo $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS $MOCK_PASSTHROUGH --rebuild" +CMD_BUILD_LIST="$CHAIN_OPTION $SRPMS_LIST" echo "" -echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" +echo "$CMD $MOCK_PASSTHROUGH --define='_tis_dist .tis' $MOCK_PASSTHROUGH --define='platform_release $PLATFORM_RELEASE' $CMD_BUILD_LIST" echo "" - -trapwrap stdbuf -o0 $CMD -m --define="_tis_dist .tis" -m --define="platform_release $PLATFORM_RELEASE" +trapwrap stdbuf -o0 $CMD $MOCK_PASSTHROUGH --define="_tis_dist .tis" $MOCK_PASSTHROUGH --define="platform_release $PLATFORM_RELEASE" $CMD_BUILD_LIST MOCKCHAIN_RC=$? echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE @@ -2357,7 +2389,7 @@ if [ $CLEAN_FLAG -eq 0 ]; then fi for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do - rsync -u $d/*.rpm $RPM_DIR + rsync -u $d/*.rpm $RPM_DIR done if [ $ALL -eq 1 ]; then @@ -2441,5 +2473,5 @@ if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then fi fi -exit 0 +exit 0 ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build-rpms-serial b/build-tools/build-rpms-serial index 5bd7dbe7..0a6afbf7 100755 --- a/build-tools/build-rpms-serial +++ b/build-tools/build-rpms-serial @@ -1,20 +1,20 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# +# # Builds rpm files from src.rpm files. # # This version compiles one package at a time. -# -# The location of packages to be built is +# +# The location of packages to be built is # $MY_WORKSPACE//rpmbuild/SRPMS. # -# The build order is a derived from the BuildRequires in the +# The build order is a derived from the BuildRequires in the # spec files in the src.rpms. Note that the BuildRequires sometimes # create dependency loops, so no correct order can be computed. In these # cases we add a retry loop. As long as one new package builds, we @@ -103,7 +103,7 @@ create-no-clean-list () { local install_groups="" local install_packages="" local p - + for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do if [[ $p == @* ]] ; then install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-)) @@ -111,37 +111,44 @@ create-no-clean-list () { install_packages=$(join_by ' ' $install_packages $p) fi done - + local noclean_last_list_len=0 local noclean_list="" local tmp_list="" local g - + for g in $install_groups; do - tmp_list=$(yum -c $MY_YUM_CONF groupinfo $g 2>> /dev/null | awk 'f;/Mandatory Packages:/{f=1}' | cut -c 5-) + # Find manditory packages in the group. + # Discard anything before (and including) 'Mandatory Packages:' + # and anything after (and including) 'Optional Packages:'. + # Also discard leading spaces or '+' characters. + tmp_list=$(${PKG_MANAGER} -c $MY_YUM_CONF groupinfo $g 2>> /dev/null \ + | awk 'f;/Mandatory Packages:/{f=1}' \ + | sed -n '/Optional Packages:/q;p' \ + | sed 's#[ +]*##') noclean_list=$(join_by ' ' $noclean_list $tmp_list) done - + noclean_list=$(join_by ' ' $noclean_list $install_packages) noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) - + while [ $noclean_list_len -gt $noclean_last_list_len ]; do noclean_last_list_len=$noclean_list_len noclean_list=$( (yum -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) done - + echo $noclean_list > $NO_CLEAN_LIST_FILE fi cat $NO_CLEAN_LIST_FILE } -str_lst_contains() { +str_lst_contains() { TARGET="$1" LST="$2" - if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then + if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then return 0 else return 1 @@ -169,7 +176,9 @@ create_lst () { recreate_repodata () { local DIR=${1} - (cd $DIR + ( + mkdir -p $DIR + cd $DIR if [ -f repodata/*comps*xml ]; then \mv repodata/*comps*xml comps.xml fi @@ -260,11 +269,11 @@ union () { echo $b fi done -} +} # # returns 0 if element is in the array -# +# # e.g. contains ARRAY $SEEKING && echo "$SEEKING is in 'ARRAY'" # contains () { @@ -311,7 +320,7 @@ build_order_recursive () { needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') ) needs_list=( $(intersection needs remainder_list) ) for((idx=0;idx<${#needs_list[@]};idx++)); do - build_order_recursive ${needs_list[idx]} + build_order_recursive ${needs_list[idx]} done echo $target break @@ -361,7 +370,7 @@ build_order () { UNORDERED_LIST=( ${original_list[@]} ) while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do element=${UNORDERED_LIST[0]} - build_order_recursive $element + build_order_recursive $element done fi } @@ -476,7 +485,7 @@ kill_descendents () recursive_promote_children="" recursive_children=$(pgrep -P $kill_pid) fi - + if [ $iteration -eq 0 ]; then other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done) if [ "$other_children" != "" ]; then @@ -491,7 +500,7 @@ kill_descendents () fi for pid in $recursive_children; do - kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) + kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) done for pid in $recursive_promote_children; do kill_descendents "$pid" 1 1 $((iteration + 1)) @@ -754,11 +763,11 @@ mock_clean_cfg () { echo "==================================" mock_clean_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --clean" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --clean + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --clean" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --clean ### Note: this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock echo "==================================" mock_clean_cache_all_cfg $CFG @@ -833,10 +842,10 @@ mock_partial_clean_cfg () { local RPMS_CLEAN_LIST="" local NEED_FULL_MOCK_CLEAN=0 for r in $RPMS_LIST; do - if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then + if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r) - else - echo "Can't remove '$r' from mock environment without a wipe"; + else + echo "Can't remove '$r' from mock environment without a wipe"; NEED_FULL_MOCK_CLEAN=1 fi done @@ -849,19 +858,19 @@ mock_partial_clean_cfg () { # Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST" CMD='LST="'$RPMS_CLEAN_LIST'"; - DELETE_LIST=""; - for r in $LST; do - FOUND=$(rpm -q $r) ; - if [ $? -eq 0 ]; then - DELETE_LIST="$DELETE_LIST $FOUND"; - fi; - done; - echo "uninstalling these packages: $DELETE_LIST"; + DELETE_LIST=""; + for r in $LST; do + FOUND=$(rpm -q $r) ; + if [ $? -eq 0 ]; then + DELETE_LIST="$DELETE_LIST $FOUND"; + fi; + done; + echo "uninstalling these packages: $DELETE_LIST"; if [ "$DELETE_LIST" != "" ]; then rpm -e --nodeps $DELETE_LIST; fi' - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -907,8 +916,8 @@ mock_clean_cache_cfg () { clean_yum_cache_cfg $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -931,8 +940,8 @@ mock_clean_cache_all_cfg () { echo "==================================" clean_yum_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" } @@ -956,8 +965,8 @@ mock_clean_metadata_cfg () { fi CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';') - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1010,7 +1019,7 @@ update_cgcs_repo () { if [ "x$NEW_UNTRACKED" != "x" ]; then NEED_REBUILD=1 fi - + if [ $NEED_REBUILD -eq 1 ]; then NEED_MOCK_CLEAN=1 echo "" @@ -1056,7 +1065,7 @@ mock_clean_mounts_dir () { fi fi if [ $RC -eq 1 ]; then - echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." + echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." echo "Ask your system administrator to umount '$MOUNT'." exit 1 fi @@ -1097,7 +1106,7 @@ clean_yum_cache_cfg () { local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum") local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock" # echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'" - + if [ "$YUM_CACHE_MOUNT" != "" ]; then mock_clean_mounts_dir "$YUM_CACHE_MOUNT" fi @@ -1131,8 +1140,8 @@ mock_update_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update echo "==================================" } @@ -1141,8 +1150,8 @@ mock_init_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init echo "==================================" } @@ -1161,14 +1170,14 @@ mock_update_or_init_cfg () { if [ -d $ROOT_DIR/root ]; then echo "Updating the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update &> $TMP RC=$? else echo "Init the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init &> $TMP RC=$? fi if [ $RC -ne 0 ]; then @@ -1198,12 +1207,12 @@ NO_DESCENDANTS=0 NO_REQUIRED=0 NO_AUTOCLEAN=0 NO_BUILD_INFO=0 -HELP=0 +HELP=0 CLEAN_FLAG=0 FORMAL_FLAG=0 CAREFUL=0 DEP_TEST_FLAG=0 - + # read the options TEMP=$(getopt -o h --long serial,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,dep-test,clean,formal,careful,help,layer: -n "$ME" -- "$@") @@ -1238,7 +1247,7 @@ while true ; do --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac -done +done # Reset variables @@ -1493,7 +1502,7 @@ clean_list () { echo "rm -rf $d" \rm -rf "$d" 2>> /dev/null done - + else rs=$(rpm_get_srpm $r) if [[ "$rs" != "$sn"-[0-9]* ]]; then @@ -1506,7 +1515,7 @@ clean_list () { fi done done - + TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s") if [ $CLEAN_FLAG -eq 1 ]; then @@ -1542,7 +1551,7 @@ clean_list () { \rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null mock_clean - else + else # If dependency test if [ $DEP_TEST_FLAG -eq 1 ]; then mock_clean @@ -1801,7 +1810,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done # For non-std build, and if non specific build targets are named, then search all - # packages that we might build and check if they require a package that DID build + # packages that we might build and check if they require a package that DID build # in the std build. If so build the package as a secondary target, even though the # primary target was from a different build_type. if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then @@ -1861,7 +1870,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done fi - # If the kernel or kernel-rt packages where absent from the primary build targets, but + # If the kernel or kernel-rt packages where absent from the primary build targets, but # added as a secondary target, then make sure all out-of-tree kernel modules are also # added. for n in kernel kernel-rt; do @@ -1996,9 +2005,9 @@ echo "SRPMS_LIST = $SRPMS_LIST" echo "RPMS_LIST = $RPMS_LIST" -echo +echo if [ $CLEAN_FLAG -eq 0 ]; then - # pre-create these directories as $USER, + # pre-create these directories as $USER, # else mock will create them as root and fails to clean them. # Note: keep these in sync with mockchain-parallel! mkdir -p $MY_WORKSPACE/mock @@ -2059,27 +2068,46 @@ mock_clean_metadata echo echo "Building" +recreate_repodata $BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR + CMD_PREFIX="" if [ -x /bin/ionice ]; then CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice " fi -CMD_OPTIONS="-m --no-clean -m --no-cleanup-after" +REAL_MOCKCHAIN=0 +MOCK_PASSTHROUGH="" +MOCKCHAIN="mock" +CHAIN_OPTION="--chain" +if file $(which mockchain) | grep -q 'Python script'; then + REAL_MOCKCHAIN=1 + MOCK_PASSTHROUGH="-m" + MOCKCHAIN="mockchain" + CHAIN_OPTION="" +fi + +CMD_OPTIONS="$MOCK_PASSTHROUGH --no-clean $MOCK_PASSTHROUGH --no-cleanup-after" if [ $CAREFUL -eq 1 ]; then - CMD_OPTIONS="-m --no-cleanup-after" + CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after" +fi +if [ $REAL_MOCKCHAIN -eq 1 ]; then + CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG" fi echo "CAREFUL=$CAREFUL" echo "CMD_OPTIONS=$CMD_OPTIONS" -CMD="$CMD_PREFIX mockchain -r $BUILD_CFG -l $BUILD_BASE --recurse --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL -m --rootdir=$BUILD_BASE/mock/root $CMD_OPTIONS -m --rebuild $SRPMS_LIST" -echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" -trapwrap stdbuf -o0 $CMD -m --define='_tis_dist .tis' -m --define="platform_release $PLATFORM_RELEASE" +CMD="$CMD_PREFIX $MOCKCHAIN --root $BUILD_CFG --localrepo $BUILD_BASE --recurse --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $MOCK_PASSTHROUGH --rootdir=$BUILD_BASE/mock/root $CMD_OPTIONS $MOCK_PASSTHROUGH --rebuild" +CMD_BUILD_LIST="$CHAIN_OPTION $SRPMS_LIST" +echo "" +echo "$CMD $MOCK_PASSTHROUGH --define='_tis_dist .tis' $MOCK_PASSTHROUGH --define='platform_release $PLATFORM_RELEASE' $CMD_BUILD_LIST" +echo "" +trapwrap stdbuf -o0 $CMD $MOCK_PASSTHROUGH --define="_tis_dist .tis" $MOCK_PASSTHROUGH --define="platform_release $PLATFORM_RELEASE" $CMD_BUILD_LIST MOCKCHAIN_RC=$? echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do - rsync -u $d/*.rpm $RPM_DIR + rsync -u $d/*.rpm $RPM_DIR done if [ $ALL -eq 1 ]; then @@ -2163,5 +2191,5 @@ if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then fi fi -exit 0 +exit 0 ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-serial_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build_iso/cgts_deps.sh b/build-tools/build_iso/cgts_deps.sh index e1ccbf96..ab45352d 100755 --- a/build-tools/build_iso/cgts_deps.sh +++ b/build-tools/build_iso/cgts_deps.sh @@ -1,5 +1,11 @@ #!/bin/env bash +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # Here's the score, kids. There are a few different places from which we can # get packages. In priority order, they are: # @@ -26,21 +32,33 @@ # export/dist/report_deps.txt # +CGTS_DEPS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source ${CGTS_DEPS_DIR}/../pkg-manager-utils.sh + # This function generates a simple file of dependencies we're trying to resolve function generate_dep_list { TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) mkdir -p $TMP_RPM_DB rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | grep -v "installing package" \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' > $DEPLISTFILE_NEW 2>&1 + cat $DEPLISTFILE_NEW >> $DEPDETAILLISTFILE + cat $DEPLISTFILE_NEW \ + | grep -v -e "error:" -e "warning:" -e "Preparing..." \ + -e "Verifying..." -e "installing package" \ + | sed -e "s/ is needed by.*$//" -e "s/ [<=>].*$//" \ + | sort -u > $DEPLISTFILE \rm -rf $TMP_RPM_DB } +join_array() { + local IFS="$1" + shift + echo "$*" +} + # Takes a list of requirements (either explcit package name, or capabilities # to provide) and install packages to meet those dependancies # @@ -55,6 +73,7 @@ function generate_dep_list { # capabilities. function install_deps { local DEP_LIST="" + local DEP_LIST_ARRAY=() local DEP_LIST_FILE="$1" # Temporary files are used in a few different ways @@ -68,7 +87,7 @@ function install_deps { while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST+=" '${DEP}'" done < $DEP_LIST_FILE echo "Debug: List of deps to resolve: ${DEP_LIST}" @@ -79,8 +98,17 @@ function install_deps { # go through each repo and convert deps to packages based on package name for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=$REPOID"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "${DEP_LIST} --qf='%{name}'" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=$REPOID \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' ${DEP_LIST} \ + | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + \rm -rf $TMP_DIR/yum-$USER-* done sort $UNSORTED_PACKAGES -u > $SORTED_PACKAGES @@ -96,19 +124,31 @@ function install_deps { # If there are any requirements not resolved, look up the packages with # --whatprovides if [ -s $UNRESOLVED_PACKAGES ]; then - DEP_LIST="" + DEP_LIST_ARRAY=() \cp $SORTED_PACKAGES $UNSORTED_PACKAGES while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST_ARRAY+=( "${DEP}" ) done < $UNRESOLVED_PACKAGES - DEP_LIST=$(echo "$DEP_LIST" | sed 's/^ //g') - if [ "$DEP_LIST" != "" ]; then + if [ "${REPOQUERY_WHATPROVIDES_DELIM}" != " " ]; then + DEP_LIST_ARRAY=( "$(join_array "${REPOQUERY_WHATPROVIDES_DELIM}" "${DEP_LIST_ARRAY[@]}" )" ) + fi + + if [ ${#DEP_LIST_ARRAY[@]} -gt 0 ]; then for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]}" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]} \ + | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + \rm -rf $TMP_DIR/yum-$USER-* done fi @@ -124,12 +164,12 @@ function install_deps { DEP_LIST=" " while read DEP do - DEP_LIST="${DEP_LIST}${DEP} " + DEP_LIST+="${DEP} " done < $SORTED_PACKAGES rm $SORTED_PACKAGES # go through each repo and install packages - local TARGETS=${DEP_LIST} + local TARGETS="${DEP_LIST}" echo "Debug: Resolved list of deps to install: ${TARGETS}" local UNRESOLVED for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do @@ -137,8 +177,20 @@ function install_deps { if [[ ! -z "${TARGETS// }" ]]; then REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V > $TMPFILE + + >&2 echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'"\ + "${REPOQUERY_RESOLVE} ${TARGETS}" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" \ + ${REPOQUERY_RESOLVE} ${TARGETS} \ + | sort -r -V > $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* while read STR @@ -157,7 +209,11 @@ function install_deps { cp $PKG_PATH . if [ $? -ne 0 ]; then >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\""\ + "${REPOQUERY_RESOLVE} ${PKG}" >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" fi @@ -172,8 +228,8 @@ function install_deps { echo " path $PKG_PATH" >> $BUILT_REPORT FOUND_UNKNOWN=1 fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" - \rm -rf $TMP_DIR/yum-$USER-* + done < $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* TARGETS="$UNRESOLVED" fi done @@ -220,6 +276,7 @@ OUTPUT_DIR=$MY_WORKSPACE/export TMP_DIR=$MY_WORKSPACE/tmp YUM=$OUTPUT_DIR/yum.conf DEPLISTFILE=$OUTPUT_DIR/deps.txt +DEPLISTFILE_NEW=$OUTPUT_DIR/deps_new.txt DEPDETAILLISTFILE=$OUTPUT_DIR/deps_detail.txt BUILT_REPORT=$OUTPUT_DIR/local.txt diff --git a/build-tools/build_minimal_iso/cgts_deps.sh b/build-tools/build_minimal_iso/cgts_deps.sh index 062b74ff..9e4a88b2 100755 --- a/build-tools/build_minimal_iso/cgts_deps.sh +++ b/build-tools/build_minimal_iso/cgts_deps.sh @@ -1,40 +1,71 @@ #!/bin/env bash + +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +CGTS_DEPS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source ${CGTS_DEPS_DIR}/../pkg-manager-utils.sh + function generate_dep_list { TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) mkdir -p $TMP_RPM_DB rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE - rm -rf $TMP_RPM_DB + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' > $DEPLISTFILE_NEW 2>&1 + cat $DEPLISTFILE_NEW >> $DEPDETAILLISTFILE + cat $DEPLISTFILE_NEW \ + | grep -v -e "error:" -e "warning:" -e "Preparing..." \ + -e "Verifying..." -e "installing package" \ + | sed -e "s/ is needed by.*$//" -e "s/ [<=>].*$//" \ + | sort -u > $DEPLISTFILE + \rm -rf $TMP_RPM_DB +} + +join_array() { + local IFS="$1" + shift + echo "$*" } function install_deps { local DEP_LIST="" + local DEP_LIST_ARRAY=() local DEP_LIST_FILE="$1" rm -f $TMPFILE while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST_ARRAY+=( "${DEP}" ) done < $DEP_LIST_FILE - echo "Debug: List of deps to resolve: ${DEP_LIST}" + if [ "${REPOQUERY_WHATPROVIDES_DELIM}" != " " ]; then + DEP_LIST_ARRAY=( "$(join_array "${REPOQUERY_WHATPROVIDES_DELIM}" "${DEP_LIST_ARRAY[@]}" )" ) + fi - if [ -z "${DEP_LIST}" ]; then + echo "Debug: List of deps to resolve: ${DEP_LIST_ARRAY[@]}" + + if [ ${#DEP_LIST_ARRAY[@]} -gt 0 ]; then return 0 fi # go through each repo and convert deps to packages - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $TMPFILE - \rm -rf $TMP_DIR/yum-$USER-* + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]}" + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]} \ + | sed "s/kernel-debug/kernel/g" >> $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* done sort $TMPFILE -u > $TMPFILE1 rm $TMPFILE @@ -42,12 +73,12 @@ function install_deps { DEP_LIST="" while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST+="${DEP} " done < $TMPFILE1 rm $TMPFILE1 # next go through each repo and install packages - local TARGETS=${DEP_LIST} + local TARGETS="${DEP_LIST}" echo "Debug: Resolved list of deps to install: ${TARGETS}" local UNRESOLVED for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do @@ -55,9 +86,19 @@ function install_deps { if [[ ! -z "${TARGETS// }" ]]; then REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V >> $TMPFILE - \rm -rf $TMP_DIR/yum-$USER-* + >&2 echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'"\ + "${REPOQUERY_RESOLVE} ${TARGETS}" + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" \ + ${REPOQUERY_RESOLVE} ${TARGETS} \ + | sort -r -V >> $TMPFILE + + \rm -rf $TMP_DIR/yum-$USER-* while read STR do @@ -75,7 +116,11 @@ function install_deps { cp $PKG_PATH . if [ $? -ne 0 ]; then >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} -c ${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\""\ + "${REPOQUERY_RESOLVE} ${PKG}" >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" fi @@ -91,8 +136,9 @@ function install_deps { echo " path $PKG_PATH" >> $BUILT_REPORT FOUND_UNKNOWN=1 fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" - \rm -rf $TMP_DIR/yum-$USER-* + done < $TMPFILE + + \rm -rf $TMP_DIR/yum-$USER-* TARGETS="$UNRESOLVED" fi done @@ -143,6 +189,7 @@ OUTPUT_DIR=${ROOT}/newDisk YUM=${ROOT}/yum.conf TMP_DIR=${ROOT}/tmp DEPLISTFILE=${ROOT}/deps.txt +DEPLISTFILE_NEW=${ROOT}/deps_new.txt DEPDETAILLISTFILE=${ROOT}/deps_detail.txt INSTALLDIR=${ROOT}/newDisk/isolinux/Packages diff --git a/build-tools/default_build_srpm b/build-tools/default_build_srpm index e74b3661..fe99ad6c 100755 --- a/build-tools/default_build_srpm +++ b/build-tools/default_build_srpm @@ -1,6 +1,12 @@ #!/bin/bash # set -x +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + source "$SRC_BASE/build-tools/spec-utils" source "$SRC_BASE/build-tools/srpm-utils" @@ -195,14 +201,14 @@ fi if [ $TAR_NEEDED -gt 0 ]; then echo "Creating tar file: $TAR_PATH ..." - echo "tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude='$SRC_DIR/$DISTRO' --exclude='pbr-*.egg' --transform 's,^$TRANSFORM,$TAR_NAME-$VERSION,'" - tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION," + echo "tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude='$SRC_DIR/$DISTRO' --exclude='pbr-*.egg' --transform 's,^$TRANSFORM,$TAR_NAME-$VERSION,' -czf $TAR_PATH $SRC_DIR" + tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION," -czf "$TAR_PATH" "$SRC_DIR" if [ $? -ne 0 ]; then if [ "x$STAGING" != "x" ]; then popd fi - echo "ERROR: default_build_srpm (${LINENO}): failed to create tar file, cmd: tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform \"s,^$TRANSFORM,$TAR_NAME-$VERSION,\"" + echo "ERROR: default_build_srpm (${LINENO}): failed to create tar file, cmd: tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform \"s,^$TRANSFORM,$TAR_NAME-$VERSION,\" -czf '$TAR_PATH' '$SRC_DIR'" exit 1 fi echo "Created tar file: $TAR_PATH" diff --git a/build-tools/mockchain-parallel b/build-tools/mockchain-parallel index 8645d30f..73029df6 100755 --- a/build-tools/mockchain-parallel +++ b/build-tools/mockchain-parallel @@ -24,10 +24,16 @@ interpreter_path () { get__version__ () { local path=${1} + local var="" if [ ! -f ${path} ]; then return 1 fi - grep __VERSION__= ${path} | cut -d '=' -f 2 | sed 's/"//g' + if file ${path} | grep -q 'Python script'; then + ver=$(grep __VERSION__= ${path} | cut -d '=' -f 2 | sed 's/"//g') + else + ver=$(${path} --version 2> /dev/null) + fi + echo $ver } VC_LESS_THAN=0 diff --git a/build-tools/mockchain-parallel-2.6 b/build-tools/mockchain-parallel-2.6 new file mode 100755 index 00000000..c159d9bf --- /dev/null +++ b/build-tools/mockchain-parallel-2.6 @@ -0,0 +1,1221 @@ +#!/usr/bin/python3 -tt +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built +from __future__ import print_function + +import cgi +# pylint: disable=deprecated-module +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import multiprocessing +import signal +import psutil + +import requests +# pylint: disable=import-error +from six.moves.urllib_parse import urlsplit + +import mockbuild.util + +from stxRpmUtils import splitRpmFilename + +# all of the variables below are substituted by the build system +__VERSION__="2.6" +SYSCONFDIR="/etc" +PYTHONDIR="/usr/lib/python3.6/site-packages" +PKGPYTHONDIR="/usr/lib/python3.6/site-packages/mockbuild" +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs + +mockconfig_path = '/etc/mock' + +def rpmName(path): + filename = os.path.basename(path) + (n, v, r, e, a) = splitRpmFilename(filename) + return n + +def createrepo(path): + global max_workers + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] + else: + comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] + cmd = subprocess.Popen( + comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + + +g_opts = optparse.Values() + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option( + '-r', '--root', default=None, dest='chroot', + metavar="CONFIG", + help="chroot config name/base to use in the mock build") + parser.add_option( + '-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option( + '-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option( + '--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option( + '--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option( + '--workers', default=1, dest='max_workers', + help="number of parallel build jobs") + parser.add_option( + '--worker-resources', default="", dest='worker_resources', + help="colon seperated list, how much mem in gb for each workers temfs") + parser.add_option( + '--basedir', default='/var/lib/mock', dest='basedir', + help="path to workspace") + parser.add_option( + '--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + parser.add_option( + '-m', '--mock-option', default=[], action='append', + dest='mock_option', + help="option to pass directly to mock") + parser.add_option( + '--mark-slow-name', default=[], action='append', + dest='slow_pkg_names_raw', + help="package name that is known to build slowly") + parser.add_option( + '--mark-slow-path', default=[], action='append', + dest='slow_pkgs_raw', + help="package path that is known to build slowly") + parser.add_option( + '--mark-big-name', default=[], action='append', + dest='big_pkg_names_raw', + help="package name that is known to require a lot of disk space to build") + parser.add_option( + '--mark-big-path', default=[], action='append', + dest='big_pkgs_raw', + help="package path that is known to require a lot of disk space to build") + parser.add_option( + '--srpm-dependency-file', default=None, + dest='srpm_dependency_file', + help="path to srpm dependency file") + parser.add_option( + '--rpm-dependency-file', default=None, + dest='rpm_dependency_file', + help="path to rpm dependency file") + parser.add_option( + '--rpm-to-srpm-map-file', default=None, + dest='rpm_to_srpm_map_file', + help="path to rpm to srpm map file") + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print("You must provide an argument to -r for the mock chroot") + sys.exit(1) + + if len(sys.argv) < 3: + print("You must specify at least 1 package to build") + sys.exit(1) + + return opts, args + + +REPOS_ID = [] + +slow_pkg_names={} +slow_pkgs={} +big_pkg_names={} +big_pkgs={} + +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): + # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['rootdir'] = config_opts['rootdir'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) + # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) + # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) + if tmpfs_size_gb > 0: + config_opts['plugin_conf']['tmpfs_enable'] = True + config_opts['plugin_conf']['tmpfs_opts'] = {} + config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 + config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb + config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' + config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) + if not os.path.isdir(config_opts['cache_topdir']): + os.makedirs(config_opts['cache_topdir'], exist_ok=True) + except (IOError, OSError): + return False, "Could not create dir: %s" % config_opts['cache_topdir'] + + cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + except (IOError, OSError): + return False, "Could not create dir: %s" % cache_dir + + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def set_basedir(infile, destfile, basedir, opts): + log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['basedir'] = basedir + config_opts['resultdir'] = '{0}/result'.format(basedir) + config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) + config_opts['root'] = 'mock/b0' + config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) + config_opts['rootdir'] = '{0}/mock/b0/root'.format(basedir) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + global config_opts + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 +cost=1 +best=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + signal.signal(signal.SIGTERM, child_signal_handler) + signal.signal(signal.SIGINT, child_signal_handler) + signal.signal(signal.SIGHUP, child_signal_handler) + signal.signal(signal.SIGABRT, child_signal_handler) + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + # return 2, None, None, None + sys.exit(2) + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + if opts.uniqueext == '': + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--root', cfg, ] + else: + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '--root', cfg, ] + + # Ensure repo is up-to-date. + # Note: Merely adding --update to mockcmd failed to update + mockcmd_update=mockcmd + mockcmd_update.append('--update') + cmd = subprocess.Popen( + mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode != 0: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + + # heuristic here, if user pass for mock "-d foo", but we must be care to leave + # "-d'foo bar'" or "--define='foo bar'" as is + compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') + compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') + for option in opts.mock_option: + r_match = compiled_re_1.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + r_match = compiled_re_2.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + mockcmd.append(option) + + print('building %s' % s_pkg) + mockcmd.append(pkg) + cmd = subprocess.Popen( + mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode == 0: + with open(success_file, 'w') as f: + f.write('done\n') + ret = 1 + else: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + with open(fail_file, 'w') as f: + f.write('undone\n') + ret = 0 + + # return ret, cmd, out, err + sys.exit(ret) + + +def log(lf, msg): + if lf: + now = time.time() + try: + with open(lf, 'a') as f: + f.write(str(now) + ':' + msg + '\n') + except (IOError, OSError) as e: + print('Could not write to logfile %s - %s' % (lf, str(e))) + print(msg) + + +config_opts = {} + +worker_data = [] +workers = 0 +max_workers = 1 + +build_env = [] + +failed = [] +built_pkgs = [] + +local_repo_dir = "" + +pkg_to_name={} +name_to_pkg={} +srpm_dependencies_direct={} +rpm_dependencies_direct={} +rpm_to_srpm_map={} +no_dep_list = [ "bash", "kernel" , "kernel-rt" ] + + +def init_build_env(slots, opts, config_opts_in): + global build_env + + orig_chroot_name=config_opts_in['chroot_name'] + orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) + # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) + for i in range(0,slots): + new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) + new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) + tmpfs_size_gb = 0 + if opts.worker_resources == "": + if i > 0: + tmpfs_size_gb = 2 * (1 + slots - i) + else: + resource_array=opts.worker_resources.split(':') + if i < len(resource_array): + tmpfs_size_gb=int(resource_array[i]) + else: + log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) + sys.exit(1) + if i == 0 and tmpfs_size_gb != 0: + log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) + sys.exit(1) + build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) + + res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + +idle_build_env_last_awarded = 0 +def get_idle_build_env(slots): + global build_env + global idle_build_env_last_awarded + visited = 0 + + if slots < 1: + return -1 + + i = idle_build_env_last_awarded - 1 + if i < 0 or i >= slots: + i = slots - 1 + + while visited < slots: + if build_env[i]['state'] == 'Idle': + build_env[i]['state'] = 'Busy' + idle_build_env_last_awarded = i + return i + visited = visited + 1 + i = i - 1 + if i < 0: + i = slots - 1 + return -1 + +def release_build_env(idx): + global build_env + + build_env[idx]['state'] = 'Idle' + +def get_best_rc(a, b): + print("get_best_rc: a=%s" % str(a)) + print("get_best_rc: b=%s" % str(b)) + if (b == {}) and (a != {}): + return a + if (a == {}) and (b != {}): + return b + + if (b['build_name'] is None) and (not a['build_name'] is None): + return a + if (a['build_name'] is None) and (not b['build_name'] is None): + return b + + if a['unbuilt_deps'] < b['unbuilt_deps']: + return a + if b['unbuilt_deps'] < a['unbuilt_deps']: + return b + + if a['depth'] < b['depth']: + return a + if b['depth'] < a['depth']: + return b + + print("get_best_rc: uncertain %s vs %s" % (a,b)) + return a + +unbuilt_dep_list_print=False +def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): + global srpm_dependencies_direct + global rpm_dependencies_direct + global rpm_to_srpm_map + global no_dep_list + global unbuilt_dep_list_print + + first_iteration=False + unbuilt = [] + if name in no_dep_list: + return unbuilt + + if checked is None: + first_iteration=True + checked=[] + + # Count unbuild dependencies + if first_iteration: + dependencies_direct=srpm_dependencies_direct + else: + dependencies_direct=rpm_dependencies_direct + + if name in dependencies_direct: + for rdep in dependencies_direct[name]: + sdep='???' + if rdep in rpm_to_srpm_map: + sdep = rpm_to_srpm_map[rdep] + if rdep != name and sdep != name and not rdep in checked: + if (not first_iteration) and (sdep in no_dep_list): + continue + checked.append(rdep) + if sdep in unbuilt_pkg_names: + if not sdep in unbuilt: + unbuilt.append(sdep) + if depth > 0: + child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) + for sub_sdep in child_unbuilt: + if sub_sdep != name: + if not sub_sdep in unbuilt: + unbuilt.append(sub_sdep) + + return unbuilt + +def can_build_at_idx(build_idx, name, opts): + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + global build_env + + fs_size_gb = 0 + size_gb = 0 + speed = 0 + pkg = name_to_pkg[name] + if name in big_pkg_names: + size_gb=big_pkg_names[name] + if pkg in big_pkgs: + size_gb=big_pkgs[pkg] + if name in slow_pkg_names: + speed=slow_pkg_names[name] + if pkg in slow_pkgs: + speed=slow_pkgs[pkg] + fs_size_gb = build_env[build_idx]['fs_size_gb'] + return fs_size_gb == 0 or fs_size_gb >= size_gb + +def schedule(build_idx, pkgs, opts): + global worker_data + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + + unbuilt_pkg_names=[] + building_pkg_names=[] + unprioritized_pkg_names=[] + + for pkg in pkgs: + name = pkg_to_name[pkg] + unbuilt_pkg_names.append(name) + unprioritized_pkg_names.append(name) + + prioritized_pkg_names=[] + + for wd in worker_data: + pkg = wd['pkg'] + if not pkg is None: + name = pkg_to_name[pkg] + building_pkg_names.append(name) + + # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) + if len(big_pkg_names) or len(big_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in big_pkg_names or pkg in big_pkgs: + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + if len(slow_pkg_names) or len(slow_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in slow_pkg_names or pkg in slow_pkgs: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + for name in unprioritized_pkg_names: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + + name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) + if not name_out is None: + pkg_out = name_to_pkg[name_out] + else: + pkg_out = None + # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) + # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) + return pkg_out + + +def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): + global pkg_to_name + global name_to_pkg + global no_dep_list + + max_depth = 3 + + if len(pkg_names) == 0: + return None + + unbuilt_deps={} + building_deps={} + for depth in range(max_depth,-1,-1): + unbuilt_deps[depth]={} + building_deps[depth]={} + + for depth in range(max_depth,-1,-1): + checked=[] + reordered_pkg_names = pkg_names[:] + # for name in reordered_pkg_names: + while len(reordered_pkg_names): + name = reordered_pkg_names.pop(0) + if name in checked: + continue + + # log(opts.logfile, "checked.append(%s)" % name) + checked.append(name) + + pkg = name_to_pkg[name] + # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) + if not name in unbuilt_deps[depth]: + unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) + if not name in building_deps[depth]: + building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) + if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + if not name in unbuilt_deps[0]: + unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) + if not name in building_deps[0]: + building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) + if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): + if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + loop = False + for dep_name in unbuilt_deps[depth][name]: + if name == dep_name: + continue + + # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) + if dep_name in checked: + continue + + # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) + if not dep_name in unbuilt_deps[depth]: + unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) + if not dep_name in building_deps[depth]: + building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) + if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if not dep_name in unbuilt_deps[0]: + unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) + if not dep_name in building_deps[0]: + building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) + if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): + if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if name in unbuilt_deps[0][dep_name]: + loop = True + # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) + + if loop and len(building_deps[depth][name]) == 0: + log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) + return name + + for dep_name in unbuilt_deps[depth][name]: + if dep_name in reordered_pkg_names: + # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) + reordered_pkg_names.remove(dep_name) + reordered_pkg_names.insert(0,dep_name) + + # log(opts.logfile, "schedule2: Nothing buildable at this time") + return None + + +def read_deps(opts): + read_srpm_deps(opts) + read_rpm_deps(opts) + read_map_deps(opts) + +def read_srpm_deps(opts): + global srpm_dependencies_direct + + if opts.srpm_dependency_file == None: + return + + if not os.path.exists(opts.srpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) + sys.exit(1) + + with open(opts.srpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + srpm_dependencies_direct[name]=deps.split(',') + +def read_rpm_deps(opts): + global rpm_dependencies_direct + + if opts.rpm_dependency_file == None: + return + + if not os.path.exists(opts.rpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) + sys.exit(1) + + with open(opts.rpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + rpm_dependencies_direct[name]=deps.split(',') + +def read_map_deps(opts): + global rpm_to_srpm_map + + if opts.rpm_to_srpm_map_file == None: + return + + if not os.path.exists(opts.rpm_to_srpm_map_file): + log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) + sys.exit(1) + + with open(opts.rpm_to_srpm_map_file) as f: + lines = f.readlines() + for line in lines: + (rpm,srpm) = line.rstrip().split(';') + rpm_to_srpm_map[rpm]=srpm + + +def reaper(opts): + global built_pkgs + global failed + global worker_data + global workers + + reaped = 0 + need_createrepo = False + last_reaped = -1 + while reaped > last_reaped: + last_reaped = reaped + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is not None: + pkg = wd['pkg'] + b = int(wd['build_index']) + p.join() + worker_data.remove(wd) + workers = workers - 1 + reaped = reaped + 1 + release_build_env(b) + + log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) + + if ret == 0: + failed.append(pkg) + log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) + if opts.recurse and not stop_signal: + log(opts.logfile, "Will try to build again (if some other package will succeed).") + else: + log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) + elif ret == 1: + log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) + built_pkgs.append(pkg) + need_createrepo = True + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if need_createrepo: + # createrepo with the new pkgs + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + + return reaped + +stop_signal = False + +def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + +def kill_proc_and_descentents(parent, need_stop=False, verbose=False): + global g_opts + + if need_stop: + if verbose: + log(g_opts.logfile, "Stop %d" % parent.pid) + + try: + parent.send_signal(signal.SIGSTOP) + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.send_signal(signal.SIGSTOP) + + try: + children = parent.children(recursive=False) + except: + children = [] + + for p in children: + kill_proc_and_descentents(p, need_stop=True, verbose=verbose) + + if verbose: + log(g_opts.logfile, "Terminate %d" % parent.pid) + + # parent.send_signal(signal.SIGTERM) + try: + parent.terminate() + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.terminate() + + if need_stop: + if verbose: + log(g_opts.logfile, "Continue %d" % parent.pid) + + parent.send_signal(signal.SIGCONT) + + +def child_signal_handler(signum, frame): + global g_opts + my_pid = os.getpid() + # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) + p = psutil.Process(my_pid) + kill_proc_and_descentents(p) + try: + sys.exit(0) + except SystemExit as e: + os._exit(0) + +def signal_handler(signum, frame): + global g_opts + global stop_signal + global workers + global worker_data + stop_signal = True + + # Signal processes to complete + log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is None: + # log(g_opts.logfile, "terminate child %d" % p.pid) + p.terminate() + else: + log(g_opts.logfile, "child return code was %d" % ret) + + # Wait for remaining processes to complete + log(g_opts.logfile, "===== wait for signaled jobs to complete =====") + while len(worker_data) > 0: + log(g_opts.logfile, " remaining workers: %d" % workers) + reaped = reaper(g_opts) + if reaped == 0: + time.sleep(0.1) + + try: + sys.exit(1) + except SystemExit as e: + os._exit(1) + +def main(args): + opts, args = parse_args(args) + # take mock config + list of pkgs + + global g_opts + global stop_signal + global build_env + global worker_data + global workers + global max_workers + + global slow_pkg_names + global slow_pkgs + global big_pkg_names + global big_pkgs + max_workers = int(opts.max_workers) + + global failed + global built_pkgs + + cfg = opts.chroot + pkgs = args[1:] + + # transform slow/big package options into dictionaries + for line in opts.slow_pkg_names_raw: + speed,name = line.split(":") + if speed != "": + slow_pkg_names[name]=int(speed) + for line in opts.slow_pkgs_raw: + speed,pkg = line.split(":") + if speed != "": + slow_pkgs[pkg]=int(speed) + for line in opts.big_pkg_names_raw: + size_gb,name = line.split(":") + if size_gb != "": + big_pkg_names[name]=int(size_gb) + for line in opts.big_pkgs_raw: + size_gb,pkg = line.split(":") + if size_gb != "": + big_pkgs[pkg]=int(size_gb) + + # Set up a mapping between pkg path and pkg name + global pkg_to_name + global name_to_pkg + for pkg in pkgs: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + continue + + try: + name = rpmName(pkg) + except OSError as e: + print("Could not parse rpm %s" % pkg) + sys.exit(1) + + pkg_to_name[pkg] = name + name_to_pkg[name] = pkg + + read_deps(opts) + + global config_opts + config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError as e: + print("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + if opts.basedir != "/var/lib/mock": + opts.uniqueext = '' + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + os.chmod(local_tmp_dir, 0o755) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + os.chmod(local_tmp_dir, 0o755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0o755) + + local_baseurl = "file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0o755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) + + # modify with localrepo + res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + # createrepo on it + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + sys.exit(1) + + init_build_env(max_workers, opts, config_opts) + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + return_code = 0 + num_of_tries = 0 + + g_opts = opts + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + signal.signal(signal.SIGABRT, signal_handler) + + while try_again and not stop_signal: + num_of_tries += 1 + failed = [] + + log(opts.logfile, "===== iteration %d start =====" % num_of_tries) + + to_be_built_scheduled = to_be_built[:] + + need_reap = False + while len(to_be_built_scheduled) > 0: + # Free up a worker + while need_reap or workers >= max_workers: + need_reap = False + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + + if workers < max_workers: + workers = workers + 1 + + b = get_idle_build_env(max_workers) + if b < 0: + log(opts.logfile, "Failed to find idle build env for: %s" % pkg) + workers = workers - 1 + need_reap = True + continue + + pkg = schedule(b, to_be_built_scheduled, opts) + if pkg is None: + if workers <= 1: + # Remember we have one build environmnet reserved, so can't test for zero workers + log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) + pkg = to_be_built_scheduled[0] + log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) + else: + release_build_env(b) + workers = workers - 1 + need_reap = True + continue + + to_be_built_scheduled.remove(pkg) + + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + release_build_env(b) + need_reap = True + continue + + elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): + url = pkg + try: + log(opts.logfile, 'Fetching %s' % url) + r = requests.get(url) + # pylint: disable=no-member + if r.status_code == requests.codes.ok: + fn = urlsplit(r.url).path.rsplit('/', 1)[1] + if 'content-disposition' in r.headers: + _, params = cgi.parse_header(r.headers['content-disposition']) + if 'filename' in params and params['filename']: + fn = params['filename'] + pkg = download_dir + '/' + fn + with open(pkg, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + except Exception as e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + release_build_env(b) + need_reap = True + continue + else: + downloaded_pkgs[pkg] = url + + log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) + # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] + p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) + worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) + p.start() + + # Wait for remaining processes to complete + log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) + while workers > 0: + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) + + if failed and opts.recurse: + log(opts.logfile, "failed=%s" % failed) + log(opts.logfile, "to_be_built=%s" % to_be_built) + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package succeeded, some failed.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) + else: + if max_workers > 1: + max_workers = 1 + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package failed under parallel build.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) + else: + log(opts.logfile, "") + log(opts.logfile, "*** Build Failed ***") + log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) + log(opts.logfile, "*** Build Failed ***") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + log(opts.logfile, "") + try_again = False + else: + try_again = False + if failed: + return_code = 2 + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "") + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "") + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log(opts.logfile, "Some packages successfully built in this order:") + else: + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build-tools/modify-build-cfg b/build-tools/modify-build-cfg index b639f521..78658623 100755 --- a/build-tools/modify-build-cfg +++ b/build-tools/modify-build-cfg @@ -1,5 +1,11 @@ #!/bin/sh +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # This script modifies a mock configuration file (typically $MY_BUILD_CFG) # to add build time environment variables to the mock environment (things # like what branch we're building on, etc). @@ -11,6 +17,11 @@ # usage: modify-build-cfg [file.cfg] [] # +MODIFY_BUILD_CFG_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set PKG_MANAGER for our build environment. +source "${MODIFY_BUILD_CFG_DIR}/pkg-manager-utils.sh" + LAYER=${2:-$LAYER} # For backward compatibility. Old repo location or new? @@ -23,6 +34,13 @@ if [ ! -d ${CENTOS_REPO} ]; then fi fi +# Preferred python +if rpm -q --whatprovides --quiet python3; then + PYTHON_PKG=python3 +else + PYTHON_PKG=python2 +fi + # Try to find a layer specific mock.cfg.proto MOCK_CFG_PROTO="${CENTOS_REPO}/mock.cfg.${LAYER}.proto" echo "==== Try MOCK_CFG_PROTO=$MOCK_CFG_PROTO ====" @@ -98,7 +116,7 @@ grep -q "config_opts\['environment'\]\['WRS_GIT_BRANCH'\]" $FILE || \ grep -q "config_opts\['environment'\]\['CGCS_GIT_BRANCH'\]" $FILE || \ echo "config_opts['environment']['CGCS_GIT_BRANCH']" >> $FILE -if [ -z $FORMAL_BUILD ]; then +if [ -z $FORMAL_BUILD ]; then grep -q "config_opts\['macros'\]\['%_no_cgcs_license_check'\] = '1'" $FILE || \ echo "config_opts['macros']['%_no_cgcs_license_check'] = '1'" >> $FILE else @@ -118,8 +136,21 @@ if [ "containers" == "$BUILD_TYPE" ]; then NETWORK_PKGS="bind-utils" fi -grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" $FILE || \ - echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" >> $FILE +BUILD_PKGS='' +if [ "${PKG_MANAGER}" == "yum" ]; then + BUILD_PKGS='@buildsys-build' +elif [ "${PKG_MANAGER}" == "dnf" ]; then + # buildsys-build group was dropped when Centos-8 switched to dnf. + # We must list all the members plus a few new ones (fedpkg-minimal, epel-rpm-macros). + BUILD_PKGS='bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +fi + +STX_PKGS='pigz lbzip2 bash' + +PKGS="${BUILD_PKGS} ${STX_PKGS} ${PKG_MANAGER} ${PYTHON_PKG} ${NETWORK_PKGS}" + +grep -q "config_opts\['chroot_setup_cmd'\] = 'install ${PKGS}'" $FILE || \ + echo "config_opts['chroot_setup_cmd'] = 'install ${PKGS}'" >> $FILE # Special case for containers. # rpmbuild_networking is required for invoking helm commands within mock diff --git a/build-tools/pkg-manager-utils.sh b/build-tools/pkg-manager-utils.sh new file mode 100755 index 00000000..17f4abc1 --- /dev/null +++ b/build-tools/pkg-manager-utils.sh @@ -0,0 +1,33 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# BASH utilities to select package manager +# +# Currently just sets some environment variables +# + +# Yum vs DNF compatibility +YUM=$(which yum 2>> /dev/null) +DNF=$(which dnf 2>> /dev/null) +PKG_MANAGER="" +REPOQUERY=$(which repoquery 2>> /dev/null) +REPOQUERY_SUB_COMMAND="" +REPOQUERY_RESOLVE="--resolve" +REPOQUERY_WHATPROVIDES_DELIM=" " +if [ ! -z ${DNF} ]; then + PKG_MANAGER="dnf" + REPOQUERY=${DNF} + REPOQUERY_SUB_COMMAND="repoquery --disable-modular-filtering" + REPOQUERY_RESOLVE="" + REPOQUERY_WHATPROVIDES_DELIM="," +elif [ ! -z ${YUM} ]; then + PKG_MANAGER="yum" +else + >&2 echo "ERROR: Couldn't find a supported package manager" + exit 1 +fi + diff --git a/build-tools/repo_files/mock.cfg.all.proto b/build-tools/repo_files/mock.cfg.all.proto deleted file mode 100644 index 6ecaa2d2..00000000 --- a/build-tools/repo_files/mock.cfg.all.proto +++ /dev/null @@ -1,58 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority -exclude=kernel-devel libvirt-devel - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.all.proto b/build-tools/repo_files/mock.cfg.all.proto new file mode 120000 index 00000000..2ba14cf5 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.all.proto @@ -0,0 +1 @@ +mock.cfg.centos7.all.proto \ No newline at end of file diff --git a/build-tools/repo_files/mock.cfg.centos7.all.proto b/build-tools/repo_files/mock.cfg.centos7.all.proto new file mode 100644 index 00000000..6ecaa2d2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.all.proto @@ -0,0 +1,58 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos7.distro.proto b/build-tools/repo_files/mock.cfg.centos7.distro.proto new file mode 100644 index 00000000..6ecaa2d2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.distro.proto @@ -0,0 +1,58 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos7.proto b/build-tools/repo_files/mock.cfg.centos7.proto new file mode 100644 index 00000000..beeea32f --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.proto @@ -0,0 +1,57 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.all.proto b/build-tools/repo_files/mock.cfg.centos8.all.proto new file mode 100644 index 00000000..c5bb65da --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.all.proto @@ -0,0 +1,63 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.distro.proto b/build-tools/repo_files/mock.cfg.centos8.distro.proto new file mode 100644 index 00000000..c5bb65da --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.distro.proto @@ -0,0 +1,63 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.proto b/build-tools/repo_files/mock.cfg.centos8.proto new file mode 100644 index 00000000..08a041b2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.proto @@ -0,0 +1,62 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.distro.proto b/build-tools/repo_files/mock.cfg.distro.proto deleted file mode 100644 index 6ecaa2d2..00000000 --- a/build-tools/repo_files/mock.cfg.distro.proto +++ /dev/null @@ -1,58 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority -exclude=kernel-devel libvirt-devel - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.distro.proto b/build-tools/repo_files/mock.cfg.distro.proto new file mode 120000 index 00000000..add71c08 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.distro.proto @@ -0,0 +1 @@ +mock.cfg.centos7.distro.proto \ No newline at end of file diff --git a/build-tools/repo_files/mock.cfg.proto b/build-tools/repo_files/mock.cfg.proto deleted file mode 100644 index beeea32f..00000000 --- a/build-tools/repo_files/mock.cfg.proto +++ /dev/null @@ -1,57 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.proto b/build-tools/repo_files/mock.cfg.proto new file mode 120000 index 00000000..55c2e026 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.proto @@ -0,0 +1 @@ +mock.cfg.centos7.proto \ No newline at end of file