From 77576b7207909f86ee2505ec90c71c1a4f216da4 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 28 Oct 2020 14:12:13 -0400 Subject: [PATCH] Add support CentOS-8/dnf/mock-2.6 based builds This update will retain support for CentOS-7/yum/mock-1.4 based builds. The build environment will be queried to discover which environment it is building in, and modify the commands we issue accordingly. In CentOS 8, DNF replaces both YUM and REPOQUERY. While DNF tries to be a transparent replacement of the old tools, there are also subtle changes to the supported arguments. I will provide independent mock.cfg.prototypes for centos7 vs centos8. Changes in generate-centos-repo.sh under stx-tools will be required to select the correct prototype. Add support for mock 2.6. Mock 2.6 is python 3, and it processes the 'root' and 'rootdir' arguments slightly differently. Also change the order of arguments to tar within default_build_srpm. The latest tar only honors '--exclude' if it precedes other arguments. Story: 2006729 Depends-On: https://review.opendev.org/762700 Signed-off-by: Scott Little Change-Id: I826be2051e535e6a4c08ad17124f453b04210668 --- build-tools/build-iso | 45 +- build-tools/build-pkgs | 14 +- build-tools/build-rpms-parallel | 202 +-- build-tools/build-rpms-serial | 172 ++- build-tools/build_iso/cgts_deps.sh | 103 +- build-tools/build_minimal_iso/cgts_deps.sh | 91 +- build-tools/default_build_srpm | 12 +- build-tools/mockchain-parallel | 8 +- build-tools/mockchain-parallel-2.6 | 1221 +++++++++++++++++ build-tools/modify-build-cfg | 37 +- build-tools/pkg-manager-utils.sh | 33 + build-tools/repo_files/mock.cfg.all.proto | 59 +- .../repo_files/mock.cfg.centos7.all.proto | 58 + .../repo_files/mock.cfg.centos7.distro.proto | 58 + build-tools/repo_files/mock.cfg.centos7.proto | 57 + .../repo_files/mock.cfg.centos8.all.proto | 63 + .../repo_files/mock.cfg.centos8.distro.proto | 63 + build-tools/repo_files/mock.cfg.centos8.proto | 62 + build-tools/repo_files/mock.cfg.distro.proto | 59 +- build-tools/repo_files/mock.cfg.proto | 58 +- 20 files changed, 2075 insertions(+), 400 deletions(-) create mode 100755 build-tools/mockchain-parallel-2.6 create mode 100755 build-tools/pkg-manager-utils.sh mode change 100644 => 120000 build-tools/repo_files/mock.cfg.all.proto create mode 100644 build-tools/repo_files/mock.cfg.centos7.all.proto create mode 100644 build-tools/repo_files/mock.cfg.centos7.distro.proto create mode 100644 build-tools/repo_files/mock.cfg.centos7.proto create mode 100644 build-tools/repo_files/mock.cfg.centos8.all.proto create mode 100644 build-tools/repo_files/mock.cfg.centos8.distro.proto create mode 100644 build-tools/repo_files/mock.cfg.centos8.proto mode change 100644 => 120000 build-tools/repo_files/mock.cfg.distro.proto mode change 100644 => 120000 build-tools/repo_files/mock.cfg.proto diff --git a/build-tools/build-iso b/build-tools/build-iso index d6c6ccc6..a830cae2 100755 --- a/build-tools/build-iso +++ b/build-tools/build-iso @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018-2019 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -18,6 +18,10 @@ BUILD_ISO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${BUILD_ISO_DIR}/image-utils.sh" source "${BUILD_ISO_DIR}/git-utils.sh" +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source "${BUILD_ISO_DIR}/pkg-manager-utils.sh" + usage () { echo "" echo "Usage: " @@ -40,8 +44,8 @@ usage () { MY_YUM_CONF="" STD_REPO_ID="local-std" RT_REPO_ID="local-rt" -LOWER_LAYER_STD_REPO_ID="StxCentos7Distro" -LOWER_LAYER_RT_REPO_ID="StxCentos7Distro-rt" +LOWER_LAYER_STD_REPO_ID="" +LOWER_LAYER_RT_REPO_ID="" NPROCS=$(nproc) @@ -77,13 +81,13 @@ function install_pkg_list { \cd $OUTPUT_DIST_DIR/isolinux/Packages $INTERNAL_REPO_ROOT/build-tools/build_iso/cgts_deps.sh --deps=$PKGLIST - + if [ $? -ne 0 ] then echo "Could not install dependencies" exit 1 fi - + # clean up echo "Removing local-std yum repo $CGCS_REPO_DIR/repodata" echo "Removing local-rt yum repo $CGCS_RT_REPO_DIR/repodata" @@ -111,7 +115,7 @@ function make_report { for PKGLIST in $PKGLISTFILES; do while read PKG; do PKG=`echo $PKG | sed "s/#.*//"`; - if [ "${PKG}x" != "x" ]; then + if [ "${PKG}x" != "x" ]; then echo $PKG >> $REPORT_FILE fi done < $PKGLIST @@ -127,7 +131,7 @@ function make_report { echo "-----------------" >> $REPORT_FILE echo " WARNINGS " >> $REPORT_FILE echo "-----------------" >> $REPORT_FILE - + # Note that the warnings file may have multiple lines for the same # missing dependency. A sort | uniq solves this so we don't duplicate # warnings @@ -158,13 +162,17 @@ function init_vars { exit 1 fi + # LOWER_LAYER_STD_REPO_ID should be something like StxCentos7Distro or StxCentos8Distro + LOWER_LAYER_STD_REPO_ID=$(grep '\[StxCentos.*Distro\]' ${MY_YUM_CONF} | sed -e 's/^\[//' -e 's/\].*//') + LOWER_LAYER_RT_REPO_ID=$(grep '\[StxCentos.*Distro-rt\]' ${MY_YUM_CONF} | sed -e 's/^\[//' -e 's/\].*//') + DISTRO_REPO_DIR=$(for d in $(grep baseurl $MY_YUM_CONF | grep file: | awk -F : '{print $2}' | sed 's:///:/:g'); do if [ -d $d/images ]; then echo $d ;fi; done) ##################################### # Output definitons # where to put stuff (curent dir unless MY_WORKSPACE defined) - OUTPUT_DIR="$PWD/export" + OUTPUT_DIR="$PWD/export" if [ ! -z "$MY_WORKSPACE" ] && [ -d "$MY_WORKSPACE" ] ; then OUTPUT_DIR="$MY_WORKSPACE/export" CGCS_REPO_DIR="$MY_WORKSPACE/std/rpmbuild/RPMS" @@ -316,7 +324,7 @@ function init_output_dir { date +%s.%N > $OUTPUT_DIST_DIR/isolinux/.discinfo echo $PLATFORM_RELEASE >> $OUTPUT_DIST_DIR/isolinux/.discinfo echo "x86_64" >> $OUTPUT_DIST_DIR/isolinux/.discinfo - + \cp -L -ru $DISTRO_REPO_DIR/isolinux/* $OUTPUT_DIST_DIR/isolinux/ \cp -L -ru $DISTRO_REPO_DIR/images/pxeboot $OUTPUT_DIST_DIR/isolinux/images/ @@ -342,7 +350,7 @@ function init_output_dir { perl -p -i -e "s/device=sda/device=${DEVICE}/g" $OUTPUT_DIST_DIR/isolinux/isolinux.cfg fi - # Copy UEFI files + # Copy UEFI files \cp -L -ru $DISTRO_REPO_DIR/EFI/* $OUTPUT_DIST_DIR/isolinux/EFI/ \cp -L "$BSP_FILES_PATH/grub.cfg" "$OUTPUT_DIST_DIR/isolinux/EFI/BOOT/grub.cfg" \cp -L "$BSP_FILES_PATH/pxeboot_grub.cfg" "$OUTPUT_DIST_DIR/isolinux/pxeboot/pxeboot_grub.cfg" @@ -472,7 +480,7 @@ function final_touches { # create the repo \cd $OUTPUT_DIST_DIR/isolinux $CREATEREPO -q -g ../comps.xml . - + # build the ISO printf "Building image $OUTPUT_FILE\n" \cd $OUTPUT_DIST_DIR @@ -485,7 +493,7 @@ function final_touches { -eltorito-alt-boot \ -e images/efiboot.img \ -no-emul-boot \ - isolinux/ + isolinux/ isohybrid --uefi $OUTPUT_FILE implantisomd5 $OUTPUT_FILE @@ -495,7 +503,7 @@ function final_touches { function extract_pkg_from_local_repo { local pkgname=$1 - local yum_conf=$2 + local pkg_mgr_conf=$2 shift 2 local repoid="" @@ -505,7 +513,14 @@ function extract_pkg_from_local_repo { repoid_arg+=" --repoid=${repoid}" done - local pkgfile=$(TMPDIR=$TMP_DIR repoquery --config=${yum_conf} ${repoid_arg} --location -q ${pkgname}) + echo "TMPDIR=$TMP_DIR"\ + "${REPOQUERY} --config=${pkg_mgr_conf} ${repoid_arg}"\ + "${REPOQUERY_SUB_COMMAND} --location"\ + "--arch=noarch,x86_64 -q ${pkgname}" + local pkgfile=$(TMPDIR=$TMP_DIR \ + ${REPOQUERY} --config=${pkg_mgr_conf} ${repoid_arg} \ + ${REPOQUERY_SUB_COMMAND} --location \ + --arch=noarch,x86_64 -q ${pkgname}) if [ -z "${pkgfile}" ]; then echo "Could not find package $pkgname in $@" exit 1 @@ -605,7 +620,7 @@ EOM echo "Error: Could not copy all files from installer" exit 1 fi - + \cp --preserve=all www/pages/feed/rel-*/LiveOS/squashfs.img $OUTPUT_DIST_DIR/isolinux/LiveOS if [ $? -ne 0 ]; then echo "Error: Could not copy squashfs from LiveOS" diff --git a/build-tools/build-pkgs b/build-tools/build-pkgs index edd980f8..da9d65d0 100755 --- a/build-tools/build-pkgs +++ b/build-tools/build-pkgs @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,6 +15,10 @@ BUILD_PKGS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" +# Set REPOQUERY and REPOQUERY_SUB_COMMAND for our build environment. +source "${BUILD_PKGS_DIR}/pkg-manager-utils.sh" + + usage () { echo "" echo "Usage: " @@ -75,11 +79,15 @@ make_cache_current_rpms () { if [ -d $MY_WORKSPACE/$build_type/rpmbuild/$rpm_type/repodata ]; then current=$MY_WORKSPACE/$build_type/rpmbuild/$rpm_type/ - repoquery \ + ${REPOQUERY} \ --repofrompath=$build_type-$rpm_type,$current \ - --repoid=$build_type-$rpm_type --arch=noarch,src,x86_64 -a \ + --repoid=$build_type-$rpm_type --arch=noarch,src,x86_64 \ + ${REPOQUERY_SUB_COMMAND} \ + --all \ --qf "%-10{repoid} %-40{name} %-10{version} %-10{release}" \ >> ${FILE} + + \rm -rf $TMP_DIR/yum-$USER-* fi done; done; diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index 4ca2202e..e2830c24 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -15,7 +15,7 @@ # The location of packages to be built is # $MY_WORKSPACE//rpmbuild/SRPMS. # -# The build order is a derived from the BuildRequires in the +# The build order is a derived from the BuildRequires in the # spec files in the src.rpms. Note that the BuildRequires sometimes # create dependency loops, so no correct order can be computed. In these # cases we add a retry loop. As long as one new package builds, we @@ -24,9 +24,12 @@ # messages go by! # - export ME=$(basename "$0") CMDLINE="$ME $@" +BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set PKG_MANAGER for our build environment. +source "${BUILD_RPMS_PARALLEL_DIR}/pkg-manager-utils.sh" # Build for distribution. Currently 'centos' is only supported value. @@ -43,22 +46,22 @@ MAX_MEM_PER_WORKER=11 # Note: tmpfs is typically 2.5 gb when compiling many small jobs MIN_MEM_PER_WORKER=3 -# Maximum number of disk based parallel build environments +# Maximum number of disk based parallel build environments MAX_DISK_BASED_WORKERS=2 # Minimum space in gb for each disk based parallel build environment MIN_DISK_PER_WORKER=20 -# How many srpms to build before we add another parallel build environment +# How many srpms to build before we add another parallel build environment MIN_TASKS_PER_CORE=3 # Max number of new concurrent builds to allow for MAX_SHARE_FACTOR=4 -# Always leave at least MEMORY_RESERVE gb of available mem for the system +# Always leave at least MEMORY_RESERVE gb of available mem for the system MEMORY_RESERVE=1 -# These two values will be reassigned in the 'compute_resources' subroutine +# These two values will be reassigned in the 'compute_resources' subroutine MOCKCHAIN_RESOURCE_ALLOCATION="" MAX_WORKERS=$ABSOLUTE_MAX_WORKERS @@ -164,7 +167,7 @@ create-no-clean-list () { local install_groups="" local install_packages="" local p - + for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do if [[ $p == @* ]] ; then install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-)) @@ -172,37 +175,44 @@ create-no-clean-list () { install_packages=$(join_by ' ' $install_packages $p) fi done - + local noclean_last_list_len=0 local noclean_list="" local tmp_list="" local g - + for g in $install_groups; do - tmp_list=$(yum -c $MY_YUM_CONF groupinfo $g 2>> /dev/null | awk 'f;/Mandatory Packages:/{f=1}' | cut -c 5-) + # Find mandatory packages in the group. + # Discard anything before (and including) 'Mandatory Packages:' + # and anything after (and including) 'Optional Packages:'. + # Also discard leading spaces or '+' characters. + tmp_list=$(${PKG_MANAGER} -c $MY_YUM_CONF groupinfo $g 2>> /dev/null \ + | awk 'f;/Mandatory Packages:/{f=1}' \ + | sed -n '/Optional Packages:/q;p' \ + | sed 's#[ +]*##') noclean_list=$(join_by ' ' $noclean_list $tmp_list) done - + noclean_list=$(join_by ' ' $noclean_list $install_packages) noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) - + while [ $noclean_list_len -gt $noclean_last_list_len ]; do noclean_last_list_len=$noclean_list_len - noclean_list=$( (yum -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) + noclean_list=$( (${PKG_MANAGER} -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) done - + echo $noclean_list > $NO_CLEAN_LIST_FILE fi cat $NO_CLEAN_LIST_FILE } -str_lst_contains() { +str_lst_contains() { TARGET="$1" LST="$2" - if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then + if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then return 0 else return 1 @@ -214,7 +224,7 @@ compute_resources () { local b echo "" - for f in $@; do + for f in $@; do b=$(basename $f) if [ -f $SOURCES_DIR/$b/BIG ] || [ ]; then weight=$((weight+MIN_TASKS_PER_CORE)) @@ -232,7 +242,7 @@ compute_resources () { local cpus=$(number_of_cpus) local num_users=$(sqrt $users) local num_build=$(number_of_builds_in_progress) - num_build=$((num_build+1)) + num_build=$((num_build+1)) echo "compute_resources: total: cpus=$cpus, mem=$mem, disk=$disk, weight=$weight, num_build=$num_build" # What fraction of the machine will we use @@ -311,7 +321,9 @@ create_lst () { recreate_repodata () { local DIR=${1} - (cd $DIR + ( + mkdir -p $DIR + cd $DIR if [ -f repodata/*comps*xml ]; then \mv repodata/*comps*xml comps.xml fi @@ -402,11 +414,11 @@ union () { echo $b fi done -} +} # # returns 0 if element is in the array -# +# # e.g. contains ARRAY $SEEKING && echo "$SEEKING is in 'ARRAY'" # contains () { @@ -453,7 +465,7 @@ build_order_recursive () { needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') ) needs_list=( $(intersection needs remainder_list) ) for((idx=0;idx<${#needs_list[@]};idx++)); do - build_order_recursive ${needs_list[idx]} + build_order_recursive ${needs_list[idx]} done echo $target break @@ -503,7 +515,7 @@ build_order () { UNORDERED_LIST=( ${original_list[@]} ) while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do element=${UNORDERED_LIST[0]} - build_order_recursive $element + build_order_recursive $element done fi } @@ -621,7 +633,7 @@ umount_mock_root_as_tmpfs () { return 0 fi mock_tmpfs_umount $mount_dir &> /dev/null - + rc=$? if [ $rc -ne 0 ]; then echo "FAILED: mock_tmpfs_umount $mount_dir" @@ -652,7 +664,7 @@ kill_descendents () recursive_promote_children="" recursive_children=$(pgrep -P $kill_pid) fi - + if [ $iteration -eq 0 ]; then other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done) if [ "$other_children" != "" ]; then @@ -667,7 +679,7 @@ kill_descendents () fi for pid in $recursive_children; do - kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) + kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) done for pid in $recursive_promote_children; do kill_descendents "$pid" 1 1 $((iteration + 1)) @@ -946,11 +958,11 @@ mock_clean_cfg () { echo "==================================" mock_clean_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --clean" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --clean + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --clean" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --clean ### Note: this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock echo "==================================" mock_clean_cache_all_cfg $CFG @@ -1034,10 +1046,10 @@ mock_partial_clean_cfg () { local RPMS_CLEAN_LIST="" local NEED_FULL_MOCK_CLEAN=0 for r in $RPMS_LIST; do - if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then + if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r) - else - echo "Can't remove '$r' from mock environment without a wipe"; + else + echo "Can't remove '$r' from mock environment without a wipe"; NEED_FULL_MOCK_CLEAN=1 fi done @@ -1050,19 +1062,19 @@ mock_partial_clean_cfg () { # Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST" CMD='LST="'$RPMS_CLEAN_LIST'"; - DELETE_LIST=""; - for r in $LST; do - FOUND=$(rpm -q $r) ; - if [ $? -eq 0 ]; then - DELETE_LIST="$DELETE_LIST $FOUND"; - fi; - done; - echo "uninstalling these packages: $DELETE_LIST"; + DELETE_LIST=""; + for r in $LST; do + FOUND=$(rpm -q $r) ; + if [ $? -eq 0 ]; then + DELETE_LIST="$DELETE_LIST $FOUND"; + fi; + done; + echo "uninstalling these packages: $DELETE_LIST"; if [ "$DELETE_LIST" != "" ]; then rpm -e --nodeps $DELETE_LIST; fi' - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1112,8 +1124,8 @@ mock_clean_cache_cfg () { clean_yum_cache_cfg $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1141,8 +1153,8 @@ mock_clean_cache_all_cfg () { echo "==================================" clean_yum_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" } @@ -1169,9 +1181,16 @@ mock_clean_metadata_cfg () { return 1 fi - CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';') - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP + CMD=$((cat $CFG; \ + grep config_opts\\[\'yum.conf\'\\\] $CFG | \ + sed 's#\\n#\n#g') | \ + grep '^[[]' | \ + grep -v main | \ + sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --enablerepo=#" -e 's#$# clean metadata#' | \ + sort -u | \ + tr '\n' ';') + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1228,7 +1247,7 @@ update_cgcs_repo () { if [ "x$NEW_UNTRACKED" != "x" ]; then NEED_REBUILD=1 fi - + if [ $NEED_REBUILD -eq 1 ]; then NEED_MOCK_CLEAN=1 echo "" @@ -1274,7 +1293,7 @@ mock_clean_mounts_dir () { fi fi if [ $RC -eq 1 ]; then - echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." + echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." echo "Ask your system administrator to umount '$MOUNT'." exit 1 fi @@ -1319,7 +1338,7 @@ clean_yum_cache_cfg () { local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum") local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock" # echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'" - + if [ "$YUM_CACHE_MOUNT" != "" ]; then mock_clean_mounts_dir "$YUM_CACHE_MOUNT" fi @@ -1358,8 +1377,8 @@ mock_update_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update echo "==================================" } @@ -1368,8 +1387,8 @@ mock_init_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init echo "==================================" } @@ -1388,14 +1407,14 @@ mock_update_or_init_cfg () { if [ -d $ROOT_DIR/root ]; then echo "Updating the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update &> $TMP RC=$? else echo "Init the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init &> $TMP RC=$? fi if [ $RC -ne 0 ]; then @@ -1411,7 +1430,7 @@ mock_update_or_init () { local PREFIX=$(echo $SUB_CFG | rev | cut -d . -f 2 | rev) ( mock_update_or_init_cfg $SUB_CFG 2>&1 | sed "s#^#${PREFIX}: #" ; exit ${PIPESTATUS[0]} ) & done - wait + wait echo "${FUNCNAME[0]}: out" } @@ -1429,12 +1448,12 @@ NO_DESCENDANTS=0 NO_REQUIRED=0 NO_AUTOCLEAN=0 NO_BUILD_INFO=0 -HELP=0 +HELP=0 CLEAN_FLAG=0 FORMAL_FLAG=0 CAREFUL=0 DEP_TEST_FLAG=0 - + # read the options TEMP=$(getopt -o h --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,dep-test,clean,tmpfs-clean,formal,careful,help,layer: -n "$ME" -- "$@") @@ -1470,7 +1489,7 @@ while true ; do --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac -done +done # Reset variables if [ -n "$MY_WORKSPACE" ]; then @@ -1729,7 +1748,7 @@ clean_list () { echo "rm -rf $d" \rm -rf "$d" 2>> /dev/null done - + else rs=$(rpm_get_srpm $r) if [[ "$rs" != "$sn"-[0-9]* ]]; then @@ -1742,7 +1761,7 @@ clean_list () { fi done done - + TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s") if [ $CLEAN_FLAG -eq 1 ]; then @@ -1782,7 +1801,7 @@ clean_list () { \rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null mock_clean - else + else # If dependency test if [ $DEP_TEST_FLAG -eq 1 ]; then mock_clean @@ -2045,7 +2064,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done # For non-std build, and if non specific build targets are named, then search all - # packages that we might build and check if they require a package that DID build + # packages that we might build and check if they require a package that DID build # in the std build. If so build the package as a secondary target, even though the # primary target was from a different build_type. if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then @@ -2105,7 +2124,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done fi - # If the kernel or kernel-rt packages where absent from the primary build targets, but + # If the kernel or kernel-rt packages where absent from the primary build targets, but # added as a secondary target, then make sure all out-of-tree kernel modules are also # added. for n in kernel kernel-rt; do @@ -2234,9 +2253,9 @@ echo "SRPMS_LIST = $SRPMS_LIST" echo "RPMS_LIST = $RPMS_LIST" -echo +echo if [ $CLEAN_FLAG -eq 0 ]; then - # pre-create these directories as $USER, + # pre-create these directories as $USER, # else mock will create them as root and fails to clean them. # Note: keep these in sync with mockchain-parallel! for i in $(seq 0 $((MAX_WORKERS-1))); do @@ -2299,15 +2318,28 @@ mock_clean_metadata echo echo "Building" +recreate_repodata $BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR + CMD_PREFIX="" if [ -x /bin/ionice ]; then CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice " fi -CMD_OPTIONS="-m --no-clean -m --no-cleanup-after" -if [ $CAREFUL -eq 1 ]; then - CMD_OPTIONS="-m --no-cleanup-after" +REAL_MOCKCHAIN=0 +MOCK_PASSTHROUGH="-m" +MOCKCHAIN="mockchain-parallel" +CHAIN_OPTION="" +if file $(which mockchain) | grep -q 'Python script'; then + REAL_MOCKCHAIN=1 fi + +CMD_OPTIONS="$MOCK_PASSTHROUGH --no-clean $MOCK_PASSTHROUGH --no-cleanup-after" +if [ $CAREFUL -eq 1 ]; then + CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after" +fi + +CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG" + echo "CAREFUL=$CAREFUL" # Sets WORKERS and MOCKCHAIN_RESOURCE_ALLOCATION @@ -2315,13 +2347,13 @@ compute_resources $SRPMS_LIST if [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then - CMD_OPTIONS="$CMD_OPTIONS --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE" + CMD_OPTIONS+=" --srpm-dependency-file $SRPM_RPM_DIRECT_REQUIRES_FILE" fi if [ -f "$RPM_DIRECT_REQUIRES_FILE" ]; then - CMD_OPTIONS="$CMD_OPTIONS --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE" + CMD_OPTIONS+=" --rpm-dependency-file $RPM_DIRECT_REQUIRES_FILE" fi if [ -f "$RPM_TO_SRPM_MAP_FILE" ]; then - CMD_OPTIONS="$CMD_OPTIONS --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE" + CMD_OPTIONS+=" --rpm-to-srpm-map-file $RPM_TO_SRPM_MAP_FILE" fi @@ -2342,12 +2374,12 @@ echo "MAX_WORKERS=$MAX_WORKERS" echo "MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION" -CMD="$CMD_PREFIX mockchain-parallel -r $BUILD_CFG -l $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS -m --rebuild $SRPMS_LIST" +CMD="$CMD_PREFIX $MOCKCHAIN --root $BUILD_CFG --localrepo $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS $MOCK_PASSTHROUGH --rebuild" +CMD_BUILD_LIST="$CHAIN_OPTION $SRPMS_LIST" echo "" -echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" +echo "$CMD $MOCK_PASSTHROUGH --define='_tis_dist .tis' $MOCK_PASSTHROUGH --define='platform_release $PLATFORM_RELEASE' $CMD_BUILD_LIST" echo "" - -trapwrap stdbuf -o0 $CMD -m --define="_tis_dist .tis" -m --define="platform_release $PLATFORM_RELEASE" +trapwrap stdbuf -o0 $CMD $MOCK_PASSTHROUGH --define="_tis_dist .tis" $MOCK_PASSTHROUGH --define="platform_release $PLATFORM_RELEASE" $CMD_BUILD_LIST MOCKCHAIN_RC=$? echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE @@ -2357,7 +2389,7 @@ if [ $CLEAN_FLAG -eq 0 ]; then fi for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do - rsync -u $d/*.rpm $RPM_DIR + rsync -u $d/*.rpm $RPM_DIR done if [ $ALL -eq 1 ]; then @@ -2441,5 +2473,5 @@ if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then fi fi -exit 0 +exit 0 ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-parallel_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build-rpms-serial b/build-tools/build-rpms-serial index 5bd7dbe7..0a6afbf7 100755 --- a/build-tools/build-rpms-serial +++ b/build-tools/build-rpms-serial @@ -1,20 +1,20 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2020 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # -# +# # Builds rpm files from src.rpm files. # # This version compiles one package at a time. -# -# The location of packages to be built is +# +# The location of packages to be built is # $MY_WORKSPACE//rpmbuild/SRPMS. # -# The build order is a derived from the BuildRequires in the +# The build order is a derived from the BuildRequires in the # spec files in the src.rpms. Note that the BuildRequires sometimes # create dependency loops, so no correct order can be computed. In these # cases we add a retry loop. As long as one new package builds, we @@ -103,7 +103,7 @@ create-no-clean-list () { local install_groups="" local install_packages="" local p - + for p in $(grep "config_opts\['chroot_setup_cmd'\]" $MY_BUILD_CFG | tail -n1 | cut -d '=' -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e "s/^'//" -e "s/'$//" -e 's/^install //'); do if [[ $p == @* ]] ; then install_groups=$(join_by ' ' $install_groups $(echo $p | cut -c 2-)) @@ -111,37 +111,44 @@ create-no-clean-list () { install_packages=$(join_by ' ' $install_packages $p) fi done - + local noclean_last_list_len=0 local noclean_list="" local tmp_list="" local g - + for g in $install_groups; do - tmp_list=$(yum -c $MY_YUM_CONF groupinfo $g 2>> /dev/null | awk 'f;/Mandatory Packages:/{f=1}' | cut -c 5-) + # Find manditory packages in the group. + # Discard anything before (and including) 'Mandatory Packages:' + # and anything after (and including) 'Optional Packages:'. + # Also discard leading spaces or '+' characters. + tmp_list=$(${PKG_MANAGER} -c $MY_YUM_CONF groupinfo $g 2>> /dev/null \ + | awk 'f;/Mandatory Packages:/{f=1}' \ + | sed -n '/Optional Packages:/q;p' \ + | sed 's#[ +]*##') noclean_list=$(join_by ' ' $noclean_list $tmp_list) done - + noclean_list=$(join_by ' ' $noclean_list $install_packages) noclean_list=$(echo $noclean_list | tr ' ' '\n' | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) - + while [ $noclean_list_len -gt $noclean_last_list_len ]; do noclean_last_list_len=$noclean_list_len noclean_list=$( (yum -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) noclean_list_len=$(echo $noclean_list | wc -w) done - + echo $noclean_list > $NO_CLEAN_LIST_FILE fi cat $NO_CLEAN_LIST_FILE } -str_lst_contains() { +str_lst_contains() { TARGET="$1" LST="$2" - if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then + if [[ $LST =~ (^|[[:space:]])$TARGET($|[[:space:]]) ]] ; then return 0 else return 1 @@ -169,7 +176,9 @@ create_lst () { recreate_repodata () { local DIR=${1} - (cd $DIR + ( + mkdir -p $DIR + cd $DIR if [ -f repodata/*comps*xml ]; then \mv repodata/*comps*xml comps.xml fi @@ -260,11 +269,11 @@ union () { echo $b fi done -} +} # # returns 0 if element is in the array -# +# # e.g. contains ARRAY $SEEKING && echo "$SEEKING is in 'ARRAY'" # contains () { @@ -311,7 +320,7 @@ build_order_recursive () { needs=( $(grep "^$target;" "$SRPM_DIRECT_REQUIRES_FILE" | sed "s/$target;//" | sed 's/,/ /g') ) needs_list=( $(intersection needs remainder_list) ) for((idx=0;idx<${#needs_list[@]};idx++)); do - build_order_recursive ${needs_list[idx]} + build_order_recursive ${needs_list[idx]} done echo $target break @@ -361,7 +370,7 @@ build_order () { UNORDERED_LIST=( ${original_list[@]} ) while [ ${#UNORDERED_LIST[@]} -gt 0 ]; do element=${UNORDERED_LIST[0]} - build_order_recursive $element + build_order_recursive $element done fi } @@ -476,7 +485,7 @@ kill_descendents () recursive_promote_children="" recursive_children=$(pgrep -P $kill_pid) fi - + if [ $iteration -eq 0 ]; then other_children=$(for relevant_child in $relevant_other_children; do pgrep -P $kill_pid $relevant_child; done) if [ "$other_children" != "" ]; then @@ -491,7 +500,7 @@ kill_descendents () fi for pid in $recursive_children; do - kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) + kill_descendents "$pid" $kill_all $need_stop $((iteration + 1)) done for pid in $recursive_promote_children; do kill_descendents "$pid" 1 1 $((iteration + 1)) @@ -754,11 +763,11 @@ mock_clean_cfg () { echo "==================================" mock_clean_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --clean" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --clean + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --clean" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --clean ### Note: this sometimes leaves behind a $MY_WORKSPACE/cache/mock/yum_cache/yumcache.lock echo "==================================" mock_clean_cache_all_cfg $CFG @@ -833,10 +842,10 @@ mock_partial_clean_cfg () { local RPMS_CLEAN_LIST="" local NEED_FULL_MOCK_CLEAN=0 for r in $RPMS_LIST; do - if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then + if ! str_lst_contains $r "$NO_CLEAN_LIST" ; then RPMS_CLEAN_LIST=$(join_by ' ' $RPMS_CLEAN_LIST $r) - else - echo "Can't remove '$r' from mock environment without a wipe"; + else + echo "Can't remove '$r' from mock environment without a wipe"; NEED_FULL_MOCK_CLEAN=1 fi done @@ -849,19 +858,19 @@ mock_partial_clean_cfg () { # Intent of following is for $RPMS_LIST to be expand now while the remaining $ varaibles are for bash inside mock to expand echo "Try to uninstall from the mock environment these packages: $RPMS_CLEAN_LIST" CMD='LST="'$RPMS_CLEAN_LIST'"; - DELETE_LIST=""; - for r in $LST; do - FOUND=$(rpm -q $r) ; - if [ $? -eq 0 ]; then - DELETE_LIST="$DELETE_LIST $FOUND"; - fi; - done; - echo "uninstalling these packages: $DELETE_LIST"; + DELETE_LIST=""; + for r in $LST; do + FOUND=$(rpm -q $r) ; + if [ $? -eq 0 ]; then + DELETE_LIST="$DELETE_LIST $FOUND"; + fi; + done; + echo "uninstalling these packages: $DELETE_LIST"; if [ "$DELETE_LIST" != "" ]; then rpm -e --nodeps $DELETE_LIST; fi' - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '$CMD'" &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -907,8 +916,8 @@ mock_clean_cache_cfg () { clean_yum_cache_cfg $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=root-cache --scrub=yum-cache --scrub=cache &>> $TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -931,8 +940,8 @@ mock_clean_cache_all_cfg () { echo "==================================" clean_yum_cache_cfg $CFG echo "==================================" - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --scrub=all + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --scrub=all echo "==================================" } @@ -956,8 +965,8 @@ mock_clean_metadata_cfg () { fi CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';') - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP RC=$? if [ $RC -ne 0 ]; then cat $TMP @@ -1010,7 +1019,7 @@ update_cgcs_repo () { if [ "x$NEW_UNTRACKED" != "x" ]; then NEED_REBUILD=1 fi - + if [ $NEED_REBUILD -eq 1 ]; then NEED_MOCK_CLEAN=1 echo "" @@ -1056,7 +1065,7 @@ mock_clean_mounts_dir () { fi fi if [ $RC -eq 1 ]; then - echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." + echo "ERROR: Directory '$MOUNT' is already mounted and will cause a build failure within mock." echo "Ask your system administrator to umount '$MOUNT'." exit 1 fi @@ -1097,7 +1106,7 @@ clean_yum_cache_cfg () { local YUM_CACHE_MOUNT=$(readlink -f "$ROOT_DIR/root/var/cache/yum") local YUM_CACHE_LOCK="$CACHE_DIR/mock/yum_cache/yumcache.lock" # echo "clean_yum_cache YUM_CACHE_MOUNT='$YUM_CACHE_MOUNT' YUM_CACHE_LOCK='$YUM_CACHE_LOCK'" - + if [ "$YUM_CACHE_MOUNT" != "" ]; then mock_clean_mounts_dir "$YUM_CACHE_MOUNT" fi @@ -1131,8 +1140,8 @@ mock_update_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update echo "==================================" } @@ -1141,8 +1150,8 @@ mock_init_cfg () { echo "${FUNCNAME[0]}: $CFG" echo "==================================" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init echo "==================================" } @@ -1161,14 +1170,14 @@ mock_update_or_init_cfg () { if [ -d $ROOT_DIR/root ]; then echo "Updating the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --update" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --update &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --update" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --update &> $TMP RC=$? else echo "Init the mock environment" set_mock_symlinks $CFG - echo "$MOCK -r $CFG --configdir $(dirname $CFG) --init" - trapwrap_n $CFG $MOCK -r $CFG --configdir $(dirname $CFG) --init &> $TMP + echo "$MOCK --root $CFG --configdir $(dirname $CFG) --init" + trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --init &> $TMP RC=$? fi if [ $RC -ne 0 ]; then @@ -1198,12 +1207,12 @@ NO_DESCENDANTS=0 NO_REQUIRED=0 NO_AUTOCLEAN=0 NO_BUILD_INFO=0 -HELP=0 +HELP=0 CLEAN_FLAG=0 FORMAL_FLAG=0 CAREFUL=0 DEP_TEST_FLAG=0 - + # read the options TEMP=$(getopt -o h --long serial,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,dep-test,clean,formal,careful,help,layer: -n "$ME" -- "$@") @@ -1238,7 +1247,7 @@ while true ; do --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac -done +done # Reset variables @@ -1493,7 +1502,7 @@ clean_list () { echo "rm -rf $d" \rm -rf "$d" 2>> /dev/null done - + else rs=$(rpm_get_srpm $r) if [[ "$rs" != "$sn"-[0-9]* ]]; then @@ -1506,7 +1515,7 @@ clean_list () { fi done done - + TARGET=$(rpm -qp --qf '%{NAME}-%{VERSION}\n' "$s") if [ $CLEAN_FLAG -eq 1 ]; then @@ -1542,7 +1551,7 @@ clean_list () { \rm -f -v $RESULT_DIR/mockchain.log 2>> /dev/null mock_clean - else + else # If dependency test if [ $DEP_TEST_FLAG -eq 1 ]; then mock_clean @@ -1801,7 +1810,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done # For non-std build, and if non specific build targets are named, then search all - # packages that we might build and check if they require a package that DID build + # packages that we might build and check if they require a package that DID build # in the std build. If so build the package as a secondary target, even though the # primary target was from a different build_type. if [ "$BUILD_TYPE" != "std" ] && [ $ALL -eq 1 ] && [ -f $SRPM_TO_RPM_MAP_FILE ] && [ -f $SRPM_RPM_DIRECT_REQUIRES_FILE ]; then @@ -1861,7 +1870,7 @@ if [ $CLEAN_FLAG -eq 0 ] && [ $NO_DESCENDANTS -eq 0 ] && [ -f $SRPM_DIRECT_DESCE done fi - # If the kernel or kernel-rt packages where absent from the primary build targets, but + # If the kernel or kernel-rt packages where absent from the primary build targets, but # added as a secondary target, then make sure all out-of-tree kernel modules are also # added. for n in kernel kernel-rt; do @@ -1996,9 +2005,9 @@ echo "SRPMS_LIST = $SRPMS_LIST" echo "RPMS_LIST = $RPMS_LIST" -echo +echo if [ $CLEAN_FLAG -eq 0 ]; then - # pre-create these directories as $USER, + # pre-create these directories as $USER, # else mock will create them as root and fails to clean them. # Note: keep these in sync with mockchain-parallel! mkdir -p $MY_WORKSPACE/mock @@ -2059,27 +2068,46 @@ mock_clean_metadata echo echo "Building" +recreate_repodata $BUILD_BASE/results/$BUILD_ENVIRONMENT_DIR + CMD_PREFIX="" if [ -x /bin/ionice ]; then CMD_PREFIX="nice -n 20 ionice -c Idle /bin/ionice " fi -CMD_OPTIONS="-m --no-clean -m --no-cleanup-after" +REAL_MOCKCHAIN=0 +MOCK_PASSTHROUGH="" +MOCKCHAIN="mock" +CHAIN_OPTION="--chain" +if file $(which mockchain) | grep -q 'Python script'; then + REAL_MOCKCHAIN=1 + MOCK_PASSTHROUGH="-m" + MOCKCHAIN="mockchain" + CHAIN_OPTION="" +fi + +CMD_OPTIONS="$MOCK_PASSTHROUGH --no-clean $MOCK_PASSTHROUGH --no-cleanup-after" if [ $CAREFUL -eq 1 ]; then - CMD_OPTIONS="-m --no-cleanup-after" + CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after" +fi +if [ $REAL_MOCKCHAIN -eq 1 ]; then + CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG" fi echo "CAREFUL=$CAREFUL" echo "CMD_OPTIONS=$CMD_OPTIONS" -CMD="$CMD_PREFIX mockchain -r $BUILD_CFG -l $BUILD_BASE --recurse --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL -m --rootdir=$BUILD_BASE/mock/root $CMD_OPTIONS -m --rebuild $SRPMS_LIST" -echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" -trapwrap stdbuf -o0 $CMD -m --define='_tis_dist .tis' -m --define="platform_release $PLATFORM_RELEASE" +CMD="$CMD_PREFIX $MOCKCHAIN --root $BUILD_CFG --localrepo $BUILD_BASE --recurse --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $MOCK_PASSTHROUGH --rootdir=$BUILD_BASE/mock/root $CMD_OPTIONS $MOCK_PASSTHROUGH --rebuild" +CMD_BUILD_LIST="$CHAIN_OPTION $SRPMS_LIST" +echo "" +echo "$CMD $MOCK_PASSTHROUGH --define='_tis_dist .tis' $MOCK_PASSTHROUGH --define='platform_release $PLATFORM_RELEASE' $CMD_BUILD_LIST" +echo "" +trapwrap stdbuf -o0 $CMD $MOCK_PASSTHROUGH --define="_tis_dist .tis" $MOCK_PASSTHROUGH --define="platform_release $PLATFORM_RELEASE" $CMD_BUILD_LIST MOCKCHAIN_RC=$? echo $PLATFORM_RELEASE > $LAST_PLATFORM_RELEASE_FILE for d in $(find $RESULT_DIR -name '*.rpm' | grep -v '[.]src[.]rpm' | xargs --max-args=1 dirname | sort -u); do - rsync -u $d/*.rpm $RPM_DIR + rsync -u $d/*.rpm $RPM_DIR done if [ $ALL -eq 1 ]; then @@ -2163,5 +2191,5 @@ if [ 0$FORMAL_BUILD -eq 1 ] && [ "$USER" == "jenkins" ]; then fi fi -exit 0 +exit 0 ) 2>&1 | stdbuf -o0 awk '{ print strftime("%H:%M:%S"), $0; fflush(); }' | tee $(date "+$MY_WORKSPACE/build-rpms-serial_%Y-%m-%d_%H-%M-%S.log") ; exit ${PIPESTATUS[0]} diff --git a/build-tools/build_iso/cgts_deps.sh b/build-tools/build_iso/cgts_deps.sh index e1ccbf96..ab45352d 100755 --- a/build-tools/build_iso/cgts_deps.sh +++ b/build-tools/build_iso/cgts_deps.sh @@ -1,5 +1,11 @@ #!/bin/env bash +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # Here's the score, kids. There are a few different places from which we can # get packages. In priority order, they are: # @@ -26,21 +32,33 @@ # export/dist/report_deps.txt # +CGTS_DEPS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source ${CGTS_DEPS_DIR}/../pkg-manager-utils.sh + # This function generates a simple file of dependencies we're trying to resolve function generate_dep_list { TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) mkdir -p $TMP_RPM_DB rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | grep -v "installing package" \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' > $DEPLISTFILE_NEW 2>&1 + cat $DEPLISTFILE_NEW >> $DEPDETAILLISTFILE + cat $DEPLISTFILE_NEW \ + | grep -v -e "error:" -e "warning:" -e "Preparing..." \ + -e "Verifying..." -e "installing package" \ + | sed -e "s/ is needed by.*$//" -e "s/ [<=>].*$//" \ + | sort -u > $DEPLISTFILE \rm -rf $TMP_RPM_DB } +join_array() { + local IFS="$1" + shift + echo "$*" +} + # Takes a list of requirements (either explcit package name, or capabilities # to provide) and install packages to meet those dependancies # @@ -55,6 +73,7 @@ function generate_dep_list { # capabilities. function install_deps { local DEP_LIST="" + local DEP_LIST_ARRAY=() local DEP_LIST_FILE="$1" # Temporary files are used in a few different ways @@ -68,7 +87,7 @@ function install_deps { while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST+=" '${DEP}'" done < $DEP_LIST_FILE echo "Debug: List of deps to resolve: ${DEP_LIST}" @@ -79,8 +98,17 @@ function install_deps { # go through each repo and convert deps to packages based on package name for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=$REPOID"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "${DEP_LIST} --qf='%{name}'" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=$REPOID \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' ${DEP_LIST} \ + | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + \rm -rf $TMP_DIR/yum-$USER-* done sort $UNSORTED_PACKAGES -u > $SORTED_PACKAGES @@ -96,19 +124,31 @@ function install_deps { # If there are any requirements not resolved, look up the packages with # --whatprovides if [ -s $UNRESOLVED_PACKAGES ]; then - DEP_LIST="" + DEP_LIST_ARRAY=() \cp $SORTED_PACKAGES $UNSORTED_PACKAGES while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST_ARRAY+=( "${DEP}" ) done < $UNRESOLVED_PACKAGES - DEP_LIST=$(echo "$DEP_LIST" | sed 's/^ //g') - if [ "$DEP_LIST" != "" ]; then + if [ "${REPOQUERY_WHATPROVIDES_DELIM}" != " " ]; then + DEP_LIST_ARRAY=( "$(join_array "${REPOQUERY_WHATPROVIDES_DELIM}" "${DEP_LIST_ARRAY[@]}" )" ) + fi + + if [ ${#DEP_LIST_ARRAY[@]} -gt 0 ]; then for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]}" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]} \ + | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + \rm -rf $TMP_DIR/yum-$USER-* done fi @@ -124,12 +164,12 @@ function install_deps { DEP_LIST=" " while read DEP do - DEP_LIST="${DEP_LIST}${DEP} " + DEP_LIST+="${DEP} " done < $SORTED_PACKAGES rm $SORTED_PACKAGES # go through each repo and install packages - local TARGETS=${DEP_LIST} + local TARGETS="${DEP_LIST}" echo "Debug: Resolved list of deps to install: ${TARGETS}" local UNRESOLVED for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do @@ -137,8 +177,20 @@ function install_deps { if [[ ! -z "${TARGETS// }" ]]; then REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V > $TMPFILE + + >&2 echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'"\ + "${REPOQUERY_RESOLVE} ${TARGETS}" + + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" \ + ${REPOQUERY_RESOLVE} ${TARGETS} \ + | sort -r -V > $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* while read STR @@ -157,7 +209,11 @@ function install_deps { cp $PKG_PATH . if [ $? -ne 0 ]; then >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\""\ + "${REPOQUERY_RESOLVE} ${PKG}" >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" fi @@ -172,8 +228,8 @@ function install_deps { echo " path $PKG_PATH" >> $BUILT_REPORT FOUND_UNKNOWN=1 fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" - \rm -rf $TMP_DIR/yum-$USER-* + done < $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* TARGETS="$UNRESOLVED" fi done @@ -220,6 +276,7 @@ OUTPUT_DIR=$MY_WORKSPACE/export TMP_DIR=$MY_WORKSPACE/tmp YUM=$OUTPUT_DIR/yum.conf DEPLISTFILE=$OUTPUT_DIR/deps.txt +DEPLISTFILE_NEW=$OUTPUT_DIR/deps_new.txt DEPDETAILLISTFILE=$OUTPUT_DIR/deps_detail.txt BUILT_REPORT=$OUTPUT_DIR/local.txt diff --git a/build-tools/build_minimal_iso/cgts_deps.sh b/build-tools/build_minimal_iso/cgts_deps.sh index 062b74ff..9e4a88b2 100755 --- a/build-tools/build_minimal_iso/cgts_deps.sh +++ b/build-tools/build_minimal_iso/cgts_deps.sh @@ -1,40 +1,71 @@ #!/bin/env bash + +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +CGTS_DEPS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set REPOQUERY, REPOQUERY_SUB_COMMAND, REPOQUERY_RESOLVE and +# REPOQUERY_WHATPROVIDES_DELIM for our build environment. +source ${CGTS_DEPS_DIR}/../pkg-manager-utils.sh + function generate_dep_list { TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) mkdir -p $TMP_RPM_DB rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE - rm -rf $TMP_RPM_DB + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' > $DEPLISTFILE_NEW 2>&1 + cat $DEPLISTFILE_NEW >> $DEPDETAILLISTFILE + cat $DEPLISTFILE_NEW \ + | grep -v -e "error:" -e "warning:" -e "Preparing..." \ + -e "Verifying..." -e "installing package" \ + | sed -e "s/ is needed by.*$//" -e "s/ [<=>].*$//" \ + | sort -u > $DEPLISTFILE + \rm -rf $TMP_RPM_DB +} + +join_array() { + local IFS="$1" + shift + echo "$*" } function install_deps { local DEP_LIST="" + local DEP_LIST_ARRAY=() local DEP_LIST_FILE="$1" rm -f $TMPFILE while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST_ARRAY+=( "${DEP}" ) done < $DEP_LIST_FILE - echo "Debug: List of deps to resolve: ${DEP_LIST}" + if [ "${REPOQUERY_WHATPROVIDES_DELIM}" != " " ]; then + DEP_LIST_ARRAY=( "$(join_array "${REPOQUERY_WHATPROVIDES_DELIM}" "${DEP_LIST_ARRAY[@]}" )" ) + fi - if [ -z "${DEP_LIST}" ]; then + echo "Debug: List of deps to resolve: ${DEP_LIST_ARRAY[@]}" + + if [ ${#DEP_LIST_ARRAY[@]} -gt 0 ]; then return 0 fi # go through each repo and convert deps to packages - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $TMPFILE - \rm -rf $TMP_DIR/yum-$USER-* + echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]}" + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf='%{name}' --whatprovides ${DEP_LIST_ARRAY[@]} \ + | sed "s/kernel-debug/kernel/g" >> $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* done sort $TMPFILE -u > $TMPFILE1 rm $TMPFILE @@ -42,12 +73,12 @@ function install_deps { DEP_LIST="" while read DEP do - DEP_LIST="${DEP_LIST} ${DEP}" + DEP_LIST+="${DEP} " done < $TMPFILE1 rm $TMPFILE1 # next go through each repo and install packages - local TARGETS=${DEP_LIST} + local TARGETS="${DEP_LIST}" echo "Debug: Resolved list of deps to install: ${TARGETS}" local UNRESOLVED for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do @@ -55,9 +86,19 @@ function install_deps { if [[ ! -z "${TARGETS// }" ]]; then REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V >> $TMPFILE - \rm -rf $TMP_DIR/yum-$USER-* + >&2 echo "TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} --config=${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'"\ + "${REPOQUERY_RESOLVE} ${TARGETS}" + TMPDIR=${TMP_DIR} \ + ${REPOQUERY} --config=${YUM} --repoid=${REPOID} \ + ${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch \ + --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" \ + ${REPOQUERY_RESOLVE} ${TARGETS} \ + | sort -r -V >> $TMPFILE + + \rm -rf $TMP_DIR/yum-$USER-* while read STR do @@ -75,7 +116,11 @@ function install_deps { cp $PKG_PATH . if [ $? -ne 0 ]; then >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " TMPDIR=${TMP_DIR}"\ + "${REPOQUERY} -c ${YUM} --repoid=${REPOID}"\ + "${REPOQUERY_SUB_COMMAND} --arch=x86_64,noarch"\ + "--qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\""\ + "${REPOQUERY_RESOLVE} ${PKG}" >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" fi @@ -91,8 +136,9 @@ function install_deps { echo " path $PKG_PATH" >> $BUILT_REPORT FOUND_UNKNOWN=1 fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" - \rm -rf $TMP_DIR/yum-$USER-* + done < $TMPFILE + + \rm -rf $TMP_DIR/yum-$USER-* TARGETS="$UNRESOLVED" fi done @@ -143,6 +189,7 @@ OUTPUT_DIR=${ROOT}/newDisk YUM=${ROOT}/yum.conf TMP_DIR=${ROOT}/tmp DEPLISTFILE=${ROOT}/deps.txt +DEPLISTFILE_NEW=${ROOT}/deps_new.txt DEPDETAILLISTFILE=${ROOT}/deps_detail.txt INSTALLDIR=${ROOT}/newDisk/isolinux/Packages diff --git a/build-tools/default_build_srpm b/build-tools/default_build_srpm index e74b3661..fe99ad6c 100755 --- a/build-tools/default_build_srpm +++ b/build-tools/default_build_srpm @@ -1,6 +1,12 @@ #!/bin/bash # set -x +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + source "$SRC_BASE/build-tools/spec-utils" source "$SRC_BASE/build-tools/srpm-utils" @@ -195,14 +201,14 @@ fi if [ $TAR_NEEDED -gt 0 ]; then echo "Creating tar file: $TAR_PATH ..." - echo "tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude='$SRC_DIR/$DISTRO' --exclude='pbr-*.egg' --transform 's,^$TRANSFORM,$TAR_NAME-$VERSION,'" - tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION," + echo "tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude='$SRC_DIR/$DISTRO' --exclude='pbr-*.egg' --transform 's,^$TRANSFORM,$TAR_NAME-$VERSION,' -czf $TAR_PATH $SRC_DIR" + tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION," -czf "$TAR_PATH" "$SRC_DIR" if [ $? -ne 0 ]; then if [ "x$STAGING" != "x" ]; then popd fi - echo "ERROR: default_build_srpm (${LINENO}): failed to create tar file, cmd: tar czf $TAR_PATH $SRC_DIR --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform \"s,^$TRANSFORM,$TAR_NAME-$VERSION,\"" + echo "ERROR: default_build_srpm (${LINENO}): failed to create tar file, cmd: tar --exclude '.git*' --exclude 'build' --exclude='.pc' --exclude='patches' --exclude="$SRC_DIR/$DISTRO" --exclude='pbr-*.egg' --transform \"s,^$TRANSFORM,$TAR_NAME-$VERSION,\" -czf '$TAR_PATH' '$SRC_DIR'" exit 1 fi echo "Created tar file: $TAR_PATH" diff --git a/build-tools/mockchain-parallel b/build-tools/mockchain-parallel index 8645d30f..73029df6 100755 --- a/build-tools/mockchain-parallel +++ b/build-tools/mockchain-parallel @@ -24,10 +24,16 @@ interpreter_path () { get__version__ () { local path=${1} + local var="" if [ ! -f ${path} ]; then return 1 fi - grep __VERSION__= ${path} | cut -d '=' -f 2 | sed 's/"//g' + if file ${path} | grep -q 'Python script'; then + ver=$(grep __VERSION__= ${path} | cut -d '=' -f 2 | sed 's/"//g') + else + ver=$(${path} --version 2> /dev/null) + fi + echo $ver } VC_LESS_THAN=0 diff --git a/build-tools/mockchain-parallel-2.6 b/build-tools/mockchain-parallel-2.6 new file mode 100755 index 00000000..c159d9bf --- /dev/null +++ b/build-tools/mockchain-parallel-2.6 @@ -0,0 +1,1221 @@ +#!/usr/bin/python3 -tt +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built +from __future__ import print_function + +import cgi +# pylint: disable=deprecated-module +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import multiprocessing +import signal +import psutil + +import requests +# pylint: disable=import-error +from six.moves.urllib_parse import urlsplit + +import mockbuild.util + +from stxRpmUtils import splitRpmFilename + +# all of the variables below are substituted by the build system +__VERSION__="2.6" +SYSCONFDIR="/etc" +PYTHONDIR="/usr/lib/python3.6/site-packages" +PKGPYTHONDIR="/usr/lib/python3.6/site-packages/mockbuild" +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs + +mockconfig_path = '/etc/mock' + +def rpmName(path): + filename = os.path.basename(path) + (n, v, r, e, a) = splitRpmFilename(filename) + return n + +def createrepo(path): + global max_workers + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] + else: + comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] + cmd = subprocess.Popen( + comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + + +g_opts = optparse.Values() + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option( + '-r', '--root', default=None, dest='chroot', + metavar="CONFIG", + help="chroot config name/base to use in the mock build") + parser.add_option( + '-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option( + '-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option( + '--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option( + '--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option( + '--workers', default=1, dest='max_workers', + help="number of parallel build jobs") + parser.add_option( + '--worker-resources', default="", dest='worker_resources', + help="colon seperated list, how much mem in gb for each workers temfs") + parser.add_option( + '--basedir', default='/var/lib/mock', dest='basedir', + help="path to workspace") + parser.add_option( + '--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + parser.add_option( + '-m', '--mock-option', default=[], action='append', + dest='mock_option', + help="option to pass directly to mock") + parser.add_option( + '--mark-slow-name', default=[], action='append', + dest='slow_pkg_names_raw', + help="package name that is known to build slowly") + parser.add_option( + '--mark-slow-path', default=[], action='append', + dest='slow_pkgs_raw', + help="package path that is known to build slowly") + parser.add_option( + '--mark-big-name', default=[], action='append', + dest='big_pkg_names_raw', + help="package name that is known to require a lot of disk space to build") + parser.add_option( + '--mark-big-path', default=[], action='append', + dest='big_pkgs_raw', + help="package path that is known to require a lot of disk space to build") + parser.add_option( + '--srpm-dependency-file', default=None, + dest='srpm_dependency_file', + help="path to srpm dependency file") + parser.add_option( + '--rpm-dependency-file', default=None, + dest='rpm_dependency_file', + help="path to rpm dependency file") + parser.add_option( + '--rpm-to-srpm-map-file', default=None, + dest='rpm_to_srpm_map_file', + help="path to rpm to srpm map file") + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print("You must provide an argument to -r for the mock chroot") + sys.exit(1) + + if len(sys.argv) < 3: + print("You must specify at least 1 package to build") + sys.exit(1) + + return opts, args + + +REPOS_ID = [] + +slow_pkg_names={} +slow_pkgs={} +big_pkg_names={} +big_pkgs={} + +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): + # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['rootdir'] = config_opts['rootdir'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) + # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) + # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) + if tmpfs_size_gb > 0: + config_opts['plugin_conf']['tmpfs_enable'] = True + config_opts['plugin_conf']['tmpfs_opts'] = {} + config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 + config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb + config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' + config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) + if not os.path.isdir(config_opts['cache_topdir']): + os.makedirs(config_opts['cache_topdir'], exist_ok=True) + except (IOError, OSError): + return False, "Could not create dir: %s" % config_opts['cache_topdir'] + + cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + except (IOError, OSError): + return False, "Could not create dir: %s" % cache_dir + + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def set_basedir(infile, destfile, basedir, opts): + log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['basedir'] = basedir + config_opts['resultdir'] = '{0}/result'.format(basedir) + config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) + config_opts['root'] = 'mock/b0' + config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) + config_opts['rootdir'] = '{0}/mock/b0/root'.format(basedir) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + global config_opts + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 +cost=1 +best=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + signal.signal(signal.SIGTERM, child_signal_handler) + signal.signal(signal.SIGINT, child_signal_handler) + signal.signal(signal.SIGHUP, child_signal_handler) + signal.signal(signal.SIGABRT, child_signal_handler) + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + # return 2, None, None, None + sys.exit(2) + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + if opts.uniqueext == '': + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--root', cfg, ] + else: + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '--root', cfg, ] + + # Ensure repo is up-to-date. + # Note: Merely adding --update to mockcmd failed to update + mockcmd_update=mockcmd + mockcmd_update.append('--update') + cmd = subprocess.Popen( + mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode != 0: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + + # heuristic here, if user pass for mock "-d foo", but we must be care to leave + # "-d'foo bar'" or "--define='foo bar'" as is + compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') + compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') + for option in opts.mock_option: + r_match = compiled_re_1.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + r_match = compiled_re_2.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + mockcmd.append(option) + + print('building %s' % s_pkg) + mockcmd.append(pkg) + cmd = subprocess.Popen( + mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode == 0: + with open(success_file, 'w') as f: + f.write('done\n') + ret = 1 + else: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + with open(fail_file, 'w') as f: + f.write('undone\n') + ret = 0 + + # return ret, cmd, out, err + sys.exit(ret) + + +def log(lf, msg): + if lf: + now = time.time() + try: + with open(lf, 'a') as f: + f.write(str(now) + ':' + msg + '\n') + except (IOError, OSError) as e: + print('Could not write to logfile %s - %s' % (lf, str(e))) + print(msg) + + +config_opts = {} + +worker_data = [] +workers = 0 +max_workers = 1 + +build_env = [] + +failed = [] +built_pkgs = [] + +local_repo_dir = "" + +pkg_to_name={} +name_to_pkg={} +srpm_dependencies_direct={} +rpm_dependencies_direct={} +rpm_to_srpm_map={} +no_dep_list = [ "bash", "kernel" , "kernel-rt" ] + + +def init_build_env(slots, opts, config_opts_in): + global build_env + + orig_chroot_name=config_opts_in['chroot_name'] + orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) + # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) + for i in range(0,slots): + new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) + new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) + tmpfs_size_gb = 0 + if opts.worker_resources == "": + if i > 0: + tmpfs_size_gb = 2 * (1 + slots - i) + else: + resource_array=opts.worker_resources.split(':') + if i < len(resource_array): + tmpfs_size_gb=int(resource_array[i]) + else: + log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) + sys.exit(1) + if i == 0 and tmpfs_size_gb != 0: + log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) + sys.exit(1) + build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) + + res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + +idle_build_env_last_awarded = 0 +def get_idle_build_env(slots): + global build_env + global idle_build_env_last_awarded + visited = 0 + + if slots < 1: + return -1 + + i = idle_build_env_last_awarded - 1 + if i < 0 or i >= slots: + i = slots - 1 + + while visited < slots: + if build_env[i]['state'] == 'Idle': + build_env[i]['state'] = 'Busy' + idle_build_env_last_awarded = i + return i + visited = visited + 1 + i = i - 1 + if i < 0: + i = slots - 1 + return -1 + +def release_build_env(idx): + global build_env + + build_env[idx]['state'] = 'Idle' + +def get_best_rc(a, b): + print("get_best_rc: a=%s" % str(a)) + print("get_best_rc: b=%s" % str(b)) + if (b == {}) and (a != {}): + return a + if (a == {}) and (b != {}): + return b + + if (b['build_name'] is None) and (not a['build_name'] is None): + return a + if (a['build_name'] is None) and (not b['build_name'] is None): + return b + + if a['unbuilt_deps'] < b['unbuilt_deps']: + return a + if b['unbuilt_deps'] < a['unbuilt_deps']: + return b + + if a['depth'] < b['depth']: + return a + if b['depth'] < a['depth']: + return b + + print("get_best_rc: uncertain %s vs %s" % (a,b)) + return a + +unbuilt_dep_list_print=False +def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): + global srpm_dependencies_direct + global rpm_dependencies_direct + global rpm_to_srpm_map + global no_dep_list + global unbuilt_dep_list_print + + first_iteration=False + unbuilt = [] + if name in no_dep_list: + return unbuilt + + if checked is None: + first_iteration=True + checked=[] + + # Count unbuild dependencies + if first_iteration: + dependencies_direct=srpm_dependencies_direct + else: + dependencies_direct=rpm_dependencies_direct + + if name in dependencies_direct: + for rdep in dependencies_direct[name]: + sdep='???' + if rdep in rpm_to_srpm_map: + sdep = rpm_to_srpm_map[rdep] + if rdep != name and sdep != name and not rdep in checked: + if (not first_iteration) and (sdep in no_dep_list): + continue + checked.append(rdep) + if sdep in unbuilt_pkg_names: + if not sdep in unbuilt: + unbuilt.append(sdep) + if depth > 0: + child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) + for sub_sdep in child_unbuilt: + if sub_sdep != name: + if not sub_sdep in unbuilt: + unbuilt.append(sub_sdep) + + return unbuilt + +def can_build_at_idx(build_idx, name, opts): + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + global build_env + + fs_size_gb = 0 + size_gb = 0 + speed = 0 + pkg = name_to_pkg[name] + if name in big_pkg_names: + size_gb=big_pkg_names[name] + if pkg in big_pkgs: + size_gb=big_pkgs[pkg] + if name in slow_pkg_names: + speed=slow_pkg_names[name] + if pkg in slow_pkgs: + speed=slow_pkgs[pkg] + fs_size_gb = build_env[build_idx]['fs_size_gb'] + return fs_size_gb == 0 or fs_size_gb >= size_gb + +def schedule(build_idx, pkgs, opts): + global worker_data + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + + unbuilt_pkg_names=[] + building_pkg_names=[] + unprioritized_pkg_names=[] + + for pkg in pkgs: + name = pkg_to_name[pkg] + unbuilt_pkg_names.append(name) + unprioritized_pkg_names.append(name) + + prioritized_pkg_names=[] + + for wd in worker_data: + pkg = wd['pkg'] + if not pkg is None: + name = pkg_to_name[pkg] + building_pkg_names.append(name) + + # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) + if len(big_pkg_names) or len(big_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in big_pkg_names or pkg in big_pkgs: + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + if len(slow_pkg_names) or len(slow_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in slow_pkg_names or pkg in slow_pkgs: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + for name in unprioritized_pkg_names: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + + name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) + if not name_out is None: + pkg_out = name_to_pkg[name_out] + else: + pkg_out = None + # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) + # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) + return pkg_out + + +def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): + global pkg_to_name + global name_to_pkg + global no_dep_list + + max_depth = 3 + + if len(pkg_names) == 0: + return None + + unbuilt_deps={} + building_deps={} + for depth in range(max_depth,-1,-1): + unbuilt_deps[depth]={} + building_deps[depth]={} + + for depth in range(max_depth,-1,-1): + checked=[] + reordered_pkg_names = pkg_names[:] + # for name in reordered_pkg_names: + while len(reordered_pkg_names): + name = reordered_pkg_names.pop(0) + if name in checked: + continue + + # log(opts.logfile, "checked.append(%s)" % name) + checked.append(name) + + pkg = name_to_pkg[name] + # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) + if not name in unbuilt_deps[depth]: + unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) + if not name in building_deps[depth]: + building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) + if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + if not name in unbuilt_deps[0]: + unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) + if not name in building_deps[0]: + building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) + if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): + if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + loop = False + for dep_name in unbuilt_deps[depth][name]: + if name == dep_name: + continue + + # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) + if dep_name in checked: + continue + + # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) + if not dep_name in unbuilt_deps[depth]: + unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) + if not dep_name in building_deps[depth]: + building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) + if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if not dep_name in unbuilt_deps[0]: + unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) + if not dep_name in building_deps[0]: + building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) + if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): + if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if name in unbuilt_deps[0][dep_name]: + loop = True + # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) + + if loop and len(building_deps[depth][name]) == 0: + log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) + return name + + for dep_name in unbuilt_deps[depth][name]: + if dep_name in reordered_pkg_names: + # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) + reordered_pkg_names.remove(dep_name) + reordered_pkg_names.insert(0,dep_name) + + # log(opts.logfile, "schedule2: Nothing buildable at this time") + return None + + +def read_deps(opts): + read_srpm_deps(opts) + read_rpm_deps(opts) + read_map_deps(opts) + +def read_srpm_deps(opts): + global srpm_dependencies_direct + + if opts.srpm_dependency_file == None: + return + + if not os.path.exists(opts.srpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) + sys.exit(1) + + with open(opts.srpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + srpm_dependencies_direct[name]=deps.split(',') + +def read_rpm_deps(opts): + global rpm_dependencies_direct + + if opts.rpm_dependency_file == None: + return + + if not os.path.exists(opts.rpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) + sys.exit(1) + + with open(opts.rpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + rpm_dependencies_direct[name]=deps.split(',') + +def read_map_deps(opts): + global rpm_to_srpm_map + + if opts.rpm_to_srpm_map_file == None: + return + + if not os.path.exists(opts.rpm_to_srpm_map_file): + log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) + sys.exit(1) + + with open(opts.rpm_to_srpm_map_file) as f: + lines = f.readlines() + for line in lines: + (rpm,srpm) = line.rstrip().split(';') + rpm_to_srpm_map[rpm]=srpm + + +def reaper(opts): + global built_pkgs + global failed + global worker_data + global workers + + reaped = 0 + need_createrepo = False + last_reaped = -1 + while reaped > last_reaped: + last_reaped = reaped + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is not None: + pkg = wd['pkg'] + b = int(wd['build_index']) + p.join() + worker_data.remove(wd) + workers = workers - 1 + reaped = reaped + 1 + release_build_env(b) + + log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) + + if ret == 0: + failed.append(pkg) + log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) + if opts.recurse and not stop_signal: + log(opts.logfile, "Will try to build again (if some other package will succeed).") + else: + log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) + elif ret == 1: + log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) + built_pkgs.append(pkg) + need_createrepo = True + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if need_createrepo: + # createrepo with the new pkgs + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + + return reaped + +stop_signal = False + +def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + +def kill_proc_and_descentents(parent, need_stop=False, verbose=False): + global g_opts + + if need_stop: + if verbose: + log(g_opts.logfile, "Stop %d" % parent.pid) + + try: + parent.send_signal(signal.SIGSTOP) + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.send_signal(signal.SIGSTOP) + + try: + children = parent.children(recursive=False) + except: + children = [] + + for p in children: + kill_proc_and_descentents(p, need_stop=True, verbose=verbose) + + if verbose: + log(g_opts.logfile, "Terminate %d" % parent.pid) + + # parent.send_signal(signal.SIGTERM) + try: + parent.terminate() + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.terminate() + + if need_stop: + if verbose: + log(g_opts.logfile, "Continue %d" % parent.pid) + + parent.send_signal(signal.SIGCONT) + + +def child_signal_handler(signum, frame): + global g_opts + my_pid = os.getpid() + # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) + p = psutil.Process(my_pid) + kill_proc_and_descentents(p) + try: + sys.exit(0) + except SystemExit as e: + os._exit(0) + +def signal_handler(signum, frame): + global g_opts + global stop_signal + global workers + global worker_data + stop_signal = True + + # Signal processes to complete + log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is None: + # log(g_opts.logfile, "terminate child %d" % p.pid) + p.terminate() + else: + log(g_opts.logfile, "child return code was %d" % ret) + + # Wait for remaining processes to complete + log(g_opts.logfile, "===== wait for signaled jobs to complete =====") + while len(worker_data) > 0: + log(g_opts.logfile, " remaining workers: %d" % workers) + reaped = reaper(g_opts) + if reaped == 0: + time.sleep(0.1) + + try: + sys.exit(1) + except SystemExit as e: + os._exit(1) + +def main(args): + opts, args = parse_args(args) + # take mock config + list of pkgs + + global g_opts + global stop_signal + global build_env + global worker_data + global workers + global max_workers + + global slow_pkg_names + global slow_pkgs + global big_pkg_names + global big_pkgs + max_workers = int(opts.max_workers) + + global failed + global built_pkgs + + cfg = opts.chroot + pkgs = args[1:] + + # transform slow/big package options into dictionaries + for line in opts.slow_pkg_names_raw: + speed,name = line.split(":") + if speed != "": + slow_pkg_names[name]=int(speed) + for line in opts.slow_pkgs_raw: + speed,pkg = line.split(":") + if speed != "": + slow_pkgs[pkg]=int(speed) + for line in opts.big_pkg_names_raw: + size_gb,name = line.split(":") + if size_gb != "": + big_pkg_names[name]=int(size_gb) + for line in opts.big_pkgs_raw: + size_gb,pkg = line.split(":") + if size_gb != "": + big_pkgs[pkg]=int(size_gb) + + # Set up a mapping between pkg path and pkg name + global pkg_to_name + global name_to_pkg + for pkg in pkgs: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + continue + + try: + name = rpmName(pkg) + except OSError as e: + print("Could not parse rpm %s" % pkg) + sys.exit(1) + + pkg_to_name[pkg] = name + name_to_pkg[name] = pkg + + read_deps(opts) + + global config_opts + config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError as e: + print("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + if opts.basedir != "/var/lib/mock": + opts.uniqueext = '' + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + os.chmod(local_tmp_dir, 0o755) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + os.chmod(local_tmp_dir, 0o755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0o755) + + local_baseurl = "file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0o755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) + + # modify with localrepo + res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + # createrepo on it + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + sys.exit(1) + + init_build_env(max_workers, opts, config_opts) + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + return_code = 0 + num_of_tries = 0 + + g_opts = opts + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + signal.signal(signal.SIGABRT, signal_handler) + + while try_again and not stop_signal: + num_of_tries += 1 + failed = [] + + log(opts.logfile, "===== iteration %d start =====" % num_of_tries) + + to_be_built_scheduled = to_be_built[:] + + need_reap = False + while len(to_be_built_scheduled) > 0: + # Free up a worker + while need_reap or workers >= max_workers: + need_reap = False + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + + if workers < max_workers: + workers = workers + 1 + + b = get_idle_build_env(max_workers) + if b < 0: + log(opts.logfile, "Failed to find idle build env for: %s" % pkg) + workers = workers - 1 + need_reap = True + continue + + pkg = schedule(b, to_be_built_scheduled, opts) + if pkg is None: + if workers <= 1: + # Remember we have one build environmnet reserved, so can't test for zero workers + log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) + pkg = to_be_built_scheduled[0] + log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) + else: + release_build_env(b) + workers = workers - 1 + need_reap = True + continue + + to_be_built_scheduled.remove(pkg) + + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + release_build_env(b) + need_reap = True + continue + + elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): + url = pkg + try: + log(opts.logfile, 'Fetching %s' % url) + r = requests.get(url) + # pylint: disable=no-member + if r.status_code == requests.codes.ok: + fn = urlsplit(r.url).path.rsplit('/', 1)[1] + if 'content-disposition' in r.headers: + _, params = cgi.parse_header(r.headers['content-disposition']) + if 'filename' in params and params['filename']: + fn = params['filename'] + pkg = download_dir + '/' + fn + with open(pkg, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + except Exception as e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + release_build_env(b) + need_reap = True + continue + else: + downloaded_pkgs[pkg] = url + + log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) + # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] + p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) + worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) + p.start() + + # Wait for remaining processes to complete + log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) + while workers > 0: + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) + + if failed and opts.recurse: + log(opts.logfile, "failed=%s" % failed) + log(opts.logfile, "to_be_built=%s" % to_be_built) + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package succeeded, some failed.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) + else: + if max_workers > 1: + max_workers = 1 + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package failed under parallel build.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) + else: + log(opts.logfile, "") + log(opts.logfile, "*** Build Failed ***") + log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) + log(opts.logfile, "*** Build Failed ***") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + log(opts.logfile, "") + try_again = False + else: + try_again = False + if failed: + return_code = 2 + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "") + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "") + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log(opts.logfile, "Some packages successfully built in this order:") + else: + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build-tools/modify-build-cfg b/build-tools/modify-build-cfg index b639f521..78658623 100755 --- a/build-tools/modify-build-cfg +++ b/build-tools/modify-build-cfg @@ -1,5 +1,11 @@ #!/bin/sh +# +# Copyright (c) 2018-2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + # This script modifies a mock configuration file (typically $MY_BUILD_CFG) # to add build time environment variables to the mock environment (things # like what branch we're building on, etc). @@ -11,6 +17,11 @@ # usage: modify-build-cfg [file.cfg] [] # +MODIFY_BUILD_CFG_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +# Set PKG_MANAGER for our build environment. +source "${MODIFY_BUILD_CFG_DIR}/pkg-manager-utils.sh" + LAYER=${2:-$LAYER} # For backward compatibility. Old repo location or new? @@ -23,6 +34,13 @@ if [ ! -d ${CENTOS_REPO} ]; then fi fi +# Preferred python +if rpm -q --whatprovides --quiet python3; then + PYTHON_PKG=python3 +else + PYTHON_PKG=python2 +fi + # Try to find a layer specific mock.cfg.proto MOCK_CFG_PROTO="${CENTOS_REPO}/mock.cfg.${LAYER}.proto" echo "==== Try MOCK_CFG_PROTO=$MOCK_CFG_PROTO ====" @@ -98,7 +116,7 @@ grep -q "config_opts\['environment'\]\['WRS_GIT_BRANCH'\]" $FILE || \ grep -q "config_opts\['environment'\]\['CGCS_GIT_BRANCH'\]" $FILE || \ echo "config_opts['environment']['CGCS_GIT_BRANCH']" >> $FILE -if [ -z $FORMAL_BUILD ]; then +if [ -z $FORMAL_BUILD ]; then grep -q "config_opts\['macros'\]\['%_no_cgcs_license_check'\] = '1'" $FILE || \ echo "config_opts['macros']['%_no_cgcs_license_check'] = '1'" >> $FILE else @@ -118,8 +136,21 @@ if [ "containers" == "$BUILD_TYPE" ]; then NETWORK_PKGS="bind-utils" fi -grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" $FILE || \ - echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" >> $FILE +BUILD_PKGS='' +if [ "${PKG_MANAGER}" == "yum" ]; then + BUILD_PKGS='@buildsys-build' +elif [ "${PKG_MANAGER}" == "dnf" ]; then + # buildsys-build group was dropped when Centos-8 switched to dnf. + # We must list all the members plus a few new ones (fedpkg-minimal, epel-rpm-macros). + BUILD_PKGS='bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +fi + +STX_PKGS='pigz lbzip2 bash' + +PKGS="${BUILD_PKGS} ${STX_PKGS} ${PKG_MANAGER} ${PYTHON_PKG} ${NETWORK_PKGS}" + +grep -q "config_opts\['chroot_setup_cmd'\] = 'install ${PKGS}'" $FILE || \ + echo "config_opts['chroot_setup_cmd'] = 'install ${PKGS}'" >> $FILE # Special case for containers. # rpmbuild_networking is required for invoking helm commands within mock diff --git a/build-tools/pkg-manager-utils.sh b/build-tools/pkg-manager-utils.sh new file mode 100755 index 00000000..17f4abc1 --- /dev/null +++ b/build-tools/pkg-manager-utils.sh @@ -0,0 +1,33 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# BASH utilities to select package manager +# +# Currently just sets some environment variables +# + +# Yum vs DNF compatibility +YUM=$(which yum 2>> /dev/null) +DNF=$(which dnf 2>> /dev/null) +PKG_MANAGER="" +REPOQUERY=$(which repoquery 2>> /dev/null) +REPOQUERY_SUB_COMMAND="" +REPOQUERY_RESOLVE="--resolve" +REPOQUERY_WHATPROVIDES_DELIM=" " +if [ ! -z ${DNF} ]; then + PKG_MANAGER="dnf" + REPOQUERY=${DNF} + REPOQUERY_SUB_COMMAND="repoquery --disable-modular-filtering" + REPOQUERY_RESOLVE="" + REPOQUERY_WHATPROVIDES_DELIM="," +elif [ ! -z ${YUM} ]; then + PKG_MANAGER="yum" +else + >&2 echo "ERROR: Couldn't find a supported package manager" + exit 1 +fi + diff --git a/build-tools/repo_files/mock.cfg.all.proto b/build-tools/repo_files/mock.cfg.all.proto deleted file mode 100644 index 6ecaa2d2..00000000 --- a/build-tools/repo_files/mock.cfg.all.proto +++ /dev/null @@ -1,58 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority -exclude=kernel-devel libvirt-devel - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.all.proto b/build-tools/repo_files/mock.cfg.all.proto new file mode 120000 index 00000000..2ba14cf5 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.all.proto @@ -0,0 +1 @@ +mock.cfg.centos7.all.proto \ No newline at end of file diff --git a/build-tools/repo_files/mock.cfg.centos7.all.proto b/build-tools/repo_files/mock.cfg.centos7.all.proto new file mode 100644 index 00000000..6ecaa2d2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.all.proto @@ -0,0 +1,58 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos7.distro.proto b/build-tools/repo_files/mock.cfg.centos7.distro.proto new file mode 100644 index 00000000..6ecaa2d2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.distro.proto @@ -0,0 +1,58 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos7.proto b/build-tools/repo_files/mock.cfg.centos7.proto new file mode 100644 index 00000000..beeea32f --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos7.proto @@ -0,0 +1,57 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['releasever'] = '7' +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos7Distro] +name=Stx-Centos-7-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority + +[StxCentos7Distro-rt] +name=Stx-Centos-7-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.all.proto b/build-tools/repo_files/mock.cfg.centos8.all.proto new file mode 100644 index 00000000..c5bb65da --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.all.proto @@ -0,0 +1,63 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.distro.proto b/build-tools/repo_files/mock.cfg.centos8.distro.proto new file mode 100644 index 00000000..c5bb65da --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.distro.proto @@ -0,0 +1,63 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +exclude=kernel-devel libvirt-devel +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.centos8.proto b/build-tools/repo_files/mock.cfg.centos8.proto new file mode 100644 index 00000000..08a041b2 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.centos8.proto @@ -0,0 +1,62 @@ +config_opts['root'] = 'BUILD_ENV/mock' +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['chroot_setup_cmd'] = 'install bash bzip2 coreutils cpio diffutils epel-release epel-rpm-macros fedpkg-minimal findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config redhat-release rpm-build sed shadow-utils tar unzip util-linux which xz' +config_opts['dist'] = 'el8' # only useful for --resultdir variable subst +config_opts['releasever'] = '8' +config_opts['package_manager'] = 'dnf' +config_opts['use_bootstrap'] = False +config_opts['use_bootstrap_image'] = False +config_opts['rpmbuild_networking'] = False + + +config_opts['yum.conf'] = """ +[main] +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# repos +[local-std] +name=local-std +baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-rt] +name=local-rt +baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[local-installer] +name=local-installer +baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 + +[StxCentos8Distro] +name=Stx-Centos-8-Distro +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary +failovermethod=priority +module_hotfixes=1 + +[StxCentos8Distro-rt] +name=Stx-Centos-8-Distro-rt +enabled=1 +baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary +failovermethod=priority +module_hotfixes=1 + +""" diff --git a/build-tools/repo_files/mock.cfg.distro.proto b/build-tools/repo_files/mock.cfg.distro.proto deleted file mode 100644 index 6ecaa2d2..00000000 --- a/build-tools/repo_files/mock.cfg.distro.proto +++ /dev/null @@ -1,58 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority -exclude=kernel-devel libvirt-devel - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.distro.proto b/build-tools/repo_files/mock.cfg.distro.proto new file mode 120000 index 00000000..add71c08 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.distro.proto @@ -0,0 +1 @@ +mock.cfg.centos7.distro.proto \ No newline at end of file diff --git a/build-tools/repo_files/mock.cfg.proto b/build-tools/repo_files/mock.cfg.proto deleted file mode 100644 index beeea32f..00000000 --- a/build-tools/repo_files/mock.cfg.proto +++ /dev/null @@ -1,57 +0,0 @@ -config_opts['root'] = 'BUILD_ENV/mock' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) -config_opts['chroot_setup_cmd'] = 'install @buildsys-build' -config_opts['dist'] = 'el7' # only useful for --resultdir variable subst -config_opts['releasever'] = '7' -config_opts['rpmbuild_networking'] = False - - -config_opts['yum.conf'] = """ -[main] -keepcache=1 -debuglevel=2 -reposdir=/dev/null -logfile=/var/log/yum.log -retries=20 -obsoletes=1 -gpgcheck=0 -assumeyes=1 -syslog_ident=mock -syslog_device= - -# repos -[local-std] -name=local-std -baseurl=LOCAL_BASE/MY_BUILD_DIR/std/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-rt] -name=local-rt -baseurl=LOCAL_BASE/MY_BUILD_DIR/rt/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[local-installer] -name=local-installer -baseurl=LOCAL_BASE/MY_BUILD_DIR/installer/rpmbuild/RPMS -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 - -[StxCentos7Distro] -name=Stx-Centos-7-Distro -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/Binary -failovermethod=priority - -[StxCentos7Distro-rt] -name=Stx-Centos-7-Distro-rt -enabled=1 -baseurl=LOCAL_BASE/MY_REPO_DIR/centos-repo/rt/Binary -failovermethod=priority - -""" diff --git a/build-tools/repo_files/mock.cfg.proto b/build-tools/repo_files/mock.cfg.proto new file mode 120000 index 00000000..55c2e026 --- /dev/null +++ b/build-tools/repo_files/mock.cfg.proto @@ -0,0 +1 @@ +mock.cfg.centos7.proto \ No newline at end of file