diff --git a/.zuul.yaml b/.zuul.yaml index 45ed5e42..dfcb7f1f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,5 +1,7 @@ - project: check: jobs: - - openstack-tox-linters: - voting: false + - openstack-tox-linters + gate: + jobs: + - openstack-tox-linters diff --git a/build-tools/branching/branch_and_tag.sh b/build-tools/branching/branch_and_tag.sh index cfe10449..dd9e3077 100755 --- a/build-tools/branching/branch_and_tag.sh +++ b/build-tools/branching/branch_and_tag.sh @@ -10,30 +10,30 @@ NEW_TAG=vCGCS_DEV_0019 OLD_BRANCH=CGCS_DEV_0018 NEW_BRANCH=CGCS_DEV_0019 -if [ -z "$MY_REPO" ]; then - echo "MY_REPO is unset" - exit 1 +if [ -z "$MY_REPO" ]; then + echo "MY_REPO is unset" + exit 1 else - echo "MY_REPO is set to '$MY_REPO'" + echo "MY_REPO is set to '$MY_REPO'" fi -if [ -d "$MY_REPO" ]; then - cd $MY_REPO - echo "checking out and pulling old branch" - wrgit checkout $OLD_BRANCH - if [ $? -ne 0 ]; then - echo "ERROR: wrgit checkout $OLD_BRANCH" - exit 1 - fi +if [ -d "$MY_REPO" ]; then + cd $MY_REPO + echo "checking out and pulling old branch" + wrgit checkout $OLD_BRANCH + if [ $? -ne 0 ]; then + echo "ERROR: wrgit checkout $OLD_BRANCH" + exit 1 + fi - wrgit pull - if [ $? -ne 0 ]; then - echo "ERROR: wrgit pull" - exit 1 - fi + wrgit pull + if [ $? -ne 0 ]; then + echo "ERROR: wrgit pull" + exit 1 + fi else - echo "Could not change to diectory '$MY_REPO'" - exit 1 + echo "Could not change to diectory '$MY_REPO'" + exit 1 fi echo "Finding subgits" @@ -42,45 +42,35 @@ SUBGITS=`find . -type d -name ".git" | sed "s%/\.git$%%"` # Go through all subgits and create the NEW_BRANCH if it does not already exist # Go through all subgits and create the NEW_TAG if it does not already exist for subgit in $SUBGITS; do - echo "" - echo "" - pushd $subgit > /dev/null - git fetch - git fetch --tags - # check if destination branch already exists - echo "$subgit" - branch_check=`git branch -a --list $NEW_BRANCH` - if [ -z "$branch_check" ] - then - echo "Creating $NEW_BRANCH" - git checkout $OLD_BRANCH - git checkout -b $NEW_BRANCH - git push origin $NEW_BRANCH:$NEW_BRANCH - else - echo "$NEW_BRANCH already exists" - fi - tag_check=`git tag -l $NEW_TAG` - if [ -z "$tag_check" ] - then - echo "Creating $NEW_TAG" - # create tag - git checkout $NEW_BRANCH - git pull origin - git tag $NEW_TAG - git push origin $NEW_TAG - else - echo "$NEW_TAG already exists" - fi + echo "" + echo "" + pushd $subgit > /dev/null + git fetch + git fetch --tags + # check if destination branch already exists + echo "$subgit" + branch_check=`git branch -a --list $NEW_BRANCH` + if [ -z "$branch_check" ]; then + echo "Creating $NEW_BRANCH" + git checkout $OLD_BRANCH + git checkout -b $NEW_BRANCH + git push origin $NEW_BRANCH:$NEW_BRANCH + else + echo "$NEW_BRANCH already exists" + fi + tag_check=`git tag -l $NEW_TAG` + if [ -z "$tag_check" ]; then + echo "Creating $NEW_TAG" + # create tag + git checkout $NEW_BRANCH + git pull origin + git tag $NEW_TAG + git push origin $NEW_TAG + else + echo "$NEW_TAG already exists" + fi - popd > /dev/null + popd > /dev/null done echo "All done. branches and tags are pushed" - - - - - - - - diff --git a/build-tools/branching/create_branches_and_tags.sh b/build-tools/branching/create_branches_and_tags.sh index 8a34dcf8..01556387 100755 --- a/build-tools/branching/create_branches_and_tags.sh +++ b/build-tools/branching/create_branches_and_tags.sh @@ -14,43 +14,41 @@ SUBGITS=`find . -type d -name ".git" | sed "s%/\.git$%%"` # Go through all subgits and create the branch and tag if they does not already exist for subgit in $SUBGITS; do - echo "" - echo "" - pushd $subgit > /dev/null + echo "" + echo "" + pushd $subgit > /dev/null # check if destination branch already exists - echo "$subgit" - branch_check=`git branch -a --list $branch` - if [ -z "$branch_check" ] - then - echo "Creating branch $branch" - git checkout -b $branch - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git checkout -b $branch" - popd > /dev/null - exit 1 - fi + echo "$subgit" + branch_check=`git branch -a --list $branch` + if [ -z "$branch_check" ]; then + echo "Creating branch $branch" + git checkout -b $branch + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git checkout -b $branch" + popd > /dev/null + exit 1 + fi # git push origin $branch:$branch - else - echo "Branch $branch already exists" - git checkout $branch - fi + else + echo "Branch $branch already exists" + git checkout $branch + fi - tag_check=`git tag -l $tag` - if [ -z "$tag_check" ] - then - echo "Creating tag $tag" - git tag $tag - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git tag $tag" - popd > /dev/null - exit 1 - fi + tag_check=`git tag -l $tag` + if [ -z "$tag_check" ]; then + echo "Creating tag $tag" + git tag $tag + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git tag $tag" + popd > /dev/null + exit 1 + fi # git push origin $tag - else - echo "Tag $tag already exists" - fi - - popd > /dev/null + else + echo "Tag $tag already exists" + fi + + popd > /dev/null done diff --git a/build-tools/branching/create_tags.sh b/build-tools/branching/create_tags.sh index b68df8e8..0a16eac3 100755 --- a/build-tools/branching/create_tags.sh +++ b/build-tools/branching/create_tags.sh @@ -12,25 +12,24 @@ SUBGITS=`find . -type d -name ".git" | sed "s%/\.git$%%"` # Go through all subgits and create the tag if it does not already exist for subgit in $SUBGITS; do - echo "" - echo "" - pushd $subgit > /dev/null + echo "" + echo "" + pushd $subgit > /dev/null - tag_check=`git tag -l $tag` - if [ -z "$tag_check" ] - then - echo "Creating tag $tag" - git tag $tag - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git tag $tag" - popd > /dev/null - exit 1 - fi + tag_check=`git tag -l $tag` + if [ -z "$tag_check" ]; then + echo "Creating tag $tag" + git tag $tag + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git tag $tag" + popd > /dev/null + exit 1 + fi # git push origin $tag - else - echo "Tag $tag already exists" - fi - - popd > /dev/null + else + echo "Tag $tag already exists" + fi + + popd > /dev/null done diff --git a/build-tools/branching/push_branches_tags.sh b/build-tools/branching/push_branches_tags.sh index 480b4751..82a7914f 100755 --- a/build-tools/branching/push_branches_tags.sh +++ b/build-tools/branching/push_branches_tags.sh @@ -15,28 +15,28 @@ SUBGITS=`find . -type d -name ".git" | sed "s%/\.git$%%"` # Go through all subgits and create the branch and tag if they does not already exist for subgit in $SUBGITS; do - echo "" - echo "" - pushd $subgit > /dev/null + echo "" + echo "" + pushd $subgit > /dev/null # check if destination branch already exists - echo "$subgit" - echo "Pushing branch $branch" - git push origin $branch:$branch - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git push origin $branch:$branch" - popd > /dev/null - exit 1 - fi + echo "$subgit" + echo "Pushing branch $branch" + git push origin $branch:$branch + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git push origin $branch:$branch" + popd > /dev/null + exit 1 + fi - echo "Pushing tag $tag" - git push origin $tag - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git push origin $tag" - popd > /dev/null - exit 1 - fi - - popd > /dev/null + echo "Pushing tag $tag" + git push origin $tag + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git push origin $tag" + popd > /dev/null + exit 1 + fi + + popd > /dev/null done diff --git a/build-tools/branching/push_tags.sh b/build-tools/branching/push_tags.sh index b32c00ae..8f1bf801 100755 --- a/build-tools/branching/push_tags.sh +++ b/build-tools/branching/push_tags.sh @@ -12,18 +12,18 @@ SUBGITS=`find . -type d -name ".git" | sed "s%/\.git$%%"` # Go through all subgits and create the tag if it does not already exist for subgit in $SUBGITS; do - echo "" - echo "" - pushd $subgit > /dev/null + echo "" + echo "" + pushd $subgit > /dev/null - echo "Creating tag $tag" - git push origin $tag - if [ $? != 0 ] ; then - echo "ERROR: Could not exec: git push origin $tag" - popd > /dev/null - exit 1 - fi - - popd > /dev/null + echo "Creating tag $tag" + git push origin $tag + if [ $? != 0 ] ; then + echo "ERROR: Could not exec: git push origin $tag" + popd > /dev/null + exit 1 + fi + + popd > /dev/null done diff --git a/build-tools/build_guest/rootfs-setup.sh b/build-tools/build_guest/rootfs-setup.sh index 1188244a..d46cac29 100755 --- a/build-tools/build_guest/rootfs-setup.sh +++ b/build-tools/build_guest/rootfs-setup.sh @@ -2,10 +2,10 @@ BUILD_MODE='' if [ "$1" == "--rt" ]; then - BUILD_MODE="rt" + BUILD_MODE="rt" fi if [ "$1" == "--std" ]; then - BUILD_MODE="std" + BUILD_MODE="std" fi # Setup boot directory for syslinux configuration (/boot/extlinux.conf) @@ -53,11 +53,11 @@ echo "touch /var/run/.init-complete" >> /etc/rc.local if [ "$BUILD_MODE" == "rt" ]; then # Adjust system tuning knobs during init when using rt kernel (CGTS-7047) - echo "echo 1 > /sys/devices/virtual/workqueue/cpumask" >> /etc/rc.local - echo "echo 1 > /sys/bus/workqueue/devices/writeback/cpumask" >> /etc/rc.local - echo "echo -1 > /proc/sys/kernel/sched_rt_runtime_us" >> /etc/rc.local - echo "echo 0 > /proc/sys/kernel/timer_migration" >> /etc/rc.local - echo "echo 10 > /proc/sys/vm/stat_interval" >> /etc/rc.local + echo "echo 1 > /sys/devices/virtual/workqueue/cpumask" >> /etc/rc.local + echo "echo 1 > /sys/bus/workqueue/devices/writeback/cpumask" >> /etc/rc.local + echo "echo -1 > /proc/sys/kernel/sched_rt_runtime_us" >> /etc/rc.local + echo "echo 0 > /proc/sys/kernel/timer_migration" >> /etc/rc.local + echo "echo 10 > /proc/sys/vm/stat_interval" >> /etc/rc.local fi # Disable audit service by default @@ -68,8 +68,8 @@ systemctl disable auditd if [ "$BUILD_MODE" == "rt" ]; then # Additional services to disable on rt guest (CGTS-7047) - systemctl disable polkit.service - systemctl disable tuned.service + systemctl disable polkit.service + systemctl disable tuned.service fi # Clean the yum cache. We don't want to maintain it on the guest file system. @@ -80,9 +80,9 @@ sed -i 's#OmitLocalLogging on#OmitLocalLogging off#g' /etc/rsyslog.conf # select correct kernel and initrd if [ "$BUILD_MODE" == "rt" ]; then - PATTERN=$(rpm -q --qf '%{VERSION}-%{RELEASE}' kernel-rt) + PATTERN=$(rpm -q --qf '%{VERSION}-%{RELEASE}' kernel-rt) else - PATTERN=$(rpm -q --qf '%{VERSION}-%{RELEASE}' kernel) + PATTERN=$(rpm -q --qf '%{VERSION}-%{RELEASE}' kernel) fi cd /boot rm -f vmlinuz initramfs.img diff --git a/build-tools/build_iso/cgts_deps.sh b/build-tools/build_iso/cgts_deps.sh index b39014a0..62b52770 100755 --- a/build-tools/build_iso/cgts_deps.sh +++ b/build-tools/build_iso/cgts_deps.sh @@ -28,16 +28,16 @@ # This function generates a simple file of dependencies we're trying to resolve function generate_dep_list { - TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) - mkdir -p $TMP_RPM_DB - rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE - \rm -rf $TMP_RPM_DB + TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) + mkdir -p $TMP_RPM_DB + rpm --initdb --dbpath $TMP_RPM_DB + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ + | grep -v "error:" \ + | grep -v "warning:" \ + | grep -v "Preparing..." \ + | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE + \rm -rf $TMP_RPM_DB } # Takes a list of requirements (either explcit package name, or capabilities @@ -53,166 +53,162 @@ function generate_dep_list { # install the "xyz" package, rather than "something-devel" which has "xyz" # capabilities. function install_deps { - local DEP_LIST="" - local DEP_LIST_FILE="$1" + local DEP_LIST="" + local DEP_LIST_FILE="$1" - # Temporary files are used in a few different ways - # Here we essenitally create variable aliases to make it easier to read - # the script - local UNSORTED_PACKAGES=$TMPFILE - local SORTED_PACKAGES=$TMPFILE1 - local UNRESOLVED_PACKAGES=$TMPFILE2 + # Temporary files are used in a few different ways + # Here we essenitally create variable aliases to make it easier to read + # the script + local UNSORTED_PACKAGES=$TMPFILE + local SORTED_PACKAGES=$TMPFILE1 + local UNRESOLVED_PACKAGES=$TMPFILE2 - rm -f $UNSORTED_PACKAGES + rm -f $UNSORTED_PACKAGES - while read DEP - do - DEP_LIST="${DEP_LIST} ${DEP}" - done < $DEP_LIST_FILE + while read DEP + do + DEP_LIST="${DEP_LIST} ${DEP}" + done < $DEP_LIST_FILE - echo "Debug: List of deps to resolve: ${DEP_LIST}" + echo "Debug: List of deps to resolve: ${DEP_LIST}" - if [ -z "${DEP_LIST}" ] - then - return 0 - fi + if [ -z "${DEP_LIST}" ]; then + return 0 + fi - # go through each repo and convert deps to packages based on package name - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + # go through each repo and convert deps to packages based on package name + for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do + echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch ${DEP_LIST} --qf='%{name}'" + TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES + \rm -rf $TMP_DIR/yum-$USER-* + done + sort $UNSORTED_PACKAGES -u > $SORTED_PACKAGES + + # figure out any dependancies which could not be resolved based on + # package name. We use --whatpovides to deal with this + # + # First, we build a new DEP_LIST based on what was NOT found in + # search-by-name attempt + sort $DEP_LIST_FILE -u > $TMPFILE + comm -2 -3 $TMPFILE $SORTED_PACKAGES > $UNRESOLVED_PACKAGES + + # If there are any requirements not resolved, look up the packages with + # --whatprovides + if [ -s $UNRESOLVED_PACKAGES ]; then + DEP_LIST="" + \cp $SORTED_PACKAGES $UNSORTED_PACKAGES + while read DEP + do + DEP_LIST="${DEP_LIST} ${DEP}" + done < $UNRESOLVED_PACKAGES + + DEP_LIST=$(echo "$DEP_LIST" | sed 's/^ //g') + if [ "$DEP_LIST" != "" ]; then + + for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do + echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" + TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES \rm -rf $TMP_DIR/yum-$USER-* - done - sort $UNSORTED_PACKAGES -u > $SORTED_PACKAGES + done + fi - # figure out any dependancies which could not be resolved based on - # package name. We use --whatpovides to deal with this - # - # First, we build a new DEP_LIST based on what was NOT found in - # search-by-name attempt - sort $DEP_LIST_FILE -u > $TMPFILE - comm -2 -3 $TMPFILE $SORTED_PACKAGES > $UNRESOLVED_PACKAGES + sort -u $UNSORTED_PACKAGES > $SORTED_PACKAGES + fi - # If there are any requirements not resolved, look up the packages with - # --whatprovides - if [ -s $UNRESOLVED_PACKAGES ]; then - DEP_LIST="" - \cp $SORTED_PACKAGES $UNSORTED_PACKAGES - while read DEP - do - DEP_LIST="${DEP_LIST} ${DEP}" - done < $UNRESOLVED_PACKAGES + # clean up + \rm -f $UNSORTED_PACKAGES $UNRESOLVED_PACKAGES - DEP_LIST=$(echo "$DEP_LIST" | sed 's/^ //g') - if [ "$DEP_LIST" != "" ]; then + # We now have, in SORTED_PACKAGES, a list of all packages that we need to install + # to meet our dependancies + DEP_LIST=" " + while read DEP + do + DEP_LIST="${DEP_LIST}${DEP} " + done < $SORTED_PACKAGES + rm $SORTED_PACKAGES - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $UNSORTED_PACKAGES - \rm -rf $TMP_DIR/yum-$USER-* - done + # go through each repo and install packages + local TARGETS=${DEP_LIST} + echo "Debug: Resolved list of deps to install: ${TARGETS}" + local UNRESOLVED + for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do + UNRESOLVED="$TARGETS" + + if [[ ! -z "${TARGETS// }" ]]; then + REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') + >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" + TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V > $TMPFILE + \rm -rf $TMP_DIR/yum-$USER-* + + while read STR + do + >&2 echo "STR=$STR" + if [ "x$STR" == "x" ]; then + continue fi - sort -u $UNSORTED_PACKAGES > $SORTED_PACKAGES - fi + PKG=`echo $STR | cut -d " " -f 1` + PKG_FILE=`echo $STR | cut -d " " -f 2` + PKG_REL_PATH=`echo $STR | cut -d " " -f 3` + PKG_PATH="${REPO_PATH}/${PKG_REL_PATH}" - # clean up - \rm -f $UNSORTED_PACKAGES $UNRESOLVED_PACKAGES + >&2 echo "Installing PKG=$PKG PKG_FILE=$PKG_FILE PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH from repo $REPOID" + cp $PKG_PATH . + if [ $? -ne 0 ]; then + >&2 echo " Here's what I have to work with..." + >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" + fi - # We now have, in SORTED_PACKAGES, a list of all packages that we need to install - # to meet our dependancies - DEP_LIST=" " - while read DEP - do - DEP_LIST="${DEP_LIST}${DEP} " - done < $SORTED_PACKAGES - rm $SORTED_PACKAGES - - # go through each repo and install packages - local TARGETS=${DEP_LIST} - echo "Debug: Resolved list of deps to install: ${TARGETS}" - local UNRESOLVED - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - UNRESOLVED="$TARGETS" - - if [[ ! -z "${TARGETS// }" ]]; then - REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V > $TMPFILE + echo $UNRESOLVED | grep $PKG >> /dev/null + if [ $? -eq 0 ]; then + echo "$PKG found in $REPOID as $PKG" >> $BUILT_REPORT + echo "$PKG_PATH" >> $BUILT_REPORT + UNRESOLVED=$(echo "$UNRESOLVED" | sed "s# $PKG # #g") + else + echo "$PKG satisfies unknown target in $REPOID" >> $BUILT_REPORT + echo " but it doesn't match targets, $UNRESOLVED" >> $BUILT_REPORT + echo " path $PKG_PATH" >> $BUILT_REPORT + FOUND_UNKNOWN=1 + fi + done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" \rm -rf $TMP_DIR/yum-$USER-* - - while read STR - do - >&2 echo "STR=$STR" - if [ "x$STR" == "x" ]; then - continue - fi - - PKG=`echo $STR | cut -d " " -f 1` - PKG_FILE=`echo $STR | cut -d " " -f 2` - PKG_REL_PATH=`echo $STR | cut -d " " -f 3` - PKG_PATH="${REPO_PATH}/${PKG_REL_PATH}" - - >&2 echo "Installing PKG=$PKG PKG_FILE=$PKG_FILE PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH from repo $REPOID" - cp $PKG_PATH . - if [ $? -ne 0 ] - then - >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" - >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" - fi - - echo $UNRESOLVED | grep $PKG >> /dev/null - if [ $? -eq 0 ]; then - echo "$PKG found in $REPOID as $PKG" >> $BUILT_REPORT - echo "$PKG_PATH" >> $BUILT_REPORT - UNRESOLVED=$(echo "$UNRESOLVED" | sed "s# $PKG # #g") - else - echo "$PKG satisfies unknown target in $REPOID" >> $BUILT_REPORT - echo " but it doesn't match targets, $UNRESOLVED" >> $BUILT_REPORT - echo " path $PKG_PATH" >> $BUILT_REPORT - FOUND_UNKNOWN=1 - fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" - \rm -rf $TMP_DIR/yum-$USER-* - TARGETS="$UNRESOLVED" - fi - done - >&2 echo "Debug: Packages still unresolved: $UNRESOLVED" - echo "Debug: Packages still unresolved: $UNRESOLVED" >> $WARNINGS_REPORT - echo "Debug: Packages still unresolved: $UNRESOLVED" >> $BUILT_REPORT - >&2 echo "" + TARGETS="$UNRESOLVED" + fi + done + >&2 echo "Debug: Packages still unresolved: $UNRESOLVED" + echo "Debug: Packages still unresolved: $UNRESOLVED" >> $WARNINGS_REPORT + echo "Debug: Packages still unresolved: $UNRESOLVED" >> $BUILT_REPORT + >&2 echo "" } -function check_all_explicit_deps_installed -{ +function check_all_explicit_deps_installed { - PKGS_TO_CHECK=" " - while read PKG_TO_ADD - do - PKGS_TO_CHECK="$PKGS_TO_CHECK ${PKG_TO_ADD}" - done < $DEPLISTFILE - rpm -qp $MY_WORKSPACE/export/dist/isolinux/Packages/*.rpm --qf="%{name}\n" --nosignature > $TMPFILE + PKGS_TO_CHECK=" " + while read PKG_TO_ADD + do + PKGS_TO_CHECK="$PKGS_TO_CHECK ${PKG_TO_ADD}" + done < $DEPLISTFILE + rpm -qp $MY_WORKSPACE/export/dist/isolinux/Packages/*.rpm --qf="%{name}\n" --nosignature > $TMPFILE - while read INSTALLED_PACKAGE - do - echo $PKGS_TO_CHECK | grep -q "${INSTALLED_PACKAGE}" - if [ $? -eq 0 ]; then - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE} //"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE} / /"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE}\$//"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE}\$//"` - fi - done < $TMPFILE + while read INSTALLED_PACKAGE + do + echo $PKGS_TO_CHECK | grep -q "${INSTALLED_PACKAGE}" + if [ $? -eq 0 ]; then + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE} //"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE} / /"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE}\$//"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE}\$//"` + fi + done < $TMPFILE - if [ -z "$PKGS_TO_CHECK" ] - then - >&2 echo "All explicitly specified packages resolved!" - else - >&2 echo "Could not resolve packages: $PKGS_TO_CHECK" - return 1 - fi - return 0 + if [ -z "$PKGS_TO_CHECK" ]; then + >&2 echo "All explicitly specified packages resolved!" + else + >&2 echo "Could not resolve packages: $PKGS_TO_CHECK" + return 1 + fi + return 0 } ATTEMPTED=0 @@ -258,39 +254,39 @@ fi # check that we resolved them all check_all_explicit_deps_installed if [ $? -ne 0 ]; then - >&2 echo "Error -- could not install all explicitly listed packages" - exit 1 + >&2 echo "Error -- could not install all explicitly listed packages" + exit 1 fi ALL_RESOLVED=0 while [ $ALL_RESOLVED -eq 0 ]; do - cp $DEPLISTFILE $DEPLISTFILE.old - generate_dep_list - if [ ! -s $DEPLISTFILE ]; then - # no more dependencies! - ALL_RESOLVED=1 - else - DIFFLINES=`diff $DEPLISTFILE.old $DEPLISTFILE | wc -l` - if [ $DIFFLINES -eq 0 ]; then - >&2 echo "Warning: Infinite loop detected in dependency resolution. See $DEPLISTFILE for details -- exiting" - >&2 echo "These RPMS had problems (likely version conflicts)" - >&2 cat $DEPLISTFILE + cp $DEPLISTFILE $DEPLISTFILE.old + generate_dep_list + if [ ! -s $DEPLISTFILE ]; then + # no more dependencies! + ALL_RESOLVED=1 + else + DIFFLINES=`diff $DEPLISTFILE.old $DEPLISTFILE | wc -l` + if [ $DIFFLINES -eq 0 ]; then + >&2 echo "Warning: Infinite loop detected in dependency resolution. See $DEPLISTFILE for details -- exiting" + >&2 echo "These RPMS had problems (likely version conflicts)" + >&2 cat $DEPLISTFILE - echo "Warning: Infinite loop detected in dependency resolution See $DEPLISTFILE for details -- exiting" >> $WARNINGS_REPORT - echo "These RPMS had problems (likely version conflicts)" >> $WARNINGS_REPORT - cat $DEPLISTFILE >> $WARNINGS_REPORT + echo "Warning: Infinite loop detected in dependency resolution See $DEPLISTFILE for details -- exiting" >> $WARNINGS_REPORT + echo "These RPMS had problems (likely version conflicts)" >> $WARNINGS_REPORT + cat $DEPLISTFILE >> $WARNINGS_REPORT - date > $LAST_TEST + date > $LAST_TEST - rm -f $DEPLISTFILE.old - exit 1 # nothing fixed - fi - install_deps $DEPLISTFILE - if [ $? -ne 0 ]; then - exit 1 - fi - fi + rm -f $DEPLISTFILE.old + exit 1 # nothing fixed + fi + install_deps $DEPLISTFILE + if [ $? -ne 0 ]; then + exit 1 + fi + fi done exit 0 diff --git a/build-tools/build_minimal_iso/build.sh b/build-tools/build_minimal_iso/build.sh index 5f857ec1..d7d58083 100755 --- a/build-tools/build_minimal_iso/build.sh +++ b/build-tools/build_minimal_iso/build.sh @@ -2,17 +2,17 @@ CREATEREPO=$(which createrepo_c) if [ $? -ne 0 ]; then - CREATEREPO="createrepo" + CREATEREPO="createrepo" fi # If a file listed in list.txt is missing, this function attempts to find the # RPM and copy it to the local directory. This should not be required normally # and is only used when collecting the source RPMs initially. function findSrc { - local lookingFor=$1 - find $MY_REPO/cgcs-centos-repo/Source -name $lookingFor | xargs -I '{}' cp '{}' . - find $MY_REPO/cgcs-tis-repo/Source -name $lookingFor | xargs -I '{}' cp '{}' . - find $MY_WORKSPACE/std/rpmbuild/SRPMS -name $lookingFor | xargs -I '{}' cp '{}' . + local lookingFor=$1 + find $MY_REPO/cgcs-centos-repo/Source -name $lookingFor | xargs -I '{}' cp '{}' . + find $MY_REPO/cgcs-tis-repo/Source -name $lookingFor | xargs -I '{}' cp '{}' . + find $MY_WORKSPACE/std/rpmbuild/SRPMS -name $lookingFor | xargs -I '{}' cp '{}' . } rm -f success.txt @@ -23,23 +23,23 @@ infile=list.txt while read p; do - if [ ! -f "$p" ]; then - findSrc $p - if [ ! -f "$p" ]; then - echo "couldn't find" >> missing.txt - echo "couldn't find $p" >> missing.txt - continue - fi - echo "found $p" - fi - - mock -r build.cfg $p --resultdir=results --no-clean - if [ $? -eq 0 ]; then - echo "$p" >> success.txt - cd results - $CREATEREPO . - cd .. - else - echo "$p" >> fail.txt - fi + if [ ! -f "$p" ]; then + findSrc $p + if [ ! -f "$p" ]; then + echo "couldn't find" >> missing.txt + echo "couldn't find $p" >> missing.txt + continue + fi + echo "found $p" + fi + + mock -r build.cfg $p --resultdir=results --no-clean + if [ $? -eq 0 ]; then + echo "$p" >> success.txt + cd results + $CREATEREPO . + cd .. + else + echo "$p" >> fail.txt + fi done < $infile diff --git a/build-tools/build_minimal_iso/build_centos.sh b/build-tools/build_minimal_iso/build_centos.sh index 1fbcb030..e56f6b98 100755 --- a/build-tools/build_minimal_iso/build_centos.sh +++ b/build-tools/build_minimal_iso/build_centos.sh @@ -4,42 +4,42 @@ CREATEREPO=$(which createrepo_c) if [ $? -ne 0 ]; then - CREATEREPO="createrepo" + CREATEREPO="createrepo" fi function final_touches { # create the repo - cd ${ROOTDIR}/${DEST}/isolinux - $CREATEREPO -g ../comps.xml . - + cd ${ROOTDIR}/${DEST}/isolinux + $CREATEREPO -g ../comps.xml . + # build the ISO - printf "Building image $OUTPUT_FILE\n" - cd ${ROOTDIR}/${DEST} - chmod 664 isolinux/isolinux.bin - mkisofs -o $OUTPUT_FILE \ - -R -D -A 'oe_iso_boot' -V 'oe_iso_boot' \ - -b isolinux.bin -c boot.cat -no-emul-boot \ - -boot-load-size 4 -boot-info-table \ - -eltorito-alt-boot \ - -e images/efiboot.img \ - -no-emul-boot \ - isolinux/ + printf "Building image $OUTPUT_FILE\n" + cd ${ROOTDIR}/${DEST} + chmod 664 isolinux/isolinux.bin + mkisofs -o $OUTPUT_FILE \ + -R -D -A 'oe_iso_boot' -V 'oe_iso_boot' \ + -b isolinux.bin -c boot.cat -no-emul-boot \ + -boot-load-size 4 -boot-info-table \ + -eltorito-alt-boot \ + -e images/efiboot.img \ + -no-emul-boot \ + isolinux/ - isohybrid --uefi $OUTPUT_FILE - implantisomd5 $OUTPUT_FILE + isohybrid --uefi $OUTPUT_FILE + implantisomd5 $OUTPUT_FILE - cd $ROOTDIR + cd $ROOTDIR } function setup_disk { - tar xJf emptyInstaller.tar.xz - mkdir ${DEST}/isolinux/Packages + tar xJf emptyInstaller.tar.xz + mkdir ${DEST}/isolinux/Packages } function install_packages { - cd ${DEST}/isolinux/Packages - ROOT=${ROOTDIR} ../../../cgts_deps.sh --deps=../../../${MINIMAL} - cd ${ROOTDIR} + cd ${DEST}/isolinux/Packages + ROOT=${ROOTDIR} ../../../cgts_deps.sh --deps=../../../${MINIMAL} + cd ${ROOTDIR} } diff --git a/build-tools/build_minimal_iso/cgts_deps.sh b/build-tools/build_minimal_iso/cgts_deps.sh index 57143b6c..062b74ff 100755 --- a/build-tools/build_minimal_iso/cgts_deps.sh +++ b/build-tools/build_minimal_iso/cgts_deps.sh @@ -1,144 +1,140 @@ #!/bin/env bash function generate_dep_list { - TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) - mkdir -p $TMP_RPM_DB - rpm --initdb --dbpath $TMP_RPM_DB - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 - rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ - | grep -v "error:" \ - | grep -v "warning:" \ - | grep -v "Preparing..." \ - | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE - rm -rf $TMP_RPM_DB + TMP_RPM_DB=$(mktemp -d $(pwd)/tmp_rpm_db_XXXXXX) + mkdir -p $TMP_RPM_DB + rpm --initdb --dbpath $TMP_RPM_DB + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' >> $DEPDETAILLISTFILE 2>&1 + rpm --dbpath $TMP_RPM_DB --test -Uvh --replacefiles '*.rpm' 2>&1 \ + | grep -v "error:" \ + | grep -v "warning:" \ + | grep -v "Preparing..." \ + | sed "s/ is needed by.*$//" | sed "s/ >=.*$//" | sort -u > $DEPLISTFILE + rm -rf $TMP_RPM_DB } function install_deps { - local DEP_LIST="" - local DEP_LIST_FILE="$1" + local DEP_LIST="" + local DEP_LIST_FILE="$1" - rm -f $TMPFILE + rm -f $TMPFILE - while read DEP - do - DEP_LIST="${DEP_LIST} ${DEP}" - done < $DEP_LIST_FILE + while read DEP + do + DEP_LIST="${DEP_LIST} ${DEP}" + done < $DEP_LIST_FILE - echo "Debug: List of deps to resolve: ${DEP_LIST}" + echo "Debug: List of deps to resolve: ${DEP_LIST}" - if [ -z "${DEP_LIST}" ] - then - return 0 - fi + if [ -z "${DEP_LIST}" ]; then + return 0 + fi - # go through each repo and convert deps to packages + # go through each repo and convert deps to packages - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $TMPFILE + for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do + echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --whatprovides ${DEP_LIST} --qf='%{name}'" + TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --qf='%{name}' --whatprovides ${DEP_LIST} | sed "s/kernel-debug/kernel/g" >> $TMPFILE \rm -rf $TMP_DIR/yum-$USER-* - done - sort $TMPFILE -u > $TMPFILE1 - rm $TMPFILE + done + sort $TMPFILE -u > $TMPFILE1 + rm $TMPFILE - DEP_LIST="" - while read DEP - do - DEP_LIST="${DEP_LIST} ${DEP}" - done < $TMPFILE1 - rm $TMPFILE1 + DEP_LIST="" + while read DEP + do + DEP_LIST="${DEP_LIST} ${DEP}" + done < $TMPFILE1 + rm $TMPFILE1 - # next go through each repo and install packages - local TARGETS=${DEP_LIST} - echo "Debug: Resolved list of deps to install: ${TARGETS}" - local UNRESOLVED - for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do - UNRESOLVED=" $TARGETS " - - if [[ ! -z "${TARGETS// }" ]]; then - REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') - >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" - TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V >> $TMPFILE + # next go through each repo and install packages + local TARGETS=${DEP_LIST} + echo "Debug: Resolved list of deps to install: ${TARGETS}" + local UNRESOLVED + for REPOID in `grep '^[[].*[]]$' $YUM | grep -v '[[]main[]]' | awk -F '[][]' '{print $2 }'`; do + UNRESOLVED=" $TARGETS " + + if [[ ! -z "${TARGETS// }" ]]; then + REPO_PATH=$(cat $YUM | sed -n "/^\[$REPOID\]\$/,\$p" | grep '^baseurl=' | head -n 1 | awk -F 'file://' '{print $2}' | sed 's:/$::') + >&2 echo "TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf='%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}'" + TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf="%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}" | sort -r -V >> $TMPFILE \rm -rf $TMP_DIR/yum-$USER-* - while read STR - do - >&2 echo "STR=$STR" - if [ "x$STR" == "x" ]; then - continue - fi + while read STR + do + >&2 echo "STR=$STR" + if [ "x$STR" == "x" ]; then + continue + fi - PKG=`echo $STR | cut -d " " -f 1` - PKG_FILE=`echo $STR | cut -d " " -f 2` - PKG_REL_PATH=`echo $STR | cut -d " " -f 3` - PKG_PATH="${REPO_PATH}/${PKG_REL_PATH}" + PKG=`echo $STR | cut -d " " -f 1` + PKG_FILE=`echo $STR | cut -d " " -f 2` + PKG_REL_PATH=`echo $STR | cut -d " " -f 3` + PKG_PATH="${REPO_PATH}/${PKG_REL_PATH}" - >&2 echo "Installing PKG=$PKG PKG_FILE=$PKG_FILE PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH from repo $REPOID" - cp $PKG_PATH . - if [ $? -ne 0 ] - then - >&2 echo " Here's what I have to work with..." - >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" - >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" - fi + >&2 echo "Installing PKG=$PKG PKG_FILE=$PKG_FILE PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH from repo $REPOID" + cp $PKG_PATH . + if [ $? -ne 0 ]; then + >&2 echo " Here's what I have to work with..." + >&2 echo " TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $PKG --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\"" + >&2 echo " PKG=$PKG PKG_FILE=$PKG_FILE REPO_PATH=$REPO_PATH PKG_REL_PATH=$PKG_REL_PATH PKG_PATH=$PKG_PATH" + fi - echo $UNRESOLVED | grep $PKG - echo $UNRESOLVED | grep $PKG >> /dev/null - if [ $? -eq 0 ]; then - echo "$PKG found in $REPOID as $PKG" >> $BUILT_REPORT - echo "$PKG_PATH" >> $BUILT_REPORT - UNRESOLVED=$(echo "$UNRESOLVED" | sed "s# $PKG # #g") - else - echo "$PKG satisfies unknown target in $REPOID" >> $BUILT_REPORT - echo " but it doesn't match targets, $UNRESOLVED" >> $BUILT_REPORT - echo " path $PKG_PATH" >> $BUILT_REPORT - FOUND_UNKNOWN=1 - fi - done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" + echo $UNRESOLVED | grep $PKG + echo $UNRESOLVED | grep $PKG >> /dev/null + if [ $? -eq 0 ]; then + echo "$PKG found in $REPOID as $PKG" >> $BUILT_REPORT + echo "$PKG_PATH" >> $BUILT_REPORT + UNRESOLVED=$(echo "$UNRESOLVED" | sed "s# $PKG # #g") + else + echo "$PKG satisfies unknown target in $REPOID" >> $BUILT_REPORT + echo " but it doesn't match targets, $UNRESOLVED" >> $BUILT_REPORT + echo " path $PKG_PATH" >> $BUILT_REPORT + FOUND_UNKNOWN=1 + fi + done < $TMPFILE #<<< "$(TMPDIR=$TMP_DIR repoquery -c $YUM --repoid=$REPOID --arch=x86_64,noarch --resolve $TARGETS --qf=\"%{name} %{name}-%{version}-%{release}.%{arch}.rpm %{relativepath}\" | sort -r -V)" \rm -rf $TMP_DIR/yum-$USER-* - TARGETS="$UNRESOLVED" - fi - done - >&2 echo "Debug: Packages still unresolved: $UNRESOLVED" - echo "Debug: Packages still unresolved: $UNRESOLVED" >> $WARNINGS_REPORT - echo "Debug: Packages still unresolved: $UNRESOLVED" >> $BUILT_REPORT - >&2 echo "" + TARGETS="$UNRESOLVED" + fi + done + >&2 echo "Debug: Packages still unresolved: $UNRESOLVED" + echo "Debug: Packages still unresolved: $UNRESOLVED" >> $WARNINGS_REPORT + echo "Debug: Packages still unresolved: $UNRESOLVED" >> $BUILT_REPORT + >&2 echo "" } -function check_all_explicit_deps_installed -{ +function check_all_explicit_deps_installed { - PKGS_TO_CHECK=" " - while read PKG_TO_ADD - do - PKGS_TO_CHECK="$PKGS_TO_CHECK ${PKG_TO_ADD}" - done < $DEPLISTFILE - rpm -qp ${INSTALLDIR}/*.rpm --qf="%{name}\n" > $TMPFILE - - echo "checking... $PKGS_TO_CHECK vs ${INSTALLED_PACKAGE}" + PKGS_TO_CHECK=" " + while read PKG_TO_ADD + do + PKGS_TO_CHECK="$PKGS_TO_CHECK ${PKG_TO_ADD}" + done < $DEPLISTFILE + rpm -qp ${INSTALLDIR}/*.rpm --qf="%{name}\n" > $TMPFILE - while read INSTALLED_PACKAGE - do - echo $PKGS_TO_CHECK | grep -q "${INSTALLED_PACKAGE}" - if [ $? -eq 0 ]; then - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE} //"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE} / /"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE}\$//"` - PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE}\$//"` - fi - done < $TMPFILE + echo "checking... $PKGS_TO_CHECK vs ${INSTALLED_PACKAGE}" - if [ -z "$PKGS_TO_CHECK" ] - then - >&2 echo "All explicitly specified packages resolved!" - else - >&2 echo "Could not resolve packages: $PKGS_TO_CHECK" - return 1 - fi - return 0 + while read INSTALLED_PACKAGE + do + echo $PKGS_TO_CHECK | grep -q "${INSTALLED_PACKAGE}" + if [ $? -eq 0 ]; then + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE} //"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE} / /"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/ ${INSTALLED_PACKAGE}\$//"` + PKGS_TO_CHECK=`echo $PKGS_TO_CHECK | sed "s/^${INSTALLED_PACKAGE}\$//"` + fi + done < $TMPFILE + + if [ -z "$PKGS_TO_CHECK" ]; then + >&2 echo "All explicitly specified packages resolved!" + else + >&2 echo "Could not resolve packages: $PKGS_TO_CHECK" + return 1 + fi + return 0 } if [ "x${ROOT}" == "x" ]; then - ROOT=/localdisk/loadbuild/centos + ROOT=/localdisk/loadbuild/centos fi ATTEMPTED=0 @@ -184,39 +180,39 @@ fi # check that we resolved them all check_all_explicit_deps_installed if [ $? -ne 0 ]; then - >&2 echo "Error -- could not install all explicitly listed packages" - exit 1 + >&2 echo "Error -- could not install all explicitly listed packages" + exit 1 fi ALL_RESOLVED=0 while [ $ALL_RESOLVED -eq 0 ]; do - cp $DEPLISTFILE $DEPLISTFILE.old - generate_dep_list - if [ ! -s $DEPLISTFILE ]; then - # no more dependencies! - ALL_RESOLVED=1 - else - DIFFLINES=`diff $DEPLISTFILE.old $DEPLISTFILE | wc -l` - if [ $DIFFLINES -eq 0 ]; then - >&2 echo "Warning: Infinite loop detected in dependency resolution. See $DEPLISTFILE for details -- exiting" - >&2 echo "These RPMS had problems (likely version conflicts)" - >&2 cat $DEPLISTFILE + cp $DEPLISTFILE $DEPLISTFILE.old + generate_dep_list + if [ ! -s $DEPLISTFILE ]; then + # no more dependencies! + ALL_RESOLVED=1 + else + DIFFLINES=`diff $DEPLISTFILE.old $DEPLISTFILE | wc -l` + if [ $DIFFLINES -eq 0 ]; then + >&2 echo "Warning: Infinite loop detected in dependency resolution. See $DEPLISTFILE for details -- exiting" + >&2 echo "These RPMS had problems (likely version conflicts)" + >&2 cat $DEPLISTFILE - echo "Warning: Infinite loop detected in dependency resolution See $DEPLISTFILE for details -- exiting" >> $WARNINGS_REPORT - echo "These RPMS had problems (likely version conflicts)" >> $WARNINGS_REPORT - cat $DEPLISTFILE >> $WARNINGS_REPORT + echo "Warning: Infinite loop detected in dependency resolution See $DEPLISTFILE for details -- exiting" >> $WARNINGS_REPORT + echo "These RPMS had problems (likely version conflicts)" >> $WARNINGS_REPORT + cat $DEPLISTFILE >> $WARNINGS_REPORT - date > $LAST_TEST + date > $LAST_TEST - rm -f $DEPLISTFILE.old - exit 1 # nothing fixed - fi - install_deps $DEPLISTFILE - if [ $? -ne 0 ]; then - exit 1 - fi - fi + rm -f $DEPLISTFILE.old + exit 1 # nothing fixed + fi + install_deps $DEPLISTFILE + if [ $? -ne 0 ]; then + exit 1 + fi + fi done exit 0 diff --git a/build-tools/git-utils.sh b/build-tools/git-utils.sh index 645f2b82..94490df1 100755 --- a/build-tools/git-utils.sh +++ b/build-tools/git-utils.sh @@ -27,9 +27,7 @@ export GIT_LIST=$(git_list "$(dirname "${MY_REPO}")") # GIT_LIST_REL: A list of root directories for all the gits under $MY_REPO/.. # as relative paths. -export GIT_LIST_REL=$(for p in $GIT_LIST; do - echo .${p#$(dirname ${MY_REPO})}; - done) +export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(dirname ${MY_REPO})}; done) # diff --git a/build-tools/make-installer-images.sh b/build-tools/make-installer-images.sh index 10e967cc..7307adce 100755 --- a/build-tools/make-installer-images.sh +++ b/build-tools/make-installer-images.sh @@ -2,25 +2,25 @@ ## this script is called by "update-pxe-network-installer" and run in "sudo" ## created by Yong Hu (yong.hu@intel.com), 05/24/2018 -function clean_rootfs() { +function clean_rootfs { rootfs_dir=$1 - echo "--> remove old files in original rootfs" + echo "--> remove old files in original rootfs" conf="$(ls ${rootfs_dir}/etc/ld.so.conf.d/kernel-*.conf)" echo "conf basename = $(basename $conf)" old_version="tbd" - if [ -f $conf ];then + if [ -f $conf ]; then old_version="$(echo $(basename $conf) | rev | cut -d'.' -f2- | rev | cut -d'-' -f2-)" fi echo "old version is $old_version" # remove old files in original initrd.img # do this in chroot to avoid accidentialy wrong operations on host root chroot $rootfs_dir /bin/bash -x < copy rootfs.img from original squashfs.img to LiveOS folder" cp -f ./squashfs.mnt/LiveOS/rootfs.img ./LiveOS/. @@ -230,7 +234,7 @@ umount $squashfs_root #rename the old version if [ -f $output_dir/new-squashfs.img ]; then mv -f $output_dir/new-squashfs.img $output_dir/squashfs.img-backup-$timestamp -fi +fi echo "--> make the new squashfs image" mksquashfs LiveOS $output_dir/new-squashfs.img -keep-as-directory -comp xz -b 1M diff --git a/build-tools/sync_jenkins.sh b/build-tools/sync_jenkins.sh index 7f303e93..dca9dc27 100755 --- a/build-tools/sync_jenkins.sh +++ b/build-tools/sync_jenkins.sh @@ -72,17 +72,17 @@ while true ; do done if [ "x$JENKINSURL" == "x" ]; then - JENKINSURL=$@ + JENKINSURL=$@ fi if [ $HELP -eq 1 ]; then - usage - exit 0 + usage + exit 0 fi if [ "x$JENKINSURL" == "x" ]; then - usage - exit 1 + usage + exit 1 fi mkdir -p $MY_WORKSPACE/export $MY_WORKSPACE/std/rpmbuild/RPMS $MY_WORKSPACE/std/rpmbuild/SRPMS $MY_WORKSPACE/rt/rpmbuild/RPMS $MY_WORKSPACE/rt/rpmbuild/SRPMS @@ -98,13 +98,13 @@ pushd $MY_REPO > /dev/null find . -type d -name ".git" | sed "s%/\.git$%%" > $TMPFILE while read hashfile; do - gitdir=`echo $hashfile | cut -d " " -f 1` - gitcommit=`echo $hashfile | sed s/.*[[:space:]]//g` - echo "doing dir $gitdir commit $gitcommit" - - pushd $gitdir >/dev/null - git checkout $gitcommit - popd + gitdir=`echo $hashfile | cut -d " " -f 1` + gitcommit=`echo $hashfile | sed s/.*[[:space:]]//g` + echo "doing dir $gitdir commit $gitcommit" + + pushd $gitdir >/dev/null + git checkout $gitcommit + popd done < $MY_WORKSPACE/$GITHASHFILE popd @@ -113,32 +113,32 @@ pushd $MY_WORKSPACE # clean stuff for build_type in std rt; do - rm -rf $MY_WORKSPACE/$build_type/rpmbuild/SRPMS - rm -rf $MY_WORKSPACE/$build_type/rpmbuild/RPMS - rm -rf $MY_WORKSPACE/$build_type/rpmbuild/inputs - rm -rf $MY_WORKSPACE/$build_type/rpmbuild/srpm_assemble + rm -rf $MY_WORKSPACE/$build_type/rpmbuild/SRPMS + rm -rf $MY_WORKSPACE/$build_type/rpmbuild/RPMS + rm -rf $MY_WORKSPACE/$build_type/rpmbuild/inputs + rm -rf $MY_WORKSPACE/$build_type/rpmbuild/srpm_assemble done # copy source rpms from jenkins for build_type in std rt; do - mkdir -p $MY_WORKSPACE/$build_type/rpmbuild/RPMS - mkdir -p $MY_WORKSPACE/$build_type/rpmbuild/SRPMS - rsync -r ${JENKINSURL}/$build_type/inputs $build_type/ - sleep 1 - rsync -r ${JENKINSURL}/$build_type/srpm_assemble $build_type/ - sleep 1 - rsync -r ${JENKINSURL}/$build_type/rpmbuild/SRPMS/* $MY_WORKSPACE/$build_type/rpmbuild/SRPMS - sleep 1 - for sub_repo in cgcs-centos-repo cgcs-tis-repo cgcs-3rd-party-repo; do - rsync ${JENKINSURL}/$build_type/$sub_repo.last_head $MY_WORKSPACE/$build_type - if [ "$build_type" == "std" ]; then - cp $MY_WORKSPACE/$build_type/$sub_repo.last_head $MY_REPO/$sub_repo/.last_head - fi - done - sleep 1 - rsync -r ${JENKINSURL}/$build_type/results $build_type/ - sleep 1 - rsync -r ${JENKINSURL}/$build_type/rpmbuild/RPMS/* $MY_WORKSPACE/$build_type/rpmbuild/RPMS + mkdir -p $MY_WORKSPACE/$build_type/rpmbuild/RPMS + mkdir -p $MY_WORKSPACE/$build_type/rpmbuild/SRPMS + rsync -r ${JENKINSURL}/$build_type/inputs $build_type/ + sleep 1 + rsync -r ${JENKINSURL}/$build_type/srpm_assemble $build_type/ + sleep 1 + rsync -r ${JENKINSURL}/$build_type/rpmbuild/SRPMS/* $MY_WORKSPACE/$build_type/rpmbuild/SRPMS + sleep 1 + for sub_repo in cgcs-centos-repo cgcs-tis-repo cgcs-3rd-party-repo; do + rsync ${JENKINSURL}/$build_type/$sub_repo.last_head $MY_WORKSPACE/$build_type + if [ "$build_type" == "std" ]; then + cp $MY_WORKSPACE/$build_type/$sub_repo.last_head $MY_REPO/$sub_repo/.last_head + fi + done + sleep 1 + rsync -r ${JENKINSURL}/$build_type/results $build_type/ + sleep 1 + rsync -r ${JENKINSURL}/$build_type/rpmbuild/RPMS/* $MY_WORKSPACE/$build_type/rpmbuild/RPMS done popd diff --git a/tox.ini b/tox.ini index 89e67e79..20ae72f5 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,9 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/test-requirements.txt [testenv:linters] +# ignore below cases +# E006 Line too long +# E010: Do not on same line as it commands whitelist_externals = bash commands = bash -c "find {toxinidir} \ @@ -20,7 +23,7 @@ commands = -not -name \*~ \ -not -name \*.md \ -name \*.sh \ - -print0 | xargs -0 bashate -v" + -print0 | xargs --no-run-if-empty -0 bashate -v -e 'E*' -i E006,E010" bash -c "find {toxinidir} \ \( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \ -o \( -name .tox -prune \) \