diff --git a/.gitignore b/.gitignore index cad81b6e..20552688 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ localrc toCOPY/.gitconfig centos-mirror-tools/logs/ centos-mirror-tools/output/ +centos-mirror-tools/mirror-check-failures.log # Sphinx documentation doc/build/ diff --git a/Dockerfile b/Dockerfile index d4cd3b25..7c1c6b20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,10 +60,8 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY* && \ VOLUME /run /tmp # Download required dependencies by mirror/build processes. -RUN groupadd -g 751 cgts && \ - echo "mock:x:751:root" >> /etc/group && \ - echo "mockbuild:x:9001:" >> /etc/group && \ - dnf install -y anaconda \ +RUN dnf install -y \ + anaconda \ anaconda-runtime \ autoconf-archive \ autogen \ @@ -90,7 +88,6 @@ RUN groupadd -g 751 cgts && \ lighttpd-fastcgi \ #lighttpd-mod_geoip \ mkisofs \ - mock \ #mongodb \ #mongodb-server \ net-tools \ @@ -126,21 +123,26 @@ RUN groupadd -g 751 cgts && \ RUN alternatives --set python /usr/bin/python3 RUN alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 -# This image requires a set of scripts and helpers -# for working correctly, in this section they are -# copied inside the image. -COPY toCOPY/finishSetup.sh /usr/local/bin -COPY toCOPY/populate_downloads.sh /usr/local/bin -COPY toCOPY/generate-local-repo.sh /usr/local/bin -COPY toCOPY/generate-centos-repo.sh /usr/local/bin -COPY toCOPY/lst_utils.sh /usr/local/bin -COPY toCOPY/.inputrc /home/$MYUNAME/ -COPY toCOPY/builder-constraints.txt /home/$MYUNAME/ +# Finally install a locked down version of mock +RUN groupadd -g 751 cgts && \ + echo "mock:x:751:root" >> /etc/group && \ + echo "mockbuild:x:9001:" >> /etc/group && \ + dnf install -y \ + http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/m/mock-2.8-1.el8.noarch.rpm \ + http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/m/mock-core-configs-33-1.el8.noarch.rpm + +# mock custumizations +# forcing chroots since a couple of packages naughtily insist on network access and +# we dont have nspawn and networks happy together. +RUN useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild && \ + rmdir /var/lib/mock && \ + ln -s /localdisk/loadbuild/mock /var/lib/mock && \ + rmdir /var/cache/mock && \ + ln -s /localdisk/loadbuild/mock-cache /var/cache/mock && \ + echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg && \ + echo "config_opts['rpmbuild_networking'] = True" >> /etc/mock/site-defaults.cfg && \ + echo >> /etc/mock/site-defaults.cfg -# Thes are included for backward compatibility, and -# should be removed after a reasonable time. -COPY toCOPY/generate-cgcs-tis-repo /usr/local/bin -COPY toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin # cpan modules, installing with cpanminus to avoid stupid questions since cpan is whack RUN cpanm --notest Fatal && \ @@ -149,10 +151,6 @@ RUN cpanm --notest Fatal && \ cpanm --notest XML::Parser && \ cpanm --notest XML::Simple -# pip installs -RUN pip3 install -c /home/$MYUNAME/builder-constraints.txt python-subunit junitxml --upgrade && \ - pip3 install -c /home/$MYUNAME/builder-constraints.txt tox --upgrade - # Install repo tool RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo && \ chmod a+x /usr/local/bin/repo @@ -164,17 +162,35 @@ RUN dnf install -y golang && \ mkdir -p ${GOPATH}/bin && \ curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -# mock time -# forcing chroots since a couple of packages naughtily insist on network access and -# we dont have nspawn and networks happy together. -RUN useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild && \ - rmdir /var/lib/mock && \ - ln -s /localdisk/loadbuild/mock /var/lib/mock && \ - rmdir /var/cache/mock && \ - ln -s /localdisk/loadbuild/mock-cache /var/cache/mock && \ - echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg && \ - echo "config_opts['rpmbuild_networking'] = True" >> /etc/mock/site-defaults.cfg && \ - echo >> /etc/mock/site-defaults.cfg +# Uprev git, repo +RUN yum install -y dh-autoreconf curl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel asciidoc xmlto docbook2X && \ + cd /tmp && \ + wget https://github.com/git/git/archive/v2.29.2.tar.gz -O git-2.29.2.tar.gz && \ + tar xzvf git-2.29.2.tar.gz && \ + cd git-2.29.2 && \ + make configure && \ + ./configure --prefix=/usr/local && \ + make all doc && \ + make install install-doc && \ + cd /tmp && \ + rm -rf git-2.29.2.tar.gz git-2.29.2 + +# Systemd Enablement +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /etc/systemd/system/*.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/* + +# pip installs +COPY toCOPY/builder-constraints.txt /home/$MYUNAME/ +RUN pip install -c /home/$MYUNAME/builder-constraints.txt pbr==5.6.0 --upgrade && \ + pip install -c /home/$MYUNAME/builder-constraints.txt git-review==2.1.0 --upgrade && \ + pip install -c /home/$MYUNAME/builder-constraints.txt python-subunit==1.4.0 junitxml==0.7 --upgrade && \ + pip install -c /home/$MYUNAME/builder-constraints.txt tox==3.23.0 --upgrade # Inherited tools for mock stuff # we at least need the mock_cache_unlock tool @@ -184,6 +200,21 @@ RUN cd /opt/mock_overlay && \ make && \ make install +# This image requires a set of scripts and helpers +# for working correctly, in this section they are +# copied inside the image. +COPY toCOPY/finishSetup.sh /usr/local/bin +COPY toCOPY/populate_downloads.sh /usr/local/bin +COPY toCOPY/generate-local-repo.sh /usr/local/bin +COPY toCOPY/generate-centos-repo.sh /usr/local/bin +COPY toCOPY/lst_utils.sh /usr/local/bin +COPY toCOPY/.inputrc /home/$MYUNAME/ + +# Thes are included for backward compatibility, and +# should be removed after a reasonable time. +COPY toCOPY/generate-cgcs-tis-repo /usr/local/bin +COPY toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin + # ENV setup RUN echo "# Load stx-builder configuration" >> /etc/profile.d/stx-builder-conf.sh && \ echo "if [[ -r \${HOME}/buildrc ]]; then" >> /etc/profile.d/stx-builder-conf.sh && \ @@ -235,30 +266,6 @@ RUN echo "$MYUNAME ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \ sed -i "s/dir-listing.activate/#dir-listing.activate/g" /etc/lighttpd/conf.d/dirlisting.conf && \ echo "dir-listing.activate = \"enable\"" >> /etc/lighttpd/conf.d/dirlisting.conf -# Uprev git, git-review, repo -RUN dnf install -y dh-autoreconf curl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel asciidoc xmlto docbook2X && \ - cd /tmp && \ - wget https://github.com/git/git/archive/v2.29.2.tar.gz -O git-2.29.2.tar.gz && \ - tar xzvf git-2.29.2.tar.gz && \ - cd git-2.29.2 && \ - make configure && \ - ./configure --prefix=/usr/local && \ - make all doc && \ - make install install-doc && \ - cd /tmp && \ - rm -rf git-2.29.2.tar.gz git-2.29.2 && \ - pip3 install git-review --upgrade - -# Systemd Enablement -RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ - rm -f /lib/systemd/system/multi-user.target.wants/*;\ - rm -f /etc/systemd/system/*.wants/*;\ - rm -f /lib/systemd/system/local-fs.target.wants/*; \ - rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ - rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ - rm -f /lib/systemd/system/basic.target.wants/*;\ - rm -f /lib/systemd/system/anaconda.target.wants/* - RUN useradd -r -u $MYUID -g cgts -m $MYUNAME && \ ln -s /home/$MYUNAME/.ssh /mySSH && \ rsync -av /etc/skel/ /home/$MYUNAME/ diff --git a/centos-mirror-tools/config/centos/mock/rpms_centos3rdparties.lst b/centos-mirror-tools/config/centos/mock/rpms_centos3rdparties.lst index 03a6d737..6022463f 100644 --- a/centos-mirror-tools/config/centos/mock/rpms_centos3rdparties.lst +++ b/centos-mirror-tools/config/centos/mock/rpms_centos3rdparties.lst @@ -15,4 +15,4 @@ nss-softokn-freebl-3.53.1-11.el8_2.x86_64.rpm systemd-239-18.el8_1.5.x86_64.rpm systemd-devel-239-18.el8_1.5.x86_64.rpm systemd-libs-239-18.el8_1.5.x86_64.rpm -util-linux-2.32.1-17.el8.x86_64.rpm \ No newline at end of file +util-linux-2.32.1-17.el8.x86_64.rpm diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index b79bb03f..852381fe 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -1,12 +1,12 @@ -#!/bin/bash -e +#!/bin/bash # # SPDX-License-Identifier: Apache-2.0 # # download RPMs/SRPMs from different sources. # this script was originated by Brian Avery, and later updated by Yong Hu -set -o errexit -set -o nounset +# set -o errexit +# set -o nounset # By default, we use "sudo" and we don't use a local dnf.conf. These can # be overridden via flags. @@ -185,6 +185,181 @@ if [ $CLEAN_LOGS_ONLY -eq 1 ];then exit 0 fi +STOP_SCHEDULING=0 +FOUND_ERRORS=0 +MAX_WORKERS=8 +workers=0 +max_workers=$MAX_WORKERS + +# An array that maps worker index to pid, or to two special values +# 'Idle' indicates no running thread. +# 'Busy' indicates the worker is allocated, but it's pid isn't known yet. +declare -A dl_env + +# +# init_dl_env: Init the array that maps worker index to pid. +# +init_dl_env () { + local i=0 + local stop + + stop=$((max_workers-1)) + for i in $(seq 0 $stop); do + dl_env[$i]='Idle' + done +} + +# +# get_idle_dl_env: Find an idle worker, mark it allocated +# and return it's index. +get_idle_dl_env () { + local i=0 + local stop + + stop=$((max_workers-1)) + if [ $stop -ge 255 ]; then + stop=254 + fi + + for i in $(seq 0 $stop); do + if [ ${dl_env[$i]} == 'Idle' ]; then + dl_env[$i]='Busy' + return $i + fi + done + + return 255 +} + +# +# set_dl_env_pid: Set the pid of a previously allocated worker +# +set_dl_env_pid () { + local idx=$1 + local val=$2 + dl_env[$idx]=$val +} + +# +# release_dl_env: Mark a worker as idle. Call after reaping the thread. +# +release_dl_env () { + local idx=$1 + dl_env[$idx]='Idle' +} + +# +# reaper: Look for worker threads that have exited. +# Check/log it's exit code, and release the worker. +# Return the number of threads reaped. +# +reaper () { + local reaped=0 + local last_reaped=-1 + local i=0 + local stop + local p=0 + local ret=0 + + stop=$((max_workers-1)) + if [ $stop -ge 255 ]; then + stop=254 + fi + + while [ $reaped -gt $last_reaped ]; do + last_reaped=$reaped + for i in $(seq 0 $stop); do + p=${dl_env[$i]} + if [ "$p" == "Idle" ] || [ "$p" == "Busy" ]; then + continue + fi + # echo "test $i $p" + kill -0 $p &> /dev/null + if [ $? -ne 0 ]; then + wait $p + ret=$? + workers=$((workers-1)) + reaped=$((reaped+1)) + release_dl_env $i + if [ $ret -ne 0 ]; then + sleep 1 + echo "ERROR: $FUNCNAME (${LINENO}): Failed to download in 'b$i'" + cat "$DL_MIRROR_LOG_DIR/$i" >> $DL_MIRROR_LOG_DIR/errors + echo "ERROR: $FUNCNAME (${LINENO}): Failed to download in 'b$i'" >> $DL_MIRROR_LOG_DIR/errors + echo "" >> $DL_MIRROR_LOG_DIR/errors + FOUND_ERRORS=1 + fi + fi + done + done + return $reaped +} + +# +# download_worker: Download one file. +# This is the entry point for a worker thread. +# +download_worker () { + local dl_idx=$1 + local ff="$2" + local _level=$3 + + local rpm_name="" + local dest_dir="" + local rc=0 + local dl_result=1 + local lvl="" + local download_cmd="" + local download_url="" + local SFILE="" + local _arch="" + + _arch=$(get_arch_from_rpm $ff) + rpm_name="$(get_rpm_name $ff)" + dest_dir="$(get_dest_directory $_arch)" + + if [ ! -e $dest_dir/$rpm_name ]; then + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + lvl=$dl_from_stx_mirror + ;; + $dl_from_upstream) + lvl=$_level + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + download_cmd="$(get_download_cmd $ff $lvl)" + + echo "Looking for $rpm_name" + echo "--> run: $download_cmd" + if $download_cmd ; then + download_url="$(get_url $ff $lvl)" + SFILE="$(get_rpm_level_name $rpm_name $lvl)" + process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" + dl_result=0 + break + else + echo "Warning: $rpm_name not found" + fi + done + + if [ $dl_result -eq 1 ]; then + echo "Error: $rpm_name not found" + echo "missing_srpm:$rpm_name" >> $LOG + echo $rpm_name >> $MISSING_SRPMS + rc=1 + fi + else + echo "Already have $dest_dir/$rpm_name" + fi + return $rc +} + # Function to download different types of RPMs in different ways download () { local _file=$1 @@ -194,70 +369,57 @@ download () { local _arch="" - local rc=0 - local download_cmd="" - local download_url="" - local rpm_name="" - local SFILE="" - local lvl - local dl_result + FOUND_ERRORS=0 _list=$(cat $_file) _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do - _arch=$(get_arch_from_rpm $ff) - rpm_name="$(get_rpm_name $ff)" - dest_dir="$(get_dest_directory $_arch)" - - if [ ! -e $dest_dir/$rpm_name ]; then - dl_result=1 - for dl_src in $dl_source; do - case $dl_src in - $dl_from_stx_mirror) - lvl=$dl_from_stx_mirror - ;; - $dl_from_upstream) - lvl=$_level - ;; - *) - echo "Error: Unknown dl_source '$dl_src'" - continue - ;; - esac - - download_cmd="$(get_download_cmd $ff $lvl)" - - echo "Looking for $rpm_name" - echo "--> run: $download_cmd" - if $download_cmd ; then - download_url="$(get_url $ff $lvl)" - SFILE="$(get_rpm_level_name $rpm_name $lvl)" - process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" - dl_result=0 - break - else - echo "Warning: $rpm_name not found" - fi - done - - if [ $dl_result -eq 1 ]; then - echo "Error: $rpm_name not found" - echo "missing_srpm:$rpm_name" >> $LOG - echo $rpm_name >> $MISSING_SRPMS - rc=1 + # Free up a worker if none available + while [ $workers -ge $max_workers ]; do + reaper + reaped=$? + if [ $reaped -eq 0 ]; then + sleep 0.1 fi - else - echo "Already have $dest_dir/$rpm_name" + done + + # Allocate a worker. b=the worker index + workers=$((workers+1)) + get_idle_dl_env + b=$? + if [ $b -ge 255 ]; then + echo "get_idle_dl_env failed to find a free slot" + exit 1 fi - echo + PREFIX="b$b" + + # Launch a thread in the background + ( download_worker $b $ff $_level 2>&1 | sed "s#^#${PREFIX}: #" | tee $DL_MIRROR_LOG_DIR/$b; exit ${PIPESTATUS[0]} ) & + + # Record the pid of background process + pp=$! + set_dl_env_pid $b $pp done - return $rc + # Wait for remaining workers to exit + while [ $workers -gt 0 ]; do + reaper + reaped=$? + if [ $reaped -eq 0 ]; then + sleep 0.1 + fi + done + + return $FOUND_ERRORS } +# Init the pool of worker threads +init_dl_env + + # Prime the cache loop_count=0 max_loop_count=5 diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index cf476ab0..77e59ee9 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash # # SPDX-License-Identifier: Apache-2.0 # @@ -19,6 +19,11 @@ cleanup () { trap "cleanup ; exit 1" INT HUP TERM QUIT trap "cleanup" EXIT +# Clear the error log before we begin +if [ -f $DL_MIRROR_LOG_DIR/errors ]; then + rm -f $DL_MIRROR_LOG_DIR/errors +fi + # A temporary compatability step to save download time # during the shift to the new DL_MIRROR_OUTPUT_DIR location. # @@ -115,8 +120,8 @@ make_stx_mirror_yum_conf="${DOWNLOAD_MIRROR_DIR}/make_stx_mirror_yum_conf.sh" # track optional arguments change_group_ids=1 -use_system_yum_conf=1 -alternate_yum_conf="" +use_system_yum_conf=0 +alternate_yum_conf="${DOWNLOAD_MIRROR_DIR}/yum.conf.sample" alternate_repo_dir="" rpm_downloader_extra_args="" tarball_downloader_extra_args="" @@ -166,6 +171,8 @@ dl_from_upstream () { MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" +TEMP_DIR="" +TEMP_DIR_CLEANUP="" multiple_dl_flag_check () { if [ "$dl_flag" != "" ]; then @@ -177,7 +184,7 @@ multiple_dl_flag_check () { # Parse out optional arguments -while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do +while getopts "c:Cd:ghI:sl:L:nt:ySuUW:" o; do case "${o}" in c) # Pass -c ("use alternate dnf.conf") to rpm downloader @@ -214,6 +221,15 @@ while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do rpm_downloader_extra_args="${rpm_downloader_extra_args} -n" SUDO="" ;; + t) + # Set TEMP_DIR + TEMP_DIR="${OPTARG}" + ;; + y) + # Use hosts /etc/yum.conf + use_system_yum_conf=1 + alternate_yum_conf="" + ;; s) # Download from StarlingX mirror only. Do not use upstream sources. multiple_dl_flag_check @@ -383,16 +399,15 @@ echo "step #0: Configuring yum repos ..." if [ ${use_system_yum_conf} -ne 0 ]; then # Restore StarlingX_3rd repos from backup - REPO_SOURCE_DIR=/localdisk/yum.repos.d REPO_DIR=/etc/yum.repos.d - if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then - ${SUDO} \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ - fi if [ $layer != "all" ]; then if [ -d ${config_dir}/${distro}/${layer}/yum.repos.d ]; then - ${SUDO} \cp -f ${config_dir}/${distro}/${layer}/yum.repos.d/*.repo $REPO_DIR + ${SUDO} \cp -f -v ${config_dir}/${distro}/${layer}/yum.repos.d/*.repo $REPO_DIR/ fi + else + # copy all layers + ${SUDO} \cp -f -v ${config_dir}/${distro}/*/yum.repos.d/*.repo $REPO_DIR/ fi fi @@ -411,7 +426,6 @@ if [ $use_system_yum_conf -eq 0 ]; then fi fi -TEMP_DIR="" rpm_downloader_extra_args="${rpm_downloader_extra_args} -D $distro" if [ "$dl_flag" != "" ]; then @@ -428,8 +442,20 @@ if ! dl_from_stx; then else # We want to use stx mirror, so we need to create a new, modified dnf.conf and yum.repos.d. # The modifications will add or substitute repos pointing to the StralingX mirror. - TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX) - TEMP_CONF="$TEMP_DIR/dnf.conf" + if [ "$TEMP_DIR" == "" ]; then + if [ "$MY_WORKSPACE" != "" ]; then + TEMP_DIR="$MY_WORKSPACE/tmp/yum" + else + TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX) + TEMP_DIR_CLEANUP="y" + fi + fi + + if [ ! -d $TEMP_DIR ]; then + mkdir -p ${TEMP_DIR} + fi + + TEMP_CONF="$TEMP_DIR/yum.conf" need_file ${make_stx_mirror_yum_conf} need_dir ${TEMP_DIR} @@ -685,7 +711,7 @@ fi # # Clean up the mktemp directory, if required. # -if [ "$TEMP_DIR" != "" ]; then +if [ "$TEMP_DIR" != "" ] && [ "$TEMP_DIR_CLEANUP" == "y" ]; then echo "${SUDO} rm -rf $TEMP_DIR" ${SUDO} \rm -rf "$TEMP_DIR" fi diff --git a/centos-mirror-tools/make_stx_mirror_yum_conf.sh b/centos-mirror-tools/make_stx_mirror_yum_conf.sh index 99521644..e59be8b5 100755 --- a/centos-mirror-tools/make_stx_mirror_yum_conf.sh +++ b/centos-mirror-tools/make_stx_mirror_yum_conf.sh @@ -15,6 +15,7 @@ MAKE_STX_MIRROR_DNF_CONF_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "$MAKE_STX_MIRROR_DNF_CONF_DIR/url_utils.sh" DISTRO="centos" +SUDO=sudo TEMP_DIR="" SRC_REPO_DIR="$MAKE_STX_MIRROR_DNF_CONF_DIR/yum.repos.d" @@ -38,6 +39,7 @@ usage () { echo " 'yum.repos.d' in same directory as this script" echo "-l = Download only packages required to build a given layer" echo "-u ,, = Add/change the repo baseurl for a lower layer" + echo "-n don't use sudo" } declare -A layer_urls @@ -61,7 +63,7 @@ set_layer_urls () { # # option processing # -while getopts "D:d:l:Rr:u:y:" o; do +while getopts "D:d:l:nRr:u:y:" o; do case "${o}" in D) DISTRO="${OPTARG}" @@ -72,6 +74,9 @@ while getopts "D:d:l:Rr:u:y:" o; do l) LAYER="${OPTARG}" ;; + n) + SUDO="" + ;; r) SRC_REPO_DIR="${OPTARG}" ;; diff --git a/centos-mirror-tools/mirror-check.sh b/centos-mirror-tools/mirror-check.sh index 2c19a9e8..7f6ff57f 100755 --- a/centos-mirror-tools/mirror-check.sh +++ b/centos-mirror-tools/mirror-check.sh @@ -40,14 +40,43 @@ ERROR_LOG_FILE="mirror-check-failures.log" truncate -s 0 $ERROR_LOG_FILE retcode=0 extra_opts="" +layer="$LAYER" +valid_layers=('compiler' 'distro' 'flock') + + +# Cloned from cgcs-root/build-tools/pkg-manager-utils.sh +# Ideally this can still be used when tools is the only git +# that has been cloned. + +# Yum vs DNF compatibility +YUM=$(which yum 2>> /dev/null) +DNF=$(which dnf 2>> /dev/null) +PKG_MANAGER="" +REPOQUERY=$(which repoquery 2>> /dev/null) +REPOQUERY_SUB_COMMAND="" +REPOQUERY_RESOLVE="--resolve" +REPOQUERY_WHATPROVIDES_DELIM=" " +if [ ! -z ${DNF} ]; then + PKG_MANAGER="dnf" + REPOQUERY=${DNF} + REPOQUERY_SUB_COMMAND="repoquery --disable-modular-filtering" + REPOQUERY_RESOLVE="" + REPOQUERY_WHATPROVIDES_DELIM="," +elif [ ! -z ${YUM} ]; then + PKG_MANAGER="yum" +else + >&2 echo "ERROR: Couldn't find a supported package manager" + exit 1 +fi usage() { - echo "$0 [-c ]" + echo "$0 [-c ] [-l ]" echo "" echo "Options:" echo " -c: Use an alternate dnf.conf rather than the system file (option passed" echo " on to subscripts when appropriate)" + echo " -l: Check specific layer (one of 'all ${valid_layers[@]}')" echo "" } @@ -79,8 +108,10 @@ get_repoquery_info() { else repoquery_opts= fi - repoquery $extra_opts ${RELEASEVER} -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ - $repoquery_opts "$_package_name" + $REPOQUERY $REPOQUERY_SUB_COMMAND \ + $extra_opts ${RELEASEVER} -C \ + --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ + $repoquery_opts "$_package_name" } _check_rpms() { @@ -113,12 +144,27 @@ check_rpms() { done } -while getopts "c:" opt; do +while getopts "c:l:" opt; do case $opt in c) extra_opts="-c ${OPTARG}" grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})" ;; + l) + layer="${OPTARG}" + if [ "$layer" == "all" ]; then + layer="" + else + case " ${valid_layers[@]} " in + *" $layer "* ) echo "found layer $layer" + ;; + *) echo "'$layer' is invalid" + usage + exit 1 + ;; + esac + fi + ;; \?) echo "Invalid option: -$OPTARG" >&2 usage @@ -133,11 +179,11 @@ if ! dnf $extra_opts ${RELEASEVER} makecache; then exit 1 fi -for rpm_list in "$RPMS_CENTOS_LIST" "$RPMS_3RD_PARTY_LIST"; do +for rpm_list in $(find config/centos/$layer -name "$RPMS_CENTOS_LIST" -o -name "$RPMS_3RD_PARTY_LIST"); do info "Reading $rpm_list..." for arch in "src" "noarch" "x86_64"; do info "Getting info for $arch packages..." - rpms=$(echo "$(grep -F "$arch.rpm" < $rpm_list)") + rpms=$(echo "$(grep -v '^#' $rpm_list | grep -F "$arch.rpm")") check_rpms "$rpms" done done diff --git a/centos-mirror-tools/utils.sh b/centos-mirror-tools/utils.sh index 4aa1363e..cd4274c0 100644 --- a/centos-mirror-tools/utils.sh +++ b/centos-mirror-tools/utils.sh @@ -74,7 +74,7 @@ get_url() { # filter urls for the desitered arch. local arr=( $(split_filename $_name) ) local arch=${arr[3]} - _ret="$($_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)" + _ret="$(${SUDO} $_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)" fi echo "$_ret" } diff --git a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.9.repo b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.9.repo index bf65e03a..8b230abd 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.9.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.9.repo @@ -86,6 +86,13 @@ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 enabled=1 +[StarlingX-C7.9.2009-rt-source] +name=StarlingX-CentOS-7.9.2009 - rt-source +baseurl=https://vault.centos.org/centos/7.9.2009/rt/Source/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + [StarlingX-C7.9.2009-rt] name=StarlingX-CentOS-7.9.2009 - rt baseurl=http://mirror.centos.org/centos/7.9.2009/rt/x86_64/ diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_3rd_linuxsoft_cern_ch.repo b/centos-mirror-tools/yum.repos.d/StarlingX_3rd_linuxsoft_cern_ch.repo index 5cc4ba85..fc4aa5f4 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_3rd_linuxsoft_cern_ch.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_3rd_linuxsoft_cern_ch.repo @@ -4,4 +4,4 @@ [Starlingx-linuxsoft.cern.ch_cern_centos_7.5_rt_Sources] name=Starlingx-linuxsoft.cern.ch_cern_centos_7.5_rt_Sources - linuxsoft.cern.ch_cern_centos_7.5_rt_Sources baseurl=http://linuxsoft.cern.ch/cern/centos/7.5/rt/Sources/ -enabled=1 +enabled=0 diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo index d3c802fb..a5f29099 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo @@ -5,7 +5,7 @@ [centos-openstack-queens] name=CentOS-7 - OpenStack queens -baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/ +baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud @@ -13,14 +13,14 @@ exclude=sip,PyQt4 [centos-openstack-queens-test] name=CentOS-7 - OpenStack queens Testing -baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-queens/ +baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-queens/ gpgcheck=0 enabled=0 exclude=sip,PyQt4 [centos-openstack-queens-debuginfo] name=CentOS-7 - OpenStack queens - Debug -baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-rocky.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-rocky.repo index cad336e6..560440e8 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-rocky.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-rocky.repo @@ -5,7 +5,7 @@ [centos-openstack-rocky] name=CentOS-7 - OpenStack rocky -baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-rocky/ +baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-rocky/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud @@ -13,14 +13,14 @@ exclude=sip,PyQt4 [centos-openstack-rocky-test] name=CentOS-7 - OpenStack rocky Testing -baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-rocky/ +baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-rocky/ gpgcheck=0 enabled=0 exclude=sip,PyQt4 [centos-openstack-rocky-debuginfo] name=CentOS-7 - OpenStack rocky - Debug -baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-stein.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-stein.repo index a32c02f0..7427329e 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-stein.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-stein.repo @@ -5,7 +5,7 @@ [centos-openstack-stein] name=CentOS-7 - OpenStack stein -baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-stein/ +baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-stein/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud @@ -13,14 +13,14 @@ exclude=sip,PyQt4 [centos-openstack-stein-test] name=CentOS-7 - OpenStack stein Testing -baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-stein/ +baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-stein/ gpgcheck=0 enabled=0 exclude=sip,PyQt4 [centos-openstack-stein-debuginfo] name=CentOS-7 - OpenStack stein - Debug -baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-train.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-train.repo index a1e06a67..4bf49369 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-train.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-train.repo @@ -5,7 +5,7 @@ [centos-openstack-train] name=CentOS-7 - OpenStack train -baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-train/ +baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-train/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud @@ -13,14 +13,14 @@ exclude=sip,PyQt4 [centos-openstack-train-test] name=CentOS-7 - OpenStack train Testing -baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-train/ +baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-train/ gpgcheck=0 enabled=0 exclude=sip,PyQt4 [centos-openstack-train-debuginfo] name=CentOS-7 - OpenStack train - Debug -baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/ gpgcheck=1 enabled=0 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/cve_support/cve_policy_filter.py b/cve_support/cve_policy_filter.py index dff356d2..1fc246f0 100644 --- a/cve_support/cve_policy_filter.py +++ b/cve_support/cve_policy_filter.py @@ -14,6 +14,16 @@ import sys import os from lp import find_lp_assigned +cves_valid = [] +cves_to_fix = [] +cves_to_fix_lp = [] +cves_to_track = [] +cves_w_errors = [] +cves_wont_fix = [] +cves_to_omit = [] +cves_report = {} + + def print_html_report(cves_report, title): """ Print the html report @@ -22,9 +32,14 @@ def print_html_report(cves_report, title): template_loader = jinja2.FileSystemLoader(searchpath="./") template_env = jinja2.Environment(loader=template_loader) - template_file = "template.txt" + if CVSS_VER == "cvssv3": + template_file = "template_v3.txt" + heads = ["cve_id", "status", "cvss3Score", "av", "ac", "ui","a"] + else: + template_file = "template.txt" + heads = ["cve_id", "status", "cvss2Score", "av", "ac", "au", "ai"] + template = template_env.get_template(template_file) - heads = ["cve_id", "status", "cvss2Score", "av", "ac", "au", "ai"] output_text = template.render(cves_to_fix=cves_report["cves_to_fix"],\ cves_to_fix_lp=cves_report["cves_to_fix_lp"],\ cves_to_track=cves_report["cves_to_track"],\ @@ -49,10 +64,16 @@ def print_report(cves_report, title): print("\n") print(cve["id"]) print("status : " + cve["status"]) - print("cvss2Score : " + str(cve["cvss2Score"])) + if CVSS_VER == "cvssv3": + print("cvss3Score : " + str(cve["cvss3Score"])) + else: + print("cvss2Score : " + str(cve["cvss2Score"])) print("Attack Vector: " + cve["av"]) print("Access Complexity : " + cve["ac"]) - print("Authentication: " + cve["au"]) + if CVSS_VER == "cvssv3": + print("User Interaction: " + cve["ui"]) + else: + print("Authentication: " + cve["au"]) print("Availability Impact :" + cve["ai"]) print("Affected packages:") print(cve["affectedpackages"]) @@ -88,8 +109,12 @@ def print_report(cves_report, title): print(cve_line) - print("\nERROR: CVEs that have no cvss2Score or cvss2Vector: %d \n" \ - % (len(cves_report["cves_w_errors"]))) + if CVSS_VER == "cvssv3": + print("\nERROR: CVEs that have no cvss3Score or cvss3Vector: %d \n" \ + % (len(cves_report["cves_w_errors"]))) + else: + print("\nERROR: CVEs that have no cvss2Score or cvss2Vector: %d \n" \ + % (len(cves_report["cves_w_errors"]))) for cve in cves_report["cves_w_errors"]: print(cve) @@ -130,49 +155,112 @@ def get_affectedpackages(data, cve_id): allfixed = "unfixed" return affectedpackages_list, allfixed -def main(): +def update_report(): + cves_report["cves_to_fix"] = cves_to_fix + cves_report["cves_to_fix_lp"] = cves_to_fix_lp + cves_report["cves_to_track"] = cves_to_track + cves_report["cves_w_errors"] = cves_w_errors + cves_report["cves_wont_fix"] = cves_wont_fix + cves_report["cves_to_omit"] = cves_to_omit + +def cvssv3_pb_alg(): """ - main function - Rules to consider a CVE valid for STX from: - https://wiki.openstack.org/wiki/StarlingX/Security/CVE_Support_Policy + Patchback algo for CVSSV3 report """ - data = {} - cves = [] - cves_valid = [] - cves_to_fix = [] - cves_to_fix_lp = [] - cves_to_track = [] - cves_w_errors = [] - cves_wont_fix = [] - cves_to_omit = [] - cves_report = {} + for cve in cves_valid: + if (cve["cvss3Score"] >= 7.8 + and cve["av"] == "N" + and cve["ac"] == "L" + and cve["ui"] == "R" + and cve["ai"] != "N"): + if cve["status"] == "fixed": + bug = find_lp_assigned(cve["id"]) + if (bug): + print(bug["status"]) + if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"): + cves_wont_fix.append(cve) + else: + cves_to_fix_lp.append(cve) + else: + cves_to_fix.append(cve) + else: + cves_to_track.append(cve) + else: + cves_to_omit.append(cve) - if len(sys.argv) < 3: - print("\nERROR : Missing arguments, the expected arguments are:") - print("\n %s \n" % (sys.argv[0])) - print("\n result.json = json file generated from: vuls report -format-json") - print("\n") - sys.exit(0) + update_report() - if os.path.isfile(sys.argv[1]): - results_json = sys.argv[1] - else: - print("%s is not a file" % sys.argv[1]) - sys.exit(0) - title = sys.argv[2] +def cvssv2_pb_alg(): + """ + Patchback algo for CVSSV2 report + """ + for cve in cves_valid: + if (cve["cvss2Score"] >= 7.0 + and cve["av"] == "N" + and cve["ac"] == "L" + and ("N" in cve["au"] or "S" in cve["au"]) + and ("P" in cve["ai"] or "C" in cve["ai"])): + if cve["status"] == "fixed": + bug = find_lp_assigned(cve["id"]) + if (bug): + print(bug["status"]) + if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"): + cves_wont_fix.append(cve) + else: + cves_to_fix_lp.append(cve) + else: + cves_to_fix.append(cve) + else: + cves_to_track.append(cve) + else: + cves_to_omit.append(cve) - try: - with open(results_json) as json_file: - data = json.load(json_file) - except ValueError as error: - print(error) + update_report() - for element in data["scannedCves"]: - cve = {} - cve["id"] = str(element.strip()) - cves.append(cve) +def cvssv3_parse_n_report(cves,title,data): + """ + Parse and generate report for CVSSV3 + """ + for cve in cves: + cve_id = cve["id"] + affectedpackages_list = [] + allfixed = "fixed" + try: + nvd2_score = data["scannedCves"][cve_id]["cveContents"]["nvd"]["cvss3Score"] + cvss3vector = data["scannedCves"][cve_id]["cveContents"]["nvd"]["cvss3Vector"] + except KeyError: + cves_w_errors.append(cve) + else: + cve["cvss3Score"] = nvd2_score + for element in cvss3vector.split("/"): + if "AV:" in element: + _av = element.split(":")[1] + if "AC:" in element: + _ac = element.split(":")[1] + if "A:" in element: + _ai = element.split(":")[1] + if "UI:" in element: + _ui = element.split(":")[1] + print(cve) + cve["av"] = str(_av) + cve["ac"] = str(_ac) + cve["ai"] = str(_ai) + cve["ui"] = str(_ui) + cve["summary"] = get_summary(data, cve_id) + cve["sourcelink"] = get_source_link(data, cve_id) + affectedpackages_list, allfixed = get_affectedpackages(data, cve_id) + cve["affectedpackages"] = affectedpackages_list + cve["status"] = allfixed + cves_valid.append(cve) + cvssv3_pb_alg() + print_report(cves_report, title) + print_html_report(cves_report, title) +def cvssv2_parse_n_report(cves,title,data): + """ + Parse and generate report for CVSSV2 + """ for cve in cves: cve_id = cve["id"] affectedpackages_list = [] @@ -203,37 +291,55 @@ def main(): cve["affectedpackages"] = affectedpackages_list cve["status"] = allfixed cves_valid.append(cve) - - for cve in cves_valid: - if (cve["cvss2Score"] >= 7.0 - and cve["av"] == "N" - and cve["ac"] == "L" - and ("N" in cve["au"] or "S" in cve["au"]) - and ("P" in cve["ai"] or "C" in cve["ai"])): - if cve["status"] == "fixed": - bug = find_lp_assigned(cve["id"]) - if (bug): - print(bug["status"]) - if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"): - cves_wont_fix.append(cve) - else: - cves_to_fix_lp.append(cve) - else: - cves_to_fix.append(cve) - else: - cves_to_track.append(cve) - else: - cves_to_omit.append(cve) - - cves_report["cves_to_fix"] = cves_to_fix - cves_report["cves_to_fix_lp"] = cves_to_fix_lp - cves_report["cves_to_track"] = cves_to_track - cves_report["cves_w_errors"] = cves_w_errors - cves_report["cves_wont_fix"] = cves_wont_fix - cves_report["cves_to_omit"] = cves_to_omit - + cvssv2_pb_alg() print_report(cves_report, title) print_html_report(cves_report, title) +def main(): + """ + main function + Rules to consider a CVE valid for STX from: + https://wiki.openstack.org/wiki/StarlingX/Security/CVE_Support_Policy + """ + data = {} + cves = [] + + + if len(sys.argv) < 4: + print("\nERROR : Missing arguments, the expected arguments are:") + print("\n %s <result.json> <title> [cvssv3|cvssv2]\n" % (sys.argv[0])) + print("\n result.json = json file generated from: vuls report -format-json") + print("\n") + sys.exit(0) + + if os.path.isfile(sys.argv[1]): + results_json = sys.argv[1] + else: + print("%s is not a file" % sys.argv[1]) + sys.exit(0) + + title = sys.argv[2] + + try: + with open(results_json) as json_file: + data = json.load(json_file) + except ValueError as error: + print(error) + + for element in data["scannedCves"]: + cve = {} + cve["id"] = str(element.strip()) + cves.append(cve) + global CVSS_VER + CVSS_VER=sys.argv[3].lower() + if CVSS_VER =="cvssv3": + cvssv3_parse_n_report(cves,title,data) + elif CVSS_VER == "cvssv2": + cvssv2_parse_n_report(cves,title,data) + else: + print("\n argument not matching \n enter [cvssv3|cvssv2] ") + sys.exit(0) + + if __name__ == "__main__": main() diff --git a/cve_support/template_v3.txt b/cve_support/template_v3.txt new file mode 100644 index 00000000..a666c9ba --- /dev/null +++ b/cve_support/template_v3.txt @@ -0,0 +1,127 @@ +<head></head> +<body> + <h1>Security report from vuls scan from {{title}}</h1> + <h2>CVEs to fix w/o a launchpad assigned: {{cves_to_fix | length}}</h2> + <table> + {% if cves_to_fix|length >= 1 %} + <tr> + {% for head in heads %} + <th>{{head}}</th> + {% endfor %} + </tr> + + {% for cve in cves_to_fix %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + <td>{{cve["cvss3Score"]}}</td> + <td>{{cve["av"]}}</td> + <td>{{cve["ac"]}}</td> + <td>{{cve["ui"]}}</td> + <td>{{cve["ai"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> + <h2>CVEs to fix w/ a launchpad assigend: {{cves_to_fix_lp | length}}</h2> + <table> + {% if cves_to_fix_lp|length >= 1 %} + <tr> + {% for head in heads %} + <th>{{head}}</th> + {% endfor %} + </tr> + + {% for cve in cves_to_fix_lp %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + <td>{{cve["cvss3Score"]}}</td> + <td>{{cve["av"]}}</td> + <td>{{cve["ac"]}}</td> + <td>{{cve["ui"]}}</td> + <td>{{cve["ai"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> + <h2> CVEs to track for incoming fix: {{cves_to_track | length}}</h2> + <table> + {% if cves_to_track|length >= 1 %} + <tr> + {% for head in heads %} + <th>{{head}}</th> + {% endfor %} + </tr> + + {% for cve in cves_to_track %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + <td>{{cve["cvss3Score"]}}</td> + <td>{{cve["av"]}}</td> + <td>{{cve["ac"]}}</td> + <td>{{cve["ui"]}}</td> + <td>{{cve["ai"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> + <h2> CVEs that are Invalid or Won't Fix: {{cves_wont_fix | length}}</h2> + <table> + {% if cves_wont_fix|length >= 1 %} + <tr> + {% for head in heads %} + <th>{{head}}</th> + {% endfor %} + </tr> + + {% for cve in cves_wont_fix %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + <td>{{cve["cvss3Score"]}}</td> + <td>{{cve["av"]}}</td> + <td>{{cve["ac"]}}</td> + <td>{{cve["ui"]}}</td> + <td>{{cve["ai"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> + + <h2> CVEs to omit: {{cves_to_omit | length}}</h2> + <table> + {% if cves_to_omit|length >= 1 %} + <tr> + {% for head in heads %} + <th>{{head}}</th> + {% endfor %} + </tr> + + {% for cve in cves_to_omit %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + <td>{{cve["cvss3Score"]}}</td> + <td>{{cve["av"]}}</td> + <td>{{cve["ac"]}}</td> + <td>{{cve["ui"]}}</td> + <td>{{cve["ai"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> + + <h2>ERROR: CVEs that have no cvss3Score or cvss2Vector:{{cves_w_errors | length}}</h2> + <table> + {% if cves_w_errors|length >= 1 %} + {% for cve in cves_w_errors %} + <tr> + <td>{{cve["id"]}}</td> + <td>{{cve["status"]}}</td> + </tr> + {% endfor %} + {% endif %} + </table> +</body> diff --git a/tb.sh b/tb.sh index 9feb9a6e..0948b609 100755 --- a/tb.sh +++ b/tb.sh @@ -24,12 +24,17 @@ CMD=$1 TC_CONTAINER_NAME=${MYUNAME}-centos-builder TC_CONTAINER_TAG=local/${MYUNAME}-stx-builder:8.2.2004 TC_DOCKERFILE=Dockerfile +NO_CACHE=0 function create_container { local EXTRA_ARGS="" if [ ! -z ${MY_EMAIL} ]; then - EXTRA_ARGS="--build-arg MY_EMAIL=${MY_EMAIL}" + EXTRA_ARGS+="--build-arg MY_EMAIL=${MY_EMAIL}" + fi + + if [ $NO_CACHE -eq 1 ]; then + EXTRA_ARGS+=" --no-cache" fi docker build \ @@ -87,7 +92,7 @@ function clean_container { } function usage { - echo "$0 [create|run|exec|env|stop|kill|clean]" + echo "$0 [create|create_no_cache|run|exec|env|stop|kill|clean]" } case $CMD in @@ -109,6 +114,10 @@ case $CMD in create) create_container ;; + create_no_cache) + NO_CACHE=1 + create_container + ;; exec) exec_container ;; diff --git a/toCOPY/lst_utils.sh b/toCOPY/lst_utils.sh index 295fdc92..e6a7aee3 100644 --- a/toCOPY/lst_utils.sh +++ b/toCOPY/lst_utils.sh @@ -52,7 +52,10 @@ merge_lst () { return 1 fi - layers=$(cat ${layer_cfgs} | sort --unique) + # Grep to ignore empty lines or whole line comments. + # Sed to drop any trailing comments. + # Side effect of grep over cat is adding any missing EOL. + layers=$(grep -h -v -e '^$' -e '^[ \t]*#' ${layer_cfgs} | sed -e 's/[ \t]*#.*$//' | sort --unique) layers+=" mock" (