Merge remote-tracking branch 'origin/master' into f/centos8
Signed-off-by: Charles Short <charles.short@windriver.com> Change-Id: I814a4565e812639c17d3c63457007e5815dd6924
This commit is contained in:
Charles Short
commit
d701c6f896
|
|
@ -5,6 +5,7 @@ localrc
|
|||
toCOPY/.gitconfig
|
||||
centos-mirror-tools/logs/
|
||||
centos-mirror-tools/output/
|
||||
centos-mirror-tools/mirror-check-failures.log
|
||||
|
||||
# Sphinx documentation
|
||||
doc/build/
|
||||
|
|
|
|||
123
Dockerfile
123
Dockerfile
|
|
@ -60,10 +60,8 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY* && \
|
|||
VOLUME /run /tmp
|
||||
|
||||
# Download required dependencies by mirror/build processes.
|
||||
RUN groupadd -g 751 cgts && \
|
||||
echo "mock:x:751:root" >> /etc/group && \
|
||||
echo "mockbuild:x:9001:" >> /etc/group && \
|
||||
dnf install -y anaconda \
|
||||
RUN dnf install -y \
|
||||
anaconda \
|
||||
anaconda-runtime \
|
||||
autoconf-archive \
|
||||
autogen \
|
||||
|
|
@ -90,7 +88,6 @@ RUN groupadd -g 751 cgts && \
|
|||
lighttpd-fastcgi \
|
||||
#lighttpd-mod_geoip \
|
||||
mkisofs \
|
||||
mock \
|
||||
#mongodb \
|
||||
#mongodb-server \
|
||||
net-tools \
|
||||
|
|
@ -126,21 +123,26 @@ RUN groupadd -g 751 cgts && \
|
|||
RUN alternatives --set python /usr/bin/python3
|
||||
RUN alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
|
||||
|
||||
# This image requires a set of scripts and helpers
|
||||
# for working correctly, in this section they are
|
||||
# copied inside the image.
|
||||
COPY toCOPY/finishSetup.sh /usr/local/bin
|
||||
COPY toCOPY/populate_downloads.sh /usr/local/bin
|
||||
COPY toCOPY/generate-local-repo.sh /usr/local/bin
|
||||
COPY toCOPY/generate-centos-repo.sh /usr/local/bin
|
||||
COPY toCOPY/lst_utils.sh /usr/local/bin
|
||||
COPY toCOPY/.inputrc /home/$MYUNAME/
|
||||
COPY toCOPY/builder-constraints.txt /home/$MYUNAME/
|
||||
# Finally install a locked down version of mock
|
||||
RUN groupadd -g 751 cgts && \
|
||||
echo "mock:x:751:root" >> /etc/group && \
|
||||
echo "mockbuild:x:9001:" >> /etc/group && \
|
||||
dnf install -y \
|
||||
http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/m/mock-2.8-1.el8.noarch.rpm \
|
||||
http://mirror.starlingx.cengn.ca/mirror/centos/epel/dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/m/mock-core-configs-33-1.el8.noarch.rpm
|
||||
|
||||
# mock custumizations
|
||||
# forcing chroots since a couple of packages naughtily insist on network access and
|
||||
# we dont have nspawn and networks happy together.
|
||||
RUN useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild && \
|
||||
rmdir /var/lib/mock && \
|
||||
ln -s /localdisk/loadbuild/mock /var/lib/mock && \
|
||||
rmdir /var/cache/mock && \
|
||||
ln -s /localdisk/loadbuild/mock-cache /var/cache/mock && \
|
||||
echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg && \
|
||||
echo "config_opts['rpmbuild_networking'] = True" >> /etc/mock/site-defaults.cfg && \
|
||||
echo >> /etc/mock/site-defaults.cfg
|
||||
|
||||
# Thes are included for backward compatibility, and
|
||||
# should be removed after a reasonable time.
|
||||
COPY toCOPY/generate-cgcs-tis-repo /usr/local/bin
|
||||
COPY toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin
|
||||
|
||||
# cpan modules, installing with cpanminus to avoid stupid questions since cpan is whack
|
||||
RUN cpanm --notest Fatal && \
|
||||
|
|
@ -149,10 +151,6 @@ RUN cpanm --notest Fatal && \
|
|||
cpanm --notest XML::Parser && \
|
||||
cpanm --notest XML::Simple
|
||||
|
||||
# pip installs
|
||||
RUN pip3 install -c /home/$MYUNAME/builder-constraints.txt python-subunit junitxml --upgrade && \
|
||||
pip3 install -c /home/$MYUNAME/builder-constraints.txt tox --upgrade
|
||||
|
||||
# Install repo tool
|
||||
RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/local/bin/repo && \
|
||||
chmod a+x /usr/local/bin/repo
|
||||
|
|
@ -164,17 +162,35 @@ RUN dnf install -y golang && \
|
|||
mkdir -p ${GOPATH}/bin && \
|
||||
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
|
||||
|
||||
# mock time
|
||||
# forcing chroots since a couple of packages naughtily insist on network access and
|
||||
# we dont have nspawn and networks happy together.
|
||||
RUN useradd -s /sbin/nologin -u 9001 -g 9001 mockbuild && \
|
||||
rmdir /var/lib/mock && \
|
||||
ln -s /localdisk/loadbuild/mock /var/lib/mock && \
|
||||
rmdir /var/cache/mock && \
|
||||
ln -s /localdisk/loadbuild/mock-cache /var/cache/mock && \
|
||||
echo "config_opts['use_nspawn'] = False" >> /etc/mock/site-defaults.cfg && \
|
||||
echo "config_opts['rpmbuild_networking'] = True" >> /etc/mock/site-defaults.cfg && \
|
||||
echo >> /etc/mock/site-defaults.cfg
|
||||
# Uprev git, repo
|
||||
RUN yum install -y dh-autoreconf curl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel asciidoc xmlto docbook2X && \
|
||||
cd /tmp && \
|
||||
wget https://github.com/git/git/archive/v2.29.2.tar.gz -O git-2.29.2.tar.gz && \
|
||||
tar xzvf git-2.29.2.tar.gz && \
|
||||
cd git-2.29.2 && \
|
||||
make configure && \
|
||||
./configure --prefix=/usr/local && \
|
||||
make all doc && \
|
||||
make install install-doc && \
|
||||
cd /tmp && \
|
||||
rm -rf git-2.29.2.tar.gz git-2.29.2
|
||||
|
||||
# Systemd Enablement
|
||||
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
|
||||
rm -f /lib/systemd/system/multi-user.target.wants/*;\
|
||||
rm -f /etc/systemd/system/*.wants/*;\
|
||||
rm -f /lib/systemd/system/local-fs.target.wants/*; \
|
||||
rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
|
||||
rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
|
||||
rm -f /lib/systemd/system/basic.target.wants/*;\
|
||||
rm -f /lib/systemd/system/anaconda.target.wants/*
|
||||
|
||||
# pip installs
|
||||
COPY toCOPY/builder-constraints.txt /home/$MYUNAME/
|
||||
RUN pip install -c /home/$MYUNAME/builder-constraints.txt pbr==5.6.0 --upgrade && \
|
||||
pip install -c /home/$MYUNAME/builder-constraints.txt git-review==2.1.0 --upgrade && \
|
||||
pip install -c /home/$MYUNAME/builder-constraints.txt python-subunit==1.4.0 junitxml==0.7 --upgrade && \
|
||||
pip install -c /home/$MYUNAME/builder-constraints.txt tox==3.23.0 --upgrade
|
||||
|
||||
# Inherited tools for mock stuff
|
||||
# we at least need the mock_cache_unlock tool
|
||||
|
|
@ -184,6 +200,21 @@ RUN cd /opt/mock_overlay && \
|
|||
make && \
|
||||
make install
|
||||
|
||||
# This image requires a set of scripts and helpers
|
||||
# for working correctly, in this section they are
|
||||
# copied inside the image.
|
||||
COPY toCOPY/finishSetup.sh /usr/local/bin
|
||||
COPY toCOPY/populate_downloads.sh /usr/local/bin
|
||||
COPY toCOPY/generate-local-repo.sh /usr/local/bin
|
||||
COPY toCOPY/generate-centos-repo.sh /usr/local/bin
|
||||
COPY toCOPY/lst_utils.sh /usr/local/bin
|
||||
COPY toCOPY/.inputrc /home/$MYUNAME/
|
||||
|
||||
# Thes are included for backward compatibility, and
|
||||
# should be removed after a reasonable time.
|
||||
COPY toCOPY/generate-cgcs-tis-repo /usr/local/bin
|
||||
COPY toCOPY/generate-cgcs-centos-repo.sh /usr/local/bin
|
||||
|
||||
# ENV setup
|
||||
RUN echo "# Load stx-builder configuration" >> /etc/profile.d/stx-builder-conf.sh && \
|
||||
echo "if [[ -r \${HOME}/buildrc ]]; then" >> /etc/profile.d/stx-builder-conf.sh && \
|
||||
|
|
@ -235,30 +266,6 @@ RUN echo "$MYUNAME ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \
|
|||
sed -i "s/dir-listing.activate/#dir-listing.activate/g" /etc/lighttpd/conf.d/dirlisting.conf && \
|
||||
echo "dir-listing.activate = \"enable\"" >> /etc/lighttpd/conf.d/dirlisting.conf
|
||||
|
||||
# Uprev git, git-review, repo
|
||||
RUN dnf install -y dh-autoreconf curl-devel expat-devel gettext-devel openssl-devel perl-devel zlib-devel asciidoc xmlto docbook2X && \
|
||||
cd /tmp && \
|
||||
wget https://github.com/git/git/archive/v2.29.2.tar.gz -O git-2.29.2.tar.gz && \
|
||||
tar xzvf git-2.29.2.tar.gz && \
|
||||
cd git-2.29.2 && \
|
||||
make configure && \
|
||||
./configure --prefix=/usr/local && \
|
||||
make all doc && \
|
||||
make install install-doc && \
|
||||
cd /tmp && \
|
||||
rm -rf git-2.29.2.tar.gz git-2.29.2 && \
|
||||
pip3 install git-review --upgrade
|
||||
|
||||
# Systemd Enablement
|
||||
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
|
||||
rm -f /lib/systemd/system/multi-user.target.wants/*;\
|
||||
rm -f /etc/systemd/system/*.wants/*;\
|
||||
rm -f /lib/systemd/system/local-fs.target.wants/*; \
|
||||
rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
|
||||
rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
|
||||
rm -f /lib/systemd/system/basic.target.wants/*;\
|
||||
rm -f /lib/systemd/system/anaconda.target.wants/*
|
||||
|
||||
RUN useradd -r -u $MYUID -g cgts -m $MYUNAME && \
|
||||
ln -s /home/$MYUNAME/.ssh /mySSH && \
|
||||
rsync -av /etc/skel/ /home/$MYUNAME/
|
||||
|
|
|
|||
|
|
@ -15,4 +15,4 @@ nss-softokn-freebl-3.53.1-11.el8_2.x86_64.rpm
|
|||
systemd-239-18.el8_1.5.x86_64.rpm
|
||||
systemd-devel-239-18.el8_1.5.x86_64.rpm
|
||||
systemd-libs-239-18.el8_1.5.x86_64.rpm
|
||||
util-linux-2.32.1-17.el8.x86_64.rpm
|
||||
util-linux-2.32.1-17.el8.x86_64.rpm
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
#!/bin/bash -e
|
||||
#!/bin/bash
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# download RPMs/SRPMs from different sources.
|
||||
# this script was originated by Brian Avery, and later updated by Yong Hu
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
# set -o errexit
|
||||
# set -o nounset
|
||||
|
||||
# By default, we use "sudo" and we don't use a local dnf.conf. These can
|
||||
# be overridden via flags.
|
||||
|
|
@ -185,6 +185,181 @@ if [ $CLEAN_LOGS_ONLY -eq 1 ];then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
STOP_SCHEDULING=0
|
||||
FOUND_ERRORS=0
|
||||
MAX_WORKERS=8
|
||||
workers=0
|
||||
max_workers=$MAX_WORKERS
|
||||
|
||||
# An array that maps worker index to pid, or to two special values
|
||||
# 'Idle' indicates no running thread.
|
||||
# 'Busy' indicates the worker is allocated, but it's pid isn't known yet.
|
||||
declare -A dl_env
|
||||
|
||||
#
|
||||
# init_dl_env: Init the array that maps worker index to pid.
|
||||
#
|
||||
init_dl_env () {
|
||||
local i=0
|
||||
local stop
|
||||
|
||||
stop=$((max_workers-1))
|
||||
for i in $(seq 0 $stop); do
|
||||
dl_env[$i]='Idle'
|
||||
done
|
||||
}
|
||||
|
||||
#
|
||||
# get_idle_dl_env: Find an idle worker, mark it allocated
|
||||
# and return it's index.
|
||||
get_idle_dl_env () {
|
||||
local i=0
|
||||
local stop
|
||||
|
||||
stop=$((max_workers-1))
|
||||
if [ $stop -ge 255 ]; then
|
||||
stop=254
|
||||
fi
|
||||
|
||||
for i in $(seq 0 $stop); do
|
||||
if [ ${dl_env[$i]} == 'Idle' ]; then
|
||||
dl_env[$i]='Busy'
|
||||
return $i
|
||||
fi
|
||||
done
|
||||
|
||||
return 255
|
||||
}
|
||||
|
||||
#
|
||||
# set_dl_env_pid: Set the pid of a previously allocated worker
|
||||
#
|
||||
set_dl_env_pid () {
|
||||
local idx=$1
|
||||
local val=$2
|
||||
dl_env[$idx]=$val
|
||||
}
|
||||
|
||||
#
|
||||
# release_dl_env: Mark a worker as idle. Call after reaping the thread.
|
||||
#
|
||||
release_dl_env () {
|
||||
local idx=$1
|
||||
dl_env[$idx]='Idle'
|
||||
}
|
||||
|
||||
#
|
||||
# reaper: Look for worker threads that have exited.
|
||||
# Check/log it's exit code, and release the worker.
|
||||
# Return the number of threads reaped.
|
||||
#
|
||||
reaper () {
|
||||
local reaped=0
|
||||
local last_reaped=-1
|
||||
local i=0
|
||||
local stop
|
||||
local p=0
|
||||
local ret=0
|
||||
|
||||
stop=$((max_workers-1))
|
||||
if [ $stop -ge 255 ]; then
|
||||
stop=254
|
||||
fi
|
||||
|
||||
while [ $reaped -gt $last_reaped ]; do
|
||||
last_reaped=$reaped
|
||||
for i in $(seq 0 $stop); do
|
||||
p=${dl_env[$i]}
|
||||
if [ "$p" == "Idle" ] || [ "$p" == "Busy" ]; then
|
||||
continue
|
||||
fi
|
||||
# echo "test $i $p"
|
||||
kill -0 $p &> /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
wait $p
|
||||
ret=$?
|
||||
workers=$((workers-1))
|
||||
reaped=$((reaped+1))
|
||||
release_dl_env $i
|
||||
if [ $ret -ne 0 ]; then
|
||||
sleep 1
|
||||
echo "ERROR: $FUNCNAME (${LINENO}): Failed to download in 'b$i'"
|
||||
cat "$DL_MIRROR_LOG_DIR/$i" >> $DL_MIRROR_LOG_DIR/errors
|
||||
echo "ERROR: $FUNCNAME (${LINENO}): Failed to download in 'b$i'" >> $DL_MIRROR_LOG_DIR/errors
|
||||
echo "" >> $DL_MIRROR_LOG_DIR/errors
|
||||
FOUND_ERRORS=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
return $reaped
|
||||
}
|
||||
|
||||
#
|
||||
# download_worker: Download one file.
|
||||
# This is the entry point for a worker thread.
|
||||
#
|
||||
download_worker () {
|
||||
local dl_idx=$1
|
||||
local ff="$2"
|
||||
local _level=$3
|
||||
|
||||
local rpm_name=""
|
||||
local dest_dir=""
|
||||
local rc=0
|
||||
local dl_result=1
|
||||
local lvl=""
|
||||
local download_cmd=""
|
||||
local download_url=""
|
||||
local SFILE=""
|
||||
local _arch=""
|
||||
|
||||
_arch=$(get_arch_from_rpm $ff)
|
||||
rpm_name="$(get_rpm_name $ff)"
|
||||
dest_dir="$(get_dest_directory $_arch)"
|
||||
|
||||
if [ ! -e $dest_dir/$rpm_name ]; then
|
||||
for dl_src in $dl_source; do
|
||||
case $dl_src in
|
||||
$dl_from_stx_mirror)
|
||||
lvl=$dl_from_stx_mirror
|
||||
;;
|
||||
$dl_from_upstream)
|
||||
lvl=$_level
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown dl_source '$dl_src'"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
download_cmd="$(get_download_cmd $ff $lvl)"
|
||||
|
||||
echo "Looking for $rpm_name"
|
||||
echo "--> run: $download_cmd"
|
||||
if $download_cmd ; then
|
||||
download_url="$(get_url $ff $lvl)"
|
||||
SFILE="$(get_rpm_level_name $rpm_name $lvl)"
|
||||
process_result "$_arch" "$dest_dir" "$download_url" "$SFILE"
|
||||
dl_result=0
|
||||
break
|
||||
else
|
||||
echo "Warning: $rpm_name not found"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $dl_result -eq 1 ]; then
|
||||
echo "Error: $rpm_name not found"
|
||||
echo "missing_srpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $MISSING_SRPMS
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "Already have $dest_dir/$rpm_name"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Function to download different types of RPMs in different ways
|
||||
download () {
|
||||
local _file=$1
|
||||
|
|
@ -194,70 +369,57 @@ download () {
|
|||
|
||||
local _arch=""
|
||||
|
||||
local rc=0
|
||||
local download_cmd=""
|
||||
local download_url=""
|
||||
local rpm_name=""
|
||||
local SFILE=""
|
||||
local lvl
|
||||
local dl_result
|
||||
|
||||
FOUND_ERRORS=0
|
||||
_list=$(cat $_file)
|
||||
_from=$(get_from $_file)
|
||||
|
||||
echo "now the rpm will come from: $_from"
|
||||
for ff in $_list; do
|
||||
_arch=$(get_arch_from_rpm $ff)
|
||||
rpm_name="$(get_rpm_name $ff)"
|
||||
dest_dir="$(get_dest_directory $_arch)"
|
||||
|
||||
if [ ! -e $dest_dir/$rpm_name ]; then
|
||||
dl_result=1
|
||||
for dl_src in $dl_source; do
|
||||
case $dl_src in
|
||||
$dl_from_stx_mirror)
|
||||
lvl=$dl_from_stx_mirror
|
||||
;;
|
||||
$dl_from_upstream)
|
||||
lvl=$_level
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown dl_source '$dl_src'"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
download_cmd="$(get_download_cmd $ff $lvl)"
|
||||
|
||||
echo "Looking for $rpm_name"
|
||||
echo "--> run: $download_cmd"
|
||||
if $download_cmd ; then
|
||||
download_url="$(get_url $ff $lvl)"
|
||||
SFILE="$(get_rpm_level_name $rpm_name $lvl)"
|
||||
process_result "$_arch" "$dest_dir" "$download_url" "$SFILE"
|
||||
dl_result=0
|
||||
break
|
||||
else
|
||||
echo "Warning: $rpm_name not found"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $dl_result -eq 1 ]; then
|
||||
echo "Error: $rpm_name not found"
|
||||
echo "missing_srpm:$rpm_name" >> $LOG
|
||||
echo $rpm_name >> $MISSING_SRPMS
|
||||
rc=1
|
||||
# Free up a worker if none available
|
||||
while [ $workers -ge $max_workers ]; do
|
||||
reaper
|
||||
reaped=$?
|
||||
if [ $reaped -eq 0 ]; then
|
||||
sleep 0.1
|
||||
fi
|
||||
else
|
||||
echo "Already have $dest_dir/$rpm_name"
|
||||
done
|
||||
|
||||
# Allocate a worker. b=the worker index
|
||||
workers=$((workers+1))
|
||||
get_idle_dl_env
|
||||
b=$?
|
||||
if [ $b -ge 255 ]; then
|
||||
echo "get_idle_dl_env failed to find a free slot"
|
||||
exit 1
|
||||
fi
|
||||
echo
|
||||
PREFIX="b$b"
|
||||
|
||||
# Launch a thread in the background
|
||||
( download_worker $b $ff $_level 2>&1 | sed "s#^#${PREFIX}: #" | tee $DL_MIRROR_LOG_DIR/$b; exit ${PIPESTATUS[0]} ) &
|
||||
|
||||
# Record the pid of background process
|
||||
pp=$!
|
||||
set_dl_env_pid $b $pp
|
||||
done
|
||||
|
||||
return $rc
|
||||
# Wait for remaining workers to exit
|
||||
while [ $workers -gt 0 ]; do
|
||||
reaper
|
||||
reaped=$?
|
||||
if [ $reaped -eq 0 ]; then
|
||||
sleep 0.1
|
||||
fi
|
||||
done
|
||||
|
||||
return $FOUND_ERRORS
|
||||
}
|
||||
|
||||
|
||||
# Init the pool of worker threads
|
||||
init_dl_env
|
||||
|
||||
|
||||
# Prime the cache
|
||||
loop_count=0
|
||||
max_loop_count=5
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/bin/bash -e
|
||||
#!/bin/bash
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
|
@ -19,6 +19,11 @@ cleanup () {
|
|||
trap "cleanup ; exit 1" INT HUP TERM QUIT
|
||||
trap "cleanup" EXIT
|
||||
|
||||
# Clear the error log before we begin
|
||||
if [ -f $DL_MIRROR_LOG_DIR/errors ]; then
|
||||
rm -f $DL_MIRROR_LOG_DIR/errors
|
||||
fi
|
||||
|
||||
# A temporary compatability step to save download time
|
||||
# during the shift to the new DL_MIRROR_OUTPUT_DIR location.
|
||||
#
|
||||
|
|
@ -115,8 +120,8 @@ make_stx_mirror_yum_conf="${DOWNLOAD_MIRROR_DIR}/make_stx_mirror_yum_conf.sh"
|
|||
|
||||
# track optional arguments
|
||||
change_group_ids=1
|
||||
use_system_yum_conf=1
|
||||
alternate_yum_conf=""
|
||||
use_system_yum_conf=0
|
||||
alternate_yum_conf="${DOWNLOAD_MIRROR_DIR}/yum.conf.sample"
|
||||
alternate_repo_dir=""
|
||||
rpm_downloader_extra_args=""
|
||||
tarball_downloader_extra_args=""
|
||||
|
|
@ -166,6 +171,8 @@ dl_from_upstream () {
|
|||
|
||||
|
||||
MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U"
|
||||
TEMP_DIR=""
|
||||
TEMP_DIR_CLEANUP=""
|
||||
|
||||
multiple_dl_flag_check () {
|
||||
if [ "$dl_flag" != "" ]; then
|
||||
|
|
@ -177,7 +184,7 @@ multiple_dl_flag_check () {
|
|||
|
||||
|
||||
# Parse out optional arguments
|
||||
while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do
|
||||
while getopts "c:Cd:ghI:sl:L:nt:ySuUW:" o; do
|
||||
case "${o}" in
|
||||
c)
|
||||
# Pass -c ("use alternate dnf.conf") to rpm downloader
|
||||
|
|
@ -214,6 +221,15 @@ while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do
|
|||
rpm_downloader_extra_args="${rpm_downloader_extra_args} -n"
|
||||
SUDO=""
|
||||
;;
|
||||
t)
|
||||
# Set TEMP_DIR
|
||||
TEMP_DIR="${OPTARG}"
|
||||
;;
|
||||
y)
|
||||
# Use hosts /etc/yum.conf
|
||||
use_system_yum_conf=1
|
||||
alternate_yum_conf=""
|
||||
;;
|
||||
s)
|
||||
# Download from StarlingX mirror only. Do not use upstream sources.
|
||||
multiple_dl_flag_check
|
||||
|
|
@ -383,16 +399,15 @@ echo "step #0: Configuring yum repos ..."
|
|||
|
||||
if [ ${use_system_yum_conf} -ne 0 ]; then
|
||||
# Restore StarlingX_3rd repos from backup
|
||||
REPO_SOURCE_DIR=/localdisk/yum.repos.d
|
||||
REPO_DIR=/etc/yum.repos.d
|
||||
if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
|
||||
${SUDO} \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
|
||||
fi
|
||||
|
||||
if [ $layer != "all" ]; then
|
||||
if [ -d ${config_dir}/${distro}/${layer}/yum.repos.d ]; then
|
||||
${SUDO} \cp -f ${config_dir}/${distro}/${layer}/yum.repos.d/*.repo $REPO_DIR
|
||||
${SUDO} \cp -f -v ${config_dir}/${distro}/${layer}/yum.repos.d/*.repo $REPO_DIR/
|
||||
fi
|
||||
else
|
||||
# copy all layers
|
||||
${SUDO} \cp -f -v ${config_dir}/${distro}/*/yum.repos.d/*.repo $REPO_DIR/
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
@ -411,7 +426,6 @@ if [ $use_system_yum_conf -eq 0 ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
TEMP_DIR=""
|
||||
rpm_downloader_extra_args="${rpm_downloader_extra_args} -D $distro"
|
||||
|
||||
if [ "$dl_flag" != "" ]; then
|
||||
|
|
@ -428,8 +442,20 @@ if ! dl_from_stx; then
|
|||
else
|
||||
# We want to use stx mirror, so we need to create a new, modified dnf.conf and yum.repos.d.
|
||||
# The modifications will add or substitute repos pointing to the StralingX mirror.
|
||||
TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX)
|
||||
TEMP_CONF="$TEMP_DIR/dnf.conf"
|
||||
if [ "$TEMP_DIR" == "" ]; then
|
||||
if [ "$MY_WORKSPACE" != "" ]; then
|
||||
TEMP_DIR="$MY_WORKSPACE/tmp/yum"
|
||||
else
|
||||
TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX)
|
||||
TEMP_DIR_CLEANUP="y"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -d $TEMP_DIR ]; then
|
||||
mkdir -p ${TEMP_DIR}
|
||||
fi
|
||||
|
||||
TEMP_CONF="$TEMP_DIR/yum.conf"
|
||||
need_file ${make_stx_mirror_yum_conf}
|
||||
need_dir ${TEMP_DIR}
|
||||
|
||||
|
|
@ -685,7 +711,7 @@ fi
|
|||
#
|
||||
# Clean up the mktemp directory, if required.
|
||||
#
|
||||
if [ "$TEMP_DIR" != "" ]; then
|
||||
if [ "$TEMP_DIR" != "" ] && [ "$TEMP_DIR_CLEANUP" == "y" ]; then
|
||||
echo "${SUDO} rm -rf $TEMP_DIR"
|
||||
${SUDO} \rm -rf "$TEMP_DIR"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ MAKE_STX_MIRROR_DNF_CONF_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
|
|||
source "$MAKE_STX_MIRROR_DNF_CONF_DIR/url_utils.sh"
|
||||
|
||||
DISTRO="centos"
|
||||
SUDO=sudo
|
||||
|
||||
TEMP_DIR=""
|
||||
SRC_REPO_DIR="$MAKE_STX_MIRROR_DNF_CONF_DIR/yum.repos.d"
|
||||
|
|
@ -38,6 +39,7 @@ usage () {
|
|||
echo " 'yum.repos.d' in same directory as this script"
|
||||
echo "-l <layer> = Download only packages required to build a given layer"
|
||||
echo "-u <lower-layer>,<build-type>,<repo_url> = Add/change the repo baseurl for a lower layer"
|
||||
echo "-n don't use sudo"
|
||||
}
|
||||
|
||||
declare -A layer_urls
|
||||
|
|
@ -61,7 +63,7 @@ set_layer_urls () {
|
|||
#
|
||||
# option processing
|
||||
#
|
||||
while getopts "D:d:l:Rr:u:y:" o; do
|
||||
while getopts "D:d:l:nRr:u:y:" o; do
|
||||
case "${o}" in
|
||||
D)
|
||||
DISTRO="${OPTARG}"
|
||||
|
|
@ -72,6 +74,9 @@ while getopts "D:d:l:Rr:u:y:" o; do
|
|||
l)
|
||||
LAYER="${OPTARG}"
|
||||
;;
|
||||
n)
|
||||
SUDO=""
|
||||
;;
|
||||
r)
|
||||
SRC_REPO_DIR="${OPTARG}"
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -40,14 +40,43 @@ ERROR_LOG_FILE="mirror-check-failures.log"
|
|||
truncate -s 0 $ERROR_LOG_FILE
|
||||
retcode=0
|
||||
extra_opts=""
|
||||
layer="$LAYER"
|
||||
valid_layers=('compiler' 'distro' 'flock')
|
||||
|
||||
|
||||
# Cloned from cgcs-root/build-tools/pkg-manager-utils.sh
|
||||
# Ideally this can still be used when tools is the only git
|
||||
# that has been cloned.
|
||||
|
||||
# Yum vs DNF compatibility
|
||||
YUM=$(which yum 2>> /dev/null)
|
||||
DNF=$(which dnf 2>> /dev/null)
|
||||
PKG_MANAGER=""
|
||||
REPOQUERY=$(which repoquery 2>> /dev/null)
|
||||
REPOQUERY_SUB_COMMAND=""
|
||||
REPOQUERY_RESOLVE="--resolve"
|
||||
REPOQUERY_WHATPROVIDES_DELIM=" "
|
||||
if [ ! -z ${DNF} ]; then
|
||||
PKG_MANAGER="dnf"
|
||||
REPOQUERY=${DNF}
|
||||
REPOQUERY_SUB_COMMAND="repoquery --disable-modular-filtering"
|
||||
REPOQUERY_RESOLVE=""
|
||||
REPOQUERY_WHATPROVIDES_DELIM=","
|
||||
elif [ ! -z ${YUM} ]; then
|
||||
PKG_MANAGER="yum"
|
||||
else
|
||||
>&2 echo "ERROR: Couldn't find a supported package manager"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
usage() {
|
||||
echo "$0 [-c <dnf.conf>]"
|
||||
echo "$0 [-c <dnf.conf>] [-l <layer>]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -c: Use an alternate dnf.conf rather than the system file (option passed"
|
||||
echo " on to subscripts when appropriate)"
|
||||
echo " -l: Check specific layer (one of 'all ${valid_layers[@]}')"
|
||||
echo ""
|
||||
}
|
||||
|
||||
|
|
@ -79,8 +108,10 @@ get_repoquery_info() {
|
|||
else
|
||||
repoquery_opts=
|
||||
fi
|
||||
repoquery $extra_opts ${RELEASEVER} -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \
|
||||
$repoquery_opts "$_package_name"
|
||||
$REPOQUERY $REPOQUERY_SUB_COMMAND \
|
||||
$extra_opts ${RELEASEVER} -C \
|
||||
--qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \
|
||||
$repoquery_opts "$_package_name"
|
||||
}
|
||||
|
||||
_check_rpms() {
|
||||
|
|
@ -113,12 +144,27 @@ check_rpms() {
|
|||
done
|
||||
}
|
||||
|
||||
while getopts "c:" opt; do
|
||||
while getopts "c:l:" opt; do
|
||||
case $opt in
|
||||
c)
|
||||
extra_opts="-c ${OPTARG}"
|
||||
grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})"
|
||||
;;
|
||||
l)
|
||||
layer="${OPTARG}"
|
||||
if [ "$layer" == "all" ]; then
|
||||
layer=""
|
||||
else
|
||||
case " ${valid_layers[@]} " in
|
||||
*" $layer "* ) echo "found layer $layer"
|
||||
;;
|
||||
*) echo "'$layer' is invalid"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
;;
|
||||
\?)
|
||||
echo "Invalid option: -$OPTARG" >&2
|
||||
usage
|
||||
|
|
@ -133,11 +179,11 @@ if ! dnf $extra_opts ${RELEASEVER} makecache; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
for rpm_list in "$RPMS_CENTOS_LIST" "$RPMS_3RD_PARTY_LIST"; do
|
||||
for rpm_list in $(find config/centos/$layer -name "$RPMS_CENTOS_LIST" -o -name "$RPMS_3RD_PARTY_LIST"); do
|
||||
info "Reading $rpm_list..."
|
||||
for arch in "src" "noarch" "x86_64"; do
|
||||
info "Getting info for $arch packages..."
|
||||
rpms=$(echo "$(grep -F "$arch.rpm" < $rpm_list)")
|
||||
rpms=$(echo "$(grep -v '^#' $rpm_list | grep -F "$arch.rpm")")
|
||||
check_rpms "$rpms"
|
||||
done
|
||||
done
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ get_url() {
|
|||
# filter urls for the desitered arch.
|
||||
local arr=( $(split_filename $_name) )
|
||||
local arch=${arr[3]}
|
||||
_ret="$($_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)"
|
||||
_ret="$(${SUDO} $_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)"
|
||||
fi
|
||||
echo "$_ret"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,6 +86,13 @@ gpgcheck=1
|
|||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
|
||||
enabled=1
|
||||
|
||||
[StarlingX-C7.9.2009-rt-source]
|
||||
name=StarlingX-CentOS-7.9.2009 - rt-source
|
||||
baseurl=https://vault.centos.org/centos/7.9.2009/rt/Source/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
|
||||
enabled=1
|
||||
|
||||
[StarlingX-C7.9.2009-rt]
|
||||
name=StarlingX-CentOS-7.9.2009 - rt
|
||||
baseurl=http://mirror.centos.org/centos/7.9.2009/rt/x86_64/
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@
|
|||
[Starlingx-linuxsoft.cern.ch_cern_centos_7.5_rt_Sources]
|
||||
name=Starlingx-linuxsoft.cern.ch_cern_centos_7.5_rt_Sources - linuxsoft.cern.ch_cern_centos_7.5_rt_Sources
|
||||
baseurl=http://linuxsoft.cern.ch/cern/centos/7.5/rt/Sources/
|
||||
enabled=1
|
||||
enabled=0
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[centos-openstack-queens]
|
||||
name=CentOS-7 - OpenStack queens
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/
|
||||
gpgcheck=1
|
||||
enabled=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -13,14 +13,14 @@ exclude=sip,PyQt4
|
|||
|
||||
[centos-openstack-queens-test]
|
||||
name=CentOS-7 - OpenStack queens Testing
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-queens/
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-queens/
|
||||
gpgcheck=0
|
||||
enabled=0
|
||||
exclude=sip,PyQt4
|
||||
|
||||
[centos-openstack-queens-debuginfo]
|
||||
name=CentOS-7 - OpenStack queens - Debug
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[centos-openstack-rocky]
|
||||
name=CentOS-7 - OpenStack rocky
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-rocky/
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-rocky/
|
||||
gpgcheck=1
|
||||
enabled=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -13,14 +13,14 @@ exclude=sip,PyQt4
|
|||
|
||||
[centos-openstack-rocky-test]
|
||||
name=CentOS-7 - OpenStack rocky Testing
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-rocky/
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-rocky/
|
||||
gpgcheck=0
|
||||
enabled=0
|
||||
exclude=sip,PyQt4
|
||||
|
||||
[centos-openstack-rocky-debuginfo]
|
||||
name=CentOS-7 - OpenStack rocky - Debug
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[centos-openstack-stein]
|
||||
name=CentOS-7 - OpenStack stein
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-stein/
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-stein/
|
||||
gpgcheck=1
|
||||
enabled=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -13,14 +13,14 @@ exclude=sip,PyQt4
|
|||
|
||||
[centos-openstack-stein-test]
|
||||
name=CentOS-7 - OpenStack stein Testing
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-stein/
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-stein/
|
||||
gpgcheck=0
|
||||
enabled=0
|
||||
exclude=sip,PyQt4
|
||||
|
||||
[centos-openstack-stein-debuginfo]
|
||||
name=CentOS-7 - OpenStack stein - Debug
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[centos-openstack-train]
|
||||
name=CentOS-7 - OpenStack train
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-train/
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-train/
|
||||
gpgcheck=1
|
||||
enabled=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -13,14 +13,14 @@ exclude=sip,PyQt4
|
|||
|
||||
[centos-openstack-train-test]
|
||||
name=CentOS-7 - OpenStack train Testing
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-train/
|
||||
baseurl=https://buildlogs.centos.org/centos/7/cloud/x86_64/openstack-train/
|
||||
gpgcheck=0
|
||||
enabled=0
|
||||
exclude=sip,PyQt4
|
||||
|
||||
[centos-openstack-train-debuginfo]
|
||||
name=CentOS-7 - OpenStack train - Debug
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/
|
||||
baseurl=http://debuginfo.centos.org/centos/7/cloud/x86_64/
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
|
|||
|
|
@ -14,6 +14,16 @@ import sys
|
|||
import os
|
||||
from lp import find_lp_assigned
|
||||
|
||||
cves_valid = []
|
||||
cves_to_fix = []
|
||||
cves_to_fix_lp = []
|
||||
cves_to_track = []
|
||||
cves_w_errors = []
|
||||
cves_wont_fix = []
|
||||
cves_to_omit = []
|
||||
cves_report = {}
|
||||
|
||||
|
||||
def print_html_report(cves_report, title):
|
||||
"""
|
||||
Print the html report
|
||||
|
|
@ -22,9 +32,14 @@ def print_html_report(cves_report, title):
|
|||
|
||||
template_loader = jinja2.FileSystemLoader(searchpath="./")
|
||||
template_env = jinja2.Environment(loader=template_loader)
|
||||
template_file = "template.txt"
|
||||
if CVSS_VER == "cvssv3":
|
||||
template_file = "template_v3.txt"
|
||||
heads = ["cve_id", "status", "cvss3Score", "av", "ac", "ui","a"]
|
||||
else:
|
||||
template_file = "template.txt"
|
||||
heads = ["cve_id", "status", "cvss2Score", "av", "ac", "au", "ai"]
|
||||
|
||||
template = template_env.get_template(template_file)
|
||||
heads = ["cve_id", "status", "cvss2Score", "av", "ac", "au", "ai"]
|
||||
output_text = template.render(cves_to_fix=cves_report["cves_to_fix"],\
|
||||
cves_to_fix_lp=cves_report["cves_to_fix_lp"],\
|
||||
cves_to_track=cves_report["cves_to_track"],\
|
||||
|
|
@ -49,10 +64,16 @@ def print_report(cves_report, title):
|
|||
print("\n")
|
||||
print(cve["id"])
|
||||
print("status : " + cve["status"])
|
||||
print("cvss2Score : " + str(cve["cvss2Score"]))
|
||||
if CVSS_VER == "cvssv3":
|
||||
print("cvss3Score : " + str(cve["cvss3Score"]))
|
||||
else:
|
||||
print("cvss2Score : " + str(cve["cvss2Score"]))
|
||||
print("Attack Vector: " + cve["av"])
|
||||
print("Access Complexity : " + cve["ac"])
|
||||
print("Authentication: " + cve["au"])
|
||||
if CVSS_VER == "cvssv3":
|
||||
print("User Interaction: " + cve["ui"])
|
||||
else:
|
||||
print("Authentication: " + cve["au"])
|
||||
print("Availability Impact :" + cve["ai"])
|
||||
print("Affected packages:")
|
||||
print(cve["affectedpackages"])
|
||||
|
|
@ -88,8 +109,12 @@ def print_report(cves_report, title):
|
|||
print(cve_line)
|
||||
|
||||
|
||||
print("\nERROR: CVEs that have no cvss2Score or cvss2Vector: %d \n" \
|
||||
% (len(cves_report["cves_w_errors"])))
|
||||
if CVSS_VER == "cvssv3":
|
||||
print("\nERROR: CVEs that have no cvss3Score or cvss3Vector: %d \n" \
|
||||
% (len(cves_report["cves_w_errors"])))
|
||||
else:
|
||||
print("\nERROR: CVEs that have no cvss2Score or cvss2Vector: %d \n" \
|
||||
% (len(cves_report["cves_w_errors"])))
|
||||
for cve in cves_report["cves_w_errors"]:
|
||||
print(cve)
|
||||
|
||||
|
|
@ -130,49 +155,112 @@ def get_affectedpackages(data, cve_id):
|
|||
allfixed = "unfixed"
|
||||
return affectedpackages_list, allfixed
|
||||
|
||||
def main():
|
||||
def update_report():
|
||||
cves_report["cves_to_fix"] = cves_to_fix
|
||||
cves_report["cves_to_fix_lp"] = cves_to_fix_lp
|
||||
cves_report["cves_to_track"] = cves_to_track
|
||||
cves_report["cves_w_errors"] = cves_w_errors
|
||||
cves_report["cves_wont_fix"] = cves_wont_fix
|
||||
cves_report["cves_to_omit"] = cves_to_omit
|
||||
|
||||
def cvssv3_pb_alg():
|
||||
"""
|
||||
main function
|
||||
Rules to consider a CVE valid for STX from:
|
||||
https://wiki.openstack.org/wiki/StarlingX/Security/CVE_Support_Policy
|
||||
Patchback algo for CVSSV3 report
|
||||
"""
|
||||
data = {}
|
||||
cves = []
|
||||
cves_valid = []
|
||||
cves_to_fix = []
|
||||
cves_to_fix_lp = []
|
||||
cves_to_track = []
|
||||
cves_w_errors = []
|
||||
cves_wont_fix = []
|
||||
cves_to_omit = []
|
||||
cves_report = {}
|
||||
for cve in cves_valid:
|
||||
if (cve["cvss3Score"] >= 7.8
|
||||
and cve["av"] == "N"
|
||||
and cve["ac"] == "L"
|
||||
and cve["ui"] == "R"
|
||||
and cve["ai"] != "N"):
|
||||
if cve["status"] == "fixed":
|
||||
bug = find_lp_assigned(cve["id"])
|
||||
if (bug):
|
||||
print(bug["status"])
|
||||
if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"):
|
||||
cves_wont_fix.append(cve)
|
||||
else:
|
||||
cves_to_fix_lp.append(cve)
|
||||
else:
|
||||
cves_to_fix.append(cve)
|
||||
else:
|
||||
cves_to_track.append(cve)
|
||||
else:
|
||||
cves_to_omit.append(cve)
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("\nERROR : Missing arguments, the expected arguments are:")
|
||||
print("\n %s <result.json> <title>\n" % (sys.argv[0]))
|
||||
print("\n result.json = json file generated from: vuls report -format-json")
|
||||
print("\n")
|
||||
sys.exit(0)
|
||||
update_report()
|
||||
|
||||
if os.path.isfile(sys.argv[1]):
|
||||
results_json = sys.argv[1]
|
||||
else:
|
||||
print("%s is not a file" % sys.argv[1])
|
||||
sys.exit(0)
|
||||
|
||||
title = sys.argv[2]
|
||||
def cvssv2_pb_alg():
|
||||
"""
|
||||
Patchback algo for CVSSV2 report
|
||||
"""
|
||||
for cve in cves_valid:
|
||||
if (cve["cvss2Score"] >= 7.0
|
||||
and cve["av"] == "N"
|
||||
and cve["ac"] == "L"
|
||||
and ("N" in cve["au"] or "S" in cve["au"])
|
||||
and ("P" in cve["ai"] or "C" in cve["ai"])):
|
||||
if cve["status"] == "fixed":
|
||||
bug = find_lp_assigned(cve["id"])
|
||||
if (bug):
|
||||
print(bug["status"])
|
||||
if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"):
|
||||
cves_wont_fix.append(cve)
|
||||
else:
|
||||
cves_to_fix_lp.append(cve)
|
||||
else:
|
||||
cves_to_fix.append(cve)
|
||||
else:
|
||||
cves_to_track.append(cve)
|
||||
else:
|
||||
cves_to_omit.append(cve)
|
||||
|
||||
try:
|
||||
with open(results_json) as json_file:
|
||||
data = json.load(json_file)
|
||||
except ValueError as error:
|
||||
print(error)
|
||||
update_report()
|
||||
|
||||
for element in data["scannedCves"]:
|
||||
cve = {}
|
||||
cve["id"] = str(element.strip())
|
||||
cves.append(cve)
|
||||
def cvssv3_parse_n_report(cves,title,data):
|
||||
"""
|
||||
Parse and generate report for CVSSV3
|
||||
"""
|
||||
for cve in cves:
|
||||
cve_id = cve["id"]
|
||||
affectedpackages_list = []
|
||||
allfixed = "fixed"
|
||||
try:
|
||||
nvd2_score = data["scannedCves"][cve_id]["cveContents"]["nvd"]["cvss3Score"]
|
||||
cvss3vector = data["scannedCves"][cve_id]["cveContents"]["nvd"]["cvss3Vector"]
|
||||
except KeyError:
|
||||
cves_w_errors.append(cve)
|
||||
else:
|
||||
cve["cvss3Score"] = nvd2_score
|
||||
for element in cvss3vector.split("/"):
|
||||
if "AV:" in element:
|
||||
_av = element.split(":")[1]
|
||||
if "AC:" in element:
|
||||
_ac = element.split(":")[1]
|
||||
if "A:" in element:
|
||||
_ai = element.split(":")[1]
|
||||
if "UI:" in element:
|
||||
_ui = element.split(":")[1]
|
||||
print(cve)
|
||||
cve["av"] = str(_av)
|
||||
cve["ac"] = str(_ac)
|
||||
cve["ai"] = str(_ai)
|
||||
cve["ui"] = str(_ui)
|
||||
cve["summary"] = get_summary(data, cve_id)
|
||||
cve["sourcelink"] = get_source_link(data, cve_id)
|
||||
affectedpackages_list, allfixed = get_affectedpackages(data, cve_id)
|
||||
cve["affectedpackages"] = affectedpackages_list
|
||||
cve["status"] = allfixed
|
||||
cves_valid.append(cve)
|
||||
cvssv3_pb_alg()
|
||||
print_report(cves_report, title)
|
||||
print_html_report(cves_report, title)
|
||||
|
||||
def cvssv2_parse_n_report(cves,title,data):
|
||||
"""
|
||||
Parse and generate report for CVSSV2
|
||||
"""
|
||||
for cve in cves:
|
||||
cve_id = cve["id"]
|
||||
affectedpackages_list = []
|
||||
|
|
@ -203,37 +291,55 @@ def main():
|
|||
cve["affectedpackages"] = affectedpackages_list
|
||||
cve["status"] = allfixed
|
||||
cves_valid.append(cve)
|
||||
|
||||
for cve in cves_valid:
|
||||
if (cve["cvss2Score"] >= 7.0
|
||||
and cve["av"] == "N"
|
||||
and cve["ac"] == "L"
|
||||
and ("N" in cve["au"] or "S" in cve["au"])
|
||||
and ("P" in cve["ai"] or "C" in cve["ai"])):
|
||||
if cve["status"] == "fixed":
|
||||
bug = find_lp_assigned(cve["id"])
|
||||
if (bug):
|
||||
print(bug["status"])
|
||||
if (bug["status"] == "Invalid" or bug["status"] == "Won't Fix"):
|
||||
cves_wont_fix.append(cve)
|
||||
else:
|
||||
cves_to_fix_lp.append(cve)
|
||||
else:
|
||||
cves_to_fix.append(cve)
|
||||
else:
|
||||
cves_to_track.append(cve)
|
||||
else:
|
||||
cves_to_omit.append(cve)
|
||||
|
||||
cves_report["cves_to_fix"] = cves_to_fix
|
||||
cves_report["cves_to_fix_lp"] = cves_to_fix_lp
|
||||
cves_report["cves_to_track"] = cves_to_track
|
||||
cves_report["cves_w_errors"] = cves_w_errors
|
||||
cves_report["cves_wont_fix"] = cves_wont_fix
|
||||
cves_report["cves_to_omit"] = cves_to_omit
|
||||
|
||||
cvssv2_pb_alg()
|
||||
print_report(cves_report, title)
|
||||
print_html_report(cves_report, title)
|
||||
|
||||
def main():
|
||||
"""
|
||||
main function
|
||||
Rules to consider a CVE valid for STX from:
|
||||
https://wiki.openstack.org/wiki/StarlingX/Security/CVE_Support_Policy
|
||||
"""
|
||||
data = {}
|
||||
cves = []
|
||||
|
||||
|
||||
if len(sys.argv) < 4:
|
||||
print("\nERROR : Missing arguments, the expected arguments are:")
|
||||
print("\n %s <result.json> <title> [cvssv3|cvssv2]\n" % (sys.argv[0]))
|
||||
print("\n result.json = json file generated from: vuls report -format-json")
|
||||
print("\n")
|
||||
sys.exit(0)
|
||||
|
||||
if os.path.isfile(sys.argv[1]):
|
||||
results_json = sys.argv[1]
|
||||
else:
|
||||
print("%s is not a file" % sys.argv[1])
|
||||
sys.exit(0)
|
||||
|
||||
title = sys.argv[2]
|
||||
|
||||
try:
|
||||
with open(results_json) as json_file:
|
||||
data = json.load(json_file)
|
||||
except ValueError as error:
|
||||
print(error)
|
||||
|
||||
for element in data["scannedCves"]:
|
||||
cve = {}
|
||||
cve["id"] = str(element.strip())
|
||||
cves.append(cve)
|
||||
global CVSS_VER
|
||||
CVSS_VER=sys.argv[3].lower()
|
||||
if CVSS_VER =="cvssv3":
|
||||
cvssv3_parse_n_report(cves,title,data)
|
||||
elif CVSS_VER == "cvssv2":
|
||||
cvssv2_parse_n_report(cves,title,data)
|
||||
else:
|
||||
print("\n argument not matching \n enter [cvssv3|cvssv2] ")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
<head></head>
|
||||
<body>
|
||||
<h1>Security report from vuls scan from {{title}}</h1>
|
||||
<h2>CVEs to fix w/o a launchpad assigned: {{cves_to_fix | length}}</h2>
|
||||
<table>
|
||||
{% if cves_to_fix|length >= 1 %}
|
||||
<tr>
|
||||
{% for head in heads %}
|
||||
<th>{{head}}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
{% for cve in cves_to_fix %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
<td>{{cve["cvss3Score"]}}</td>
|
||||
<td>{{cve["av"]}}</td>
|
||||
<td>{{cve["ac"]}}</td>
|
||||
<td>{{cve["ui"]}}</td>
|
||||
<td>{{cve["ai"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
<h2>CVEs to fix w/ a launchpad assigend: {{cves_to_fix_lp | length}}</h2>
|
||||
<table>
|
||||
{% if cves_to_fix_lp|length >= 1 %}
|
||||
<tr>
|
||||
{% for head in heads %}
|
||||
<th>{{head}}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
{% for cve in cves_to_fix_lp %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
<td>{{cve["cvss3Score"]}}</td>
|
||||
<td>{{cve["av"]}}</td>
|
||||
<td>{{cve["ac"]}}</td>
|
||||
<td>{{cve["ui"]}}</td>
|
||||
<td>{{cve["ai"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
<h2> CVEs to track for incoming fix: {{cves_to_track | length}}</h2>
|
||||
<table>
|
||||
{% if cves_to_track|length >= 1 %}
|
||||
<tr>
|
||||
{% for head in heads %}
|
||||
<th>{{head}}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
{% for cve in cves_to_track %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
<td>{{cve["cvss3Score"]}}</td>
|
||||
<td>{{cve["av"]}}</td>
|
||||
<td>{{cve["ac"]}}</td>
|
||||
<td>{{cve["ui"]}}</td>
|
||||
<td>{{cve["ai"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
<h2> CVEs that are Invalid or Won't Fix: {{cves_wont_fix | length}}</h2>
|
||||
<table>
|
||||
{% if cves_wont_fix|length >= 1 %}
|
||||
<tr>
|
||||
{% for head in heads %}
|
||||
<th>{{head}}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
{% for cve in cves_wont_fix %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
<td>{{cve["cvss3Score"]}}</td>
|
||||
<td>{{cve["av"]}}</td>
|
||||
<td>{{cve["ac"]}}</td>
|
||||
<td>{{cve["ui"]}}</td>
|
||||
<td>{{cve["ai"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
|
||||
<h2> CVEs to omit: {{cves_to_omit | length}}</h2>
|
||||
<table>
|
||||
{% if cves_to_omit|length >= 1 %}
|
||||
<tr>
|
||||
{% for head in heads %}
|
||||
<th>{{head}}</th>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
|
||||
{% for cve in cves_to_omit %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
<td>{{cve["cvss3Score"]}}</td>
|
||||
<td>{{cve["av"]}}</td>
|
||||
<td>{{cve["ac"]}}</td>
|
||||
<td>{{cve["ui"]}}</td>
|
||||
<td>{{cve["ai"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
|
||||
<h2>ERROR: CVEs that have no cvss3Score or cvss2Vector:{{cves_w_errors | length}}</h2>
|
||||
<table>
|
||||
{% if cves_w_errors|length >= 1 %}
|
||||
{% for cve in cves_w_errors %}
|
||||
<tr>
|
||||
<td>{{cve["id"]}}</td>
|
||||
<td>{{cve["status"]}}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</table>
|
||||
</body>
|
||||
13
tb.sh
13
tb.sh
|
|
@ -24,12 +24,17 @@ CMD=$1
|
|||
TC_CONTAINER_NAME=${MYUNAME}-centos-builder
|
||||
TC_CONTAINER_TAG=local/${MYUNAME}-stx-builder:8.2.2004
|
||||
TC_DOCKERFILE=Dockerfile
|
||||
NO_CACHE=0
|
||||
|
||||
function create_container {
|
||||
local EXTRA_ARGS=""
|
||||
|
||||
if [ ! -z ${MY_EMAIL} ]; then
|
||||
EXTRA_ARGS="--build-arg MY_EMAIL=${MY_EMAIL}"
|
||||
EXTRA_ARGS+="--build-arg MY_EMAIL=${MY_EMAIL}"
|
||||
fi
|
||||
|
||||
if [ $NO_CACHE -eq 1 ]; then
|
||||
EXTRA_ARGS+=" --no-cache"
|
||||
fi
|
||||
|
||||
docker build \
|
||||
|
|
@ -87,7 +92,7 @@ function clean_container {
|
|||
}
|
||||
|
||||
function usage {
|
||||
echo "$0 [create|run|exec|env|stop|kill|clean]"
|
||||
echo "$0 [create|create_no_cache|run|exec|env|stop|kill|clean]"
|
||||
}
|
||||
|
||||
case $CMD in
|
||||
|
|
@ -109,6 +114,10 @@ case $CMD in
|
|||
create)
|
||||
create_container
|
||||
;;
|
||||
create_no_cache)
|
||||
NO_CACHE=1
|
||||
create_container
|
||||
;;
|
||||
exec)
|
||||
exec_container
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -52,7 +52,10 @@ merge_lst () {
|
|||
return 1
|
||||
fi
|
||||
|
||||
layers=$(cat ${layer_cfgs} | sort --unique)
|
||||
# Grep to ignore empty lines or whole line comments.
|
||||
# Sed to drop any trailing comments.
|
||||
# Side effect of grep over cat is adding any missing EOL.
|
||||
layers=$(grep -h -v -e '^$' -e '^[ \t]*#' ${layer_cfgs} | sed -e 's/[ \t]*#.*$//' | sort --unique)
|
||||
layers+=" mock"
|
||||
|
||||
(
|
||||
|
|
|
|||
Loading…
Reference in New Issue