diff --git a/.gitignore b/.gitignore index 766c4b1c..acb93792 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,9 @@ localrc toCOPY/.gitconfig centos-mirror-tools/logs/ centos-mirror-tools/output/ + +# Sphinx documentation +docs/build/ + +# Release Notes documentation +releasenotes/build diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 83d040b1..1c5a6060 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -31,6 +31,11 @@ for ff in $all; do if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name else + if [ -e "$save_path/$_name" ]; then + echo "Already have $save_path/$_name" + continue + fi + echo "remote path: $url_prefix/$_name" echo "local path: $save_path/$_name" if wget $url_prefix/$_name; then diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 767761a6..29c0ae48 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -1,25 +1,86 @@ #!/bin/bash -e +# +# SPDX-License-Identifier: Apache-2.0 +# # download RPMs/SRPMs from different sources. # this script was originated by Brian Avery, and later updated by Yong Hu -if [ $# -lt 3 ]; then - echo "$0 " - echo "rpm_list: a list of RPM files to be downloaded." - echo "match_level: value could be L1, L2 or L3:" - echo " L1: use name, major version and minor version:" - echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" - echo " L2: use name and major version:" - echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" - echo " L3: use name:" - echo " using vim to search vim-7.4.160-2.el7.src.rpm" - echo "from_where: where to download the RPMs: 'centos'from CentOS Repos," - echo "otherwise from 3rd-party websets" - exit -1 +usage() { + echo "$0 [-n] [-c ] " + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations" + echo " -c: Use an alternate yum.conf rather than the system file" + echo " -x: Clean log files only, do not run." + echo " rpm_list: a list of RPM files to be downloaded." + echo " match_level: value could be L1, L2 or L3:" + echo " L1: use name, major version and minor version:" + echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" + echo " L2: use name and major version:" + echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" + echo " L3: use name:" + echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " K1: Use Koji rather than yum repos as a source." + echo " Koji has a longer retention period than epel mirrors." + echo "" + echo "Returns: 0 = All files downloaded successfully" + echo " 1 = Some files could not be downloaded" + echo " 2 = Bad arguements or other error" + echo "" +} + +get_from() { + list=$1 + base=$(basename $list .lst) # removing lst extension + base=$(basename $base .log) # removing log extension + from=$(echo $base | rev | cut -d'_' -f1-1 | rev) + echo $from +} + +# By default, we use "sudo" and we don't use a local yum.conf. These can +# be overridden via flags. +SUDOCMD="sudo -E" +YUMCONFOPT="" + +CLEAN_LOGS_ONLY=0 +dl_rc=0 + +# Parse option flags +while getopts "c:nxh" o; do + case "${o}" in + n) + # No-sudo + SUDOCMD="" + ;; + x) + # Clean only + CLEAN_LOGS_ONLY=1 + ;; + c) + # Use an alternate yum.conf + YUMCONFOPT="-c $OPTARG" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 2 + ;; + esac +done +shift $((OPTIND-1)) + +if [ $# -lt 2 ]; then + usage + exit 2 fi if [ "$1" == "" ]; then echo "Need to supply the rpm file list" - exit -1; + exit 2; else rpms_list=$1 echo "using $rpms_list as the download name lists" @@ -31,142 +92,222 @@ if [ ! -z "$2" -a "$2" != " " ];then match_level=$2 fi -from=$3 timestamp=$(date +%F_%H%M) echo $timestamp + + DESTDIR="output" MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source mkdir -p $MDIR_SRC MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary mkdir -p $MDIR_BIN -FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt" -FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt" -MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt" -URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt" - -cat /dev/null > $FAIL_MOVE_SRPMS -cat /dev/null > $FOUND_SRPMS +LOGSDIR="logs" +from=$(get_from $rpms_list) +LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log" +MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log" +MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log" +FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log" +FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log" +cat /dev/null > $LOG cat /dev/null > $MISSING_SRPMS -cat /dev/null > $URL_SRPMS - -FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt" -FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt" -MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt" -URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt" - -cat /dev/null > $FAIL_MOVE_RPMS -cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS -cat /dev/null > $URL_RPMS +cat /dev/null > $FOUND_SRPMS +cat /dev/null > $FOUND_RPMS -#function to download different type of RPMs in different ways + +if [ $CLEAN_LOGS_ONLY -eq 1 ];then + exit 0 +fi + +# Function to split an rpm filename into parts. +# +# Returns a space seperated list containing: +# +# +split_filename () { + local rpm_filename=$1 + + local RPM="" + local SFILE="" + local ARCH="" + local RELEASE="" + local VERSION="" + local NAME="" + + RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) + ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) + RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + + if [[ $NAME = *":"* ]]; then + EPOCH=$(echo $NAME | cut -d':' -f-1) + NAME=$(echo $NAME | cut -d':' -f2-) + fi + + echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" +} + +# Function to predict the URL where a rpm might be found. +# Assumes the rpm was compile for EPEL by fedora's koji. +koji_url () { + local rpm_filename=$1 + + local arr=( $(split_filename $rpm_filename) ) + + local n=${arr[0]} + local v=${arr[1]} + local r=${arr[2]} + local a=${arr[3]} + local e=${arr[4]} + + echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" +} + +# Function to download different types of RPMs in different ways download () { - _list=$1 - _level=$2 - _from=$3 - _type=$4 + local _file=$1 + local _level=$2 + + local _list + local _from + local _type="" + + local rc=0 + local download_cmd="" + local download_url_cmd="" + local rpm_name="" + local rpm_url="" + local SFILE="" + + _list=$(cat $_file) + _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do - ## download RPM from CentOS repos - if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then + download_cmd="" + download_url_cmd="" + _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) + + # Decide if the list will be downloaded using yumdownloader or wget + if [[ $ff != *"#"* ]]; then rpm_name=$ff - if [ $_level == "L1" ]; then + + if [ $_level == "K1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - elif [ $match_level == "L2" ];then - SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + rpm_url=$(koji_url $rpm_name) + download_cmd="wget $rpm_url)" + download_url_cmd="echo $rpm_url)" else - SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` - fi - echo " ------ using $SFILE to search $rpm_name ------" - if [ "$_type" == "src" ];then - download_cmd="sudo -E yumdownloader -q -C --source $SFILE" - download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE" - else - download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64" - download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64" + if [ $_level == "L1" ]; then + SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` + elif [ $match_level == "L2" ];then + SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + else + SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` + fi + echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 + if [ "$_type" == "src" ];then + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" + else + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + fi fi else + # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" + download_url_cmd="echo $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" + + # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice - $download_url_cmd | sort --unique >> $URL_SRPMS + URL=$($download_url_cmd | sort --unique) + echo "The url is: $URL" + echo "url_srpm:$URL" >> $LOG if ! mv -f $SFILE* $MDIR_SRC ; then echo "FAILED to move $rpm_name" - echo $rpm_name >> $FAIL_MOVE_SRPMS + echo "fail_move_srpm:$rpm_name" >> $LOG fi + echo "found_srpm:$rpm_name" >> $LOG echo $rpm_name >> $FOUND_SRPMS else + echo "Warning: $rpm_name not found" + echo "missing_srpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_SRPMS + rc=1 fi else - echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" - echo $rpm_name >> $FOUND_SRPMS + echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" + echo "already_there_srpm:$rpm_name" >> $LOG fi else ## noarch or x86_64 if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then echo "Looking for $rpm_name..." + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice - $download_url_cmd | sort --unique >> $URL_RPMS + URL=$($download_url_cmd | sort --unique) + echo "The url is: $URL" + echo "url_rpm:$URL" >> $LOG mkdir -p $MDIR_BIN/${_type} if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then echo "FAILED to move $rpm_name" - echo $rpm_name >> $FAIL_MOVE_RPMS + echo "fail_move_rpm:$rpm_name" >> $LOG fi + echo "found_rpm:$rpm_name" >> $LOG echo $rpm_name >> $FOUND_RPMS else + echo "Warning: $rpm_name not found" + echo "missing_rpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_RPMS + rc=1 fi else echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" - echo $rpm_name >> $FOUND_RPMS + echo "already_there_rpm:$rpm_name" >> $LOG fi fi done + + return $rc } -# prime the cache -sudo -E yum makecache +# Prime the cache +${SUDOCMD} yum ${YUMCONFOPT} makecache -#go to download *.noarch.rpm files -noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"` -if [ ! -z "$noarch_rpms" ];then - echo "--> start searching noarch RPMs ....." - download "$noarch_rpms" $match_level $from "noarch" -fi - -#go to download *.x86_64.rpm files -x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"` -if [ ! -z "$x86_64_rpms" ];then - echo "--> start searching x86_64 RPMs ....." - download "$x86_64_rpms" $match_level $from "x86_64" -fi - -#go to download *.src.rpm files -src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"` -if [ ! -z "$src_rpms" ];then - echo "--> start searching source RPMs ....." - download "$src_rpms" $match_level $from "src" +# Download files +if [ -s "$rpms_list" ];then + echo "--> start searching $rpms_list" + download $rpms_list $match_level + if [ $? -ne 0 ]; then + dl_rc=1 + fi fi echo "done!!" -exit 0 +exit $dl_rc + diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index fe090ad9..0658a99b 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + # Remove leading '!' if present + tarball_name="${tarball_name//!/}" + # - For the General category and the Puppet category: # - Packages have a common process: download, decompressed, # change the directory path and compressed. @@ -87,12 +90,16 @@ for line in $(cat $tarball_file); do download_directory=$output_tarball fi + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + # We have 6 packages from the text file starting with the character "!": # they require special handling besides the common process: remove directory, # remove text from some files, clone a git repository, etc. if [[ "$line" =~ ^'!' ]]; then - tarball_name="${tarball_name//!/}" echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then @@ -145,13 +152,18 @@ for line in $(cat $tarball_file); do elif [ "$pkg_version" = "4.3-1.0.1.0" ]; then cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.1.0.1.1.g8509e41.src.rpm" . cp "$srpm_path/rdma-core-43mlnx1-1.43101.src.rpm" . + cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.0.1.8.43101.src.rpm" . elif [ "$pkg_version" = "4.3-3.0.2.1" ]; then cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm" . cp "$srpm_path/rdma-core-43mlnx1-1.43302.src.rpm" . + cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.2.1.6.43302.src.rpm" . else echo "$pkg_version : unknown version" fi - rm -f "$tarball_name" + # Don't delete the original MLNX_OFED_LINUX tarball. + # We don't use it, but it will prevent re-downloading this file. + # rm -f "$tarball_name" + rm -rf "MLNX_OFED_SRC-${pkg_version}" rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 07e9ced3..6b6731e1 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -1,124 +1,265 @@ #!/bin/bash -e -echo "--------------------------------------------------------------" +# +# SPDX-License-Identifier: Apache-2.0 +# -echo "WARNING: this script HAS TO access internet (http/https/ftp)," -echo "so please make sure your network working properly!!" +usage() { + echo "$0 [-n] [-c ] [-g]" + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations (option passed on to" + echo " subscripts when appropriate)" + echo " -c: Use an alternate yum.conf rather than the system file (option passed" + echo " on to subscripts when appropriate)" + echo " -g: do not change group IDs of downloaded artifacts" + echo "" +} -mkdir -p ./logs +generate_log_name() { + filename=$1 + level=$2 + base=$(basename $filename .lst) + echo $LOGSDIR"/"$base"_download_"$level".log" +} need_file(){ for f in $*; do if [ ! -e $f ]; then echo "ERROR: $f does not exist." - exit -1 + exit 1 fi done } +# Downloader scripts +rpm_downloader="./dl_rpms.sh" +tarball_downloader="./dl_tarball.sh" +other_downloader="./dl_other_from_centos_repo.sh" + +# track optional arguments +change_group_ids=1 +use_system_yum_conf=1 +rpm_downloader_extra_args="" +tarball_downloader_extra_args="" + +# lst files to use as input +rpms_from_3rd_parties="./rpms_3rdparties.lst" +rpms_from_centos_repo="./rpms_centos.lst" +rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" +other_downloads="./other_downloads.lst" + +# Overall success +success=1 + +# Parse out optional -c or -n arguments +while getopts "c:ngh" o; do + case "${o}" in + n) + # Pass -n ("no-sudo") to rpm downloader + rpm_downloader_extra_args="${rpm_downloader_extra_args} -n" + ;; + c) + # Pass -c ("use alternate yum.conf") to rpm downloader + use_system_yum_conf=0 + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}" + ;; + g) + # Do not attempt to change group IDs on downloaded packages + change_group_ids=0 + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +echo "--------------------------------------------------------------" + +echo "WARNING: this script HAS TO access internet (http/https/ftp)," +echo "so please make sure your network working properly!!" + + +LOGSDIR="logs" +mkdir -p $LOGSDIR + + # Check extistence of prerequisites files -need_file dl_rpms.sh dl_other_from_centos_repo.sh tarball-dl.sh -need_file rpms_from_3rd_parties.lst -need_file rpms_from_centos_3rd_parties.lst -need_file rpms_from_centos_repo.lst -need_file other_downloads.lst +need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader} +need_file ${rpms_from_3rd_parties} +need_file ${rpms_from_centos_3rd_parties} +need_file ${rpms_from_centos_repo} +need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst + #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." -# Restore StarlingX_3rd repos from backup -REPO_SOURCE_DIR=/localdisk/yum.repos.d -REPO_DIR=/etc/yum.repos.d -if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then - \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ +if [ ${use_system_yum_conf} -ne 0 ]; then + # Restore StarlingX_3rd repos from backup + REPO_SOURCE_DIR=/localdisk/yum.repos.d + REPO_DIR=/etc/yum.repos.d + if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then + \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ + fi fi -rpm_downloader="./dl_rpms.sh" -$rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt -if [ $? != 0 ];then - echo "ERROR: something wrong with downloading, please check the log!!" +list=${rpms_from_3rd_parties} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in $list." + echo " Please check the log at $(pwd)/$logfile !" + echo "" + success=0 fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -$rpm_downloader ./rpms_from_centos_3rd_parties.lst L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt +list=${rpms_from_centos_3rd_parties} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in $list." + echo " Please check the log at $(pwd)/$logfile !" + echo "" + success=0 +fi + +if [ ${use_system_yum_conf} -eq 1 ]; then + # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. + \rm -f $REPO_DIR/StarlingX_3rd*.repo +fi -# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. -\rm -f $REPO_DIR/StarlingX_3rd*.repo echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" +list=${rpms_from_centos_repo} +level=L1 +logfile=$(generate_log_name $list $level) +$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile +retcode=${PIPESTATUS[0]} -$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt -if [ $? == 0 ]; then +K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1) +if [ $retcode -ne 1 ]; then + # K1 step not needed. Clear any K1 logs from previous download attempts. + $rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile +fi + +if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" - if [ -e "./output/centos_rpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` - if [ "$missing_num" != "0" ];then - echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" +elif [ $retcode -eq 1 ]; then + echo "finish 1st round of RPM downloading with missing files!" + if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then + + echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." + $rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile + retcode=${PIPESTATUS[0]} + if [ $retcode -eq 0 ]; then + echo "finish 2nd round of RPM downloading successfully!" + elif [ $retcode -eq 1 ]; then + echo "finish 2nd round of RPM downloading with missing files!" + if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then + echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !" + fi + fi + + # Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent + # false reporting of missing files. + grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log > $LOGSDIR/L1_rpms_missing_centos.tmp + mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log + + + missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1` + if [ "$missing_num" != "0" ];then + echo "ERROR: -------RPMs missing: $missing_num ---------------" + retcode=1 fi - #echo "start 2nd round of downloading Binary RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_rpms_missing_L1.txt L2 centos | tee ./logs/log_download_rpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of RPM downloading successfully!" - # if [ -e "./output/rpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing RPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi - if [ -e "./output/centos_srpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` + if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then + missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: --------- SRPMs missing: $missing_num ---------------" + retcode=1 fi - #echo "start 2nd round of downloading Source RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_srpms_missing_L1.txt L2 centos | tee ./logs/log_download_srpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of SRPM downloading successfully!" - # if [ -e "./srpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing SRPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi -else - echo "finish 1st round with failures!" +fi + +if [ $retcode -ne 0 ]; then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." + echo " Please check the logs at $(pwd)/$logfile" + echo " and $(pwd)/logs/$K1_logfile !" + echo "" + success=0 fi ## verify all RPMs SRPMs we download for the GPG keys -find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt +find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt # remove all i686.rpms to avoid pollute the chroot dep chain -find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt +find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt find ./output -name "*.i686.rpm" | xargs rm -f -line1=`wc -l rpms_from_3rd_parties.lst | cut -d " " -f1-1` -line2=`wc -l rpms_from_centos_repo.lst | cut -d " " -f1-1` -line3=`wc -l rpms_from_centos_3rd_parties.lst | cut -d " " -f1-1` +line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` +line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` +line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` let total_line=$line1+$line2+$line3 -echo "We expect to download $total_line RPMs." +echo "We expected to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` echo "There are $num_of_downloaded_rpms RPMs in output directory." if [ "$total_line" != "$num_of_downloaded_rpms" ]; then echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs." fi -# change "./output" and sub-folders to 751 (cgcs) group -chown 751:751 -R ./output +if [ $change_group_ids -eq 1 ]; then + # change "./output" and sub-folders to 751 (cgcs) group + chown 751:751 -R ./output +fi echo "step #3: start downloading other files ..." -other_downloader="./dl_other_from_centos_repo.sh" -$other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt -if [ $? == 0 ];then +logfile=$LOGSDIR"/otherfiles_centos_download.log" +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then echo "step #3: done successfully" +else + echo "step #3: finished with errors" + echo "ERROR: Something wrong with downloading from ${other_downloads}." + echo " Please check the log at $(pwd)/$logfile!" + echo "" + success=0 fi + # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -./dl_tarball.sh +logfile=$LOGSDIR"/tarballs_download.log" +${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then + echo "step #4: done successfully" +else + echo "step #4: finished with errors" + echo "ERROR: Something wrong with downloading tarballs." + echo " Please check the log at $(pwd)/$logfile !" + echo "" + success=0 +fi echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" @@ -127,3 +268,11 @@ echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" +echo "" +if [ $success -ne 1 ]; then + echo "Warning: Not all download steps succeeded. You are likely missing files." + exit 1 +fi + +echo "Success" +exit 0 diff --git a/centos-mirror-tools/mirror-check.sh b/centos-mirror-tools/mirror-check.sh new file mode 100755 index 00000000..6a8c4776 --- /dev/null +++ b/centos-mirror-tools/mirror-check.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This script checks if the required packages in the .lst file list is +# actually downloadable. Sometimes, the release number in upstream is +# changed and that causes a mismatch in the build requirements. +# We can find this problems in an early stage without the need to +# download all the packages. +# +# The yum cache contains this information, more specific the primary_db +# files, so iterating over the content of .lst, parse the name of the +# package and get the information on what is available to download +# should be enough to know the status of the mirror. +# +# If a package is not found then the script will try to get the avai- +# lable version and log that into the error log. By this way we get +# notified on what changed in the external repositories. +# +# How to run: +# This script is intended to be run inside the downloader container. +# It needs that all the CentOS repositories are well setup. +# +# ./mirror-check.sh +# +# And you should see the checking in progress. + +_print_msg() { echo -en "$(date -u +"%Y-%m-%d %H-%M-%S") ==> $1"; } +info() { _print_msg "INFO: $1\n"; } +info_c() { _print_msg "INFO: $1"; } +warning() { _print_msg "WARN: $1\n"; } +error() { _print_msg "ERROR: $1\n"; } + +RPMS_CENTOS_LIST="rpms_centos.lst" +RPMS_3RD_PARTY_LIST="rpms_centos3rdparties.lst" +ERROR_LOG_FILE="mirror-check-failures.log" +truncate -s 0 $ERROR_LOG_FILE +retcode=0 +extra_opts="" + +usage() { + echo "$0 [-c ]" + echo "" + echo "Options:" + echo " -c: Use an alternate yum.conf rather than the system file (option passed" + echo " on to subscripts when appropriate)" + echo "" +} + +get_rpm_name() { + _rpm_file_name=$1 + rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'-' -f3- | rev) + echo "$rpm_name" +} + +get_rpm_full_name() { + _rpm_file_name=$1 + rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'.' -f2- | rev) + echo "$rpm_name" +} + +get_rpm_arch() { + arch=$(echo "$1" | rev | cut -d'.' -f2 | rev) + echo "$arch" +} + +get_repoquery_info() { + _arch=$1 + _package_name=$2 + if [ "$_arch" == "x86_64" ]; then + # To filter out the i686 packages + repoquery_opts="--archlist=x86_64" + elif [ "$_arch" == "src" ]; then + repoquery_opts="--archlist=src" + else + repoquery_opts= + fi + repoquery $extra_opts -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ + $repoquery_opts "$_package_name" +} + +_check_rpms() { + p=$1 + full_name=$(get_rpm_full_name "$p") + rpm_name=$(get_rpm_name "$p") + arch=$(get_rpm_arch "$p") + info_c "Checking $full_name... " + _repoquery=$(get_repoquery_info "$arch" "$full_name") + if [ -z "$_repoquery" ]; then + echo -e "FAILED!" + available_pkgs=$(get_repoquery_info "$arch" "$rpm_name") + echo -e "Package $full_name not found, available $available_pkgs" >> $ERROR_LOG_FILE + retcode=1 + else + if [ "$full_name" == "$_repoquery" ]; then + echo -e "OK" + else + echo -e "FAILED!" + retcode=1 + echo -e "Required $full_name but found $_repoquery" >> $ERROR_LOG_FILE + fi + fi +} + +check_rpms() { + _rpms_list=$1 + for p in $_rpms_list; do + _check_rpms "$p" + done +} + +while getopts "c:" opt; do + case $opt in + c) + extra_opts="-c ${OPTARG}" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + esac +done + +info "Getting yum cache" +if ! yum $extra_opts makecache; then + error "There was a problem getting yum cache" + exit 1 +fi + +for rpm_list in "$RPMS_CENTOS_LIST" "$RPMS_3RD_PARTY_LIST"; do + info "Reading $rpm_list..." + for arch in "src" "noarch" "x86_64"; do + info "Getting info for $arch packages..." + rpms=$(echo "$(grep -F "$arch.rpm" < $rpm_list)") + check_rpms "$rpms" + done +done + +if [ $retcode -ne 0 ]; then + error "Failures found, error log:" + error "==========================" + cat $ERROR_LOG_FILE +fi + +exit $retcode diff --git a/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization b/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization new file mode 100644 index 00000000..00006eb1 --- /dev/null +++ b/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization @@ -0,0 +1,20 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQENBFWB31YBCAC4dFmTzBDOcq4R1RbvQXLkyYfF+yXcsMA5kwZy7kjxnFqBoNPv +aAjFm3e5huTw2BMZW0viLGJrHZGnsXsE5iNmzom2UgCtrvcG2f65OFGlC1HZ3ajA +8ZIfdgNQkPpor61xqBCLzIsp55A7YuPNDvatk/+MqGdNv8Ug7iVmhQvI0p1bbaZR +0GuavmC5EZ/+mDlZ2kHIQOUoInHqLJaX7iw46iLRUnvJ1vATOzTnKidoFapjhzIt +i4ZSIRaalyJ4sT+oX4CoRzerNnUtIe2k9Hw6cEu4YKGCO7nnuXjMKz7Nz5GgP2Ou +zIA/fcOmQkSGcn7FoXybWJ8DqBExvkJuDljPABEBAAG0bENlbnRPUyBWaXJ0dWFs +aXphdGlvbiBTSUcgKGh0dHA6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lhbEludGVy +ZXN0R3JvdXAvVmlydHVhbGl6YXRpb24pIDxzZWN1cml0eUBjZW50b3Mub3JnPokB +OQQTAQIAIwUCVYHfVgIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEHrr +voJh6IBsRd0H/A62i5CqfftuySOCE95xMxZRw8+voWO84QS9zYvDEnzcEQpNnHyo +FNZTpKOghIDtETWxzpY2ThLixcZOTubT+6hUL1n+cuLDVMu4OVXBPoUkRy56defc +qkWR+UVwQitmlq1ngzwmqVZaB8Hf/mFZiB3B3Jr4dvVgWXRv58jcXFOPb8DdUoAc +S3u/FLvri92lCaXu08p8YSpFOfT5T55kFICeneqETNYS2E3iKLipHFOLh7EWGM5b +Wsr7o0r+KltI4Ehy/TjvNX16fa/t9p5pUs8rKyG8SZndxJCsk0MW55G9HFvQ0FmP +A6vX9WQmbP+ml7jsUxtEJ6MOGJ39jmaUvPc= +=ZzP+ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/centos-mirror-tools/rpms_from_3rd_parties.lst b/centos-mirror-tools/rpms_3rdparties.lst similarity index 89% rename from centos-mirror-tools/rpms_from_3rd_parties.lst rename to centos-mirror-tools/rpms_3rdparties.lst index 9061a2b0..6961bfc7 100644 --- a/centos-mirror-tools/rpms_from_3rd_parties.lst +++ b/centos-mirror-tools/rpms_3rdparties.lst @@ -5,5 +5,4 @@ python2-pytest-httpbin-0.2.3-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/pa python2-pytest-mock-1.6.0-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-pytest-mock/1.6.0/2.el7/noarch/python2-pytest-mock-1.6.0-2.el7.noarch.rpm python2-storops-0.4.7-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-storops/0.4.7/2.el7/noarch/python2-storops-0.4.7-2.el7.noarch.rpm python-gunicorn-19.7.1-1.fc27.src.rpm#https://kojipkgs.fedoraproject.org/packages/python-gunicorn/19.7.1/1.fc27/src/python-gunicorn-19.7.1-1.fc27.src.rpm -kubernetes-1.10.0-1.el7.src.rpm#http://cbs.centos.org/kojifiles/packages/kubernetes/1.10.0/1.el7/src/kubernetes-1.10.0-1.el7.src.rpm influxdb-0.9.5.1-1.x86_64.rpm#https://s3.amazonaws.com/influxdb/influxdb-0.9.5.1-1.x86_64.rpm diff --git a/centos-mirror-tools/rpms_from_centos_repo.lst b/centos-mirror-tools/rpms_centos.lst similarity index 98% rename from centos-mirror-tools/rpms_from_centos_repo.lst rename to centos-mirror-tools/rpms_centos.lst index d99b284e..e4758b54 100644 --- a/centos-mirror-tools/rpms_from_centos_repo.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -524,6 +524,7 @@ kmod-libs-20-21.el7.x86_64.rpm kpartx-0.4.9-119.el7.x86_64.rpm krb5-devel-1.15.1-19.el7.x86_64.rpm krb5-libs-1.15.1-19.el7.x86_64.rpm +kubernetes-1.10.0-1.el7.src.rpm lapack-3.4.2-8.el7.x86_64.rpm latex2html-2012-3.el7.noarch.rpm lato-fonts-2.015-1.el7.noarch.rpm @@ -573,6 +574,7 @@ libedit-3.0-12.20121213cvs.el7.x86_64.rpm libedit-devel-3.0-12.20121213cvs.el7.x86_64.rpm libepoxy-1.3.1-1.el7.x86_64.rpm libepoxy-devel-1.3.1-1.el7.x86_64.rpm +liberasurecode-1.5.0-1.el7.x86_64.rpm liberation-fonts-common-1.07.2-16.el7.noarch.rpm liberation-mono-fonts-1.07.2-16.el7.noarch.rpm liberation-narrow-fonts-1.07.2-16.el7.noarch.rpm @@ -884,6 +886,11 @@ openstack-aodh-3.0.4-1.el7.src.rpm openstack-macros-2018.1.0-0.noarch.rpm openstack-nova-cert-14.0.8-1.el7.noarch.rpm openstack-panko-3.0.0-1.el7.src.rpm +openstack-swift-account-2.15.1-1.el7.noarch.rpm +openstack-swift-container-2.15.1-1.el7.noarch.rpm +openstack-swift-doc-2.15.1-1.el7.noarch.rpm +openstack-swift-object-2.15.1-1.el7.noarch.rpm +openstack-swift-proxy-2.15.1-1.el7.noarch.rpm openvswitch-2.9.0-3.el7.src.rpm orc-0.4.26-1.el7.x86_64.rpm osinfo-db-tools-1.1.0-1.el7.x86_64.rpm @@ -1093,6 +1100,7 @@ python2-cursive-0.1.2-1.el7.noarch.rpm python2-cycler-0.10.0-2.el7.noarch.rpm python2-Cython-0.25.2-3.el7.x86_64.rpm python2-daiquiri-1.2.1-1.el7.noarch.rpm +python2-dateutil-2.6.1-1.el7.noarch.rpm python2-ddt-1.1.3-1.el7.noarch.rpm python2-debtcollector-1.17.0-1.el7.noarch.rpm python2-deprecation-1.0-3.el7.noarch.rpm @@ -1113,6 +1121,7 @@ python2-gabbi-1.33.0-1.el7.noarch.rpm python2-gevent-1.1.2-2.el7.x86_64.rpm python2-gflags-2.0-5.el7.noarch.rpm python2-gnocchiclient-3.3.1-1.el7.noarch.rpm +python2-google-auth-1.3.0-1.el7.noarch.rpm python2-greenlet-0.4.9-1.el7.x86_64.rpm python2-hacking-0.13.0-1.el7.noarch.rpm python2-idna-2.5-1.el7.noarch.rpm @@ -1128,7 +1137,7 @@ python2-jsonschema-2.5.1-3.el7.noarch.rpm python2-keystoneauth1-3.1.0-1.el7.noarch.rpm python2-keystoneclient-3.13.0-1.el7.noarch.rpm python2-kombu-4.0.2-5.el7.noarch.rpm -python2-kubernetes-1.0.0-0.3.b3.el7.noarch.rpm +python2-kubernetes-4.0.0-1.el7.noarch.rpm python2-lz4-0.9.0-1.el7.x86_64.rpm python2-marathon-0.8.8-1.el7.noarch.rpm python2-markupsafe-0.23-16.el7.x86_64.rpm @@ -1191,16 +1200,17 @@ python2-pycadf-2.6.0-1.el7.noarch.rpm python2-pycodestyle-2.0.0-5.el7.noarch.rpm python2-pygments-2.2.0-7.el7.noarch.rpm python2-PyMySQL-0.7.11-1.el7.noarch.rpm -python2-pyngus-2.2.3-1.el7.noarch.rpm +python2-pyngus-2.2.4-1.el7.noarch.rpm python2-pyOpenSSL-16.2.0-3.el7.noarch.rpm python2-pyparsing-2.1.10-3.el7.noarch.rpm python2-pyroute2-0.4.19-1.el7.noarch.rpm python2-pysaml2-3.0.2-2.el7.noarch.rpm python2-pysnmp-4.3.2-3.el7.noarch.rpm python2-pytest-3.0.6-2.el7.noarch.rpm -python2-qpid-proton-0.22.0-1.el7.x86_64.rpm +python2-qpid-proton-0.24.0-1.el7.x86_64.rpm python2-rcssmin-1.0.6-2.el7.x86_64.rpm python2-reno-2.5.0-1.el7.noarch.rpm +python2-requests-oauthlib-0.8.0-5.el7.noarch.rpm python2-requestsexceptions-1.3.0-1.el7.noarch.rpm python2-retryz-0.1.8-1.el7.noarch.rpm python2-rfc3986-0.3.1-1.el7.noarch.rpm @@ -1276,6 +1286,7 @@ python-betamax-0.7.0-1.el7.noarch.rpm python-Bottleneck-0.7.0-1.el7.x86_64.rpm python-bson-3.0.3-1.el7.x86_64.rpm python-cachetools-1.1.6-2.el7.noarch.rpm +python-ceilometermiddleware-1.1.0-1.el7.noarch.rpm python-cffi-1.6.0-5.el7.x86_64.rpm python-characteristic-14.3.0-4.el7.noarch.rpm python-chardet-2.2.1-1.el7_1.noarch.rpm @@ -1290,6 +1301,7 @@ python-croniter-0.3.4-2.el7.noarch.rpm python-d2to1-0.2.11-1.el7.noarch.rpm python-daemon-1.6-4.el7.noarch.rpm python-dateutil-2.4.2-1.el7.noarch.rpm +python-dateutil-2.6.1-1.el7.src.rpm python-decorator-3.4.0-3.el7.noarch.rpm python-deltarpm-3.6-3.el7.x86_64.rpm python-django-1.8.14-1.el7.noarch.rpm @@ -1300,6 +1312,7 @@ python-django-compressor-2.0-1.el7.noarch.rpm python-django-nose-1.4.3-1.el7.noarch.rpm python-django-openstack-auth-3.5.0-1.el7.src.rpm python-django-pyscss-2.0.2-1.el7.noarch.rpm +python-dns-1.12.0-4.20150617git465785f.el7.noarch.rpm python-docker-pycreds-1.10.6-1.el7.noarch.rpm python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm python-dogpile-cache-0.6.2-1.el7.noarch.rpm @@ -1316,6 +1329,7 @@ python-flake8-2.4.1-2.el7.noarch.rpm python-flask-0.10.1-4.el7.noarch.rpm python-freezegun-0.3.8-2.el7.noarch.rpm python-glance-store-0.22.0-1.el7.src.rpm +python-google-auth-1.3.0-1.el7.src.rpm python-heatclient-1.11.0-1.el7.src.rpm python-httplib2-0.9.2-1.el7.noarch.rpm python-iniparse-0.4-9.el7.noarch.rpm @@ -1336,6 +1350,7 @@ python-keystoneclient-3.13.0-1.el7.src.rpm python-keystonemiddleware-4.17.0-1.el7.src.rpm python-kitchen-1.1.1-5.el7.noarch.rpm python-kmod-0.9-4.el7.x86_64.rpm +python-kubernetes-4.0.0-1.el7.src.rpm python-ldap-2.4.15-2.el7.x86_64.rpm python-ldappool-1.0-4.el7.noarch.rpm python-lesscpy-0.9j-4.el7.noarch.rpm @@ -1395,6 +1410,7 @@ python-py-1.4.32-1.el7.noarch.rpm python-pycadf-common-2.6.0-1.el7.noarch.rpm python-pycparser-2.14-1.el7.noarch.rpm python-pycurl-7.19.0-19.el7.x86_64.rpm +python-pyeclib-1.5.0-1.el7.x86_64.rpm python-pyelftools-0.22-0.5.git20130619.a1d9681.el7.noarch.rpm python-pymongo-3.0.3-1.el7.x86_64.rpm python-pytest-cov-2.5.1-1.el7.noarch.rpm @@ -1422,6 +1438,8 @@ python-sqlparse-0.1.18-5.el7.noarch.rpm python-srpm-macros-3-22.el7.noarch.rpm python-subprocess32-3.2.6-4.el7.x86_64.rpm python-subunit-1.0.0-1.el7.noarch.rpm +python-swift-2.15.1-1.el7.noarch.rpm +python-swift-tests-2.15.1-1.el7.noarch.rpm python-sysv_ipc-0.4.2-11.el7.x86_64.rpm python-tables-3.2.0-1.el7.x86_64.rpm python-tablib-0.10.0-1.el7.noarch.rpm @@ -1466,7 +1484,7 @@ pytz-2016.10-2.el7.noarch.rpm pyxattr-0.5.1-5.el7.x86_64.rpm PyYAML-3.10-11.el7.x86_64.rpm qdox-1.12.1-10.el7.noarch.rpm -qpid-proton-c-0.22.0-1.el7.x86_64.rpm +qpid-proton-c-0.24.0-1.el7.x86_64.rpm qrencode-devel-3.4.1-3.el7.x86_64.rpm qrencode-libs-3.4.1-3.el7.x86_64.rpm qt-4.8.5-13.el7.x86_64.rpm @@ -1514,6 +1532,7 @@ sanlock-3.6.0-1.el7.src.rpm sazanami-fonts-common-0.20040629-22.el7.noarch.rpm sazanami-gothic-fonts-0.20040629-22.el7.noarch.rpm sazanami-mincho-fonts-0.20040629-22.el7.noarch.rpm +screen-4.1.0-0.25.20120314git3c2946.el7.x86_64.rpm scrub-2.5.2-7.el7.x86_64.rpm scsi-target-utils-1.0.55-4.el7.src.rpm SDL-1.2.15-14.el7.x86_64.rpm diff --git a/centos-mirror-tools/rpms_from_centos_3rd_parties.lst b/centos-mirror-tools/rpms_centos3rdparties.lst similarity index 100% rename from centos-mirror-tools/rpms_from_centos_3rd_parties.lst rename to centos-mirror-tools/rpms_centos3rdparties.lst diff --git a/centos-mirror-tools/tarball-dl.lst b/centos-mirror-tools/tarball-dl.lst index 8b452fdb..7b6c8226 100644 --- a/centos-mirror-tools/tarball-dl.lst +++ b/centos-mirror-tools/tarball-dl.lst @@ -4,7 +4,6 @@ drbd-8.4.3.tar.gz#drbd-8.4.3#http://www.linbit.com/downloads/drbd/8.4/archive/dr drbd-8.4.7-1.tar.gz#drbd-8.4.7-1#http://www.linbit.com/downloads/drbd/8.4/drbd-8.4.7-1.tar.gz dtc-1.4.4.tar.gz#dtc-1.4.4#https://www.kernel.org/pub/software/utils/dtc/dtc-1.4.4.tar.gz e1000e-3.4.1.1.tar.gz#e1000e-3.4.1.1#https://sourceforge.net/projects/e1000/files/e1000e%20stable/3.4.1.1/e1000e-3.4.1.1.tar.gz -expect-lite_4.9.0.tar.gz#expect-lite.proj#https://sourceforge.net/projects/expect-lite/files/expect-lite/expect-lite_4.9.0/expect-lite_4.9.0.tar.gz gnulib-ffc927e.tar.gz#gnulib-ffc927e#http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-ffc927eef29016a5219cd969daad8928af6a1f4d.tar.gz i40e-2.4.10.tar.gz#i40e-2.4.10#https://sourceforge.net/projects/e1000/files/i40e%20stable/2.4.10/i40e-2.4.10.tar.gz/download i40evf-3.5.13.tar.gz#i40evf-3.5.13#https://sourceforge.net/projects/e1000/files/i40evf%20stable/3.5.13/i40evf-3.5.13.tar.gz/download diff --git a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo index 925486cd..f05db07a 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo @@ -100,3 +100,11 @@ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 enabled=1 +[Starlingx-C7.5.1804-virt-kubernetes-source] +name=Starlingx-CentOS-7.5.1804 - virt-kubernetes-source +baseurl=http://vault.centos.org/7.5.1804/virt/Source/kubernetes110 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + + diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo new file mode 100644 index 00000000..3d74243f --- /dev/null +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo @@ -0,0 +1,41 @@ +# CentOS-OpenStack-queens.repo +# +# Please see http://wiki.centos.org/SpecialInterestGroup/Cloud for more +# information + +[centos-openstack-queens] +name=CentOS-7 - OpenStack queens +baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[centos-openstack-queens-test] +name=CentOS-7 - OpenStack queens Testing +baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-queens/ +gpgcheck=0 +enabled=0 +exclude=sip,PyQt4 + +[centos-openstack-queens-debuginfo] +name=CentOS-7 - OpenStack queens - Debug +baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[centos-openstack-queens-source] +name=CentOS-7 - OpenStack queens - Source +baseurl=http://vault.centos.org/centos/7/cloud/Source/openstack-queens/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[rdo-trunk-queens-tested] +name=OpenStack queens Trunk Tested +baseurl=https://trunk.rdoproject.org/centos7-queens/current-passed-ci/ +gpgcheck=0 +enabled=0 diff --git a/deployment/README.rst b/deployment/README.rst new file mode 100644 index 00000000..9bcb7a14 --- /dev/null +++ b/deployment/README.rst @@ -0,0 +1,141 @@ +StarlingX Deployment in Virtualized Environments +================================================ + +A StarlingX system can be installed in a variety of platforms with the following +deployment options: + +- Standard Controller + + - Dedicated Storage + - Controller Storage + +- All-in-one + + - Duplex + - Simplex + +Deployment options uses a variety of configurations based on 3 node identities: + +- Controller +- Storage +- Compute + +Standard Controller :: Dedicated Storage +---------------------------------------- + +The software installation workflow for an initial Ceph-backed block +storage on dedicated storage nodes is: + +- Controller-0 Installation and Provisioning +- Controller-1 / Compute Host / Storage Host Installation +- Controller-1 Provisioning +- Provider Network Configuration +- Compute Host Provisioning +- Storage Host Provisioning + +Standard Controller :: Controller Storage +----------------------------------------- + +The software installation workflow for an initial LVM-backed block +storage on controller nodes is: + +- Controller-0 Installation +- Controller-0 and System Provisioning +- Controller-1 / Compute Host Installation +- Controller-1 Provisioning +- Compute Host Provisioning + +All-in-one :: Duplex +-------------------- + +The software installation workflow for two combined controller / compute +nodes is: + +- Controller-0 Installation and Provisioning +- Controller-1 Installation and Provisioning + +All-in-one :: Simplex +--------------------- + +The software installation workflow for a single combined controller / compute +node is: + +- Controller-0 Installation and Provisioning + +Virtualization Environments +--------------------------- + +The available virtualization products where StarlingX has been deployed +are: + +- VirtualBox +- Libvirt/QEMU + +Directory Structure +------------------- + +Deployment directory hosts a total of 3 directories and 18 files:: + + $ tree -L 3 deployment/ + deployment/ + ├── libvirt + │   ├── compute.xml + │   ├── controller_allinone.xml + │   ├── controller.xml + │   ├── destroy_allinone.sh + │   ├── destroy_standard_controller.sh + │   ├── install_packages.sh + │   ├── setup_allinone.sh + │   └── setup_standard_controller.sh + ├── provision + │   ├── simplex_stage_1.sh + │   └── simplex_stage_2.sh + └── virtualbox + ├── all_in_one.conf + ├── serial_vm.sh + ├── setup_vm.sh + ├── standard_controller.conf + ├── start_vm.sh + └── stop_vm.sh + +Directory: libvirt +~~~~~~~~~~~~~~~~~~ + +Deployment under Libvirt/QEMU uses a set of xml files to define the node +identity: + +- Controller All-in-one +- Controller +- Compute + +These nodes are used to create the virtual machines and the network interfaces +to setup the StarlingX system: + +- Setup All-in-one + + - 2 Controllers + +- Setup Standard Controller + + - 2 Controllers + - 2 Computes + +Directory: virtualbox +~~~~~~~~~~~~~~~~~~~~~ + +Deployment under VirtualBox uses a set of configuration files to define the +StarlingX system: + +- All-in-one Configuration +- Standard Controller Configuration + +These configurations files are used to create the virtual machines and the +network interfaces from a single script: + +- Setup VM + +Directory: provision +~~~~~~~~~~~~~~~~~~~~ + +A set of scripts are provided to automate the provisioning of data interfaces and +local storage resources for the compute function for StarlingX Duplex or Simplex. diff --git a/deployment/libvirt/README.rst b/deployment/libvirt/README.rst new file mode 100644 index 00000000..b5f0396a --- /dev/null +++ b/deployment/libvirt/README.rst @@ -0,0 +1,65 @@ +StarlingX Deployment on Libvirt +=============================== + +This is a quick reference for deploying StarlingX on libvirt/qemu systems. +It assumes you have a working libvirt/qemu installation for a non-root user +and that your user has NOPASSWD sudo permissions. + +Overview +-------- + +We create 4 bridges to use for the STX cloud. This is done in an initial step +separate from the VM management. + +Depending on which basic configuration is chosen, we create a number of VMs +for one or more controllers and storage nodes. + +These scripts are configured using environment variables that all have built-in +defaults. On shared systems you probably do not want to use the defaults. +The simplest way to handle this is to keep an rc file that can be sourced into +an interactive shell that configures everything. Here's an example:: + + export CONTROLLER=madcloud + export COMPUTE=madnode + export BRIDGE_INTERFACE=madbr + export INTERNAL_NETWORK=172.30.20.0/24 + export INTERNAL_IP=172.30.20.1/24 + export EXTERNAL_NETWORK=192.168.20.0/24 + export EXTERNAL_IP=192.168.20.1/24 + +There is also a script ``cleanup_network.sh`` that will remove networking +configuration from libvirt. + +Networking +---------- + +Configure the bridges using ``setup_network.sh`` before doing anything else. It +will create 4 bridges named ``stxbr1``, ``stxbr2``, ``stxbr3`` and ``stxbr4``. +Set the BRIDGE_INTERFACE environment variable if you need to change stxbr to +something unique. + +The ``destroy_network.sh`` script does the reverse, and should not be used lightly. +It should also only be used after all of the VMs created below have been destroyed. + +Controllers +----------- + +There are two scripts for creating the controllers: ``setup_allinone.sh`` and +``setup_standard_controller.sh``. They are operated in the same manner but build +different StarlingX cloud configurations. Choose wisely. + +You need an ISO file for the installation, these scripts take a name with the +``-i`` option:: + + ./setup_allinone.sh -i stx-2018-08-28-93.iso + +And the setup will begin. The scripts create one or more VMs and start the boot +of the first controller, named oddly enough ``controller-0``. If you have Xwindows +available you will get virt-manager running. If not, Ctrl-C out of that attempt if +it doesn't return to a shell prompt. Then connect to the serial console:: + + virsh console madcloud-0 + +Continue the usual SterlingX installation from this point forward. + +Tear down the VMs using ``destroy_allinone.sh`` and ``destroy_standard_controller.sh``. diff --git a/deployment/libvirt/cleanup_network.sh b/deployment/libvirt/cleanup_network.sh new file mode 100755 index 00000000..ebace0bb --- /dev/null +++ b/deployment/libvirt/cleanup_network.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# cleanup_network.sh - Cleans up network interfaces - not safe to run blindly! + +NETWORK_DEFAULT=${NETWORK_DEFAULT:-default} +BRIDGE_INTERFACE=${BRIDGE_INTERFACE=stxbr0} + +if virsh net-list --name | grep ${NETWORK_DEFAULT} ; then + sudo virsh net-destroy ${NETWORK_DEFAULT} + sudo virsh net-undefine ${NETWORK_DEFAULT} + sudo rm -rf /etc/libvirt/qemu/networks/autostart/${NETWORK_DEFAULT}.xml +fi + +if [ -d "/sys/class/net/${BRIDGE_INTERFACE}" ]; then + sudo ifconfig ${BRIDGE_INTERFACE} down || true + sudo brctl delbr ${BRIDGE_INTERFACE} || true +fi diff --git a/deployment/libvirt/compute.xml b/deployment/libvirt/compute.xml new file mode 100644 index 00000000..85474c03 --- /dev/null +++ b/deployment/libvirt/compute.xml @@ -0,0 +1,115 @@ + + NAME + 16777216 + 16777216 + 4 + + /machine + + + hvm + + + + + + + + + + + + + + destroy + restart + destroy + + /usr/bin/qemu-system-x86_64 + + + + + + + +
+ + + + + + + +
+ + + +
+ + + + + + +
+ + + + + + +
+ + + + + + + +
+ + + + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + +