Merge remote-tracking branch 'origin/master' into f/centos75-merge

Change-Id: I413bca255dc6513f2c8e412f4e789e96f7d8badf

.gitignore

@@ -5,3 +5,9 @@ localrc
 toCOPY/.gitconfig
 centos-mirror-tools/logs/
 centos-mirror-tools/output/
+
+# Sphinx documentation
+docs/build/
+
+# Release Notes documentation
+releasenotes/build

centos-mirror-tools/dl_other_from_centos_repo.sh

@@ -31,6 +31,11 @@ for ff in $all; do
     if [ "$_type" == "folder" ];then
         mkdir -p $save_path/$_name
     else
+        if [ -e "$save_path/$_name" ]; then
+            echo "Already have $save_path/$_name"
+            continue
+        fi
+
         echo "remote path: $url_prefix/$_name"
         echo "local path: $save_path/$_name"
         if wget $url_prefix/$_name; then

centos-mirror-tools/dl_rpms.sh

@@ -1,25 +1,86 @@
 #!/bin/bash -e
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 # download RPMs/SRPMs from different sources.
 # this script was originated by Brian Avery, and later updated by Yong Hu
 
-if [ $# -lt 3 ]; then
-    echo "$0 <rpms_list> <match_level> <from_where>"
-    echo "rpm_list: a list of RPM files to be downloaded."
-    echo "match_level: value could be L1, L2 or L3:"
-    echo "  L1: use name, major version and minor version:"
-    echo "      vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
-    echo "  L2: use name and major version:"
-    echo "      using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
-    echo "  L3: use name:"
-    echo "      using vim to search vim-7.4.160-2.el7.src.rpm"
-    echo "from_where: where to download the RPMs: 'centos'from CentOS Repos,"
-    echo "otherwise from 3rd-party websets"
-    exit -1
+usage() {
+    echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
+    echo ""
+    echo "Options:"
+    echo "  -n: Do not use sudo when performing operations"
+    echo "  -c: Use an alternate yum.conf rather than the system file"
+    echo "  -x: Clean log files only, do not run."
+    echo "  rpm_list: a list of RPM files to be downloaded."
+    echo "  match_level: value could be L1, L2 or L3:"
+    echo "    L1: use name, major version and minor version:"
+    echo "        vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
+    echo "    L2: use name and major version:"
+    echo "        using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
+    echo "    L3: use name:"
+    echo "        using vim to search vim-7.4.160-2.el7.src.rpm"
+    echo "    K1: Use Koji rather than yum repos as a source."
+    echo "        Koji has a longer retention period than epel mirrors."
+    echo ""
+    echo "Returns: 0 = All files downloaded successfully"
+    echo "         1 = Some files could not be downloaded"
+    echo "         2 = Bad arguements or other error"
+    echo ""
+}
+
+get_from() {
+    list=$1
+    base=$(basename $list .lst) # removing lst extension 
+    base=$(basename $base .log) # removing log extension
+    from=$(echo $base | rev | cut -d'_' -f1-1 | rev)
+    echo $from
+}
+
+# By default, we use "sudo" and we don't use a local yum.conf. These can
+# be overridden via flags.
+SUDOCMD="sudo -E"
+YUMCONFOPT=""
+
+CLEAN_LOGS_ONLY=0
+dl_rc=0
+
+# Parse option flags
+while getopts "c:nxh" o; do
+    case "${o}" in
+        n)
+            # No-sudo
+            SUDOCMD=""
+            ;;
+        x)
+            # Clean only
+            CLEAN_LOGS_ONLY=1
+            ;;
+        c)
+            # Use an alternate yum.conf
+            YUMCONFOPT="-c $OPTARG"
+            ;;
+        h)
+            # Help
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 2
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ $# -lt 2 ]; then
+    usage
+    exit 2
 fi
 
 if [ "$1" == "" ]; then
     echo "Need to supply the rpm file list"
-    exit -1;
+    exit 2;
 else
     rpms_list=$1
     echo "using $rpms_list as the download name lists"
@@ -31,142 +92,222 @@ if [ ! -z "$2" -a "$2" != " " ];then
     match_level=$2
 fi
 
-from=$3
 
 timestamp=$(date +%F_%H%M)
 echo $timestamp
 
+
+
 DESTDIR="output"
 MDIR_SRC=$DESTDIR/stx-r1/CentOS/pike/Source
 mkdir -p $MDIR_SRC
 MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
 mkdir -p $MDIR_BIN
 
-FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
-FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
-MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
-URL_SRPMS="$DESTDIR/${from}_srpms_urls_${match_level}.txt"
-
-cat /dev/null > $FAIL_MOVE_SRPMS
-cat /dev/null > $FOUND_SRPMS
+LOGSDIR="logs"
+from=$(get_from $rpms_list)
+LOG="$LOGSDIR/${match_level}_failmoved_url_${from}.log"
+MISSING_SRPMS="$LOGSDIR/${match_level}_srpms_missing_${from}.log"
+MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${from}.log"
+FOUND_SRPMS="$LOGSDIR/${match_level}_srpms_found_${from}.log"
+FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${from}.log"
+cat /dev/null > $LOG
 cat /dev/null > $MISSING_SRPMS
-cat /dev/null > $URL_SRPMS
-
-FAIL_MOVE_RPMS="$DESTDIR/${from}_rpms_fail_move_${match_level}.txt"
-FOUND_RPMS="$DESTDIR/${from}_rpms_found_${match_level}.txt"
-MISSING_RPMS="$DESTDIR/${from}_rpms_missing_${match_level}.txt"
-URL_RPMS="$DESTDIR/${from}_rpms_urls_${match_level}.txt"
-
-cat /dev/null > $FAIL_MOVE_RPMS
-cat /dev/null > $FOUND_RPMS
 cat /dev/null > $MISSING_RPMS
-cat /dev/null > $URL_RPMS
+cat /dev/null > $FOUND_SRPMS
+cat /dev/null > $FOUND_RPMS
 
-#function to download different type of RPMs in different ways
+
+if [ $CLEAN_LOGS_ONLY -eq 1 ];then
+    exit 0
+fi
+
+# Function to split an rpm filename into parts.
+#
+# Returns a space seperated list containing:
+#    <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
+#
+split_filename () {
+    local rpm_filename=$1
+
+    local RPM=""
+    local SFILE=""
+    local ARCH=""
+    local RELEASE=""
+    local VERSION=""
+    local NAME=""
+
+    RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
+    ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
+    RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+    VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
+    NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
+
+    if [[ $NAME = *":"* ]]; then
+        EPOCH=$(echo $NAME | cut -d':' -f-1)
+        NAME=$(echo $NAME | cut -d':' -f2-)
+    fi
+
+    echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
+}
+
+# Function to predict the URL where a rpm might be found.
+# Assumes the rpm was compile for EPEL by fedora's koji.
+koji_url () {
+    local rpm_filename=$1
+
+    local arr=( $(split_filename $rpm_filename) )
+
+    local n=${arr[0]}
+    local v=${arr[1]}
+    local r=${arr[2]}
+    local a=${arr[3]}
+    local e=${arr[4]}
+
+    echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
+}
+
+# Function to download different types of RPMs in different ways
 download () {
-    _list=$1
-    _level=$2
-    _from=$3
-    _type=$4
+    local _file=$1
+    local _level=$2
+
+    local _list
+    local _from
+    local _type=""
+
+    local rc=0
+    local download_cmd=""
+    local download_url_cmd=""
+    local rpm_name=""
+    local rpm_url=""
+    local SFILE=""
+
+    _list=$(cat $_file)
+    _from=$(get_from $_file)
 
     echo "now the rpm will come from: $_from"
     for ff in $_list; do
-        ## download RPM from CentOS repos
-        if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then
+        download_cmd=""
+        download_url_cmd=""
+        _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
+
+        # Decide if the list will be downloaded using yumdownloader or wget
+        if [[ $ff != *"#"* ]]; then
             rpm_name=$ff
-            if [ $_level == "L1" ]; then
+
+            if [ $_level == "K1" ]; then
                 SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
-            elif [ $match_level == "L2" ];then
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
+                rpm_url=$(koji_url $rpm_name)
+                download_cmd="wget $rpm_url)"
+                download_url_cmd="echo $rpm_url)"
             else
-                SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
-            fi
-            echo " ------ using $SFILE to search $rpm_name ------"
-            if [ "$_type" == "src" ];then
-                download_cmd="sudo -E yumdownloader -q -C --source $SFILE"
-                download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE"
-            else
-                download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64"
-                download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64"
+                if [ $_level == "L1" ]; then
+                    SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
+                elif [ $match_level == "L2" ];then
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
+                else
+                    SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
+                fi
+                echo " ------ using $SFILE to search $rpm_name ------"
+                # Yumdownloader with the appropriate flag for src, noarch or x86_64
+                if [ "$_type" == "src" ];then
+                    download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
+                else
+                    download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                    download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
+                fi
             fi
         else
+            # Buid wget command
             rpm_name=`echo $ff | cut -d"#" -f1-1`
             rpm_url=`echo $ff | cut -d"#" -f2-2`
             download_cmd="wget $rpm_url"
+            download_url_cmd="echo $rpm_url"
             SFILE=$rpm_name
         fi
-            echo "--> run: $download_cmd"
+
+        # Put the RPM in the Binary or Source directory
         if [ "$_type" == "src" ]; then
             if [ ! -e $MDIR_SRC/$rpm_name ]; then
                 echo "Looking for $rpm_name"
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
-                    # Use 'sort --unique' because sometimes 
+                    # Use 'sort --unique' because sometimes
                     # yumdownloader reports the url twice
-                    $download_url_cmd | sort --unique >> $URL_SRPMS
+                    URL=$($download_url_cmd | sort --unique)
+                    echo "The url is: $URL"
+                    echo "url_srpm:$URL" >> $LOG
 
                     if ! mv -f $SFILE* $MDIR_SRC ; then
                         echo "FAILED to move $rpm_name"
-                        echo $rpm_name >> $FAIL_MOVE_SRPMS
+                        echo "fail_move_srpm:$rpm_name" >> $LOG
                     fi
+                    echo "found_srpm:$rpm_name" >> $LOG
                     echo $rpm_name >> $FOUND_SRPMS
                 else
+                    echo "Warning: $rpm_name not found"
+                    echo "missing_srpm:$rpm_name" >> $LOG
                     echo $rpm_name >> $MISSING_SRPMS
+                    rc=1
                 fi
             else
-                echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
-                echo $rpm_name >> $FOUND_SRPMS
+                echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
+                echo "already_there_srpm:$rpm_name" >> $LOG
             fi
         else  ## noarch or x86_64
             if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
                 echo "Looking for $rpm_name..."
+                echo "--> run: $download_cmd"
                 if $download_cmd ; then
                     # Success!   Record download URL.
-                    # Use 'sort --unique' because sometimes 
+                    # Use 'sort --unique' because sometimes
                     # yumdownloader reports the url twice
-                    $download_url_cmd | sort --unique >> $URL_RPMS
+                    URL=$($download_url_cmd | sort --unique)
+                    echo "The url is: $URL"
+                    echo "url_rpm:$URL" >> $LOG
 
                     mkdir -p $MDIR_BIN/${_type}
                     if ! mv -f $SFILE* $MDIR_BIN/${_type}/ ; then
                         echo "FAILED to move $rpm_name"
-                        echo $rpm_name >> $FAIL_MOVE_RPMS
+                        echo "fail_move_rpm:$rpm_name" >> $LOG
                     fi
+                    echo "found_rpm:$rpm_name" >> $LOG
                     echo $rpm_name >> $FOUND_RPMS
                 else
+                    echo "Warning: $rpm_name not found"
+                    echo "missing_rpm:$rpm_name" >> $LOG
                     echo $rpm_name >> $MISSING_RPMS
+                    rc=1
                 fi
             else
                 echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
-                echo $rpm_name >> $FOUND_RPMS
+                echo "already_there_rpm:$rpm_name" >> $LOG
             fi
         fi
     done
+
+    return $rc
 }
 
-# prime the cache
-sudo -E yum makecache
+# Prime the cache
+${SUDOCMD} yum ${YUMCONFOPT} makecache
 
-#go to download *.noarch.rpm files
-noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"`
-if [ ! -z "$noarch_rpms" ];then
-    echo "--> start searching noarch RPMs ....."
-    download "$noarch_rpms" $match_level $from "noarch"
-fi
-
-#go to download *.x86_64.rpm files
-x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"`
-if [ ! -z "$x86_64_rpms" ];then
-    echo "--> start searching x86_64 RPMs ....."
-    download "$x86_64_rpms" $match_level $from "x86_64"
-fi
-
-#go to download *.src.rpm files
-src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"`
-if [ ! -z "$src_rpms" ];then
-    echo "--> start searching source RPMs ....."
-    download "$src_rpms" $match_level $from "src"
+# Download files
+if [ -s "$rpms_list" ];then
+    echo "--> start searching $rpms_list"
+    download $rpms_list $match_level
+    if [ $? -ne 0 ]; then
+        dl_rc=1
+    fi
 fi
 
 echo "done!!"
 
-exit 0
+exit $dl_rc
+

centos-mirror-tools/dl_tarball.sh

@@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do
     directory_name=$(echo $line | cut -d"#" -f2-2)
     tarball_url=$(echo $line | cut -d"#" -f3-3)
 
+    # Remove leading '!' if present
+    tarball_name="${tarball_name//!/}"
+
     # - For the General category and the Puppet category:
     #   - Packages have a common process: download, decompressed,
     #     change the directory path and compressed.
@@ -87,12 +90,16 @@ for line in $(cat $tarball_file); do
         download_directory=$output_tarball
     fi
 
+    if [ -e $download_path ]; then
+        echo "Already have $download_path"
+        continue
+    fi
+
     # We have 6 packages from the text file starting with the character "!":
     # they require special handling besides the common process: remove directory,
     # remove text from some files, clone a git repository, etc.
 
     if [[ "$line" =~ ^'!' ]]; then
-        tarball_name="${tarball_name//!/}"
         echo $tarball_name
         pushd $output_tarball
         if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
@@ -145,13 +152,18 @@ for line in $(cat $tarball_file); do
             elif [ "$pkg_version" = "4.3-1.0.1.0" ]; then
                 cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.1.0.1.1.g8509e41.src.rpm" .
                 cp "$srpm_path/rdma-core-43mlnx1-1.43101.src.rpm" .
+                cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.0.1.8.43101.src.rpm" .
             elif [ "$pkg_version" = "4.3-3.0.2.1" ]; then
                 cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm" .
                 cp "$srpm_path/rdma-core-43mlnx1-1.43302.src.rpm" .
+                cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.2.1.6.43302.src.rpm" .
             else
                 echo "$pkg_version : unknown version"
             fi
-            rm -f "$tarball_name"
+            # Don't delete the original MLNX_OFED_LINUX tarball.
+            # We don't use it, but it will prevent re-downloading this file.
+            #   rm -f "$tarball_name"
+
             rm -rf "MLNX_OFED_SRC-${pkg_version}"
             rm -rf "$directory_name"
         elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then

centos-mirror-tools/download_mirror.sh

@@ -1,124 +1,265 @@
 #!/bin/bash -e
-echo "--------------------------------------------------------------"
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 
-echo "WARNING: this script HAS TO access internet (http/https/ftp),"
-echo "so please make sure your network working properly!!"
+usage() {
+    echo "$0 [-n] [-c <yum.conf>] [-g]"
+    echo ""
+    echo "Options:"
+    echo "  -n: Do not use sudo when performing operations (option passed on to"
+    echo "      subscripts when appropriate)"
+    echo "  -c: Use an alternate yum.conf rather than the system file (option passed"
+    echo "      on to subscripts when appropriate)"
+    echo "  -g: do not change group IDs of downloaded artifacts"
+    echo ""
+}
 
-mkdir -p ./logs
+generate_log_name() {
+    filename=$1
+    level=$2
+    base=$(basename $filename .lst)
+    echo $LOGSDIR"/"$base"_download_"$level".log"
+}
 
 need_file(){
     for f in $*; do
         if [ ! -e $f ]; then
             echo "ERROR: $f does not exist."
-            exit -1
+            exit 1
         fi
     done
 }
 
+# Downloader scripts
+rpm_downloader="./dl_rpms.sh"
+tarball_downloader="./dl_tarball.sh"
+other_downloader="./dl_other_from_centos_repo.sh"
+
+# track optional arguments
+change_group_ids=1
+use_system_yum_conf=1
+rpm_downloader_extra_args=""
+tarball_downloader_extra_args=""
+
+# lst files to use as input
+rpms_from_3rd_parties="./rpms_3rdparties.lst"
+rpms_from_centos_repo="./rpms_centos.lst"
+rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
+other_downloads="./other_downloads.lst"
+
+# Overall success
+success=1
+
+# Parse out optional -c or -n arguments
+while getopts "c:ngh" o; do
+    case "${o}" in
+        n)
+            # Pass -n ("no-sudo") to rpm downloader
+            rpm_downloader_extra_args="${rpm_downloader_extra_args} -n"
+            ;;
+        c)
+            # Pass -c ("use alternate yum.conf") to rpm downloader
+            use_system_yum_conf=0
+            rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}"
+            ;;
+        g)
+            # Do not attempt to change group IDs on downloaded packages
+            change_group_ids=0
+            ;;
+        h)
+            # Help
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+echo "--------------------------------------------------------------"
+
+echo "WARNING: this script HAS TO access internet (http/https/ftp),"
+echo "so please make sure your network working properly!!"
+
+
+LOGSDIR="logs"
+mkdir -p $LOGSDIR
+
+
 # Check extistence of prerequisites files
-need_file dl_rpms.sh dl_other_from_centos_repo.sh tarball-dl.sh
-need_file rpms_from_3rd_parties.lst
-need_file rpms_from_centos_3rd_parties.lst
-need_file rpms_from_centos_repo.lst
-need_file other_downloads.lst
+need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader}
+need_file ${rpms_from_3rd_parties}
+need_file ${rpms_from_centos_3rd_parties}
+need_file ${rpms_from_centos_repo}
+need_file ${other_downloads}
 need_file tarball-dl.lst mvn-artifacts.lst
 
+
 #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
 echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
 
-# Restore StarlingX_3rd repos from backup
-REPO_SOURCE_DIR=/localdisk/yum.repos.d
-REPO_DIR=/etc/yum.repos.d
-if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
-    \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
+if [ ${use_system_yum_conf} -ne 0 ]; then
+    # Restore StarlingX_3rd repos from backup
+    REPO_SOURCE_DIR=/localdisk/yum.repos.d
+    REPO_DIR=/etc/yum.repos.d
+    if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then
+        \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/
+    fi
 fi
 
-rpm_downloader="./dl_rpms.sh"
-$rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
-if [ $? != 0 ];then
-    echo "ERROR: something wrong with downloading, please check the log!!"
+list=${rpms_from_3rd_parties}
+level=L1
+logfile=$(generate_log_name $list $level)
+$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -ne 0 ];then
+    echo "ERROR: Something wrong with downloading files listed in $list."
+    echo "   Please check the log at $(pwd)/$logfile !"
+    echo ""
+    success=0
 fi
 
 # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
-$rpm_downloader ./rpms_from_centos_3rd_parties.lst L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
+list=${rpms_from_centos_3rd_parties}
+level=L1
+logfile=$(generate_log_name $list $level)
+$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -ne 0 ];then
+    echo "ERROR: Something wrong with downloading files listed in $list."
+    echo "   Please check the log at $(pwd)/$logfile !"
+    echo ""
+    success=0
+fi
+
+if [ ${use_system_yum_conf} -eq 1 ]; then
+    # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
+    \rm -f $REPO_DIR/StarlingX_3rd*.repo
+fi
 
-# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
-\rm -f $REPO_DIR/StarlingX_3rd*.repo
 
 echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
 #download RPMs/SRPMs from CentOS repos by "yumdownloader"
+list=${rpms_from_centos_repo}
+level=L1
+logfile=$(generate_log_name $list $level)
+$rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
+retcode=${PIPESTATUS[0]}
 
-$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt
 
-if [ $? == 0 ]; then
+K1_logfile=$(generate_log_name ${rpms_from_centos_repo} K1)
+if [ $retcode -ne 1 ]; then
+    # K1 step not needed. Clear any K1 logs from previous download attempts.
+    $rpm_downloader -x $LOGSDIR/L1_rpms_missing_centos.log K1 |& tee $K1_logfile
+fi
+
+if [ $retcode -eq 0 ]; then
     echo "finish 1st round of RPM downloading successfully!"
-    if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
-        missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
-        if [ "$missing_num" != "0" ];then
-            echo "ERROR:  -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
+elif [ $retcode -eq 1 ]; then
+    echo "finish 1st round of RPM downloading with missing files!"
+    if [ -e "$LOGSDIR/L1_rpms_missing_centos.log" ]; then
+
+        echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
+        $rpm_downloader $LOGSDIR/L1_rpms_missing_centos.log K1 centos |& tee $K1_logfile
+        retcode=${PIPESTATUS[0]}
+        if [ $retcode -eq 0 ]; then
+            echo "finish 2nd round of RPM downloading successfully!"
+        elif [ $retcode -eq 1 ]; then
+            echo "finish 2nd round of RPM downloading with missing files!"
+            if [ -e "$LOGSDIR/rpms_missing_K1.log" ]; then
+                echo "WARNING: missing RPMs listed in $LOGSDIR/centos_rpms_missing_K1.log !"
+            fi
+        fi
+
+        # Remove files found by K1 download from L1_rpms_missing_centos.txt to prevent
+        # false reporting of missing files.
+        grep -v -x -F -f $LOGSDIR/K1_rpms_found_centos.log $LOGSDIR/L1_rpms_missing_centos.log  > $LOGSDIR/L1_rpms_missing_centos.tmp
+        mv -f $LOGSDIR/L1_rpms_missing_centos.tmp $LOGSDIR/L1_rpms_missing_centos.log
+
+
+        missing_num=`wc -l $LOGSDIR/K1_rpms_missing_centos.log | cut -d " " -f1-1`
+        if [ "$missing_num" != "0" ];then
+            echo "ERROR:  -------RPMs missing: $missing_num ---------------"
+            retcode=1
         fi
-        #echo "start 2nd round of downloading Binary RPMs with L2 match criteria..."
-        #$rpm_downloader ./output/centos_rpms_missing_L1.txt L2 centos | tee ./logs/log_download_rpms_from_centos_L2.txt
-        #if [ $? == 0 ]; then
-        #    echo "finish 2nd round of RPM downloading successfully!"
-        #    if [ -e "./output/rpms_missing_L2.txt" ]; then
-        #        echo "WARNING: we're still missing RPMs listed in ./rpms_missing_L2.txt !"
-        #    fi
-        #fi
     fi
 
-    if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
-        missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
+    if [ -e "$LOGSDIR/L1_srpms_missing_centos.log" ]; then
+        missing_num=`wc -l $LOGSDIR/L1_srpms_missing_centos.log | cut -d " " -f1-1`
         if [ "$missing_num" != "0" ];then
-            echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
+            echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
+            retcode=1
         fi
-        #echo "start 2nd round of downloading Source RPMs with L2 match criteria..."
-        #$rpm_downloader ./output/centos_srpms_missing_L1.txt L2 centos | tee ./logs/log_download_srpms_from_centos_L2.txt
-        #if [ $? == 0 ]; then
-        #    echo "finish 2nd round of SRPM downloading successfully!"
-        #    if [ -e "./srpms_missing_L2.txt" ]; then
-        #        echo "WARNING: we're still missing SRPMs listed in ./rpms_missing_L2.txt !"
-        #    fi
-        #fi
     fi
-else
-    echo "finish 1st round with failures!"
+fi
+
+if [ $retcode -ne 0 ]; then
+    echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
+    echo "   Please check the logs at $(pwd)/$logfile"
+    echo "   and $(pwd)/logs/$K1_logfile !"
+    echo ""
+    success=0
 fi
 
 ## verify all RPMs SRPMs we download for the GPG keys
-find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./rpm-gpg-key-missing.txt
+find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > $LOGSDIR/rpm-gpg-key-missing.txt
 
 # remove all i686.rpms to avoid pollute the chroot dep chain
-find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt
+find ./output -name "*.i686.rpm" | tee $LOGSDIR/all_i686.txt
 find ./output -name "*.i686.rpm" | xargs rm -f
 
-line1=`wc -l rpms_from_3rd_parties.lst | cut -d " " -f1-1`
-line2=`wc -l rpms_from_centos_repo.lst | cut -d " " -f1-1`
-line3=`wc -l rpms_from_centos_3rd_parties.lst | cut -d " " -f1-1`
+line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
+line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
+line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
 let total_line=$line1+$line2+$line3
-echo "We expect to download $total_line RPMs."
+echo "We expected to download $total_line RPMs."
 num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
 echo "There are $num_of_downloaded_rpms RPMs in output directory."
 if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
     echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs."
 fi
 
-# change "./output" and sub-folders to 751 (cgcs) group
-chown  751:751 -R ./output
+if [ $change_group_ids -eq 1 ]; then
+    # change "./output" and sub-folders to 751 (cgcs) group
+    chown  751:751 -R ./output
+fi
 
 
 echo "step #3: start downloading other files ..."
 
-other_downloader="./dl_other_from_centos_repo.sh"
-$other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
-if [ $? == 0 ];then
+logfile=$LOGSDIR"/otherfiles_centos_download.log"
+${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -eq 0 ];then
     echo "step #3: done successfully"
+else
+    echo "step #3: finished with errors"
+    echo "ERROR: Something wrong with downloading from ${other_downloads}."
+    echo "   Please check the log at $(pwd)/$logfile!"
+    echo ""
+    success=0
 fi
 
+
 # StarlingX requires a group of source code pakages, in this section
 # they will be downloaded.
 echo "step #4: start downloading tarball compressed files"
-./dl_tarball.sh
+logfile=$LOGSDIR"/tarballs_download.log"
+${tarball_downloader} ${tarball_downloader_extra_args}  |& tee $logfile
+retcode=${PIPESTATUS[0]}
+if [ $retcode -eq 0 ];then
+    echo "step #4: done successfully"
+else
+    echo "step #4: finished with errors"
+    echo "ERROR: Something wrong with downloading tarballs."
+    echo "   Please check the log at $(pwd)/$logfile !"
+    echo ""
+    success=0
+fi
 
 echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
 echo "on them, the workable images for StarlingX could be generated by"
@@ -127,3 +268,11 @@ echo "    - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
 
+echo ""
+if [ $success -ne 1 ]; then
+    echo "Warning: Not all download steps succeeded.  You are likely missing files."
+    exit 1
+fi
+
+echo "Success"
+exit 0

centos-mirror-tools/mirror-check.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# This script checks if the required packages in the .lst file list is
+# actually downloadable. Sometimes, the release number in upstream is
+# changed and that causes a mismatch in the build requirements.
+# We can find this problems in an early stage without the need to
+# download all the packages.
+#
+# The yum cache contains this information, more specific the primary_db
+# files, so iterating over the content of .lst, parse the name of the
+# package and get the information on what is available to download
+# should be enough to know the status of the mirror.
+#
+# If a package is not found then the script will try to get the avai-
+# lable version and log that into the error log. By this way we get
+# notified on what changed in the external repositories.
+#
+# How to run:
+# This script is intended to be run inside the downloader container.
+# It needs that all the CentOS repositories are well setup.
+#
+# ./mirror-check.sh
+#
+# And you should see the checking in progress.
+
+_print_msg() { echo -en "$(date -u +"%Y-%m-%d %H-%M-%S") ==> $1"; }
+info() { _print_msg "INFO: $1\n"; }
+info_c() { _print_msg "INFO: $1"; }
+warning() { _print_msg "WARN: $1\n"; }
+error() { _print_msg "ERROR: $1\n"; }
+
+RPMS_CENTOS_LIST="rpms_centos.lst"
+RPMS_3RD_PARTY_LIST="rpms_centos3rdparties.lst"
+ERROR_LOG_FILE="mirror-check-failures.log"
+truncate -s 0 $ERROR_LOG_FILE
+retcode=0
+extra_opts=""
+
+usage() {
+    echo "$0 [-c <yum.conf>]"
+    echo ""
+    echo "Options:"
+    echo "  -c: Use an alternate yum.conf rather than the system file (option passed"
+    echo "      on to subscripts when appropriate)"
+    echo ""
+}
+
+get_rpm_name() {
+    _rpm_file_name=$1
+    rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'-' -f3- | rev)
+    echo "$rpm_name"
+}
+
+get_rpm_full_name() {
+    _rpm_file_name=$1
+    rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'.' -f2- | rev)
+    echo "$rpm_name"
+}
+
+get_rpm_arch() {
+    arch=$(echo "$1" | rev | cut -d'.' -f2 | rev)
+    echo "$arch"
+}
+
+get_repoquery_info() {
+    _arch=$1
+    _package_name=$2
+    if [ "$_arch" == "x86_64" ]; then
+        # To filter out the i686 packages
+        repoquery_opts="--archlist=x86_64"
+    elif [ "$_arch" == "src" ]; then
+        repoquery_opts="--archlist=src"
+    else
+        repoquery_opts=
+    fi
+    repoquery $extra_opts -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \
+              $repoquery_opts "$_package_name"
+}
+
+_check_rpms() {
+    p=$1
+    full_name=$(get_rpm_full_name "$p")
+    rpm_name=$(get_rpm_name "$p")
+    arch=$(get_rpm_arch "$p")
+    info_c "Checking $full_name... "
+    _repoquery=$(get_repoquery_info "$arch" "$full_name")
+    if [ -z "$_repoquery" ]; then
+        echo -e "FAILED!"
+        available_pkgs=$(get_repoquery_info "$arch" "$rpm_name")
+        echo -e "Package $full_name not found, available $available_pkgs" >> $ERROR_LOG_FILE
+        retcode=1
+    else
+        if [ "$full_name" == "$_repoquery" ]; then
+            echo -e "OK"
+        else
+            echo -e "FAILED!"
+            retcode=1
+            echo -e "Required $full_name but found $_repoquery" >> $ERROR_LOG_FILE
+        fi
+    fi
+}
+
+check_rpms() {
+    _rpms_list=$1
+    for p in $_rpms_list; do
+        _check_rpms "$p"
+    done
+}
+
+while getopts "c:" opt; do
+    case $opt in
+        c)
+            extra_opts="-c ${OPTARG}"
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG" >&2
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+info "Getting yum cache"
+if ! yum $extra_opts makecache; then
+    error "There was a problem getting yum cache"
+    exit 1
+fi
+
+for rpm_list in "$RPMS_CENTOS_LIST" "$RPMS_3RD_PARTY_LIST"; do
+    info "Reading $rpm_list..."
+    for arch in "src" "noarch" "x86_64"; do
+        info "Getting info for $arch packages..."
+        rpms=$(echo "$(grep -F "$arch.rpm" < $rpm_list)")
+        check_rpms "$rpms"
+    done
+done
+
+if [ $retcode -ne 0 ]; then
+    error "Failures found, error log:"
+    error "=========================="
+    cat $ERROR_LOG_FILE
+fi
+
+exit $retcode

centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization

@@ -0,0 +1,20 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v2.0.22 (GNU/Linux)
+
+mQENBFWB31YBCAC4dFmTzBDOcq4R1RbvQXLkyYfF+yXcsMA5kwZy7kjxnFqBoNPv
+aAjFm3e5huTw2BMZW0viLGJrHZGnsXsE5iNmzom2UgCtrvcG2f65OFGlC1HZ3ajA
+8ZIfdgNQkPpor61xqBCLzIsp55A7YuPNDvatk/+MqGdNv8Ug7iVmhQvI0p1bbaZR
+0GuavmC5EZ/+mDlZ2kHIQOUoInHqLJaX7iw46iLRUnvJ1vATOzTnKidoFapjhzIt
+i4ZSIRaalyJ4sT+oX4CoRzerNnUtIe2k9Hw6cEu4YKGCO7nnuXjMKz7Nz5GgP2Ou
+zIA/fcOmQkSGcn7FoXybWJ8DqBExvkJuDljPABEBAAG0bENlbnRPUyBWaXJ0dWFs
+aXphdGlvbiBTSUcgKGh0dHA6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lhbEludGVy
+ZXN0R3JvdXAvVmlydHVhbGl6YXRpb24pIDxzZWN1cml0eUBjZW50b3Mub3JnPokB
+OQQTAQIAIwUCVYHfVgIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEHrr
+voJh6IBsRd0H/A62i5CqfftuySOCE95xMxZRw8+voWO84QS9zYvDEnzcEQpNnHyo
+FNZTpKOghIDtETWxzpY2ThLixcZOTubT+6hUL1n+cuLDVMu4OVXBPoUkRy56defc
+qkWR+UVwQitmlq1ngzwmqVZaB8Hf/mFZiB3B3Jr4dvVgWXRv58jcXFOPb8DdUoAc
+S3u/FLvri92lCaXu08p8YSpFOfT5T55kFICeneqETNYS2E3iKLipHFOLh7EWGM5b
+Wsr7o0r+KltI4Ehy/TjvNX16fa/t9p5pUs8rKyG8SZndxJCsk0MW55G9HFvQ0FmP
+A6vX9WQmbP+ml7jsUxtEJ6MOGJ39jmaUvPc=
+=ZzP+
+-----END PGP PUBLIC KEY BLOCK-----

centos-mirror-tools/rpms_3rdparties.lst

@@ -5,5 +5,4 @@ python2-pytest-httpbin-0.2.3-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/pa
 python2-pytest-mock-1.6.0-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-pytest-mock/1.6.0/2.el7/noarch/python2-pytest-mock-1.6.0-2.el7.noarch.rpm
 python2-storops-0.4.7-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-storops/0.4.7/2.el7/noarch/python2-storops-0.4.7-2.el7.noarch.rpm
 python-gunicorn-19.7.1-1.fc27.src.rpm#https://kojipkgs.fedoraproject.org/packages/python-gunicorn/19.7.1/1.fc27/src/python-gunicorn-19.7.1-1.fc27.src.rpm
-kubernetes-1.10.0-1.el7.src.rpm#http://cbs.centos.org/kojifiles/packages/kubernetes/1.10.0/1.el7/src/kubernetes-1.10.0-1.el7.src.rpm
 influxdb-0.9.5.1-1.x86_64.rpm#https://s3.amazonaws.com/influxdb/influxdb-0.9.5.1-1.x86_64.rpm

centos-mirror-tools/rpms_centos.lst

@@ -524,6 +524,7 @@ kmod-libs-20-21.el7.x86_64.rpm
 kpartx-0.4.9-119.el7.x86_64.rpm
 krb5-devel-1.15.1-19.el7.x86_64.rpm
 krb5-libs-1.15.1-19.el7.x86_64.rpm
+kubernetes-1.10.0-1.el7.src.rpm
 lapack-3.4.2-8.el7.x86_64.rpm
 latex2html-2012-3.el7.noarch.rpm
 lato-fonts-2.015-1.el7.noarch.rpm
@@ -573,6 +574,7 @@ libedit-3.0-12.20121213cvs.el7.x86_64.rpm
 libedit-devel-3.0-12.20121213cvs.el7.x86_64.rpm
 libepoxy-1.3.1-1.el7.x86_64.rpm
 libepoxy-devel-1.3.1-1.el7.x86_64.rpm
+liberasurecode-1.5.0-1.el7.x86_64.rpm
 liberation-fonts-common-1.07.2-16.el7.noarch.rpm
 liberation-mono-fonts-1.07.2-16.el7.noarch.rpm
 liberation-narrow-fonts-1.07.2-16.el7.noarch.rpm
@@ -884,6 +886,11 @@ openstack-aodh-3.0.4-1.el7.src.rpm
 openstack-macros-2018.1.0-0.noarch.rpm
 openstack-nova-cert-14.0.8-1.el7.noarch.rpm
 openstack-panko-3.0.0-1.el7.src.rpm
+openstack-swift-account-2.15.1-1.el7.noarch.rpm
+openstack-swift-container-2.15.1-1.el7.noarch.rpm
+openstack-swift-doc-2.15.1-1.el7.noarch.rpm
+openstack-swift-object-2.15.1-1.el7.noarch.rpm
+openstack-swift-proxy-2.15.1-1.el7.noarch.rpm
 openvswitch-2.9.0-3.el7.src.rpm
 orc-0.4.26-1.el7.x86_64.rpm
 osinfo-db-tools-1.1.0-1.el7.x86_64.rpm
@@ -1093,6 +1100,7 @@ python2-cursive-0.1.2-1.el7.noarch.rpm
 python2-cycler-0.10.0-2.el7.noarch.rpm
 python2-Cython-0.25.2-3.el7.x86_64.rpm
 python2-daiquiri-1.2.1-1.el7.noarch.rpm
+python2-dateutil-2.6.1-1.el7.noarch.rpm
 python2-ddt-1.1.3-1.el7.noarch.rpm
 python2-debtcollector-1.17.0-1.el7.noarch.rpm
 python2-deprecation-1.0-3.el7.noarch.rpm
@@ -1113,6 +1121,7 @@ python2-gabbi-1.33.0-1.el7.noarch.rpm
 python2-gevent-1.1.2-2.el7.x86_64.rpm
 python2-gflags-2.0-5.el7.noarch.rpm
 python2-gnocchiclient-3.3.1-1.el7.noarch.rpm
+python2-google-auth-1.3.0-1.el7.noarch.rpm
 python2-greenlet-0.4.9-1.el7.x86_64.rpm
 python2-hacking-0.13.0-1.el7.noarch.rpm
 python2-idna-2.5-1.el7.noarch.rpm
@@ -1128,7 +1137,7 @@ python2-jsonschema-2.5.1-3.el7.noarch.rpm
 python2-keystoneauth1-3.1.0-1.el7.noarch.rpm
 python2-keystoneclient-3.13.0-1.el7.noarch.rpm
 python2-kombu-4.0.2-5.el7.noarch.rpm
-python2-kubernetes-1.0.0-0.3.b3.el7.noarch.rpm
+python2-kubernetes-4.0.0-1.el7.noarch.rpm
 python2-lz4-0.9.0-1.el7.x86_64.rpm
 python2-marathon-0.8.8-1.el7.noarch.rpm
 python2-markupsafe-0.23-16.el7.x86_64.rpm
@@ -1191,16 +1200,17 @@ python2-pycadf-2.6.0-1.el7.noarch.rpm
 python2-pycodestyle-2.0.0-5.el7.noarch.rpm
 python2-pygments-2.2.0-7.el7.noarch.rpm
 python2-PyMySQL-0.7.11-1.el7.noarch.rpm
-python2-pyngus-2.2.3-1.el7.noarch.rpm
+python2-pyngus-2.2.4-1.el7.noarch.rpm
 python2-pyOpenSSL-16.2.0-3.el7.noarch.rpm
 python2-pyparsing-2.1.10-3.el7.noarch.rpm
 python2-pyroute2-0.4.19-1.el7.noarch.rpm
 python2-pysaml2-3.0.2-2.el7.noarch.rpm
 python2-pysnmp-4.3.2-3.el7.noarch.rpm
 python2-pytest-3.0.6-2.el7.noarch.rpm
-python2-qpid-proton-0.22.0-1.el7.x86_64.rpm
+python2-qpid-proton-0.24.0-1.el7.x86_64.rpm
 python2-rcssmin-1.0.6-2.el7.x86_64.rpm
 python2-reno-2.5.0-1.el7.noarch.rpm
+python2-requests-oauthlib-0.8.0-5.el7.noarch.rpm
 python2-requestsexceptions-1.3.0-1.el7.noarch.rpm
 python2-retryz-0.1.8-1.el7.noarch.rpm
 python2-rfc3986-0.3.1-1.el7.noarch.rpm
@@ -1276,6 +1286,7 @@ python-betamax-0.7.0-1.el7.noarch.rpm
 python-Bottleneck-0.7.0-1.el7.x86_64.rpm
 python-bson-3.0.3-1.el7.x86_64.rpm
 python-cachetools-1.1.6-2.el7.noarch.rpm
+python-ceilometermiddleware-1.1.0-1.el7.noarch.rpm
 python-cffi-1.6.0-5.el7.x86_64.rpm
 python-characteristic-14.3.0-4.el7.noarch.rpm
 python-chardet-2.2.1-1.el7_1.noarch.rpm
@@ -1290,6 +1301,7 @@ python-croniter-0.3.4-2.el7.noarch.rpm
 python-d2to1-0.2.11-1.el7.noarch.rpm
 python-daemon-1.6-4.el7.noarch.rpm
 python-dateutil-2.4.2-1.el7.noarch.rpm
+python-dateutil-2.6.1-1.el7.src.rpm
 python-decorator-3.4.0-3.el7.noarch.rpm
 python-deltarpm-3.6-3.el7.x86_64.rpm
 python-django-1.8.14-1.el7.noarch.rpm
@@ -1300,6 +1312,7 @@ python-django-compressor-2.0-1.el7.noarch.rpm
 python-django-nose-1.4.3-1.el7.noarch.rpm
 python-django-openstack-auth-3.5.0-1.el7.src.rpm
 python-django-pyscss-2.0.2-1.el7.noarch.rpm
+python-dns-1.12.0-4.20150617git465785f.el7.noarch.rpm
 python-docker-pycreds-1.10.6-1.el7.noarch.rpm
 python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm
 python-dogpile-cache-0.6.2-1.el7.noarch.rpm
@@ -1316,6 +1329,7 @@ python-flake8-2.4.1-2.el7.noarch.rpm
 python-flask-0.10.1-4.el7.noarch.rpm
 python-freezegun-0.3.8-2.el7.noarch.rpm
 python-glance-store-0.22.0-1.el7.src.rpm
+python-google-auth-1.3.0-1.el7.src.rpm
 python-heatclient-1.11.0-1.el7.src.rpm
 python-httplib2-0.9.2-1.el7.noarch.rpm
 python-iniparse-0.4-9.el7.noarch.rpm
@@ -1336,6 +1350,7 @@ python-keystoneclient-3.13.0-1.el7.src.rpm
 python-keystonemiddleware-4.17.0-1.el7.src.rpm
 python-kitchen-1.1.1-5.el7.noarch.rpm
 python-kmod-0.9-4.el7.x86_64.rpm
+python-kubernetes-4.0.0-1.el7.src.rpm
 python-ldap-2.4.15-2.el7.x86_64.rpm
 python-ldappool-1.0-4.el7.noarch.rpm
 python-lesscpy-0.9j-4.el7.noarch.rpm
@@ -1395,6 +1410,7 @@ python-py-1.4.32-1.el7.noarch.rpm
 python-pycadf-common-2.6.0-1.el7.noarch.rpm
 python-pycparser-2.14-1.el7.noarch.rpm
 python-pycurl-7.19.0-19.el7.x86_64.rpm
+python-pyeclib-1.5.0-1.el7.x86_64.rpm
 python-pyelftools-0.22-0.5.git20130619.a1d9681.el7.noarch.rpm
 python-pymongo-3.0.3-1.el7.x86_64.rpm
 python-pytest-cov-2.5.1-1.el7.noarch.rpm
@@ -1422,6 +1438,8 @@ python-sqlparse-0.1.18-5.el7.noarch.rpm
 python-srpm-macros-3-22.el7.noarch.rpm
 python-subprocess32-3.2.6-4.el7.x86_64.rpm
 python-subunit-1.0.0-1.el7.noarch.rpm
+python-swift-2.15.1-1.el7.noarch.rpm
+python-swift-tests-2.15.1-1.el7.noarch.rpm
 python-sysv_ipc-0.4.2-11.el7.x86_64.rpm
 python-tables-3.2.0-1.el7.x86_64.rpm
 python-tablib-0.10.0-1.el7.noarch.rpm
@@ -1466,7 +1484,7 @@ pytz-2016.10-2.el7.noarch.rpm
 pyxattr-0.5.1-5.el7.x86_64.rpm
 PyYAML-3.10-11.el7.x86_64.rpm
 qdox-1.12.1-10.el7.noarch.rpm
-qpid-proton-c-0.22.0-1.el7.x86_64.rpm
+qpid-proton-c-0.24.0-1.el7.x86_64.rpm
 qrencode-devel-3.4.1-3.el7.x86_64.rpm
 qrencode-libs-3.4.1-3.el7.x86_64.rpm
 qt-4.8.5-13.el7.x86_64.rpm
@@ -1514,6 +1532,7 @@ sanlock-3.6.0-1.el7.src.rpm
 sazanami-fonts-common-0.20040629-22.el7.noarch.rpm
 sazanami-gothic-fonts-0.20040629-22.el7.noarch.rpm
 sazanami-mincho-fonts-0.20040629-22.el7.noarch.rpm
+screen-4.1.0-0.25.20120314git3c2946.el7.x86_64.rpm
 scrub-2.5.2-7.el7.x86_64.rpm
 scsi-target-utils-1.0.55-4.el7.src.rpm
 SDL-1.2.15-14.el7.x86_64.rpm

centos-mirror-tools/tarball-dl.lst

@@ -4,7 +4,6 @@ drbd-8.4.3.tar.gz#drbd-8.4.3#http://www.linbit.com/downloads/drbd/8.4/archive/dr
 drbd-8.4.7-1.tar.gz#drbd-8.4.7-1#http://www.linbit.com/downloads/drbd/8.4/drbd-8.4.7-1.tar.gz
 dtc-1.4.4.tar.gz#dtc-1.4.4#https://www.kernel.org/pub/software/utils/dtc/dtc-1.4.4.tar.gz
 e1000e-3.4.1.1.tar.gz#e1000e-3.4.1.1#https://sourceforge.net/projects/e1000/files/e1000e%20stable/3.4.1.1/e1000e-3.4.1.1.tar.gz
-expect-lite_4.9.0.tar.gz#expect-lite.proj#https://sourceforge.net/projects/expect-lite/files/expect-lite/expect-lite_4.9.0/expect-lite_4.9.0.tar.gz
 gnulib-ffc927e.tar.gz#gnulib-ffc927e#http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-ffc927eef29016a5219cd969daad8928af6a1f4d.tar.gz
 i40e-2.4.10.tar.gz#i40e-2.4.10#https://sourceforge.net/projects/e1000/files/i40e%20stable/2.4.10/i40e-2.4.10.tar.gz/download
 i40evf-3.5.13.tar.gz#i40evf-3.5.13#https://sourceforge.net/projects/e1000/files/i40evf%20stable/3.5.13/i40evf-3.5.13.tar.gz/download

centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo

@@ -100,3 +100,11 @@ gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
 
+[Starlingx-C7.5.1804-virt-kubernetes-source]
+name=Starlingx-CentOS-7.5.1804 - virt-kubernetes-source
+baseurl=http://vault.centos.org/7.5.1804/virt/Source/kubernetes110
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
+enabled=1
+
+

centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo

@@ -0,0 +1,41 @@
+# CentOS-OpenStack-queens.repo
+#
+# Please see http://wiki.centos.org/SpecialInterestGroup/Cloud for more
+# information
+
+[centos-openstack-queens]
+name=CentOS-7 - OpenStack queens
+baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/
+gpgcheck=1
+enabled=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
+exclude=sip,PyQt4
+
+[centos-openstack-queens-test]
+name=CentOS-7 - OpenStack queens Testing
+baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-queens/
+gpgcheck=0
+enabled=0
+exclude=sip,PyQt4
+
+[centos-openstack-queens-debuginfo]
+name=CentOS-7 - OpenStack queens - Debug
+baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/
+gpgcheck=1
+enabled=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
+exclude=sip,PyQt4
+
+[centos-openstack-queens-source]
+name=CentOS-7 - OpenStack queens - Source
+baseurl=http://vault.centos.org/centos/7/cloud/Source/openstack-queens/
+gpgcheck=1
+enabled=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
+exclude=sip,PyQt4
+
+[rdo-trunk-queens-tested]
+name=OpenStack queens Trunk Tested
+baseurl=https://trunk.rdoproject.org/centos7-queens/current-passed-ci/
+gpgcheck=0
+enabled=0

deployment/README.rst

@@ -0,0 +1,141 @@
+StarlingX Deployment in Virtualized Environments
+================================================
+
+A StarlingX system can be installed in a variety of platforms with the following
+deployment options:
+
+- Standard Controller
+
+  - Dedicated Storage
+  - Controller Storage
+
+- All-in-one
+
+  - Duplex
+  - Simplex
+
+Deployment options uses a variety of configurations based on 3 node identities:
+
+- Controller
+- Storage
+- Compute
+
+Standard Controller :: Dedicated Storage
+----------------------------------------
+
+The software installation workflow for an initial Ceph-backed block
+storage on dedicated storage nodes is:
+
+- Controller-0 Installation and Provisioning
+- Controller-1 / Compute Host / Storage Host Installation
+- Controller-1 Provisioning
+- Provider Network Configuration
+- Compute Host Provisioning
+- Storage Host Provisioning
+
+Standard Controller :: Controller Storage
+-----------------------------------------
+
+The software installation workflow for an initial LVM-backed block
+storage on controller nodes is:
+
+- Controller-0 Installation
+- Controller-0 and System Provisioning
+- Controller-1 / Compute Host Installation
+- Controller-1 Provisioning
+- Compute Host Provisioning
+
+All-in-one :: Duplex
+--------------------
+
+The software installation workflow for two combined controller / compute
+nodes is:
+
+- Controller-0 Installation and Provisioning
+- Controller-1 Installation and Provisioning
+
+All-in-one :: Simplex
+---------------------
+
+The software installation workflow for a single combined controller / compute
+node is:
+
+- Controller-0 Installation and Provisioning
+
+Virtualization Environments
+---------------------------
+
+The available virtualization products where StarlingX has been deployed
+are:
+
+- VirtualBox
+- Libvirt/QEMU
+
+Directory Structure
+-------------------
+
+Deployment directory hosts a total of 3 directories and 18 files::
+
+    $ tree -L 3 deployment/
+    deployment/
+    ├── libvirt
+    │   ├── compute.xml
+    │   ├── controller_allinone.xml
+    │   ├── controller.xml
+    │   ├── destroy_allinone.sh
+    │   ├── destroy_standard_controller.sh
+    │   ├── install_packages.sh
+    │   ├── setup_allinone.sh
+    │   └── setup_standard_controller.sh
+    ├── provision
+    │   ├── simplex_stage_1.sh
+    │   └── simplex_stage_2.sh
+    └── virtualbox
+        ├── all_in_one.conf
+        ├── serial_vm.sh
+        ├── setup_vm.sh
+        ├── standard_controller.conf
+        ├── start_vm.sh
+        └── stop_vm.sh
+
+Directory: libvirt
+~~~~~~~~~~~~~~~~~~
+
+Deployment under Libvirt/QEMU uses a set of xml files to define the node
+identity:
+
+- Controller All-in-one
+- Controller
+- Compute
+
+These nodes are used to create the virtual machines and the network interfaces
+to setup the StarlingX system:
+
+- Setup All-in-one
+
+  - 2 Controllers
+
+- Setup Standard Controller
+
+  - 2 Controllers
+  - 2 Computes
+
+Directory: virtualbox
+~~~~~~~~~~~~~~~~~~~~~
+
+Deployment under VirtualBox uses a set of configuration files to define the
+StarlingX system:
+
+- All-in-one Configuration
+- Standard Controller Configuration
+
+These configurations files are used to create the virtual machines and the
+network interfaces from a single script:
+
+- Setup VM
+
+Directory: provision
+~~~~~~~~~~~~~~~~~~~~
+
+A set of scripts are provided to automate the provisioning of data interfaces and
+local storage resources for the compute function for StarlingX Duplex or Simplex.

deployment/libvirt/README.rst

@@ -0,0 +1,65 @@
+StarlingX Deployment on Libvirt
+===============================
+
+This is a quick reference for deploying StarlingX on libvirt/qemu systems.
+It assumes you have a working libvirt/qemu installation for a non-root user
+and that your user has NOPASSWD sudo permissions.
+
+Overview
+--------
+
+We create 4 bridges to use for the STX cloud.  This is done in an initial step
+separate from the VM management.
+
+Depending on which basic configuration is chosen, we create a number of VMs
+for one or more controllers and storage nodes.
+
+These scripts are configured using environment variables that all have built-in
+defaults.  On shared systems you probably do not want to use the defaults.
+The simplest way to handle this is to keep an rc file that can be sourced into
+an interactive shell that configures everything.  Here's an example::
+
+	export CONTROLLER=madcloud
+	export COMPUTE=madnode
+	export BRIDGE_INTERFACE=madbr
+	export INTERNAL_NETWORK=172.30.20.0/24
+	export INTERNAL_IP=172.30.20.1/24
+	export EXTERNAL_NETWORK=192.168.20.0/24
+    export EXTERNAL_IP=192.168.20.1/24
+
+There is also a script ``cleanup_network.sh`` that will remove networking
+configuration from libvirt.
+
+Networking
+----------
+
+Configure the bridges using ``setup_network.sh`` before doing anything else. It
+will create 4 bridges named ``stxbr1``, ``stxbr2``, ``stxbr3`` and ``stxbr4``.
+Set the BRIDGE_INTERFACE environment variable if you need to change stxbr to
+something unique.
+
+The ``destroy_network.sh`` script does the reverse, and should not be used lightly.
+It should also only be used after all of the VMs created below have been destroyed.
+
+Controllers
+-----------
+
+There are two scripts for creating the controllers: ``setup_allinone.sh`` and
+``setup_standard_controller.sh``.  They are operated in the same manner but build
+different StarlingX cloud configurations.  Choose wisely.
+
+You need an ISO file for the installation, these scripts take a name with the
+``-i`` option::
+
+	./setup_allinone.sh -i stx-2018-08-28-93.iso
+
+And the setup will begin.  The scripts create one or more VMs and start the boot
+of the first controller, named oddly enough ``controller-0``.  If you have Xwindows
+available you will get virt-manager running.  If not, Ctrl-C out of that attempt if
+it doesn't return to a shell prompt.  Then connect to the serial console::
+
+	virsh console madcloud-0
+
+Continue the usual SterlingX installation from this point forward.
+
+Tear down the VMs using ``destroy_allinone.sh`` and ``destroy_standard_controller.sh``.

deployment/libvirt/cleanup_network.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+#
+# cleanup_network.sh - Cleans up network interfaces - not safe to run blindly!
+
+NETWORK_DEFAULT=${NETWORK_DEFAULT:-default}
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE=stxbr0}
+
+if virsh net-list --name | grep ${NETWORK_DEFAULT} ; then
+    sudo virsh net-destroy ${NETWORK_DEFAULT}
+    sudo virsh net-undefine ${NETWORK_DEFAULT}
+    sudo rm -rf /etc/libvirt/qemu/networks/autostart/${NETWORK_DEFAULT}.xml
+fi
+
+if [ -d "/sys/class/net/${BRIDGE_INTERFACE}" ]; then
+    sudo ifconfig ${BRIDGE_INTERFACE} down || true
+    sudo brctl delbr ${BRIDGE_INTERFACE} || true
+fi

deployment/libvirt/compute.xml

@@ -0,0 +1,115 @@
+<domain type='kvm' id='187'>
+  <name>NAME</name>
+  <memory unit='KiB'>16777216</memory>
+  <currentMemory unit='KiB'>16777216</currentMemory>
+  <vcpu placement='static'>4</vcpu>
+  <resource>
+    <partition>/machine</partition>
+  </resource>
+  <os>
+    <type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <cpu mode='host-model'>
+    <model fallback='allow'/>
+    <topology sockets='1' cores='4' threads='1'/>
+    <feature policy='optional' name='vmx'/>
+    <feature policy='optional' name='svm'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK0'/>
+      <backingStore/>
+      <target dev='sda' bus='sata'/>
+      <boot order='1'/>
+      <alias name='sata0-0-0'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK1'/>
+      <backingStore/>
+      <target dev='sdb' bus='sata'/>
+      <alias name='sata0-0-1'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='1'/>
+    </disk>
+    <controller type='usb' index='0'>
+      <alias name='usb'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'>
+      <alias name='pci.0'/>
+    </controller>
+    <controller type='sata' index='0'>
+      <alias name='sata0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </controller>
+    <interface type='bridge'>
+      <source bridge='%BR1%'/>
+      <target dev='vnet8'/>
+      <model type='e1000'/>
+      <alias name='net0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <source bridge='%BR2%'/>
+      <target dev='vnet9'/>
+      <model type='e1000'/>
+      <boot order='2'/>
+      <alias name='net1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <source bridge='%BR3%'/>
+      <target dev='vnet10'/>
+      <model type='virtio'/>
+      <alias name='net2'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <source bridge='%BR4%'/>
+      <target dev='vnet11'/>
+      <model type='virtio'/>
+      <alias name='net3'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
+    </interface>
+    <serial type='pty'>
+      <source path='/dev/pts/12'/>
+      <target port='0'/>
+      <alias name='serial0'/>
+    </serial>
+    <console type='pty' tty='/dev/pts/12'>
+      <source path='/dev/pts/12'/>
+      <target type='serial' port='0'/>
+      <alias name='serial0'/>
+    </console>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='vnc' port='5903' autoport='yes' listen='127.0.0.1' keymap='en-us'>
+      <listen type='address' address='127.0.0.1'/>
+    </graphics>
+    <video>
+      <model type='cirrus' vram='16384' heads='1'/>
+      <alias name='video0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </video>
+    <memballoon model='virtio'>
+      <alias name='balloon0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
+    </memballoon>
+  </devices>
+  <seclabel type='dynamic' model='apparmor' relabel='yes'>
+    <label>libvirt-608ab5c8-8d11-4bdd-885f-f8b5fee12ff0</label>
+    <imagelabel>libvirt-608ab5c8-8d11-4bdd-885f-f8b5fee12ff0</imagelabel>
+  </seclabel>
+</domain>

deployment/libvirt/controller.xml

@@ -0,0 +1,130 @@
+<domain type='kvm' id='164'>
+  <name>NAME</name>
+  <memory unit='KiB'>16777216</memory>
+  <currentMemory unit='KiB'>16777216</currentMemory>
+  <vcpu placement='static'>4</vcpu>
+  <resource>
+    <partition>/machine</partition>
+  </resource>
+  <os>
+    <type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <cpu mode='host-model'>
+    <model fallback='allow'/>
+    <topology sockets='1' cores='4' threads='1'/>
+    <feature policy='optional' name='vmx'/>
+    <feature policy='optional' name='svm'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK0'/>
+      <backingStore/>
+      <target dev='sda' bus='sata'/>
+      <boot order='1'/>
+      <alias name='sata0-0-0'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK1'/>
+      <backingStore/>
+      <target dev='sdb' bus='sata'/>
+      <alias name='sata0-0-1'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='1'/>
+    </disk>
+    <disk type='file' device='cdrom'>
+      <driver name='qemu' type='raw'/>
+      <source file='ISO'/>
+      <backingStore/>
+      <target dev='sdc' bus='sata'/>
+      <readonly/>
+      <boot order='2'/>
+      <alias name='sata0-0-2'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='2'/>
+    </disk>
+    <controller type='usb' index='0'>
+      <alias name='usb'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'>
+      <alias name='pci.0'/>
+    </controller>
+    <controller type='sata' index='0'>
+      <alias name='sata0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </controller>
+    <interface type='bridge'>
+      <mac address='52:54:00:da:8c:ad'/>
+      <source bridge='%BR1%'/>
+      <target dev='vnet0'/>
+      <model type='e1000'/>
+      <alias name='net0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:da:48:ff'/>
+      <source bridge='%BR2%'/>
+      <target dev='vnet1'/>
+      <model type='e1000'/>
+      <boot order='3'/>
+      <alias name='net1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:54:e6:c2'/>
+      <source bridge='%BR3%'/>
+      <target dev='vnet2'/>
+      <model type='virtio'/>
+      <alias name='net2'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:c5:62:6d'/>
+      <source bridge='%BR4%'/>
+      <target dev='vnet3'/>
+      <model type='virtio'/>
+      <alias name='net3'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
+    </interface>
+    <serial type='pty'>
+      <source path='/dev/pts/8'/>
+      <target port='0'/>
+      <alias name='serial0'/>
+    </serial>
+    <console type='pty' tty='/dev/pts/8'>
+      <source path='/dev/pts/8'/>
+      <target type='serial' port='0'/>
+      <alias name='serial0'/>
+    </console>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='vnc' port='5900' autoport='yes' listen='127.0.0.1' keymap='en-us'>
+      <listen type='address' address='127.0.0.1'/>
+    </graphics>
+    <video>
+      <model type='cirrus' vram='16384' heads='1'/>
+      <alias name='video0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </video>
+    <memballoon model='virtio'>
+      <alias name='balloon0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
+    </memballoon>
+  </devices>
+  <seclabel type='dynamic' model='apparmor' relabel='yes'>
+    <label>libvirt-6afab2ba-0ed0-45cb-b1bd-985e211a48de</label>
+    <imagelabel>libvirt-6afab2ba-0ed0-45cb-b1bd-985e211a48de</imagelabel>
+  </seclabel>
+</domain>
+

deployment/libvirt/controller_allinone.xml

@@ -0,0 +1,130 @@
+<domain type='kvm' id='164'>
+  <name>controller-0-allinone</name>
+  <memory unit='MiB'>24576</memory>
+  <currentMemory unit='MiB'>24576</currentMemory>
+  <vcpu placement='static'>4</vcpu>
+  <resource>
+    <partition>/machine</partition>
+  </resource>
+  <os>
+    <type arch='x86_64' machine='pc-i440fx-xenial'>hvm</type>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <cpu mode='host-model'>
+    <model fallback='allow'/>
+    <topology sockets='1' cores='4' threads='1'/>
+    <feature policy='optional' name='vmx'/>
+    <feature policy='optional' name='svm'/>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK0'/>
+      <backingStore/>
+      <target dev='sda' bus='sata'/>
+      <boot order='1'/>
+      <alias name='sata0-0-0'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='DISK1'/>
+      <backingStore/>
+      <target dev='sdb' bus='sata'/>
+      <alias name='sata0-0-1'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='1'/>
+    </disk>
+    <disk type='file' device='cdrom'>
+      <driver name='qemu' type='raw'/>
+      <source file='ISO'/>
+      <backingStore/>
+      <target dev='sdc' bus='sata'/>
+      <readonly/>
+      <boot order='2'/>
+      <alias name='sata0-0-2'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='2'/>
+    </disk>
+    <controller type='usb' index='0'>
+      <alias name='usb'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'>
+      <alias name='pci.0'/>
+    </controller>
+    <controller type='sata' index='0'>
+      <alias name='sata0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </controller>
+    <interface type='bridge'>
+      <mac address='52:54:00:da:8c:ad'/>
+      <source bridge='%BR1%'/>
+      <target dev='vnet0'/>
+      <model type='e1000'/>
+      <alias name='net0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:da:48:ff'/>
+      <source bridge='%BR2%'/>
+      <target dev='vnet1'/>
+      <model type='e1000'/>
+      <boot order='3'/>
+      <alias name='net1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:54:e6:c2'/>
+      <source bridge='%BR3%'/>
+      <target dev='vnet2'/>
+      <model type='virtio'/>
+      <alias name='net2'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+    </interface>
+    <interface type='bridge'>
+      <mac address='52:54:00:c5:62:6d'/>
+      <source bridge='%BR4%'/>
+      <target dev='vnet3'/>
+      <model type='virtio'/>
+      <alias name='net3'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
+    </interface>
+    <serial type='pty'>
+      <source path='/dev/pts/8'/>
+      <target port='0'/>
+      <alias name='serial0'/>
+    </serial>
+    <console type='pty' tty='/dev/pts/8'>
+      <source path='/dev/pts/8'/>
+      <target type='serial' port='0'/>
+      <alias name='serial0'/>
+    </console>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='vnc' port='5900' autoport='yes' listen='127.0.0.1' keymap='en-us'>
+      <listen type='address' address='127.0.0.1'/>
+    </graphics>
+    <video>
+      <model type='cirrus' vram='16384' heads='1'/>
+      <alias name='video0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </video>
+    <memballoon model='virtio'>
+      <alias name='balloon0'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
+    </memballoon>
+  </devices>
+  <seclabel type='dynamic' model='apparmor' relabel='yes'>
+    <label>libvirt-6afab2ba-0ed0-45cb-b1bd-985e211a48de</label>
+    <imagelabel>libvirt-6afab2ba-0ed0-45cb-b1bd-985e211a48de</imagelabel>
+  </seclabel>
+</domain>
+

deployment/libvirt/destroy_allinone.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+CONTROLLER=${CONTROLLER:-controller-allinone}
+DOMAIN_DIRECTORY=vms
+
+for i in {0..1}; do
+    CONTROLLER_NODE=${CONTROLLER}-${i}
+    DOMAIN_FILE=$DOMAIN_DIRECTORY/$CONTROLLER_NODE.xml
+    if virsh list --all --name | grep ${CONTROLLER_NODE}; then
+        STATUS=$(virsh list --all | grep ${CONTROLLER_NODE} | awk '{ print $3}')
+        if ([ "$STATUS" == "running" ])
+        then
+            sudo virsh destroy ${CONTROLLER_NODE}
+        fi
+        sudo virsh undefine ${CONTROLLER_NODE}
+        sudo rm -rf /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img
+        sudo rm -rf /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img
+        [ -e ${DOMAIN_FILE} ] && rm ${DOMAIN_FILE}
+    fi
+done

deployment/libvirt/destroy_network.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+INTERNAL_NETWORK=${INTERNAL_NETWORK:-10.10.10.0/24}
+INTERNAL_IP=${INTERNAL_IP:-10.10.10.1/24}
+EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-192.168.204.0/24}
+EXTERNAL_IP=${EXTERNAL_IP:-192.168.204.1/24}
+
+for i in {1..4}; do
+    BRIDGE_INTERFACE_NAME=${BRIDGE_INTERFACE}$i
+    if [ -d "/sys/class/net/${BRIDGE_INTERFACE_NAME}" ]; then
+        sudo ifconfig ${BRIDGE_INTERFACE_NAME} down
+        sudo brctl delbr ${BRIDGE_INTERFACE_NAME}
+    fi
+done

deployment/libvirt/destroy_standard_controller.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+CONTROLLER=${CONTROLLER:-controller}
+COMPUTE=${COMPUTE:-compute}
+DOMAIN_DIRECTORY=vms
+
+for i in {0..1}; do
+    CONTROLLER_NODE=${CONTROLLER}-${i}
+    DOMAIN_FILE=$DOMAIN_DIRECTORY/$CONTROLLER_NODE.xml
+    if virsh list --all --name | grep ${CONTROLLER_NODE}; then
+        STATUS=$(virsh list --all | grep ${CONTROLLER_NODE} | awk '{ print $3}')
+        if ([ "$STATUS" == "running" ])
+        then
+            sudo virsh destroy ${CONTROLLER_NODE}
+        fi
+        sudo virsh undefine ${CONTROLLER_NODE}
+        sudo rm -rf /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img
+        sudo rm -rf /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img
+        [ -e ${DOMAIN_FILE} ] && rm ${DOMAIN_FILE}
+    fi
+done
+
+for i in {0..1}; do
+    COMPUTE_NODE=${COMPUTE}-${i}
+    DOMAIN_FILE=$DOMAIN_DIRECTORY/$COMPUTE_NODE.xml
+    if virsh list --all --name | grep ${COMPUTE_NODE}; then
+        STATUS=$(virsh list --all | grep ${COMPUTE_NODE} | awk '{ print $3}')
+        if ([ "$STATUS" == "running" ])
+        then
+            sudo virsh destroy ${COMPUTE_NODE}
+        fi
+        sudo virsh undefine ${COMPUTE_NODE}
+        sudo rm -rf /var/lib/libvirt/images/${COMPUTE_NODE}-0.img
+        sudo rm -rf /var/lib/libvirt/images/${COMPUTE_NODE}-1.img
+        [ -e ${DOMAIN_FILE} ] && rm ${DOMAIN_FILE}
+    fi
+done

deployment/libvirt/install_packages.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# install_packages.sh - install required packages
+
+sudo apt-get install virt-manager libvirt-bin qemu-system -y
+
+cat << EOF | sudo tee /etc/libvirt/qemu.conf
+user = "root"
+group = "root"
+EOF
+
+sudo service libvirt-bin restart

deployment/libvirt/setup_allinone.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo "$0 [-h] [-i <iso image>]"
+    echo ""
+    echo "Options:"
+    echo "  -i: StarlingX ISO image"
+    echo ""
+}
+
+while getopts "i:" o; do
+    case "${o}" in
+        i)
+            ISOIMAGE="$OPTARG"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${ISOIMAGE}" ]; then
+    usage
+    exit -1
+fi
+
+ISOIMAGE=$(readlink -f "$ISOIMAGE")
+FILETYPE=$(file --mime-type -b ${ISOIMAGE})
+if ([ "$FILETYPE" != "application/x-iso9660-image" ]); then
+    echo "$ISOIMAGE is not an application/x-iso9660-image type"
+    exit -1
+fi
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+CONTROLLER=${CONTROLLER:-controller-allinone}
+DOMAIN_DIRECTORY=vms
+DOMAIN_FILE=$DOMAIN_DIRECTORY/$CONTROLLER.xml
+
+bash destroy_allinone.sh
+
+[ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY}
+
+for i in {0..1}; do
+    CONTROLLER_NODE=${CONTROLLER}-${i}
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img 600G
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img 200G
+    ISOIMAGE=${ISOIMAGE}
+    DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
+    cp controller.xml ${DOMAIN_FILE}
+    sed -i -e "
+        s,NAME,${CONTROLLER_NODE},
+        s,DISK0,/var/lib/libvirt/images/${CONTROLLER_NODE}-0.img,
+        s,DISK1,/var/lib/libvirt/images/${CONTROLLER_NODE}-1.img,
+        s,%BR1%,${BRIDGE_INTERFACE}1,
+        s,%BR2%,${BRIDGE_INTERFACE}2,
+        s,%BR3%,${BRIDGE_INTERFACE}3,
+        s,%BR4%,${BRIDGE_INTERFACE}4,
+    " ${DOMAIN_FILE}
+    if [ $i -eq 0 ]; then
+        sed -i -e "s,ISO,${ISOIMAGE}," ${DOMAIN_FILE}
+    else
+        sed -i -e "s,ISO,," ${DOMAIN_FILE}
+    fi
+    sudo virsh define ${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
+    if [ $i -eq 0 ]; then
+        sudo virsh start ${CONTROLLER_NODE}
+    fi
+done
+
+sudo virt-manager

deployment/libvirt/setup_network.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+usage() {
+    echo "$0 [-h]"
+    echo ""
+    echo "Options:"
+#    echo "  -i: StarlingX ISO image"
+    echo ""
+}
+
+while getopts "i:" o; do
+    case "${o}" in
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+INTERNAL_NETWORK=${INTERNAL_NETWORK:-10.10.10.0/24}
+INTERNAL_IP=${INTERNAL_IP:-10.10.10.1/24}
+EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-192.168.204.0/24}
+EXTERNAL_IP=${EXTERNAL_IP:-192.168.204.1/24}
+
+if [[ -r /sys/class/net/${BRIDGE_INTERFACE}1 ]]; then
+    echo "${BRIDGE_INTERFACE}1 exists, cowardly refusing to overwrite it, exiting..."
+    exit 1
+fi
+
+for i in {1..4}; do
+    sudo brctl addbr ${BRIDGE_INTERFACE}$i
+done
+
+sudo ifconfig ${BRIDGE_INTERFACE}1 $INTERNAL_IP up
+sudo ifconfig ${BRIDGE_INTERFACE}2 $EXTERNAL_IP up
+sudo ifconfig ${BRIDGE_INTERFACE}3 up
+sudo ifconfig ${BRIDGE_INTERFACE}4 up
+sudo iptables -t nat -A POSTROUTING -s $EXTERNAL_NETWORK -j MASQUERADE

deployment/libvirt/setup_standard_controller.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+#set -x
+
+usage() {
+    echo "$0 [-h] [-i <iso image>]"
+    echo ""
+    echo "Options:"
+    echo "  -i: StarlingX ISO image"
+    echo ""
+}
+
+while getopts "i:" o; do
+    case "${o}" in
+        i)
+            ISOIMAGE="$OPTARG"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${ISOIMAGE}" ]; then
+    usage
+    exit -1
+fi
+
+ISOIMAGE=$(readlink -f "$ISOIMAGE")
+FILETYPE=$(file --mime-type -b ${ISOIMAGE})
+if ([ "$FILETYPE" != "application/x-iso9660-image" ]); then
+    echo "$ISOIMAGE is not an application/x-iso9660-image type"
+    exit -1
+fi
+
+BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
+CONTROLLER=${CONTROLLER:-controller}
+COMPUTE=${COMPUTE:-compute}
+DOMAIN_DIRECTORY=vms
+
+bash destroy_standard_controller.sh
+
+[ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY}
+
+for i in {0..1}; do
+    CONTROLLER_NODE=${CONTROLLER}-${i}
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img 200G
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img 200G
+    ISOIMAGE=${ISOIMAGE}
+    DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
+    cp controller.xml ${DOMAIN_FILE}
+    sed -i -e "
+        s,NAME,${CONTROLLER_NODE},
+        s,DISK0,/var/lib/libvirt/images/${CONTROLLER_NODE}-0.img,
+        s,DISK1,/var/lib/libvirt/images/${CONTROLLER_NODE}-1.img,
+        s,%BR1%,${BRIDGE_INTERFACE}1,
+        s,%BR2%,${BRIDGE_INTERFACE}2,
+        s,%BR3%,${BRIDGE_INTERFACE}3,
+        s,%BR4%,${BRIDGE_INTERFACE}4,
+    " ${DOMAIN_FILE}
+    if [ $i -eq 0 ]; then
+        sed -i -e "s,ISO,${ISOIMAGE}," ${DOMAIN_FILE}
+    else
+        sed -i -e "s,ISO,," ${DOMAIN_FILE}
+    fi
+    sudo virsh define ${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml
+    if [ $i -eq 0 ]; then
+        sudo virsh start ${CONTROLLER_NODE}
+    fi
+done
+
+for i in {0..1}; do
+    COMPUTE_NODE=${COMPUTE}-${i}
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${COMPUTE_NODE}-0.img 200G
+    sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${COMPUTE_NODE}-1.img 200G
+    DOMAIN_FILE=${DOMAIN_DIRECTORY}/${COMPUTE_NODE}.xml
+    cp compute.xml ${DOMAIN_FILE}
+    sed -i -e "
+        s,NAME,${COMPUTE_NODE},;
+        s,DISK0,/var/lib/libvirt/images/${COMPUTE_NODE}-0.img,;
+        s,DISK1,/var/lib/libvirt/images/${COMPUTE_NODE}-1.img,
+        s,%BR1%,${BRIDGE_INTERFACE}1,
+        s,%BR2%,${BRIDGE_INTERFACE}2,
+        s,%BR3%,${BRIDGE_INTERFACE}3,
+        s,%BR4%,${BRIDGE_INTERFACE}4,
+    " ${DOMAIN_FILE}
+    sudo virsh define ${DOMAIN_FILE}
+done
+
+sudo virt-manager

deployment/provision/simplex_stage_1.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+## To be run AFTER "sudo config_controller --config-file TiS_config.ini_vb_simplex"
+source /etc/nova/openrc
+system license-install license.lic
+system host-disk-list controller-0
+
+NODE=controller-0
+DEVICE=/dev/sdb
+SIZE=$(system host-disk-list $NODE | grep $DEVICE | awk '{print $12}')
+DISK=$(system host-disk-list $NODE | grep $DEVICE | awk '{print $2}')
+# Create a partition for Cinder
+system host-disk-partition-add $NODE $DISK $SIZE -t lvm_phys_vol
+# Create the Volume Group
+system host-lvg-add $NODE cinder-volumes
+# Wait for partition to be created
+while true; do
+    system host-disk-partition-list $NODE --nowrap | grep $DEVICE | grep Ready;
+    if [ $? -eq 0 ]; then
+        break;
+    fi;
+    sleep 1;
+    echo "Waiting for Disk Partition for $DEVICE:$NODE"
+done
+
+PARTITION=$(system host-disk-partition-list $NODE --disk $DISK --nowrap | grep part1 | awk '{print $2}')
+# Create the PV
+sleep 1
+system host-pv-add $NODE cinder-volumes $PARTITION
+sleep 1
+#Enable LVM Backend.
+
+system storage-backend-add lvm -s cinder --confirmed
+
+#Wait for backend to be configured:
+echo " This can take a few minutes..."
+while true; do
+    system storage-backend-list | grep lvm | grep configured;
+    if [ $? -eq 0 ]; then
+        break;
+    else sleep 10;
+    fi;
+    echo "Waiting for backend to be configured"
+done
+system storage-backend-list
+
+# Add provider networks and assign segmentation ranges
+PHYSNET0='providernet-a'
+PHYSNET1='providernet-b'
+neutron providernet-create ${PHYSNET0} --type vlan
+neutron providernet-create ${PHYSNET1} --type vlan
+neutron providernet-range-create ${PHYSNET0} --name ${PHYSNET0}-a --range 400-499
+neutron providernet-range-create ${PHYSNET0} --name ${PHYSNET0}-b --range 10-10 --shared
+neutron providernet-range-create ${PHYSNET1} --name ${PHYSNET1}-a --range 500-599
+
+# Create data interfaces
+DATA0IF=eth1000
+DATA1IF=eth1001
+COMPUTE='controller-0'
+system host-list --nowrap &> /dev/null && NOWRAP="--nowrap"
+SPL=/tmp/tmp-system-port-list
+SPIL=/tmp/tmp-system-host-if-list
+system host-port-list ${COMPUTE} $NOWRAP > ${SPL}
+system host-if-list -a ${COMPUTE} $NOWRAP > ${SPIL}
+DATA0PCIADDR=$(cat $SPL | grep $DATA0IF |awk '{print $8}')
+DATA1PCIADDR=$(cat $SPL | grep $DATA1IF |awk '{print $8}')
+DATA0PORTUUID=$(cat $SPL | grep ${DATA0PCIADDR} | awk '{print $2}')
+DATA1PORTUUID=$(cat $SPL | grep ${DATA1PCIADDR} | awk '{print $2}')
+DATA0PORTNAME=$(cat $SPL | grep ${DATA0PCIADDR} | awk '{print $4}')
+DATA1PORTNAME=$(cat  $SPL | grep ${DATA1PCIADDR} | awk '{print $4}')
+DATA0IFUUID=$(cat $SPIL | awk -v DATA0PORTNAME=$DATA0PORTNAME '($12 ~ DATA0PORTNAME) {print $2}')
+DATA1IFUUID=$(cat $SPIL | awk -v DATA1PORTNAME=$DATA1PORTNAME '($12 ~ DATA1PORTNAME) {print $2}')
+system host-if-modify -m 1500 -n data0 -p ${PHYSNET0} -nt data ${COMPUTE} ${DATA0IFUUID}
+system host-if-modify -m 1500 -n data1 -p ${PHYSNET1} -nt data ${COMPUTE} ${DATA1IFUUID}
+
+# Add nova local backend
+system host-lvg-add ${COMPUTE} nova-local
+ROOT_DISK=$(system host-show ${COMPUTE} | grep rootfs | awk '{print $4}')
+ROOT_DISK_UUID=$(system host-disk-list ${COMPUTE} --nowrap | grep ${ROOT_DISK} | awk '{print $2}')
+PARTITION_SIZE=$(($(system host-disk-list ${COMPUTE} --nowrap   | grep ${ROOT_DISK} | awk '{print $12;}')/2))
+CGTS_PARTITION=$(system host-disk-partition-add -t lvm_phys_vol ${COMPUTE} ${ROOT_DISK_UUID} ${PARTITION_SIZE})
+
+while true; do
+    system host-disk-partition-list ${COMPUTE} | grep /dev/sda5 | grep Ready
+    if [ $? -eq 0 ]; then
+        break;
+    else sleep 2;
+    fi;
+    echo "Waiting to add disk partition"
+done
+system host-disk-partition-list ${COMPUTE}
+
+CGTS_PARTITION_UUID=$(echo ${CGTS_PARTITION} | grep -ow "| uuid | [a-z0-9\-]* |" | awk '{print $4}')
+sleep 1
+system host-pv-add ${COMPUTE} cgts-vg ${CGTS_PARTITION_UUID}
+sleep 1
+NOVA_PARTITION=$(system host-disk-partition-add -t lvm_phys_vol ${COMPUTE} ${ROOT_DISK_UUID} ${PARTITION_SIZE})
+
+while true; do
+    system host-disk-partition-list ${COMPUTE} | grep /dev/sda6 | grep Ready
+    if [ $? -eq 0 ]; then
+        break;
+    else sleep 2;
+    fi;
+    echo "Waiting to add disk partition"
+done
+system host-disk-partition-list ${COMPUTE}
+
+NOVA_PARTITION_UUID=$(echo ${NOVA_PARTITION} | grep -ow "| uuid | [a-z0-9\-]* |" | awk '{print $4}')
+system host-pv-add ${COMPUTE} nova-local ${NOVA_PARTITION_UUID}
+sleep 1
+system host-lvg-modify -b image -s 10240 ${COMPUTE} nova-local
+sleep 10
+
+### This will result in a reboot.
+system host-unlock controller-0
+echo " Watch CONSOLE to see progress. You will see things like "
+echo "    Applying manifest 127.168.204.3_patching.pp..."
+echo "    [DONE]"
+echo " Tailing /var/log/platform.log until reboot..."
+tail -f /var/log/platform.log

deployment/provision/simplex_stage_2.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+
+
+
+## To be run AFTER "provision_simplexR5_stage1.sh" and a reboot cycle
+source /etc/nova/openrc
+
+#  Create tenant networks and routers
+echo "Setting up tenant networks"
+ADMINID=`openstack project list | grep admin | awk '{print $2}'`
+PHYSNET0='providernet-a'
+PHYSNET1='providernet-b'
+PUBLICNET='public-net0'
+PRIVATENET='private-net0'
+INTERNALNET='internal-net0'
+EXTERNALNET='external-net0'
+PUBLICSUBNET='public-subnet0'
+PRIVATESUBNET='private-subnet0'
+INTERNALSUBNET='internal-subnet0'
+EXTERNALSUBNET='external-subnet0'
+PUBLICROUTER='public-router0'
+PRIVATEROUTER='private-router0'
+
+neutron net-create --tenant-id ${ADMINID} --provider:network_type=vlan --provider:physical_network=${PHYSNET0} --provider:segmentation_id=10 --router:external ${EXTERNALNET}
+neutron net-create --tenant-id ${ADMINID} --provider:network_type=vlan --provider:physical_network=${PHYSNET0} --provider:segmentation_id=400 ${PUBLICNET}
+neutron net-create --tenant-id ${ADMINID} --provider:network_type=vlan --provider:physical_network=${PHYSNET1} --provider:segmentation_id=500 ${PRIVATENET}
+neutron net-create --tenant-id ${ADMINID} ${INTERNALNET}
+PUBLICNETID=`neutron net-list | grep ${PUBLICNET} | awk '{print $2}'`
+PRIVATENETID=`neutron net-list | grep ${PRIVATENET} | awk '{print $2}'`
+INTERNALNETID=`neutron net-list | grep ${INTERNALNET} | awk '{print $2}'`
+EXTERNALNETID=`neutron net-list | grep ${EXTERNALNET} | awk '{print $2}'`
+neutron subnet-create --tenant-id ${ADMINID} --name ${PUBLICSUBNET} ${PUBLICNET} 192.168.101.0/24
+neutron subnet-create --tenant-id ${ADMINID} --name ${PRIVATESUBNET} ${PRIVATENET} 192.168.201.0/24
+neutron subnet-create --tenant-id ${ADMINID} --name ${INTERNALSUBNET} --no-gateway  ${INTERNALNET} 10.10.0.0/24
+neutron subnet-create --tenant-id ${ADMINID} --name ${EXTERNALSUBNET} --gateway 192.168.1.1 --disable-dhcp ${EXTERNALNET} 192.168.1.0/24
+echo "Setting up tenant routers"
+neutron router-create ${PUBLICROUTER}
+neutron router-create ${PRIVATEROUTER}
+PRIVATEROUTERID=`neutron router-list | grep ${PRIVATEROUTER} | awk '{print $2}'`
+PUBLICROUTERID=`neutron router-list | grep ${PUBLICROUTER} | awk '{print $2}'`
+neutron router-gateway-set --disable-snat ${PUBLICROUTERID} ${EXTERNALNETID}
+neutron router-gateway-set --disable-snat ${PRIVATEROUTERID} ${EXTERNALNETID}
+neutron router-interface-add ${PUBLICROUTER} ${PUBLICSUBNET}
+neutron router-interface-add ${PRIVATEROUTER} ${PRIVATESUBNET}
+
+# Create a flavor
+echo "Making a flavour"
+nova flavor-create s.p1 auto 512 1 1
+nova flavor-key s.p1 set hw:cpu_policy=dedicated
+nova flavor-key s.p1 set hw:mem_page_size=2048
+
+# Import an image
+# due to dns/proxy, just use host and horizon
+echo "use Horizon to upload an image. cirros is a good small one"
+echo "http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img"

deployment/virtualbox/all_in_one.conf

@@ -0,0 +1,26 @@
+# Virtual machine prefix name
+VM_PREFIX_NAME=$(whoami)-simplex-
+
+# Number of controller nodes
+TIC_CONTROLLER_NUM=1
+# Number of compute nodes
+TIC_COMPUTE_NUM=0
+
+# vCPU
+TIC_CONTROLLER_CPUS="8"
+# Memory size, in MB
+TIC_CONTROLLER_MEM="24576"
+# Disk1 size, in 1 MB units
+TIC_CONTROLLER_DISK1="600000"
+# Disk2 size, in 1 MB units
+TIC_CONTROLLER_DISK2="16240"
+# Disk2 size, in 1 MB units
+TIC_CONTROLLER_DISK3="16240"
+
+# First port for VRDE
+# This is for a Virtualbox Remote Display Port
+# https://www.virtualbox.org/manual/ch07.html#vrde
+TIC_VDEPORT_START=1$(id -u)
+
+# Install image
+TIC_INSTALL_ISO="$PWD/bootimage.iso"

deployment/virtualbox/serial_vm.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+socat UNIX-CONNECT:"/tmp/serial_$1" stdio,raw,echo=0,icanon=0

deployment/virtualbox/setup_vm.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+CONFFILE="${1:-standard_controller.conf}"
+source $SCRIPTPATH/$CONFFILE
+
+VM_PREFIX_NAME="${VM_PREFIX_NAME:-default-}"
+CONTROLLER_CPUS="${TIC_CONTROLLER_CPUS:-4}"
+CONTROLLER_MEM="${TIC_CONTROLLER_MEM:-8192}"
+CONTROLLER_DISK1="${TIC_CONTROLLER_DISK1:-81920}"
+CONTROLLER_DISK2="${TIC_CONTROLLER_DISK2:-10240}"
+CONTROLLER_DISK3="${TIC_CONTROLLER_DISK3:-4096}"
+ISO="${TIC_INSTALL_ISO:-$SCRIPTPATH/bootimage.iso}"
+
+HOSTADD_SCRIPT="$SCRIPTPATH/add_host.sh"
+
+declare -a CREATED_VMS
+machine_folder=`VBoxManage list systemproperties | grep "Default machine folder:" | cut -d : -f 2 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'`
+
+# VRDE port for 1st VM
+vrdeport="${TIC_VDEPORT_START:-13389}"
+function set_vrde {
+    vm=$1
+    VBoxManage modifyvm "$vm" --vrde on --vrdeaddress 127.0.0.1 --vrdeport $vrdeport
+    let vrdeport=vrdeport+1
+}
+
+function my_error {
+    echo "Error: $1"
+    my_trap_clean
+    exit 1
+}
+
+function my_trap_clean {
+    echo "deleting created VMS ${CREATED_VMS[@]}..."
+    for vm in ${CREATED_VMS[@]}; do
+        VBoxManage unregistervm "$vm" --delete
+    done
+}
+
+function init_hostonly_net {
+    # Create hostonly networks
+    VBoxManage list hostonlyifs | grep vboxnet0 && vboxnet="1"
+    if [ "x$vboxnet" != "x" ]; then
+        VBoxManage list hostonlyifs
+        read -r -p "Hostonly network vboxnet0 already existed. Are you sure to reconfigure it? [y/N] " response
+        case $response in
+            [yY][eE][sS]|[yY])
+                echo
+                ;;
+            *)
+                my_error "Please make sure it's safe before you remove hostonly network vboxnet0!"
+                ;;
+        esac
+    else
+        VBoxManage hostonlyif create
+    fi
+    VBoxManage hostonlyif ipconfig vboxnet0 --ip 10.10.10.1 --netmask 255.255.255.0
+}
+
+function createvm {
+    vm=$1
+    cpus=$2
+    mem=$3
+
+    echo "creating VM ${vm}..."
+    # Find if VM already existing
+    VBoxManage showvminfo "$vm" &>/dev/null && my_error "VM $vm already existed. Please delete it first"
+    CREATED_VMS+=("$vm")
+    # Create VM
+    VBoxManage createvm --name "$vm" --register
+
+    # Configure controller VM
+    # CPU
+    VBoxManage modifyvm "$vm" --ostype Linux_64 --cpus "$cpus" --pae on --longmode on --x2apic on --largepages off
+    # Memory
+    VBoxManage modifyvm "$vm" --memory "$mem"
+    # Network
+    VBoxManage modifyvm "$vm" --cableconnected1 on --nic1 hostonly --nictype1 82540EM --hostonlyadapter1 vboxnet0
+    VBoxManage modifyvm "$vm" --cableconnected2 on --nic2 intnet --nictype2 82540EM --intnet2 intnet-management-$(whoami) --nicpromisc2 allow-all --nicbootprio2 1
+    VBoxManage modifyvm "$vm" --cableconnected3 on --nic3 intnet --nictype3 virtio --intnet3 intnet-data1-$(whoami) --nicpromisc3 allow-all
+    VBoxManage modifyvm "$vm" --cableconnected4 on --nic4 intnet --nictype4 virtio --intnet4 intnet-data2-$(whoami) --nicpromisc4 allow-all
+    # Storage Medium
+    VBoxManage createmedium disk --filename "${machine_folder}/${vm}/${vm}-disk1.vdi" --size $CONTROLLER_DISK1 --format VDI
+    VBoxManage createmedium disk --filename "${machine_folder}/${vm}/${vm}-disk2.vdi" --size $CONTROLLER_DISK2 --format VDI
+    VBoxManage createmedium disk --filename "${machine_folder}/${vm}/${vm}-disk3.vdi" --size $CONTROLLER_DISK3 --format VDI
+    VBoxManage storagectl "$vm" --name SATA --add sata --controller IntelAhci --portcount 4 --hostiocache on --bootable on
+    VBoxManage storageattach "$vm" --storagectl SATA --port 0 --device 0 --type hdd --medium "${machine_folder}/${vm}/${vm}-disk1.vdi"
+    VBoxManage storageattach "$vm" --storagectl SATA --port 1 --device 0 --type hdd --medium "${machine_folder}/${vm}/${vm}-disk2.vdi"
+    VBoxManage storageattach "$vm" --storagectl SATA --port 2 --device 0 --type hdd --medium "${machine_folder}/${vm}/${vm}-disk3.vdi"
+    VBoxManage storageattach "$vm" --storagectl SATA --port 3 --device 0 --type dvddrive --medium emptydrive
+    # Display
+    VBoxManage modifyvm "$vm" --vram 16
+    # Audio
+    VBoxManage modifyvm "$vm" --audio none
+    # Boot Order
+    VBoxManage modifyvm "$vm" --boot1 dvd --boot2 disk --boot3 net --boot4 none
+    # Other
+    VBoxManage modifyvm "$vm" --ioapic on --rtcuseutc on
+    # VM sepcific
+    # Serial
+    VBoxManage modifyvm "$vm" --uart1 0x3F8 4 --uartmode1 server "/tmp/serial_$vm"
+    set_vrde "$vm"
+}
+
+function clonevm {
+    src=$1
+    target=$2
+    echo "creating VM ${target} from ${src}..."
+    # Find if vm already existing
+    VBoxManage showvminfo "$target" &>/dev/null && my_error "VM $target already existed. Please delete it first"
+    VBoxManage clonevm  "$src" --mode machine --name "$target" --register
+    CREATED_VMS+=("$target")
+    # Serial
+    VBoxManage modifyvm "$target" --uart1 0x3F8 4 --uartmode1 server "/tmp/serial_$target"
+    set_vrde "$target"
+}
+
+trap my_trap_clean SIGINT SIGTERM
+
+set -e
+
+[[ -f $ISO ]] || my_error "Can not fild install image $ISO"
+
+# Init hostonly network
+init_hostonly_net
+
+# Create host_add.sh for Compute and Controller node
+rm -f "$HOSTADD_SCRIPT"
+cat <<EOF > "$HOSTADD_SCRIPT"
+#!/usr/bin/env bash
+source /etc/nova/openrc
+EOF
+chmod +x "$HOSTADD_SCRIPT"
+
+# Create Contoller VM, at least controller0
+createvm "${VM_PREFIX_NAME}controller-0" $CONTROLLER_CPUS $CONTROLLER_MEM
+COUNTER=1
+while [  $COUNTER -lt $TIC_CONTROLLER_NUM  ]; do
+    clonevm ${VM_PREFIX_NAME}controller-0 "${VM_PREFIX_NAME}controller-$COUNTER"
+    mac=`VBoxManage showvminfo "${VM_PREFIX_NAME}controller-$COUNTER" | grep intnet-management | grep -o "MAC: [0-9a-fA-F]*" | awk '{ print $2 }' | sed 's/../&:/g;s/:$//'`
+    echo "system host-add -n ${VM_PREFIX_NAME}controller-$COUNTER -p controller -m $mac" >> "$HOSTADD_SCRIPT"
+    let COUNTER=COUNTER+1
+done
+
+# Create Compute VM
+COUNTER=0
+while [  $COUNTER -lt $TIC_COMPUTE_NUM  ]; do
+    clonevm ${VM_PREFIX_NAME}controller-0 "${VM_PREFIX_NAME}compute-$COUNTER"
+    mac=`VBoxManage showvminfo "${VM_PREFIX_NAME}compute-$COUNTER" | grep intnet-management | grep -o "MAC: [0-9a-fA-F]*" | awk '{ print $2 }' | sed 's/../&:/g;s/:$//'`
+    echo "system host-add -n ${VM_PREFIX_NAME}compute-$COUNTER -p compute -m $mac" >> "$HOSTADD_SCRIPT"
+    let COUNTER=COUNTER+1
+done
+
+# Start Controller-0 with bootiso.img
+VBoxManage storageattach ${VM_PREFIX_NAME}controller-0 --storagectl SATA --port 3 --device 0 --type dvddrive --medium "$ISO"
+$SCRIPTPATH/start_vm.sh ${VM_PREFIX_NAME}controller-0

deployment/virtualbox/standard_controller.conf

@@ -0,0 +1,20 @@
+# Number of Controller nodes
+TIC_CONTROLLER_NUM=1
+# Number of Compute nodes
+TIC_COMPUTE_NUM=1
+
+# vCPU
+TIC_CONTROLLER_CPUS="4"
+# Memory size, in MB
+TIC_CONTROLLER_MEM="8192"
+# Disk1 size, in 1 MB units
+TIC_CONTROLLER_DISK1="81920"
+# Disk2 size, in 1 MB units
+TIC_CONTROLLER_DISK2="10240"
+# First port for VRDE
+# This is for a Virtualbox Remote Display Port
+# https://www.virtualbox.org/manual/ch07.html#vrde
+TIC_VDEPORT_START=13389
+
+# Install image
+TIC_INSTALL_ISO="$PWD/bootimage.iso"

deployment/virtualbox/start_vm.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+rdeport=`VBoxManage showvminfo $1 | grep "^VRDE:" | grep -o "Ports [0-9]*" | cut -d ' ' -f 2`
+if [ "x$rdeport" == "x" ]; then
+    echo "Vm $1 not found or not configured to use rde".
+    exit 1
+fi
+
+VBoxManage startvm "$1" --type headless
+
+sleep 3
+echo rdesktop-vrdp -a 16 -N "127.0.0.1:$rdeport"
+if xdpyinfo 2>&1 >> /dev/null; then
+    rdesktop-vrdp -a 16 -N "127.0.0.1:$rdeport" &
+else
+    echo "Running without X display. Use a tunnel from your laptop"
+    echo "ssh -L $rdeport:127.0.0.1:$rdeport -N -f -l <uname> madbuild01.ostc.intel.com"
+    echo "Then run rdesktop-vrdp -a 16 -N 127.0.0.1:$rdeport from your laptop"
+fi

deployment/virtualbox/stop_vm.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+
+VBoxManage controlvm "$1" poweroff
+

doc/requirements.txt

@@ -0,0 +1,5 @@
+sphinx>=1.6.2
+openstackdocstheme>=1.19.0 # Apache-2.0
+
+# Release Notes
+reno>=0.1.1 # Apache2

doc/source/conf.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another
+# directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown
+# here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'stx-tools'
+copyright = u'2018, StarlingX'
+author = u'StarlingX'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u'0.1'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['openstackdocstheme'
+              ]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+bug_project = '1027'
+bug_tag = 'stx.bug'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'openstackdocs'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'stx-toolsdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'stx-tools.tex', u'stx-tools Documentation',
+     u'StarlingX', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'stx-tools', u'stx-tools Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'stx-tools', u'stx-tools Documentation',
+     author, 'stx-tools', 'One line description of project.',
+     'Miscellaneous'),
+]

doc/source/index.rst

@@ -0,0 +1,15 @@
+stx-tools Documentation
+=======================
+
+StarlingX Build Tools
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`search`

release/branch-stx.sh

@@ -57,12 +57,14 @@ function update_gitreview {
 defaultbranch=$branch"
     echo "$grcontents" > .gitreview
     git add .gitreview
-    git commit -s -m "Update .gitreview for $branch"
-    git show
-    if [[ -z $DRY_RUN ]]; then
-        git review -t "create-${branch}"
+    if ! git commit -s -m "Update .gitreview for $branch"; then
+        if [[ -z $DRY_RUN ]]; then
+            git review -t "create-${branch}"
+        else
+            echo "### skipping .gitreview submission to $branch"
+        fi
     else
-        echo "### skipping review submission to $branch"
+        echo "### no changes required for .gitreview"
     fi
 }
 
@@ -89,7 +91,7 @@ function branch_repo {
     fi
 
     # tag branch point at $sha
-    git tag -f $tag $sha
+    git tag -s -m "Branch $branch" -f $tag $sha
 
     # Push the new goodness back up
     if [[ "$r" == "starlingx" ]]; then
@@ -100,7 +102,7 @@ function branch_repo {
 
         # push
         if [[ -z $DRY_RUN ]]; then
-            git push gerrit $branch
+            git push --tags gerrit $branch
         else
             echo "### skipping push to $branch"
         fi

releasenotes/source/conf.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# stx-tools Release Notes documentation build configuration file.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'openstackdocstheme',
+    'reno.sphinxext',
+]
+
+bug_project = '1027'
+bug_tag = 'stx.bug'
+
+# Add any paths that contain templates here, relative to this directory.
+# templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# Release notes are version independent, no need to set version and release
+release = ''
+version = ''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'openstackdocs'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+# html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+# html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%Y-%m-%d %H:%M'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'stx-toolsreleasenotesdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    # 'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    ('index', 'stx-toolsreleasenotes.tex', u'stx-tools Release Notes',
+     u'StarlingX', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'stx-toolsreleasenotes', u'stx-tools Release Notes',
+     [u'StarlingX'], 1)
+]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    ('index', 'stx-toolsReleaseNotes', u'stx-tools Release Notes',
+     u'StarlingX', 'stx-toolsreleasenotes',
+     'StarlingX build tools',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+# texinfo_no_detailmenu = False
+
+# -- Options for Internationalization output ------------------------------
+locale_dirs = ['locale/']

releasenotes/source/index.rst

@@ -0,0 +1,7 @@
+stx-tools Release Notes
+=======================
+
+.. toctree::
+   :maxdepth: 2
+
+   unreleased

releasenotes/source/unreleased.rst

@@ -0,0 +1,5 @@
+============================
+Current Series Release Notes
+============================
+
+.. release-notes::

toCOPY/generate-cgcs-centos-repo.sh

@@ -27,37 +27,66 @@ comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml
 mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto
 comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml
 
-if [[ ( ! -d $mirror_dir/Binary ) || ( ! -d $mirror_dir/Source ) ]]; then
-    echo "The mirror $mirror_dir doesn't has the Binary and Source"
+lst_file_dir="$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools"
+rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst"
+other_lst_file="other_downloads.lst"
+missing_rpms_file=missing.txt
+
+rm -f ${missing_rpms_file}
+
+# Strip trailing / from mirror_dir if it was specified...
+mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%")
+
+if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then
+    echo "The mirror ${mirror_dir} doesn't has the Binary and Source"
     echo "folders. Please provide a valid mirror"
     exit -1
 fi
 
-if [ ! -d "$dest_dir" ]; then
-    mkdir -p "$dest_dir"
+if [ ! -d "${dest_dir}" ]; then
+    mkdir -p "${dest_dir}"
 fi
 
 for t in "Binary" "Source" ; do
-    target_dir=$dest_dir/$t
+    target_dir=${dest_dir}/$t
     if [ ! -d "$target_dir" ]; then
         mkdir -p "$target_dir"
     else
         mv -f "$target_dir" "$target_dir-backup-$timestamp"
         mkdir -p "$target_dir"
     fi
+done
 
-    pushd "$mirror_dir/$t"|| exit 1
-    find . -type d -exec mkdir -p "${target_dir}"/{} \;
-    all_files=$(find . -type f -name "*")
-    popd || exit 1
+unsuccessful_file=0
+for lst_file in ${rpm_lst_files} ; do
+    grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do
+        if [ -z "${rpmname}" ]; then
+            continue
+        fi
+        mirror_file=$(find ${mirror_dir} -name ${rpmname})
+        if [ -z "${mirror_file}" ]; then
+            echo "Error -- could not find requested ${rpmname} in ${mirror_dir}"
+            echo ${rpmname} >> ${missing_rpms_file}
+            unsuccessful_file=1
+            continue
+        fi
 
-
-    for ff in $all_files; do
+        # Great, we found the file!  Let's strip the mirror_dir prefix from it...
+        ff=$(echo ${mirror_file} | sed "s%^${mirror_dir}/%%")
         f_name=$(basename "$ff")
         sub_dir=$(dirname "$ff")
-        ln -sf "$mirror_dir/$t/$ff" "$target_dir/$sub_dir"
-        echo "Creating symlink for $target_dir/$sub_dir/$f_name"
-        echo "------------------------------"
+
+        # Make sure we have a subdir (so we don't symlink the first file as
+        # the subdir name)
+        mkdir -p ${dest_dir}/${sub_dir}
+
+        # Link it!
+        echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}"
+        ln -sf "${mirror_dir}/$ff" "${dest_dir}/${sub_dir}"
+        if [ $? -ne 0 ]; then
+            echo "Failed ${mirror_file}: ln -sf \"${mirror_dir}/$ff\" \"${dest_dir}/${sub_dir}\""
+            unsuccessful_file=1
+        fi
     done
 done
 
@@ -75,13 +104,30 @@ fi
 echo "Copying mock.cfg.proto and comps.xml files."
 
 if [ -f "$mock_cfg_dest_file" ]; then
-    cp "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp"
+    \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp"
 fi
 cp "$mock_cfg_file" "$mock_cfg_dest_file"
 
 if [ -f "$comps_xml_dest_file" ]; then
-    cp "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp"
+    \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp"
 fi
 cp "$comps_xml_file" "$comps_xml_dest_file"
 
-echo "Done"
+# Populate the contents from other list files
+cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do
+    if [ "${targettype}" == "folder" ]; then
+        echo "Creating folder ${item}"
+        mkdir -p $MY_REPO/cgcs-centos-repo/Binary/${item}
+    fi
+
+    if [ "${targettype}" == "file" ]; then
+        mkdir -p $MY_REPO/cgcs-centos-repo/Binary/$(dirname ${item})
+        echo "Creating symlink for $MY_REPO/cgcs-centos-repo/Binary/${item}"
+        ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/cgcs-centos-repo/Binary/${item}
+    fi
+done
+
+echo "Done creating repo directory"
+if [ ${unsuccessful_file} -ne 0 ]; then
+    echo "WARNING: Some targets could not be found.  Your repo may be incomplete."
+fi

toCOPY/populate_downloads.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+usage () {
+    echo "$0 <mirror-path>"
+}
+
+if [ $# -ne 1 ]; then
+    usage
+    exit -1
+fi
+
+if [ -z "$MY_REPO" ]; then
+    echo "\$MY_REPO is not set. Ensure you are running this script"
+    echo "from the container and \$MY_REPO points to the root of"
+    echo "your folder tree."
+    exit -1
+fi
+
+mirror_dir=$1
+tarball_lst=${MY_REPO_ROOT_DIR}/stx-tools/centos-mirror-tools/tarball-dl.lst
+downloads_dir=${MY_REPO}/stx/downloads
+extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.3.2.1.6.43302.src.rpm rdma-core-43mlnx1-1.43302.src.rpm"
+
+mkdir -p ${MY_REPO}/stx/downloads
+
+grep -v "^#" ${tarball_lst} | while read x; do
+    if [ -z "$x" ]; then
+        continue
+    fi
+
+    # Get first element of item & strip leading ! if appropriate
+    tarball_file=$(echo $x | sed "s/#.*//" | sed "s/^!//")
+
+    # put the file in downloads
+    source_file=$(find ${mirror_dir}/downloads -name "${tarball_file}")
+    if [ -z ${source_file} ]; then
+        echo "Could not find ${tarball_file}"
+    else
+        rel_path=$(echo ${source_file} | sed "s%^${mirror_dir}/downloads/%%")
+        rel_dir_name=$(dirname ${rel_path})
+        if [ ! -e ${downloads_dir}/${rel_dir_name}/${tarball_file} ]; then
+            mkdir -p ${downloads_dir}/${rel_dir_name}
+            ln -sf ${source_file} ${downloads_dir}/${rel_dir_name}/
+        fi
+    fi
+done
+
+for x in ${extra_downloads}; do
+    ln -sf ${mirror_dir}/downloads/$x ${downloads_dir}
+done

tox.ini

@@ -5,14 +5,18 @@ skipsdist = True
 
 [testenv]
 basepython = python3
-install_command = pip install -U {opts} {packages}
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
          OS_STDOUT_CAPTURE=1
          OS_STDERR_CAPTURE=1
          OS_TEST_TIMEOUT=60
 deps = -r{toxinidir}/test-requirements.txt
+whitelist_externals = reno
 
 [testenv:linters]
+# bashate ignore:
+#  E006 - accept long lines
+#  E040 - false positive on |& syntax (new in bash 4)
 whitelist_externals = bash
 commands =
   bash -c "find {toxinidir}        \
@@ -26,7 +30,7 @@ commands =
          -not -name \*~                       \
          -not -name \*.md                     \
          -name \*.sh                          \
-         -print0 | xargs -0 bashate -v -iE006"
+         -print0 | xargs -0 bashate -v -iE006,E040"
 
 [testenv:venv]
 commands = {posargs}
@@ -36,3 +40,19 @@ commands = {posargs}
 envdir = {toxworkdir}/venv
 commands = true
 whitelist_externals = true
+
+[testenv:docs]
+basepython = python3
+deps = -r{toxinidir}/doc/requirements.txt
+commands =
+    rm -rf doc/build
+    sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html
+whitelist_externals = rm
+
+[testenv:releasenotes]
+basepython = python3
+deps = -r{toxinidir}/doc/requirements.txt
+commands =
+    rm -rf releasenotes/build
+    sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
+whitelist_externals = rm