Merge remote-tracking branch 'gerrit/master' into f/centos76

Change-Id: If87051f274a45887aa5487ef8b037489290b9e82
Signed-off-by: Saul Wold <sgw@linux.intel.com>

README.rst

@@ -291,7 +291,7 @@ links if this is the first build or the mirror has been updated.
 
 .. code-block:: bash
 
-    generate-cgcs-centos-repo.sh /import/mirror/CentOS/pike
+    generate-cgcs-centos-repo.sh /import/mirrors/CentOS/pike
 
 Where the argument to the script is the path of the mirror.
 

centos-mirror-tools/rpms_centos.lst

@@ -163,7 +163,6 @@ cracklib-dicts-2.9.0-11.el7.x86_64.rpm
 createrepo-0.9.9-28.el7.noarch.rpm
 cronie-1.4.11-19.el7.x86_64.rpm
 cronie-anacron-1.4.11-19.el7.x86_64.rpm
-crontabs-1.11-6.20121102git.el7.src.rpm
 crontabs-1.11-6.20121102git.el7.noarch.rpm
 crudini-0.9-2.el7.noarch.rpm
 cryptsetup-1.7.4-4.el7.x86_64.rpm
@@ -472,7 +471,6 @@ iptables-utils-1.4.21-24.1.el7_5.x86_64.rpm
 iptables-services-1.4.21-24.1.el7_5.x86_64.rpm
 iputils-20160308-10.el7.x86_64.rpm
 ipxe-roms-qemu-20170123-1.git4e85b27.el7_4.1.noarch.rpm
-iscsi-initiator-utils-6.2.0.874-7.el7.src.rpm
 iscsi-initiator-utils-6.2.0.874-7.el7.x86_64.rpm
 iscsi-initiator-utils-iscsiuio-6.2.0.874-7.el7.x86_64.rpm
 iso-codes-3.46-2.el7.noarch.rpm
@@ -868,7 +866,6 @@ nspr-devel-4.19.0-1.el7_5.x86_64.rpm
 nss-3.36.0-5.el7_5.x86_64.rpm
 nss_compat_ossl-0.9.6-8.el7.x86_64.rpm
 nss-devel-3.36.0-5.el7_5.x86_64.rpm
-nss-pam-ldapd-0.8.13-16.el7.src.rpm
 nss-pam-ldapd-0.8.13-16.el7.x86_64.rpm
 nss-pem-1.0.3-4.el7.x86_64.rpm
 nss-sysinit-3.36.0-5.el7_5.x86_64.rpm

centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh

@@ -24,6 +24,13 @@ YUM_REPOS_DIR="$YUM_CONF_DIR/yum.repos.d"
 DOWNLOAD_PATH_ROOT="/export/mirror/centos"
 URL_UTILS="url_utils.sh"
 
+# These variables drive the download of the centos installer
+# and other non-repo files found under the os/x86_64 subdirectory.
+OS_PATH_PREFIX=/export/mirror/centos/centos
+OS_PATH_SUFFIX=os/x86_64
+OS_FILES="EULA GPL"
+OS_DIRS="EFI LiveOS images isolinux"
+
 DAILY_REPO_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 
 if [ -f "$DAILY_REPO_SYNC_DIR/$URL_UTILS" ]; then
@@ -118,6 +125,32 @@ for REPO in $(find $YUM_REPOS_DIR -name '*.repo'); do
             continue
         fi
 
+        # The following will download the centos installer and other non-repo
+        # files and directories found under the os/x86_64 subdirectory.
+        if [[ "$DOWNLOAD_PATH" == "$OS_PATH_PREFIX"/*/"$OS_PATH_SUFFIX" ]]; then
+            for f in $OS_FILES; do
+                CMD="wget '$REPO_URL/$f' --output-document='$DOWNLOAD_PATH/$f'"
+                echo "$CMD"
+                eval $CMD
+                if [ $? -ne 0 ]; then
+                    echo "Error: $CMD"
+                    ERR_COUNT=$((ERR_COUNT+1))
+                    continue
+                fi
+            done
+
+            for d in $OS_DIRS; do
+                CMD="wget -r -N -l 3 -nv -np -e robots=off --reject-regex '.*[?].*' --reject index.html '$REPO_URL/$d/' -P '$OS_PATH_PREFIX/'"
+                echo "$CMD"
+                eval $CMD
+                if [ $? -ne 0 ]; then
+                    echo "Error: $CMD"
+                    ERR_COUNT=$((ERR_COUNT+1))
+                    continue
+                fi
+            done
+        fi
+
         popd
     done
 done | tee $LOGFILE

centos-mirror-tools/tarball-dl.lst

@@ -41,7 +41,7 @@ puppet-ldap-480f13af6d17d1d3fcf0dc7b4bd04b49fa4099e1.tar.gz#puppet-ldap-master#h
 puppet-network-7deacd5fdc22c0543455878a8d1872f2f5417c1d.tar.gz#packstack/puppet/modules/network#https://github.com/voxpupuli/puppet-network/tarball/7deacd5fdc22c0543455878a8d1872f2f5417c1d#http##
 puppet-nslcd-b8c19b1ada89865f2e50758e054583798ad8011a.tar.gz#packstack/puppet/modules/nslcd#https://github.com/jlyheden/puppet-nslcd/tarball/b8c19b1ada89865f2e50758e054583798ad8011a#http##
 puppi-c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81.tar.gz#puppi-master#https://github.com/example42/puppi/tarball/c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81#http##
-helm-v2.11.0-linux-amd64.tar.gz#linux-amd64#https://storage.googleapis.com/kubernetes-helm/helm-v2.11.0-linux-amd64.tar.gz#http##
+helm-v2.12.1-linux-amd64.tar.gz#linux-amd64#https://storage.googleapis.com/kubernetes-helm/helm-v2.12.1-linux-amd64.tar.gz#http##
 openstack-helm-9d72fe1a501bc609a875eebf7b6274e18600ed70.tar.gz#openstack-helm#https://github.com/openstack/openstack-helm/archive/9d72fe1a501bc609a875eebf7b6274e18600ed70.tar.gz#http##
 openstack-helm-infra-5d356f9265b337b75f605dee839faa8cd0ed3ab2.tar.gz#openstack-helm-infra#https://github.com/openstack/openstack-helm-infra/archive/5d356f9265b337b75f605dee839faa8cd0ed3ab2.tar.gz#http##
 gnocchi-4.2.5.tar.gz#gnocchi-4.2.5#https://pypi.io/packages/source/g/gnocchi/gnocchi-4.2.5.tar.gz#http##

centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo

@@ -3,7 +3,7 @@
 #
 [Starlingx-C7.5.1804-cloud-pike]
 name=Starlingx-CentOS-7.5.1804 - Cloud-pike
-baseurl=http://mirror.centos.org/centos/7.5.1804/cloud/x86_64/openstack-pike/
+baseurl=http://vault.centos.org/7.5.1804/cloud/x86_64/openstack-pike/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -17,7 +17,7 @@ enabled=1
 
 [Starlingx-C7.5.1804-extras]
 name=Starlingx-CentOS-7.5.1804 - extras
-baseurl=http://mirror.centos.org/centos/7.5.1804/extras/x86_64/
+baseurl=http://vault.centos.org/7.5.1804/extras/x86_64/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -31,7 +31,7 @@ enabled=1
 
 [Starlingx-C7.5.1804-os]
 name=Starlingx-CentOS-7.5.1804 - os
-baseurl=http://mirror.centos.org/centos/7.5.1804/os/x86_64/
+baseurl=http://vault.centos.org/7.5.1804/os/x86_64/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -45,14 +45,14 @@ enabled=1
 
 [Starlingx-C7.5.1804-paas-openshift]
 name=Starlingx-CentOS-7.5.1804 - pass-openshift
-baseurl=http://mirror.centos.org/centos/7.5.1804/paas/x86_64/openshift-origin311/
+baseurl=http://vault.centos.org/7.5.1804/paas/x86_64/openshift-origin311/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
 
 [Starlingx-C7.5.1804-rt]
 name=Starlingx-CentOS-7.5.1804 - rt
-baseurl=http://mirror.centos.org/centos/7.5.1804/rt/x86_64/
+baseurl=http://vault.centos.org/7.5.1804/rt/x86_64/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -67,7 +67,7 @@ enabled=1
 
 [Starlingx-C7.5.1804-updates]
 name=Starlingx-CentOS-7.5.1804 - updates
-baseurl=http://mirror.centos.org/centos/7.5.1804/updates/x86_64/
+baseurl=http://vault.centos.org/7.5.1804/updates/x86_64/
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -81,7 +81,7 @@ enabled=1
 
 [Starlingx-C7.5.1804-virt-kvm]
 name=Starlingx-CentOS-7.5.1804 - virt-kvm
-baseurl=http://mirror.centos.org/centos/7.5.1804/virt/x86_64/kvm-common
+baseurl=http://vault.centos.org/7.5.1804/virt/x86_64/kvm-common
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=1
@@ -95,7 +95,7 @@ enabled=1
 
 [Starlingx-C7.5.1804-virt-libvirt]
 name=Starlingx-CentOS-7.5.1804 - virt-libvirt
-baseurl=http://mirror.centos.org/centos/7.5.1804/virt/x86_64/libvirt-latest
+baseurl=http://vault.centos.org/7.5.1804/virt/x86_64/libvirt-latest
 gpgcheck=1
 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 enabled=0

deployment/libvirt/README.rst

@@ -5,8 +5,8 @@ This is a quick reference for deploying StarlingX on libvirt/qemu systems.
 It assumes you have a working libvirt/qemu installation for a non-root user
 and that your user has NOPASSWD sudo permissions.
 
-Refer also to pages "Installation Guide Virtual Environment", "Testing Guide"
+Refer also to pages "Installation Guide" on the StarlingX Documentation:
-on the StarlingX wiki: https://wiki.openstack.org/wiki/StarlingX
+https://docs.starlingx.io/installation_guide/index.html
 
 Overview
 --------
@@ -45,16 +45,19 @@ It should also only be used after all of the VMs created below have been destroy
 Controllers
 -----------
 
-There are two scripts for creating the controllers: ``setup_allinone.sh`` and
+There is one script for creating the controllers: ``setup_configuration.sh``. It
-``setup_standard_controller.sh``.  They are operated in the same manner but build
+builds different StarlingX cloud configurations:
-different StarlingX cloud configurations. Choose wisely.
 
-You need an ISO file for the installation, these scripts take a name with the
+- allinone
+- standardcontroller
+
+You need an StarlingX ISO file for the installation. The script takes the
+configuration name with the ``-c`` option and the ISO file name with the
 ``-i`` option::
 
-	./setup_allinone.sh -i stx-2018-08-28-93.iso
+	./setup_configuration.sh -c allinone -i stx-2018-08-28-93.iso
 
-And the setup will begin.  The scripts create one or more VMs and start the boot
+And the setup will begin.  The script create one or more VMs and start the boot
 of the first controller, named oddly enough ``controller-0``.  If you have Xwindows
 available you will get virt-manager running.
 If not, Ctrl-C out of that attempt if it doesn't return to a shell prompt.
@@ -64,4 +67,6 @@ Then connect to the serial console::
 
 Continue the usual StarlingX installation from this point forward.
 
-Tear down the VMs using ``destroy_allinone.sh`` and ``destroy_standard_controller.sh``.
+Tear down the VMs giving the configuration name with the ``-c`` option::
+
+>-------./destroy_configuration.sh -c allinone

deployment/libvirt/destroy_allinone.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
-
-source ${SCRIPT_DIR}/functions.sh
-
-CONFIGURATION="allinone"
-CONTROLLER=${CONTROLLER:-controller}
-DOMAIN_DIRECTORY=vms
-
-destroy_controller ${CONFIGURATION} ${CONTROLLER}

deployment/libvirt/destroy_configuration.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+
+source ${SCRIPT_DIR}/functions.sh
+
+while getopts "c:" o; do
+    case "${o}" in
+        c)
+            CONFIGURATION=${OPTARG}
+            ;;
+        *)
+            usage_destroy
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [[ -z ${CONFIGURATION} ]]; then
+    usage_destroy
+    exit -1
+fi
+
+CONFIGURATION=${CONFIGURATION:-allinone}
+CONTROLLER=${CONTROLLER:-controller}
+DOMAIN_DIRECTORY=vms
+
+destroy_controller ${CONFIGURATION} ${CONTROLLER}
+
+if ([ "$CONFIGURATION" == "standardcontroller" ]); then
+    COMPUTE=${COMPUTE:-compute}
+    COMPUTE_NODES_NUMBER=${COMPUTE_NODES_NUMBER:-1}
+    for ((i=0; i<=$COMPUTE_NODES_NUMBER; i++)); do
+        COMPUTE_NODE=${COMPUTE}-${i}
+        destroy_compute $COMPUTE_NODE
+    done
+fi

deployment/libvirt/destroy_standard_controller.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
-
-source ${SCRIPT_DIR}/functions.sh
-
-CONFIGURATION="standardcontroller"
-CONTROLLER=${CONTROLLER:-controller}
-COMPUTE=${COMPUTE:-compute}
-COMPUTE_NODES_NUMBER=${COMPUTE_NODES_NUMBER:-1}
-DOMAIN_DIRECTORY=vms
-
-destroy_controller ${CONFIGURATION} ${CONTROLLER}
-
-for ((i=0; i<=$COMPUTE_NODES_NUMBER; i++)); do
-    COMPUTE_NODE=${COMPUTE}-${i}
-    destroy_compute $COMPUTE_NODE
-done

deployment/libvirt/functions.sh

@@ -1,18 +1,26 @@
 #!/usr/bin/env bash
 
 usage() {
-    echo "$0 [-h] [-i <iso image>]"
+    echo "$0 [-h] [-c <configuration>] [-i <iso image>]"
     echo ""
     echo "Options:"
+    echo "  -c: Configuration: allinone, standardcontroller"
     echo "  -i: StarlingX ISO image"
     echo ""
 }
 
+usage_destroy() {
+    echo "$0 [-h] [-c <configuration>]"
+    echo ""
+    echo "Options:"
+    echo "  -c: Configuration: allinone, standardcontroller"
+    echo ""
+}
+
 iso_image_check() {
     local ISOIMAGE=$1
-    FILETYPE=$(file --mime-type -b ${ISOIMAGE})
+    if ! file ${ISOIMAGE} | grep "DOS/MBR" > /dev/null; then
-    if ([ "$FILETYPE" != "application/x-iso9660-image" ]); then
+        echo "$ISOIMAGE is not an ISO type"
-        echo "$ISOIMAGE is not an application/x-iso9660-image type"
         exit -1
     fi
 }

deployment/libvirt/setup_allinone.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
-source ${SCRIPT_DIR}/functions.sh
-
-while getopts "i:" o; do
-    case "${o}" in
-        i)
-            ISOIMAGE=$(readlink -f "$OPTARG")
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${ISOIMAGE}" ]; then
-    usage
-    exit -1
-fi
-
-iso_image_check ${ISOIMAGE}
-
-CONFIGURATION="allinone"
-BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
-CONTROLLER=${CONTROLLER:-controller}
-DOMAIN_DIRECTORY=vms
-
-bash ${SCRIPT_DIR}/destroy_allinone.sh
-
-[ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY}
-
-create_controller $CONFIGURATION $CONTROLLER $BRIDGE_INTERFACE $ISOIMAGE
-
-sudo virt-manager

deployment/libvirt/setup_configuration.sh

@@ -3,8 +3,11 @@
 SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 source ${SCRIPT_DIR}/functions.sh
 
-while getopts "i:" o; do
+while getopts "c:i:" o; do
     case "${o}" in
+        c)
+            CONFIGURATION="$OPTARG"
+            ;;
         i)
             ISOIMAGE=$(readlink -f "$OPTARG")
             ;;
@@ -16,29 +19,31 @@ while getopts "i:" o; do
 done
 shift $((OPTIND-1))
 
-if [ -z "${ISOIMAGE}" ]; then
+if [[ -z ${CONFIGURATION} ]] || [[ -z "${ISOIMAGE}" ]]; then
     usage
     exit -1
 fi
 
 iso_image_check ${ISOIMAGE}
 
-CONFIGURATION="standardcontroller"
+CONFIGURATION=${CONFIGURATION:-allinone}
 BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr}
 CONTROLLER=${CONTROLLER:-controller}
 COMPUTE=${COMPUTE:-compute}
 COMPUTE_NODES_NUMBER=${COMPUTE_NODES_NUMBER:-1}
 DOMAIN_DIRECTORY=vms
 
-bash ${SCRIPT_DIR}/destroy_standard_controller.sh
+bash ${SCRIPT_DIR}/destroy_configuration.sh -c $CONFIGURATION
 
 [ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY}
 
 create_controller $CONFIGURATION $CONTROLLER $BRIDGE_INTERFACE $ISOIMAGE
 
+if ([ "$CONFIGURATION" == "standardcontroller" ]); then
     for ((i=0; i<=$COMPUTE_NODES_NUMBER; i++)); do
         COMPUTE_NODE=${COMPUTE}-${i}
         create_compute ${COMPUTE_NODE}
     done
+fi
 
 sudo virt-manager