Merge remote-tracking branch 'gerrit/master' into f/centos76

Change-Id: I66ff26289b850d0fcfcb7c4c3db32ded81beba32 Signed-off-by: Saul Wold <sgw@linux.intel.com>
2018-12-20 12:13:13 -08:00 · 2018-12-20 12:13:13 -08:00 · 8312ffad07
parent aeaf791ff8 550c28895f
commit 8312ffad07
7 changed files with 121 additions and 26 deletions
--- a/build-tools/build-docker-images/README
+++ b/build-tools/build-docker-images/README
@ -1,34 +1,51 @@
 ## Example commands for building StarlingX images

 PRIVATE_REGISTRY_USERID=myuser
-PRIVATE_REGISTRY=xxx.xxx.xxx.xxx
+PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001
+VERSION=2018.11.13
+OS=centos
+OS_VERSION=7.5.1804
+OPENSTACK_RELEASE=pike
+HOST_PORT=8088

 ## Step 1: Build stx-centos
 time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \
-    --version 2018.11.13 \
-    --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \
+    --os ${OS} \
+    --os-version ${OS_VERSION} \
+    --version ${VERSION} \
+    --user ${PRIVATE_REGISTRY_USERID} \
+    --registry ${PRIVATE_REGISTRY} \
    --push \
-    --repo stx-local-build,http://${HOSTNAME}:8088/localdisk/loadbuild/jenkins/StarlingX_Upstream_build/2018-11-13_20-18-00/std/rpmbuild/RPMS \
-    --repo stx-mirror-distro,http://${HOSTNAME}:8088/localdisk/designer/jenkins/StarlingX_upstream/cgcs-root/cgcs-centos-repo/Binary \
+    --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \
+    --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \
    --clean


 ## Step 2: Build wheels (output as tarball)
 time $MY_REPO/build-tools/build-wheels/build-wheel-tarball.sh \
-    --os centos \
-    --release pike
+    --os ${OS} \
+    --os-version ${OS_VERSION} \
+    --release ${OPENSTACK_RELEASE}

 ## Step 3: Build images
 time $MY_REPO/build-tools/build-docker-images/build-stx-images.sh \
-    --os centos \
-    --base ${PRIVATE_REGISTRY}:9001/${PRIVATE_REGISTRY_USERID}/stx-centos:2018.11.13 \
-    --wheels http://${HOSTNAME}:8088/$MY_WORKSPACE/std/build-wheels-centos-pike/stx-centos-pike-wheels.tar \
-    --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \
+    --os ${OS} \
+    --version ${VERSION} \
+    --release ${OPENSTACK_RELEASE} \
+    --base ${PRIVATE_REGISTRY}/${PRIVATE_REGISTRY_USERID}/stx-${OS}:${VERSION} \
+    --wheels http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/stx-${OS}-${OPENSTACK_RELEASE}-wheels.tar \
+    --user ${PRIVATE_REGISTRY_USERID} \
+    --registry ${PRIVATE_REGISTRY} \
    --push --latest \
    --clean


+## Note: Verify that lighttpd is not bound to "localhost"
+vi /etc/lighttpd/lighttpd.conf
+# server.bind = "localhost"
+systemctl restart lighttpd
+
 ## Note: You may need to add an iptables rule to allow the docker
 ## containers to access the http server on your host. For example:
-iptables -I INPUT 6 -i docker0 -p tcp --dport 8088 -m state --state NEW,ESTABLISHED -j ACCEPT
+iptables -I INPUT 6 -i docker0 -p tcp --dport ${HOST_PORT} -m state --state NEW,ESTABLISHED -j ACCEPT

--- a/build-tools/build-docker-images/build-stx-base.sh
+++ b/build-tools/build-docker-images/build-stx-base.sh
@ -26,6 +26,9 @@ declare -a REPO_LIST
 REPO_OPTS=
 LOCAL=no
 CLEAN=no
+TAG_LATEST=no
+LATEST_TAG=latest
+HOST=${HOSTNAME}

 function usage {
    cat >&2 <<EOF
@ -39,14 +42,17 @@ Options:
    --repo:       Software repository (Format: name,baseurl), can be specified multiple times
    --local:      Use local build for software repository (cannot be used with --repo)
    --push:       Push to docker repo
+    --latest:     Add a 'latest' tag when pushing
+    --latest-tag: Use the provided tag when pushing latest.
    --user:       Docker repo userid
    --registry:   Docker registry
    --clean:      Remove image(s) from local registry
+    --hostname:   build repo host

 EOF
 }

-OPTS=$(getopt -o h -l help,os:,os-version:,version:,repo:,push,user:,registry:,local,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,version:,repo:,push,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -85,6 +91,14 @@ while true; do
            PUSH=yes
            shift
            ;;
+        --latest)
+            TAG_LATEST=yes
+            shift
+            ;;
+        --latest-tag)
+            LATEST_TAG=$2
+            shift 2
+            ;;
        --user)
            DOCKER_USER=$2
            shift 2
@ -98,6 +112,10 @@ while true; do
            CLEAN=yes
            shift
            ;;
+        --hostname)
+            HOST=$2
+            shift 2
+            ;;
        -h | --help )
            usage
            exit 1
@ -130,8 +148,8 @@ fi
 if [ ${#REPO_LIST[@]} -eq 0 ]; then
    # Either --repo or --local must be specified
    if [ "${LOCAL}" = "yes" ]; then
-        REPO_LIST+=("local-std,http://${HOSTNAME}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
-        REPO_LIST+=("stx-distro,http://${HOSTNAME}:8088${MY_REPO}/cgcs-centos-repo/Binary")
+        REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
+        REPO_LIST+=("stx-distro,http://${HOST}:8088${MY_REPO}/cgcs-${OS}-repo/Binary")
    else
        echo "Either --local or --repo must be specified" >&2
        exit 1
@ -191,6 +209,7 @@ BASE_IMAGE_PRESENT=$?

 # Build the image
 IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
+IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}

 docker build \
    --build-arg RELEASE=${OS_VERSION} \
@ -209,6 +228,16 @@ if [ "${PUSH}" = "yes" ]; then
        echo "Failed running docker push command" >&2
        exit 1
    fi
+
+    if [ "$TAG_LATEST" = "yes" ]; then
+        docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST}
+        echo "Pushing image: ${IMAGE_NAME_LATEST}"
+        docker push ${IMAGE_NAME_LATEST}
+        if [ $? -ne 0 ]; then
+            echo "Failed running docker push command on latest" >&2
+            exit 1
+        fi
+    fi
 fi

 if [ "${CLEAN}" = "yes" ]; then
--- a/build-tools/build-docker-images/build-stx-images.sh
+++ b/build-tools/build-docker-images/build-stx-images.sh
@ -21,13 +21,15 @@ SUPPORTED_OS_ARGS=('centos')
 OS=centos
 OPENSTACK_RELEASE=pike
 IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
-RELEASE=dev
+PREFIX=dev
+LATEST_PREFIX=""
 PUSH=no
 DOCKER_USER=${USER}
 DOCKER_REGISTRY=
 BASE=
 WHEELS=
 CLEAN=no
+TAG_LATEST=no
 declare -a ONLY
 declare -a SKIP

@ -45,6 +47,9 @@ Options:
    --push:       Push to docker repo
    --user:       Docker repo userid
    --registry:   Docker registry
+    --prefix:     Prefix on the image tag (default: dev)
+    --latest:     Add a 'latest' tag when pushing
+    --latest-prefix: Alternative prefix on the latest image tag
    --clean:      Remove image(s) from local registry
    --only <image> : Only build the specified image(s). Multiple images
                     can be specified with a comma-separated list, or with
@ -344,7 +349,7 @@ function build_image {
    esac
 }

-OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,latest,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -379,6 +384,14 @@ while true; do
            OPENSTACK_RELEASE=$2
            shift 2
            ;;
+        --prefix)
+            PREFIX=$2
+            shift 2
+            ;;
+        --latest-prefix)
+            LATEST_PREFIX=$2
+            shift 2
+            ;;
        --push)
            PUSH=yes
            shift
@ -445,9 +458,24 @@ if [ -z "${BASE}" ]; then
    exit 1
 fi

-IMAGE_TAG_BUILD="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-build"
-IMAGE_TAG="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-${IMAGE_VERSION}"
-IMAGE_TAG_LATEST="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-latest"
+IMAGE_TAG="${OS}-${OPENSTACK_RELEASE}"
+IMAGE_TAG_LATEST="${IMAGE_TAG}-latest"
+
+if [ -n "${LATEST_PREFIX}" ]; then
+    IMAGE_TAG_LATEST="${LATEST_PREFIX}-${IMAGE_TAG_LATEST}"
+elif [ -n "${PREFIX}" ]; then
+    IMAGE_TAG_LATEST="${PREFIX}-${IMAGE_TAG_LATEST}"
+fi
+
+if [ -n "${PREFIX}" ]; then
+    IMAGE_TAG="${PREFIX}-${IMAGE_TAG}"
+fi
+
+IMAGE_TAG_BUILD="${IMAGE_TAG}-build"
+
+if [ -n "${IMAGE_VERSION}" ]; then
+    IMAGE_TAG="${IMAGE_TAG}-${IMAGE_VERSION}"
+fi

 WORKDIR=${MY_WORKSPACE}/std/build-images
 mkdir -p ${WORKDIR}
--- a/build-tools/build-wheels/build-base-wheels.sh
+++ b/build-tools/build-wheels/build-base-wheels.sh
@ -36,7 +36,7 @@ Options:
 EOF
 }

-OPTS=$(getopt -o h -l help,os:,keep-image,keep-container,release: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -83,8 +83,13 @@ while true; do
 done

 BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base
+
 BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}"

+# BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME.
+# The following will substitute caps with lower case.
+BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}"
+
 DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile
 WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg

--- a/build-tools/build-wheels/build-wheel-tarball.sh
+++ b/build-tools/build-wheels/build-wheel-tarball.sh
@ -17,6 +17,7 @@ fi

 SUPPORTED_OS_ARGS=('centos')
 OS=centos
+OS_VERSION=7.5.1804
 OPENSTACK_RELEASE=pike
 VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PUSH=no
@ -46,6 +47,7 @@ $(basename $0)

 Options:
    --os:         Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
+    --os-version:     Specify OS version
    --release:    Openstack release (default: pike)
    --push:       Push to docker repo
    --user:       Docker repo userid
@ -54,7 +56,7 @@ Options:
 EOF
 }

-OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -73,6 +75,10 @@ while true; do
            OS=$2
            shift 2
            ;;
+        --os-version)
+            OS_VERSION=$2
+            shift 2
+            ;;
        --push)
            PUSH=yes
            shift
@ -119,7 +125,7 @@ if [ ${VALID_OS} -ne 0 ]; then
 fi

 # Build the base wheels and retrieve the StarlingX wheels
-${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE}
+${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE}
 if [ $? -ne 0 ]; then
    echo "Failure running build-base-wheels.sh" >&2
    exit 1
--- a/build-tools/build-wheels/docker/pike-wheels.cfg
+++ b/build-tools/build-wheels/docker/pike-wheels.cfg
@ -54,6 +54,7 @@ itsdangerous-0.24-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d
 jaeger_client-3.10.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ee/1d/66cfa8ef438980e523c7ca3ed98c1b865b4e18df4167f32133ae3053c86c/jaeger-client-3.10.0.tar.gz|jaeger-client-3.10.0
 jdcal-1.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/9b/fa/40beb2aa43a13f740dd5be367a10a03270043787833409c61b79e69f1dfd/jdcal-1.3.tar.gz|jdcal-1.3
 jsonpath_rw-1.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/71/7c/45001b1f19af8c4478489fbae4fc657b21c4c669d7a5a036a86882581d85/jsonpath-rw-1.4.0.tar.gz|jsonpath-rw-1.4.0
+keyring-17.0.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/c8/4d/e1f19eebda5cd58030ecc1f61321f71661cfdb9d911d0b1053ab0ca7351c/keyring-17.0.0-py2.py3-none-any.whl
 krest-1.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fb/d2/9dbbd3a76f2385041720a0eb51ddab676e688fa8bee8a1489470839616cf/krest-1.3.1.tar.gz|krest-1.3.1
 #libvirt_python-4.4.0-cp27-none-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2b/8d/1160cf34dc3d296896eb5c8f4944439ea368b87d2d2431f58d08d6bdf374/libvirt-python-4.4.0.tar.gz|libvirt-python-4.4.0|fix_setup
 logutils-0.3.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/b2/b57450889bf73da26027f8b995fd5fbfab258ec24ef967e4c1892f7cb121/logutils-0.3.5.tar.gz|logutils-0.3.5|fix_setup
@ -90,7 +91,7 @@ osc_placement-1.3.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages
 os_net_config-7.3.8-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/f5/f0/17c140a29b5100db11cc186095aa48a90b3ab659844992cfbb5224f75704/os_net_config-7.3.8-py2-none-any.whl
 osprofiler-1.11.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/80/46/395e951f6c5797f8a320e5df7a8217118b20ccf47441ae29c97b377ffd6c/osprofiler-1.11.0-py2-none-any.whl
 os_service_types-1.0.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/44/b3/8d03e393fea829f654f737e740059b5f528ecbb991efba1859fa7133ae5e/os_service_types-1.0.0-py2-none-any.whl
-os_vif-1.7.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/74/66/c2727e4bb7302029d2406065233c3069176edef803be721c3ccf44a0f9ed/os_vif-1.7.0-py2-none-any.whl
+os_vif-1.9.1-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/95/ee/3fe8ccb644b20f73e7c6bbd78d78ceecc30b9bc23e7a96b8eee54a0a9d5e/os_vif-1.9.1-py2-none-any.whl
 ovs-2.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/8a/14/c0cf1101406f24de2b72fd239ae638c489674d856c8d6dda41da2daa55ba/ovs-2.7.0.tar.gz|ovs-2.7.0
 pathlib-1.0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz|pathlib-1.0.1|fix_setup
 pifpaf-1.9.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/38/4d/0e594192743aa7184051119951431e15e989cc222428d5a4856c528f57db/pifpaf-1.9.2.tar.gz|pifpaf-1.9.2
--- a/build-tools/modify-build-cfg
+++ b/build-tools/modify-build-cfg
@ -92,8 +92,13 @@ if [ -f /usr/lib64/nosync/nosync.so ]; then
        echo "config_opts['nosync'] = True" >> $FILE
 fi

-grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum'" $FILE || \
-    echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum'" >> $FILE
+NETWORK_PKGS=""
+if [ "containers" == "$BUILD_TYPE" ]; then
+    NETWORK_PKGS="bind-utils"
+fi
+
+grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" $FILE || \
+    echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" >> $FILE

 # Special case for containers.
 # rpmbuild_networking is required for invoking helm commands within mock
@ -101,6 +106,10 @@ grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip
 if [ "containers" == "$BUILD_TYPE" ]; then
    grep -q "config_opts\['rpmbuild_networking'\] = True" $FILE || \
        echo "config_opts['rpmbuild_networking'] = True" >> $FILE
+
+    grep -q "config_opts\['use_host_resolv'\] = True" $FILE || \
+        echo "config_opts['use_host_resolv'] = True" >> $FILE
+
   sed -i "/^\[local-std\]/,/^\[/ s/enabled=0/enabled=1/" $FILE
 fi