Add greater flexibility in docker image naming

The existing user interface for docker image and wheel creation
is not flexible enough to name images is described in the spec,
and to push those images to the starlingx organization at docker.hub

This update adds new arguements that will allow creation of
docker images with compliant names, and will allow them to
be pushed to https://hub.docker.com/r/starlingx.

I will also change build-base-wheels.sh to allow the
'os-version' to be pushed through to build-wheels-tarball.sh.

Story: 2003907
Task: 28370

Change-Id: I4378db63d5ff906ffd636034351e580afa5e3472
Signed-off-by: Scott Little <scott.little@windriver.com>

build-tools/build-docker-images/README

@@ -1,34 +1,51 @@
 ## Example commands for building StarlingX images
 
 PRIVATE_REGISTRY_USERID=myuser
-PRIVATE_REGISTRY=xxx.xxx.xxx.xxx
+PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001
+VERSION=2018.11.13
+OS=centos
+OS_VERSION=7.5.1804
+OPENSTACK_RELEASE=pike
+HOST_PORT=8088
 
 ## Step 1: Build stx-centos
 time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \
-    --version 2018.11.13 \
+    --os ${OS} \
-    --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \
+    --os-version ${OS_VERSION} \
+    --version ${VERSION} \
+    --user ${PRIVATE_REGISTRY_USERID} \
+    --registry ${PRIVATE_REGISTRY} \
     --push \
-    --repo stx-local-build,http://${HOSTNAME}:8088/localdisk/loadbuild/jenkins/StarlingX_Upstream_build/2018-11-13_20-18-00/std/rpmbuild/RPMS \
+    --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \
-    --repo stx-mirror-distro,http://${HOSTNAME}:8088/localdisk/designer/jenkins/StarlingX_upstream/cgcs-root/cgcs-centos-repo/Binary \
+    --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \
     --clean
 
 
 ## Step 2: Build wheels (output as tarball)
 time $MY_REPO/build-tools/build-wheels/build-wheel-tarball.sh \
-    --os centos \
+    --os ${OS} \
-    --release pike
+    --os-version ${OS_VERSION} \
+    --release ${OPENSTACK_RELEASE}
 
 ## Step 3: Build images
 time $MY_REPO/build-tools/build-docker-images/build-stx-images.sh \
-    --os centos \
+    --os ${OS} \
-    --base ${PRIVATE_REGISTRY}:9001/${PRIVATE_REGISTRY_USERID}/stx-centos:2018.11.13 \
+    --version ${VERSION} \
-    --wheels http://${HOSTNAME}:8088/$MY_WORKSPACE/std/build-wheels-centos-pike/stx-centos-pike-wheels.tar \
+    --release ${OPENSTACK_RELEASE} \
-    --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \
+    --base ${PRIVATE_REGISTRY}/${PRIVATE_REGISTRY_USERID}/stx-${OS}:${VERSION} \
+    --wheels http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/stx-${OS}-${OPENSTACK_RELEASE}-wheels.tar \
+    --user ${PRIVATE_REGISTRY_USERID} \
+    --registry ${PRIVATE_REGISTRY} \
     --push --latest \
     --clean
 
 
+## Note: Verify that lighttpd is not bound to "localhost"
+vi /etc/lighttpd/lighttpd.conf
+# server.bind = "localhost"
+systemctl restart lighttpd
+
 ## Note: You may need to add an iptables rule to allow the docker
 ## containers to access the http server on your host. For example:
-iptables -I INPUT 6 -i docker0 -p tcp --dport 8088 -m state --state NEW,ESTABLISHED -j ACCEPT
+iptables -I INPUT 6 -i docker0 -p tcp --dport ${HOST_PORT} -m state --state NEW,ESTABLISHED -j ACCEPT
 

build-tools/build-docker-images/build-stx-base.sh

@@ -26,6 +26,9 @@ declare -a REPO_LIST
 REPO_OPTS=
 LOCAL=no
 CLEAN=no
+TAG_LATEST=no
+LATEST_TAG=latest
+HOST=${HOSTNAME}
 
 function usage {
     cat >&2 <<EOF
@@ -39,14 +42,17 @@ Options:
     --repo:       Software repository (Format: name,baseurl), can be specified multiple times
     --local:      Use local build for software repository (cannot be used with --repo)
     --push:       Push to docker repo
+    --latest:     Add a 'latest' tag when pushing
+    --latest-tag: Use the provided tag when pushing latest.
     --user:       Docker repo userid
     --registry:   Docker registry
     --clean:      Remove image(s) from local registry
+    --hostname:   build repo host
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,version:,repo:,push,user:,registry:,local,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,version:,repo:,push,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -85,6 +91,14 @@ while true; do
             PUSH=yes
             shift
             ;;
+        --latest)
+            TAG_LATEST=yes
+            shift
+            ;;
+        --latest-tag)
+            LATEST_TAG=$2
+            shift 2
+            ;;
         --user)
             DOCKER_USER=$2
             shift 2
@@ -98,6 +112,10 @@ while true; do
             CLEAN=yes
             shift
             ;;
+        --hostname)
+            HOST=$2
+            shift 2
+            ;;
         -h | --help )
             usage
             exit 1
@@ -130,8 +148,8 @@ fi
 if [ ${#REPO_LIST[@]} -eq 0 ]; then
     # Either --repo or --local must be specified
     if [ "${LOCAL}" = "yes" ]; then
-        REPO_LIST+=("local-std,http://${HOSTNAME}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
+        REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
-        REPO_LIST+=("stx-distro,http://${HOSTNAME}:8088${MY_REPO}/cgcs-centos-repo/Binary")
+        REPO_LIST+=("stx-distro,http://${HOST}:8088${MY_REPO}/cgcs-${OS}-repo/Binary")
     else
         echo "Either --local or --repo must be specified" >&2
         exit 1
@@ -191,6 +209,7 @@ BASE_IMAGE_PRESENT=$?
 
 # Build the image
 IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
+IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
 
 docker build \
     --build-arg RELEASE=${OS_VERSION} \
@@ -209,6 +228,16 @@ if [ "${PUSH}" = "yes" ]; then
         echo "Failed running docker push command" >&2
         exit 1
     fi
+
+    if [ "$TAG_LATEST" = "yes" ]; then
+        docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST}
+        echo "Pushing image: ${IMAGE_NAME_LATEST}"
+        docker push ${IMAGE_NAME_LATEST}
+        if [ $? -ne 0 ]; then
+            echo "Failed running docker push command on latest" >&2
+            exit 1
+        fi
+    fi
 fi
 
 if [ "${CLEAN}" = "yes" ]; then

build-tools/build-docker-images/build-stx-images.sh

@@ -21,13 +21,15 @@ SUPPORTED_OS_ARGS=('centos')
 OS=centos
 OPENSTACK_RELEASE=pike
 IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
-RELEASE=dev
+PREFIX=dev
+LATEST_PREFIX=""
 PUSH=no
 DOCKER_USER=${USER}
 DOCKER_REGISTRY=
 BASE=
 WHEELS=
 CLEAN=no
+TAG_LATEST=no
 declare -a ONLY
 declare -a SKIP
 
@@ -45,6 +47,9 @@ Options:
     --push:       Push to docker repo
     --user:       Docker repo userid
     --registry:   Docker registry
+    --prefix:     Prefix on the image tag (default: dev)
+    --latest:     Add a 'latest' tag when pushing
+    --latest-prefix: Alternative prefix on the latest image tag
     --clean:      Remove image(s) from local registry
     --only <image> : Only build the specified image(s). Multiple images
                      can be specified with a comma-separated list, or with
@@ -344,7 +349,7 @@ function build_image {
     esac
 }
 
-OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,latest,clean -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -379,6 +384,14 @@ while true; do
             OPENSTACK_RELEASE=$2
             shift 2
             ;;
+        --prefix)
+            PREFIX=$2
+            shift 2
+            ;;
+        --latest-prefix)
+            LATEST_PREFIX=$2
+            shift 2
+            ;;
         --push)
             PUSH=yes
             shift
@@ -445,9 +458,24 @@ if [ -z "${BASE}" ]; then
     exit 1
 fi
 
-IMAGE_TAG_BUILD="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-build"
+IMAGE_TAG="${OS}-${OPENSTACK_RELEASE}"
-IMAGE_TAG="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-${IMAGE_VERSION}"
+IMAGE_TAG_LATEST="${IMAGE_TAG}-latest"
-IMAGE_TAG_LATEST="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-latest"
+
+if [ -n "${LATEST_PREFIX}" ]; then
+    IMAGE_TAG_LATEST="${LATEST_PREFIX}-${IMAGE_TAG_LATEST}"
+elif [ -n "${PREFIX}" ]; then
+    IMAGE_TAG_LATEST="${PREFIX}-${IMAGE_TAG_LATEST}"
+fi
+
+if [ -n "${PREFIX}" ]; then
+    IMAGE_TAG="${PREFIX}-${IMAGE_TAG}"
+fi
+
+IMAGE_TAG_BUILD="${IMAGE_TAG}-build"
+
+if [ -n "${IMAGE_VERSION}" ]; then
+    IMAGE_TAG="${IMAGE_TAG}-${IMAGE_VERSION}"
+fi
 
 WORKDIR=${MY_WORKSPACE}/std/build-images
 mkdir -p ${WORKDIR}

build-tools/build-wheels/build-base-wheels.sh

@@ -36,7 +36,7 @@ Options:
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,keep-image,keep-container,release: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -83,8 +83,13 @@ while true; do
 done
 
 BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base
+
 BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}"
 
+# BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME.
+# The following will substitute caps with lower case.
+BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}"
+
 DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile
 WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg
 

build-tools/build-wheels/build-wheel-tarball.sh

@@ -17,6 +17,7 @@ fi
 
 SUPPORTED_OS_ARGS=('centos')
 OS=centos
+OS_VERSION=7.5.1804
 OPENSTACK_RELEASE=pike
 VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PUSH=no
@@ -46,6 +47,7 @@ $(basename $0)
 
 Options:
     --os:         Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
+    --os-version:     Specify OS version
     --release:    Openstack release (default: pike)
     --push:       Push to docker repo
     --user:       Docker repo userid
@@ -54,7 +56,7 @@ Options:
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -73,6 +75,10 @@ while true; do
             OS=$2
             shift 2
             ;;
+        --os-version)
+            OS_VERSION=$2
+            shift 2
+            ;;
         --push)
             PUSH=yes
             shift
@@ -119,7 +125,7 @@ if [ ${VALID_OS} -ne 0 ]; then
 fi
 
 # Build the base wheels and retrieve the StarlingX wheels
-${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE}
+${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE}
 if [ $? -ne 0 ]; then
     echo "Failure running build-base-wheels.sh" >&2
     exit 1