Update docker image and wheels tools for build on master

This update adds support to the docker image and wheels build tools
to allow the build of StarlingX service images from openstack master
branches.

The master wheels tarball makes use of the loci-built master-OS wheel
image as a base, with any additional base wheels not provided by loci.

Change-Id: I6de67e29a9b05189f2a0bad3dfdd1668fbdab42b
Story: 2003907
Task: 28179
Signed-off-by: Don Penney <don.penney@windriver.com>

build-tools/build-docker-images/build-stx-images.sh

@@ -185,7 +185,7 @@ function build_image_loci {
 
     docker build ${WORKDIR}/loci --no-cache \
         "${BUILD_ARGS[@]}" \
-        --tag ${build_image_name}  2>&1 | tee ${WORKDIR}/docker-${LABEL}.log
+        --tag ${build_image_name}  2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to build ${LABEL}... Aborting"
         RESULTS_FAILED+=(${LABEL})
@@ -297,7 +297,7 @@ function build_image_docker {
 
     docker build ${docker_src} --no-cache \
         --build-arg "BASE=${BASE}" \
-        --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}.log
+        --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log
     if [ ${PIPESTATUS[0]} -ne 0 ]; then
         echo "Failed to build ${LABEL}... Aborting"
         RESULTS_FAILED+=(${LABEL})

build-tools/build-wheels/build-base-wheels.sh

@@ -100,6 +100,11 @@ if [ ! -f ${DOCKER_FILE} ]; then
     exit 1
 fi
 
+if [ ! -f ${WHEELS_CFG} ]; then
+    echo "Required file does not exist: ${WHEELS_CFG}" >&2
+    exit 1
+fi
+
 #
 # Check build output directory for unexpected files,
 # ie. wheels from old builds that are no longer in wheels.cfg
@@ -130,6 +135,45 @@ for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do
     fi
 done
 
+if [ "${OPENSTACK_RELEASE}" = "master" ]; then
+    # Download the master wheel from loci, so we're only building pieces not covered by it
+    MASTER_WHEELS_IMAGE="loci/requirements:master-${OS}"
+
+    # Check to see if the wheels are already present.
+    # If so, we'll still pull to ensure the image is updated,
+    # but we won't delete it after
+    docker images --format '{{.Repository}}:{{.Tag}}' ${MASTER_WHEELS_IMAGE} | grep -q "^${MASTER_WHEELS_IMAGE}$"
+    MASTER_WHEELS_PRESENT=$?
+
+    docker pull ${MASTER_WHEELS_IMAGE}
+    if [ $? -ne 0 ]; then
+        echo "Failed to pull ${MASTER_WHEELS_IMAGE}" >&2
+        exit 1
+    fi
+
+    # Export the image to a tarball.
+    # The "docker run" will always fail, due to the construct of the wheels image,
+    # so just ignore it
+    docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null
+
+    echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}"
+    docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl'
+    if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then
+        echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2
+        docker rm ${USER}_inspect_wheels
+        if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
+            docker image rm ${MASTER_WHEELS_IMAGE}
+        fi
+        exit 1
+    fi
+
+    docker rm ${USER}_inspect_wheels
+
+    if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
+        docker image rm ${MASTER_WHEELS_IMAGE}
+    fi
+fi
+
 if [ "${BUILD_NEEDED}" = "no" ]; then
     echo "All base wheels are already present. Skipping build."
     exit 0

build-tools/build-wheels/build-wheel-tarball.sh

@@ -147,18 +147,33 @@ if [ -f ${TARBALL_FNAME} ]; then
 fi
 
 # Download the global-requirements.txt and upper-constraints.txt files
-wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/global-requirements.txt
+if [ "${OPENSTACK_RELEASE}" = "master" ]; then
+    OPENSTACK_BRANCH=${OPENSTACK_RELEASE}
+else
+    OPENSTACK_BRANCH=stable/${OPENSTACK_RELEASE}
+fi
+
+wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download global-requirements.txt" >&2
     exit 1
 fi
 
-wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/upper-constraints.txt
+wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
 if [ $? -ne 0 ]; then
     echo "Failed to download upper-constraints.txt" >&2
     exit 1
 fi
 
+# Delete $SKIP_CONSTRAINTS from upper-constraints.txt, if any present
+for name in ${SKIP_CONSTRAINTS[@]}; do
+    grep -q "^${name}===" upper-constraints.txt
+    if [ $? -eq 0 ]; then
+        # Delete the module
+        sed -i "/^${name}===/d" upper-constraints.txt
+    fi
+done
+
 # Copy the base and stx wheels, updating upper-constraints.txt as necessary
 for wheel in ../base/*.whl ../stx/wheels/*.whl; do
     # Get the wheel name and version from the METADATA

build-tools/build-wheels/docker/master-wheels.cfg

@@ -0,0 +1,9 @@
+#
+# git: wheelname|git|git-source|basedir|branch
+# tar: wheelname|tar|wget-source|basedir
+# pypi: wheelname|pypi|wget-source
+# zip: wheelname|zip|wget-source|basedir
+#
+# If fix_setup must be called, add |fix_setup at the end of the line
+#
+lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0

build-tools/build-wheels/get-stx-wheels.sh

@@ -84,8 +84,8 @@ function get_wheels_files {
     find ${GIT_LIST} -maxdepth 1 -name "${OS}_${OPENSTACK_RELEASE}_wheels.inc"
 }
 
-WHEELS_FILES=$(get_wheels_files)
-if [ $(echo -n "$WHEELS_FILES" | wc -l) -eq 0 ]; then
+declare -a WHEELS_FILES=($(get_wheels_files))
+if [ ${#WHEELS_FILES[@]} -eq 0 ]; then
     echo "Could not find ${OS} wheels.inc files" >&2
     exit 1
 fi
@@ -100,7 +100,7 @@ cd ${BUILD_OUTPUT_PATH}
 
 # Extract the wheels
 declare -a FAILED
-for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES} | sort -u); do
+for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES[@]} | sort -u); do
     case $OS in
         centos)
             # Bash globbing does not handle [^\-] well,