From a9cdcae974d0d4e702f85f141aaedd25b6f441ec Mon Sep 17 00:00:00 2001 From: Don Penney Date: Fri, 30 Nov 2018 10:06:20 -0600 Subject: [PATCH] Update docker image and wheels tools for build on master This update adds support to the docker image and wheels build tools to allow the build of StarlingX service images from openstack master branches. The master wheels tarball makes use of the loci-built master-OS wheel image as a base, with any additional base wheels not provided by loci. Change-Id: I6de67e29a9b05189f2a0bad3dfdd1668fbdab42b Story: 2003907 Task: 28179 Signed-off-by: Don Penney --- .../build-docker-images/build-stx-images.sh | 4 +- build-tools/build-wheels/build-base-wheels.sh | 44 +++++++++++++++++++ .../build-wheels/build-wheel-tarball.sh | 19 +++++++- .../build-wheels/docker/master-wheels.cfg | 9 ++++ build-tools/build-wheels/get-stx-wheels.sh | 6 +-- 5 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 build-tools/build-wheels/docker/master-wheels.cfg diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 085c5d77..345b040b 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -185,7 +185,7 @@ function build_image_loci { docker build ${WORKDIR}/loci --no-cache \ "${BUILD_ARGS[@]}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}.log + --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) @@ -297,7 +297,7 @@ function build_image_docker { docker build ${docker_src} --no-cache \ --build-arg "BASE=${BASE}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}.log + --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index ad3bb9ad..f97dc993 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -100,6 +100,11 @@ if [ ! -f ${DOCKER_FILE} ]; then exit 1 fi +if [ ! -f ${WHEELS_CFG} ]; then + echo "Required file does not exist: ${WHEELS_CFG}" >&2 + exit 1 +fi + # # Check build output directory for unexpected files, # ie. wheels from old builds that are no longer in wheels.cfg @@ -130,6 +135,45 @@ for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do fi done +if [ "${OPENSTACK_RELEASE}" = "master" ]; then + # Download the master wheel from loci, so we're only building pieces not covered by it + MASTER_WHEELS_IMAGE="loci/requirements:master-${OS}" + + # Check to see if the wheels are already present. + # If so, we'll still pull to ensure the image is updated, + # but we won't delete it after + docker images --format '{{.Repository}}:{{.Tag}}' ${MASTER_WHEELS_IMAGE} | grep -q "^${MASTER_WHEELS_IMAGE}$" + MASTER_WHEELS_PRESENT=$? + + docker pull ${MASTER_WHEELS_IMAGE} + if [ $? -ne 0 ]; then + echo "Failed to pull ${MASTER_WHEELS_IMAGE}" >&2 + exit 1 + fi + + # Export the image to a tarball. + # The "docker run" will always fail, due to the construct of the wheels image, + # so just ignore it + docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null + + echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}" + docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl' + if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then + echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2 + docker rm ${USER}_inspect_wheels + if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then + docker image rm ${MASTER_WHEELS_IMAGE} + fi + exit 1 + fi + + docker rm ${USER}_inspect_wheels + + if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then + docker image rm ${MASTER_WHEELS_IMAGE} + fi +fi + if [ "${BUILD_NEEDED}" = "no" ]; then echo "All base wheels are already present. Skipping build." exit 0 diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index a134e31f..4fa8b7ea 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -147,18 +147,33 @@ if [ -f ${TARBALL_FNAME} ]; then fi # Download the global-requirements.txt and upper-constraints.txt files -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/global-requirements.txt +if [ "${OPENSTACK_RELEASE}" = "master" ]; then + OPENSTACK_BRANCH=${OPENSTACK_RELEASE} +else + OPENSTACK_BRANCH=stable/${OPENSTACK_RELEASE} +fi + +wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt if [ $? -ne 0 ]; then echo "Failed to download global-requirements.txt" >&2 exit 1 fi -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/upper-constraints.txt +wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 fi +# Delete $SKIP_CONSTRAINTS from upper-constraints.txt, if any present +for name in ${SKIP_CONSTRAINTS[@]}; do + grep -q "^${name}===" upper-constraints.txt + if [ $? -eq 0 ]; then + # Delete the module + sed -i "/^${name}===/d" upper-constraints.txt + fi +done + # Copy the base and stx wheels, updating upper-constraints.txt as necessary for wheel in ../base/*.whl ../stx/wheels/*.whl; do # Get the wheel name and version from the METADATA diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg new file mode 100644 index 00000000..07fc5463 --- /dev/null +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -0,0 +1,9 @@ +# +# git: wheelname|git|git-source|basedir|branch +# tar: wheelname|tar|wget-source|basedir +# pypi: wheelname|pypi|wget-source +# zip: wheelname|zip|wget-source|basedir +# +# If fix_setup must be called, add |fix_setup at the end of the line +# +lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 diff --git a/build-tools/build-wheels/get-stx-wheels.sh b/build-tools/build-wheels/get-stx-wheels.sh index f5f553c6..e24528b8 100755 --- a/build-tools/build-wheels/get-stx-wheels.sh +++ b/build-tools/build-wheels/get-stx-wheels.sh @@ -84,8 +84,8 @@ function get_wheels_files { find ${GIT_LIST} -maxdepth 1 -name "${OS}_${OPENSTACK_RELEASE}_wheels.inc" } -WHEELS_FILES=$(get_wheels_files) -if [ $(echo -n "$WHEELS_FILES" | wc -l) -eq 0 ]; then +declare -a WHEELS_FILES=($(get_wheels_files)) +if [ ${#WHEELS_FILES[@]} -eq 0 ]; then echo "Could not find ${OS} wheels.inc files" >&2 exit 1 fi @@ -100,7 +100,7 @@ cd ${BUILD_OUTPUT_PATH} # Extract the wheels declare -a FAILED -for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES} | sort -u); do +for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES[@]} | sort -u); do case $OS in centos) # Bash globbing does not handle [^\-] well,