diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 085c5d77..345b040b 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -185,7 +185,7 @@ function build_image_loci { docker build ${WORKDIR}/loci --no-cache \ "${BUILD_ARGS[@]}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}.log + --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) @@ -297,7 +297,7 @@ function build_image_docker { docker build ${docker_src} --no-cache \ --build-arg "BASE=${BASE}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}.log + --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index ad3bb9ad..f97dc993 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -100,6 +100,11 @@ if [ ! -f ${DOCKER_FILE} ]; then exit 1 fi +if [ ! -f ${WHEELS_CFG} ]; then + echo "Required file does not exist: ${WHEELS_CFG}" >&2 + exit 1 +fi + # # Check build output directory for unexpected files, # ie. wheels from old builds that are no longer in wheels.cfg @@ -130,6 +135,45 @@ for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do fi done +if [ "${OPENSTACK_RELEASE}" = "master" ]; then + # Download the master wheel from loci, so we're only building pieces not covered by it + MASTER_WHEELS_IMAGE="loci/requirements:master-${OS}" + + # Check to see if the wheels are already present. + # If so, we'll still pull to ensure the image is updated, + # but we won't delete it after + docker images --format '{{.Repository}}:{{.Tag}}' ${MASTER_WHEELS_IMAGE} | grep -q "^${MASTER_WHEELS_IMAGE}$" + MASTER_WHEELS_PRESENT=$? + + docker pull ${MASTER_WHEELS_IMAGE} + if [ $? -ne 0 ]; then + echo "Failed to pull ${MASTER_WHEELS_IMAGE}" >&2 + exit 1 + fi + + # Export the image to a tarball. + # The "docker run" will always fail, due to the construct of the wheels image, + # so just ignore it + docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null + + echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}" + docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl' + if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then + echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2 + docker rm ${USER}_inspect_wheels + if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then + docker image rm ${MASTER_WHEELS_IMAGE} + fi + exit 1 + fi + + docker rm ${USER}_inspect_wheels + + if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then + docker image rm ${MASTER_WHEELS_IMAGE} + fi +fi + if [ "${BUILD_NEEDED}" = "no" ]; then echo "All base wheels are already present. Skipping build." exit 0 diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index a134e31f..4fa8b7ea 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -147,18 +147,33 @@ if [ -f ${TARBALL_FNAME} ]; then fi # Download the global-requirements.txt and upper-constraints.txt files -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/global-requirements.txt +if [ "${OPENSTACK_RELEASE}" = "master" ]; then + OPENSTACK_BRANCH=${OPENSTACK_RELEASE} +else + OPENSTACK_BRANCH=stable/${OPENSTACK_RELEASE} +fi + +wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt if [ $? -ne 0 ]; then echo "Failed to download global-requirements.txt" >&2 exit 1 fi -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/upper-constraints.txt +wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 fi +# Delete $SKIP_CONSTRAINTS from upper-constraints.txt, if any present +for name in ${SKIP_CONSTRAINTS[@]}; do + grep -q "^${name}===" upper-constraints.txt + if [ $? -eq 0 ]; then + # Delete the module + sed -i "/^${name}===/d" upper-constraints.txt + fi +done + # Copy the base and stx wheels, updating upper-constraints.txt as necessary for wheel in ../base/*.whl ../stx/wheels/*.whl; do # Get the wheel name and version from the METADATA diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg new file mode 100644 index 00000000..07fc5463 --- /dev/null +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -0,0 +1,9 @@ +# +# git: wheelname|git|git-source|basedir|branch +# tar: wheelname|tar|wget-source|basedir +# pypi: wheelname|pypi|wget-source +# zip: wheelname|zip|wget-source|basedir +# +# If fix_setup must be called, add |fix_setup at the end of the line +# +lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 diff --git a/build-tools/build-wheels/get-stx-wheels.sh b/build-tools/build-wheels/get-stx-wheels.sh index f5f553c6..e24528b8 100755 --- a/build-tools/build-wheels/get-stx-wheels.sh +++ b/build-tools/build-wheels/get-stx-wheels.sh @@ -84,8 +84,8 @@ function get_wheels_files { find ${GIT_LIST} -maxdepth 1 -name "${OS}_${OPENSTACK_RELEASE}_wheels.inc" } -WHEELS_FILES=$(get_wheels_files) -if [ $(echo -n "$WHEELS_FILES" | wc -l) -eq 0 ]; then +declare -a WHEELS_FILES=($(get_wheels_files)) +if [ ${#WHEELS_FILES[@]} -eq 0 ]; then echo "Could not find ${OS} wheels.inc files" >&2 exit 1 fi @@ -100,7 +100,7 @@ cd ${BUILD_OUTPUT_PATH} # Extract the wheels declare -a FAILED -for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES} | sort -u); do +for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES[@]} | sort -u); do case $OS in centos) # Bash globbing does not handle [^\-] well,