Merge "Enhancements to wheel build tools"
This commit is contained in:
Zuul
Gerrit Code Review
commit
ae8c653f47
|
|
@ -15,11 +15,11 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
|||
fi
|
||||
|
||||
DOCKER_PATH=${MY_REPO}/build-tools/build-wheels/docker
|
||||
WHEELS_CFG=${DOCKER_PATH}/wheels.cfg
|
||||
KEEP_IMAGE=no
|
||||
KEEP_CONTAINER=no
|
||||
OS=centos
|
||||
OS_RELEASE=pike
|
||||
OS_VERSION=7.5.1804
|
||||
OPENSTACK_RELEASE=pike
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -28,6 +28,7 @@ $(basename $0) [ --os <os> ] [ --keep-image ] [ --keep-container ] [ --release <
|
|||
|
||||
Options:
|
||||
--os: Specify base OS (eg. centos)
|
||||
--os-version: Specify OS version
|
||||
--keep-image: Skip deletion of the wheel build image in docker
|
||||
--keep-container: Skip deletion of container used for the build
|
||||
--release: Openstack release (default: pike)
|
||||
|
|
@ -54,6 +55,10 @@ while true; do
|
|||
OS=$2
|
||||
shift 2
|
||||
;;
|
||||
--os-version)
|
||||
OS_VERSION=$2
|
||||
shift 2
|
||||
;;
|
||||
--keep-image)
|
||||
KEEP_IMAGE=yes
|
||||
shift
|
||||
|
|
@ -63,7 +68,7 @@ while true; do
|
|||
shift
|
||||
;;
|
||||
--release)
|
||||
OS_RELEASE=$2
|
||||
OPENSTACK_RELEASE=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
|
|
@ -77,10 +82,11 @@ while true; do
|
|||
esac
|
||||
done
|
||||
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/base
|
||||
BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OS_RELEASE}"
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base
|
||||
BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}"
|
||||
|
||||
DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile
|
||||
WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg
|
||||
|
||||
function supported_os_list {
|
||||
for f in ${DOCKER_PATH}/*-dockerfile; do
|
||||
|
|
@ -115,8 +121,29 @@ else
|
|||
fi
|
||||
fi
|
||||
|
||||
# Check to see if we need to build anything
|
||||
BUILD_NEEDED=no
|
||||
for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do
|
||||
if [ ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]; then
|
||||
BUILD_NEEDED=yes
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${BUILD_NEEDED}" = "no" ]; then
|
||||
echo "All base wheels are already present. Skipping build."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check to see if the OS image is already pulled
|
||||
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
|
||||
BASE_IMAGE_PRESENT=$?
|
||||
|
||||
# Create the builder image
|
||||
docker build --build-arg OS_RELEASE=${OS_RELEASE} -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}
|
||||
docker build \
|
||||
--build-arg RELEASE=${OS_VERSION} \
|
||||
--build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE} \
|
||||
-t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to create build image in docker" >&2
|
||||
exit 1
|
||||
|
|
@ -131,11 +158,20 @@ docker run ${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels -i -t ${BUILD_IMAGE_NAME} /
|
|||
|
||||
if [ "${KEEP_IMAGE}" = "no" ]; then
|
||||
# Delete the builder image
|
||||
echo "Removing docker build image ${BUILD_IMAGE_NAME}"
|
||||
echo "Removing docker image ${BUILD_IMAGE_NAME}"
|
||||
docker image rm ${BUILD_IMAGE_NAME}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to delete build image from docker" >&2
|
||||
fi
|
||||
|
||||
if [ ${BASE_IMAGE_PRESENT} -ne 0 ]; then
|
||||
# The base image was not already present, so delete it
|
||||
echo "Removing docker image ${OS}:${OS_VERSION}"
|
||||
docker image rm ${OS}:${OS_VERSION}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to delete base image from docker" >&2
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check for failures
|
||||
|
|
|
|||
|
|
@ -16,83 +16,50 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
|||
fi
|
||||
|
||||
SUPPORTED_OS_ARGS=('centos')
|
||||
OS=centos
|
||||
OPENSTACK_RELEASE=pike
|
||||
VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
|
||||
PUSH=no
|
||||
CLEAN=no
|
||||
DOCKER_USER=${USER}
|
||||
|
||||
# List of top-level services for images, which should not be listed in upper-constraints.txt
|
||||
SKIP_CONSTRAINTS=(
|
||||
ceilometer
|
||||
cinder
|
||||
glance
|
||||
gnocchi
|
||||
heat
|
||||
horizon
|
||||
ironic
|
||||
keystone
|
||||
magnum
|
||||
murano
|
||||
neutron
|
||||
nova
|
||||
)
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
Usage:
|
||||
$(basename $0) [ --os <os> ] [ --push ] [ --user <userid> ] [ --release <release> ]
|
||||
$(basename $0)
|
||||
|
||||
Options:
|
||||
--os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
|
||||
--release: Openstack release (default: pike)
|
||||
--push: Push to docker repo
|
||||
--user: Docker repo userid
|
||||
--release: Openstack release (default: pike)
|
||||
--version: Version for pushed image (if used with --push)
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
#
|
||||
# Function to get an auth token from docker
|
||||
#
|
||||
function get_docker_token {
|
||||
local auth_server="auth.docker.io"
|
||||
local registry="registry.docker.io"
|
||||
local scope="repository:${REPO_USER}/${IMAGE_NAME}:pull"
|
||||
curl -k -sSL -X GET "https://${auth_server}/token?service=${registry}&scope=${scope}" \
|
||||
| python -c '
|
||||
import sys, yaml, json
|
||||
y=yaml.load(sys.stdin.read())
|
||||
print y["token"]
|
||||
' 2>/dev/null
|
||||
}
|
||||
|
||||
#
|
||||
# Function to query docker for the highest current version
|
||||
#
|
||||
function get_current_version {
|
||||
local registry_server="registry-1.docker.io"
|
||||
local token=
|
||||
token=$(get_docker_token)
|
||||
|
||||
curl -H "Authorization: Bearer ${token}" -k -sSL -X GET https://${registry_server}/v2/${REPO_USER}/${IMAGE_NAME}/tags/list \
|
||||
| python -c '
|
||||
import sys, yaml, json, re
|
||||
y=yaml.load(sys.stdin.read())
|
||||
print max([float(v) for v in filter(lambda x: re.search(r"\d+\.\d+", x), y["tags"])])
|
||||
' 2>/dev/null
|
||||
}
|
||||
|
||||
#
|
||||
# Function to increment the image version above the current docker version
|
||||
#
|
||||
function increment_version {
|
||||
local CUR_VERSION=
|
||||
CUR_VERSION=$(get_current_version)
|
||||
|
||||
if [ -z "${CUR_VERSION}" ]; then
|
||||
echo "0.1"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "${CUR_VERSION}" | while IFS='.' read x y; do
|
||||
let -i y++
|
||||
echo "${x}.${y}"
|
||||
done
|
||||
}
|
||||
|
||||
OPTS=$(getopt -o h -l help,os:,push,user:,release: -- "$@")
|
||||
OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@")
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
OS=centos
|
||||
OS_RELEASE=pike
|
||||
PUSH=no
|
||||
REPO_USER=${USER}
|
||||
# List of top-level services for images, which should not be listed in upper-constraints.txt
|
||||
SKIP_CONSTRAINTS=(ceilometer cinder glance gnocchi heat horizon keystone neutron nova)
|
||||
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
while true; do
|
||||
|
|
@ -110,12 +77,20 @@ while true; do
|
|||
PUSH=yes
|
||||
shift
|
||||
;;
|
||||
--clean)
|
||||
CLEAN=yes
|
||||
shift
|
||||
;;
|
||||
--user)
|
||||
REPO_USER=$2
|
||||
DOCKER_USER=$2
|
||||
shift 2
|
||||
;;
|
||||
--release)
|
||||
OS_RELEASE=$2
|
||||
OPENSTACK_RELEASE=$2
|
||||
shift 2
|
||||
;;
|
||||
--version)
|
||||
VERSION=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
|
|
@ -144,19 +119,19 @@ if [ ${VALID_OS} -ne 0 ]; then
|
|||
fi
|
||||
|
||||
# Build the base wheels and retrieve the StarlingX wheels
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OS_RELEASE}
|
||||
${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running build-base-wheels.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
${MY_SCRIPT_DIR}/get-stx-wheels.sh --os ${OS} --release ${OS_RELEASE}
|
||||
${MY_SCRIPT_DIR}/get-stx-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failure running get-stx-wheels.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/tarball
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/tarball
|
||||
if [ -d ${BUILD_OUTPUT_PATH} ]; then
|
||||
# Wipe out the existing dir to ensure there are no stale files
|
||||
rm -rf ${BUILD_OUTPUT_PATH}
|
||||
|
|
@ -164,21 +139,21 @@ fi
|
|||
mkdir -p ${BUILD_OUTPUT_PATH}
|
||||
cd ${BUILD_OUTPUT_PATH}
|
||||
|
||||
IMAGE_NAME=stx-${OS}-${OS_RELEASE}-wheels
|
||||
IMAGE_NAME=stx-${OS}-${OPENSTACK_RELEASE}-wheels
|
||||
|
||||
TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/${IMAGE_NAME}.tar
|
||||
TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/${IMAGE_NAME}.tar
|
||||
if [ -f ${TARBALL_FNAME} ]; then
|
||||
rm -f ${TARBALL_FNAME}
|
||||
fi
|
||||
|
||||
# Download the global-requirements.txt and upper-constraints.txt files
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/global-requirements.txt
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/global-requirements.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download global-requirements.txt" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/upper-constraints.txt
|
||||
wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/upper-constraints.txt
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed to download upper-constraints.txt" >&2
|
||||
exit 1
|
||||
|
|
@ -246,35 +221,43 @@ if [ "${PUSH}" = "yes" ]; then
|
|||
#
|
||||
# Push generated wheels tarball to docker registry
|
||||
#
|
||||
VERSION=$(increment_version)
|
||||
|
||||
docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}
|
||||
docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed command:" >&2
|
||||
echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2
|
||||
echo "docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest
|
||||
docker tag ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed command:" >&2
|
||||
echo "docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest" >&2
|
||||
echo "docker tag ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker push $REPO_USER/${IMAGE_NAME}:${VERSION}
|
||||
docker push ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed command:" >&2
|
||||
echo "docker push $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2
|
||||
echo "docker push ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker push $REPO_USER/${IMAGE_NAME}:latest
|
||||
docker push ${DOCKER_USER}/${IMAGE_NAME}:latest
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed command:" >&2
|
||||
echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2
|
||||
echo "docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${CLEAN}" = "yes" ]; then
|
||||
echo "Deleting docker images ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest"
|
||||
docker image rm ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Failed command:" >&2
|
||||
echo "docker image rm ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
FROM centos:7.5.1804
|
||||
ARG RELEASE=7.5.1804
|
||||
FROM centos:${RELEASE}
|
||||
|
||||
ARG OS_RELEASE=pike
|
||||
ARG OPENSTACK_RELEASE=pike
|
||||
|
||||
# Install the necessary packages for building the python modules.
|
||||
# Some of these are dependencies of the specific modules, and could
|
||||
|
|
@ -14,5 +15,5 @@ RUN yum install -y epel-release centos-release-openstack-queens ;\
|
|||
pip install --upgrade pip setuptools
|
||||
|
||||
COPY docker-build-wheel.sh /
|
||||
COPY ${OS_RELEASE}-wheels.cfg /wheels.cfg
|
||||
COPY ${OPENSTACK_RELEASE}-wheels.cfg /wheels.cfg
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
SUPPORTED_OS_ARGS=('centos')
|
||||
OS=centos
|
||||
OS_RELEASE=pike
|
||||
OPENSTACK_RELEASE=pike
|
||||
|
||||
function usage {
|
||||
cat >&2 <<EOF
|
||||
|
|
@ -49,7 +50,7 @@ while true; do
|
|||
shift 2
|
||||
;;
|
||||
--release)
|
||||
OS_RELEASE=$2
|
||||
OPENSTACK_RELEASE=$2
|
||||
shift 2
|
||||
;;
|
||||
-h | --help )
|
||||
|
|
@ -64,7 +65,6 @@ while true; do
|
|||
done
|
||||
|
||||
# Validate the OS option
|
||||
SUPPORTED_OS_ARGS=('centos')
|
||||
VALID_OS=1
|
||||
for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
|
||||
if [ "$OS" = "${supported_os}" ]; then
|
||||
|
|
@ -81,7 +81,7 @@ fi
|
|||
source ${MY_REPO}/build-tools/git-utils.sh
|
||||
|
||||
function get_wheels_files {
|
||||
find ${GIT_LIST} -maxdepth 1 -name "${OS}_wheels.inc"
|
||||
find ${GIT_LIST} -maxdepth 1 -name "${OS}_${OPENSTACK_RELEASE}_wheels.inc"
|
||||
}
|
||||
|
||||
WHEELS_FILES=$(get_wheels_files)
|
||||
|
|
@ -90,7 +90,7 @@ if [ $(echo -n "$WHEELS_FILES" | wc -l) -eq 0 ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/stx
|
||||
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/stx
|
||||
if [ -d ${BUILD_OUTPUT_PATH} ]; then
|
||||
# Wipe out the existing dir to ensure there are no stale files
|
||||
rm -rf ${BUILD_OUTPUT_PATH}
|
||||
|
|
@ -103,7 +103,9 @@ declare -a FAILED
|
|||
for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES} | sort -u); do
|
||||
case $OS in
|
||||
centos)
|
||||
wheelfile=${MY_WORKSPACE}/std/rpmbuild/RPMS/${wheel}-[^-]*-[^-]*.rpm
|
||||
# Bash globbing does not handle [^\-] well,
|
||||
# so use grep instead
|
||||
wheelfile=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/${wheel}-* | grep -- '[^\-]*-[^\-]*.rpm')
|
||||
|
||||
if [ ! -f ${wheelfile} ]; then
|
||||
echo "Could not find ${wheelfile}" >&2
|
||||
|
|
|
|||
Loading…
Reference in New Issue