diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 79e79710..bf556a61 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -15,11 +15,11 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then fi DOCKER_PATH=${MY_REPO}/build-tools/build-wheels/docker -WHEELS_CFG=${DOCKER_PATH}/wheels.cfg KEEP_IMAGE=no KEEP_CONTAINER=no OS=centos -OS_RELEASE=pike +OS_VERSION=7.5.1804 +OPENSTACK_RELEASE=pike function usage { cat >&2 < ] [ --keep-image ] [ --keep-container ] [ --release < Options: --os: Specify base OS (eg. centos) + --os-version: Specify OS version --keep-image: Skip deletion of the wheel build image in docker --keep-container: Skip deletion of container used for the build --release: Openstack release (default: pike) @@ -54,6 +55,10 @@ while true; do OS=$2 shift 2 ;; + --os-version) + OS_VERSION=$2 + shift 2 + ;; --keep-image) KEEP_IMAGE=yes shift @@ -63,7 +68,7 @@ while true; do shift ;; --release) - OS_RELEASE=$2 + OPENSTACK_RELEASE=$2 shift 2 ;; -h | --help ) @@ -77,10 +82,11 @@ while true; do esac done -BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/base -BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OS_RELEASE}" +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base +BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}" DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile +WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg function supported_os_list { for f in ${DOCKER_PATH}/*-dockerfile; do @@ -115,8 +121,29 @@ else fi fi +# Check to see if we need to build anything +BUILD_NEEDED=no +for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do + if [ ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]; then + BUILD_NEEDED=yes + break + fi +done + +if [ "${BUILD_NEEDED}" = "no" ]; then + echo "All base wheels are already present. Skipping build." + exit 0 +fi + +# Check to see if the OS image is already pulled +docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$" +BASE_IMAGE_PRESENT=$? + # Create the builder image -docker build --build-arg OS_RELEASE=${OS_RELEASE} -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH} +docker build \ + --build-arg RELEASE=${OS_VERSION} \ + --build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE} \ + -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH} if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -131,11 +158,20 @@ docker run ${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels -i -t ${BUILD_IMAGE_NAME} / if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image - echo "Removing docker build image ${BUILD_IMAGE_NAME}" + echo "Removing docker image ${BUILD_IMAGE_NAME}" docker image rm ${BUILD_IMAGE_NAME} if [ $? -ne 0 ]; then echo "Failed to delete build image from docker" >&2 fi + + if [ ${BASE_IMAGE_PRESENT} -ne 0 ]; then + # The base image was not already present, so delete it + echo "Removing docker image ${OS}:${OS_VERSION}" + docker image rm ${OS}:${OS_VERSION} + if [ $? -ne 0 ]; then + echo "Failed to delete base image from docker" >&2 + fi + fi fi # Check for failures diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index b2d184a6..a134e31f 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -16,83 +16,50 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then fi SUPPORTED_OS_ARGS=('centos') +OS=centos +OPENSTACK_RELEASE=pike +VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp +PUSH=no +CLEAN=no +DOCKER_USER=${USER} + +# List of top-level services for images, which should not be listed in upper-constraints.txt +SKIP_CONSTRAINTS=( + ceilometer + cinder + glance + gnocchi + heat + horizon + ironic + keystone + magnum + murano + neutron + nova +) function usage { cat >&2 < ] [ --push ] [ --user ] [ --release ] +$(basename $0) Options: --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --release: Openstack release (default: pike) --push: Push to docker repo --user: Docker repo userid - --release: Openstack release (default: pike) + --version: Version for pushed image (if used with --push) EOF } -# -# Function to get an auth token from docker -# -function get_docker_token { - local auth_server="auth.docker.io" - local registry="registry.docker.io" - local scope="repository:${REPO_USER}/${IMAGE_NAME}:pull" - curl -k -sSL -X GET "https://${auth_server}/token?service=${registry}&scope=${scope}" \ - | python -c ' -import sys, yaml, json -y=yaml.load(sys.stdin.read()) -print y["token"] -' 2>/dev/null -} - -# -# Function to query docker for the highest current version -# -function get_current_version { - local registry_server="registry-1.docker.io" - local token= - token=$(get_docker_token) - - curl -H "Authorization: Bearer ${token}" -k -sSL -X GET https://${registry_server}/v2/${REPO_USER}/${IMAGE_NAME}/tags/list \ - | python -c ' -import sys, yaml, json, re -y=yaml.load(sys.stdin.read()) -print max([float(v) for v in filter(lambda x: re.search(r"\d+\.\d+", x), y["tags"])]) -' 2>/dev/null -} - -# -# Function to increment the image version above the current docker version -# -function increment_version { - local CUR_VERSION= - CUR_VERSION=$(get_current_version) - - if [ -z "${CUR_VERSION}" ]; then - echo "0.1" - return - fi - - echo "${CUR_VERSION}" | while IFS='.' read x y; do - let -i y++ - echo "${x}.${y}" - done -} - -OPTS=$(getopt -o h -l help,os:,push,user:,release: -- "$@") +OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@") if [ $? -ne 0 ]; then usage exit 1 fi -OS=centos -OS_RELEASE=pike -PUSH=no -REPO_USER=${USER} -# List of top-level services for images, which should not be listed in upper-constraints.txt -SKIP_CONSTRAINTS=(ceilometer cinder glance gnocchi heat horizon keystone neutron nova) - eval set -- "${OPTS}" while true; do @@ -110,12 +77,20 @@ while true; do PUSH=yes shift ;; + --clean) + CLEAN=yes + shift + ;; --user) - REPO_USER=$2 + DOCKER_USER=$2 shift 2 ;; --release) - OS_RELEASE=$2 + OPENSTACK_RELEASE=$2 + shift 2 + ;; + --version) + VERSION=$2 shift 2 ;; -h | --help ) @@ -144,19 +119,19 @@ if [ ${VALID_OS} -ne 0 ]; then fi # Build the base wheels and retrieve the StarlingX wheels -${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OS_RELEASE} +${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 fi -${MY_SCRIPT_DIR}/get-stx-wheels.sh --os ${OS} --release ${OS_RELEASE} +${MY_SCRIPT_DIR}/get-stx-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE} if [ $? -ne 0 ]; then echo "Failure running get-stx-wheels.sh" >&2 exit 1 fi -BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/tarball +BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/tarball if [ -d ${BUILD_OUTPUT_PATH} ]; then # Wipe out the existing dir to ensure there are no stale files rm -rf ${BUILD_OUTPUT_PATH} @@ -164,21 +139,21 @@ fi mkdir -p ${BUILD_OUTPUT_PATH} cd ${BUILD_OUTPUT_PATH} -IMAGE_NAME=stx-${OS}-${OS_RELEASE}-wheels +IMAGE_NAME=stx-${OS}-${OPENSTACK_RELEASE}-wheels -TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${OS_RELEASE}/${IMAGE_NAME}.tar +TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/${IMAGE_NAME}.tar if [ -f ${TARBALL_FNAME} ]; then rm -f ${TARBALL_FNAME} fi # Download the global-requirements.txt and upper-constraints.txt files -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/global-requirements.txt +wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/global-requirements.txt if [ $? -ne 0 ]; then echo "Failed to download global-requirements.txt" >&2 exit 1 fi -wget https://raw.githubusercontent.com/openstack/requirements/stable/${OS_RELEASE}/upper-constraints.txt +wget https://raw.githubusercontent.com/openstack/requirements/stable/${OPENSTACK_RELEASE}/upper-constraints.txt if [ $? -ne 0 ]; then echo "Failed to download upper-constraints.txt" >&2 exit 1 @@ -246,35 +221,43 @@ if [ "${PUSH}" = "yes" ]; then # # Push generated wheels tarball to docker registry # - VERSION=$(increment_version) - - docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION} + docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} if [ $? -ne 0 ]; then echo "Failed command:" >&2 - echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + echo "docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2 exit 1 fi - docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest + docker tag ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest if [ $? -ne 0 ]; then echo "Failed command:" >&2 - echo "docker tag $REPO_USER/${IMAGE_NAME}:${VERSION} $REPO_USER/${IMAGE_NAME}:latest" >&2 + echo "docker tag ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest" >&2 exit 1 fi - docker push $REPO_USER/${IMAGE_NAME}:${VERSION} + docker push ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} if [ $? -ne 0 ]; then echo "Failed command:" >&2 - echo "docker push $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + echo "docker push ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2 exit 1 fi - docker push $REPO_USER/${IMAGE_NAME}:latest + docker push ${DOCKER_USER}/${IMAGE_NAME}:latest if [ $? -ne 0 ]; then echo "Failed command:" >&2 - echo "docker import ${TARBALL_FNAME} $REPO_USER/${IMAGE_NAME}:${VERSION}" >&2 + echo "docker import ${TARBALL_FNAME} ${DOCKER_USER}/${IMAGE_NAME}:${VERSION}" >&2 exit 1 fi + + if [ "${CLEAN}" = "yes" ]; then + echo "Deleting docker images ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest" + docker image rm ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest + if [ $? -ne 0 ]; then + echo "Failed command:" >&2 + echo "docker image rm ${DOCKER_USER}/${IMAGE_NAME}:${VERSION} ${DOCKER_USER}/${IMAGE_NAME}:latest" >&2 + exit 1 + fi + fi fi exit 0 diff --git a/build-tools/build-wheels/docker/centos-dockerfile b/build-tools/build-wheels/docker/centos-dockerfile index e7e8d7c6..f5a48449 100644 --- a/build-tools/build-wheels/docker/centos-dockerfile +++ b/build-tools/build-wheels/docker/centos-dockerfile @@ -1,6 +1,7 @@ -FROM centos:7.5.1804 +ARG RELEASE=7.5.1804 +FROM centos:${RELEASE} -ARG OS_RELEASE=pike +ARG OPENSTACK_RELEASE=pike # Install the necessary packages for building the python modules. # Some of these are dependencies of the specific modules, and could @@ -14,5 +15,5 @@ RUN yum install -y epel-release centos-release-openstack-queens ;\ pip install --upgrade pip setuptools COPY docker-build-wheel.sh / -COPY ${OS_RELEASE}-wheels.cfg /wheels.cfg +COPY ${OPENSTACK_RELEASE}-wheels.cfg /wheels.cfg diff --git a/build-tools/build-wheels/get-stx-wheels.sh b/build-tools/build-wheels/get-stx-wheels.sh index 3447e8e6..f5f553c6 100755 --- a/build-tools/build-wheels/get-stx-wheels.sh +++ b/build-tools/build-wheels/get-stx-wheels.sh @@ -14,8 +14,9 @@ if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then exit 1 fi +SUPPORTED_OS_ARGS=('centos') OS=centos -OS_RELEASE=pike +OPENSTACK_RELEASE=pike function usage { cat >&2 <&2