From 87cbff88228aec768a8dfa33f06397c21a6f25e6 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Wed, 23 Jan 2019 11:43:19 -0500 Subject: [PATCH 1/5] Add panko to master wheels tarball The stx-ceilometer master image requires a specific version of panko, which does not have a pre-built wheel available in pypi, so we're adding it to the master wheels tarball to make it available to the image build. Change-Id: I854b75577b6dbc3f0a8171190f5a1aa839412dc8 Story: 2004520 Task: 29046 Signed-off-by: Don Penney --- build-tools/build-wheels/build-wheel-tarball.sh | 1 + build-tools/build-wheels/docker/master-wheels.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 39dd3071..736eee72 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -38,6 +38,7 @@ SKIP_CONSTRAINTS=( murano neutron nova + panko ) function usage { diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg index 07fc5463..6c239102 100644 --- a/build-tools/build-wheels/docker/master-wheels.cfg +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -7,3 +7,4 @@ # If fix_setup must be called, add |fix_setup at the end of the line # lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 +panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 From d41eff31eae1ed107609cedb8132a99602511ad6 Mon Sep 17 00:00:00 2001 From: Felipe de Jesus Ruiz Garcia Date: Mon, 21 Jan 2019 16:33:37 -0500 Subject: [PATCH 2/5] Remove unused variables Partial cleanup of the python code. Story: 2004825 Task: 29286 Change-Id: I146c626342a8be2c93da21cf6cb229a39e8eac10 Signed-off-by: Felipe de Jesus Ruiz Garcia --- build-tools/create_dependancy_cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools/create_dependancy_cache.py b/build-tools/create_dependancy_cache.py index a7b6f453..8ab6192a 100755 --- a/build-tools/create_dependancy_cache.py +++ b/build-tools/create_dependancy_cache.py @@ -177,7 +177,6 @@ def file_search(dir, pattern, recursive_depth=0): # rpm_type= 'RPM' or 'SRPM' # arch= e.g. x86_64, only relevant of rpm_type=='RPM' def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list): - rpm_repo_roots = [] rpm_repodata_roots = [] repodata_list = [] @@ -206,7 +205,6 @@ def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list): # rpm_type= 'RPM' or 'SRPM' # arch= e.g. x86_64, only relevant of rpm_type=='RPM' def get_repo_filelists_data_list(rpm_type='RPM', arch_list=default_arch_list): - rpm_repo_roots = [] rpm_repodata_roots = [] repodata_list = [] From d98601c46a7dcad1e7f914675482bdd490178902 Mon Sep 17 00:00:00 2001 From: Mario Alfredo Carrillo Arevalo Date: Sat, 2 Feb 2019 03:01:36 +0000 Subject: [PATCH 3/5] Add support to set proxy as argument When the building scripts involved in the containerization processes are executed behind a proxy, the user must set the proxy manually making modifications in the scripts or dockerfiles. These changes allows to set it as an argument. Story: 2004925 Task: 29305 Change-Id: I52e13e4832daf488d8ff29334686a20daa7d2826 Signed-off-by: Mario Alfredo Carrillo Arevalo --- .../build-docker-images/build-stx-base.sh | 24 ++++++++++--- .../build-docker-images/build-stx-images.sh | 23 +++++++++--- build-tools/build-wheels/build-base-wheels.sh | 35 +++++++++++++++---- .../build-wheels/build-wheel-tarball.sh | 16 +++++++-- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index c28d746c..ae5fba9a 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -21,6 +21,7 @@ OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike IMAGE_VERSION= PUSH=no +PROXY="" DOCKER_USER=${USER} DOCKER_REGISTRY= declare -a REPO_LIST @@ -44,6 +45,7 @@ Options: --repo: Software repository (Format: name,baseurl), can be specified multiple times --local: Use local build for software repository (cannot be used with --repo) --push: Push to docker repo + --proxy: Set proxy : --latest: Add a 'latest' tag when pushing --latest-tag: Use the provided tag when pushing latest. --user: Docker repo userid @@ -54,7 +56,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,version:,release:,repo:,push,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,version:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -97,6 +99,10 @@ while true; do PUSH=yes shift ;; + --proxy) + PROXY=$2 + shift 2 + ;; --latest) TAG_LATEST=yes shift @@ -217,10 +223,18 @@ BASE_IMAGE_PRESENT=$? IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION} IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG} -docker build \ - --build-arg RELEASE=${OS_VERSION} \ - --build-arg REPO_OPTS="${REPO_OPTS}" \ - --tag ${IMAGE_NAME} ${BUILDDIR} +local -a BUILD_ARGS +BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION}) +BUILD_ARGS+=(--build-arg REPO_OPTS=${REPO_OPTS}) + +# Add proxy to docker build +if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) +fi +BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR}) + +# Build base image +docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed running docker build command" >&2 exit 1 diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 309794eb..a2491f21 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -24,6 +24,7 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PREFIX=dev LATEST_PREFIX="" PUSH=no +PROXY="" DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= @@ -45,6 +46,7 @@ Options: --base: Specify base docker image (required option) --wheels: Specify path to wheels tarball or image, URL or docker tag (required option) --push: Push to docker repo + --proxy: Set proxy : --user: Docker repo userid --registry: Docker registry --prefix: Prefix on the image tag (default: dev) @@ -169,6 +171,9 @@ function build_image_loci { BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO}) BUILD_ARGS+=(--build-arg FROM=${BASE}) BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) + if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + fi if [ -n "${PROJECT_REF}" ]; then BUILD_ARGS+=(--build-arg PROJECT_REF=${PROJECT_REF}) @@ -300,9 +305,15 @@ function build_image_docker { local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - docker build ${docker_src} --no-cache \ - --build-arg "BASE=${BASE}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log + local -a BASE_BUILD_ARGS + BASE_BUILD_ARGS+=(${docker_src} --no-cache) + BASE_BUILD_ARGS+=(--build-arg "BASE=${BASE}") + if [ ! -z "$PROXY" ]; then + BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + fi + BASE_BUILD_ARGS+=(--tag ${build_image_name}) + docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log + if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) @@ -349,7 +360,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,push,proxy:,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -396,6 +407,10 @@ while true; do PUSH=yes shift ;; + --proxy) + PROXY=$2 + shift 2 + ;; --user) DOCKER_USER=$2 shift 2 diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 5fb9f892..a72c0326 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -20,6 +20,7 @@ KEEP_CONTAINER=no OS=centos OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike +PROXY="" function usage { cat >&2 <: --release: Openstack release (default: pike) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,proxy: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -71,6 +73,10 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --proxy) + PROXY=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -189,10 +195,18 @@ docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q BASE_IMAGE_PRESENT=$? # Create the builder image -docker build \ - --build-arg RELEASE=${OS_VERSION} \ - --build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE} \ - -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH} +local -a BUILD_ARGS +BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION}) +BUILD_ARGS+=(--build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE}) +if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + BUILD_ARGS+=(--build-arg https_proxy=$PROXY) +fi +BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) +BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}) + +# Build image +docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -203,7 +217,16 @@ RM_OPT= if [ "${KEEP_CONTAINER}" = "no" ]; then RM_OPT="--rm" fi -docker run ${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh + +local -a RUN_ARGS +if [ ! -z "$PROXY" ]; then + RUN_ARGS+=(--env http_proxy=$PROXY) + RUN_ARGS+=(--env https_proxy=$PROXY) +fi +RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) + +# Run container to build wheels +docker run ${RUN_ARGS[@]} if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 736eee72..5857e592 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -21,6 +21,7 @@ OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PUSH=no +PROXY="" CLEAN=no DOCKER_USER=${USER} @@ -51,13 +52,14 @@ Options: --os-version: Specify OS version --release: Openstack release (default: pike) --push: Push to docker repo + --proxy: Set proxy : --user: Docker repo userid --version: Version for pushed image (if used with --push) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,proxy:,version: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -96,6 +98,10 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --proxy) + PROXY=$2 + shift 2 + ;; --version) VERSION=$2 shift 2 @@ -126,7 +132,13 @@ if [ ${VALID_OS} -ne 0 ]; then fi # Build the base wheels and retrieve the StarlingX wheels -${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE} +local -a BUILD_BASE_WL_ARGS +BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE}) +if [ ! -z "$PROXY" ]; then + BUILD_BASE_WL_ARGS+=(--proxy ${PROXY}) +fi + +${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 From 88494909715fafe35250eb344c7b633157d87b86 Mon Sep 17 00:00:00 2001 From: Al Bailey Date: Tue, 5 Feb 2019 16:36:02 -0600 Subject: [PATCH 4/5] Fix docker image build failures due to google-api-python-client This enforces google-api-python-client 1.7.7 which is available on pypi site as a wheel that is both python2 and python3. This affected only master docker images, pike was not impacted Closes-Bug: 1814835 Change-Id: I1bae1a5a3a1b0fcd323d072a113381c3340f9134 Signed-off-by: Al Bailey --- build-tools/build-wheels/docker/master-wheels.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg index 6c239102..bfdf4a9d 100644 --- a/build-tools/build-wheels/docker/master-wheels.cfg +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -8,3 +8,4 @@ # lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 +google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl From 1f56c5036eb1a32e95fed2c03efbd389138f9abd Mon Sep 17 00:00:00 2001 From: Don Penney Date: Thu, 7 Feb 2019 10:52:11 -0500 Subject: [PATCH 5/5] Add support for building wheels from master branches This update enhances build-base-wheels.sh to support using a wildcard in wheels.cfg for the wheel name, to allow for whatever the version is when building from master for a given repo. This also updates master-wheels.cfg to build the following modules from master, to pick up the latest code for these supporting modules: neutron-lib python-openstackclient openstacksdk Story: 2004751 Task: 29395 Change-Id: Ifab8a209170a00f58e0f4a53b6cbd7e4fe86e8f0 Signed-off-by: Don Penney --- build-tools/build-wheels/build-base-wheels.sh | 2 +- build-tools/build-wheels/docker/master-wheels.cfg | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index a72c0326..20c6ad55 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -140,7 +140,7 @@ fi # Check to see if we need to build anything BUILD_NEEDED=no for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do - if [ ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]; then + if [[ "${wheel}" =~ \* || ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]]; then BUILD_NEEDED=yes break fi diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg index bfdf4a9d..b227dc7f 100644 --- a/build-tools/build-wheels/docker/master-wheels.cfg +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -9,3 +9,6 @@ lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl +neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master +python_openstackclient-*.whl|git|https://github.com/donpenney/python-openstackclient|python-openstackclient|master +openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master