diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index c28d746c..ae5fba9a 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -21,6 +21,7 @@ OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike IMAGE_VERSION= PUSH=no +PROXY="" DOCKER_USER=${USER} DOCKER_REGISTRY= declare -a REPO_LIST @@ -44,6 +45,7 @@ Options: --repo: Software repository (Format: name,baseurl), can be specified multiple times --local: Use local build for software repository (cannot be used with --repo) --push: Push to docker repo + --proxy: Set proxy : --latest: Add a 'latest' tag when pushing --latest-tag: Use the provided tag when pushing latest. --user: Docker repo userid @@ -54,7 +56,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,version:,release:,repo:,push,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,version:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -97,6 +99,10 @@ while true; do PUSH=yes shift ;; + --proxy) + PROXY=$2 + shift 2 + ;; --latest) TAG_LATEST=yes shift @@ -217,10 +223,18 @@ BASE_IMAGE_PRESENT=$? IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION} IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG} -docker build \ - --build-arg RELEASE=${OS_VERSION} \ - --build-arg REPO_OPTS="${REPO_OPTS}" \ - --tag ${IMAGE_NAME} ${BUILDDIR} +local -a BUILD_ARGS +BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION}) +BUILD_ARGS+=(--build-arg REPO_OPTS=${REPO_OPTS}) + +# Add proxy to docker build +if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) +fi +BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR}) + +# Build base image +docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed running docker build command" >&2 exit 1 diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 309794eb..a2491f21 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -24,6 +24,7 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PREFIX=dev LATEST_PREFIX="" PUSH=no +PROXY="" DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= @@ -45,6 +46,7 @@ Options: --base: Specify base docker image (required option) --wheels: Specify path to wheels tarball or image, URL or docker tag (required option) --push: Push to docker repo + --proxy: Set proxy : --user: Docker repo userid --registry: Docker registry --prefix: Prefix on the image tag (default: dev) @@ -169,6 +171,9 @@ function build_image_loci { BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO}) BUILD_ARGS+=(--build-arg FROM=${BASE}) BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) + if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + fi if [ -n "${PROJECT_REF}" ]; then BUILD_ARGS+=(--build-arg PROJECT_REF=${PROJECT_REF}) @@ -300,9 +305,15 @@ function build_image_docker { local build_image_name="${USER}/${LABEL}:${IMAGE_TAG_BUILD}" - docker build ${docker_src} --no-cache \ - --build-arg "BASE=${BASE}" \ - --tag ${build_image_name} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log + local -a BASE_BUILD_ARGS + BASE_BUILD_ARGS+=(${docker_src} --no-cache) + BASE_BUILD_ARGS+=(--build-arg "BASE=${BASE}") + if [ ! -z "$PROXY" ]; then + BASE_BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + fi + BASE_BUILD_ARGS+=(--tag ${build_image_name}) + docker build ${BASE_BUILD_ARGS[@]} 2>&1 | tee ${WORKDIR}/docker-${LABEL}-${OS}-${OPENSTACK_RELEASE}.log + if [ ${PIPESTATUS[0]} -ne 0 ]; then echo "Failed to build ${LABEL}... Aborting" RESULTS_FAILED+=(${LABEL}) @@ -349,7 +360,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,push,proxy:,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -396,6 +407,10 @@ while true; do PUSH=yes shift ;; + --proxy) + PROXY=$2 + shift 2 + ;; --user) DOCKER_USER=$2 shift 2 diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 5fb9f892..20c6ad55 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -20,6 +20,7 @@ KEEP_CONTAINER=no OS=centos OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike +PROXY="" function usage { cat >&2 <: --release: Openstack release (default: pike) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,proxy: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -71,6 +73,10 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --proxy) + PROXY=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -134,7 +140,7 @@ fi # Check to see if we need to build anything BUILD_NEEDED=no for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do - if [ ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]; then + if [[ "${wheel}" =~ \* || ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]]; then BUILD_NEEDED=yes break fi @@ -189,10 +195,18 @@ docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q BASE_IMAGE_PRESENT=$? # Create the builder image -docker build \ - --build-arg RELEASE=${OS_VERSION} \ - --build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE} \ - -t ${BUILD_IMAGE_NAME} -f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH} +local -a BUILD_ARGS +BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION}) +BUILD_ARGS+=(--build-arg OPENSTACK_RELEASE=${OPENSTACK_RELEASE}) +if [ ! -z "$PROXY" ]; then + BUILD_ARGS+=(--build-arg http_proxy=$PROXY) + BUILD_ARGS+=(--build-arg https_proxy=$PROXY) +fi +BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) +BUILD_ARGS+=(-f ${DOCKER_PATH}/${OS}-dockerfile ${DOCKER_PATH}) + +# Build image +docker build "${BUILD_ARGS[@]}" if [ $? -ne 0 ]; then echo "Failed to create build image in docker" >&2 exit 1 @@ -203,7 +217,16 @@ RM_OPT= if [ "${KEEP_CONTAINER}" = "no" ]; then RM_OPT="--rm" fi -docker run ${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh + +local -a RUN_ARGS +if [ ! -z "$PROXY" ]; then + RUN_ARGS+=(--env http_proxy=$PROXY) + RUN_ARGS+=(--env https_proxy=$PROXY) +fi +RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh) + +# Run container to build wheels +docker run ${RUN_ARGS[@]} if [ "${KEEP_IMAGE}" = "no" ]; then # Delete the builder image diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 39dd3071..5857e592 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -21,6 +21,7 @@ OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PUSH=no +PROXY="" CLEAN=no DOCKER_USER=${USER} @@ -38,6 +39,7 @@ SKIP_CONSTRAINTS=( murano neutron nova + panko ) function usage { @@ -50,13 +52,14 @@ Options: --os-version: Specify OS version --release: Openstack release (default: pike) --push: Push to docker repo + --proxy: Set proxy : --user: Docker repo userid --version: Version for pushed image (if used with --push) EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,proxy:,version: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -95,6 +98,10 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --proxy) + PROXY=$2 + shift 2 + ;; --version) VERSION=$2 shift 2 @@ -125,7 +132,13 @@ if [ ${VALID_OS} -ne 0 ]; then fi # Build the base wheels and retrieve the StarlingX wheels -${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE} +local -a BUILD_BASE_WL_ARGS +BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE}) +if [ ! -z "$PROXY" ]; then + BUILD_BASE_WL_ARGS+=(--proxy ${PROXY}) +fi + +${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 diff --git a/build-tools/build-wheels/docker/master-wheels.cfg b/build-tools/build-wheels/docker/master-wheels.cfg index 07fc5463..b227dc7f 100644 --- a/build-tools/build-wheels/docker/master-wheels.cfg +++ b/build-tools/build-wheels/docker/master-wheels.cfg @@ -7,3 +7,8 @@ # If fix_setup must be called, add |fix_setup at the end of the line # lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 +panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0 +google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl +neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master +python_openstackclient-*.whl|git|https://github.com/donpenney/python-openstackclient|python-openstackclient|master +openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master diff --git a/build-tools/create_dependancy_cache.py b/build-tools/create_dependancy_cache.py index a7b6f453..8ab6192a 100755 --- a/build-tools/create_dependancy_cache.py +++ b/build-tools/create_dependancy_cache.py @@ -177,7 +177,6 @@ def file_search(dir, pattern, recursive_depth=0): # rpm_type= 'RPM' or 'SRPM' # arch= e.g. x86_64, only relevant of rpm_type=='RPM' def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list): - rpm_repo_roots = [] rpm_repodata_roots = [] repodata_list = [] @@ -206,7 +205,6 @@ def get_repo_primary_data_list(rpm_type='RPM', arch_list=default_arch_list): # rpm_type= 'RPM' or 'SRPM' # arch= e.g. x86_64, only relevant of rpm_type=='RPM' def get_repo_filelists_data_list(rpm_type='RPM', arch_list=default_arch_list): - rpm_repo_roots = [] rpm_repodata_roots = [] repodata_list = []