Merge remote-tracking branch 'origin/master' into f/centos8

Signed-off-by: Charles Short <charles.short@windriver.com>
Change-Id: I6b79402dd076ea2d4cd3ba52ae66926fcb29f759

build-tools/branching/create_branches_and_tags.sh

@@ -24,7 +24,7 @@ CREATE_BRANCHES_AND_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )"
 source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh"
 
 usage () {
-    echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest [ --lock-down ]]"
+    echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --gitreview-default ] [ --manifest [ --lock-down  | --soft-lock-down ] [ --default-revision ]]"
     echo ""
     echo "Create a branch and a tag in all listed projects, and all"
     echo "projects hosted by all listed remotes.  Lists are comma separated."
@@ -33,13 +33,19 @@ usage () {
     echo "If the tag is omitted, one is automativally generate by adding the"
     echo "prefix 'v' to the branch name."
     echo ""
-    echo "If a manifest is requested, it will recieve the name '<branch>.xml' and"
-    echo "it will specify the branch as the revision for all tagged projects."
+    echo "If a manifest is requested, the current manifest is modified."
+    echo "to specify the new branch for all select remotes and projects."
     echo "If lockdown is requested, all other projects get the current"
     echo "HEAD's sha set as the revision."
+    echo "If default-revision is selected, then the manifest default revision"
+    wcho "will be set."
+    echo ""
+    echo "If a gitreview-default is selected, then all branched projects"
+    echo "with a .gitreview file will have a defaultbranch entry added"
+    echo "or updated."
 }
 
-TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,help -n 'create_branches_and_tags.sh' -- "$@")
+TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,help -n 'create_branches_and_tags.sh' -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -49,6 +55,8 @@ eval set -- "$TEMP"
 HELP=0
 MANIFEST=0
 LOCK_DOWN=0
+GITREVIEW_DEFAULT=0
+SET_DEFAULT_REVISION=0
 remotes=""
 projects=""
 branch=""
@@ -59,15 +67,19 @@ repo_root_dir=""
 
 while true ; do
     case "$1" in
-        -h|--help)         HELP=1 ; shift ;;
-        --remotes)         remotes+=$(echo "$2 " | tr ',' ' '); shift 2;;
-        --projects)        projects+=$(echo "$2 " | tr ',' ' '); shift 2;;
-        --branch)          branch=$2; shift 2;;
-        --tag)             tag=$2; shift 2;;
-        --manifest)        MANIFEST=1 ; shift ;;
-        --lock-down)       LOCK_DOWN=1 ; shift ;;
-        --)                shift ; break ;;
-        *)                 usage; exit 1 ;;
+        -h|--help)           HELP=1 ; shift ;;
+        --remotes)           remotes+=$(echo "$2 " | tr ',' ' '); shift 2;;
+        --projects)          projects+=$(echo "$2 " | tr ',' ' '); shift 2;;
+        --branch)            branch=$2; shift 2;;
+        --tag)               tag=$2; shift 2;;
+        --manifest)          MANIFEST=1 ; shift ;;
+        --lock-down)         LOCK_DOWN=2 ; shift ;;
+        --hard-lock-down)    LOCK_DOWN=2 ; shift ;;
+        --soft-lock-down)    LOCK_DOWN=1 ; shift ;;
+        --default-revision)  SET_DEFAULT_REVISION=1 ; shift ;;
+        --gitreview-default) GITREVIEW_DEFAULT=1 ; shift ;;
+        --)                  shift ; break ;;
+        *)                   usage; exit 1 ;;
     esac
 done
 
@@ -88,6 +100,37 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
+update_gitreview () {
+    local DIR=$1
+    (
+    cd $DIR || exit 1
+    if [ $GITREVIEW_DEFAULT -eq 1 ] && [ -f .gitreview ]; then
+        if ! grep -q "^defaultbranch=$branch$" .gitreview; then
+            echo "Updating defaultbranch in ${DIR}/.gitreview"
+            if grep -q defaultbranch= .gitreview; then
+                sed "s#\(defaultbranch=\).*#\1$branch#" -i .gitreview
+            else
+                echo "defaultbranch=$branch" >>  .gitreview
+            fi
+
+            git add .gitreview
+            if [ $? != 0 ] ; then
+                echo_stderr "ERROR: failed to add .gitreview in ${DIR}"
+                exit 1
+            fi
+
+            git commit -s -m "Update .gitreview for $branch"
+            if [ $? != 0 ] ; then
+                echo_stderr "ERROR: failed to commit .gitreview in ${DIR}"
+                exit 1
+            fi
+        else
+            echo "defaultbranch in ${DIR}/.gitreview already set"
+        fi
+    fi
+    )
+}
+
 if [ $MANIFEST -eq 1 ]; then
     manifest=$(repo_manifest $repo_root_dir)
     if [ $? -ne 0 ]; then
@@ -205,6 +248,7 @@ for subgit in $SUBGITS; do
         git checkout $branch
     fi
 
+    # check if destination tag already exists
     tag_check=$(git tag -l $tag)
     if [ -z "$tag_check" ]; then
         echo "Creating tag $tag in ${subgit}"
@@ -216,6 +260,8 @@ for subgit in $SUBGITS; do
     else
         echo "Tag '$tag' already exists in ${subgit}"
     fi
+
+    update_gitreview ${subgit} || exit 1
     ) || exit 1
 done
 ) || exit 1
@@ -276,8 +322,10 @@ if [ $MANIFEST -eq 1 ]; then
         exit 1
     fi
 
+    update_gitreview ${manifest_dir} || exit 1
+
     echo "Creating manifest ${new_manifest_name}"
-    manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} $projects || exit 1
+    manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} ${SET_DEFAULT_REVISION} $projects || exit 1
 
     echo "Move manifest ${new_manifest_name}, overwriting ${manifest_name}"
     \cp -f "${manifest}" "${manifest}.save"

build-tools/branching/push_branches_tags.sh

@@ -17,6 +17,7 @@
 PUSH_BRANCHES_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 
 source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh"
+source "${PUSH_BRANCHES_TAGS_SH_DIR}/../url_utils.sh"
 
 usage () {
     echo "push_branches_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest ]"
@@ -151,6 +152,8 @@ for subgit in $SUBGITS; do
     (
     cd $subgit
 
+    git fetch --all
+
     branch_check=$(git branch -a --list $branch)
     if [ -z "$branch_check" ]; then
         echo_stderr "ERROR: Expected branch '$branch' to exist in ${subgit}"
@@ -169,21 +172,49 @@ for subgit in $SUBGITS; do
         exit 1
     fi
 
-    if [ "${review_method}" == "gerrit" ]; then
-        remote=$(git_repo_review_remote)
-    else
-        remote=$(git_repo_remote)
+    remote=$(git_remote)
+    if [ "${remote}" == "" ]; then
+        echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}"
+        exit 1
     fi
 
-    if [ "${remote}" == "" ]; then
-        echo_stderr "ERROR: Failed to determine remote in ${subgit}"
+    if [ "${review_method}" == "gerrit" ]; then
+        review_remote=$(git_repo_review_remote)
+    else
+        review_remote=${remote}
+    fi
+
+    if [ "${review_remote}" == "" ]; then
+        echo_stderr "ERROR: Failed to determine review_remote in ${subgit}"
         exit 1
     fi
 
+    branch_check=$(git branch -a --list $remote/$branch)
+    if [ "${branch_check}" != "" ]; then
+        echo "Branch $branch already exists in ${subgit}"
+        exit 0
+    fi
+
     echo "Pushing branch $branch in ${subgit}"
     if [ "${review_method}" == "gerrit" ]; then
-        echo "git push --tags ${remote} ${branch}"
-        git push --tags ${remote} ${branch}
+        url=$(git_repo_review_url)
+        if [ "${review_remote}" == "" ]; then
+            echo_stderr "ERROR: Failed to determine review_url in ${subgit}"
+            exit 1
+        fi
+
+        host=$(url_server "${url}")
+        port=$(url_port "${url}")
+        path=$(url_path "${url}")
+        if [ "${host}" == "review.opendev.org" ]; then
+            git push ${review_remote} ${tag} && \
+            ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
+            git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
+            git review --topic="${branch}"
+        else
+            echo "git push --tags ${remote} ${branch}"
+            git push --tags ${remote} ${branch}
+        fi
     else
         echo "git push --tags --set-upstream ${remote} ${branch}"
         git push --tags --set-upstream ${remote} ${branch}
@@ -232,23 +263,44 @@ if [ $MANIFEST -eq 1 ]; then
         exit 1
     fi
 
-
-    remote=$(git_review_remote)
+    remote=$(git_remote)
     if [ "${remote}" == "" ]; then
         echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}"
         exit 1
     fi
 
+    review_remote=$(git_review_remote)
+    if [ "${review_remote}" == "" ]; then
+        echo_stderr "ERROR: Failed to determine review_remote in ${manifest_dir}"
+        exit 1
+    fi
+
     echo "Pushing branch $branch in ${manifest_dir}"
     if [ "${review_method}" == "gerrit" ]; then
         # Is a reviewless push possible as part of creating a new branch in gerrit?
-        git push --tags ${remote} ${branch}
+        url=$(git_review_url)
+        if [ "${review_remote}" == "" ]; then
+            echo_stderr "ERROR: Failed to determine review_url in ${subgit}"
+            exit 1
+        fi
+
+        host=$(url_server "${url}")
+        port=$(url_port "${url}")
+        path=$(url_path "${url}")
+        if [ "${host}" == "review.opendev.org" ]; then
+            git push ${review_remote} ${tag} && \
+            ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
+            git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
+            git review --yes --topic="${branch}"
+        else
+            git push --tags ${review_remote} ${branch}
+        fi
     else
-        git push --tags --set-upstream ${remote} ${branch}
+        git push --tags --set-upstream ${review_remote} ${branch}
     fi
 
     if [ $? != 0 ] ; then
-        echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${remote}' in  ${manifest_dir}"
+        echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${review_remote}' in  ${manifest_dir}"
         exit 1
     fi
     ) || exit 1

build-tools/build-docker-images/build-stx-images.sh

@@ -26,7 +26,6 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PREFIX=dev
 LATEST_PREFIX=""
 PUSH=no
-CONFIG_FILE=""
 HTTP_PROXY=""
 HTTPS_PROXY=""
 NO_PROXY=""
@@ -34,16 +33,13 @@ DOCKER_USER=${USER}
 DOCKER_REGISTRY=
 BASE=
 WHEELS=
-WHEELS_ALTERNATE=
-DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images"
-DEFAULT_CONFIG_FILE_PREFIX="docker-image-build"
+WHEELS_PY2=
 CLEAN=no
 TAG_LATEST=no
 TAG_LIST_FILE=
 TAG_LIST_LATEST_FILE=
 declare -a ONLY
 declare -a SKIP
-declare -a SERVICES_ALTERNATE
 declare -i MAX_ATTEMPTS=1
 
 function usage {
@@ -52,30 +48,32 @@ Usage:
 $(basename $0)
 
 Options:
-    --os:         Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
-    --version:    Specify version for output image
-    --stream:     Build stream, stable or dev (default: stable)
-    --base:       Specify base docker image (required option)
-    --wheels:     Specify path to wheels tarball or image, URL or docker tag (required option)
-    --wheels-alternate: Specify path to alternate wheels tarball or image, URL or docker tag
-    --push:       Push to docker repo
-    --http_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
-    --https_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
-    --no_proxy: Set proxy <URL>, urls splitted with ","
-    --user:       Docker repo userid
-    --registry:   Docker registry
-    --prefix:     Prefix on the image tag (default: dev)
-    --latest:     Add a 'latest' tag when pushing
+    --os:            Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
+    --version:       Specify version for output image
+    --stream:        Build stream, stable or dev (default: stable)
+    --base:          Specify base docker image (required option)
+    --wheels:        Specify path to wheels tarball or image, URL or docker tag
+                     (required when building loci projects)
+    --wheels-py2:    Use this wheels tarball for Python2 projects
+                     (default: work out from --wheels)
+    --wheels-alternate: same as --wheels-py2
+    --push:          Push to docker repo
+    --http_proxy:    Set proxy <URL>:<PORT>, urls splitted with ","
+    --https_proxy:   Set proxy <URL>:<PORT>, urls splitted with ","
+    --no_proxy:      Set proxy <URL>, urls splitted with ","
+    --user:          Docker repo userid
+    --registry:      Docker registry
+    --prefix:        Prefix on the image tag (default: dev)
+    --latest:        Add a 'latest' tag when pushing
     --latest-prefix: Alternative prefix on the latest image tag
-    --clean:      Remove image(s) from local registry
+    --clean:         Remove image(s) from local registry
     --only <image> : Only build the specified image(s). Multiple images
                      can be specified with a comma-separated list, or with
                      multiple --only arguments.
     --skip <image> : Skip building the specified image(s). Multiple images
                      can be specified with a comma-separated list, or with
                      multiple --skip arguments.
-    --attempts:   Max attempts, in case of failure (default: 1)
-    --config-file:Specify a path to a config file which will specify additional arguments to be passed into the the command
+    --attempts:      Max attempts, in case of failure (default: 1)
 
 
 EOF
@@ -97,63 +95,6 @@ function is_empty {
     test $# -eq 0
 }
 
-function get_args_from_file {
-    # get additional build args from specified file.
-    local -a config_items
-
-    echo "Get args from file: $1"
-    for i in $(cat $1)
-    do
-        config_items=($(echo $i | sed s/=/\ /g))
-        echo "--${config_items[0]} ${config_items[1]}"
-        case ${config_items[0]} in
-            base)
-                if [ -z "${BASE}" ]; then
-                    BASE=${config_items[1]}
-                fi
-                ;;
-            user)
-                if [ -z "${DOCKER_USER}" ]; then
-                    DOCKER_USER=${config_items[1]}
-                fi
-                ;;
-            proxy)
-                if [ -z "${PROXY}" ]; then
-                    PROXY=${config_items[1]}
-                fi
-                ;;
-            registry)
-                if [ -z "${DOCKER_REGISTRY}" ]; then
-                    # Add a trailing / if needed
-                    DOCKER_REGISTRY="${config_items[1]%/}/"
-                fi
-                ;;
-            only)
-                # Read comma-separated values into array
-                if [ -z "${ONLY}" ]; then
-                    # Read comma-separated values into array
-                    ONLY=(`echo ${config_items[1]} | sed s/,/\ /g`)
-                fi
-                ;;
-            wheels)
-                if [ -z "${WHEELS}" ]; then
-                    WHEELS=${config_items[1]}
-                fi
-                ;;
-            wheels_alternate)
-                if [ -z "${WHEELS_ALTERNATE}" ]; then
-                    WHEELS_ALTERNATE=${config_items[1]}
-                    echo "WHEELS_ALTERNATE: ${WHEELS_ALTERNATE}" >&2
-                fi
-                ;;
-            services_alternate)
-                SERVICES_ALTERNATE=(`echo ${config_items[1]} | sed s/,/\ /g`)
-                echo "SERVICES_ALTERNATE: ${SERVICES_ALTERNATE[@]}" >&2
-                ;;
-        esac
-    done
-}
-
 #
 # get_git: Clones a git into a subdirectory of ${WORKDIR}, and
 #          leaves you in that directory.  On error the directory
@@ -393,27 +334,54 @@ function build_image_loci {
     PROFILES=$(source ${image_build_file} && echo ${PROFILES})
     local PYTHON3
     PYTHON3=$(source ${image_build_file} && echo ${PYTHON3})
-
-    if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
-
-    if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
+    local MIRROR_LOCAL
+    MIRROR_LOCAL=$(source ${image_build_file} && echo ${MIRROR_LOCAL})
 
     echo "Building ${LABEL}"
 
+    local ORIGWD=${PWD}
+
+    if [ "${MIRROR_LOCAL}" = "yes" ]; then
+        # Setup a local mirror of PROJECT_REPO
+
+        local BARE_CLONES=${WORKDIR}/bare_clones
+        mkdir -p ${BARE_CLONES}
+        if [ $? -ne 0 ]; then
+            echo "Failed to create ${BARE_CLONES}" >&2
+            RESULTS_FAILED+=(${LABEL})
+            return 1
+        fi
+
+        local CLONE_DIR=${BARE_CLONES}/${PROJECT}.git
+
+        # Remove prior clone dir, if it exists
+        \rm -rf ${CLONE_DIR}
+
+        echo "Creating bare clone of ${PROJECT_REPO} for ${LABEL} build..."
+        git clone --bare ${PROJECT_REPO} ${CLONE_DIR} \
+            && mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \
+            && chmod a+x ${CLONE_DIR}/hooks/post-update \
+            && cd ${CLONE_DIR} \
+            && git update-server-info \
+            && cd ${ORIGWD}
+        if [ $? -ne 0 ]; then
+            echo "Failed to clone ${PROJECT_REPO}... Aborting ${LABEL} build"
+            RESULTS_FAILED+=(${LABEL})
+            cd ${ORIGWD}
+            return 1
+        fi
+
+        PROJECT_REPO=http://${HOSTNAME}:8088/${CLONE_DIR}
+    fi
+
     local -a BUILD_ARGS=
     BUILD_ARGS=(--build-arg PROJECT=${PROJECT})
     BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO})
     BUILD_ARGS+=(--build-arg FROM=${BASE})
 
-    if is_in ${LABEL} ${SERVICES_ALTERNATE[@]}; then
+    if [ "${PYTHON3}" != "yes" ] ; then
         echo "Python2 service ${LABEL}"
-        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_ALTERNATE})
+        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
     else
         echo "Python3 service ${LABEL}"
         BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
@@ -518,16 +486,6 @@ function build_image_docker {
     local DOCKER_PATCHES
     DOCKER_PATCHES=$(source ${image_build_file} && for p in ${DOCKER_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done)
 
-    if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
-
-    if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
-
     echo "Building ${LABEL}"
 
     local real_docker_context
@@ -625,16 +583,6 @@ function build_image_script {
     local SOURCE_PATCHES
     SOURCE_PATCHES=$(source ${image_build_file} && for p in ${SOURCE_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done)
 
-    if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
-
-    if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
-        echo "Skipping ${LABEL}"
-        return 0
-    fi
-
     # Validate the COMMAND option
     SUPPORTED_COMMAND_ARGS=('bash')
     local VALID_COMMAND=1
@@ -715,7 +663,7 @@ function build_image {
     esac
 }
 
-OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@")
+OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -742,8 +690,8 @@ while true; do
             WHEELS=$2
             shift 2
             ;;
-        --wheels-alternate)
-            WHEELS_ALTERNATE=$2
+        --wheels-alternate|--wheels-py2)
+            WHEELS_PY2=$2
             shift 2
             ;;
         --version)
@@ -813,10 +761,6 @@ while true; do
             MAX_ATTEMPTS=$2
             shift 2
             ;;
-        --config-file)
-            CONFIG_FILE=$2
-            shift 2
-            ;;
         -h | --help )
             usage
             exit 1
@@ -842,37 +786,80 @@ if [ ${VALID_OS} -ne 0 ]; then
     exit 1
 fi
 
-DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg"
-
-# Read additional arguments from config file if it exists.
-if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then
-    CONFIG_FILE=${DEFAULT_CONFIG_FILE}
-fi
-if [[ ! -z  ${CONFIG_FILE} ]]; then
-    if [[ -f ${CONFIG_FILE} ]]; then
-        get_args_from_file ${CONFIG_FILE}
-    else
-        echo "Config file not found: ${CONFIG_FILE}"
-        exit 1
-    fi
-fi
-
-if [ -z "${WHEELS}" ]; then
-    echo "Path to wheels tarball must be specified with --wheels option." >&2
-    exit 1
-fi
-
-if [ ${#SERVICES_ALTERNATE[@]} -ne 0 ] && [ -z "${WHEELS_ALTERNATE}" ]; then
-    echo "Path to wheels-alternate tarball must be specified with --wheels-alternate option"\
-         "if python2 based services need to be build!" >&2
-    exit 1
-fi
-
 if [ -z "${BASE}" ]; then
     echo "Base image must be specified with --base option." >&2
     exit 1
 fi
 
+# Guess WHEELS_PY2 if missing
+if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then
+    # http://foo/bar.tar?xxx#yyy => http://foo/bar-py2.tar?xxx#yyy
+    WHEELS_PY2="$(echo "$WHEELS" | sed -r 's,^([^#?]*)(\.tar)(\.gz|\.bz2|\.xz)?([#?].*)?$,\1-py2\2\3\4,i')"
+    if [[ "$WHEELS" == "$WHEELS_PY2" ]]; then
+        echo "Unable to guess --wheels-py2, please specify it explicitly" >&2
+        exit 1
+    fi
+fi
+
+# Find the directives files
+IMAGE_BUILD_FILES=()
+function find_image_build_files {
+    local image_build_inc_file image_build_dir image_build_file
+    local -A all_labels
+
+    for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do
+        basedir=$(dirname ${image_build_inc_file})
+        for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do
+            for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do
+
+                # reset & read image build directive vars
+                local BUILDER=
+                local PROJECT=
+                local LABEL=
+                local PYTHON3=
+                PROJECT="$(source ${image_build_file} && echo ${PROJECT})"
+                BUILDER="$(source ${image_build_file} && echo ${BUILDER})"
+                LABEL="$(source ${image_build_file} && echo ${LABEL})"
+                PYTHON3="$(source ${image_build_file} && echo ${PYTHON3})"
+
+                # make sure labels are unique
+                if [[ -n "${all_labels["$LABEL"]}" ]] ; then
+                    echo "The following files define the same LABEL $LABEL" >&2
+                    echo "  ${all_labels["$LABEL"]}" >&2
+                    echo "  ${image_build_file}" >&2
+                    exit 1
+                fi
+                all_labels["$LABEL"]="$image_build_file"
+
+                # skip images we don't want to build
+                if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
+                    continue
+                fi
+                if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
+                    continue
+                fi
+
+                # loci builders require a wheels tarball
+                if [[ "${BUILDER}" == "loci" ]] ; then
+                    # python3 projects require $WHEELS
+                    if [[ "${PYTHON3}" == "yes" && -z "${WHEELS}" ]] ; then
+                        echo "You are building python3 services with loci, but you didn't specify --wheels!" >&2
+                        exit 1
+                    # python2 projects require WHEELS_PY2
+                    elif [[ "${PYTHON3}" != "yes" && -z "${WHEELS_PY2}" ]] ; then
+                        echo "You are building python2 services with loci, but you didn't specify --wheels-py2!" >&2
+                        exit 1
+                    fi
+                fi
+
+                # Save image build file in the global list
+                IMAGE_BUILD_FILES+=("$image_build_file")
+            done
+        done
+    done
+}
+find_image_build_files
+
 IMAGE_TAG="${OS}-${BUILD_STREAM}"
 IMAGE_TAG_LATEST="${IMAGE_TAG}-latest"
 
@@ -933,15 +920,10 @@ if ! (grep -q rh-python36-mod_wsgi ${WORKDIR}/loci/bindep.txt); then
     echo 'rh-python36-mod_wsgi        [platform:rpm !platform:suse (apache python3)]' >>  ${WORKDIR}/loci/bindep.txt
 fi
 
-# Find the directives files
-for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do
-    basedir=$(dirname ${image_build_inc_file})
-    for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do
-        for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do
-            # Failures are reported by the build functions
-            build_image ${image_build_file}
-        done
-    done
+# Build everything
+for image_build_file in "${IMAGE_BUILD_FILES[@]}" ; do
+    # Failures are reported by the build functions
+    build_image ${image_build_file}
 done
 
 if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then

build-tools/build-docker-images/docker-image-build-centos-dev.cfg

@@ -1,2 +0,0 @@
-services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients
-wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_dev-wheels.tar

build-tools/build-docker-images/docker-image-build-centos-stable.cfg

@@ -1,2 +0,0 @@
-services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients
-wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_stable-wheels.tar

build-tools/build-docker-images/tag-management/image-tags.yaml

@@ -2,10 +2,10 @@
 ---
 images:
   - name: docker.io/starlingx/k8s-cni-sriov
-    src_build_tag: master-centos-stable-20191203T153530Z.0
-    src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4
-    # Tag determined based on release tag associated with upstream commit
-    tag: stx.4.0-v2.2
+    src_build_tag: master-centos-stable-20210218T003113Z.0
+    src_ref: https://opendev.org/starlingx/integ/commit/eccff3b0e661592084d9114a9a41816761e1f9b5
+    # Version determined by running 'git describe --tags' in clone of upstream repo
+    tag: stx.5.0-v2.6-7-gb18123d8
   - name: docker.io/starlingx/k8s-plugins-sriov-network-device
     src_build_tag: master-centos-stable-20200512T184214Z.0
     src_ref: https://opendev.org/starlingx/integ/commit/e2dc5c2dd0042788697ade268ac5c24fe9dc2f8c
@@ -37,10 +37,10 @@ images:
     # Version determined by running 'git describe --tags' in clone of upstream repo
     tag: stx.4.0-v0.11.0-109-gc48c502
   - name: docker.io/starlingx/stx-oidc-client
-    src_build_tag: master-centos-stable-20200901T001315Z.0
-    src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/957fc7c2092c7574a0a931012a0ec1cf5bb66429
-    # StarlingX app. Setting version to v1.0.3
-    tag: stx.5.0-v1.0.3
+    src_build_tag: master-centos-stable-20210119T015305Z.0
+    src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/70147e64e910e9878dd5bdf464cfd9672894ba18
+    # StarlingX app. Setting version to v1.0.4
+    tag: stx.5.0-v1.0.4
   - name: docker.io/starlingx/dex
     src_build_tag: master-centos-stable-20200204T162546Z.0
     src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/5d6701bdf214e77f460f2e3dd2b6f7d3186830c8
@@ -55,9 +55,9 @@ images:
     src_ref: https://opendev.org/starlingx/metal/commit/d46c9c55a9a9b7ea09e8d0fe66c8cfbeeb9ac75f
     tag: stx.5.0-v1.0.0
   - name: docker.io/starlingx/stx-platformclients
-    src_build_tag: master-centos-stable-20200803T230630Z.0
-    src_ref: https://opendev.org/starlingx/distcloud-client/commit/7036f1fd11cd3bbae743aee89908e8195e4ded40
-    tag: stx.5.0-v1.4.0
+    src_build_tag: master-centos-stable-20210512T053357Z.0
+    src_ref: https://opendev.org/starlingx/distcloud-client/commit/d52a9080082db5fda2e77fb9e342f812ea8c17e1
+    tag: stx.5.0-v1.4.3
   - name: docker.io/starlingx/stx-vault-manager
     src_build_tag: master-centos-stable-20200722T035334Z.0
     src_ref: https://opendev.org/starlingx/vault-armada-app/commit/2cd206d6703cc2733e39ecad4539c0d5f1600550
@@ -68,3 +68,27 @@ images:
     src_ref: https://opendev.org/starlingx/portieris-armada-app/commit/a6123ffebb77f23d5182576be17e69d62fd8d701
     # Tag based on upstream package version
     tag: stx.5.0-v0.7.0
+  - name: docker.io/starlingx/stx-snmp
+    src_build_tag: master-centos-stable-20210105T023146Z.0
+    src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/2b370655a7f9a506ea139cfffa6c466d1a82cce4
+    tag: stx.5.0-v1.0.0
+  - name: docker.io/starlingx/stx-fm-subagent
+    src_build_tag: master-centos-stable-20210105T023146Z.0
+    src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/2b370655a7f9a506ea139cfffa6c466d1a82cce4
+    tag: stx.5.0-v1.0.0
+  - name: docker.io/starlingx/stx-fm-trap-subagent
+    src_build_tag: master-centos-stable-20210314T171252Z.0
+    src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/5aca0dd1661bc87a7927c00cf95e0c8aa6f2e2a0
+    tag: stx.5.0-v1.0.1
+  - name: docker.io/starlingx/notificationservice-base
+    src_build_tag: master-centos-stable-20210504T193232Z.0
+    src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/eb4458e37ebe170f4c1289362f9cbc55fb1f32aa
+    tag: stx.5.0-v1.0.4
+  - name: docker.io/starlingx/locationservice-base
+    src_build_tag: master-centos-stable-20210204T224209Z.0
+    src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/545e6b6bb093235c2f8dab8d171f30c6ae8682d3
+    tag: stx.5.0-v1.0.1
+  - name: docker.io/starlingx/notificationclient-base
+    src_build_tag: master-centos-stable-20210503T050004Z.0
+    src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/6e87c185baf927b28d0bcff6e2763a1e62c8145e
+    tag: stx.5.0-v1.0.4

build-tools/build-img

@@ -566,7 +566,7 @@ if [ ${#RPM_ADDON_LIST[@]} -gt 0 ] ; then
     pushd $MY_WORKSPACE
     patch_file="PATCH.img-addon"
     patched_iso="$TEMPFILES_DIR/bootimage_${AUTO_MODE}${GRAPHICAL_SUFFIX}_patched.iso"
-    cmd=("$PATCH_BUILD" --id "${patch_file}" --summary "additional packages for qcow2 image" --desc "Adds customizations to qcow2 image")
+    cmd=("$PATCH_BUILD" --id "${patch_file}" --summary "additional packages for qcow2 image" --desc "Adds customizations to qcow2 image" --status "REL" --reboot-required "N")
     for rpm_addon in "${RPM_ADDON_LIST[@]}"; do
         cmd+=(--all-nodes "${rpm_addon}")
     done

build-tools/build-rpms-parallel

@@ -122,6 +122,10 @@ number_of_users () {
     users | tr ' ' '\n' | sort --uniq | wc -l
 }
 
+total_mem_gb () {
+    free -g | grep 'Mem:' | awk '{ print $2 }'
+}
+
 available_mem_gb () {
     free -g | grep 'Mem:' | awk '{ print $7 }'
 }
@@ -238,26 +242,41 @@ compute_resources () {
     local users=$(number_of_users)
     if [ $users -lt 1 ]; then users=1; fi
     local mem=$(available_mem_gb)
+    local total_mem=$(total_mem_gb)
     local disk=$(available_disk_gb)
     local cpus=$(number_of_cpus)
     local num_users=$(sqrt $users)
     local num_build=$(number_of_builds_in_progress)
     num_build=$((num_build+1))
-    echo "compute_resources: total: cpus=$cpus, mem=$mem, disk=$disk, weight=$weight, num_build=$num_build"
+    echo "compute_resources: total: cpus=$cpus, total_mem=$total_mem, avail_mem=$mem, disk=$disk, weight=$weight, num_build=$num_build"
 
     # What fraction of the machine will we use
     local share_factor=$num_users
     if [ $share_factor -gt $((MAX_SHARE_FACTOR+num_build-1)) ]; then share_factor=$((MAX_SHARE_FACTOR+num_build-1)); fi
     if [ $share_factor -lt $num_build ]; then share_factor=$num_build; fi
-    local mem_share_factor=$((share_factor-num_build))
+
+    # What fraction of free memory can we use.  
+    #  e.g.
+    #  We intend to support 4 concurrent builds (share_factor)
+    #  Two builds (excluding ours) are already underway (num_build-1)
+    #  So we should be able to support 2 more builds (mem_share_factor)
+    local mem_share_factor=$((share_factor-(num_build-1)))
     if [ $mem_share_factor -lt 1 ]; then mem_share_factor=1; fi
+
     echo "compute_resources: share_factor=$share_factor mem_share_factor=$mem_share_factor"
 
     # What resources are we permitted to use
+    #   Continuing the example from above ... memory share is the lesser of
+    #   - Half the available memory     (mem/mem_share_factor)
+    #   - A quarter of the total memory (total_mem/share_factor)
     local mem_share=$(((mem-MEMORY_RESERVE)/mem_share_factor))
     if [ $mem_share -lt 0 ]; then mem_share=0; fi
+    local total_mem_share=$(((total_mem-MEMORY_RESERVE)/share_factor))
+    if [ $total_mem_share -lt 0 ]; then total_mem_share=0; fi
+    if [ $mem_share -gt $total_mem_share ]; then mem_share=$total_mem_share; fi
     local disk_share=$((disk/share_factor))
     local cpus_share=$((cpus/share_factor))
+
     echo "compute_resources: our share: cpus=$cpus_share, mem=$mem_share, disk=$disk_share"
 
     # How many build jobs, how many jobs will use tmpfs, and how much mem for each tmpfs
@@ -293,7 +312,7 @@ compute_resources () {
         fi
     done
 
-    # Our output is saved in environmnet variables
+    # Our output is saved in environment variables
     MOCKCHAIN_RESOURCE_ALLOCATION=$(echo $x | sed 's#^:##')
     MAX_WORKERS=$workers
     echo "compute_resources: MAX_WORKERS=$MAX_WORKERS, MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"
@@ -654,7 +673,7 @@ kill_descendents ()
 
     local relevant_recursive_children="$ME"
     local relevant_recursive_promote_children="mock"
-    local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16"
+    local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16 mockchain-parallel-2.6 mockchain-parallel-2.7"
 
     local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
     local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
@@ -1181,14 +1200,24 @@ mock_clean_metadata_cfg () {
       return 1
    fi
 
-   CMD=$((cat $CFG; \
-                 grep config_opts\\[\'yum.conf\'\\\] $CFG | \
-                 sed 's#\\n#\n#g') | \
-                 grep '^[[]' | \
-                 grep -v main | \
-                 sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --enablerepo=#" -e 's#$# clean metadata#' | \
-                 sort -u | \
-                 tr '\n' ';')
+   #
+   # From mock config, extract the embedded yum/dnf config.
+   # Then extract the repo definitions,
+   # and convert to a series of yum commands to clean the 
+   # metadata one repo at a time.   e.g.
+   # CMD="yum --disablerepo=* --enablerepo=StxCentos7Distro clean metadata; \
+   #      yum --disablerepo=* --enablerepo=StxCentos7Distro-rt clean metadata;
+   #      ...
+   #     "
+   #
+   CMD=$((grep -e config_opts\\[\'yum.conf\'\\\] $CFG \
+               -e config_opts\\[\'dnf.conf\'\\\] $CFG | \
+          sed 's#\\n#\n#g') | \
+         grep '^[[]' | \
+         grep -v main | \
+         sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --disablerepo=* --enablerepo=#" -e 's#$# clean metadata#' | \
+         sort -u | \
+         tr '\n' ';')
    echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
    trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP
    RC=$?
@@ -2338,6 +2367,7 @@ if [ $CAREFUL -eq 1 ]; then
    CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after"
 fi
 
+CMD_OPTIONS+=" $MOCK_PASSTHROUGH --enable-plugin=package_state"
 CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG"
 
 echo "CAREFUL=$CAREFUL"

build-tools/build-rpms-serial

@@ -25,7 +25,14 @@
 
 export ME=$(basename "$0")
 CMDLINE="$ME $@"
+BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 
+# Set PKG_MANAGER for our build environment.
+source "${BUILD_RPMS_PARALLEL_DIR}/pkg-manager-utils.sh"
+
+
+# Build for distribution.  Currently 'centos' is only supported value.
+export DISTRO="centos"
 
 CREATEREPO=$(which createrepo_c)
 if [ $? -ne 0 ]; then
@@ -42,6 +49,7 @@ if [ ! -d ${LOCAL_REPO} ]; then
     fi
 fi
 
+# Make sure we have a dependency cache
 DEPENDANCY_DIR="${LOCAL_REPO}/dependancy-cache"
 SRPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires"
 SRPM_TRANSITIVE_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-transitive-requires"
@@ -118,7 +126,7 @@ create-no-clean-list () {
        local g
 
        for g in $install_groups; do
-           # Find manditory packages in the group.
+           # Find mandatory packages in the group.
            # Discard anything before (and including) 'Mandatory Packages:'
            # and anything after (and including) 'Optional Packages:'.
            # Also discard leading spaces or '+' characters.
@@ -135,7 +143,7 @@ create-no-clean-list () {
 
        while [ $noclean_list_len -gt $noclean_last_list_len ]; do
            noclean_last_list_len=$noclean_list_len
-           noclean_list=$( (dnf -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq)
+           noclean_list=$( (${PKG_MANAGER} -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq)
            noclean_list_len=$(echo $noclean_list | wc -w)
        done
 
@@ -475,7 +483,7 @@ kill_descendents ()
 
     local relevant_recursive_children="$ME"
     local relevant_recursive_promote_children="mock"
-    local relevant_other_children="mockchain-parallel"
+    local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16 mockchain-parallel-2.6 mockchain-parallel-2.7"
 
     local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
     local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
@@ -964,7 +972,24 @@ mock_clean_metadata_cfg () {
       return 1
    fi
 
-   CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';')
+   #
+   # From mock config, extract the embedded yum/dnf config.
+   # Then extract the repo definitions,
+   # and convert to a series of yum commands to clean the 
+   # metadata one repo at a time.   e.g.
+   # CMD="yum --disablerepo=* --enablerepo=StxCentos7Distro clean metadata; \
+   #      yum --disablerepo=* --enablerepo=StxCentos7Distro-rt clean metadata;
+   #      ...
+   #     "
+   #
+   CMD=$((grep -e config_opts\\[\'yum.conf\'\\\] $CFG \
+               -e config_opts\\[\'dnf.conf\'\\\] $CFG | \
+          sed 's#\\n#\n#g') | \
+         grep '^[[]' | \
+         grep -v main | \
+         sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --disablerepo=* --enablerepo=#" -e 's#$# clean metadata#' | \
+         sort -u | \
+         tr '\n' ';')
    echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
    trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP
    RC=$?
@@ -1129,6 +1154,7 @@ clean_yum_cache_cfg () {
    return $RC
 }
 
+
 clean_yum_cache () {
    echo "${FUNCNAME[0]}: in"
    clean_yum_cache_cfg $BUILD_CFG
@@ -1249,7 +1275,6 @@ while true ; do
     esac
 done
 
-
 # Reset variables
 if [ -n "$MY_WORKSPACE" ]; then
    export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE}

build-tools/build-wheels/build-base-wheels.sh

@@ -114,8 +114,6 @@ while true; do
     esac
 done
 
-BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base
-
 BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD_STREAM}"
 
 # BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME.
@@ -123,7 +121,6 @@ BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD
 BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}"
 
 DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile
-WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg
 
 function supported_os_list {
     for f in ${DOCKER_PATH}/*-dockerfile; do
@@ -137,40 +134,75 @@ if [ ! -f ${DOCKER_FILE} ]; then
     exit 1
 fi
 
-if [ ! -f ${WHEELS_CFG} ]; then
-    echo "Required file does not exist: ${WHEELS_CFG}" >&2
-    exit 1
-fi
+# Print a loud message
+function notice {
+    (
+        set +x
+        echo
+        echo ======================================
+        for s in "$@" ; do
+            echo "$s"
+        done
+        echo ======================================
+        echo
+    ) 2>&1
+}
 
-#
-# Check build output directory for unexpected files,
-# ie. wheels from old builds that are no longer in wheels.cfg
-#
-if [ -d ${BUILD_OUTPUT_PATH} ]; then
+# prefix each line of a command's output
+# also redirects command's STDERR to STDOUT
+log_prefix() {
+    local prefix="$1" ; shift
+    "$@" 2>&1 | awk -v prefix="$prefix" '{print prefix $0}'
+    # return false if the command (rather than awk) failed
+    [ ${PIPESTATUS[0]} -eq 0 ]
+}
 
-    for f in ${BUILD_OUTPUT_PATH}/*; do
-        grep -q "^$(basename $f)|" ${WHEELS_CFG}
-        if [ $? -ne 0 ]; then
-            echo "Deleting stale file: $f"
-            rm -f $f
-        fi
-    done
-else
-    mkdir -p ${BUILD_OUTPUT_PATH}
-    if [ $? -ne 0 ]; then
-        echo "Failed to create directory: ${BUILD_OUTPUT_PATH}" >&2
+
+# Make sure a file exists, exit otherwise
+function require_file {
+    if [ ! -f "${1}" ]; then
+        echo "Required file does not exist: ${1}" >&2
         exit 1
     fi
-fi
+}
 
-# Check to see if we need to build anything
-BUILD_NEEDED=no
-for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do
-    if [[ "${wheel}" =~ \* || ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]]; then
-        BUILD_NEEDED=yes
-        break
+# Check build output directory for unexpected files,
+# ie. wheels from old builds that are no longer in wheels.cfg
+function prepare_output_dir {
+    local output_dir="$1"
+    local wheels_cfg="$2"
+    if [ -d ${output_dir} ]; then
+        local f
+        for f in ${output_dir}/*; do
+            if [ -f $f ] ; then
+                grep -q "^$(basename $f)|" ${wheels_cfg}
+                if [ $? -ne 0 ]; then
+                    echo "Deleting stale file: $f"
+                    rm -f $f
+                fi
+            fi
+        done
+    else
+        mkdir -p ${output_dir}
+        if [ $? -ne 0 ]; then
+            echo "Failed to create directory: ${output_dir}" >&2
+            exit 1
+        fi
     fi
-done
+}
+
+BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base
+BUILD_OUTPUT_PATH_PY2=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base-py2
+WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg
+WHEELS_CFG_PY2=${DOCKER_PATH}/${BUILD_STREAM}-wheels-py2.cfg
+
+# make sure .cfg files exist
+require_file "${WHEELS_CFG}"
+require_file "${WHEELS_CFG_PY2}"
+
+# prepare output directories
+prepare_output_dir "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}"
+prepare_output_dir "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}"
 
 if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
     # Download the master wheel from loci, so we're only building pieces not covered by it
@@ -194,16 +226,30 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
     docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null
 
     echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}"
-    docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl'
+    rm -rf "${BUILD_OUTPUT_PATH}-loci"
+    mkdir -p "$BUILD_OUTPUT_PATH-loci"
+    docker export ${USER}_inspect_wheels | tar x -C "${BUILD_OUTPUT_PATH}-loci" '*.whl'
     if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then
         echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2
         docker rm ${USER}_inspect_wheels
         if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
             docker image rm ${MASTER_WHEELS_IMAGE}
         fi
+        rm -rf "${BUILD_OUTPUT_PATH}-loci"
         exit 1
     fi
 
+    # copy loci wheels in base and base-py2 directories
+    if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH}"/ ; then
+        echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH}" >&2
+        exit 1
+    fi
+    if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH_PY2}"/ ; then
+        echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH_PY2}" >&2
+        exit 1
+    fi
+    rm -rf "${BUILD_OUTPUT_PATH}-loci"
+
     docker rm ${USER}_inspect_wheels
 
     if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
@@ -211,7 +257,21 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
     fi
 fi
 
-if [ "${BUILD_NEEDED}" = "no" ]; then
+# check if there are any wheels missing
+function all_wheels_exist {
+    local output_dir="$1"
+    local wheels_cfg="$2"
+    local wheel
+    for wheel in $(cat "${wheels_cfg}" | sed 's/#.*//' | awk -F '|' '{print $1}'); do
+        if [[ "${wheel}" =~ \* || ! -f ${output_dir}/${wheel} ]]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+if all_wheels_exist "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}" && \
+   all_wheels_exist "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}" ; then
     echo "All base wheels are already present. Skipping build."
     exit 0
 fi
@@ -247,12 +307,10 @@ if [ $? -ne 0 ]; then
 fi
 
 # Run the image, executing the build-wheel.sh script
-RM_OPT=
-if [ "${KEEP_CONTAINER}" = "no" ]; then
-    RM_OPT="--rm"
-fi
-
 declare -a RUN_ARGS
+if [ "${KEEP_CONTAINER}" = "no" ]; then
+    RUN_ARGS+=(--rm)
+fi
 if [ ! -z "$HTTP_PROXY" ]; then
     RUN_ARGS+=(--env http_proxy=$HTTP_PROXY)
 fi
@@ -262,11 +320,23 @@ fi
 if [ ! -z "$NO_PROXY" ]; then
     RUN_ARGS+=(--env no_proxy=$NO_PROXY)
 fi
-
-RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
+RUN_ARGS+=(--env DISPLAY_RESULT=no)
 
 # Run container to build wheels
-with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]}
+rm -f ${BUILD_OUTPUT_PATH}/failed.lst
+rm -f ${BUILD_OUTPUT_PATH_PY2}/failed.lst
+
+notice "building python3 wheels"
+log_prefix "[python3] " \
+    with_retries ${MAX_ATTEMPTS} \
+        docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
+BUILD_STATUS=$?
+
+notice "building python2 wheels"
+log_prefix "[python2] " \
+    with_retries ${MAX_ATTEMPTS} \
+        docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH_PY2}:/wheels --env PYTHON=python2 ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
+BUILD_STATUS_PY2=$?
 
 if [ "${KEEP_IMAGE}" = "no" ]; then
     # Delete the builder image
@@ -287,8 +357,52 @@ if [ "${KEEP_IMAGE}" = "no" ]; then
 fi
 
 # Check for failures
-if [ -f ${BUILD_OUTPUT_PATH}/failed.lst ]; then
-    # Failures would already have been reported
+check_result() {
+    local python="$1"
+    local status="$2"
+    local dir="$3"
+
+    # There's a failed images list
+    if [ -f "${dir}/failed.lst" ]; then
+        let failures=$(cat "${dir}/failed.lst" | wc -l)
+
+        cat <<EOF
+
+############################################################
+The following ${python} module(s) failed to build:
+$(cat ${dir}/failed.lst)
+
+Summary:
+${failures} build failure(s).
+
+EOF
+        return 1
+    fi
+
+    # No failed images list, but build script failed nonetheless
+    if [ "${status}" != 0 ] ; then
+        cat <<EOF
+
+############################################################
+Build script failed for ${python}
+
+EOF
+        return 1
+    fi
+
+    # success
+    cat <<EOF
+
+############################################################
+All ${python} wheels have been successfully built.
+
+EOF
+    return 0
+}
+
+if ! check_result "python3" "${BUILD_STATUS}" "${BUILD_OUTPUT_PATH}" || \
+   ! check_result "python2" "${BUILD_STATUS_PY2}" "${BUILD_OUTPUT_PATH_PY3}" ; then
     exit 1
 fi
+exit 0
 

build-tools/build-wheels/build-wheel-tarball.sh

@@ -21,7 +21,6 @@ SUPPORTED_OS_ARGS=('centos')
 OS=centos
 OS_VERSION=7.5.1804
 BUILD_STREAM=stable
-CURRENT_STABLE_OPENSTACK=ussuri
 VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
 PUSH=no
 HTTP_PROXY=""
@@ -30,6 +29,13 @@ NO_PROXY=""
 CLEAN=no
 DOCKER_USER=${USER}
 declare -i MAX_ATTEMPTS=1
+PYTHON2=no
+
+# Requirement/constraint URLs -- these will be read from openstack.cfg
+STABLE_OPENSTACK_REQ_URL=
+MASTER_OPENSTACK_REQ_URL=
+STABLE_OPENSTACK_REQ_URL_PY2=
+MASTER_OPENSTACK_REQ_URL_PY2=
 
 # List of top-level services for images, which should not be listed in upper-constraints.txt
 SKIP_CONSTRAINTS=(
@@ -47,6 +53,7 @@ SKIP_CONSTRAINTS=(
     nova
     panko
 )
+SKIP_CONSTRAINTS_PY2=("${SKIP_CONSTRAINTS_PY[@]}")
 
 function usage {
     cat >&2 <<EOF
@@ -64,11 +71,12 @@ Options:
     --user:       Docker repo userid
     --version:    Version for pushed image (if used with --push)
     --attempts:   Max attempts, in case of failure (default: 1)
+    --python2:    Build a python2 tarball
 
 EOF
 }
 
-OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2 -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -131,6 +139,10 @@ while true; do
             MAX_ATTEMPTS=$2
             shift 2
             ;;
+        --python2)
+            PYTHON2=yes
+            shift
+            ;;
         -h | --help )
             usage
             exit 1
@@ -156,6 +168,15 @@ if [ ${VALID_OS} -ne 0 ]; then
     exit 1
 fi
 
+# Read openstack URLs
+source "${MY_SCRIPT_DIR}/openstack.cfg" || exit 1
+
+# Set python version-specific variables
+if [ "${PYTHON2}" = "yes" ]; then
+    SKIP_CONSTRAINTS=("${SKIP_CONSTRAINTS_PY2[@]}")
+    PY_SUFFIX="-py2"
+fi
+
 # Build the base wheels and retrieve the StarlingX wheels
 declare -a BUILD_BASE_WL_ARGS
 BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --stream ${BUILD_STREAM})
@@ -183,7 +204,7 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/tarball
+BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/tarball${PY_SUFFIX}
 if [ -d ${BUILD_OUTPUT_PATH} ]; then
     # Wipe out the existing dir to ensure there are no stale files
     rm -rf ${BUILD_OUTPUT_PATH}
@@ -191,7 +212,7 @@ fi
 mkdir -p ${BUILD_OUTPUT_PATH}
 cd ${BUILD_OUTPUT_PATH}
 
-IMAGE_NAME=stx-${OS}-${BUILD_STREAM}-wheels
+IMAGE_NAME=stx-${OS}-${BUILD_STREAM}-wheels${PY_SUFFIX}
 
 TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/${IMAGE_NAME}.tar
 if [ -f ${TARBALL_FNAME} ]; then
@@ -199,19 +220,27 @@ if [ -f ${TARBALL_FNAME} ]; then
 fi
 
 # Download the global-requirements.txt and upper-constraints.txt files
-if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
-    OPENSTACK_BRANCH=master
+if [ "${PYTHON2}" = "yes" ]; then
+    if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
+        OPENSTACK_REQ_URL="${MASTER_OPENSTACK_REQ_URL_PY2}"
+    else
+        OPENSTACK_REQ_URL="${STABLE_OPENSTACK_REQ_URL_PY2}"
+    fi
 else
-    OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK}
+    if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
+        OPENSTACK_REQ_URL="${MASTER_OPENSTACK_REQ_URL}"
+    else
+        OPENSTACK_REQ_URL="${STABLE_OPENSTACK_REQ_URL}"
+    fi
 fi
 
-with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt
+with_retries ${MAX_ATTEMPTS} wget "${OPENSTACK_REQ_URL}/global-requirements.txt"
 if [ $? -ne 0 ]; then
     echo "Failed to download global-requirements.txt" >&2
     exit 1
 fi
 
-with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt
+with_retries ${MAX_ATTEMPTS} wget "${OPENSTACK_REQ_URL}/upper-constraints.txt"
 if [ $? -ne 0 ]; then
     echo "Failed to download upper-constraints.txt" >&2
     exit 1
@@ -230,7 +259,7 @@ done
 shopt -s nullglob
 
 # Copy the base and stx wheels, updating upper-constraints.txt as necessary
-for wheel in ../base/*.whl ../stx/wheels/*.whl; do
+for wheel in ../base${PY_SUFFIX}/*.whl ../stx/wheels/*.whl; do
     # Get the wheel name and version from the METADATA
     METADATA=$(unzip -p ${wheel} '*/METADATA')
     name=$(echo "${METADATA}" | grep '^Name:' | awk '{print $2}')

build-tools/build-wheels/docker/centos-dockerfile

@@ -20,3 +20,10 @@ RUN set -ex ;\
 COPY docker-build-wheel.sh /
 COPY ${BUILD_STREAM}-wheels.cfg /wheels.cfg
 
+# Python2 packages
+RUN set -ex; \
+    yum -y install python python-devel ;\
+    wget https://bootstrap.pypa.io/pip/2.7/get-pip.py ;\
+    python get-pip.py
+COPY ${BUILD_STREAM}-wheels-py2.cfg /wheels-py2.cfg
+

build-tools/build-wheels/docker/dev-wheels-py2.cfg

@@ -0,0 +1,18 @@
+#
+# git: wheelname|git|git-source|basedir|branch
+# tar: wheelname|tar|wget-source|basedir
+# pypi: wheelname|pypi|wget-source
+# zip: wheelname|zip|wget-source|basedir
+#
+# If fix_setup must be called, add |fix_setup at the end of the line
+#
+amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
+lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
+panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
+google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl
+neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master
+python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master
+openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master
+networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
+croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
+pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3

build-tools/build-wheels/docker/dev-wheels.cfg

@@ -6,13 +6,14 @@
 #
 # If fix_setup must be called, add |fix_setup at the end of the line
 #
-amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
-lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
-panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
+amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
+croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
 google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl
+lz4-0.9.0-cp36-cp36m-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
+networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
 neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master
 python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master
 openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master
-networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
-croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
-pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3
+panko-5.0.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
+pecan-1.3.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3
+

build-tools/build-wheels/docker/docker-build-wheel.sh

@@ -10,8 +10,14 @@
 
 CFGFILE=/wheels.cfg
 OUTPUTDIR=/wheels
-FAILED_LOG=$OUTPUTDIR/failed.lst
+FAILED_LOG="${OUTPUTDIR}/failed.lst"
+: ${DISPLAY_RESULT=yes}
 declare -i MAX_ATTEMPTS=5
+: ${PYTHON=python3}
+if [[ "${PYTHON}" == "python2" ]] ; then
+    CFGFILE=/wheels-py2.cfg
+    FAILED_LOG="${OUTPUTDIR}/failed-py2.lst"
+fi
 
 #
 # Function to log the start of a build
@@ -184,7 +190,7 @@ function from_git {
         fi
 
         # Build the wheel
-        python3 setup.py bdist_wheel
+        ${PYTHON} setup.py bdist_wheel
         if [ -f dist/$wheelname ]; then
             cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
         else
@@ -244,7 +250,7 @@ function from_tar {
         fi
 
         # Build the wheel
-        python3 setup.py bdist_wheel
+        ${PYTHON} setup.py bdist_wheel
         if [ -f dist/$wheelname ]; then
             cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
         else
@@ -295,7 +301,7 @@ function from_zip {
         fi
 
         # Build the wheel
-        python3 setup.py bdist_wheel
+        ${PYTHON} setup.py bdist_wheel
         if [ -f dist/$wheelname ]; then
             cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
         else
@@ -339,24 +345,28 @@ from_tar
 from_zip
 from_pypi
 
-if [ -f $FAILED_LOG ]; then
-    let failures=$(cat $FAILED_LOG | wc -l)
+if [ -f "${FAILED_LOG}" ]; then
+    if [ "${DISPLAY_RESULT}" = yes ] ; then
+        let failures=$(cat "${FAILED_LOG}" | wc -l)
 
-    cat <<EOF
+        cat <<EOF
 ############################################################
-The following module(s) failed to build:
-$(cat $FAILED_LOG)
+The following ${PYTHON} module(s) failed to build:
+$(cat ${FAILED_LOG})
 
 Summary:
 ${failures} build failure(s).
 EOF
+    fi
     exit 1
 fi
 
-cat <<EOF
+if [ "${DISPLAY_RESULT}" = yes ] ; then
+    cat <<EOF
 ############################################################
-All wheels have been successfully built.
+All ${PYTHON} wheels have been successfully built.
 EOF
+fi
 
 exit 0
 

build-tools/build-wheels/docker/stable-wheels-py2.cfg

@@ -0,0 +1,175 @@
+#
+# git: wheelname|git|git-source|basedir|branch
+# tar: wheelname|tar|wget-source|basedir
+# pypi: wheelname|pypi|wget-source
+# zip: wheelname|zip|wget-source|basedir
+#
+# If fix_setup must be called, add |fix_setup at the end of the line
+#
+abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
+alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0
+amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
+anyjson-0.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c3/4d/d4089e1a3dd25b46bebdb55a992b0797cff657b4477bc32ce28038fdecbc/anyjson-0.3.3.tar.gz|anyjson-0.3.3
+backports.ssl_match_hostname-3.7.0.1-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ff/2b/8265224812912bc5b7a607c44bf7b027554e1b9775e9ee0de8032e3de4b2/backports.ssl_match_hostname-3.7.0.1.tar.gz|backports.ssl_match_hostname-3.7.0.1|fix_setup
+bottle-0.12.17-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c4/a5/6bf41779860e9b526772e1b3b31a65a22bd97535572988d16028c5ab617d/bottle-0.12.17.tar.gz|bottle-0.12.17
+cassandra_driver-3.19.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1c/fe/e4df42a3e864b6b7b2c7f6050b66cafc7fba8b46da0dfb9d51867e171a77/cassandra-driver-3.19.0.tar.gz|cassandra-driver-3.19.0
+cmd2-0.8.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/40/a71caa2aaff10c73612a7106e2d35f693e85b8cf6e37ab0774274bca3cf9/cmd2-0.8.9-py2.py3-none-any.whl
+construct-2.8.22-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e5/c6/3e3aeef38bb0c27364af3d21493d9690c7c3925f298559bca3c48b7c9419/construct-2.8.22.tar.gz|construct-2.8.22
+crc16-0.1.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz|crc16-0.1.1|fix_setup
+demjson-2.2.4-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/96/67/6db789e2533158963d4af689f961b644ddd9200615b8ce92d6cad695c65a/demjson-2.2.4.tar.gz|demjson-2.2.4|fix_setup
+django_floppyforms-1.7.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8c/18/30a9137c7ae279a27ccdeb10f6fe8be18ee98551d01ec030b6cfe8b2d2e2/django-floppyforms-1.7.0.tar.gz|django-floppyforms-1.7.0
+django_pyscss-2.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4b/7f/d771802305184aac6010826f60a0b2ecaa3f57d19ab0e405f0c8db07e809/django-pyscss-2.0.2.tar.gz|django-pyscss-2.0.2
+docopt-0.6.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz|docopt-0.6.2
+dogpile.cache-0.7.1-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/84/3e/dbf1cfc5228f1d3dca80ef714db2c5aaec5cd9efaf54d7e3daef6bc48b19/dogpile.cache-0.7.1.tar.gz|dogpile.cache-0.7.1
+enum_compat-0.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/95/6e/26bdcba28b66126f66cf3e4cd03bcd63f7ae330d29ee68b1f6b623550bfa/enum-compat-0.0.2.tar.gz|enum-compat-0.0.2
+etcd3-0.10.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/09/f1/93603a26daf7a993a0acbbcfd32afce8b2fdf30a765d5651571ab635969b/etcd3-0.10.0.tar.gz|etcd3-0.10.0
+exabgp-4.1.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/b9/f1/f2417bc82c9caa220fcd369a3b55ac895088bcc8afc262e4bb07d48aa40c/exabgp-4.1.2.tar.gz|exabgp-4.1.2
+flask_keystone-0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/1f/ca/3938de8c5f4a3d1c5dd4278bedb9d31d79816feba4d088293c620a366fb1/flask_keystone-0.2.tar.gz|flask_keystone-0.2
+flask_oslolog-0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a7/62/fec02ce761b548b1289680bb1be1aa0bce2b2c4017d5b31bd6c67c78aef9/flask_oslolog-0.1.tar.gz|flask_oslolog-0.1
+fortiosclient-0.0.3-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/e9/aa/b2c0705d5e52c8d9af35422d940800b49c562758fbdad3179a6fbf6e92f5/fortiosclient-0.0.3.tar.gz|fortiosclient-0.0.3
+frozendict-1.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4e/55/a12ded2c426a4d2bee73f88304c9c08ebbdbadb82569ebdd6a0c007cfd08/frozendict-1.2.tar.gz|frozendict-1.2
+funcparserlib-0.3.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/cb/f7/b4a59c3ccf67c0082546eaeb454da1a6610e924d2e7a2a21f337ecae7b40/funcparserlib-0.3.6.tar.gz|funcparserlib-0.3.6
+functools32-3.2.3.post2-py2-none-any.whl|git|https://github.com/MiCHiLU/python-functools32|python-functools32|3.2.3-2|fix_setup
+future-0.17.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/90/52/e20466b85000a181e1e144fd8305caf2cf475e2f9674e797b222f8105f5f/future-0.17.1.tar.gz|future-0.17.1
+happybase-1.2.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/d1/9c/f5f7bdb5439cda2b7da4e20ac24ec0e2455fd68aade8397f211d2994c39d/happybase-1.2.0.tar.gz|happybase-1.2.0
+hiredis-1.0.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9e/e0/c160dbdff032ffe68e4b3c576cba3db22d8ceffc9513ae63368296d1bcc8/hiredis-1.0.0.tar.gz|hiredis-1.0.0
+httplib2-0.13.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/78/23/bb9606e87a66fd8c72a2b1a75b049d3859a122bc2648915be845bc44e04f/httplib2-0.13.1.tar.gz|httplib2-0.13.1
+itsdangerous-1.1.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/76/ae/44b03b253d6fade317f32c24d100b3b35c2239807046a4c953c7b89fa49e/itsdangerous-1.1.0-py2.py3-none-any.whl
+jaeger_client-4.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/f1/da/569a4f1bc3d0c412c7f903053f09ef62fa10949374ca90bc852b22dd3860/jaeger-client-4.1.0.tar.gz|jaeger-client-4.1.0
+jsonpath_rw-1.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/71/7c/45001b1f19af8c4478489fbae4fc657b21c4c669d7a5a036a86882581d85/jsonpath-rw-1.4.0.tar.gz|jsonpath-rw-1.4.0
+krest-1.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fb/d2/9dbbd3a76f2385041720a0eb51ddab676e688fa8bee8a1489470839616cf/krest-1.3.1.tar.gz|krest-1.3.1
+#libvirt_python-4.4.0-cp27-none-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2b/8d/1160cf34dc3d296896eb5c8f4944439ea368b87d2d2431f58d08d6bdf374/libvirt-python-4.4.0.tar.gz|libvirt-python-4.4.0|fix_setup
+logutils-0.3.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/b2/b57450889bf73da26027f8b995fd5fbfab258ec24ef967e4c1892f7cb121/logutils-0.3.5.tar.gz|logutils-0.3.5|fix_setup
+lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
+Mako-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz|Mako-1.1.0
+marathon-0.11.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/97/e3/f036af0d94f98d199233faa71b5bcbef8b8e8e634551940d98c95d276e4f/marathon-0.11.0-py2.py3-none-any.whl
+MarkupSafe-1.1.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz|MarkupSafe-1.1.1
+mox-0.5.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/a1/64740c638cc5fae807022368f4141700518ee343b53eb3e90bf3cc15a4d4/mox-0.5.3.tar.gz|mox-0.5.3|fix_setup
+migrate-0.3.8-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ce/31/1a4cbf8dc0536c55f41072e8ea37b3df1e412262dc731c57e5bb099eb9b2/migrate-0.3.8.tar.gz|migrate-0.3.8
+mpmath-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32/mpmath-1.1.0.tar.gz|mpmath-1.1.0|fix_setup
+msgpack_python-0.4.8-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/msgpack/msgpack-python.git|msgpack-python|0.4.8
+munch-2.3.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/68/f4/260ec98ea840757a0da09e0ed8135333d59b8dfebe9752a365b04857660a/munch-2.3.2.tar.gz|munch-2.3.2
+ndg_httpsclient-0.5.1-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/bf/b2/26470fde7ff55169df8e071fb42cb1f83e22bd952520ab2b5c5a5edc2acd/ndg_httpsclient-0.5.1-py2-none-any.whl
+netifaces-0.10.9-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/0d/18/fd6e9c71a35b67a73160ec80a49da63d1eed2d2055054cc2995714949132/netifaces-0.10.9.tar.gz|netifaces-0.10.9
+networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
+networkx-2.2-py2.py3-none-any.whl|zip|https://files.pythonhosted.org/packages/f3/f4/7e20ef40b118478191cec0b58c3192f822cace858c19505c7670961b76b2/networkx-2.2.zip|networkx-2.2
+neutron_lib-1.29.1-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6b/dd/548cbb7a936de18aa642372927e409540d8f5d96a2f7650c4d1197845f3c/neutron_lib-1.29.1-py2.py3-none-any.whl
+nodeenv-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/00/6e/ed417bd1ed417ab3feada52d0c89ab0ed87d150f91590badf84273e047c9/nodeenv-1.3.3.tar.gz|nodeenv-1.3.3
+nose_exclude-0.5.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/63/cf/90c4be56bf11b7bc8801086d9445baf731aa36b8e8fc5791731e8e604dcd/nose-exclude-0.5.0.tar.gz|nose-exclude-0.5.0
+nosehtmloutput-0.0.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/f7/6cb16c0b233d3f2d62be38ddb7d7c1bc967188c41575ecf0312e6575730d/nosehtmloutput-0.0.5.tar.gz|nosehtmloutput-0.0.5
+openshift-0.8.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/73/ed/c92c0ba23b6c4c8e5542151a1b89cb8ff01f68a72fe68f6c95a28d885ebe/openshift-0.8.6.tar.gz|openshift-0.8.6
+openstack.nose_plugin-0.11-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bc/83/e7c9b9297e1a501d2c2617f98d6176199570e8ee32f0e72669c8852c6c81/openstack.nose_plugin-0.11.tar.gz|openstack.nose_plugin-0.11
+opentracing-2.2.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/94/9f/289424136addf621fb4c75624ef9a3a80e8575da3993a87950c57e93217e/opentracing-2.2.0.tar.gz|opentracing-2.2.0
+ovs-2.11.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/81/06/387b2159ac073de95e484aa6e2f108a232cd906e350307168843061f899f/ovs-2.11.0.tar.gz|ovs-2.11.0
+panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
+pathlib-1.0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz|pathlib-1.0.1|fix_setup
+pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3
+pifpaf-2.2.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/33/dc/4f276c55d94cd73fc1f94e2d23f34b476fea38d240e3e17b837a5749bc9f/pifpaf-2.2.2-py2.py3-none-any.whl
+pika_pool-0.1.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ec/48/50c8f02a3eef4cb824bec50661ec1713040402cc1b2a38954dc977a59c23/pika-pool-0.1.3.tar.gz|pika-pool-0.1.3
+Pint-0.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/15/9d/bf177ebbc57d25e9e296addc14a1303d1e34d7964af5df428a8332349c42/Pint-0.9-py2.py3-none-any.whl
+ply-3.11-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl
+positional-1.1.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8c/16/64a4fa0967c486380468dca18867d22ac1c17bba06349e31ace77c7757f7/positional-1.1.2.tar.gz|positional-1.1.2
+prettytable-0.7.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz|prettytable-0.7.2
+proboscis-1.2.6.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/3c/c8/c187818ab8d0faecdc3c16c1e0b2e522f3b38570f0fb91dcae21662019d0/proboscis-1.2.6.0.tar.gz|proboscis-1.2.6.0
+psutil-5.6.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1c/ca/5b8c1fe032a458c2c4bcbe509d1401dca9dda35c7fc46b36bb81c2834740/psutil-5.6.3.tar.gz|psutil-5.6.3
+psycopg2-2.8.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/5c/1c/6997288da181277a0c29bc39a5f9143ff20b8c99f2a7d059cfb55163e165/psycopg2-2.8.3.tar.gz|psycopg2-2.8.3
+PuLP-1.6.10-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/2d/33/3ae6d9d2ac8c7068937af6372fd8828ac605e62a8b17106fe57110930d38/PuLP-1.6.10.zip|PuLP-1.6.10
+pycparser-2.19-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a/pycparser-2.19.tar.gz|pycparser-2.19
+pycrypto-2.6.1-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/dlitz/pycrypto|pycrypto|v2.6.1|fix_setup
+pycryptodomex-3.9.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e4/90/a01cafbbad7466491e3a630bf1d734294a32ff1b10e7429e9a4e8478669e/pycryptodomex-3.9.0.tar.gz|pycryptodomex-3.9.0
+pydot-1.4.1-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/33/d1/b1479a770f66d962f545c2101630ce1d5592d90cb4f083d38862e93d16d2/pydot-1.4.1-py2.py3-none-any.whl
+pydotplus-2.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/60/bf/62567830b700d9f6930e9ab6831d6ba256f7b0b730acb37278b0ccdffacf/pydotplus-2.0.2.tar.gz|pydotplus-2.0.2
+pyeclib-1.6.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/aa/d6/ca6bba5e66fc7a9810a995b17a3675492da2bec405806d8ac3db18cfd93b/pyeclib-1.6.0.tar.gz|pyeclib-1.6.0|fix_setup
+pyinotify-0.9.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e3/c0/fd5b18dde17c1249658521f69598f3252f11d9d7a980c5be8619970646e1/pyinotify-0.9.6.tar.gz|pyinotify-0.9.6
+pykerberos-1.2.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9a/b8/1ec56b6fa8a2e2a81420bd3d90e70b59fc83f6b857fb2c2c37accddc8be3/pykerberos-1.2.1.tar.gz|pykerberos-1.2.1
+PyKMIP-0.9.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/24/b2/258332aea85163f49a187337e8c85ee4529eb499b84fe0a6fe2d1a9c8d25/PyKMIP-0.9.1.tar.gz|PyKMIP-0.9.1
+pylxd-2.2.10-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/9a/eba58646721ffbff40dc41571b13c9528fdc4e26a82252318c997cdbe26a/pylxd-2.2.10.tar.gz|pylxd-2.2.10
+pyngus-2.3.0-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/58/b1/336b8f64e7e4efa9b95027af71e02cd4cfacca8f919345badb852381878a/pyngus-2.3.0.zip|pyngus-2.3.0
+pyperclip-1.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2d/0f/4eda562dffd085945d57c2d9a5da745cfb5228c02bc90f2c74bbac746243/pyperclip-1.7.0.tar.gz|pyperclip-1.7.0
+pyroute2-0.5.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/f6/80/16a604075345f0c253537d55e5c5282a37c61a1fc8ee0fcc42d1fd2a0739/pyroute2-0.5.6.tar.gz|pyroute2-0.5.6|fix_setup
+pyrsistent-0.15.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/66/b2638d96a2d128b168d0dba60fdc77b7800a9b4a5340cefcc5fc4eae6295/pyrsistent-0.15.4.tar.gz|pyrsistent-0.15.4
+pyScss-1.3.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1d/4a/221ae7561c8f51c4f28b2b172366ccd0820b14bb947350df82428dfce381/pyScss-1.3.4.tar.gz|pyScss-1.3.4
+pysendfile-2.0.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/cd/3f/4aa268afd0252f06b3b487c296a066a01ddd4222a46b7a3748599c8fc8c3/pysendfile-2.0.1.tar.gz|pysendfile-2.0.1
+pystache-0.5.4-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d6/fd/eb8c212053addd941cc90baac307c00ac246ac3fce7166b86434c6eae963/pystache-0.5.4.tar.gz|pystache-0.5.4
+python_cinderclient-4.3.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/f1/09/760c454c5bf67509d7f8479d583a3e84411f51ec2a1942aea3741a49b090/python_cinderclient-4.3.0-py2.py3-none-any.whl
+python_consul-1.1.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/3f/d0/59bc5f1c6c4d4b498c41d8ce7052ee9e9d68be19e16038a55252018a6c4d/python_consul-1.1.0-py2.py3-none-any.whl
+python_editor-1.0.4-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/55/a0/3c0ba1c10f2ca381645dd46cb7afbb73fddc8de9f957e1f9e726a846eabc/python_editor-1.0.4-py2-none-any.whl
+python_etcd-0.4.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a1/da/616a4d073642da5dd432e5289b7c1cb0963cc5dde23d1ecb8d726821ab41/python-etcd-0.4.5.tar.gz|python-etcd-0.4.5
+python_ldap-3.2.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/ea/93/596f875e003c770447f4b99267820a0c769dd2dc3ae3ed19afe460fcbad0/python-ldap-3.2.0.tar.gz|python-ldap-3.2.0
+python_memcached-1.59-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/f5/90/19d3908048f70c120ec66a39e61b92c253e834e6e895cd104ce5e46cbe53/python_memcached-1.59-py2.py3-none-any.whl
+python_nss-1.0.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/6b/29/629098e34951c358b1f04f13a70b3590eb0cf2df817d945bd05c4169d71b/python-nss-1.0.1.tar.bz2|python-nss-1.0.1|fix_setup
+python_pcre-0.7-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9d/af/61435bd163f01fe3709fca9b1f79e4978d8089ee671d2e004fc85e10de29/python-pcre-0.7.tar.gz|python-pcre-0.7|fix_setup
+python_pytun-2.3.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/52/a4/a062106c739eac79c8160fcf5779ebc84afc1c38b016ab216ed1e6da69b6/python-pytun-2.3.0.tar.gz|python-pytun-2.3.0|fix_setup
+python_qpid_proton-0.28.0-cp27-cp27mu-linux_x86_64.whl|zip|https://files.pythonhosted.org/packages/96/35/2c86d844aec1acdfe7778966994aa270fcf03f076df393003bd4fc07dfa9/python-qpid-proton-0.28.0.zip|python-qpid-proton-0.28.0|fix_setup
+python_string_utils-0.6.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/5d/13/216f2d4a71307f5a4e5782f1f59e6e8e5d6d6c00eaadf9f92aeccfbb900c/python-string-utils-0.6.0.tar.gz|python-string-utils-0.6.0|fix_setup
+pyudev-0.21.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bc/a2/31a07829acea8e70a28c247f43fa5d981229ae0f9edfeddedf52de00709b/pyudev-0.21.0.tar.gz|pyudev-0.21.0
+PyYAML-5.1.2-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/e8/b3212641ee2718d556df0f23f78de8303f068fe29cdaa7a91018849582fe/PyYAML-5.1.2.tar.gz|PyYAML-5.1.2
+pyzabbix-0.7.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/11/ad/24e19d0cf16d05b7ee19f337f02058ee9b760649171865469ccceef83027/pyzabbix-0.7.5.tar.gz|pyzabbix-0.7.5
+qpid_python-1.36.0.post1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2a/33/026ac50a29a85d5d54dd7784a98d624f6142cb07ce185ed268ef9bd3b6dc/qpid-python-1.36.0-1.tar.gz|qpid-python-1.36.0-1|fix_setup
+rcssmin-1.0.6-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e2/5f/852be8aa80d1c24de9b030cdb6532bc7e7a1c8461554f6edbe14335ba890/rcssmin-1.0.6.tar.gz|rcssmin-1.0.6|fix_setup
+repoze.lru-0.7-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz|repoze.lru-0.7
+requests_aws-0.1.8-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/5e/2f/4da17752036c04cf4c9af7a2da0d41ef2205043f1c61008006475aa24b8b/requests-aws-0.1.8.tar.gz|requests-aws-0.1.8
+restructuredtext_lint-1.3.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/62/76/bd8760de759fb74d7863e6935200af101cb128a7de008741a4e22341d03c/restructuredtext_lint-1.3.0.tar.gz|restructuredtext_lint-1.3.0
+retrying-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/44/ef/beae4b4ef80902f22e3af073397f079c96969c69b2c7d52a57ea9ae61c9d/retrying-1.3.3.tar.gz|retrying-1.3.3
+rfc3986-1.4.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl
+rjsmin-1.1.0-cp27-cp27mu-manylinux1_x86_64.whl|pypi|https://files.pythonhosted.org/packages/c3/8e/079b7cc3a0fc9934ab05d868a00183c7aafd90b5d6138313d98ac2b9f666/rjsmin-1.1.0-cp27-cp27mu-manylinux1_x86_64.whl
+rtslib_fb-2.1.69-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fc/1a/77a26207bdad13cc39b93d874b3a1b04e5a0b0332fb716e4d654537bacdb/rtslib-fb-2.1.69.tar.gz|rtslib-fb-2.1.69
+scandir-1.10.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz|scandir-1.10.0
+scrypt-0.8.13-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/80/3d/141eb80e754b86f6c25a2ffaf6c3af3acdb65a3e3700829a05ab0c5d965d/scrypt-0.8.13.tar.gz|scrypt-0.8.13
+SecretStorage-2.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a5/a5/0830cfe34a4cfd0d1c3c8b614ede1edb2aaf999091ac8548dd19cb352e79/SecretStorage-2.3.1.tar.gz|SecretStorage-2.3.1
+setproctitle-1.1.10-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz|setproctitle-1.1.10
+simplegeneric-0.8.1-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip|simplegeneric-0.8.1
+simplejson-3.16.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz|simplejson-3.16.0
+skydive_client-0.5.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e4/68/78a246619d9b16bb226562c155f18f798283f86db8f01a89c30b97ac7a27/skydive-client-0.5.0.tar.gz|skydive-client-0.5.0
+smmap2-2.0.5-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/55/d2/866d45e3a121ee15a1dc013824d58072fd5c7799c9c34d01378eb262ca8f/smmap2-2.0.5-py2.py3-none-any.whl
+sphinxcontrib_fulltoc-1.2.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8e/a6/d1297db9b75650681e5429e92e13df139ee6b64303ff1b2eea4ebd32c0a9/sphinxcontrib-fulltoc-1.2.0.tar.gz|sphinxcontrib-fulltoc-1.2.0
+sphinxcontrib_pecanwsme-0.10.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/2b/105d07f47485ecf774cd80b881c29e148182b72a3a60596abdd016c87fce/sphinxcontrib-pecanwsme-0.10.0.tar.gz|sphinxcontrib-pecanwsme-0.10.0
+SQLAlchemy-1.3.8-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/fc/49/82d64d705ced344ba458197dadab30cfa745f9650ee22260ac2b275d288c/SQLAlchemy-1.3.8.tar.gz|SQLAlchemy-1.3.8
+SQLAlchemy_Utils-0.34.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/45/61/3bdd2931e86253fa7df6445a26929fbcc9bc43ad6b27a10f991eb6ecde75/SQLAlchemy-Utils-0.34.2.tar.gz|SQLAlchemy-Utils-0.34.2
+stomp.py-4.1.22-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/52/7e/22ca617f61e0d5904e06c1ebd5d453adf30099526c0b64dca8d74fff0cad/stomp.py-4.1.22.tar.gz|stomp.py-4.1.22
+subprocess32-3.5.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz|subprocess32-3.5.4
+suds_jurko-0.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bd/6f/54fbf0999a606680d27c69b1ad12dfff62768ecb9fe48524cebda6eb4423/suds-jurko-0.6.tar.bz2|suds-jurko-0.6
+systemd_python-234-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e8/a8/00ba0f605837a8f69523e6c3a4fb14675a6430c163f836540129c50b3aef/systemd-python-234.tar.gz|systemd-python-234|fix_setup
+sysv_ipc-1.0.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/08/7d/a862f3045fa191eeece23650725273f2ccaf9ac6b95443dfe4cac6508638/sysv_ipc-1.0.0.tar.gz|sysv_ipc-1.0.0|fix_setup
+Tempita-0.5.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz|Tempita-0.5.2
+termcolor-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz|termcolor-1.1.0|fix_setup
+testrepository-0.0.20-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/85/f495b58b2b0ac907def07385219e9747b75840fa01280f228546a4a5ad7f/testrepository-0.0.20.tar.gz|testrepository-0.0.20
+thrift-0.11.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c6/b4/510617906f8e0c5660e7d96fbc5585113f83ad547a3989b80297ac72a74c/thrift-0.11.0.tar.gz|thrift-0.11.0
+thriftpy-0.3.9-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/f4/19/cca118cf7d2087310dbc8bd70dc7df0c1320f2652873a93d06d7ba356d4a/thriftpy-0.3.9.tar.gz|thriftpy-0.3.9
+thriftpy2-0.4.8-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2c/23/57b00b3d5d3d0ae66d79844a39d3c3b92dde3063c901036808602137d3ab/thriftpy2-0.4.8.tar.gz|thriftpy2-0.4.8
+tinyrpc-1.0.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/21/7a/ff1a74256e1bcc04fbaa414c13a2bb79a29ac9918b25f2238592b991e3bc/tinyrpc-1.0.3.tar.gz|tinyrpc-1.0.3
+tornado-4.5.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/7b/e29ab3d51c8df66922fea216e2bddfcb6430fb29620e5165b16a216e0d3c/tornado-4.5.3.tar.gz|tornado-4.5.3
+trollius-2.2.post1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0b/31/356ae13ad4df58f963e9954d55118f6cffdb3a903c1547973ad7bc347fb9/trollius-2.2.post1.tar.gz|trollius-2.2.post1
+ujson-1.35-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/16/c4/79f3409bc710559015464e5f49b9879430d8f87498ecdc335899732e5377/ujson-1.35.tar.gz|ujson-1.35
+unicodecsv-0.14.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/6f/a4/691ab63b17505a26096608cc309960b5a6bdf39e4ba1a793d5f9b1a53270/unicodecsv-0.14.1.tar.gz|unicodecsv-0.14.1
+uWSGI-2.0.17.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a2/c9/a2d5737f63cd9df4317a4acc15d1ddf4952e28398601d8d7d706c16381e0/uwsgi-2.0.17.1.tar.gz|uwsgi-2.0.17.1
+voluptuous-0.11.7-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/24/3b/fe531688c0d9e057fccc0bc9430c0a3d4b90e0d2f015326e659c2944e328/voluptuous-0.11.7.tar.gz|voluptuous-0.11.7
+warlock-1.3.3-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/c2/36/178b26a338cd6d30523246da4721b1114306f588deb813f3f503052825ee/warlock-1.3.3.tar.gz|warlock-1.3.3
+weakrefmethod-1.0.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/99/82/73a21e3eab9a1ff76d12375f7301fba5c6325b9598eed0ae5b0cf5243656/weakrefmethod-1.0.3.tar.gz|weakrefmethod-1.0.3
+websockify-0.9.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c4/5b/16ec1e9f4fc536846d95a01a77d97da12f8042ca5cf83cdf3dd0442e881c/websockify-0.9.0.tar.gz|websockify-0.9.0
+whereto-0.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/80/83/371a699ce90257608592dadca400a7ecd9a2db6137d78f6f433c7c5e3197/whereto-0.4.0.tar.gz|whereto-0.4.0
+wrapt-1.11.2-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/23/84/323c2415280bc4fc880ac5050dddfb3c8062c2552b34c2e512eb4aa68f79/wrapt-1.11.2.tar.gz|wrapt-1.11.2|fix_setup
+ws4py-0.5.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/53/20/4019a739b2eefe9282d3822ef6a225250af964b117356971bd55e274193c/ws4py-0.5.1.tar.gz|ws4py-0.5.1
+WSME-0.9.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d1/b6/8027248bfca3ce192bc54d46fcda4324c86c8beabe344cbb80fb57a6c868/WSME-0.9.3.tar.gz|WSME-0.9.3
+xattr-0.9.6-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/60/80/a1f35bfd3c7ffb78791b2a6a15c233584a102a20547fd96d48933ec453e7/xattr-0.9.6.tar.gz|xattr-0.9.6
+XStatic-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/36/78/c0ffaf14216517a14d3daa67ff24fbb60b4703e95ce1059a48fd508e6b8c/XStatic-1.0.2.tar.gz|XStatic-1.0.2
+XStatic_Angular_FileUpload-12.0.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4d/fd/c3051915d2f12e8fa11f59c01162ce85e38eca15d9ec73a3d7b271b49744/XStatic-Angular-FileUpload-12.0.4.0.tar.gz|XStatic-Angular-FileUpload-12.0.4.0
+XStatic_Angular_lrdragndrop-1.0.2.4-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/80/ea/ffdde05892eabe468f22403f75299cf5d991f0af4f1400bebbf3af04bc9a/XStatic_Angular_lrdragndrop-1.0.2.4-py2.py3-none-any.whl
+XStatic_Angular_Schema_Form-0.8.13.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/57/71/ceea2c0a72e2ee2d316d6ab1c06b21faa9f5cbc4b36a4127d7847b7079c5/XStatic-Angular-Schema-Form-0.8.13.0.tar.gz|XStatic-Angular-Schema-Form-0.8.13.0
+XStatic_Bootstrap_Datepicker-1.3.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/91/4f/832f14478e714815bb3d44d01dfe8dbe19ccf9f823e0bc7ac1a8cf7fa6b3/XStatic-Bootstrap-Datepicker-1.3.1.0.tar.gz|XStatic-Bootstrap-Datepicker-1.3.1.0
+XStatic_Hogan-2.0.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/21/fe/37d5c8247f24738e7e368d27ebf945de1ea29fbc3112ac5e75b1b7f1d0c9/XStatic-Hogan-2.0.0.2.tar.gz|XStatic-Hogan-2.0.0.2
+XStatic_Jasmine-2.4.1.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/05/43/ceac7def3b6eaf82b6f593e3db2b03a9693a7b002b569e664e382aecddbc/XStatic_Jasmine-2.4.1.2-py2.py3-none-any.whl
+XStatic_jQuery-1.12.4.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/67/f1/c18c14fc4aab386e4aba587c5d10c268de222c75bf5e271b6f68a2ea6e77/XStatic-jQuery-1.12.4.1.tar.gz|XStatic-jQuery-1.12.4.1
+XStatic_JQuery_Migrate-1.2.1.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/7c/fc/edbfcb4574ec3cf0b68a0613dd1904c9139e3bf6dede792d2e7edcf13023/XStatic-JQuery-Migrate-1.2.1.1.tar.gz|XStatic-JQuery-Migrate-1.2.1.1
+XStatic_JQuery.quicksearch-2.0.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ea/ab/f934d06a78ce2c6bb594e9a426f6966b3192c4c279467c9898be6fd284d3/XStatic-JQuery.quicksearch-2.0.3.1.tar.gz|XStatic-JQuery.quicksearch-2.0.3.1
+XStatic_JQuery.TableSorter-2.14.5.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c1/6c/d6b0807906af90536e793a3b23cca557869fa5a27156639f0029de8b1f1f/XStatic-JQuery.TableSorter-2.14.5.1.tar.gz|XStatic-JQuery.TableSorter-2.14.5.1
+XStatic_jquery_ui-1.12.1.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e6/5a/883b22dad1d3e01708312d71c5bc63d543d66cef9b448c1cf85379d64fb3/XStatic-jquery-ui-1.12.1.1.tar.gz|XStatic-jquery-ui-1.12.1.1
+XStatic_mdi-1.6.50.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/73/49/13b9f7ce9fbcc7fabe086b7ac1b056118cbd4c9abf185e01cc4a54631136/XStatic_mdi-1.6.50.2-py2.py3-none-any.whl
+XStatic_objectpath-1.2.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/23/6c/56de25d9d3be430e7de2fcf4baac10279dad78d7b16cbda339cf014c2fe5/XStatic-objectpath-1.2.1.0.tar.gz|XStatic-objectpath-1.2.1.0
+XStatic_Rickshaw-1.5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/45/c6/39aa4d02ea96b04ff372d1e3558587155790b1c5444855a97b89c255be38/XStatic-Rickshaw-1.5.0.0.tar.gz|XStatic-Rickshaw-1.5.0.0
+XStatic_Spin-1.2.5.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/af/21/cca7f0b7abfe008cdd03dd4c4255aad3087f4a892a010c0f6f1452d7344b/XStatic-Spin-1.2.5.2.tar.gz|XStatic-Spin-1.2.5.2
+XStatic_term.js-0.0.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/63/7a/7bfec29f5f28fdda7170ebbbb2204aeb1d33d6050f3476a807590de06434/XStatic-term.js-0.0.7.0.tar.gz|XStatic-term.js-0.0.7.0
+XStatic_tv4-1.2.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2b/26/b07115af27b339c861b8c9a775a621524b421c898e26e015880dfb888c49/XStatic-tv4-1.2.7.0.tar.gz|XStatic-tv4-1.2.7.0
+xvfbwrapper-0.2.9-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/57/b6/4920eabda9b49630dea58745e79f9919aba6408d460afe758bf6e9b21a04/xvfbwrapper-0.2.9.tar.gz|xvfbwrapper-0.2.9
+yappi-1.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/d2/92/7cd637a19fa2a10c0e55a44f8b36bcb83f0e1943ba8f1fb5edb15c819f2e/yappi-1.0.tar.gz|yappi-1.0
+zVMCloudConnector-1.4.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/11/92/9f704de9759816e7b9897b9fb41285b421498b4642551b6fbcccd2850008/zVMCloudConnector-1.4.1.tar.gz|zVMCloudConnector-1.4.1

build-tools/build-wheels/docker/stable-wheels.cfg

@@ -17,7 +17,6 @@ cmd2-0.8.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/
 construct-2.8.22-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/e5/c6/3e3aeef38bb0c27364af3d21493d9690c7c3925f298559bca3c48b7c9419/construct-2.8.22.tar.gz|construct-2.8.22
 crc16-0.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz|crc16-0.1.1|fix_setup
 demjson-2.2.4-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/96/67/6db789e2533158963d4af689f961b644ddd9200615b8ce92d6cad695c65a/demjson-2.2.4.tar.gz|demjson-2.2.4|fix_setup
-Django-2.1.5-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/36/50/078a42b4e9bedb94efd3e0278c0eb71650ed9672cdc91bd5542953bec17f/Django-2.1.5-py3-none-any.whl
 django_debreach-2.0.1-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/2a/92/8c363cf5d1ee33d4c3b999b41c127c5cd3c64d4c20aa47bdfb6c386c9309/django_debreach-2.0.1-py3-none-any.whl
 django_floppyforms-1.8.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/d2/498b883ac309b56b70c26877974bd50927615dd3f6433f5463e2668b1128/django_floppyforms-1.8.0-py2.py3-none-any.whl
 django_pyscss-2.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/4b/7f/d771802305184aac6010826f60a0b2ecaa3f57d19ab0e405f0c8db07e809/django-pyscss-2.0.2.tar.gz|django-pyscss-2.0.2
@@ -46,6 +45,7 @@ lz4-0.9.0-cp36-cp36m-linux_x86_64.whl|git|https://github.com/python-lz4/python-l
 Mako-1.1.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/50/78/f6ade1e18aebda570eed33b7c534378d9659351cadce2fcbc7b31be5f615/Mako-1.1.2-py2.py3-none-any.whl
 marathon-0.12.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/41/66/814432693297dfb076958ae5ac781e3a88fd70d335473a57f4f2c6329515/marathon-0.12.0-py2.py3-none-any.whl
 MarkupSafe-1.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz|MarkupSafe-1.1.1
+migrate-0.3.8-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ce/31/1a4cbf8dc0536c55f41072e8ea37b3df1e412262dc731c57e5bb099eb9b2/migrate-0.3.8.tar.gz|migrate-0.3.8
 mox-0.5.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/a1/64740c638cc5fae807022368f4141700518ee343b53eb3e90bf3cc15a4d4/mox-0.5.3.tar.gz|mox-0.5.3|fix_setup
 mpmath-1.1.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32/mpmath-1.1.0.tar.gz|mpmath-1.1.0|fix_setup
 msgpack_python-0.4.8-cp36-cp36m-linux_x86_64.whl|git|https://github.com/msgpack/msgpack-python.git|msgpack-python|0.4.8
@@ -113,6 +113,7 @@ repoze.lru-0.7-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/12/b
 requests_aws-0.1.8-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/5e/2f/4da17752036c04cf4c9af7a2da0d41ef2205043f1c61008006475aa24b8b/requests-aws-0.1.8.tar.gz|requests-aws-0.1.8
 restructuredtext_lint-1.3.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/62/76/bd8760de759fb74d7863e6935200af101cb128a7de008741a4e22341d03c/restructuredtext_lint-1.3.0.tar.gz|restructuredtext_lint-1.3.0
 retrying-1.3.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/44/ef/beae4b4ef80902f22e3af073397f079c96969c69b2c7d52a57ea9ae61c9d/retrying-1.3.3.tar.gz|retrying-1.3.3
+rfc3986-1.4.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl
 rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl|pypi|https://files.pythonhosted.org/packages/62/ee/574b170bbe7a059314e7239305cb829379232a408901585019e012e71170/rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl
 rtslib_fb-2.1.71-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9e/1b/c26bc038888b1e6042d35ec97599cef05181fb6a7a7ecdbb0c041c3f50ea/rtslib-fb-2.1.71.tar.gz|rtslib-fb-2.1.71|
 scandir-1.10.0-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz|scandir-1.10.0
@@ -177,4 +178,4 @@ XStatic_Font_Awesome-4.7.0.0-py2.py3-none-any.whl|pypi|https://files.pythonhoste
 xvfbwrapper-0.2.9-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/57/b6/4920eabda9b49630dea58745e79f9919aba6408d460afe758bf6e9b21a04/xvfbwrapper-0.2.9.tar.gz|xvfbwrapper-0.2.9
 yappi-1.2.3-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/37/dc/86bbe1822cdc6dbf46c644061bd24217f6a0f056f00162a3697c9bea7575/yappi-1.2.3.tar.gz|yappi-1.2.3
 yaql-1.1.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/77/89/cfee017cf4f2d6f5e7159bbf13fe4131c7dbf20d675b78c9928ae9aa9df8/yaql-1.1.3.tar.gz|yaql-1.1.3
-zVMCloudConnector-1.4.1-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/11/92/9f704de9759816e7b9897b9fb41285b421498b4642551b6fbcccd2850008/zVMCloudConnector-1.4.1.tar.gz|zVMCloudConnector-1.4.1
+zVMCloudConnector-1.4.1-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/11/92/9f704de9759816e7b9897b9fb41285b421498b4642551b6fbcccd2850008/zVMCloudConnector-1.4.1.tar.gz|zVMCloudConnector-1.4.1

build-tools/build-wheels/openstack.cfg

@@ -0,0 +1,13 @@
+# This file specifies constraint/requirement URLs for current and python2
+# openstack branches
+
+# Current/stable
+STABLE_OPENSTACK_REQ_URL="https://raw.githubusercontent.com/openstack/requirements/stable/ussuri"
+# Current/experimental (for dev images)
+MASTER_OPENSTACK_REQ_URL="https://raw.githubusercontent.com/openstack/requirements/master"
+
+# Python2/stable
+STABLE_OPENSTACK_REQ_URL_PY2="https://opendev.org/openstack/requirements/raw/commit/2da5c5045118b0e36fb14427872e4b9b37335071"
+# Python2/experimental (for dev images)
+MASTER_OPENSTACK_REQ_URL_PY2="https://raw.githubusercontent.com/openstack/requirements/stable/train"
+

build-tools/build_iso/image-dev.inc

@@ -2,3 +2,5 @@
 #
 # They are exceptional packages only to be included in developer builds
 enable-dev-patch
+fio
+dstat

build-tools/build_iso/image.inc

@@ -81,3 +81,7 @@ kata-runtime
 
 # For nvme disk firmware update
 nvme-cli
+
+# Add openscap tools
+openscap
+openscap-scanner

build-tools/create-yum-conf

@@ -63,11 +63,16 @@ if [ ! -f "$MY_YUM_CONF" ]; then
         mock_cfg_to_yum_conf.py "$MOCK_CFG_PROTO"      > "$MY_YUM_CONF"
         sed -i "s%\[main\]%&\ncachedir=$YUM_CACHE%"      "$MY_YUM_CONF"
         sed -i "s%logfile=.*%logfile=$YUM_DIR/yum.log%"  "$MY_YUM_CONF"
+        # eg: LOCAL_BASE/MY_BUILD_DIR => file:///MY_BUILD_DIR
         sed -i "s%LOCAL_BASE%file://%g"                  "$MY_YUM_CONF"
         sed -i "s%MIRROR_BASE%file:///import/mirrors%g"  "$MY_YUM_CONF"
         sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g"     "$MY_YUM_CONF"
+        # eg: file:///MY_BUILD_DIR => file:///localdisk/loadbuild/...
         sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR%g"         "$MY_YUM_CONF"
         sed -i "s%/MY_REPO_DIR%$MY_REPO%g"               "$MY_YUM_CONF"
+        # eg = MY_BUILD_DIR/xyz => /localdisk/loadbuild/.../xyz
+        sed -i "s%MY_BUILD_DIR%$MY_BUILD_DIR%g"          "$MY_YUM_CONF"
+        sed -i "s%MY_REPO_DIR%$MY_REPO%g"                "$MY_YUM_CONF"
     else
         echo "ERROR: Could not find yum.conf or MOCK_CFG_PROTO"
         exit 1

build-tools/modify-build-cfg

@@ -83,11 +83,16 @@ if [ ! -f $FILE ]; then
       exit 1
    fi
 
+   # eg: LOCAL_BASE/MY_BUILD_DIR => http://127.0.0.1:8088/MY_BUILD_DIR
    sed -i "s%LOCAL_BASE%http://127.0.0.1:8088%g"  "$FILE"
    sed -i "s%MIRROR_BASE%http://127.0.0.1:8088%g" "$FILE"
    sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g"   "$FILE"
+   # eg http://127.0.0.1:8088/MY_BUILD_DIR => http://12.0.0.1:8088/localdisk/loadbuild/...
    sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR_TOP%g"   "$FILE"
    sed -i "s%/MY_REPO_DIR%$MY_REPO%g"             "$FILE"
+   # eg = MY_BUILD_DIR/xyz => /localdisk/loadbuild/.../xyz
+   sed -i "s%MY_BUILD_DIR%$MY_BUILD_DIR_TOP%g"    "$FILE"
+   sed -i "s%MY_REPO_DIR%$MY_REPO%g"              "$FILE"
 
    # Disable all local-* repos for the build-types other than the current one
    for bt in std rt; do

build-tools/repo-utils.sh

@@ -140,6 +140,50 @@ repo_is_project () {
 }
 
 
+#
+# manifest_get_revision_of_project <manifest> <project-name>
+#
+# Extract the revision of a project within the manifest.
+# The default revision is supplied in the absence
+# of an explicit project revision.
+#
+#    manifest = Path to manifest.
+#    project-name = name of project.
+#
+manifest_get_revision_of_project () {
+    local manifest="${1}"
+    local project="${2}"
+
+    local default_revision=""
+    local revision=""
+
+    default_revision=$(manifest_get_default_revision "${manifest}")
+    revision=$(grep '<project' "${manifest}" | \
+                grep -e "name=${project}" -e "name=\"${project}\"" | \
+                grep 'revision=' | \
+                sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g")
+    if [ "${revision}" != "" ]; then
+        echo "${revision}"
+    elif [ "${default_revision}" != "" ]; then
+        echo "${default_revision}"
+    else
+        return 1
+    fi
+}
+
+#
+# manifest_get_default_revision <manifest>
+#
+# Extract the default revision of the manifest, if any.
+#
+#    manifest = Path to manifest.
+#
+manifest_get_default_revision () {
+    local manifest="${1}"
+
+    grep '<default' $manifest |sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g"
+}
+
 #
 # manifest_set_revision <old_manifest> <new_manifest> <revision> <lock_down> <project-list>
 #
@@ -149,8 +193,10 @@ repo_is_project () {
 #    revision     = A branch, tag ,or sha.  Branch and SHA can be used
 #                   directly, but repo requires that a tag be in the form
 #                   "refs/tags/<tag-name>".
-#    lock_down    = 0 or 1.  If 1, set a revision on all other non-listed
+#    lock_down    = 0,1 or 2.  If 2, set a revision on all other non-listed
 #                   projects to equal the SHA of the current git head.
+#                   If 1, similar to 2, but only if the project doesn't have
+#                   some other form of revision specified.
 #    project-list = A space seperated list of projects.  Listed projects
 #                   will have their revision set to the provided revision
 #                   value.
@@ -160,9 +206,11 @@ manifest_set_revision () {
     local new_manifest="${2}"
     local revision="${3}"
     local lock_down="${4}"
-    shift 4
+    local set_default="${5}"
+    shift 5
     local projects="${@}"
 
+    local old_default_revision=""
     local repo_root_dir=""
     local line=""
     local FOUND=0
@@ -192,11 +240,32 @@ manifest_set_revision () {
         return 1
     fi
 
+    old_default_revision=$(manifest_get_default_revision "${old_manifest}")
+    if [ ${set_default} -eq 1 ] && [ "${old_default_revision}" == "" ]; then
+        # We only know how to alter an existing default revision, not set a
+        # new one, so continue without setting a default.
+        set_default=0
+    fi
+
     while IFS= read -r line; do
         echo "${line}" | grep -q '<project'
         if [ $? -ne 0 ]; then
-            # Line does not define a project, do not modify
-            echo "${line}"
+            # Line does not define a project
+            if [ ${set_default} -eq 0 ] || [ "${old_default_revision}" == "" ]; then
+                # No further processing, do not modify
+                echo "${line}"
+                continue
+            fi
+
+            # ok setting the default
+            echo "${line}" | grep -q '<default'
+            if [ $? -ne 0 ]; then
+                # Line does not set defaults, do not modify
+                echo "${line}"
+                continue
+            fi
+
+            echo "${line}" | sed "s#${old_default_revision}#${revision}#"
             continue
         fi
 
@@ -211,15 +280,58 @@ manifest_set_revision () {
         done
 
         rev=${revision}
+        old_rev=$(echo "${line}" | grep 'revision=' | sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g")
         if [ $FOUND -eq 0 ]; then
             # A non-selected project
-            if [ ${lock_down} -eq 0 ]; then
-                echo "${line}"
+            if [ ${lock_down} -eq 2 ]; then
+                # Hard lock-down
+                # Set the revision to current HEAD SHA.
+                path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
+                rev=$(cd "${path}"; git rev-parse HEAD)
+            elif [ ${lock_down} -eq 1 ] && [ "${old_rev}" == "" ]; then
+                # Soft lock-down but no revision is currently set on the project.
+                # Set the revision to current HEAD SHA.
+                path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
+                rev=$(cd "${path}"; git rev-parse HEAD)
+            elif [ ${lock_down} -eq 1 ] && [ "${old_rev}" == "master" ]; then
+                # Soft lock-down and project has revision set to 'master' which is definitly unstable.
+                # Set the revision to current HEAD SHA.
+                path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
+                rev=$(cd "${path}"; git rev-parse HEAD)
+            else
+                if [ ${set_default} -eq 0 ] || [ "${old_default_revision}" == "${revision}" ]; then
+                    # default revision unchanged, leave it be
+                    echo "${line}"
+                    continue
+                fi
+
+                if [ "${old_rev}" != "" ]; then
+                    # Non-selected project has an explicit revision, leave it be
+                    echo "${line}"
+                    continue
+                fi
+
+                # The default revision will change, but this project, which
+                # relied on the old default, is not supposed to change,
+                # so it's revision must now be explicitly set to point to
+                # the old default revision.
+                rev="${old_default_revision}"
+            fi
+        else
+            # A selected project
+            if [ ${set_default} -eq 1 ]; then
+                # Selected project does not need to set a revision.
+                # The revision will come from the default
+                if [ "${old_rev}" == "" ]; then
+                    # project has no revision to delete
+                    echo "${line}"
+                    continue
+                fi
+
+                # delete any revision present
+                echo "${line}" | sed 's#\(.*\)revision=[^ ]*\(.*\)#\1\2#'
                 continue
             fi
-
-            path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
-            rev=$(cd "${path}"; git rev-parse HEAD)
         fi
 
         # Need to set revision on selected project

build-tools/repo_files/mock.cfg.centos7.all.proto

@@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
 config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
 config_opts['releasever'] = '7'
 config_opts['package_manager'] = 'yum'
+config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
 config_opts['use_bootstrap'] = False
 config_opts['use_bootstrap_image'] = False
 config_opts['rpmbuild_networking'] = False

build-tools/repo_files/mock.cfg.centos7.distro.proto

@@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
 config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
 config_opts['releasever'] = '7'
 config_opts['package_manager'] = 'yum'
+config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
 config_opts['use_bootstrap'] = False
 config_opts['use_bootstrap_image'] = False
 config_opts['rpmbuild_networking'] = False

build-tools/repo_files/mock.cfg.centos7.proto

@@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
 config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
 config_opts['releasever'] = '7'
 config_opts['package_manager'] = 'yum'
+config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
 config_opts['use_bootstrap'] = False
 config_opts['use_bootstrap_image'] = False
 config_opts['rpmbuild_networking'] = False

build-tools/update-pxe-network-installer

@@ -100,6 +100,7 @@ find_and_copy_rpm 'e1000e kernel module'             'kmod-e1000e-[0-9]*.x86_64.
 find_and_copy_rpm 'i40e kernel module'               'kmod-i40e-[0-9]*.x86_64.rpm'                  std "$kernel_rpms_std"
 find_and_copy_rpm 'ixgbe kernel module'              'kmod-ixgbe-[0-9]*.x86_64.rpm'                 std "$kernel_rpms_std"
 find_and_copy_rpm 'mlnx-ofa kernel module'           'mlnx-ofa_kernel-modules-[0-9]*.x86_64.rpm'    std "$kernel_rpms_std"
+find_and_copy_rpm 'ice kernel module'                'kmod-ice-[0-9]*.x86_64.rpm'                   std "$kernel_rpms_std"
 echo " -------- successfully found standard kernel rpm and related kernel modules --------"
 echo ""
 

build-tools/url_utils.sh

@@ -0,0 +1,267 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# A set of bash utility functions to parse a URL.
+# This script was originated by Scott Little
+#
+
+url_protocol () {
+    local URL="$1"
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    if echo "$URL" | grep -q '[:][/][/]' ;then
+        echo "$URL" | sed 's#^\(.*\)://.*$#\1#'
+    else
+        echo "http"
+    fi
+    return 0
+}
+
+url_login () {
+    local URL="$1"
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    echo "$URL" | sed 's#^.*://\([^/]*\)/.*$#\1#'
+    return 0
+}
+
+url_user () {
+    local URL="$1"
+    local LOGIN
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    url_login "$URL" | sed -e '/@/! s#.*## ; s#\([^@]*\)@.*#\1#'
+    if [ ${PIPESTATUS[0]} -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
+        return 1
+    fi
+
+    return 0
+}
+
+url_port () {
+    local URL="$1"
+    local LOGIN
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    url_login "$URL" | sed -e '/:/! s#.*## ; s#[^:]*:\([^:]*\)#\1#'
+    if [ ${PIPESTATUS[0]} -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
+        return 1
+    fi
+
+    return 0
+}
+
+url_server () {
+    local URL="$1"
+    local LOGIN
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    url_login "$URL" | sed 's#^.*@## ; s#:.*$##'
+    if [ ${PIPESTATUS[0]} -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
+        return 1
+    fi
+
+    return 0
+}
+
+url_path () {
+    local URL="$1"
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    echo "$URL" | sed 's#^.*://[^/]*/\(.*\)$#\1#'
+    return 0
+}
+
+#
+# url_path_to_fs_path:
+#
+# Convert url format path to file system format.
+# e.g. replace %20 with ' '.
+#
+# Note: Does NOT test the output path to ensure there are
+#       no illegal file system characters.
+#
+url_path_to_fs_path () {
+    local INPUT_PATH="$1"
+    local TEMP
+
+    if [ "$INPUT_PATH" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    # Deviate from URI spec by not substituding '+' with ' '.
+    # It would alias '%20' and we need unique mappings.
+    # TEMP="${INPUT_PATH//+/ }"
+
+    TEMP="$INPUT_PATH"
+    printf '%b' "${TEMP//%/\\x}"
+    return 0
+}
+
+#
+# fs_path_to_url_path:
+#
+# Convert file system format path to url format.
+# e.g. replace ' ' with %20.
+#
+fs_path_to_url_path () {
+    local INPUT_PATH="$1"
+    local LENGTH
+    local POS
+    local CHAR
+
+    if [ "$INPUT_PATH" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    LENGTH="${#INPUT_PATH}"
+    for (( POS = 0; POS < LENGTH; POS++ )); do
+        CHAR="${1:POS:1}"
+        case $CHAR in
+            [/a-zA-Z0-9.~_-])
+                # Reference https://metacpan.org/pod/URI::Escape
+                printf "$CHAR"
+                ;;
+            *)
+                printf '%%%02X' "'$CHAR"
+                ;;
+        esac
+    done
+
+    return 0
+}
+
+#
+# normalize_path:
+#
+# 1) replace // with /
+# 2) replace /./ with /
+# 3) Remove trailing /
+# 4) Remove leading ./
+#
+
+normalize_path () {
+    local INPUT_PATH="$1"
+
+    if [ "$INPUT_PATH" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    echo "$INPUT_PATH" | sed 's#[/]\+#/#g ; s#[/][.][/]#/#g ; s#/$## ; s#^[.]/##'
+    return 0
+}
+
+
+#
+# repo_url_to_sub_path:
+#
+repo_url_to_sub_path () {
+    local URL="$1"
+    local FAMILY=""
+    local SERVER=""
+    local URL_PATH=""
+    local FS_PATH=""
+
+    if [ "$URL" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    # set FAMILY from URL
+    echo $URL | grep -q 'centos[.]org' && FAMILY=centos
+    echo $URL | grep -q 'fedoraproject[.]org[/]pub[/]epel' && FAMILY=epel
+
+    SERVER=$(url_server "$URL")
+    if [ $? -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_server '$URL'"
+        return 1
+    fi
+
+    URL_PATH="$(url_path "$URL")"
+    if [ $? -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_path '$URL'"
+        return 1
+    fi
+
+    FS_PATH="$(url_path_to_fs_path "$URL_PATH")"
+    if [ $? -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): url_path_to_fs_path '$URL_PATH'"
+        return 1
+    fi
+
+    FS_PATH="$(normalize_path "$FS_PATH")"
+    if [ $? -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path '$FS_PATH'"
+        return 1
+    fi
+
+    normalize_path "./$FAMILY/$SERVER/$FS_PATH"
+    if [ $? -ne 0  ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path './$FAMILY/$SERVER/$FS_PATH'"
+        return 1
+    fi
+
+    return 0
+}
+
+CENGN_PROTOCOL="http"
+CENGN_HOST="mirror.starlingx.cengn.ca"
+CENGN_PORT="80"
+CENGN_URL_ROOT="mirror"
+
+url_to_stx_mirror_url () {
+    local URL="$1"
+    local DISTRO="$2"
+    local URL_PATH=""
+    local FS_PATH=""
+
+    if [ "$URL" == "" ] || [ "$DISTRO" == "" ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
+        return 1
+    fi
+
+    FS_PATH="$(repo_url_to_sub_path "$URL")"
+    if [ $? -ne 0 ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): repo_url_to_sub_path '$URL'"
+        return 1
+    fi
+
+    URL_PATH=$(fs_path_to_url_path "$FS_PATH")
+    if [ $? -ne 0 ]; then
+        >&2 echo "Error: $FUNCNAME (${LINENO}): fs_path_to_url_path '$FS_PATH'"
+        return 1
+    fi
+
+    echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH"
+    return 0
+}

build-tools/yum-builddep-wrapper

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Old versions of yum-builddep leave a stale yum.pid file behind.
+# Remove that file if necessary after yum-builddep exits
+
+# find yum-builddep
+YUM_BUILDDEP=$(which yum-builddep 2>/dev/null)
+
+# dnf: call it directly
+if [[ -z $YUM_BUILDDEP ]] || grep -q -F dnf.cli "$YUM_BUILDDEP" ; then
+    yum-builddep "$@"
+    exit $?
+fi
+
+
+# old yum: scan command line for --installroot
+ROOT_PREFIX=
+YUM_CONF=/etc/yum.conf
+find_root_prefix() {
+    while [[ "$#" -gt 0 ]] ; do
+        case "$1" in
+            --installroot)
+                ROOT_PREFIX="$2"
+                shift
+                ;;
+            --installroot=*)
+                ROOT_PREFIX="${1#*=}"
+                ;;
+            -c|--config)
+                YUM_CONF="$2"
+                shift
+                ;;
+            --config=*)
+                YUM_CONF="${1#*=}"
+                ;;
+        esac
+        shift
+    done
+    if [[ -z "$ROOT_PREFIX" ]] && [[ -f "$YUM_CONF" ]] ; then
+        ROOT_PREFIX=$(sed -rn 's/^\s*installroot\s*=\s*(\S+)\s*$/\1/p' $YUM_CONF)
+    fi
+}
+find_root_prefix "$@"
+
+# ignore signals -- always wait for yum-builddep
+trap "" INT TERM HUP PIPE
+
+# run it in the background to get its PID
+"$YUM_BUILDDEP" "$@" &
+pid="$!"
+
+# wait for it
+wait "$pid"
+res="$?"
+
+# if yum.pid remains and contains yum-builddep's PID, delete it
+if [[ -f "${ROOT_PREFIX}/run/yum.pid" ]] ; then
+    lock_owner=
+    read lock_owner <"${ROOT_PREFIX}/run/yum.pid" || :
+    if [[ -n $lock_owner && $lock_owner == $pid ]] ; then
+        rm -f "${ROOT_PREFIX}/run/yum.pid"
+    fi
+fi
+
+# done
+exit $res