starlingx
/
root
Code Issues Proposed changes

Merge remote-tracking branch 'origin/master' into f/centos8 

Signed-off-by: Charles Short <charles.short@windriver.com>
Change-Id: I6b79402dd076ea2d4cd3ba52ae66926fcb29f759
This commit is contained in:
Charles Short 2021-05-19 14:13:29 -04:00
parent 13d213dc46 99be4a94c7
commit 7cb55d2811
29 changed files with 1284 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
build-tools/branching/create_branches_and_tags.sh
View File

@ -24,7 +24,7 @@ CREATE_BRANCHES_AND_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )"
source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh" source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh"
usage () { usage () {
echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest [ --lock-down ]]" echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --gitreview-default ] [ --manifest [ --lock-down | --soft-lock-down ] [ --default-revision ]]"
echo "" echo ""
echo "Create a branch and a tag in all listed projects, and all" echo "Create a branch and a tag in all listed projects, and all"
echo "projects hosted by all listed remotes. Lists are comma separated." echo "projects hosted by all listed remotes. Lists are comma separated."
@ -33,13 +33,19 @@ usage () {
echo "If the tag is omitted, one is automativally generate by adding the" echo "If the tag is omitted, one is automativally generate by adding the"
echo "prefix 'v' to the branch name." echo "prefix 'v' to the branch name."
echo "" echo ""
echo "If a manifest is requested, it will recieve the name '<branch>.xml' and" echo "If a manifest is requested, the current manifest is modified."
echo "it will specify the branch as the revision for all tagged projects." echo "to specify the new branch for all select remotes and projects."
echo "If lockdown is requested, all other projects get the current" echo "If lockdown is requested, all other projects get the current"
echo "HEAD's sha set as the revision." echo "HEAD's sha set as the revision."
echo "If default-revision is selected, then the manifest default revision"
wcho "will be set."
echo ""
echo "If a gitreview-default is selected, then all branched projects"
echo "with a .gitreview file will have a defaultbranch entry added"
echo "or updated."
} }
TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,help -n 'create_branches_and_tags.sh' -- "$@") TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,help -n 'create_branches_and_tags.sh' -- "$@")
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
usage usage
exit 1 exit 1
@ -49,6 +55,8 @@ eval set -- "$TEMP"
HELP=0 HELP=0
MANIFEST=0 MANIFEST=0
LOCK_DOWN=0 LOCK_DOWN=0
GITREVIEW_DEFAULT=0
SET_DEFAULT_REVISION=0
remotes="" remotes=""
projects="" projects=""
branch="" branch=""
@ -65,7 +73,11 @@ while true ; do
--branch) branch=$2; shift 2;; --branch) branch=$2; shift 2;;
--tag) tag=$2; shift 2;; --tag) tag=$2; shift 2;;
--manifest) MANIFEST=1 ; shift ;; --manifest) MANIFEST=1 ; shift ;;
--lock-down) LOCK_DOWN=1 ; shift ;; --lock-down) LOCK_DOWN=2 ; shift ;;
--hard-lock-down) LOCK_DOWN=2 ; shift ;;
--soft-lock-down) LOCK_DOWN=1 ; shift ;;
--default-revision) SET_DEFAULT_REVISION=1 ; shift ;;
--gitreview-default) GITREVIEW_DEFAULT=1 ; shift ;;
--) shift ; break ;; --) shift ; break ;;
*) usage; exit 1 ;; *) usage; exit 1 ;;
esac esac
@ -88,6 +100,37 @@ if [ $? -ne 0 ]; then
exit 1 exit 1
fi fi
update_gitreview () {
local DIR=$1
(
cd $DIR || exit 1
if [ $GITREVIEW_DEFAULT -eq 1 ] && [ -f .gitreview ]; then
if ! grep -q "^defaultbranch=$branch$" .gitreview; then
echo "Updating defaultbranch in ${DIR}/.gitreview"
if grep -q defaultbranch= .gitreview; then
sed "s#\(defaultbranch=\).*#\1$branch#" -i .gitreview
else
echo "defaultbranch=$branch" >> .gitreview
fi
git add .gitreview
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to add .gitreview in ${DIR}"
exit 1
fi
git commit -s -m "Update .gitreview for $branch"
if [ $? != 0 ] ; then
echo_stderr "ERROR: failed to commit .gitreview in ${DIR}"
exit 1
fi
else
echo "defaultbranch in ${DIR}/.gitreview already set"
fi
fi
)
}
if [ $MANIFEST -eq 1 ]; then if [ $MANIFEST -eq 1 ]; then
manifest=$(repo_manifest $repo_root_dir) manifest=$(repo_manifest $repo_root_dir)
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
@ -205,6 +248,7 @@ for subgit in $SUBGITS; do
git checkout $branch git checkout $branch
fi fi
# check if destination tag already exists
tag_check=$(git tag -l $tag) tag_check=$(git tag -l $tag)
if [ -z "$tag_check" ]; then if [ -z "$tag_check" ]; then
echo "Creating tag $tag in ${subgit}" echo "Creating tag $tag in ${subgit}"
@ -216,6 +260,8 @@ for subgit in $SUBGITS; do
else else
echo "Tag '$tag' already exists in ${subgit}" echo "Tag '$tag' already exists in ${subgit}"
fi fi
update_gitreview ${subgit} || exit 1
) || exit 1 ) || exit 1
done done
) || exit 1 ) || exit 1
@ -276,8 +322,10 @@ if [ $MANIFEST -eq 1 ]; then
exit 1 exit 1
fi fi
update_gitreview ${manifest_dir} || exit 1
echo "Creating manifest ${new_manifest_name}" echo "Creating manifest ${new_manifest_name}"
manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} $projects || exit 1 manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} ${SET_DEFAULT_REVISION} $projects || exit 1
echo "Move manifest ${new_manifest_name}, overwriting ${manifest_name}" echo "Move manifest ${new_manifest_name}, overwriting ${manifest_name}"
\cp -f "${manifest}" "${manifest}.save" \cp -f "${manifest}" "${manifest}.save"

74
build-tools/branching/push_branches_tags.sh
View File

@ -17,6 +17,7 @@
PUSH_BRANCHES_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" PUSH_BRANCHES_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh" source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh"
source "${PUSH_BRANCHES_TAGS_SH_DIR}/../url_utils.sh"
usage () { usage () {
echo "push_branches_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest ]" echo "push_branches_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest ]"
@ -151,6 +152,8 @@ for subgit in $SUBGITS; do
( (
cd $subgit cd $subgit
git fetch --all
branch_check=$(git branch -a --list $branch) branch_check=$(git branch -a --list $branch)
if [ -z "$branch_check" ]; then if [ -z "$branch_check" ]; then
echo_stderr "ERROR: Expected branch '$branch' to exist in ${subgit}" echo_stderr "ERROR: Expected branch '$branch' to exist in ${subgit}"
@ -169,21 +172,49 @@ for subgit in $SUBGITS; do
exit 1 exit 1
fi fi
if [ "${review_method}" == "gerrit" ]; then remote=$(git_remote)
remote=$(git_repo_review_remote) if [ "${remote}" == "" ]; then
else echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}"
remote=$(git_repo_remote) exit 1
fi fi
if [ "${remote}" == "" ]; then if [ "${review_method}" == "gerrit" ]; then
echo_stderr "ERROR: Failed to determine remote in ${subgit}" review_remote=$(git_repo_review_remote)
else
review_remote=${remote}
fi
if [ "${review_remote}" == "" ]; then
echo_stderr "ERROR: Failed to determine review_remote in ${subgit}"
exit 1 exit 1
fi fi
branch_check=$(git branch -a --list $remote/$branch)
if [ "${branch_check}" != "" ]; then
echo "Branch $branch already exists in ${subgit}"
exit 0
fi
echo "Pushing branch $branch in ${subgit}" echo "Pushing branch $branch in ${subgit}"
if [ "${review_method}" == "gerrit" ]; then if [ "${review_method}" == "gerrit" ]; then
url=$(git_repo_review_url)
if [ "${review_remote}" == "" ]; then
echo_stderr "ERROR: Failed to determine review_url in ${subgit}"
exit 1
fi
host=$(url_server "${url}")
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ]; then
git push ${review_remote} ${tag} && \
ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
git review --topic="${branch}"
else
echo "git push --tags ${remote} ${branch}" echo "git push --tags ${remote} ${branch}"
git push --tags ${remote} ${branch} git push --tags ${remote} ${branch}
fi
else else
echo "git push --tags --set-upstream ${remote} ${branch}" echo "git push --tags --set-upstream ${remote} ${branch}"
git push --tags --set-upstream ${remote} ${branch} git push --tags --set-upstream ${remote} ${branch}
@ -232,23 +263,44 @@ if [ $MANIFEST -eq 1 ]; then
exit 1 exit 1
fi fi
remote=$(git_remote)
remote=$(git_review_remote)
if [ "${remote}" == "" ]; then if [ "${remote}" == "" ]; then
echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}" echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}"
exit 1 exit 1
fi fi
review_remote=$(git_review_remote)
if [ "${review_remote}" == "" ]; then
echo_stderr "ERROR: Failed to determine review_remote in ${manifest_dir}"
exit 1
fi
echo "Pushing branch $branch in ${manifest_dir}" echo "Pushing branch $branch in ${manifest_dir}"
if [ "${review_method}" == "gerrit" ]; then if [ "${review_method}" == "gerrit" ]; then
# Is a reviewless push possible as part of creating a new branch in gerrit? # Is a reviewless push possible as part of creating a new branch in gerrit?
git push --tags ${remote} ${branch} url=$(git_review_url)
if [ "${review_remote}" == "" ]; then
echo_stderr "ERROR: Failed to determine review_url in ${subgit}"
exit 1
fi
host=$(url_server "${url}")
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ]; then
git push ${review_remote} ${tag} && \
ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
git review --yes --topic="${branch}"
else else
git push --tags --set-upstream ${remote} ${branch} git push --tags ${review_remote} ${branch}
fi
else
git push --tags --set-upstream ${review_remote} ${branch}
fi fi
if [ $? != 0 ] ; then if [ $? != 0 ] ; then
echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${remote}' in ${manifest_dir}" echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${review_remote}' in ${manifest_dir}"
exit 1 exit 1
fi fi
) || exit 1 ) || exit 1

256
build-tools/build-docker-images/build-stx-images.sh
View File

@ -26,7 +26,6 @@ IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
PREFIX=dev PREFIX=dev
LATEST_PREFIX="" LATEST_PREFIX=""
PUSH=no PUSH=no
CONFIG_FILE=""
HTTP_PROXY="" HTTP_PROXY=""
HTTPS_PROXY="" HTTPS_PROXY=""
NO_PROXY="" NO_PROXY=""
@ -34,16 +33,13 @@ DOCKER_USER=${USER}
DOCKER_REGISTRY= DOCKER_REGISTRY=
BASE= BASE=
WHEELS= WHEELS=
WHEELS_ALTERNATE= WHEELS_PY2=
DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images"
DEFAULT_CONFIG_FILE_PREFIX="docker-image-build"
CLEAN=no CLEAN=no
TAG_LATEST=no TAG_LATEST=no
TAG_LIST_FILE= TAG_LIST_FILE=
TAG_LIST_LATEST_FILE= TAG_LIST_LATEST_FILE=
declare -a ONLY declare -a ONLY
declare -a SKIP declare -a SKIP
declare -a SERVICES_ALTERNATE
declare -i MAX_ATTEMPTS=1 declare -i MAX_ATTEMPTS=1
function usage { function usage {
@ -56,8 +52,11 @@ Options:
--version: Specify version for output image --version: Specify version for output image
--stream: Build stream, stable or dev (default: stable) --stream: Build stream, stable or dev (default: stable)
--base: Specify base docker image (required option) --base: Specify base docker image (required option)
--wheels: Specify path to wheels tarball or image, URL or docker tag (required option) --wheels: Specify path to wheels tarball or image, URL or docker tag
--wheels-alternate: Specify path to alternate wheels tarball or image, URL or docker tag (required when building loci projects)
--wheels-py2: Use this wheels tarball for Python2 projects
(default: work out from --wheels)
--wheels-alternate: same as --wheels-py2
--push: Push to docker repo --push: Push to docker repo
--http_proxy: Set proxy <URL>:<PORT>, urls splitted with "," --http_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
--https_proxy: Set proxy <URL>:<PORT>, urls splitted with "," --https_proxy: Set proxy <URL>:<PORT>, urls splitted with ","
@ -75,7 +74,6 @@ Options:
can be specified with a comma-separated list, or with can be specified with a comma-separated list, or with
multiple --skip arguments. multiple --skip arguments.
--attempts: Max attempts, in case of failure (default: 1) --attempts: Max attempts, in case of failure (default: 1)
--config-file:Specify a path to a config file which will specify additional arguments to be passed into the the command
EOF EOF
@ -97,63 +95,6 @@ function is_empty {
test $# -eq 0 test $# -eq 0
} }
function get_args_from_file {
# get additional build args from specified file.
local -a config_items
echo "Get args from file: $1"
for i in $(cat $1)
do
config_items=($(echo $i | sed s/=/\ /g))
echo "--${config_items[0]} ${config_items[1]}"
case ${config_items[0]} in
base)
if [ -z "${BASE}" ]; then
BASE=${config_items[1]}
fi
;;
user)
if [ -z "${DOCKER_USER}" ]; then
DOCKER_USER=${config_items[1]}
fi
;;
proxy)
if [ -z "${PROXY}" ]; then
PROXY=${config_items[1]}
fi
;;
registry)
if [ -z "${DOCKER_REGISTRY}" ]; then
# Add a trailing / if needed
DOCKER_REGISTRY="${config_items[1]%/}/"
fi
;;
only)
# Read comma-separated values into array
if [ -z "${ONLY}" ]; then
# Read comma-separated values into array
ONLY=(`echo ${config_items[1]} | sed s/,/\ /g`)
fi
;;
wheels)
if [ -z "${WHEELS}" ]; then
WHEELS=${config_items[1]}
fi
;;
wheels_alternate)
if [ -z "${WHEELS_ALTERNATE}" ]; then
WHEELS_ALTERNATE=${config_items[1]}
echo "WHEELS_ALTERNATE: ${WHEELS_ALTERNATE}" >&2
fi
;;
services_alternate)
SERVICES_ALTERNATE=(`echo ${config_items[1]} | sed s/,/\ /g`)
echo "SERVICES_ALTERNATE: ${SERVICES_ALTERNATE[@]}" >&2
;;
esac
done
}
# #
# get_git: Clones a git into a subdirectory of ${WORKDIR}, and # get_git: Clones a git into a subdirectory of ${WORKDIR}, and
# leaves you in that directory. On error the directory # leaves you in that directory. On error the directory
@ -393,27 +334,54 @@ function build_image_loci {
PROFILES=$(source ${image_build_file} && echo ${PROFILES}) PROFILES=$(source ${image_build_file} && echo ${PROFILES})
local PYTHON3 local PYTHON3
PYTHON3=$(source ${image_build_file} && echo ${PYTHON3}) PYTHON3=$(source ${image_build_file} && echo ${PYTHON3})
local MIRROR_LOCAL
if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then MIRROR_LOCAL=$(source ${image_build_file} && echo ${MIRROR_LOCAL})
echo "Skipping ${LABEL}"
return 0
fi
if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
echo "Skipping ${LABEL}"
return 0
fi
echo "Building ${LABEL}" echo "Building ${LABEL}"
local ORIGWD=${PWD}
if [ "${MIRROR_LOCAL}" = "yes" ]; then
# Setup a local mirror of PROJECT_REPO
local BARE_CLONES=${WORKDIR}/bare_clones
mkdir -p ${BARE_CLONES}
if [ $? -ne 0 ]; then
echo "Failed to create ${BARE_CLONES}" >&2
RESULTS_FAILED+=(${LABEL})
return 1
fi
local CLONE_DIR=${BARE_CLONES}/${PROJECT}.git
# Remove prior clone dir, if it exists
\rm -rf ${CLONE_DIR}
echo "Creating bare clone of ${PROJECT_REPO} for ${LABEL} build..."
git clone --bare ${PROJECT_REPO} ${CLONE_DIR} \
&& mv ${CLONE_DIR}/hooks/post-update.sample ${CLONE_DIR}/hooks/post-update \
&& chmod a+x ${CLONE_DIR}/hooks/post-update \
&& cd ${CLONE_DIR} \
&& git update-server-info \
&& cd ${ORIGWD}
if [ $? -ne 0 ]; then
echo "Failed to clone ${PROJECT_REPO}... Aborting ${LABEL} build"
RESULTS_FAILED+=(${LABEL})
cd ${ORIGWD}
return 1
fi
PROJECT_REPO=http://${HOSTNAME}:8088/${CLONE_DIR}
fi
local -a BUILD_ARGS= local -a BUILD_ARGS=
BUILD_ARGS=(--build-arg PROJECT=${PROJECT}) BUILD_ARGS=(--build-arg PROJECT=${PROJECT})
BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO}) BUILD_ARGS+=(--build-arg PROJECT_REPO=${PROJECT_REPO})
BUILD_ARGS+=(--build-arg FROM=${BASE}) BUILD_ARGS+=(--build-arg FROM=${BASE})
if is_in ${LABEL} ${SERVICES_ALTERNATE[@]}; then if [ "${PYTHON3}" != "yes" ] ; then
echo "Python2 service ${LABEL}" echo "Python2 service ${LABEL}"
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_ALTERNATE}) BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
else else
echo "Python3 service ${LABEL}" echo "Python3 service ${LABEL}"
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS}) BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
@ -518,16 +486,6 @@ function build_image_docker {
local DOCKER_PATCHES local DOCKER_PATCHES
DOCKER_PATCHES=$(source ${image_build_file} && for p in ${DOCKER_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done) DOCKER_PATCHES=$(source ${image_build_file} && for p in ${DOCKER_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done)
if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
echo "Skipping ${LABEL}"
return 0
fi
if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
echo "Skipping ${LABEL}"
return 0
fi
echo "Building ${LABEL}" echo "Building ${LABEL}"
local real_docker_context local real_docker_context
@ -625,16 +583,6 @@ function build_image_script {
local SOURCE_PATCHES local SOURCE_PATCHES
SOURCE_PATCHES=$(source ${image_build_file} && for p in ${SOURCE_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done) SOURCE_PATCHES=$(source ${image_build_file} && for p in ${SOURCE_PATCHES}; do echo $(dirname ${image_build_file})/${p}; done)
if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
echo "Skipping ${LABEL}"
return 0
fi
if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
echo "Skipping ${LABEL}"
return 0
fi
# Validate the COMMAND option # Validate the COMMAND option
SUPPORTED_COMMAND_ARGS=('bash') SUPPORTED_COMMAND_ARGS=('bash')
local VALID_COMMAND=1 local VALID_COMMAND=1
@ -715,7 +663,7 @@ function build_image {
esac esac
} }
OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@") OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,http_proxy:,https_proxy:,no_proxy:,user:,registry:,base:,wheels:,wheels-alternate:,wheels-py2:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@")
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
usage usage
exit 1 exit 1
@ -742,8 +690,8 @@ while true; do
WHEELS=$2 WHEELS=$2
shift 2 shift 2
;; ;;
--wheels-alternate) --wheels-alternate|--wheels-py2)
WHEELS_ALTERNATE=$2 WHEELS_PY2=$2
shift 2 shift 2
;; ;;
--version) --version)
@ -813,10 +761,6 @@ while true; do
MAX_ATTEMPTS=$2 MAX_ATTEMPTS=$2
shift 2 shift 2
;; ;;
--config-file)
CONFIG_FILE=$2
shift 2
;;
-h | --help ) -h | --help )
usage usage
exit 1 exit 1
@ -842,37 +786,80 @@ if [ ${VALID_OS} -ne 0 ]; then
exit 1 exit 1
fi fi
DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg"
# Read additional arguments from config file if it exists.
if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then
CONFIG_FILE=${DEFAULT_CONFIG_FILE}
fi
if [[ ! -z ${CONFIG_FILE} ]]; then
if [[ -f ${CONFIG_FILE} ]]; then
get_args_from_file ${CONFIG_FILE}
else
echo "Config file not found: ${CONFIG_FILE}"
exit 1
fi
fi
if [ -z "${WHEELS}" ]; then
echo "Path to wheels tarball must be specified with --wheels option." >&2
exit 1
fi
if [ ${#SERVICES_ALTERNATE[@]} -ne 0 ] && [ -z "${WHEELS_ALTERNATE}" ]; then
echo "Path to wheels-alternate tarball must be specified with --wheels-alternate option"\
"if python2 based services need to be build!" >&2
exit 1
fi
if [ -z "${BASE}" ]; then if [ -z "${BASE}" ]; then
echo "Base image must be specified with --base option." >&2 echo "Base image must be specified with --base option." >&2
exit 1 exit 1
fi fi
# Guess WHEELS_PY2 if missing
if [[ -z "$WHEELS_PY2" && -n "$WHEELS" ]]; then
# http://foo/bar.tar?xxx#yyy => http://foo/bar-py2.tar?xxx#yyy
WHEELS_PY2="$(echo "$WHEELS" | sed -r 's,^([^#?]*)(\.tar)(\.gz|\.bz2|\.xz)?([#?].*)?$,\1-py2\2\3\4,i')"
if [[ "$WHEELS" == "$WHEELS_PY2" ]]; then
echo "Unable to guess --wheels-py2, please specify it explicitly" >&2
exit 1
fi
fi
# Find the directives files
IMAGE_BUILD_FILES=()
function find_image_build_files {
local image_build_inc_file image_build_dir image_build_file
local -A all_labels
for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do
basedir=$(dirname ${image_build_inc_file})
for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do
for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do
# reset & read image build directive vars
local BUILDER=
local PROJECT=
local LABEL=
local PYTHON3=
PROJECT="$(source ${image_build_file} && echo ${PROJECT})"
BUILDER="$(source ${image_build_file} && echo ${BUILDER})"
LABEL="$(source ${image_build_file} && echo ${LABEL})"
PYTHON3="$(source ${image_build_file} && echo ${PYTHON3})"
# make sure labels are unique
if [[ -n "${all_labels["$LABEL"]}" ]] ; then
echo "The following files define the same LABEL $LABEL" >&2
echo " ${all_labels["$LABEL"]}" >&2
echo " ${image_build_file}" >&2
exit 1
fi
all_labels["$LABEL"]="$image_build_file"
# skip images we don't want to build
if is_in ${PROJECT} ${SKIP[@]} || is_in ${LABEL} ${SKIP[@]}; then
continue
fi
if ! is_empty ${ONLY[@]} && ! is_in ${PROJECT} ${ONLY[@]} && ! is_in ${LABEL} ${ONLY[@]}; then
continue
fi
# loci builders require a wheels tarball
if [[ "${BUILDER}" == "loci" ]] ; then
# python3 projects require $WHEELS
if [[ "${PYTHON3}" == "yes" && -z "${WHEELS}" ]] ; then
echo "You are building python3 services with loci, but you didn't specify --wheels!" >&2
exit 1
# python2 projects require WHEELS_PY2
elif [[ "${PYTHON3}" != "yes" && -z "${WHEELS_PY2}" ]] ; then
echo "You are building python2 services with loci, but you didn't specify --wheels-py2!" >&2
exit 1
fi
fi
# Save image build file in the global list
IMAGE_BUILD_FILES+=("$image_build_file")
done
done
done
}
find_image_build_files
IMAGE_TAG="${OS}-${BUILD_STREAM}" IMAGE_TAG="${OS}-${BUILD_STREAM}"
IMAGE_TAG_LATEST="${IMAGE_TAG}-latest" IMAGE_TAG_LATEST="${IMAGE_TAG}-latest"
@ -933,16 +920,11 @@ if ! (grep -q rh-python36-mod_wsgi ${WORKDIR}/loci/bindep.txt); then
echo 'rh-python36-mod_wsgi [platform:rpm !platform:suse (apache python3)]' >> ${WORKDIR}/loci/bindep.txt echo 'rh-python36-mod_wsgi [platform:rpm !platform:suse (apache python3)]' >> ${WORKDIR}/loci/bindep.txt
fi fi
# Find the directives files # Build everything
for image_build_inc_file in $(find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_docker_images.inc"); do for image_build_file in "${IMAGE_BUILD_FILES[@]}" ; do
basedir=$(dirname ${image_build_inc_file})
for image_build_dir in $(sed -e 's/#.*//' ${image_build_inc_file} | sort -u); do
for image_build_file in ${basedir}/${image_build_dir}/${OS}/*.${BUILD_STREAM}_docker_image; do
# Failures are reported by the build functions # Failures are reported by the build functions
build_image ${image_build_file} build_image ${image_build_file}
done done
done
done
if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then if [ "${CLEAN}" = "yes" -a ${#RESULTS_BUILT[@]} -gt 0 ]; then
# Delete the images # Delete the images

2
build-tools/build-docker-images/docker-image-build-centos-dev.cfg
View File

@ -1,2 +0,0 @@
services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients
wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_dev-wheels.tar

2
build-tools/build-docker-images/docker-image-build-centos-stable.cfg
View File

@ -1,2 +0,0 @@
services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients
wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_stable-wheels.tar

46
build-tools/build-docker-images/tag-management/image-tags.yaml
View File

@ -2,10 +2,10 @@
--- ---
images: images:
- name: docker.io/starlingx/k8s-cni-sriov - name: docker.io/starlingx/k8s-cni-sriov
src_build_tag: master-centos-stable-20191203T153530Z.0 src_build_tag: master-centos-stable-20210218T003113Z.0
src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 src_ref: https://opendev.org/starlingx/integ/commit/eccff3b0e661592084d9114a9a41816761e1f9b5
# Tag determined based on release tag associated with upstream commit # Version determined by running 'git describe --tags' in clone of upstream repo
tag: stx.4.0-v2.2 tag: stx.5.0-v2.6-7-gb18123d8
- name: docker.io/starlingx/k8s-plugins-sriov-network-device - name: docker.io/starlingx/k8s-plugins-sriov-network-device
src_build_tag: master-centos-stable-20200512T184214Z.0 src_build_tag: master-centos-stable-20200512T184214Z.0
src_ref: https://opendev.org/starlingx/integ/commit/e2dc5c2dd0042788697ade268ac5c24fe9dc2f8c src_ref: https://opendev.org/starlingx/integ/commit/e2dc5c2dd0042788697ade268ac5c24fe9dc2f8c
@ -37,10 +37,10 @@ images:
# Version determined by running 'git describe --tags' in clone of upstream repo # Version determined by running 'git describe --tags' in clone of upstream repo
tag: stx.4.0-v0.11.0-109-gc48c502 tag: stx.4.0-v0.11.0-109-gc48c502
- name: docker.io/starlingx/stx-oidc-client - name: docker.io/starlingx/stx-oidc-client
src_build_tag: master-centos-stable-20200901T001315Z.0 src_build_tag: master-centos-stable-20210119T015305Z.0
src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/957fc7c2092c7574a0a931012a0ec1cf5bb66429 src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/70147e64e910e9878dd5bdf464cfd9672894ba18
# StarlingX app. Setting version to v1.0.3 # StarlingX app. Setting version to v1.0.4
tag: stx.5.0-v1.0.3 tag: stx.5.0-v1.0.4
- name: docker.io/starlingx/dex - name: docker.io/starlingx/dex
src_build_tag: master-centos-stable-20200204T162546Z.0 src_build_tag: master-centos-stable-20200204T162546Z.0
src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/5d6701bdf214e77f460f2e3dd2b6f7d3186830c8 src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/5d6701bdf214e77f460f2e3dd2b6f7d3186830c8
@ -55,9 +55,9 @@ images:
src_ref: https://opendev.org/starlingx/metal/commit/d46c9c55a9a9b7ea09e8d0fe66c8cfbeeb9ac75f src_ref: https://opendev.org/starlingx/metal/commit/d46c9c55a9a9b7ea09e8d0fe66c8cfbeeb9ac75f
tag: stx.5.0-v1.0.0 tag: stx.5.0-v1.0.0
- name: docker.io/starlingx/stx-platformclients - name: docker.io/starlingx/stx-platformclients
src_build_tag: master-centos-stable-20200803T230630Z.0 src_build_tag: master-centos-stable-20210512T053357Z.0
src_ref: https://opendev.org/starlingx/distcloud-client/commit/7036f1fd11cd3bbae743aee89908e8195e4ded40 src_ref: https://opendev.org/starlingx/distcloud-client/commit/d52a9080082db5fda2e77fb9e342f812ea8c17e1
tag: stx.5.0-v1.4.0 tag: stx.5.0-v1.4.3
- name: docker.io/starlingx/stx-vault-manager - name: docker.io/starlingx/stx-vault-manager
src_build_tag: master-centos-stable-20200722T035334Z.0 src_build_tag: master-centos-stable-20200722T035334Z.0
src_ref: https://opendev.org/starlingx/vault-armada-app/commit/2cd206d6703cc2733e39ecad4539c0d5f1600550 src_ref: https://opendev.org/starlingx/vault-armada-app/commit/2cd206d6703cc2733e39ecad4539c0d5f1600550
@ -68,3 +68,27 @@ images:
src_ref: https://opendev.org/starlingx/portieris-armada-app/commit/a6123ffebb77f23d5182576be17e69d62fd8d701 src_ref: https://opendev.org/starlingx/portieris-armada-app/commit/a6123ffebb77f23d5182576be17e69d62fd8d701
# Tag based on upstream package version # Tag based on upstream package version
tag: stx.5.0-v0.7.0 tag: stx.5.0-v0.7.0
- name: docker.io/starlingx/stx-snmp
src_build_tag: master-centos-stable-20210105T023146Z.0
src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/2b370655a7f9a506ea139cfffa6c466d1a82cce4
tag: stx.5.0-v1.0.0
- name: docker.io/starlingx/stx-fm-subagent
src_build_tag: master-centos-stable-20210105T023146Z.0
src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/2b370655a7f9a506ea139cfffa6c466d1a82cce4
tag: stx.5.0-v1.0.0
- name: docker.io/starlingx/stx-fm-trap-subagent
src_build_tag: master-centos-stable-20210314T171252Z.0
src_ref: https://opendev.org/starlingx/snmp-armada-app/commit/5aca0dd1661bc87a7927c00cf95e0c8aa6f2e2a0
tag: stx.5.0-v1.0.1
- name: docker.io/starlingx/notificationservice-base
src_build_tag: master-centos-stable-20210504T193232Z.0
src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/eb4458e37ebe170f4c1289362f9cbc55fb1f32aa
tag: stx.5.0-v1.0.4
- name: docker.io/starlingx/locationservice-base
src_build_tag: master-centos-stable-20210204T224209Z.0
src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/545e6b6bb093235c2f8dab8d171f30c6ae8682d3
tag: stx.5.0-v1.0.1
- name: docker.io/starlingx/notificationclient-base
src_build_tag: master-centos-stable-20210503T050004Z.0
src_ref: https://opendev.org/starlingx/ptp-notification-armada-app/commit/6e87c185baf927b28d0bcff6e2763a1e62c8145e
tag: stx.5.0-v1.0.4

2
build-tools/build-img
View File

@ -566,7 +566,7 @@ if [ ${#RPM_ADDON_LIST[@]} -gt 0 ] ; then
pushd $MY_WORKSPACE pushd $MY_WORKSPACE
patch_file="PATCH.img-addon" patch_file="PATCH.img-addon"
patched_iso="$TEMPFILES_DIR/bootimage_${AUTO_MODE}${GRAPHICAL_SUFFIX}_patched.iso" patched_iso="$TEMPFILES_DIR/bootimage_${AUTO_MODE}${GRAPHICAL_SUFFIX}_patched.iso"
cmd=("$PATCH_BUILD" --id "${patch_file}" --summary "additional packages for qcow2 image" --desc "Adds customizations to qcow2 image") cmd=("$PATCH_BUILD" --id "${patch_file}" --summary "additional packages for qcow2 image" --desc "Adds customizations to qcow2 image" --status "REL" --reboot-required "N")
for rpm_addon in "${RPM_ADDON_LIST[@]}"; do for rpm_addon in "${RPM_ADDON_LIST[@]}"; do
cmd+=(--all-nodes "${rpm_addon}") cmd+=(--all-nodes "${rpm_addon}")
done done

44
build-tools/build-rpms-parallel
View File

@ -122,6 +122,10 @@ number_of_users () {
users | tr ' ' '\n' | sort --uniq | wc -l users | tr ' ' '\n' | sort --uniq | wc -l
} }
total_mem_gb () {
free -g | grep 'Mem:' | awk '{ print $2 }'
}
available_mem_gb () { available_mem_gb () {
free -g | grep 'Mem:' | awk '{ print $7 }' free -g | grep 'Mem:' | awk '{ print $7 }'
} }
@ -238,26 +242,41 @@ compute_resources () {
local users=$(number_of_users) local users=$(number_of_users)
if [ $users -lt 1 ]; then users=1; fi if [ $users -lt 1 ]; then users=1; fi
local mem=$(available_mem_gb) local mem=$(available_mem_gb)
local total_mem=$(total_mem_gb)
local disk=$(available_disk_gb) local disk=$(available_disk_gb)
local cpus=$(number_of_cpus) local cpus=$(number_of_cpus)
local num_users=$(sqrt $users) local num_users=$(sqrt $users)
local num_build=$(number_of_builds_in_progress) local num_build=$(number_of_builds_in_progress)
num_build=$((num_build+1)) num_build=$((num_build+1))
echo "compute_resources: total: cpus=$cpus, mem=$mem, disk=$disk, weight=$weight, num_build=$num_build" echo "compute_resources: total: cpus=$cpus, total_mem=$total_mem, avail_mem=$mem, disk=$disk, weight=$weight, num_build=$num_build"
# What fraction of the machine will we use # What fraction of the machine will we use
local share_factor=$num_users local share_factor=$num_users
if [ $share_factor -gt $((MAX_SHARE_FACTOR+num_build-1)) ]; then share_factor=$((MAX_SHARE_FACTOR+num_build-1)); fi if [ $share_factor -gt $((MAX_SHARE_FACTOR+num_build-1)) ]; then share_factor=$((MAX_SHARE_FACTOR+num_build-1)); fi
if [ $share_factor -lt $num_build ]; then share_factor=$num_build; fi if [ $share_factor -lt $num_build ]; then share_factor=$num_build; fi
local mem_share_factor=$((share_factor-num_build))
# What fraction of free memory can we use.
# e.g.
# We intend to support 4 concurrent builds (share_factor)
# Two builds (excluding ours) are already underway (num_build-1)
# So we should be able to support 2 more builds (mem_share_factor)
local mem_share_factor=$((share_factor-(num_build-1)))
if [ $mem_share_factor -lt 1 ]; then mem_share_factor=1; fi if [ $mem_share_factor -lt 1 ]; then mem_share_factor=1; fi
echo "compute_resources: share_factor=$share_factor mem_share_factor=$mem_share_factor" echo "compute_resources: share_factor=$share_factor mem_share_factor=$mem_share_factor"
# What resources are we permitted to use # What resources are we permitted to use
# Continuing the example from above ... memory share is the lesser of
# - Half the available memory (mem/mem_share_factor)
# - A quarter of the total memory (total_mem/share_factor)
local mem_share=$(((mem-MEMORY_RESERVE)/mem_share_factor)) local mem_share=$(((mem-MEMORY_RESERVE)/mem_share_factor))
if [ $mem_share -lt 0 ]; then mem_share=0; fi if [ $mem_share -lt 0 ]; then mem_share=0; fi
local total_mem_share=$(((total_mem-MEMORY_RESERVE)/share_factor))
if [ $total_mem_share -lt 0 ]; then total_mem_share=0; fi
if [ $mem_share -gt $total_mem_share ]; then mem_share=$total_mem_share; fi
local disk_share=$((disk/share_factor)) local disk_share=$((disk/share_factor))
local cpus_share=$((cpus/share_factor)) local cpus_share=$((cpus/share_factor))
echo "compute_resources: our share: cpus=$cpus_share, mem=$mem_share, disk=$disk_share" echo "compute_resources: our share: cpus=$cpus_share, mem=$mem_share, disk=$disk_share"
# How many build jobs, how many jobs will use tmpfs, and how much mem for each tmpfs # How many build jobs, how many jobs will use tmpfs, and how much mem for each tmpfs
@ -293,7 +312,7 @@ compute_resources () {
fi fi
done done
# Our output is saved in environmnet variables # Our output is saved in environment variables
MOCKCHAIN_RESOURCE_ALLOCATION=$(echo $x | sed 's#^:##') MOCKCHAIN_RESOURCE_ALLOCATION=$(echo $x | sed 's#^:##')
MAX_WORKERS=$workers MAX_WORKERS=$workers
echo "compute_resources: MAX_WORKERS=$MAX_WORKERS, MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION" echo "compute_resources: MAX_WORKERS=$MAX_WORKERS, MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION"
@ -654,7 +673,7 @@ kill_descendents ()
local relevant_recursive_children="$ME" local relevant_recursive_children="$ME"
local relevant_recursive_promote_children="mock" local relevant_recursive_promote_children="mock"
local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16" local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16 mockchain-parallel-2.6 mockchain-parallel-2.7"
local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done) local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done) local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
@ -1181,12 +1200,22 @@ mock_clean_metadata_cfg () {
return 1 return 1
fi fi
CMD=$((cat $CFG; \ #
grep config_opts\\[\'yum.conf\'\\\] $CFG | \ # From mock config, extract the embedded yum/dnf config.
# Then extract the repo definitions,
# and convert to a series of yum commands to clean the
# metadata one repo at a time. e.g.
# CMD="yum --disablerepo=* --enablerepo=StxCentos7Distro clean metadata; \
# yum --disablerepo=* --enablerepo=StxCentos7Distro-rt clean metadata;
# ...
# "
#
CMD=$((grep -e config_opts\\[\'yum.conf\'\\\] $CFG \
-e config_opts\\[\'dnf.conf\'\\\] $CFG | \
sed 's#\\n#\n#g') | \ sed 's#\\n#\n#g') | \
grep '^[[]' | \ grep '^[[]' | \
grep -v main | \ grep -v main | \
sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --enablerepo=#" -e 's#$# clean metadata#' | \ sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --disablerepo=* --enablerepo=#" -e 's#$# clean metadata#' | \
sort -u | \ sort -u | \
tr '\n' ';') tr '\n' ';')
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
@ -2338,6 +2367,7 @@ if [ $CAREFUL -eq 1 ]; then
CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after" CMD_OPTIONS="$MOCK_PASSTHROUGH --no-cleanup-after"
fi fi
CMD_OPTIONS+=" $MOCK_PASSTHROUGH --enable-plugin=package_state"
CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG" CMD_OPTIONS+=" --log=$MOCKCHAIN_LOG"
echo "CAREFUL=$CAREFUL" echo "CAREFUL=$CAREFUL"

35
build-tools/build-rpms-serial
View File

@ -25,7 +25,14 @@
export ME=$(basename "$0") export ME=$(basename "$0")
CMDLINE="$ME $@" CMDLINE="$ME $@"
BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
# Set PKG_MANAGER for our build environment.
source "${BUILD_RPMS_PARALLEL_DIR}/pkg-manager-utils.sh"
# Build for distribution. Currently 'centos' is only supported value.
export DISTRO="centos"
CREATEREPO=$(which createrepo_c) CREATEREPO=$(which createrepo_c)
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
@ -42,6 +49,7 @@ if [ ! -d ${LOCAL_REPO} ]; then
fi fi
fi fi
# Make sure we have a dependency cache
DEPENDANCY_DIR="${LOCAL_REPO}/dependancy-cache" DEPENDANCY_DIR="${LOCAL_REPO}/dependancy-cache"
SRPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires" SRPM_DIRECT_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-direct-requires"
SRPM_TRANSITIVE_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-transitive-requires" SRPM_TRANSITIVE_REQUIRES_FILE="$DEPENDANCY_DIR/SRPM-transitive-requires"
@ -118,7 +126,7 @@ create-no-clean-list () {
local g local g
for g in $install_groups; do for g in $install_groups; do
# Find manditory packages in the group. # Find mandatory packages in the group.
# Discard anything before (and including) 'Mandatory Packages:' # Discard anything before (and including) 'Mandatory Packages:'
# and anything after (and including) 'Optional Packages:'. # and anything after (and including) 'Optional Packages:'.
# Also discard leading spaces or '+' characters. # Also discard leading spaces or '+' characters.
@ -135,7 +143,7 @@ create-no-clean-list () {
while [ $noclean_list_len -gt $noclean_last_list_len ]; do while [ $noclean_list_len -gt $noclean_last_list_len ]; do
noclean_last_list_len=$noclean_list_len noclean_last_list_len=$noclean_list_len
noclean_list=$( (dnf -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq) noclean_list=$( (${PKG_MANAGER} -c $MY_YUM_CONF deplist $noclean_list 2>> /dev/null | grep provider: | awk '{ print $2 }' | awk -F . '{ print $1 }'; for p in $noclean_list; do echo $p; done) | sort --uniq)
noclean_list_len=$(echo $noclean_list | wc -w) noclean_list_len=$(echo $noclean_list | wc -w)
done done
@ -475,7 +483,7 @@ kill_descendents ()
local relevant_recursive_children="$ME" local relevant_recursive_children="$ME"
local relevant_recursive_promote_children="mock" local relevant_recursive_promote_children="mock"
local relevant_other_children="mockchain-parallel" local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16 mockchain-parallel-2.6 mockchain-parallel-2.7"
local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done) local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done)
local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done) local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done)
@ -964,7 +972,24 @@ mock_clean_metadata_cfg () {
return 1 return 1
fi fi
CMD=$((cat $CFG; grep config_opts\\[\'yum.conf\'\\\] $CFG | sed 's#\\n#\n#g') | grep '^[[]' | grep -v main | sed 's/[][]//g' | sed 's#^#yum --enablerepo=#' | sed 's#$# clean metadata#' | sort -u | tr '\n' ';') #
# From mock config, extract the embedded yum/dnf config.
# Then extract the repo definitions,
# and convert to a series of yum commands to clean the
# metadata one repo at a time. e.g.
# CMD="yum --disablerepo=* --enablerepo=StxCentos7Distro clean metadata; \
# yum --disablerepo=* --enablerepo=StxCentos7Distro-rt clean metadata;
# ...
# "
#
CMD=$((grep -e config_opts\\[\'yum.conf\'\\\] $CFG \
-e config_opts\\[\'dnf.conf\'\\\] $CFG | \
sed 's#\\n#\n#g') | \
grep '^[[]' | \
grep -v main | \
sed -e 's/[][]//g' -e "s#^#${PKG_MANAGER} --disablerepo=* --enablerepo=#" -e 's#$# clean metadata#' | \
sort -u | \
tr '\n' ';')
echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP echo "$MOCK --root $CFG --configdir $(dirname $CFG) --chroot bash -c $CMD" &> $TMP
trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP trapwrap_n $CFG $MOCK --root $CFG --configdir $(dirname $CFG) --chroot "bash -c '($CMD)'" &>>$TMP
RC=$? RC=$?
@ -1129,6 +1154,7 @@ clean_yum_cache_cfg () {
return $RC return $RC
} }
clean_yum_cache () { clean_yum_cache () {
echo "${FUNCNAME[0]}: in" echo "${FUNCNAME[0]}: in"
clean_yum_cache_cfg $BUILD_CFG clean_yum_cache_cfg $BUILD_CFG
@ -1249,7 +1275,6 @@ while true ; do
esac esac
done done
# Reset variables # Reset variables
if [ -n "$MY_WORKSPACE" ]; then if [ -n "$MY_WORKSPACE" ]; then
export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE} export MY_WORKSPACE_TOP=${MY_WORKSPACE_TOP:-$MY_WORKSPACE}

182
build-tools/build-wheels/build-base-wheels.sh
View File

@ -114,8 +114,6 @@ while true; do
esac esac
done done
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base
BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD_STREAM}" BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD_STREAM}"
# BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME. # BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME.
@ -123,7 +121,6 @@ BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${BUILD
BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}" BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}"
DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile
WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg
function supported_os_list { function supported_os_list {
for f in ${DOCKER_PATH}/*-dockerfile; do for f in ${DOCKER_PATH}/*-dockerfile; do
@ -137,40 +134,75 @@ if [ ! -f ${DOCKER_FILE} ]; then
exit 1 exit 1
fi fi
if [ ! -f ${WHEELS_CFG} ]; then # Print a loud message
echo "Required file does not exist: ${WHEELS_CFG}" >&2 function notice {
(
set +x
echo
echo ======================================
for s in "$@" ; do
echo "$s"
done
echo ======================================
echo
) 2>&1
}
# prefix each line of a command's output
# also redirects command's STDERR to STDOUT
log_prefix() {
local prefix="$1" ; shift
"$@" 2>&1 | awk -v prefix="$prefix" '{print prefix $0}'
# return false if the command (rather than awk) failed
[ ${PIPESTATUS[0]} -eq 0 ]
}
# Make sure a file exists, exit otherwise
function require_file {
if [ ! -f "${1}" ]; then
echo "Required file does not exist: ${1}" >&2
exit 1 exit 1
fi fi
}
#
# Check build output directory for unexpected files, # Check build output directory for unexpected files,
# ie. wheels from old builds that are no longer in wheels.cfg # ie. wheels from old builds that are no longer in wheels.cfg
# function prepare_output_dir {
if [ -d ${BUILD_OUTPUT_PATH} ]; then local output_dir="$1"
local wheels_cfg="$2"
for f in ${BUILD_OUTPUT_PATH}/*; do if [ -d ${output_dir} ]; then
grep -q "^$(basename $f)|" ${WHEELS_CFG} local f
for f in ${output_dir}/*; do
if [ -f $f ] ; then
grep -q "^$(basename $f)|" ${wheels_cfg}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Deleting stale file: $f" echo "Deleting stale file: $f"
rm -f $f rm -f $f
fi fi
fi
done done
else else
mkdir -p ${BUILD_OUTPUT_PATH} mkdir -p ${output_dir}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Failed to create directory: ${BUILD_OUTPUT_PATH}" >&2 echo "Failed to create directory: ${output_dir}" >&2
exit 1 exit 1
fi fi
fi fi
}
# Check to see if we need to build anything BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base
BUILD_NEEDED=no BUILD_OUTPUT_PATH_PY2=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/base-py2
for wheel in $(cat ${WHEELS_CFG} | sed 's/#.*//' | awk -F '|' '{print $1}'); do WHEELS_CFG=${DOCKER_PATH}/${BUILD_STREAM}-wheels.cfg
if [[ "${wheel}" =~ \* || ! -f ${BUILD_OUTPUT_PATH}/${wheel} ]]; then WHEELS_CFG_PY2=${DOCKER_PATH}/${BUILD_STREAM}-wheels-py2.cfg
BUILD_NEEDED=yes
break # make sure .cfg files exist
fi require_file "${WHEELS_CFG}"
done require_file "${WHEELS_CFG_PY2}"
# prepare output directories
prepare_output_dir "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}"
prepare_output_dir "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}"
if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
# Download the master wheel from loci, so we're only building pieces not covered by it # Download the master wheel from loci, so we're only building pieces not covered by it
@ -194,16 +226,30 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null docker run --name ${USER}_inspect_wheels ${MASTER_WHEELS_IMAGE} noop 2>/dev/null
echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}" echo "Extracting wheels from ${MASTER_WHEELS_IMAGE}"
docker export ${USER}_inspect_wheels | tar x -C ${BUILD_OUTPUT_PATH} '*.whl' rm -rf "${BUILD_OUTPUT_PATH}-loci"
mkdir -p "$BUILD_OUTPUT_PATH-loci"
docker export ${USER}_inspect_wheels | tar x -C "${BUILD_OUTPUT_PATH}-loci" '*.whl'
if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then
echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2 echo "Failed to extract wheels from ${MASTER_WHEELS_IMAGE}" >&2
docker rm ${USER}_inspect_wheels docker rm ${USER}_inspect_wheels
if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
docker image rm ${MASTER_WHEELS_IMAGE} docker image rm ${MASTER_WHEELS_IMAGE}
fi fi
rm -rf "${BUILD_OUTPUT_PATH}-loci"
exit 1 exit 1
fi fi
# copy loci wheels in base and base-py2 directories
if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH}"/ ; then
echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH}" >&2
exit 1
fi
if ! cp "${BUILD_OUTPUT_PATH}-loci"/*.whl "${BUILD_OUTPUT_PATH_PY2}"/ ; then
echo "Failed to copy wheels to ${BUILD_OPUTPUT_PATH_PY2}" >&2
exit 1
fi
rm -rf "${BUILD_OUTPUT_PATH}-loci"
docker rm ${USER}_inspect_wheels docker rm ${USER}_inspect_wheels
if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then if [ ${MASTER_WHEELS_PRESENT} -ne 0 ]; then
@ -211,7 +257,21 @@ if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
fi fi
fi fi
if [ "${BUILD_NEEDED}" = "no" ]; then # check if there are any wheels missing
function all_wheels_exist {
local output_dir="$1"
local wheels_cfg="$2"
local wheel
for wheel in $(cat "${wheels_cfg}" | sed 's/#.*//' | awk -F '|' '{print $1}'); do
if [[ "${wheel}" =~ \* || ! -f ${output_dir}/${wheel} ]]; then
return 1
fi
done
return 0
}
if all_wheels_exist "${BUILD_OUTPUT_PATH}" "${WHEELS_CFG}" && \
all_wheels_exist "${BUILD_OUTPUT_PATH_PY2}" "${WHEELS_CFG_PY2}" ; then
echo "All base wheels are already present. Skipping build." echo "All base wheels are already present. Skipping build."
exit 0 exit 0
fi fi
@ -247,12 +307,10 @@ if [ $? -ne 0 ]; then
fi fi
# Run the image, executing the build-wheel.sh script # Run the image, executing the build-wheel.sh script
RM_OPT=
if [ "${KEEP_CONTAINER}" = "no" ]; then
RM_OPT="--rm"
fi
declare -a RUN_ARGS declare -a RUN_ARGS
if [ "${KEEP_CONTAINER}" = "no" ]; then
RUN_ARGS+=(--rm)
fi
if [ ! -z "$HTTP_PROXY" ]; then if [ ! -z "$HTTP_PROXY" ]; then
RUN_ARGS+=(--env http_proxy=$HTTP_PROXY) RUN_ARGS+=(--env http_proxy=$HTTP_PROXY)
fi fi
@ -262,11 +320,23 @@ fi
if [ ! -z "$NO_PROXY" ]; then if [ ! -z "$NO_PROXY" ]; then
RUN_ARGS+=(--env no_proxy=$NO_PROXY) RUN_ARGS+=(--env no_proxy=$NO_PROXY)
fi fi
RUN_ARGS+=(--env DISPLAY_RESULT=no)
RUN_ARGS+=(${RM_OPT} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh)
# Run container to build wheels # Run container to build wheels
with_retries ${MAX_ATTEMPTS} docker run ${RUN_ARGS[@]} rm -f ${BUILD_OUTPUT_PATH}/failed.lst
rm -f ${BUILD_OUTPUT_PATH_PY2}/failed.lst
notice "building python3 wheels"
log_prefix "[python3] " \
with_retries ${MAX_ATTEMPTS} \
docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH}:/wheels ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
BUILD_STATUS=$?
notice "building python2 wheels"
log_prefix "[python2] " \
with_retries ${MAX_ATTEMPTS} \
docker run ${RUN_ARGS[@]} -v ${BUILD_OUTPUT_PATH_PY2}:/wheels --env PYTHON=python2 ${BUILD_IMAGE_NAME} /docker-build-wheel.sh
BUILD_STATUS_PY2=$?
if [ "${KEEP_IMAGE}" = "no" ]; then if [ "${KEEP_IMAGE}" = "no" ]; then
# Delete the builder image # Delete the builder image
@ -287,8 +357,52 @@ if [ "${KEEP_IMAGE}" = "no" ]; then
fi fi
# Check for failures # Check for failures
if [ -f ${BUILD_OUTPUT_PATH}/failed.lst ]; then check_result() {
# Failures would already have been reported local python="$1"
exit 1 local status="$2"
local dir="$3"
# There's a failed images list
if [ -f "${dir}/failed.lst" ]; then
let failures=$(cat "${dir}/failed.lst" | wc -l)
cat <<EOF
############################################################
The following ${python} module(s) failed to build:
$(cat ${dir}/failed.lst)
Summary:
${failures} build failure(s).
EOF
return 1
fi fi
# No failed images list, but build script failed nonetheless
if [ "${status}" != 0 ] ; then
cat <<EOF
############################################################
Build script failed for ${python}
EOF
return 1
fi
# success
cat <<EOF
############################################################
All ${python} wheels have been successfully built.
EOF
return 0
}
if ! check_result "python3" "${BUILD_STATUS}" "${BUILD_OUTPUT_PATH}" || \
! check_result "python2" "${BUILD_STATUS_PY2}" "${BUILD_OUTPUT_PATH_PY3}" ; then
exit 1
fi
exit 0

47
build-tools/build-wheels/build-wheel-tarball.sh
View File

@ -21,7 +21,6 @@ SUPPORTED_OS_ARGS=('centos')
OS=centos OS=centos
OS_VERSION=7.5.1804 OS_VERSION=7.5.1804
BUILD_STREAM=stable BUILD_STREAM=stable
CURRENT_STABLE_OPENSTACK=ussuri
VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp
PUSH=no PUSH=no
HTTP_PROXY="" HTTP_PROXY=""
@ -30,6 +29,13 @@ NO_PROXY=""
CLEAN=no CLEAN=no
DOCKER_USER=${USER} DOCKER_USER=${USER}
declare -i MAX_ATTEMPTS=1 declare -i MAX_ATTEMPTS=1
PYTHON2=no
# Requirement/constraint URLs -- these will be read from openstack.cfg
STABLE_OPENSTACK_REQ_URL=
MASTER_OPENSTACK_REQ_URL=
STABLE_OPENSTACK_REQ_URL_PY2=
MASTER_OPENSTACK_REQ_URL_PY2=
# List of top-level services for images, which should not be listed in upper-constraints.txt # List of top-level services for images, which should not be listed in upper-constraints.txt
SKIP_CONSTRAINTS=( SKIP_CONSTRAINTS=(
@ -47,6 +53,7 @@ SKIP_CONSTRAINTS=(
nova nova
panko panko
) )
SKIP_CONSTRAINTS_PY2=("${SKIP_CONSTRAINTS_PY[@]}")
function usage { function usage {
cat >&2 <<EOF cat >&2 <<EOF
@ -64,11 +71,12 @@ Options:
--user: Docker repo userid --user: Docker repo userid
--version: Version for pushed image (if used with --push) --version: Version for pushed image (if used with --push)
--attempts: Max attempts, in case of failure (default: 1) --attempts: Max attempts, in case of failure (default: 1)
--python2: Build a python2 tarball
EOF EOF
} }
OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts: -- "$@") OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2 -- "$@")
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
usage usage
exit 1 exit 1
@ -131,6 +139,10 @@ while true; do
MAX_ATTEMPTS=$2 MAX_ATTEMPTS=$2
shift 2 shift 2
;; ;;
--python2)
PYTHON2=yes
shift
;;
-h | --help ) -h | --help )
usage usage
exit 1 exit 1
@ -156,6 +168,15 @@ if [ ${VALID_OS} -ne 0 ]; then
exit 1 exit 1
fi fi
# Read openstack URLs
source "${MY_SCRIPT_DIR}/openstack.cfg" || exit 1
# Set python version-specific variables
if [ "${PYTHON2}" = "yes" ]; then
SKIP_CONSTRAINTS=("${SKIP_CONSTRAINTS_PY2[@]}")
PY_SUFFIX="-py2"
fi
# Build the base wheels and retrieve the StarlingX wheels # Build the base wheels and retrieve the StarlingX wheels
declare -a BUILD_BASE_WL_ARGS declare -a BUILD_BASE_WL_ARGS
BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --stream ${BUILD_STREAM}) BUILD_BASE_WL_ARGS+=(--os ${OS} --os-version ${OS_VERSION} --stream ${BUILD_STREAM})
@ -183,7 +204,7 @@ if [ $? -ne 0 ]; then
exit 1 exit 1
fi fi
BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/tarball BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/tarball${PY_SUFFIX}
if [ -d ${BUILD_OUTPUT_PATH} ]; then if [ -d ${BUILD_OUTPUT_PATH} ]; then
# Wipe out the existing dir to ensure there are no stale files # Wipe out the existing dir to ensure there are no stale files
rm -rf ${BUILD_OUTPUT_PATH} rm -rf ${BUILD_OUTPUT_PATH}
@ -191,7 +212,7 @@ fi
mkdir -p ${BUILD_OUTPUT_PATH} mkdir -p ${BUILD_OUTPUT_PATH}
cd ${BUILD_OUTPUT_PATH} cd ${BUILD_OUTPUT_PATH}
IMAGE_NAME=stx-${OS}-${BUILD_STREAM}-wheels IMAGE_NAME=stx-${OS}-${BUILD_STREAM}-wheels${PY_SUFFIX}
TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/${IMAGE_NAME}.tar TARBALL_FNAME=${MY_WORKSPACE}/std/build-wheels-${OS}-${BUILD_STREAM}/${IMAGE_NAME}.tar
if [ -f ${TARBALL_FNAME} ]; then if [ -f ${TARBALL_FNAME} ]; then
@ -199,19 +220,27 @@ if [ -f ${TARBALL_FNAME} ]; then
fi fi
# Download the global-requirements.txt and upper-constraints.txt files # Download the global-requirements.txt and upper-constraints.txt files
if [ "${PYTHON2}" = "yes" ]; then
if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
OPENSTACK_BRANCH=master OPENSTACK_REQ_URL="${MASTER_OPENSTACK_REQ_URL_PY2}"
else else
OPENSTACK_BRANCH=stable/${CURRENT_STABLE_OPENSTACK} OPENSTACK_REQ_URL="${STABLE_OPENSTACK_REQ_URL_PY2}"
fi
else
if [ "${BUILD_STREAM}" = "dev" -o "${BUILD_STREAM}" = "master" ]; then
OPENSTACK_REQ_URL="${MASTER_OPENSTACK_REQ_URL}"
else
OPENSTACK_REQ_URL="${STABLE_OPENSTACK_REQ_URL}"
fi
fi fi
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/global-requirements.txt with_retries ${MAX_ATTEMPTS} wget "${OPENSTACK_REQ_URL}/global-requirements.txt"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Failed to download global-requirements.txt" >&2 echo "Failed to download global-requirements.txt" >&2
exit 1 exit 1
fi fi
with_retries ${MAX_ATTEMPTS} wget https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_BRANCH}/upper-constraints.txt with_retries ${MAX_ATTEMPTS} wget "${OPENSTACK_REQ_URL}/upper-constraints.txt"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Failed to download upper-constraints.txt" >&2 echo "Failed to download upper-constraints.txt" >&2
exit 1 exit 1
@ -230,7 +259,7 @@ done
shopt -s nullglob shopt -s nullglob
# Copy the base and stx wheels, updating upper-constraints.txt as necessary # Copy the base and stx wheels, updating upper-constraints.txt as necessary
for wheel in ../base/*.whl ../stx/wheels/*.whl; do for wheel in ../base${PY_SUFFIX}/*.whl ../stx/wheels/*.whl; do
# Get the wheel name and version from the METADATA # Get the wheel name and version from the METADATA
METADATA=$(unzip -p ${wheel} '*/METADATA') METADATA=$(unzip -p ${wheel} '*/METADATA')
name=$(echo "${METADATA}" | grep '^Name:' | awk '{print $2}') name=$(echo "${METADATA}" | grep '^Name:' | awk '{print $2}')

7
build-tools/build-wheels/docker/centos-dockerfile
View File

@ -20,3 +20,10 @@ RUN set -ex ;\
COPY docker-build-wheel.sh / COPY docker-build-wheel.sh /
COPY ${BUILD_STREAM}-wheels.cfg /wheels.cfg COPY ${BUILD_STREAM}-wheels.cfg /wheels.cfg
# Python2 packages
RUN set -ex; \
yum -y install python python-devel ;\
wget https://bootstrap.pypa.io/pip/2.7/get-pip.py ;\
python get-pip.py
COPY ${BUILD_STREAM}-wheels-py2.cfg /wheels-py2.cfg

18
build-tools/build-wheels/docker/dev-wheels-py2.cfg Normal file
View File

@ -0,0 +1,18 @@
#
# git: wheelname|git|git-source|basedir|branch
# tar: wheelname|tar|wget-source|basedir
# pypi: wheelname|pypi|wget-source
# zip: wheelname|zip|wget-source|basedir
#
# If fix_setup must be called, add |fix_setup at the end of the line
#
amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl
neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master
python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master
openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master
networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3

13
build-tools/build-wheels/docker/dev-wheels.cfg
View File

@ -6,13 +6,14 @@
# #
# If fix_setup must be called, add |fix_setup at the end of the line # If fix_setup must be called, add |fix_setup at the end of the line
# #
amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2 amqplib-1.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
lz4-0.9.0-cp27-none-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0 croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl
panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl google_api_python_client-1.7.7-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/d7/47/940908e52487440f61fb93ad55cbbe3a28235d3bb143b26affb17b37dd28/google_api_python_client-1.7.7-py2.py3-none-any.whl
lz4-0.9.0-cp36-cp36m-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master neutron_lib-*.whl|git|https://github.com/openstack/neutron-lib|neutron-lib|master
python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master python_openstackclient-*.whl|git|https://github.com/openstack/python-openstackclient|python-openstackclient|master
openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master openstacksdk-*.whl|git|https://github.com/openstack/openstacksdk|openstacksdk|master
networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl panko-5.0.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
croniter-0.3.29-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/c9/11182a2507798c661b04a7914739ea8ca73a738e6869a23742029f51bc1a/croniter-0.3.29-py2.py3-none-any.whl pecan-1.3.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3
pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3

28
build-tools/build-wheels/docker/docker-build-wheel.sh
View File

@ -10,8 +10,14 @@
CFGFILE=/wheels.cfg CFGFILE=/wheels.cfg
OUTPUTDIR=/wheels OUTPUTDIR=/wheels
FAILED_LOG=$OUTPUTDIR/failed.lst FAILED_LOG="${OUTPUTDIR}/failed.lst"
: ${DISPLAY_RESULT=yes}
declare -i MAX_ATTEMPTS=5 declare -i MAX_ATTEMPTS=5
: ${PYTHON=python3}
if [[ "${PYTHON}" == "python2" ]] ; then
CFGFILE=/wheels-py2.cfg
FAILED_LOG="${OUTPUTDIR}/failed-py2.lst"
fi
# #
# Function to log the start of a build # Function to log the start of a build
@ -184,7 +190,7 @@ function from_git {
fi fi
# Build the wheel # Build the wheel
python3 setup.py bdist_wheel ${PYTHON} setup.py bdist_wheel
if [ -f dist/$wheelname ]; then if [ -f dist/$wheelname ]; then
cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
else else
@ -244,7 +250,7 @@ function from_tar {
fi fi
# Build the wheel # Build the wheel
python3 setup.py bdist_wheel ${PYTHON} setup.py bdist_wheel
if [ -f dist/$wheelname ]; then if [ -f dist/$wheelname ]; then
cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
else else
@ -295,7 +301,7 @@ function from_zip {
fi fi
# Build the wheel # Build the wheel
python3 setup.py bdist_wheel ${PYTHON} setup.py bdist_wheel
if [ -f dist/$wheelname ]; then if [ -f dist/$wheelname ]; then
cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG cp dist/$wheelname $OUTPUTDIR || echo $wheelname >> $FAILED_LOG
else else
@ -339,24 +345,28 @@ from_tar
from_zip from_zip
from_pypi from_pypi
if [ -f $FAILED_LOG ]; then if [ -f "${FAILED_LOG}" ]; then
let failures=$(cat $FAILED_LOG | wc -l) if [ "${DISPLAY_RESULT}" = yes ] ; then
let failures=$(cat "${FAILED_LOG}" | wc -l)
cat <<EOF cat <<EOF
############################################################ ############################################################
The following module(s) failed to build: The following ${PYTHON} module(s) failed to build:
$(cat $FAILED_LOG) $(cat ${FAILED_LOG})
Summary: Summary:
${failures} build failure(s). ${failures} build failure(s).
EOF EOF
fi
exit 1 exit 1
fi fi
if [ "${DISPLAY_RESULT}" = yes ] ; then
cat <<EOF cat <<EOF
############################################################ ############################################################
All wheels have been successfully built. All ${PYTHON} wheels have been successfully built.
EOF EOF
fi
exit 0 exit 0

175
build-tools/build-wheels/docker/stable-wheels-py2.cfg Normal file
View File

@ -0,0 +1,175 @@
#
# git: wheelname|git|git-source|basedir|branch
# tar: wheelname|tar|wget-source|basedir
# pypi: wheelname|pypi|wget-source
# zip: wheelname|zip|wget-source|basedir
#
# If fix_setup must be called, add |fix_setup at the end of the line
#
abclient-0.2.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/eb/091b02c1e36d68927adfb746706e2c80f7e7bfb3f16e3cbcfec2632118ab/abclient-0.2.3.tar.gz|abclient-0.2.3
alembic-1.1.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9a/0f/a5e8997d58882da8ecd288360dddf133a83145de6480216774923b393422/alembic-1.1.0.tar.gz|alembic-1.1.0
amqplib-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz|amqplib-1.0.2
anyjson-0.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c3/4d/d4089e1a3dd25b46bebdb55a992b0797cff657b4477bc32ce28038fdecbc/anyjson-0.3.3.tar.gz|anyjson-0.3.3
backports.ssl_match_hostname-3.7.0.1-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ff/2b/8265224812912bc5b7a607c44bf7b027554e1b9775e9ee0de8032e3de4b2/backports.ssl_match_hostname-3.7.0.1.tar.gz|backports.ssl_match_hostname-3.7.0.1|fix_setup
bottle-0.12.17-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c4/a5/6bf41779860e9b526772e1b3b31a65a22bd97535572988d16028c5ab617d/bottle-0.12.17.tar.gz|bottle-0.12.17
cassandra_driver-3.19.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1c/fe/e4df42a3e864b6b7b2c7f6050b66cafc7fba8b46da0dfb9d51867e171a77/cassandra-driver-3.19.0.tar.gz|cassandra-driver-3.19.0
cmd2-0.8.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/40/a71caa2aaff10c73612a7106e2d35f693e85b8cf6e37ab0774274bca3cf9/cmd2-0.8.9-py2.py3-none-any.whl
construct-2.8.22-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e5/c6/3e3aeef38bb0c27364af3d21493d9690c7c3925f298559bca3c48b7c9419/construct-2.8.22.tar.gz|construct-2.8.22
crc16-0.1.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz|crc16-0.1.1|fix_setup
demjson-2.2.4-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/96/67/6db789e2533158963d4af689f961b644ddd9200615b8ce92d6cad695c65a/demjson-2.2.4.tar.gz|demjson-2.2.4|fix_setup
django_floppyforms-1.7.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8c/18/30a9137c7ae279a27ccdeb10f6fe8be18ee98551d01ec030b6cfe8b2d2e2/django-floppyforms-1.7.0.tar.gz|django-floppyforms-1.7.0
django_pyscss-2.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4b/7f/d771802305184aac6010826f60a0b2ecaa3f57d19ab0e405f0c8db07e809/django-pyscss-2.0.2.tar.gz|django-pyscss-2.0.2
docopt-0.6.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz|docopt-0.6.2
dogpile.cache-0.7.1-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/84/3e/dbf1cfc5228f1d3dca80ef714db2c5aaec5cd9efaf54d7e3daef6bc48b19/dogpile.cache-0.7.1.tar.gz|dogpile.cache-0.7.1
enum_compat-0.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/95/6e/26bdcba28b66126f66cf3e4cd03bcd63f7ae330d29ee68b1f6b623550bfa/enum-compat-0.0.2.tar.gz|enum-compat-0.0.2
etcd3-0.10.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/09/f1/93603a26daf7a993a0acbbcfd32afce8b2fdf30a765d5651571ab635969b/etcd3-0.10.0.tar.gz|etcd3-0.10.0
exabgp-4.1.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/b9/f1/f2417bc82c9caa220fcd369a3b55ac895088bcc8afc262e4bb07d48aa40c/exabgp-4.1.2.tar.gz|exabgp-4.1.2
flask_keystone-0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/1f/ca/3938de8c5f4a3d1c5dd4278bedb9d31d79816feba4d088293c620a366fb1/flask_keystone-0.2.tar.gz|flask_keystone-0.2
flask_oslolog-0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a7/62/fec02ce761b548b1289680bb1be1aa0bce2b2c4017d5b31bd6c67c78aef9/flask_oslolog-0.1.tar.gz|flask_oslolog-0.1
fortiosclient-0.0.3-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/e9/aa/b2c0705d5e52c8d9af35422d940800b49c562758fbdad3179a6fbf6e92f5/fortiosclient-0.0.3.tar.gz|fortiosclient-0.0.3
frozendict-1.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4e/55/a12ded2c426a4d2bee73f88304c9c08ebbdbadb82569ebdd6a0c007cfd08/frozendict-1.2.tar.gz|frozendict-1.2
funcparserlib-0.3.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/cb/f7/b4a59c3ccf67c0082546eaeb454da1a6610e924d2e7a2a21f337ecae7b40/funcparserlib-0.3.6.tar.gz|funcparserlib-0.3.6
functools32-3.2.3.post2-py2-none-any.whl|git|https://github.com/MiCHiLU/python-functools32|python-functools32|3.2.3-2|fix_setup
future-0.17.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/90/52/e20466b85000a181e1e144fd8305caf2cf475e2f9674e797b222f8105f5f/future-0.17.1.tar.gz|future-0.17.1
happybase-1.2.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/d1/9c/f5f7bdb5439cda2b7da4e20ac24ec0e2455fd68aade8397f211d2994c39d/happybase-1.2.0.tar.gz|happybase-1.2.0
hiredis-1.0.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9e/e0/c160dbdff032ffe68e4b3c576cba3db22d8ceffc9513ae63368296d1bcc8/hiredis-1.0.0.tar.gz|hiredis-1.0.0
httplib2-0.13.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/78/23/bb9606e87a66fd8c72a2b1a75b049d3859a122bc2648915be845bc44e04f/httplib2-0.13.1.tar.gz|httplib2-0.13.1
itsdangerous-1.1.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/76/ae/44b03b253d6fade317f32c24d100b3b35c2239807046a4c953c7b89fa49e/itsdangerous-1.1.0-py2.py3-none-any.whl
jaeger_client-4.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/f1/da/569a4f1bc3d0c412c7f903053f09ef62fa10949374ca90bc852b22dd3860/jaeger-client-4.1.0.tar.gz|jaeger-client-4.1.0
jsonpath_rw-1.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/71/7c/45001b1f19af8c4478489fbae4fc657b21c4c669d7a5a036a86882581d85/jsonpath-rw-1.4.0.tar.gz|jsonpath-rw-1.4.0
krest-1.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fb/d2/9dbbd3a76f2385041720a0eb51ddab676e688fa8bee8a1489470839616cf/krest-1.3.1.tar.gz|krest-1.3.1
#libvirt_python-4.4.0-cp27-none-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2b/8d/1160cf34dc3d296896eb5c8f4944439ea368b87d2d2431f58d08d6bdf374/libvirt-python-4.4.0.tar.gz|libvirt-python-4.4.0|fix_setup
logutils-0.3.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/b2/b57450889bf73da26027f8b995fd5fbfab258ec24ef967e4c1892f7cb121/logutils-0.3.5.tar.gz|logutils-0.3.5|fix_setup
lz4-0.9.0-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/python-lz4/python-lz4|python-lz4|v0.9.0
Mako-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/b0/3c/8dcd6883d009f7cae0f3157fb53e9afb05a0d3d33b3db1268ec2e6f4a56b/Mako-1.1.0.tar.gz|Mako-1.1.0
marathon-0.11.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/97/e3/f036af0d94f98d199233faa71b5bcbef8b8e8e634551940d98c95d276e4f/marathon-0.11.0-py2.py3-none-any.whl
MarkupSafe-1.1.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz|MarkupSafe-1.1.1
mox-0.5.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/a1/64740c638cc5fae807022368f4141700518ee343b53eb3e90bf3cc15a4d4/mox-0.5.3.tar.gz|mox-0.5.3|fix_setup
migrate-0.3.8-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ce/31/1a4cbf8dc0536c55f41072e8ea37b3df1e412262dc731c57e5bb099eb9b2/migrate-0.3.8.tar.gz|migrate-0.3.8
mpmath-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32/mpmath-1.1.0.tar.gz|mpmath-1.1.0|fix_setup
msgpack_python-0.4.8-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/msgpack/msgpack-python.git|msgpack-python|0.4.8
munch-2.3.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/68/f4/260ec98ea840757a0da09e0ed8135333d59b8dfebe9752a365b04857660a/munch-2.3.2.tar.gz|munch-2.3.2
ndg_httpsclient-0.5.1-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/bf/b2/26470fde7ff55169df8e071fb42cb1f83e22bd952520ab2b5c5a5edc2acd/ndg_httpsclient-0.5.1-py2-none-any.whl
netifaces-0.10.9-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/0d/18/fd6e9c71a35b67a73160ec80a49da63d1eed2d2055054cc2995714949132/netifaces-0.10.9.tar.gz|netifaces-0.10.9
networking_sfc-8.0.0.0b2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6a/a8/0e9bdd1f87dfb50682f23a01f590530ec8fa715e51127cf9f58d1905886c/networking_sfc-8.0.0.0b2-py2.py3-none-any.whl
networkx-2.2-py2.py3-none-any.whl|zip|https://files.pythonhosted.org/packages/f3/f4/7e20ef40b118478191cec0b58c3192f822cace858c19505c7670961b76b2/networkx-2.2.zip|networkx-2.2
neutron_lib-1.29.1-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/6b/dd/548cbb7a936de18aa642372927e409540d8f5d96a2f7650c4d1197845f3c/neutron_lib-1.29.1-py2.py3-none-any.whl
nodeenv-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/00/6e/ed417bd1ed417ab3feada52d0c89ab0ed87d150f91590badf84273e047c9/nodeenv-1.3.3.tar.gz|nodeenv-1.3.3
nose_exclude-0.5.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/63/cf/90c4be56bf11b7bc8801086d9445baf731aa36b8e8fc5791731e8e604dcd/nose-exclude-0.5.0.tar.gz|nose-exclude-0.5.0
nosehtmloutput-0.0.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/f7/6cb16c0b233d3f2d62be38ddb7d7c1bc967188c41575ecf0312e6575730d/nosehtmloutput-0.0.5.tar.gz|nosehtmloutput-0.0.5
openshift-0.8.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/73/ed/c92c0ba23b6c4c8e5542151a1b89cb8ff01f68a72fe68f6c95a28d885ebe/openshift-0.8.6.tar.gz|openshift-0.8.6
openstack.nose_plugin-0.11-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bc/83/e7c9b9297e1a501d2c2617f98d6176199570e8ee32f0e72669c8852c6c81/openstack.nose_plugin-0.11.tar.gz|openstack.nose_plugin-0.11
opentracing-2.2.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/94/9f/289424136addf621fb4c75624ef9a3a80e8575da3993a87950c57e93217e/opentracing-2.2.0.tar.gz|opentracing-2.2.0
ovs-2.11.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/81/06/387b2159ac073de95e484aa6e2f108a232cd906e350307168843061f899f/ovs-2.11.0.tar.gz|ovs-2.11.0
panko-5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a9/89/d666e0889d869e41c9b7f87a0a34858b2520782b82e025da84c98e0db8f6/panko-5.0.0.tar.gz|panko-5.0.0
pathlib-1.0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz|pathlib-1.0.1|fix_setup
pecan-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/93/98/889d7615595e894f4f7e4c17d4008c822c8e39e650c8ab390cc6c39b99c4/pecan-1.3.3.tar.gz|pecan-1.3.3
pifpaf-2.2.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/33/dc/4f276c55d94cd73fc1f94e2d23f34b476fea38d240e3e17b837a5749bc9f/pifpaf-2.2.2-py2.py3-none-any.whl
pika_pool-0.1.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ec/48/50c8f02a3eef4cb824bec50661ec1713040402cc1b2a38954dc977a59c23/pika-pool-0.1.3.tar.gz|pika-pool-0.1.3
Pint-0.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/15/9d/bf177ebbc57d25e9e296addc14a1303d1e34d7964af5df428a8332349c42/Pint-0.9-py2.py3-none-any.whl
ply-3.11-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl
positional-1.1.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8c/16/64a4fa0967c486380468dca18867d22ac1c17bba06349e31ace77c7757f7/positional-1.1.2.tar.gz|positional-1.1.2
prettytable-0.7.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz|prettytable-0.7.2
proboscis-1.2.6.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/3c/c8/c187818ab8d0faecdc3c16c1e0b2e522f3b38570f0fb91dcae21662019d0/proboscis-1.2.6.0.tar.gz|proboscis-1.2.6.0
psutil-5.6.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1c/ca/5b8c1fe032a458c2c4bcbe509d1401dca9dda35c7fc46b36bb81c2834740/psutil-5.6.3.tar.gz|psutil-5.6.3
psycopg2-2.8.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/5c/1c/6997288da181277a0c29bc39a5f9143ff20b8c99f2a7d059cfb55163e165/psycopg2-2.8.3.tar.gz|psycopg2-2.8.3
PuLP-1.6.10-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/2d/33/3ae6d9d2ac8c7068937af6372fd8828ac605e62a8b17106fe57110930d38/PuLP-1.6.10.zip|PuLP-1.6.10
pycparser-2.19-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a/pycparser-2.19.tar.gz|pycparser-2.19
pycrypto-2.6.1-cp27-cp27mu-linux_x86_64.whl|git|https://github.com/dlitz/pycrypto|pycrypto|v2.6.1|fix_setup
pycryptodomex-3.9.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e4/90/a01cafbbad7466491e3a630bf1d734294a32ff1b10e7429e9a4e8478669e/pycryptodomex-3.9.0.tar.gz|pycryptodomex-3.9.0
pydot-1.4.1-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/33/d1/b1479a770f66d962f545c2101630ce1d5592d90cb4f083d38862e93d16d2/pydot-1.4.1-py2.py3-none-any.whl
pydotplus-2.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/60/bf/62567830b700d9f6930e9ab6831d6ba256f7b0b730acb37278b0ccdffacf/pydotplus-2.0.2.tar.gz|pydotplus-2.0.2
pyeclib-1.6.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/aa/d6/ca6bba5e66fc7a9810a995b17a3675492da2bec405806d8ac3db18cfd93b/pyeclib-1.6.0.tar.gz|pyeclib-1.6.0|fix_setup
pyinotify-0.9.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e3/c0/fd5b18dde17c1249658521f69598f3252f11d9d7a980c5be8619970646e1/pyinotify-0.9.6.tar.gz|pyinotify-0.9.6
pykerberos-1.2.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9a/b8/1ec56b6fa8a2e2a81420bd3d90e70b59fc83f6b857fb2c2c37accddc8be3/pykerberos-1.2.1.tar.gz|pykerberos-1.2.1
PyKMIP-0.9.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/24/b2/258332aea85163f49a187337e8c85ee4529eb499b84fe0a6fe2d1a9c8d25/PyKMIP-0.9.1.tar.gz|PyKMIP-0.9.1
pylxd-2.2.10-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/9a/eba58646721ffbff40dc41571b13c9528fdc4e26a82252318c997cdbe26a/pylxd-2.2.10.tar.gz|pylxd-2.2.10
pyngus-2.3.0-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/58/b1/336b8f64e7e4efa9b95027af71e02cd4cfacca8f919345badb852381878a/pyngus-2.3.0.zip|pyngus-2.3.0
pyperclip-1.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2d/0f/4eda562dffd085945d57c2d9a5da745cfb5228c02bc90f2c74bbac746243/pyperclip-1.7.0.tar.gz|pyperclip-1.7.0
pyroute2-0.5.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/f6/80/16a604075345f0c253537d55e5c5282a37c61a1fc8ee0fcc42d1fd2a0739/pyroute2-0.5.6.tar.gz|pyroute2-0.5.6|fix_setup
pyrsistent-0.15.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/66/b2638d96a2d128b168d0dba60fdc77b7800a9b4a5340cefcc5fc4eae6295/pyrsistent-0.15.4.tar.gz|pyrsistent-0.15.4
pyScss-1.3.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/1d/4a/221ae7561c8f51c4f28b2b172366ccd0820b14bb947350df82428dfce381/pyScss-1.3.4.tar.gz|pyScss-1.3.4
pysendfile-2.0.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/cd/3f/4aa268afd0252f06b3b487c296a066a01ddd4222a46b7a3748599c8fc8c3/pysendfile-2.0.1.tar.gz|pysendfile-2.0.1
pystache-0.5.4-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d6/fd/eb8c212053addd941cc90baac307c00ac246ac3fce7166b86434c6eae963/pystache-0.5.4.tar.gz|pystache-0.5.4
python_cinderclient-4.3.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/f1/09/760c454c5bf67509d7f8479d583a3e84411f51ec2a1942aea3741a49b090/python_cinderclient-4.3.0-py2.py3-none-any.whl
python_consul-1.1.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/3f/d0/59bc5f1c6c4d4b498c41d8ce7052ee9e9d68be19e16038a55252018a6c4d/python_consul-1.1.0-py2.py3-none-any.whl
python_editor-1.0.4-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/55/a0/3c0ba1c10f2ca381645dd46cb7afbb73fddc8de9f957e1f9e726a846eabc/python_editor-1.0.4-py2-none-any.whl
python_etcd-0.4.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a1/da/616a4d073642da5dd432e5289b7c1cb0963cc5dde23d1ecb8d726821ab41/python-etcd-0.4.5.tar.gz|python-etcd-0.4.5
python_ldap-3.2.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/ea/93/596f875e003c770447f4b99267820a0c769dd2dc3ae3ed19afe460fcbad0/python-ldap-3.2.0.tar.gz|python-ldap-3.2.0
python_memcached-1.59-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/f5/90/19d3908048f70c120ec66a39e61b92c253e834e6e895cd104ce5e46cbe53/python_memcached-1.59-py2.py3-none-any.whl
python_nss-1.0.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/6b/29/629098e34951c358b1f04f13a70b3590eb0cf2df817d945bd05c4169d71b/python-nss-1.0.1.tar.bz2|python-nss-1.0.1|fix_setup
python_pcre-0.7-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/9d/af/61435bd163f01fe3709fca9b1f79e4978d8089ee671d2e004fc85e10de29/python-pcre-0.7.tar.gz|python-pcre-0.7|fix_setup
python_pytun-2.3.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/52/a4/a062106c739eac79c8160fcf5779ebc84afc1c38b016ab216ed1e6da69b6/python-pytun-2.3.0.tar.gz|python-pytun-2.3.0|fix_setup
python_qpid_proton-0.28.0-cp27-cp27mu-linux_x86_64.whl|zip|https://files.pythonhosted.org/packages/96/35/2c86d844aec1acdfe7778966994aa270fcf03f076df393003bd4fc07dfa9/python-qpid-proton-0.28.0.zip|python-qpid-proton-0.28.0|fix_setup
python_string_utils-0.6.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/5d/13/216f2d4a71307f5a4e5782f1f59e6e8e5d6d6c00eaadf9f92aeccfbb900c/python-string-utils-0.6.0.tar.gz|python-string-utils-0.6.0|fix_setup
pyudev-0.21.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bc/a2/31a07829acea8e70a28c247f43fa5d981229ae0f9edfeddedf52de00709b/pyudev-0.21.0.tar.gz|pyudev-0.21.0
PyYAML-5.1.2-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/e8/b3212641ee2718d556df0f23f78de8303f068fe29cdaa7a91018849582fe/PyYAML-5.1.2.tar.gz|PyYAML-5.1.2
pyzabbix-0.7.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/11/ad/24e19d0cf16d05b7ee19f337f02058ee9b760649171865469ccceef83027/pyzabbix-0.7.5.tar.gz|pyzabbix-0.7.5
qpid_python-1.36.0.post1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2a/33/026ac50a29a85d5d54dd7784a98d624f6142cb07ce185ed268ef9bd3b6dc/qpid-python-1.36.0-1.tar.gz|qpid-python-1.36.0-1|fix_setup
rcssmin-1.0.6-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e2/5f/852be8aa80d1c24de9b030cdb6532bc7e7a1c8461554f6edbe14335ba890/rcssmin-1.0.6.tar.gz|rcssmin-1.0.6|fix_setup
repoze.lru-0.7-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/12/bc/595a77c4b5e204847fdf19268314ef59c85193a9dc9f83630fc459c0fee5/repoze.lru-0.7.tar.gz|repoze.lru-0.7
requests_aws-0.1.8-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/5e/2f/4da17752036c04cf4c9af7a2da0d41ef2205043f1c61008006475aa24b8b/requests-aws-0.1.8.tar.gz|requests-aws-0.1.8
restructuredtext_lint-1.3.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/62/76/bd8760de759fb74d7863e6935200af101cb128a7de008741a4e22341d03c/restructuredtext_lint-1.3.0.tar.gz|restructuredtext_lint-1.3.0
retrying-1.3.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/44/ef/beae4b4ef80902f22e3af073397f079c96969c69b2c7d52a57ea9ae61c9d/retrying-1.3.3.tar.gz|retrying-1.3.3
rfc3986-1.4.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl
rjsmin-1.1.0-cp27-cp27mu-manylinux1_x86_64.whl|pypi|https://files.pythonhosted.org/packages/c3/8e/079b7cc3a0fc9934ab05d868a00183c7aafd90b5d6138313d98ac2b9f666/rjsmin-1.1.0-cp27-cp27mu-manylinux1_x86_64.whl
rtslib_fb-2.1.69-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fc/1a/77a26207bdad13cc39b93d874b3a1b04e5a0b0332fb716e4d654537bacdb/rtslib-fb-2.1.69.tar.gz|rtslib-fb-2.1.69
scandir-1.10.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz|scandir-1.10.0
scrypt-0.8.13-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/80/3d/141eb80e754b86f6c25a2ffaf6c3af3acdb65a3e3700829a05ab0c5d965d/scrypt-0.8.13.tar.gz|scrypt-0.8.13
SecretStorage-2.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/a5/a5/0830cfe34a4cfd0d1c3c8b614ede1edb2aaf999091ac8548dd19cb352e79/SecretStorage-2.3.1.tar.gz|SecretStorage-2.3.1
setproctitle-1.1.10-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/5a/0d/dc0d2234aacba6cf1a729964383e3452c52096dc695581248b548786f2b3/setproctitle-1.1.10.tar.gz|setproctitle-1.1.10
simplegeneric-0.8.1-py2-none-any.whl|zip|https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip|simplegeneric-0.8.1
simplejson-3.16.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/24/c35fb1c1c315fc0fffe61ea00d3f88e85469004713dab488dee4f35b0aff/simplejson-3.16.0.tar.gz|simplejson-3.16.0
skydive_client-0.5.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e4/68/78a246619d9b16bb226562c155f18f798283f86db8f01a89c30b97ac7a27/skydive-client-0.5.0.tar.gz|skydive-client-0.5.0
smmap2-2.0.5-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/55/d2/866d45e3a121ee15a1dc013824d58072fd5c7799c9c34d01378eb262ca8f/smmap2-2.0.5-py2.py3-none-any.whl
sphinxcontrib_fulltoc-1.2.0-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/8e/a6/d1297db9b75650681e5429e92e13df139ee6b64303ff1b2eea4ebd32c0a9/sphinxcontrib-fulltoc-1.2.0.tar.gz|sphinxcontrib-fulltoc-1.2.0
sphinxcontrib_pecanwsme-0.10.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/75/2b/105d07f47485ecf774cd80b881c29e148182b72a3a60596abdd016c87fce/sphinxcontrib-pecanwsme-0.10.0.tar.gz|sphinxcontrib-pecanwsme-0.10.0
SQLAlchemy-1.3.8-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/fc/49/82d64d705ced344ba458197dadab30cfa745f9650ee22260ac2b275d288c/SQLAlchemy-1.3.8.tar.gz|SQLAlchemy-1.3.8
SQLAlchemy_Utils-0.34.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/45/61/3bdd2931e86253fa7df6445a26929fbcc9bc43ad6b27a10f991eb6ecde75/SQLAlchemy-Utils-0.34.2.tar.gz|SQLAlchemy-Utils-0.34.2
stomp.py-4.1.22-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/52/7e/22ca617f61e0d5904e06c1ebd5d453adf30099526c0b64dca8d74fff0cad/stomp.py-4.1.22.tar.gz|stomp.py-4.1.22
subprocess32-3.5.4-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz|subprocess32-3.5.4
suds_jurko-0.6-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/bd/6f/54fbf0999a606680d27c69b1ad12dfff62768ecb9fe48524cebda6eb4423/suds-jurko-0.6.tar.bz2|suds-jurko-0.6
systemd_python-234-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e8/a8/00ba0f605837a8f69523e6c3a4fb14675a6430c163f836540129c50b3aef/systemd-python-234.tar.gz|systemd-python-234|fix_setup
sysv_ipc-1.0.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/08/7d/a862f3045fa191eeece23650725273f2ccaf9ac6b95443dfe4cac6508638/sysv_ipc-1.0.0.tar.gz|sysv_ipc-1.0.0|fix_setup
Tempita-0.5.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz|Tempita-0.5.2
termcolor-1.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz|termcolor-1.1.0|fix_setup
testrepository-0.0.20-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/85/f495b58b2b0ac907def07385219e9747b75840fa01280f228546a4a5ad7f/testrepository-0.0.20.tar.gz|testrepository-0.0.20
thrift-0.11.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c6/b4/510617906f8e0c5660e7d96fbc5585113f83ad547a3989b80297ac72a74c/thrift-0.11.0.tar.gz|thrift-0.11.0
thriftpy-0.3.9-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/f4/19/cca118cf7d2087310dbc8bd70dc7df0c1320f2652873a93d06d7ba356d4a/thriftpy-0.3.9.tar.gz|thriftpy-0.3.9
thriftpy2-0.4.8-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2c/23/57b00b3d5d3d0ae66d79844a39d3c3b92dde3063c901036808602137d3ab/thriftpy2-0.4.8.tar.gz|thriftpy2-0.4.8
tinyrpc-1.0.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/21/7a/ff1a74256e1bcc04fbaa414c13a2bb79a29ac9918b25f2238592b991e3bc/tinyrpc-1.0.3.tar.gz|tinyrpc-1.0.3
tornado-4.5.3-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/e3/7b/e29ab3d51c8df66922fea216e2bddfcb6430fb29620e5165b16a216e0d3c/tornado-4.5.3.tar.gz|tornado-4.5.3
trollius-2.2.post1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/0b/31/356ae13ad4df58f963e9954d55118f6cffdb3a903c1547973ad7bc347fb9/trollius-2.2.post1.tar.gz|trollius-2.2.post1
ujson-1.35-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/16/c4/79f3409bc710559015464e5f49b9879430d8f87498ecdc335899732e5377/ujson-1.35.tar.gz|ujson-1.35
unicodecsv-0.14.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/6f/a4/691ab63b17505a26096608cc309960b5a6bdf39e4ba1a793d5f9b1a53270/unicodecsv-0.14.1.tar.gz|unicodecsv-0.14.1
uWSGI-2.0.17.1-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a2/c9/a2d5737f63cd9df4317a4acc15d1ddf4952e28398601d8d7d706c16381e0/uwsgi-2.0.17.1.tar.gz|uwsgi-2.0.17.1
voluptuous-0.11.7-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/24/3b/fe531688c0d9e057fccc0bc9430c0a3d4b90e0d2f015326e659c2944e328/voluptuous-0.11.7.tar.gz|voluptuous-0.11.7
warlock-1.3.3-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/c2/36/178b26a338cd6d30523246da4721b1114306f588deb813f3f503052825ee/warlock-1.3.3.tar.gz|warlock-1.3.3
weakrefmethod-1.0.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/99/82/73a21e3eab9a1ff76d12375f7301fba5c6325b9598eed0ae5b0cf5243656/weakrefmethod-1.0.3.tar.gz|weakrefmethod-1.0.3
websockify-0.9.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c4/5b/16ec1e9f4fc536846d95a01a77d97da12f8042ca5cf83cdf3dd0442e881c/websockify-0.9.0.tar.gz|websockify-0.9.0
whereto-0.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/80/83/371a699ce90257608592dadca400a7ecd9a2db6137d78f6f433c7c5e3197/whereto-0.4.0.tar.gz|whereto-0.4.0
wrapt-1.11.2-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/23/84/323c2415280bc4fc880ac5050dddfb3c8062c2552b34c2e512eb4aa68f79/wrapt-1.11.2.tar.gz|wrapt-1.11.2|fix_setup
ws4py-0.5.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/53/20/4019a739b2eefe9282d3822ef6a225250af964b117356971bd55e274193c/ws4py-0.5.1.tar.gz|ws4py-0.5.1
WSME-0.9.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d1/b6/8027248bfca3ce192bc54d46fcda4324c86c8beabe344cbb80fb57a6c868/WSME-0.9.3.tar.gz|WSME-0.9.3
xattr-0.9.6-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/60/80/a1f35bfd3c7ffb78791b2a6a15c233584a102a20547fd96d48933ec453e7/xattr-0.9.6.tar.gz|xattr-0.9.6
XStatic-1.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/36/78/c0ffaf14216517a14d3daa67ff24fbb60b4703e95ce1059a48fd508e6b8c/XStatic-1.0.2.tar.gz|XStatic-1.0.2
XStatic_Angular_FileUpload-12.0.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/4d/fd/c3051915d2f12e8fa11f59c01162ce85e38eca15d9ec73a3d7b271b49744/XStatic-Angular-FileUpload-12.0.4.0.tar.gz|XStatic-Angular-FileUpload-12.0.4.0
XStatic_Angular_lrdragndrop-1.0.2.4-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/80/ea/ffdde05892eabe468f22403f75299cf5d991f0af4f1400bebbf3af04bc9a/XStatic_Angular_lrdragndrop-1.0.2.4-py2.py3-none-any.whl
XStatic_Angular_Schema_Form-0.8.13.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/57/71/ceea2c0a72e2ee2d316d6ab1c06b21faa9f5cbc4b36a4127d7847b7079c5/XStatic-Angular-Schema-Form-0.8.13.0.tar.gz|XStatic-Angular-Schema-Form-0.8.13.0
XStatic_Bootstrap_Datepicker-1.3.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/91/4f/832f14478e714815bb3d44d01dfe8dbe19ccf9f823e0bc7ac1a8cf7fa6b3/XStatic-Bootstrap-Datepicker-1.3.1.0.tar.gz|XStatic-Bootstrap-Datepicker-1.3.1.0
XStatic_Hogan-2.0.0.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/21/fe/37d5c8247f24738e7e368d27ebf945de1ea29fbc3112ac5e75b1b7f1d0c9/XStatic-Hogan-2.0.0.2.tar.gz|XStatic-Hogan-2.0.0.2
XStatic_Jasmine-2.4.1.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/05/43/ceac7def3b6eaf82b6f593e3db2b03a9693a7b002b569e664e382aecddbc/XStatic_Jasmine-2.4.1.2-py2.py3-none-any.whl
XStatic_jQuery-1.12.4.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/67/f1/c18c14fc4aab386e4aba587c5d10c268de222c75bf5e271b6f68a2ea6e77/XStatic-jQuery-1.12.4.1.tar.gz|XStatic-jQuery-1.12.4.1
XStatic_JQuery_Migrate-1.2.1.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/7c/fc/edbfcb4574ec3cf0b68a0613dd1904c9139e3bf6dede792d2e7edcf13023/XStatic-JQuery-Migrate-1.2.1.1.tar.gz|XStatic-JQuery-Migrate-1.2.1.1
XStatic_JQuery.quicksearch-2.0.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ea/ab/f934d06a78ce2c6bb594e9a426f6966b3192c4c279467c9898be6fd284d3/XStatic-JQuery.quicksearch-2.0.3.1.tar.gz|XStatic-JQuery.quicksearch-2.0.3.1
XStatic_JQuery.TableSorter-2.14.5.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/c1/6c/d6b0807906af90536e793a3b23cca557869fa5a27156639f0029de8b1f1f/XStatic-JQuery.TableSorter-2.14.5.1.tar.gz|XStatic-JQuery.TableSorter-2.14.5.1
XStatic_jquery_ui-1.12.1.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/e6/5a/883b22dad1d3e01708312d71c5bc63d543d66cef9b448c1cf85379d64fb3/XStatic-jquery-ui-1.12.1.1.tar.gz|XStatic-jquery-ui-1.12.1.1
XStatic_mdi-1.6.50.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/73/49/13b9f7ce9fbcc7fabe086b7ac1b056118cbd4c9abf185e01cc4a54631136/XStatic_mdi-1.6.50.2-py2.py3-none-any.whl
XStatic_objectpath-1.2.1.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/23/6c/56de25d9d3be430e7de2fcf4baac10279dad78d7b16cbda339cf014c2fe5/XStatic-objectpath-1.2.1.0.tar.gz|XStatic-objectpath-1.2.1.0
XStatic_Rickshaw-1.5.0.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/45/c6/39aa4d02ea96b04ff372d1e3558587155790b1c5444855a97b89c255be38/XStatic-Rickshaw-1.5.0.0.tar.gz|XStatic-Rickshaw-1.5.0.0
XStatic_Spin-1.2.5.2-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/af/21/cca7f0b7abfe008cdd03dd4c4255aad3087f4a892a010c0f6f1452d7344b/XStatic-Spin-1.2.5.2.tar.gz|XStatic-Spin-1.2.5.2
XStatic_term.js-0.0.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/63/7a/7bfec29f5f28fdda7170ebbbb2204aeb1d33d6050f3476a807590de06434/XStatic-term.js-0.0.7.0.tar.gz|XStatic-term.js-0.0.7.0
XStatic_tv4-1.2.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/2b/26/b07115af27b339c861b8c9a775a621524b421c898e26e015880dfb888c49/XStatic-tv4-1.2.7.0.tar.gz|XStatic-tv4-1.2.7.0
xvfbwrapper-0.2.9-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/57/b6/4920eabda9b49630dea58745e79f9919aba6408d460afe758bf6e9b21a04/xvfbwrapper-0.2.9.tar.gz|xvfbwrapper-0.2.9
yappi-1.0-cp27-cp27mu-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/d2/92/7cd637a19fa2a10c0e55a44f8b36bcb83f0e1943ba8f1fb5edb15c819f2e/yappi-1.0.tar.gz|yappi-1.0
zVMCloudConnector-1.4.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/11/92/9f704de9759816e7b9897b9fb41285b421498b4642551b6fbcccd2850008/zVMCloudConnector-1.4.1.tar.gz|zVMCloudConnector-1.4.1

3
build-tools/build-wheels/docker/stable-wheels.cfg
View File

@ -17,7 +17,6 @@ cmd2-0.8.9-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/e9/
construct-2.8.22-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/e5/c6/3e3aeef38bb0c27364af3d21493d9690c7c3925f298559bca3c48b7c9419/construct-2.8.22.tar.gz|construct-2.8.22 construct-2.8.22-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/e5/c6/3e3aeef38bb0c27364af3d21493d9690c7c3925f298559bca3c48b7c9419/construct-2.8.22.tar.gz|construct-2.8.22
crc16-0.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz|crc16-0.1.1|fix_setup crc16-0.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/a6/e0/70a44c4385f2b33df82e518005aae16b5c1feaf082c73c0acebe3426fc0a/crc16-0.1.1.tar.gz|crc16-0.1.1|fix_setup
demjson-2.2.4-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/96/67/6db789e2533158963d4af689f961b644ddd9200615b8ce92d6cad695c65a/demjson-2.2.4.tar.gz|demjson-2.2.4|fix_setup demjson-2.2.4-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/96/67/6db789e2533158963d4af689f961b644ddd9200615b8ce92d6cad695c65a/demjson-2.2.4.tar.gz|demjson-2.2.4|fix_setup
Django-2.1.5-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/36/50/078a42b4e9bedb94efd3e0278c0eb71650ed9672cdc91bd5542953bec17f/Django-2.1.5-py3-none-any.whl
django_debreach-2.0.1-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/2a/92/8c363cf5d1ee33d4c3b999b41c127c5cd3c64d4c20aa47bdfb6c386c9309/django_debreach-2.0.1-py3-none-any.whl django_debreach-2.0.1-py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/2a/92/8c363cf5d1ee33d4c3b999b41c127c5cd3c64d4c20aa47bdfb6c386c9309/django_debreach-2.0.1-py3-none-any.whl
django_floppyforms-1.8.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/d2/498b883ac309b56b70c26877974bd50927615dd3f6433f5463e2668b1128/django_floppyforms-1.8.0-py2.py3-none-any.whl django_floppyforms-1.8.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/a9/d2/498b883ac309b56b70c26877974bd50927615dd3f6433f5463e2668b1128/django_floppyforms-1.8.0-py2.py3-none-any.whl
django_pyscss-2.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/4b/7f/d771802305184aac6010826f60a0b2ecaa3f57d19ab0e405f0c8db07e809/django-pyscss-2.0.2.tar.gz|django-pyscss-2.0.2 django_pyscss-2.0.2-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/4b/7f/d771802305184aac6010826f60a0b2ecaa3f57d19ab0e405f0c8db07e809/django-pyscss-2.0.2.tar.gz|django-pyscss-2.0.2
@ -46,6 +45,7 @@ lz4-0.9.0-cp36-cp36m-linux_x86_64.whl|git|https://github.com/python-lz4/python-l
Mako-1.1.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/50/78/f6ade1e18aebda570eed33b7c534378d9659351cadce2fcbc7b31be5f615/Mako-1.1.2-py2.py3-none-any.whl Mako-1.1.2-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/50/78/f6ade1e18aebda570eed33b7c534378d9659351cadce2fcbc7b31be5f615/Mako-1.1.2-py2.py3-none-any.whl
marathon-0.12.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/41/66/814432693297dfb076958ae5ac781e3a88fd70d335473a57f4f2c6329515/marathon-0.12.0-py2.py3-none-any.whl marathon-0.12.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/41/66/814432693297dfb076958ae5ac781e3a88fd70d335473a57f4f2c6329515/marathon-0.12.0-py2.py3-none-any.whl
MarkupSafe-1.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz|MarkupSafe-1.1.1 MarkupSafe-1.1.1-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/b9/2e/64db92e53b86efccfaea71321f597fa2e1b2bd3853d8ce658568f7a13094/MarkupSafe-1.1.1.tar.gz|MarkupSafe-1.1.1
migrate-0.3.8-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ce/31/1a4cbf8dc0536c55f41072e8ea37b3df1e412262dc731c57e5bb099eb9b2/migrate-0.3.8.tar.gz|migrate-0.3.8
mox-0.5.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/a1/64740c638cc5fae807022368f4141700518ee343b53eb3e90bf3cc15a4d4/mox-0.5.3.tar.gz|mox-0.5.3|fix_setup mox-0.5.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/0c/a1/64740c638cc5fae807022368f4141700518ee343b53eb3e90bf3cc15a4d4/mox-0.5.3.tar.gz|mox-0.5.3|fix_setup
mpmath-1.1.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32/mpmath-1.1.0.tar.gz|mpmath-1.1.0|fix_setup mpmath-1.1.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32/mpmath-1.1.0.tar.gz|mpmath-1.1.0|fix_setup
msgpack_python-0.4.8-cp36-cp36m-linux_x86_64.whl|git|https://github.com/msgpack/msgpack-python.git|msgpack-python|0.4.8 msgpack_python-0.4.8-cp36-cp36m-linux_x86_64.whl|git|https://github.com/msgpack/msgpack-python.git|msgpack-python|0.4.8
@ -113,6 +113,7 @@ repoze.lru-0.7-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/12/b
requests_aws-0.1.8-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/5e/2f/4da17752036c04cf4c9af7a2da0d41ef2205043f1c61008006475aa24b8b/requests-aws-0.1.8.tar.gz|requests-aws-0.1.8 requests_aws-0.1.8-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/5e/2f/4da17752036c04cf4c9af7a2da0d41ef2205043f1c61008006475aa24b8b/requests-aws-0.1.8.tar.gz|requests-aws-0.1.8
restructuredtext_lint-1.3.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/62/76/bd8760de759fb74d7863e6935200af101cb128a7de008741a4e22341d03c/restructuredtext_lint-1.3.0.tar.gz|restructuredtext_lint-1.3.0 restructuredtext_lint-1.3.0-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/62/76/bd8760de759fb74d7863e6935200af101cb128a7de008741a4e22341d03c/restructuredtext_lint-1.3.0.tar.gz|restructuredtext_lint-1.3.0
retrying-1.3.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/44/ef/beae4b4ef80902f22e3af073397f079c96969c69b2c7d52a57ea9ae61c9d/retrying-1.3.3.tar.gz|retrying-1.3.3 retrying-1.3.3-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/44/ef/beae4b4ef80902f22e3af073397f079c96969c69b2c7d52a57ea9ae61c9d/retrying-1.3.3.tar.gz|retrying-1.3.3
rfc3986-1.4.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/78/be/7b8b99fd74ff5684225f50dd0e865393d2265656ef3b4ba9eaaaffe622b8/rfc3986-1.4.0-py2.py3-none-any.whl
rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl|pypi|https://files.pythonhosted.org/packages/62/ee/574b170bbe7a059314e7239305cb829379232a408901585019e012e71170/rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl|pypi|https://files.pythonhosted.org/packages/62/ee/574b170bbe7a059314e7239305cb829379232a408901585019e012e71170/rjsmin-1.1.0-cp36-cp36m-manylinux1_x86_64.whl
rtslib_fb-2.1.71-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9e/1b/c26bc038888b1e6042d35ec97599cef05181fb6a7a7ecdbb0c041c3f50ea/rtslib-fb-2.1.71.tar.gz|rtslib-fb-2.1.71| rtslib_fb-2.1.71-py3-none-any.whl|tar|https://files.pythonhosted.org/packages/9e/1b/c26bc038888b1e6042d35ec97599cef05181fb6a7a7ecdbb0c041c3f50ea/rtslib-fb-2.1.71.tar.gz|rtslib-fb-2.1.71|
scandir-1.10.0-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz|scandir-1.10.0 scandir-1.10.0-cp36-cp36m-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/df/f5/9c052db7bd54d0cbf1bc0bb6554362bba1012d03e5888950a4f5c5dadc4e/scandir-1.10.0.tar.gz|scandir-1.10.0

13
build-tools/build-wheels/openstack.cfg Normal file
View File

@ -0,0 +1,13 @@
# This file specifies constraint/requirement URLs for current and python2
# openstack branches
# Current/stable
STABLE_OPENSTACK_REQ_URL="https://raw.githubusercontent.com/openstack/requirements/stable/ussuri"
# Current/experimental (for dev images)
MASTER_OPENSTACK_REQ_URL="https://raw.githubusercontent.com/openstack/requirements/master"
# Python2/stable
STABLE_OPENSTACK_REQ_URL_PY2="https://opendev.org/openstack/requirements/raw/commit/2da5c5045118b0e36fb14427872e4b9b37335071"
# Python2/experimental (for dev images)
MASTER_OPENSTACK_REQ_URL_PY2="https://raw.githubusercontent.com/openstack/requirements/stable/train"

2
build-tools/build_iso/image-dev.inc
View File

@ -2,3 +2,5 @@
# #
# They are exceptional packages only to be included in developer builds # They are exceptional packages only to be included in developer builds
enable-dev-patch enable-dev-patch
fio
dstat

4
build-tools/build_iso/image.inc
View File

@ -81,3 +81,7 @@ kata-runtime
# For nvme disk firmware update # For nvme disk firmware update
nvme-cli nvme-cli
# Add openscap tools
openscap
openscap-scanner

5
build-tools/create-yum-conf
View File

@ -63,11 +63,16 @@ if [ ! -f "$MY_YUM_CONF" ]; then
mock_cfg_to_yum_conf.py "$MOCK_CFG_PROTO" > "$MY_YUM_CONF" mock_cfg_to_yum_conf.py "$MOCK_CFG_PROTO" > "$MY_YUM_CONF"
sed -i "s%\[main\]%&\ncachedir=$YUM_CACHE%" "$MY_YUM_CONF" sed -i "s%\[main\]%&\ncachedir=$YUM_CACHE%" "$MY_YUM_CONF"
sed -i "s%logfile=.*%logfile=$YUM_DIR/yum.log%" "$MY_YUM_CONF" sed -i "s%logfile=.*%logfile=$YUM_DIR/yum.log%" "$MY_YUM_CONF"
# eg: LOCAL_BASE/MY_BUILD_DIR => file:///MY_BUILD_DIR
sed -i "s%LOCAL_BASE%file://%g" "$MY_YUM_CONF" sed -i "s%LOCAL_BASE%file://%g" "$MY_YUM_CONF"
sed -i "s%MIRROR_BASE%file:///import/mirrors%g" "$MY_YUM_CONF" sed -i "s%MIRROR_BASE%file:///import/mirrors%g" "$MY_YUM_CONF"
sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g" "$MY_YUM_CONF" sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g" "$MY_YUM_CONF"
# eg: file:///MY_BUILD_DIR => file:///localdisk/loadbuild/...
sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR%g" "$MY_YUM_CONF" sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR%g" "$MY_YUM_CONF"
sed -i "s%/MY_REPO_DIR%$MY_REPO%g" "$MY_YUM_CONF" sed -i "s%/MY_REPO_DIR%$MY_REPO%g" "$MY_YUM_CONF"
# eg = MY_BUILD_DIR/xyz => /localdisk/loadbuild/.../xyz
sed -i "s%MY_BUILD_DIR%$MY_BUILD_DIR%g" "$MY_YUM_CONF"
sed -i "s%MY_REPO_DIR%$MY_REPO%g" "$MY_YUM_CONF"
else else
echo "ERROR: Could not find yum.conf or MOCK_CFG_PROTO" echo "ERROR: Could not find yum.conf or MOCK_CFG_PROTO"
exit 1 exit 1

5
build-tools/modify-build-cfg
View File

@ -83,11 +83,16 @@ if [ ! -f $FILE ]; then
exit 1 exit 1
fi fi
# eg: LOCAL_BASE/MY_BUILD_DIR => http://127.0.0.1:8088/MY_BUILD_DIR
sed -i "s%LOCAL_BASE%http://127.0.0.1:8088%g" "$FILE" sed -i "s%LOCAL_BASE%http://127.0.0.1:8088%g" "$FILE"
sed -i "s%MIRROR_BASE%http://127.0.0.1:8088%g" "$FILE" sed -i "s%MIRROR_BASE%http://127.0.0.1:8088%g" "$FILE"
sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g" "$FILE" sed -i "s%BUILD_ENV%$MY_BUILD_ENVIRONMENT%g" "$FILE"
# eg http://127.0.0.1:8088/MY_BUILD_DIR => http://12.0.0.1:8088/localdisk/loadbuild/...
sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR_TOP%g" "$FILE" sed -i "s%/MY_BUILD_DIR%$MY_BUILD_DIR_TOP%g" "$FILE"
sed -i "s%/MY_REPO_DIR%$MY_REPO%g" "$FILE" sed -i "s%/MY_REPO_DIR%$MY_REPO%g" "$FILE"
# eg = MY_BUILD_DIR/xyz => /localdisk/loadbuild/.../xyz
sed -i "s%MY_BUILD_DIR%$MY_BUILD_DIR_TOP%g" "$FILE"
sed -i "s%MY_REPO_DIR%$MY_REPO%g" "$FILE"
# Disable all local-* repos for the build-types other than the current one # Disable all local-* repos for the build-types other than the current one
for bt in std rt; do for bt in std rt; do

124
build-tools/repo-utils.sh
View File

@ -140,6 +140,50 @@ repo_is_project () {
} }
#
# manifest_get_revision_of_project <manifest> <project-name>
#
# Extract the revision of a project within the manifest.
# The default revision is supplied in the absence
# of an explicit project revision.
#
# manifest = Path to manifest.
# project-name = name of project.
#
manifest_get_revision_of_project () {
local manifest="${1}"
local project="${2}"
local default_revision=""
local revision=""
default_revision=$(manifest_get_default_revision "${manifest}")
revision=$(grep '<project' "${manifest}" | \
grep -e "name=${project}" -e "name=\"${project}\"" | \
grep 'revision=' | \
sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g")
if [ "${revision}" != "" ]; then
echo "${revision}"
elif [ "${default_revision}" != "" ]; then
echo "${default_revision}"
else
return 1
fi
}
#
# manifest_get_default_revision <manifest>
#
# Extract the default revision of the manifest, if any.
#
# manifest = Path to manifest.
#
manifest_get_default_revision () {
local manifest="${1}"
grep '<default' $manifest |sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g"
}
# #
# manifest_set_revision <old_manifest> <new_manifest> <revision> <lock_down> <project-list> # manifest_set_revision <old_manifest> <new_manifest> <revision> <lock_down> <project-list>
# #
@ -149,8 +193,10 @@ repo_is_project () {
# revision = A branch, tag ,or sha. Branch and SHA can be used # revision = A branch, tag ,or sha. Branch and SHA can be used
# directly, but repo requires that a tag be in the form # directly, but repo requires that a tag be in the form
# "refs/tags/<tag-name>". # "refs/tags/<tag-name>".
# lock_down = 0 or 1. If 1, set a revision on all other non-listed # lock_down = 0,1 or 2. If 2, set a revision on all other non-listed
# projects to equal the SHA of the current git head. # projects to equal the SHA of the current git head.
# If 1, similar to 2, but only if the project doesn't have
# some other form of revision specified.
# project-list = A space seperated list of projects. Listed projects # project-list = A space seperated list of projects. Listed projects
# will have their revision set to the provided revision # will have their revision set to the provided revision
# value. # value.
@ -160,9 +206,11 @@ manifest_set_revision () {
local new_manifest="${2}" local new_manifest="${2}"
local revision="${3}" local revision="${3}"
local lock_down="${4}" local lock_down="${4}"
shift 4 local set_default="${5}"
shift 5
local projects="${@}" local projects="${@}"
local old_default_revision=""
local repo_root_dir="" local repo_root_dir=""
local line="" local line=""
local FOUND=0 local FOUND=0
@ -192,14 +240,35 @@ manifest_set_revision () {
return 1 return 1
fi fi
old_default_revision=$(manifest_get_default_revision "${old_manifest}")
if [ ${set_default} -eq 1 ] && [ "${old_default_revision}" == "" ]; then
# We only know how to alter an existing default revision, not set a
# new one, so continue without setting a default.
set_default=0
fi
while IFS= read -r line; do while IFS= read -r line; do
echo "${line}" | grep -q '<project' echo "${line}" | grep -q '<project'
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
# Line does not define a project, do not modify # Line does not define a project
if [ ${set_default} -eq 0 ] || [ "${old_default_revision}" == "" ]; then
# No further processing, do not modify
echo "${line}" echo "${line}"
continue continue
fi fi
# ok setting the default
echo "${line}" | grep -q '<default'
if [ $? -ne 0 ]; then
# Line does not set defaults, do not modify
echo "${line}"
continue
fi
echo "${line}" | sed "s#${old_default_revision}#${revision}#"
continue
fi
# check if project name is selected # check if project name is selected
FOUND=0 FOUND=0
for project in ${projects}; do for project in ${projects}; do
@ -211,15 +280,58 @@ manifest_set_revision () {
done done
rev=${revision} rev=${revision}
old_rev=$(echo "${line}" | grep 'revision=' | sed -e 's#.*revision=\([^ ]*\).*#\1#' -e 's#"##g' -e "s#'##g")
if [ $FOUND -eq 0 ]; then if [ $FOUND -eq 0 ]; then
# A non-selected project # A non-selected project
if [ ${lock_down} -eq 0 ]; then if [ ${lock_down} -eq 2 ]; then
# Hard lock-down
# Set the revision to current HEAD SHA.
path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
rev=$(cd "${path}"; git rev-parse HEAD)
elif [ ${lock_down} -eq 1 ] && [ "${old_rev}" == "" ]; then
# Soft lock-down but no revision is currently set on the project.
# Set the revision to current HEAD SHA.
path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
rev=$(cd "${path}"; git rev-parse HEAD)
elif [ ${lock_down} -eq 1 ] && [ "${old_rev}" == "master" ]; then
# Soft lock-down and project has revision set to 'master' which is definitly unstable.
# Set the revision to current HEAD SHA.
path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')"
rev=$(cd "${path}"; git rev-parse HEAD)
else
if [ ${set_default} -eq 0 ] || [ "${old_default_revision}" == "${revision}" ]; then
# default revision unchanged, leave it be
echo "${line}" echo "${line}"
continue continue
fi fi
path="${repo_root_dir}/$(echo "${line}" | sed 's#.*path="\([^"]*\)".*#\1#')" if [ "${old_rev}" != "" ]; then
rev=$(cd "${path}"; git rev-parse HEAD) # Non-selected project has an explicit revision, leave it be
echo "${line}"
continue
fi
# The default revision will change, but this project, which
# relied on the old default, is not supposed to change,
# so it's revision must now be explicitly set to point to
# the old default revision.
rev="${old_default_revision}"
fi
else
# A selected project
if [ ${set_default} -eq 1 ]; then
# Selected project does not need to set a revision.
# The revision will come from the default
if [ "${old_rev}" == "" ]; then
# project has no revision to delete
echo "${line}"
continue
fi
# delete any revision present
echo "${line}" | sed 's#\(.*\)revision=[^ ]*\(.*\)#\1\2#'
continue
fi
fi fi
# Need to set revision on selected project # Need to set revision on selected project

1
build-tools/repo_files/mock.cfg.centos7.all.proto
View File

@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['dist'] = 'el7' # only useful for --resultdir variable subst
config_opts['releasever'] = '7' config_opts['releasever'] = '7'
config_opts['package_manager'] = 'yum' config_opts['package_manager'] = 'yum'
config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
config_opts['use_bootstrap'] = False config_opts['use_bootstrap'] = False
config_opts['use_bootstrap_image'] = False config_opts['use_bootstrap_image'] = False
config_opts['rpmbuild_networking'] = False config_opts['rpmbuild_networking'] = False

1
build-tools/repo_files/mock.cfg.centos7.distro.proto
View File

@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['dist'] = 'el7' # only useful for --resultdir variable subst
config_opts['releasever'] = '7' config_opts['releasever'] = '7'
config_opts['package_manager'] = 'yum' config_opts['package_manager'] = 'yum'
config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
config_opts['use_bootstrap'] = False config_opts['use_bootstrap'] = False
config_opts['use_bootstrap_image'] = False config_opts['use_bootstrap_image'] = False
config_opts['rpmbuild_networking'] = False config_opts['rpmbuild_networking'] = False

1
build-tools/repo_files/mock.cfg.centos7.proto
View File

@ -5,6 +5,7 @@ config_opts['chroot_setup_cmd'] = 'install @buildsys-build'
config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['dist'] = 'el7' # only useful for --resultdir variable subst
config_opts['releasever'] = '7' config_opts['releasever'] = '7'
config_opts['package_manager'] = 'yum' config_opts['package_manager'] = 'yum'
config_opts['yum_builddep_command'] = 'MY_REPO_DIR/build-tools/yum-builddep-wrapper'
config_opts['use_bootstrap'] = False config_opts['use_bootstrap'] = False
config_opts['use_bootstrap_image'] = False config_opts['use_bootstrap_image'] = False
config_opts['rpmbuild_networking'] = False config_opts['rpmbuild_networking'] = False

1
build-tools/update-pxe-network-installer
View File

@ -100,6 +100,7 @@ find_and_copy_rpm 'e1000e kernel module' 'kmod-e1000e-[0-9]*.x86_64.
find_and_copy_rpm 'i40e kernel module' 'kmod-i40e-[0-9]*.x86_64.rpm' std "$kernel_rpms_std" find_and_copy_rpm 'i40e kernel module' 'kmod-i40e-[0-9]*.x86_64.rpm' std "$kernel_rpms_std"
find_and_copy_rpm 'ixgbe kernel module' 'kmod-ixgbe-[0-9]*.x86_64.rpm' std "$kernel_rpms_std" find_and_copy_rpm 'ixgbe kernel module' 'kmod-ixgbe-[0-9]*.x86_64.rpm' std "$kernel_rpms_std"
find_and_copy_rpm 'mlnx-ofa kernel module' 'mlnx-ofa_kernel-modules-[0-9]*.x86_64.rpm' std "$kernel_rpms_std" find_and_copy_rpm 'mlnx-ofa kernel module' 'mlnx-ofa_kernel-modules-[0-9]*.x86_64.rpm' std "$kernel_rpms_std"
find_and_copy_rpm 'ice kernel module' 'kmod-ice-[0-9]*.x86_64.rpm' std "$kernel_rpms_std"
echo " -------- successfully found standard kernel rpm and related kernel modules --------" echo " -------- successfully found standard kernel rpm and related kernel modules --------"
echo "" echo ""

267
build-tools/url_utils.sh Executable file
View File

@ -0,0 +1,267 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# A set of bash utility functions to parse a URL.
# This script was originated by Scott Little
#
url_protocol () {
local URL="$1"
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
if echo "$URL" | grep -q '[:][/][/]' ;then
echo "$URL" | sed 's#^\(.*\)://.*$#\1#'
else
echo "http"
fi
return 0
}
url_login () {
local URL="$1"
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
echo "$URL" | sed 's#^.*://\([^/]*\)/.*$#\1#'
return 0
}
url_user () {
local URL="$1"
local LOGIN
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
url_login "$URL" | sed -e '/@/! s#.*## ; s#\([^@]*\)@.*#\1#'
if [ ${PIPESTATUS[0]} -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
return 1
fi
return 0
}
url_port () {
local URL="$1"
local LOGIN
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
url_login "$URL" | sed -e '/:/! s#.*## ; s#[^:]*:\([^:]*\)#\1#'
if [ ${PIPESTATUS[0]} -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
return 1
fi
return 0
}
url_server () {
local URL="$1"
local LOGIN
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
url_login "$URL" | sed 's#^.*@## ; s#:.*$##'
if [ ${PIPESTATUS[0]} -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed"
return 1
fi
return 0
}
url_path () {
local URL="$1"
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
echo "$URL" | sed 's#^.*://[^/]*/\(.*\)$#\1#'
return 0
}
#
# url_path_to_fs_path:
#
# Convert url format path to file system format.
# e.g. replace %20 with ' '.
#
# Note: Does NOT test the output path to ensure there are
# no illegal file system characters.
#
url_path_to_fs_path () {
local INPUT_PATH="$1"
local TEMP
if [ "$INPUT_PATH" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
# Deviate from URI spec by not substituding '+' with ' '.
# It would alias '%20' and we need unique mappings.
# TEMP="${INPUT_PATH//+/ }"
TEMP="$INPUT_PATH"
printf '%b' "${TEMP//%/\\x}"
return 0
}
#
# fs_path_to_url_path:
#
# Convert file system format path to url format.
# e.g. replace ' ' with %20.
#
fs_path_to_url_path () {
local INPUT_PATH="$1"
local LENGTH
local POS
local CHAR
if [ "$INPUT_PATH" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
LENGTH="${#INPUT_PATH}"
for (( POS = 0; POS < LENGTH; POS++ )); do
CHAR="${1:POS:1}"
case $CHAR in
[/a-zA-Z0-9.~_-])
# Reference https://metacpan.org/pod/URI::Escape
printf "$CHAR"
;;
*)
printf '%%%02X' "'$CHAR"
;;
esac
done
return 0
}
#
# normalize_path:
#
# 1) replace // with /
# 2) replace /./ with /
# 3) Remove trailing /
# 4) Remove leading ./
#
normalize_path () {
local INPUT_PATH="$1"
if [ "$INPUT_PATH" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
echo "$INPUT_PATH" | sed 's#[/]\+#/#g ; s#[/][.][/]#/#g ; s#/$## ; s#^[.]/##'
return 0
}
#
# repo_url_to_sub_path:
#
repo_url_to_sub_path () {
local URL="$1"
local FAMILY=""
local SERVER=""
local URL_PATH=""
local FS_PATH=""
if [ "$URL" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
# set FAMILY from URL
echo $URL | grep -q 'centos[.]org' && FAMILY=centos
echo $URL | grep -q 'fedoraproject[.]org[/]pub[/]epel' && FAMILY=epel
SERVER=$(url_server "$URL")
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_server '$URL'"
return 1
fi
URL_PATH="$(url_path "$URL")"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_path '$URL'"
return 1
fi
FS_PATH="$(url_path_to_fs_path "$URL_PATH")"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): url_path_to_fs_path '$URL_PATH'"
return 1
fi
FS_PATH="$(normalize_path "$FS_PATH")"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path '$FS_PATH'"
return 1
fi
normalize_path "./$FAMILY/$SERVER/$FS_PATH"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path './$FAMILY/$SERVER/$FS_PATH'"
return 1
fi
return 0
}
CENGN_PROTOCOL="http"
CENGN_HOST="mirror.starlingx.cengn.ca"
CENGN_PORT="80"
CENGN_URL_ROOT="mirror"
url_to_stx_mirror_url () {
local URL="$1"
local DISTRO="$2"
local URL_PATH=""
local FS_PATH=""
if [ "$URL" == "" ] || [ "$DISTRO" == "" ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): empty argument"
return 1
fi
FS_PATH="$(repo_url_to_sub_path "$URL")"
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): repo_url_to_sub_path '$URL'"
return 1
fi
URL_PATH=$(fs_path_to_url_path "$FS_PATH")
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): fs_path_to_url_path '$FS_PATH'"
return 1
fi
echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH"
return 0
}

66
build-tools/yum-builddep-wrapper Executable file
View File

@ -0,0 +1,66 @@
#!/bin/bash
# Old versions of yum-builddep leave a stale yum.pid file behind.
# Remove that file if necessary after yum-builddep exits
# find yum-builddep
YUM_BUILDDEP=$(which yum-builddep 2>/dev/null)
# dnf: call it directly
if [[ -z $YUM_BUILDDEP ]] || grep -q -F dnf.cli "$YUM_BUILDDEP" ; then
yum-builddep "$@"
exit $?
fi
# old yum: scan command line for --installroot
ROOT_PREFIX=
YUM_CONF=/etc/yum.conf
find_root_prefix() {
while [[ "$#" -gt 0 ]] ; do
case "$1" in
--installroot)
ROOT_PREFIX="$2"
shift
;;
--installroot=*)
ROOT_PREFIX="${1#*=}"
;;
-c|--config)
YUM_CONF="$2"
shift
;;
--config=*)
YUM_CONF="${1#*=}"
;;
esac
shift
done
if [[ -z "$ROOT_PREFIX" ]] && [[ -f "$YUM_CONF" ]] ; then
ROOT_PREFIX=$(sed -rn 's/^\s*installroot\s*=\s*(\S+)\s*$/\1/p' $YUM_CONF)
fi
}
find_root_prefix "$@"
# ignore signals -- always wait for yum-builddep
trap "" INT TERM HUP PIPE
# run it in the background to get its PID
"$YUM_BUILDDEP" "$@" &
pid="$!"
# wait for it
wait "$pid"
res="$?"
# if yum.pid remains and contains yum-builddep's PID, delete it
if [[ -f "${ROOT_PREFIX}/run/yum.pid" ]] ; then
lock_owner=
read lock_owner <"${ROOT_PREFIX}/run/yum.pid" || :
if [[ -n $lock_owner && $lock_owner == $pid ]] ; then
rm -f "${ROOT_PREFIX}/run/yum.pid"
fi
fi
# done
exit $res