From f7bf479a58f3faf3f29b88d4559fba5627b0d9f4 Mon Sep 17 00:00:00 2001 From: Saul Wold Date: Thu, 30 Aug 2018 14:35:53 -0700 Subject: [PATCH 01/13] read releasever from yum.conf iff -c yum.conf is set This solves a problem when the default releasever is not set for distros that dont have a YUM as a standard tool. Read from the yum.conf if set on the command line since that is mostly likely going to be when we need to know this, in the default case (no -c yum.conf) it will get the correct value from the system. Change-Id: I21b0e8f881d054d8cdf6cdd831836a7f3e3020b2 Signed-off-by: Saul Wold --- centos-mirror-tools/dl_rpms.sh | 12 +++++++----- centos-mirror-tools/mirror-check.sh | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 29c0ae48..67cf7924 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -40,6 +40,7 @@ get_from() { # By default, we use "sudo" and we don't use a local yum.conf. These can # be overridden via flags. SUDOCMD="sudo -E" +RELEASEVER="--releasever=7" YUMCONFOPT="" CLEAN_LOGS_ONLY=0 @@ -59,6 +60,7 @@ while getopts "c:nxh" o; do c) # Use an alternate yum.conf YUMCONFOPT="-c $OPTARG" + RELEASEVER="--$(grep releasever= ${OPTARG})" ;; h) # Help @@ -215,11 +217,11 @@ download () { echo " ------ using $SFILE to search $rpm_name ------" # Yumdownloader with the appropriate flag for src, noarch or x86_64 if [ "$_type" == "src" ];then - download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --arcgglist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64" fi fi else @@ -296,7 +298,7 @@ download () { } # Prime the cache -${SUDOCMD} yum ${YUMCONFOPT} makecache +${SUDOCMD} yum ${YUMCONFOPT} ${RELEASEVER} makecache # Download files if [ -s "$rpms_list" ];then diff --git a/centos-mirror-tools/mirror-check.sh b/centos-mirror-tools/mirror-check.sh index 6a8c4776..43a92a17 100755 --- a/centos-mirror-tools/mirror-check.sh +++ b/centos-mirror-tools/mirror-check.sh @@ -39,6 +39,7 @@ truncate -s 0 $ERROR_LOG_FILE retcode=0 extra_opts="" + usage() { echo "$0 [-c ]" echo "" @@ -76,7 +77,7 @@ get_repoquery_info() { else repoquery_opts= fi - repoquery $extra_opts -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ + repoquery $extra_opts ${RELEASEVER} -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ $repoquery_opts "$_package_name" } @@ -114,6 +115,7 @@ while getopts "c:" opt; do case $opt in c) extra_opts="-c ${OPTARG}" + RELEASEVER="--$(grep releasever= ${OPTARG})" ;; \?) echo "Invalid option: -$OPTARG" >&2 @@ -124,7 +126,7 @@ while getopts "c:" opt; do done info "Getting yum cache" -if ! yum $extra_opts makecache; then +if ! yum $extra_opts ${RELEASEVER} makecache; then error "There was a problem getting yum cache" exit 1 fi From 774ca8b6d0c0858a7a2894816fbcbf57c07ebb0a Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Fri, 17 Aug 2018 17:31:54 -0500 Subject: [PATCH 02/13] Support feature branches in branch-stx.sh This changes branch-stx.sh to not assume that a manifest file will be used to create the list of repos to process, and allow direct repo URLs on the command line. Change-Id: Ie2bbfa314b33d2b2b810ee479b4a0158e9d819bf Signed-off-by: Dean Troyer --- release/branch-stx.sh | 135 ++++++++++++++++++++++++++++++++---------- 1 file changed, 103 insertions(+), 32 deletions(-) diff --git a/release/branch-stx.sh b/release/branch-stx.sh index b4f53184..6a7b9284 100755 --- a/release/branch-stx.sh +++ b/release/branch-stx.sh @@ -1,11 +1,24 @@ #!/bin/bash -# branch-stx.sh - create STX branches based on today +# branch-stx.sh - create STX branches # -# branch-stx.sh [--dry-run] [] +# branch-stx.sh [--dry-run|-n] [-l] [-m ] [ ...] # -# * get the repo list from stx-manifest in both starlingx and stx-staging remotes -# * create a new branch -# * tag the new branch with an initial release identifier +# --dry-run|-n Do all work except pushing back to the remote repo. +# Useful to validate everything locally before pushing. +# +# -l List the repo URLS that would be processed and exit +# +# -m Extract the repo list from for starlingx +# and stx-staging remotes +# +# Specify one or more direct repo URLs to branch (ie git remote) +# These are appended to the list of repos extracted from the +# manifest if one is specified. +# +# For each repo: +# * create a new branch $BRANCH +# * tag the new branch with an initial release identifier if $TAG is set +# * update the .gitreview file to default to the new branch (Gerrit repos only) # # Some environment variables are available for modifying this script's behaviour: # @@ -17,18 +30,59 @@ # 'r/' (for periodic releases) to SERIES. # # TAG is the release tag that represents the actual release, derived by adding -# a 'patch' version to SERIES, initially '0'. +# a 'patch' version to SERIES, initially '0'. If TAG is unset no tag is created. +# +# Notes: +# * This script is used for creating milestone, release and feature branches. +# * The default action is to create a milestone branch with prefix 'm/'. +# * To create a release branch set BRANCH directly using a 'r/' prefix. +# * To create a feature branch set BRANCH directly using a 'f/' prefix and set +# TAG="" to skip tagging the branch point. +# * The detection to use Gerrit or Github is determined by the presence of +# 'git.starlingx.io' in the repo URL. This may be sub-optimal. The only actual +# difference in execution is .gitreview updates are only prepared for Gerrit repos. set -e -# Grab options -if [[ "$1" == "--dry-run" ]]; then - DRY_RUN=1 - shift; -fi +# Defaults +MANIFEST="" -# Where to get the repo list -MANIFEST=${1:-default.xml} +optspec="lm:n-:" +while getopts "$optspec" o; do + case "${o}" in + # Hack in longopt support + -) + case "${OPTARG}" in + dry-run) + DRY_RUN=1 + ;; + *) + if [[ "$OPTERR" = 1 ]] && [[ "${optspec:0:1}" != ":" ]]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + + esac + ;; + l) + LIST=1 + ;; + m) + MANIFEST=${OPTARG} + ;; + n) + DRY_RUN=1 + ;; + esac +done +shift $((OPTIND-1)) + +# See if we can build a repo list +if [[ $# == 0 && -z $MANIFEST ]]; then + echo "ERROR: No repos to process" + echo "Usage: $0 [--dry-run|-n] [-l] [-m ] [ ...]" + exit 1 +fi # SERIES is the base of the branch and release tag names: year.month (YYYY.MM) SERIES=${SERIES:-$(date '+%Y.%m')} @@ -57,7 +111,7 @@ function update_gitreview { defaultbranch=$branch" echo "$grcontents" > .gitreview git add .gitreview - if ! git commit -s -m "Update .gitreview for $branch"; then + if git commit -s -m "Update .gitreview for $branch"; then if [[ -z $DRY_RUN ]]; then git review -t "create-${branch}" else @@ -68,21 +122,20 @@ defaultbranch=$branch" fi } -# branch_repo +# branch_repo function branch_repo { - local remote=$1 - local repo=$2 - local sha=$3 - local branch=$4 - local tag=$5 + local repo=$1 + local sha=$2 + local branch=$3 + local tag=$4 local repo_dir=${repo##*/} if [[ ! -d $repo_dir ]]; then - git clone $i $repo_dir || true + git clone $repo $repo_dir || true fi - cd $repo_dir + pushd $repo_dir >/dev/null git checkout master if ! git branch | grep $BRANCH; then @@ -90,11 +143,13 @@ function branch_repo { git branch $branch $sha fi - # tag branch point at $sha - git tag -s -m "Branch $branch" -f $tag $sha + if [[ -n $tag ]]; then + # tag branch point at $sha + git tag -s -m "Branch $branch" -f $tag $sha + fi # Push the new goodness back up - if [[ "$r" == "starlingx" ]]; then + if [[ "$repo" =~ "git.starlingx.io" ]]; then # Do the Gerrit way # set up gerrit remote @@ -118,14 +173,30 @@ function branch_repo { fi fi - cd - + popd >/dev/null } -for r in $REMOTES; do - repos=$($script_dir/getrepo.sh $MANIFEST $r) - # crap, convert github URLs to git: - repos=$(sed -e 's|https://github.com/starlingx-staging|git@github.com:starlingx-staging|g' <<<$repos) - for i in $repos; do - branch_repo $r $i HEAD $BRANCH $TAG +repo_list="" + +if [[ -n $MANIFEST ]]; then + # First get repos from the manifest + for r in $REMOTES; do + repos=$($script_dir/getrepo.sh $MANIFEST $r) + # crap, convert github URLs to git: + repos=$(sed -e 's|https://github.com/starlingx-staging|git@github.com:starlingx-staging|g' <<<$repos) + repo_list+=" $repos" done +fi + +if [[ $# != 0 ]]; then + # Then add whatever is on the command line + repo_list+=" $@" +fi + +for i in $repo_list; do + if [[ -z $LIST ]]; then + branch_repo $i HEAD $BRANCH $TAG + else + echo "$i" + fi done From 88ba4d5f229ab21bcf883f4e2792a0cc8c740455 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Wed, 5 Sep 2018 14:33:42 -0500 Subject: [PATCH 03/13] Add docs, releasenotes jobs Also: * Use build-openstack-docs-pti job template for docs * Use build-openstack-releasenotes job for release notes (We can't use the OpenStack releasenotes template as it includes publish jobs, stx needs its own) * Add newnote tox environment as convenience for creating new release notes, re-using the releasenotes venv. * Create a release summary note. * Remove the unused 'true' tox environment. * Fix doc/build in .gitignore Change-Id: Ibe9d298bcc76f023c0500038db55d8384f5ff731 Signed-off-by: Dean Troyer --- .gitignore | 2 +- .zuul.yaml | 4 ++++ .../notes/release-summary-9c35dd1bd9b020f1.yaml | 15 +++++++++++++++ tox.ini | 12 ++++++------ 4 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/release-summary-9c35dd1bd9b020f1.yaml diff --git a/.gitignore b/.gitignore index acb93792..cad81b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ centos-mirror-tools/logs/ centos-mirror-tools/output/ # Sphinx documentation -docs/build/ +doc/build/ # Release Notes documentation releasenotes/build diff --git a/.zuul.yaml b/.zuul.yaml index 7ccbd0a6..6c524d97 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,8 +1,12 @@ --- - project: + templates: + - build-openstack-docs-pti check: jobs: + - build-openstack-releasenotes - openstack-tox-linters gate: jobs: + - build-openstack-releasenotes - openstack-tox-linters diff --git a/releasenotes/notes/release-summary-9c35dd1bd9b020f1.yaml b/releasenotes/notes/release-summary-9c35dd1bd9b020f1.yaml new file mode 100644 index 00000000..8b4abd40 --- /dev/null +++ b/releasenotes/notes/release-summary-9c35dd1bd9b020f1.yaml @@ -0,0 +1,15 @@ +--- +prelude: > + These release notes cover the initial release of StarlingX. + The StarlingX Tools project is a new repository of tools used in the + development, build, test and release of StarlingX. +features: + - | + Deployment: Scripts to assist in deploying StarlingX in virtual + environments, supports both QEMU/KVM and VirtualBox. + - | + Mirror Tools: Scripts to build and maintain the mirror required to + complete the StarlingX build process. + - | + Release Tools: Scripts used to automate the steps in producing StarlingX + milestones and releases. diff --git a/tox.ini b/tox.ini index 47d8c1b6..963d5ee2 100644 --- a/tox.ini +++ b/tox.ini @@ -35,12 +35,6 @@ commands = [testenv:venv] commands = {posargs} -[testenv:true] -# Use same environment directory as venv to save space and install time -envdir = {toxworkdir}/venv -commands = true -whitelist_externals = true - [testenv:docs] basepython = python3 deps = -r{toxinidir}/doc/requirements.txt @@ -56,3 +50,9 @@ commands = rm -rf releasenotes/build sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html whitelist_externals = rm + +[testenv:newnote] +# Re-use the releasenotes venv +envdir = {toxworkdir}/releasenotes +deps = -r{toxinidir}/doc/requirements.txt +commands = reno new {posargs} From d56955c5f1d2f16e14ee0d033819c498ce092afe Mon Sep 17 00:00:00 2001 From: Jason McKenna Date: Thu, 6 Sep 2018 10:52:09 -0400 Subject: [PATCH 04/13] Check for releasever in yum.conf before grepping The dl_rpms.sh script may be passed an optional yum.conf file. If this file is used, then the releasever variable is read from the rather than use the default releasever=7. If the releasever is not defined in the file, the grep which searches for the variable will fail, and the dl_rpms.sh script exits with an error. This change checks for the presence of releasever before attempting to read it from the file. Change made to dl_rpms.sh and mirror-check.sh Tested using yum.conf files both with and without releasever present. Change-Id: Ie13241e5662188d75b5d147537915c0a35ea7397 Closes-Bug: 1791113 Signed-off-by: Jason McKenna --- centos-mirror-tools/dl_rpms.sh | 2 +- centos-mirror-tools/mirror-check.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 67cf7924..5fd66eef 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -60,7 +60,7 @@ while getopts "c:nxh" o; do c) # Use an alternate yum.conf YUMCONFOPT="-c $OPTARG" - RELEASEVER="--$(grep releasever= ${OPTARG})" + grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})" ;; h) # Help diff --git a/centos-mirror-tools/mirror-check.sh b/centos-mirror-tools/mirror-check.sh index 43a92a17..919cfb80 100755 --- a/centos-mirror-tools/mirror-check.sh +++ b/centos-mirror-tools/mirror-check.sh @@ -115,7 +115,7 @@ while getopts "c:" opt; do case $opt in c) extra_opts="-c ${OPTARG}" - RELEASEVER="--$(grep releasever= ${OPTARG})" + grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})" ;; \?) echo "Invalid option: -$OPTARG" >&2 From b8a52f5c583352d22a81dbb1e7563c2d39a515eb Mon Sep 17 00:00:00 2001 From: slin14 Date: Fri, 7 Sep 2018 06:03:32 +0800 Subject: [PATCH 05/13] Fix typo error for word archlist Story: 2003687 Task: 26211 Change-Id: Ia3712da38a0a47ce23a45d3f494e3f146801baf3 Signed-off-by: slin14 --- centos-mirror-tools/dl_rpms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 67cf7924..09f67513 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -220,7 +220,7 @@ download () { download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT} ${RELEASEVER} -C --source $SFILE" else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --arcgglist=noarch,x86_64" + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64" download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} ${RELEASEVER} $SFILE --archlist=noarch,x86_64" fi fi From c7a0f5113922bd8c321c6b7197f2f4fc4ef6c19a Mon Sep 17 00:00:00 2001 From: Jack Ding Date: Thu, 6 Sep 2018 16:16:25 -0400 Subject: [PATCH 06/13] Add memcached binaries Customization to memcached package is removed so no need to download source rpm anymore. Download the binary rpm instead. Story: 2002826 Task: 26346 Change-Id: I3acd809c81556350461f4e017f91b00a3bb97473 Signed-off-by: Jack Ding --- centos-mirror-tools/rpms_centos.lst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 00637f12..2af32954 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -800,7 +800,7 @@ mcstrans-0.3.4-5.el7.x86_64.rpm mdadm-4.0-13.el7.x86_64.rpm mdi-common-1.4.57.0-4.el7.noarch.rpm mdi-fonts-1.4.57.0-4.el7.noarch.rpm -memcached-1.4.39-1.el7.src.rpm +memcached-1.4.39-1.el7.x86_64.rpm mesa-libGLU-9.0.0-4.el7.x86_64.rpm mesa-libGLU-devel-9.0.0-4.el7.x86_64.rpm microcode_ctl-2.1-29.el7.x86_64.rpm From 3637aa6f78385a6106b3a46106938b2e8c1261ac Mon Sep 17 00:00:00 2001 From: slin14 Date: Fri, 7 Sep 2018 23:51:54 +0800 Subject: [PATCH 07/13] replace novnc with rpm package Story: 2003340 Task: 24378 No patch is needed for this package, so replace it with rpm package Change-Id: I21eecb9fb898cf0c7484eb03376fc4703c81204e Signed-off-by: slin14 --- centos-mirror-tools/rpms_3rdparties.lst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/rpms_3rdparties.lst b/centos-mirror-tools/rpms_3rdparties.lst index 6961bfc7..eaeec2ca 100644 --- a/centos-mirror-tools/rpms_3rdparties.lst +++ b/centos-mirror-tools/rpms_3rdparties.lst @@ -1,5 +1,5 @@ libvirt-python-3.5.0-1.fc24.src.rpm#https://libvirt.org/sources/python/libvirt-python-3.5.0-1.fc24.src.rpm -novnc-0.6.2-1.el7.src.rpm#http://cbs.centos.org/kojifiles/packages/novnc/0.6.2/1.el7/src/novnc-0.6.2-1.el7.src.rpm +novnc-0.6.2-1.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/novnc/0.6.2/1.el7/noarch/novnc-0.6.2-1.el7.noarch.rpm python2-httpbin-0.5.0-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-httpbin/0.5.0/6.el7/noarch/python2-httpbin-0.5.0-6.el7.noarch.rpm python2-pytest-httpbin-0.2.3-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-pytest-httpbin/0.2.3/6.el7/noarch/python2-pytest-httpbin-0.2.3-6.el7.noarch.rpm python2-pytest-mock-1.6.0-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-pytest-mock/1.6.0/2.el7/noarch/python2-pytest-mock-1.6.0-2.el7.noarch.rpm From e56cbd7890a32453085189c9a77f66baf1bde4a2 Mon Sep 17 00:00:00 2001 From: Michel Thebeau Date: Thu, 6 Sep 2018 15:43:56 -0400 Subject: [PATCH 08/13] deployment: libvirt: remove macs from xml The xml template is used to define both controllers, and so the two will have the same macs. Libvirt will fill in the omitted mac addresses. Related-Bug: #1790716 Change-Id: Ib9544b5b8cbc2e5cf8ea941129ee9720cba3df97 Signed-off-by: Michel Thebeau --- deployment/libvirt/controller.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deployment/libvirt/controller.xml b/deployment/libvirt/controller.xml index e346813e..1c786adc 100644 --- a/deployment/libvirt/controller.xml +++ b/deployment/libvirt/controller.xml @@ -65,7 +65,6 @@
- @@ -73,7 +72,6 @@
- @@ -82,7 +80,6 @@
- @@ -90,7 +87,6 @@
- From 10daff0c34546266dbb5b093a14d8932a576ffe9 Mon Sep 17 00:00:00 2001 From: Michel Thebeau Date: Thu, 6 Sep 2018 17:44:05 -0400 Subject: [PATCH 09/13] deployment: libvirt: remove pci, bus, controllers Some of the entries are invalid when changing the machine type, which is the next commit to follow this one. Libvirt will fill these things in correctly for the specified hardware. Related-Bug: #1790716 Change-Id: I674657cca117563de0c0ff04231224198623d3dd Signed-off-by: Michel Thebeau --- deployment/libvirt/compute.xml | 21 --------------------- deployment/libvirt/controller.xml | 23 ----------------------- 2 files changed, 44 deletions(-) diff --git a/deployment/libvirt/compute.xml b/deployment/libvirt/compute.xml index 85474c03..19e052ad 100644 --- a/deployment/libvirt/compute.xml +++ b/deployment/libvirt/compute.xml @@ -32,34 +32,18 @@ - -
- -
- - -
- - - - - - -
- -
@@ -67,21 +51,18 @@ -
-
-
@@ -101,11 +82,9 @@