From 7dd3b26d0a611e8c4a098ab34c2f3ffd421a9364 Mon Sep 17 00:00:00 2001 From: zhipengl Date: Thu, 9 Aug 2018 00:36:26 +0800 Subject: [PATCH 01/27] Switch kubernetes-1.10.0-1.el7.src.rpm from 3rd-party download to centos_repo Story: 2003342 Task: 24380 Change-Id: Icaa8f5ed96e300e6fdee70aa19b7ab405fb1b1c2 Signed-off-by: zhipengl --- centos-mirror-tools/rpms_from_3rd_parties.lst | 1 - centos-mirror-tools/rpms_from_centos_repo.lst | 1 + centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo | 8 ++++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/centos-mirror-tools/rpms_from_3rd_parties.lst b/centos-mirror-tools/rpms_from_3rd_parties.lst index 9061a2b0..6961bfc7 100644 --- a/centos-mirror-tools/rpms_from_3rd_parties.lst +++ b/centos-mirror-tools/rpms_from_3rd_parties.lst @@ -5,5 +5,4 @@ python2-pytest-httpbin-0.2.3-6.el7.noarch.rpm#http://cbs.centos.org/kojifiles/pa python2-pytest-mock-1.6.0-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-pytest-mock/1.6.0/2.el7/noarch/python2-pytest-mock-1.6.0-2.el7.noarch.rpm python2-storops-0.4.7-2.el7.noarch.rpm#http://cbs.centos.org/kojifiles/packages/python-storops/0.4.7/2.el7/noarch/python2-storops-0.4.7-2.el7.noarch.rpm python-gunicorn-19.7.1-1.fc27.src.rpm#https://kojipkgs.fedoraproject.org/packages/python-gunicorn/19.7.1/1.fc27/src/python-gunicorn-19.7.1-1.fc27.src.rpm -kubernetes-1.10.0-1.el7.src.rpm#http://cbs.centos.org/kojifiles/packages/kubernetes/1.10.0/1.el7/src/kubernetes-1.10.0-1.el7.src.rpm influxdb-0.9.5.1-1.x86_64.rpm#https://s3.amazonaws.com/influxdb/influxdb-0.9.5.1-1.x86_64.rpm diff --git a/centos-mirror-tools/rpms_from_centos_repo.lst b/centos-mirror-tools/rpms_from_centos_repo.lst index 34c232a1..188082ab 100644 --- a/centos-mirror-tools/rpms_from_centos_repo.lst +++ b/centos-mirror-tools/rpms_from_centos_repo.lst @@ -522,6 +522,7 @@ kmod-libs-20-21.el7.x86_64.rpm kpartx-0.4.9-119.el7.x86_64.rpm krb5-devel-1.15.1-19.el7.x86_64.rpm krb5-libs-1.15.1-19.el7.x86_64.rpm +kubernetes-1.10.0-1.el7.src.rpm lapack-3.4.2-8.el7.x86_64.rpm latex2html-2012-3.el7.noarch.rpm lato-fonts-2.015-1.el7.noarch.rpm diff --git a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo index 925486cd..f05db07a 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX-Centos-7.5.repo @@ -100,3 +100,11 @@ gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 enabled=1 +[Starlingx-C7.5.1804-virt-kubernetes-source] +name=Starlingx-CentOS-7.5.1804 - virt-kubernetes-source +baseurl=http://vault.centos.org/7.5.1804/virt/Source/kubernetes110 +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +enabled=1 + + From 2f6fa5e4e27189930fab6d80d75a4689ebe2cfe8 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Fri, 10 Aug 2018 13:25:59 -0500 Subject: [PATCH 02/27] Skip .gitreview review when unnecessary This allows the process to run more than once in a given repo without erroring on the already-patched .gitreview checkout. Change-Id: Iebdf64c87d0ec497b16ce66c48c0c5e4dee5554e Signed-off-by: Dean Troyer --- release/branch-stx.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/release/branch-stx.sh b/release/branch-stx.sh index 7530825b..734bd567 100755 --- a/release/branch-stx.sh +++ b/release/branch-stx.sh @@ -57,12 +57,14 @@ function update_gitreview { defaultbranch=$branch" echo "$grcontents" > .gitreview git add .gitreview - git commit -s -m "Update .gitreview for $branch" - git show - if [[ -z $DRY_RUN ]]; then - git review -t "create-${branch}" + if ! git commit -s -m "Update .gitreview for $branch"; then + if [[ -z $DRY_RUN ]]; then + git review -t "create-${branch}" + else + echo "### skipping .gitreview submission to $branch" + fi else - echo "### skipping review submission to $branch" + echo "### no changes required for .gitreview" fi } From aef1670fde1d3db5492ff165f7394bfbbd785b9e Mon Sep 17 00:00:00 2001 From: Marcela Rosales Date: Fri, 10 Aug 2018 12:42:41 -0500 Subject: [PATCH 03/27] Fixing retcode error in download_mirror.sh tee returns 0, we want to check downloader retcode Change-Id: I80bd843ca49cb78f99f4a78115c8efce3c1b05da Signed-off-by: Marcela Rosales --- centos-mirror-tools/download_mirror.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 07e9ced3..f21d4018 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -16,7 +16,7 @@ need_file(){ } # Check extistence of prerequisites files -need_file dl_rpms.sh dl_other_from_centos_repo.sh tarball-dl.sh +need_file dl_rpms.sh dl_other_from_centos_repo.sh dl_tarball.sh need_file rpms_from_3rd_parties.lst need_file rpms_from_centos_3rd_parties.lst need_file rpms_from_centos_repo.lst @@ -35,7 +35,8 @@ fi rpm_downloader="./dl_rpms.sh" $rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt -if [ $? != 0 ];then +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then echo "ERROR: something wrong with downloading, please check the log!!" fi @@ -49,8 +50,8 @@ echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match crite #download RPMs/SRPMs from CentOS repos by "yumdownloader" $rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt - -if [ $? == 0 ]; then +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` @@ -111,7 +112,8 @@ echo "step #3: start downloading other files ..." other_downloader="./dl_other_from_centos_repo.sh" $other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt -if [ $? == 0 ];then +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then echo "step #3: done successfully" fi From 8966e888518e14826478d27ef9b7517053ab2987 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Fri, 10 Aug 2018 13:28:48 -0500 Subject: [PATCH 04/27] Correctly push tags to Gerrit The tags were missing from the branch push to Gerrit. Also, tags pushed to Gerrit must be signed, so gpg and git must both be properly configured for this to run now. Change-Id: I97f3a4364f090e1b42ac0fc2ba7554c164013345 Signed-off-by: Dean Troyer --- release/branch-stx.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/branch-stx.sh b/release/branch-stx.sh index 734bd567..b4f53184 100755 --- a/release/branch-stx.sh +++ b/release/branch-stx.sh @@ -91,7 +91,7 @@ function branch_repo { fi # tag branch point at $sha - git tag -f $tag $sha + git tag -s -m "Branch $branch" -f $tag $sha # Push the new goodness back up if [[ "$r" == "starlingx" ]]; then @@ -102,7 +102,7 @@ function branch_repo { # push if [[ -z $DRY_RUN ]]; then - git push gerrit $branch + git push --tags gerrit $branch else echo "### skipping push to $branch" fi From ea97b45e8e35d0d527aa2dd834f500340bd99386 Mon Sep 17 00:00:00 2001 From: Abraham Arce Date: Thu, 9 Aug 2018 11:38:37 -0500 Subject: [PATCH 05/27] [Doc] Building docs following Docs Contrib Guide Baseline changes to comply with OpenStack Documentation Contributor Guide [0] starting with the following sections: - Project guide setup - [1] sphinx-quickstart - [2] doc/source/ layout - Building documentation - [3] tox -e docs - Using documentation tools - [4] openstackdocstheme [0] https://docs.openstack.org/doc-contrib-guide [1] http://www.sphinx-doc.org/en/master/usage/quickstart.html [2] https://docs.openstack.org/doc-contrib-guide/project-guides.html [3] https://docs.openstack.org/doc-contrib-guide/docs-builds.html [4] https://docs.openstack.org/openstackdocstheme/ Story: 2002708 Task: 24529 Story: 2002813 Task: 24530 Change-Id: Id9b49e84c79523df2ad6bca183f3407ef84d0f1a Signed-off-by: Abraham Arce --- .gitignore | 3 + doc/requirements.txt | 2 + doc/source/conf.py | 160 +++++++++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 15 ++++ tox.ini | 10 ++- 5 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 doc/requirements.txt create mode 100644 doc/source/conf.py create mode 100644 doc/source/index.rst diff --git a/.gitignore b/.gitignore index 766c4b1c..c2424721 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ localrc toCOPY/.gitconfig centos-mirror-tools/logs/ centos-mirror-tools/output/ + +# Sphinx documentation +docs/build/ diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..0b077dc9 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,2 @@ +sphinx>=1.6.2 +openstackdocstheme>=1.19.0 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 00000000..f637d48b --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another +# directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown +# here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = u'stx-tools' +copyright = u'2018, StarlingX' +author = u'StarlingX' + +# The short X.Y version +version = u'' +# The full version, including alpha/beta/rc tags +release = u'0.1' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['openstackdocstheme' + ] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +bug_project = '1027' +bug_tag = 'stx.bug' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'stx-toolsdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'stx-tools.tex', u'stx-tools Documentation', + u'StarlingX', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'stx-tools', u'stx-tools Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'stx-tools', u'stx-tools Documentation', + author, 'stx-tools', 'One line description of project.', + 'Miscellaneous'), +] diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 00000000..47551a29 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,15 @@ +stx-tools Documentation +======================= + +StarlingX Build Tools + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`search` diff --git a/tox.ini b/tox.ini index e1fc1e54..743d3582 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skipsdist = True [testenv] basepython = python3 -install_command = pip install -U {opts} {packages} +install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} setenv = VIRTUAL_ENV={envdir} OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 @@ -36,3 +36,11 @@ commands = {posargs} envdir = {toxworkdir}/venv commands = true whitelist_externals = true + +[testenv:docs] +basepython = python3 +deps = -r{toxinidir}/doc/requirements.txt +commands = + rm -rf doc/build + sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html +whitelist_externals = rm From ed79155570ac88872704a93296a92130cda3329a Mon Sep 17 00:00:00 2001 From: Abraham Arce Date: Fri, 10 Aug 2018 02:00:29 -0500 Subject: [PATCH 06/27] [Doc] Release Notes Management Baseline changes to comply with Release Notes Management based in Reno [0] a release notes manager. [0] https://docs.openstack.org/reno/latest/ Story: 2003101 Task: 24554 Change-Id: I812b92a60599edc28ace84f286200522a706e3cd Signed-off-by: Abraham Arce --- .gitignore | 3 + doc/requirements.txt | 3 + releasenotes/source/conf.py | 267 +++++++++++++++++++++++++++++ releasenotes/source/index.rst | 7 + releasenotes/source/unreleased.rst | 5 + tox.ini | 9 + 6 files changed, 294 insertions(+) create mode 100644 releasenotes/source/conf.py create mode 100644 releasenotes/source/index.rst create mode 100644 releasenotes/source/unreleased.rst diff --git a/.gitignore b/.gitignore index c2424721..acb93792 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ centos-mirror-tools/output/ # Sphinx documentation docs/build/ + +# Release Notes documentation +releasenotes/build diff --git a/doc/requirements.txt b/doc/requirements.txt index 0b077dc9..79b7802e 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,2 +1,5 @@ sphinx>=1.6.2 openstackdocstheme>=1.19.0 # Apache-2.0 + +# Release Notes +reno>=0.1.1 # Apache2 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 00000000..764221de --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# stx-tools Release Notes documentation build configuration file. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'openstackdocstheme', + 'reno.sphinxext', +] + +bug_project = '1027' +bug_tag = 'stx.bug' + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# Release notes are version independent, no need to set version and release +release = '' +version = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +html_last_updated_fmt = '%Y-%m-%d %H:%M' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'stx-toolsreleasenotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'stx-toolsreleasenotes.tex', u'stx-tools Release Notes', + u'StarlingX', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'stx-toolsreleasenotes', u'stx-tools Release Notes', + [u'StarlingX'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'stx-toolsReleaseNotes', u'stx-tools Release Notes', + u'StarlingX', 'stx-toolsreleasenotes', + 'StarlingX build tools', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 00000000..11f0b40b --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,7 @@ +stx-tools Release Notes +======================= + +.. toctree:: + :maxdepth: 2 + + unreleased diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst new file mode 100644 index 00000000..875030f9 --- /dev/null +++ b/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================ +Current Series Release Notes +============================ + +.. release-notes:: diff --git a/tox.ini b/tox.ini index 743d3582..82da1e95 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ setenv = VIRTUAL_ENV={envdir} OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 deps = -r{toxinidir}/test-requirements.txt +whitelist_externals = reno [testenv:linters] whitelist_externals = bash @@ -44,3 +45,11 @@ commands = rm -rf doc/build sphinx-build -a -E -W -d doc/build/doctrees -b html doc/source doc/build/html whitelist_externals = rm + +[testenv:releasenotes] +basepython = python3 +deps = -r{toxinidir}/doc/requirements.txt +commands = + rm -rf releasenotes/build + sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html +whitelist_externals = rm From 13f92f4668a6d4b4f7fda4062b526ee34b63e639 Mon Sep 17 00:00:00 2001 From: jmckenna Date: Fri, 10 Aug 2018 10:20:30 -0400 Subject: [PATCH 07/27] Improved local mirror support in tools This change improves support for organizational or personal local mirrors. Download tools work in a wider variety of environmnets, and the population of build environments is now driven by .lst files. The download_mirror.sh (and related files) now have options to better support running in non-containerized environments. This includes options to run using local yum.conf files (rather than system files) and not executing sudo or chgrp commands. These options are enabled by flags. Default behaviour remains unchanged. The generate-cgcs-centos-repo.sh tool now populates build environment as driven by lst files. This allows a mirror to be maintained (and grown) at an organizational level, while allowing a developer to produce exactly the environment they need as per the lst files they have checked out. generate-cgcs-centos-repo.sh now also populates the downloads directory by invoking the populate_downloads.sh script (also driven by lst files) Change-Id: I8aae32c776df7618a5ef545a0ce43903b41112d0 Story: 2003422 Task: 24586 Signed-off-by: jmckenna --- centos-mirror-tools/dl_rpms.sh | 77 +++++++++++---- centos-mirror-tools/dl_tarball.sh | 4 + centos-mirror-tools/download_mirror.sh | 132 +++++++++++++++++-------- toCOPY/generate-cgcs-centos-repo.sh | 80 +++++++++++---- toCOPY/populate_downloads.sh | 53 ++++++++++ 5 files changed, 270 insertions(+), 76 deletions(-) create mode 100755 toCOPY/populate_downloads.sh diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 767761a6..ac51ea6e 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -1,19 +1,60 @@ #!/bin/bash -e +# +# SPDX-License-Identifier: Apache-2.0 +# # download RPMs/SRPMs from different sources. # this script was originated by Brian Avery, and later updated by Yong Hu +usage() { + echo "$0 [-n] [-c ] " + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations" + echo " -c: Use an alternate yum.conf rather than the system file" + echo " rpm_list: a list of RPM files to be downloaded." + echo " match_level: value could be L1, L2 or L3:" + echo " L1: use name, major version and minor version:" + echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" + echo " L2: use name and major version:" + echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" + echo " L3: use name:" + echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " from_where: where to download the RPMs: 'centos'from CentOS Repos," + echo " otherwise from 3rd-party websets" + echo "" +} + +# By default, we use "sudo" and we don't use a local yum.conf. These can +# be overridden via flags. +SUDOCMD="sudo -E" +YUMCONFOPT="" + +# Parse option flags +while getopts "c:nh" o; do + case "${o}" in + n) + # No-sudo + SUDOCMD="" + ;; + c) + # Use an alternate yum.conf + YUMCONFOPT="-c $OPTARG" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + if [ $# -lt 3 ]; then - echo "$0 " - echo "rpm_list: a list of RPM files to be downloaded." - echo "match_level: value could be L1, L2 or L3:" - echo " L1: use name, major version and minor version:" - echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" - echo " L2: use name and major version:" - echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" - echo " L3: use name:" - echo " using vim to search vim-7.4.160-2.el7.src.rpm" - echo "from_where: where to download the RPMs: 'centos'from CentOS Repos," - echo "otherwise from 3rd-party websets" + usage exit -1 fi @@ -83,11 +124,11 @@ download () { fi echo " ------ using $SFILE to search $rpm_name ------" if [ "$_type" == "src" ];then - download_cmd="sudo -E yumdownloader -q -C --source $SFILE" - download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE" + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" else - download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64" - download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64" + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" fi else rpm_name=`echo $ff | cut -d"#" -f1-1` @@ -95,13 +136,13 @@ download () { download_cmd="wget $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" + echo "--> run: $download_cmd" if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice $download_url_cmd | sort --unique >> $URL_SRPMS @@ -144,7 +185,7 @@ download () { } # prime the cache -sudo -E yum makecache +${SUDOCMD} yum ${YUMCONFOPT} makecache #go to download *.noarch.rpm files noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"` diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index fe090ad9..c49c7ccd 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -93,6 +93,10 @@ for line in $(cat $tarball_file); do if [[ "$line" =~ ^'!' ]]; then tarball_name="${tarball_name//!/}" + if [ -e "$output_tarball/$tarball_name" ]; then + echo "Already have $tarball_name" + continue + fi echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index f21d4018..3da64d60 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -1,4 +1,65 @@ #!/bin/bash -e +# +# SPDX-License-Identifier: Apache-2.0 +# + +usage() { + echo "$0 [-n] [-c ] [-g]" + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations (option passed on to" + echo " subscripts when appropriate)" + echo " -c: Use an alternate yum.conf rather than the system file (option passed" + echo " on to subscripts when appropriate)" + echo " -g: do not change group IDs of downloaded artifacts" + echo "" +} + +rpm_downloader="./dl_rpms.sh" +tarball_downloader="./dl_tarball.sh" +other_downloader="./dl_other_from_centos_repo.sh" + +# track optional arguments +change_group_ids=1 +use_system_yum_conf=1 +rpm_downloader_extra_args="" +tarball_downloader_extra_args="" + +# lst files to use as input +rpms_from_3rd_parties="./rpms_from_3rd_parties.lst" +rpms_from_centos_repo="./rpms_from_centos_repo.lst" +rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst" +other_downloads="./other_downloads.lst" + +# Parse out optional -c or -n arguments +while getopts "c:ngh" o; do + case "${o}" in + n) + # Pass -n ("no-sudo") to rpm downloader + rpm_downloader_extra_args="${rpm_downloader_extra_args} -n" + ;; + c) + # Pass -c ("use alternate yum.conf") to rpm downloader + use_system_yum_conf=0 + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}" + ;; + g) + # Do not attempt to change group IDs on downloaded packages + change_group_ids=0 + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + echo "--------------------------------------------------------------" echo "WARNING: this script HAS TO access internet (http/https/ftp)," @@ -16,56 +77,51 @@ need_file(){ } # Check extistence of prerequisites files -need_file dl_rpms.sh dl_other_from_centos_repo.sh dl_tarball.sh -need_file rpms_from_3rd_parties.lst -need_file rpms_from_centos_3rd_parties.lst -need_file rpms_from_centos_repo.lst -need_file other_downloads.lst +need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader} +need_file ${rpms_from_3rd_parties} +need_file ${rpms_from_centos_3rd_parties} +need_file ${rpms_from_centos_repo} +need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." -# Restore StarlingX_3rd repos from backup -REPO_SOURCE_DIR=/localdisk/yum.repos.d -REPO_DIR=/etc/yum.repos.d -if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then - \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ +if [ ${use_system_yum_conf} -eq 0 ]; then + # Restore StarlingX_3rd repos from backup + REPO_SOURCE_DIR=/localdisk/yum.repos.d + REPO_DIR=/etc/yum.repos.d + if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then + \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ + fi fi -rpm_downloader="./dl_rpms.sh" -$rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ];then echo "ERROR: something wrong with downloading, please check the log!!" fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -$rpm_downloader ./rpms_from_centos_3rd_parties.lst L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt - -# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. -\rm -f $REPO_DIR/StarlingX_3rd*.repo +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt +if [ ${use_system_yum_conf} -eq 1 ]; then + # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. + \rm -f $REPO_DIR/StarlingX_3rd*.repo +fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" $rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt + retcode=${PIPESTATUS[0]} -if [ $retcode -eq 0 ]; then +if [ $retcode -ne 0 ]; then echo "finish 1st round of RPM downloading successfully!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" fi - #echo "start 2nd round of downloading Binary RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_rpms_missing_L1.txt L2 centos | tee ./logs/log_download_rpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of RPM downloading successfully!" - # if [ -e "./output/rpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing RPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi if [ -e "./output/centos_srpms_missing_L1.txt" ]; then @@ -73,14 +129,6 @@ if [ $retcode -eq 0 ]; then if [ "$missing_num" != "0" ];then echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" fi - #echo "start 2nd round of downloading Source RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_srpms_missing_L1.txt L2 centos | tee ./logs/log_download_srpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of SRPM downloading successfully!" - # if [ -e "./srpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing SRPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi else echo "finish 1st round with failures!" @@ -93,9 +141,10 @@ find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./ find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt find ./output -name "*.i686.rpm" | xargs rm -f -line1=`wc -l rpms_from_3rd_parties.lst | cut -d " " -f1-1` -line2=`wc -l rpms_from_centos_repo.lst | cut -d " " -f1-1` -line3=`wc -l rpms_from_centos_3rd_parties.lst | cut -d " " -f1-1` +line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` +line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` +line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` + let total_line=$line1+$line2+$line3 echo "We expect to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` @@ -104,14 +153,15 @@ if [ "$total_line" != "$num_of_downloaded_rpms" ]; then echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs." fi -# change "./output" and sub-folders to 751 (cgcs) group -chown 751:751 -R ./output +if [ $change_group_ids -eq 1 ]; then + # change "./output" and sub-folders to 751 (cgcs) group + chown 751:751 -R ./output +fi echo "step #3: start downloading other files ..." -other_downloader="./dl_other_from_centos_repo.sh" -$other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" @@ -120,7 +170,7 @@ fi # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -./dl_tarball.sh +${tarball_downloader} ${tarball_downloader_extra_args} echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" diff --git a/toCOPY/generate-cgcs-centos-repo.sh b/toCOPY/generate-cgcs-centos-repo.sh index 0bd9cf74..a8d89d99 100755 --- a/toCOPY/generate-cgcs-centos-repo.sh +++ b/toCOPY/generate-cgcs-centos-repo.sh @@ -27,37 +27,66 @@ comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml -if [[ ( ! -d $mirror_dir/Binary ) || ( ! -d $mirror_dir/Source ) ]]; then - echo "The mirror $mirror_dir doesn't has the Binary and Source" +lst_file_dir="$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools" +rpm_lst_files="rpms_from_3rd_parties.lst rpms_from_centos_3rd_parties.lst rpms_from_centos_repo.lst" +other_lst_file="other_downloads.lst" +missing_rpms_file=missing.txt + +rm -f ${missing_rpms_file} + +# Strip trailing / from mirror_dir if it was specified... +mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%") + +if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then + echo "The mirror ${mirror_dir} doesn't has the Binary and Source" echo "folders. Please provide a valid mirror" exit -1 fi -if [ ! -d "$dest_dir" ]; then - mkdir -p "$dest_dir" +if [ ! -d "${dest_dir}" ]; then + mkdir -p "${dest_dir}" fi for t in "Binary" "Source" ; do - target_dir=$dest_dir/$t + target_dir=${dest_dir}/$t if [ ! -d "$target_dir" ]; then mkdir -p "$target_dir" else mv -f "$target_dir" "$target_dir-backup-$timestamp" mkdir -p "$target_dir" fi +done - pushd "$mirror_dir/$t"|| exit 1 - find . -type d -exec mkdir -p "${target_dir}"/{} \; - all_files=$(find . -type f -name "*") - popd || exit 1 +unsuccessful_file=0 +for lst_file in ${rpm_lst_files} ; do + grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do + if [ -z "${rpmname}" ]; then + continue + fi + mirror_file=$(find ${mirror_dir} -name ${rpmname}) + if [ -z "${mirror_file}" ]; then + echo "Error -- could not find requested ${rpmname} in ${mirror_dir}" + echo ${rpmname} >> ${missing_rpms_file} + unsuccessful_file=1 + continue + fi - - for ff in $all_files; do + # Great, we found the file! Let's strip the mirror_dir prefix from it... + ff=$(echo ${mirror_file} | sed "s%^${mirror_dir}/%%") f_name=$(basename "$ff") sub_dir=$(dirname "$ff") - ln -sf "$mirror_dir/$t/$ff" "$target_dir/$sub_dir" - echo "Creating symlink for $target_dir/$sub_dir/$f_name" - echo "------------------------------" + + # Make sure we have a subdir (so we don't symlink the first file as + # the subdir name) + mkdir -p ${dest_dir}/${sub_dir} + + # Link it! + echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}" + ln -sf "${mirror_dir}/$ff" "${dest_dir}/${sub_dir}" + if [ $? -ne 0 ]; then + echo "Failed ${mirror_file}: ln -sf \"${mirror_dir}/$ff\" \"${dest_dir}/${sub_dir}\"" + unsuccessful_file=1 + fi done done @@ -75,13 +104,30 @@ fi echo "Copying mock.cfg.proto and comps.xml files." if [ -f "$mock_cfg_dest_file" ]; then - cp "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" + \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" fi cp "$mock_cfg_file" "$mock_cfg_dest_file" if [ -f "$comps_xml_dest_file" ]; then - cp "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" + \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" fi cp "$comps_xml_file" "$comps_xml_dest_file" -echo "Done" +# Populate the contents from other list files +cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do + if [ "${targettype}" == "folder" ]; then + echo "Creating folder ${item}" + mkdir -p $MY_REPO/cgcs-centos-repo/Binary/${item} + fi + + if [ "${targettype}" == "file" ]; then + mkdir -p $MY_REPO/cgcs-centos-repo/Binary/$(dirname ${item}) + echo "Creating symlink for $MY_REPO/cgcs-centos-repo/Binary/${item}" + ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/cgcs-centos-repo/Binary/${item} + fi +done + +echo "Done creating repo directory" +if [ ${unsuccessful_file} -ne 0 ]; then + echo "WARNING: Some targets could not be found. Your repo may be incomplete." +fi diff --git a/toCOPY/populate_downloads.sh b/toCOPY/populate_downloads.sh new file mode 100755 index 00000000..1df08e10 --- /dev/null +++ b/toCOPY/populate_downloads.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# + +usage () { + echo "$0 " +} + +if [ $# -ne 1 ]; then + usage + exit -1 +fi + +if [ -z "$MY_REPO" ]; then + echo "\$MY_REPO is not set. Ensure you are running this script" + echo "from the container and \$MY_REPO points to the root of" + echo "your folder tree." + exit -1 +fi + +mirror_dir=$1 +tarball_lst=${MY_REPO_ROOT_DIR}/stx-tools/centos-mirror-tools/tarball-dl.lst +downloads_dir=${MY_REPO}/stx/downloads +extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm rdma-core-43mlnx1-1.43302.src.rpm" + +mkdir -p ${MY_REPO}/stx/downloads + +grep -v "^#" ${tarball_lst} | while read x; do + if [ -z "$x" ]; then + continue + fi + + # Get first element of item & strip leading ! if appropriate + tarball_file=$(echo $x | sed "s/#.*//" | sed "s/^!//") + + # put the file in downloads + source_file=$(find ${mirror_dir}/downloads -name "${tarball_file}") + if [ -z ${source_file} ]; then + echo "Could not find ${tarball_file}" + else + rel_path=$(echo ${source_file} | sed "s%^${mirror_dir}/downloads/%%") + rel_dir_name=$(dirname ${rel_path}) + if [ ! -e ${downloads_dir}/${rel_dir_name}/${tarball_file} ]; then + mkdir -p ${downloads_dir}/${rel_dir_name} + ln -sf ${source_file} ${downloads_dir}/${rel_dir_name}/ + fi + fi +done + +for x in ${extra_downloads}; do + ln -sf ${mirror_dir}/downloads/$x ${downloads_dir} +done From fdd64ad1c6b3abe6c1ca9cbfd96c68362b8ee444 Mon Sep 17 00:00:00 2001 From: jmckenna Date: Wed, 15 Aug 2018 10:35:00 -0400 Subject: [PATCH 08/27] Fix inversion error in download_mirrors.sh Flag to copy back repo files was incorrectly parsed in download_mirrors.sh. This change parses the flag as intended. Change-Id: I7b999d8c85e1495df72eff3410445a948d627363 Story: 2003057 Task: 24645 Signed-off-by: jmckenna --- centos-mirror-tools/download_mirror.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 3da64d60..52075c67 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -87,7 +87,7 @@ need_file tarball-dl.lst mvn-artifacts.lst #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." -if [ ${use_system_yum_conf} -eq 0 ]; then +if [ ${use_system_yum_conf} -ne 0 ]; then # Restore StarlingX_3rd repos from backup REPO_SOURCE_DIR=/localdisk/yum.repos.d REPO_DIR=/etc/yum.repos.d From 1a539ac2a5e663d199b5ea9ac1b2a1c9541b478e Mon Sep 17 00:00:00 2001 From: Marcela Rosales Date: Wed, 8 Aug 2018 10:40:00 -0500 Subject: [PATCH 09/27] Change lists names and downloading scripts -By standardizing the list files name it's possible to eliminate one parameter in the download function in dl_rpms and reduce complexity. -Now the download function does not receive it as a parameter anymore, and instead it gets it from the RPM name. -Also the download function now decides if build a wget command or a yumdownloader commnad based on the content of the list (by identifying the # character) instead of using the name of the list. I adapted download_mirror.sh. Change-Id: I041fc9c704156215f06149e5b4c16cd92990e17c Signed-off-by: Marcela Rosales --- centos-mirror-tools/dl_rpms.sh | 64 +++++++++---------- centos-mirror-tools/download_mirror.sh | 22 +++---- ...om_3rd_parties.lst => rpms_3rdparties.lst} | 0 ...s_from_centos_repo.lst => rpms_centos.lst} | 0 ..._parties.lst => rpms_centos3rdparties.lst} | 0 5 files changed, 42 insertions(+), 44 deletions(-) rename centos-mirror-tools/{rpms_from_3rd_parties.lst => rpms_3rdparties.lst} (100%) rename centos-mirror-tools/{rpms_from_centos_repo.lst => rpms_centos.lst} (100%) rename centos-mirror-tools/{rpms_from_centos_3rd_parties.lst => rpms_centos3rdparties.lst} (100%) diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index ac51ea6e..303bf94e 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -6,7 +6,7 @@ # this script was originated by Brian Avery, and later updated by Yong Hu usage() { - echo "$0 [-n] [-c ] " + echo "$0 [-n] [-c ] " echo "" echo "Options:" echo " -n: Do not use sudo when performing operations" @@ -19,11 +19,16 @@ usage() { echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" echo " L3: use name:" echo " using vim to search vim-7.4.160-2.el7.src.rpm" - echo " from_where: where to download the RPMs: 'centos'from CentOS Repos," - echo " otherwise from 3rd-party websets" echo "" } +get_from() { + list=$1 + base=$(basename $list .lst) + from=$(echo $base | cut -d'_' -f2-2) + echo $from +} + # By default, we use "sudo" and we don't use a local yum.conf. These can # be overridden via flags. SUDOCMD="sudo -E" @@ -53,7 +58,7 @@ while getopts "c:nh" o; do done shift $((OPTIND-1)) -if [ $# -lt 3 ]; then +if [ $# -lt 2 ]; then usage exit -1 fi @@ -72,7 +77,6 @@ if [ ! -z "$2" -a "$2" != " " ];then match_level=$2 fi -from=$3 timestamp=$(date +%F_%H%M) echo $timestamp @@ -83,6 +87,8 @@ mkdir -p $MDIR_SRC MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary mkdir -p $MDIR_BIN + +from=$(get_from $rpms_list) FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt" FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt" MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt" @@ -103,18 +109,21 @@ cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS cat /dev/null > $URL_RPMS -#function to download different type of RPMs in different ways +# Function to download different types of RPMs in different ways download () { - _list=$1 + _file=$1 _level=$2 - _from=$3 - _type=$4 - + _list=$(cat $_file) + _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do - ## download RPM from CentOS repos - if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then + _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) + + # Decide if the list will be downloaded using yumdownloader or wget + if [[ $ff != *"#"* ]]; then rpm_name=$ff + + # Cut the rpm name for the specified level (L1, L2 or L3) if [ $_level == "L1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` elif [ $match_level == "L2" ];then @@ -123,6 +132,7 @@ download () { SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` fi echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 if [ "$_type" == "src" ];then download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" @@ -131,12 +141,15 @@ download () { download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" fi else + # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" SFILE=$rpm_name fi + echo "--> run: $download_cmd" + # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" @@ -163,7 +176,7 @@ download () { echo "Looking for $rpm_name..." if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice $download_url_cmd | sort --unique >> $URL_RPMS @@ -184,28 +197,13 @@ download () { done } -# prime the cache +# Prime the cache ${SUDOCMD} yum ${YUMCONFOPT} makecache -#go to download *.noarch.rpm files -noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"` -if [ ! -z "$noarch_rpms" ];then - echo "--> start searching noarch RPMs ....." - download "$noarch_rpms" $match_level $from "noarch" -fi - -#go to download *.x86_64.rpm files -x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"` -if [ ! -z "$x86_64_rpms" ];then - echo "--> start searching x86_64 RPMs ....." - download "$x86_64_rpms" $match_level $from "x86_64" -fi - -#go to download *.src.rpm files -src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"` -if [ ! -z "$src_rpms" ];then - echo "--> start searching source RPMs ....." - download "$src_rpms" $match_level $from "src" +# Download files +if [ -s "$rpms_list" ];then + echo "--> start searching "$rpms_list + download $rpms_list $match_level fi echo "done!!" diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 52075c67..963f7661 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -26,9 +26,9 @@ rpm_downloader_extra_args="" tarball_downloader_extra_args="" # lst files to use as input -rpms_from_3rd_parties="./rpms_from_3rd_parties.lst" -rpms_from_centos_repo="./rpms_from_centos_repo.lst" -rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst" +rpms_from_3rd_parties="./rpms_3rdparties.lst" +rpms_from_centos_repo="./rpms_centos.lst" +rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" other_downloads="./other_downloads.lst" # Parse out optional -c or -n arguments @@ -96,14 +96,17 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi fi -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt +logfile="log_download_3rdparties_L1.txt" +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ];then +if [ $retcode -ne 0 ]; then echo "ERROR: something wrong with downloading, please check the log!!" fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt +logfile="log_download_centos3rdparties_L1.txt" +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile + if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo @@ -111,9 +114,8 @@ fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" - -$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt - +logfile="log_download_centos_L1.txt" +$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ]; then echo "finish 1st round of RPM downloading successfully!" @@ -144,7 +146,6 @@ find ./output -name "*.i686.rpm" | xargs rm -f line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` - let total_line=$line1+$line2+$line3 echo "We expect to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` @@ -178,4 +179,3 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\"" echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" - diff --git a/centos-mirror-tools/rpms_from_3rd_parties.lst b/centos-mirror-tools/rpms_3rdparties.lst similarity index 100% rename from centos-mirror-tools/rpms_from_3rd_parties.lst rename to centos-mirror-tools/rpms_3rdparties.lst diff --git a/centos-mirror-tools/rpms_from_centos_repo.lst b/centos-mirror-tools/rpms_centos.lst similarity index 100% rename from centos-mirror-tools/rpms_from_centos_repo.lst rename to centos-mirror-tools/rpms_centos.lst diff --git a/centos-mirror-tools/rpms_from_centos_3rd_parties.lst b/centos-mirror-tools/rpms_centos3rdparties.lst similarity index 100% rename from centos-mirror-tools/rpms_from_centos_3rd_parties.lst rename to centos-mirror-tools/rpms_centos3rdparties.lst From 76dfc5bfc46c54b689c6e5b5d2958072b0ea8b4a Mon Sep 17 00:00:00 2001 From: yhu6 Date: Wed, 15 Aug 2018 15:00:42 +0800 Subject: [PATCH 10/27] fix: old libibverbs-41mlnx1 mismatched with newer mlnx-ofa_kernel Story: 2003443 Change-Id: Ia039b168cade4c6f1751deed7481e14d27557b06 Signed-off-by: yhu6 --- centos-mirror-tools/dl_tarball.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index c49c7ccd..e0a64331 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -149,9 +149,11 @@ for line in $(cat $tarball_file); do elif [ "$pkg_version" = "4.3-1.0.1.0" ]; then cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.1.0.1.1.g8509e41.src.rpm" . cp "$srpm_path/rdma-core-43mlnx1-1.43101.src.rpm" . + cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.0.1.8.43101.src.rpm" . elif [ "$pkg_version" = "4.3-3.0.2.1" ]; then cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm" . cp "$srpm_path/rdma-core-43mlnx1-1.43302.src.rpm" . + cp "$srpm_path/libibverbs-41mlnx1-OFED.4.3.2.1.6.43302.src.rpm" . else echo "$pkg_version : unknown version" fi From 390e0432b582747a5788656c1289884aaa437cfa Mon Sep 17 00:00:00 2001 From: Abraham Arce Date: Thu, 16 Aug 2018 09:40:23 -0500 Subject: [PATCH 11/27] Tarball: expect-lite removal Removed code for expect-lite as it is not used anymore in stx-integ based in: https://review.openstack.org/#/c/578113/ Change-Id: Ie3e6306b5e9008875e6d7c6ae18e78895b06781a Signed-off-by: Abraham Arce --- centos-mirror-tools/tarball-dl.lst | 1 - 1 file changed, 1 deletion(-) diff --git a/centos-mirror-tools/tarball-dl.lst b/centos-mirror-tools/tarball-dl.lst index 8b452fdb..7b6c8226 100644 --- a/centos-mirror-tools/tarball-dl.lst +++ b/centos-mirror-tools/tarball-dl.lst @@ -4,7 +4,6 @@ drbd-8.4.3.tar.gz#drbd-8.4.3#http://www.linbit.com/downloads/drbd/8.4/archive/dr drbd-8.4.7-1.tar.gz#drbd-8.4.7-1#http://www.linbit.com/downloads/drbd/8.4/drbd-8.4.7-1.tar.gz dtc-1.4.4.tar.gz#dtc-1.4.4#https://www.kernel.org/pub/software/utils/dtc/dtc-1.4.4.tar.gz e1000e-3.4.1.1.tar.gz#e1000e-3.4.1.1#https://sourceforge.net/projects/e1000/files/e1000e%20stable/3.4.1.1/e1000e-3.4.1.1.tar.gz -expect-lite_4.9.0.tar.gz#expect-lite.proj#https://sourceforge.net/projects/expect-lite/files/expect-lite/expect-lite_4.9.0/expect-lite_4.9.0.tar.gz gnulib-ffc927e.tar.gz#gnulib-ffc927e#http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-ffc927eef29016a5219cd969daad8928af6a1f4d.tar.gz i40e-2.4.10.tar.gz#i40e-2.4.10#https://sourceforge.net/projects/e1000/files/i40e%20stable/2.4.10/i40e-2.4.10.tar.gz/download i40evf-3.5.13.tar.gz#i40evf-3.5.13#https://sourceforge.net/projects/e1000/files/i40evf%20stable/3.5.13/i40evf-3.5.13.tar.gz/download From 863c5bfecb975ab5366a50190e3e4de185869104 Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Thu, 16 Aug 2018 15:26:41 -0500 Subject: [PATCH 12/27] Add missing key for CentOS Virtualization. Story: 2003470 Task: 24730 Change-Id: I78b8b876473f267323bb7585ce74268a12bd5879 Signed-off-by: Erich Cordoba --- .../RPM-GPG-KEY-CentOS-SIG-Virtualization | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization diff --git a/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization b/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization new file mode 100644 index 00000000..00006eb1 --- /dev/null +++ b/centos-mirror-tools/rpm-gpg-keys/RPM-GPG-KEY-CentOS-SIG-Virtualization @@ -0,0 +1,20 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQENBFWB31YBCAC4dFmTzBDOcq4R1RbvQXLkyYfF+yXcsMA5kwZy7kjxnFqBoNPv +aAjFm3e5huTw2BMZW0viLGJrHZGnsXsE5iNmzom2UgCtrvcG2f65OFGlC1HZ3ajA +8ZIfdgNQkPpor61xqBCLzIsp55A7YuPNDvatk/+MqGdNv8Ug7iVmhQvI0p1bbaZR +0GuavmC5EZ/+mDlZ2kHIQOUoInHqLJaX7iw46iLRUnvJ1vATOzTnKidoFapjhzIt +i4ZSIRaalyJ4sT+oX4CoRzerNnUtIe2k9Hw6cEu4YKGCO7nnuXjMKz7Nz5GgP2Ou +zIA/fcOmQkSGcn7FoXybWJ8DqBExvkJuDljPABEBAAG0bENlbnRPUyBWaXJ0dWFs +aXphdGlvbiBTSUcgKGh0dHA6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lhbEludGVy +ZXN0R3JvdXAvVmlydHVhbGl6YXRpb24pIDxzZWN1cml0eUBjZW50b3Mub3JnPokB +OQQTAQIAIwUCVYHfVgIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEHrr +voJh6IBsRd0H/A62i5CqfftuySOCE95xMxZRw8+voWO84QS9zYvDEnzcEQpNnHyo +FNZTpKOghIDtETWxzpY2ThLixcZOTubT+6hUL1n+cuLDVMu4OVXBPoUkRy56defc +qkWR+UVwQitmlq1ngzwmqVZaB8Hf/mFZiB3B3Jr4dvVgWXRv58jcXFOPb8DdUoAc +S3u/FLvri92lCaXu08p8YSpFOfT5T55kFICeneqETNYS2E3iKLipHFOLh7EWGM5b +Wsr7o0r+KltI4Ehy/TjvNX16fa/t9p5pUs8rKyG8SZndxJCsk0MW55G9HFvQ0FmP +A6vX9WQmbP+ml7jsUxtEJ6MOGJ39jmaUvPc= +=ZzP+ +-----END PGP PUBLIC KEY BLOCK----- From 6b893381bf104246861771743825f1484dee70fe Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Thu, 16 Aug 2018 23:27:08 -0500 Subject: [PATCH 13/27] Update file names in cgcs-centos-repo generator The script was updated as the filenames were changed. Change-Id: I1ef3dae3752cc377a9c9a087628f741b30cb31b5 Signed-off-by: Erich Cordoba --- toCOPY/generate-cgcs-centos-repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toCOPY/generate-cgcs-centos-repo.sh b/toCOPY/generate-cgcs-centos-repo.sh index a8d89d99..6c6e51ae 100755 --- a/toCOPY/generate-cgcs-centos-repo.sh +++ b/toCOPY/generate-cgcs-centos-repo.sh @@ -28,7 +28,7 @@ mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml lst_file_dir="$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools" -rpm_lst_files="rpms_from_3rd_parties.lst rpms_from_centos_3rd_parties.lst rpms_from_centos_repo.lst" +rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst" other_lst_file="other_downloads.lst" missing_rpms_file=missing.txt From fa995d69812c43b0bc908f96648e3170a92bfad5 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Thu, 9 Aug 2018 14:22:40 -0400 Subject: [PATCH 14/27] Use kojipkgs.fedoraproject.org as a backup rpm source. EPEL7 rpms builds are visible at kojipkgs.fedoraproject.org and often remain available there long after they age out of the official repo. Use kojipkgs.fedoraproject.org as a backup source for packages when not found in the repo. Log seperately so that these dropouts can be identified and corrected in the lst files. Also improved the tests for already downloaded packages in several places to accelerate the test cycle. Change-Id: Iaaa9fe05ac605a3604acf0048c571c6e88303692 Story: 2003157 Task: 23292 Signed-off-by: Scott Little --- .../dl_other_from_centos_repo.sh | 5 + centos-mirror-tools/dl_rpms.sh | 145 +++++++++++++++--- centos-mirror-tools/dl_tarball.sh | 18 ++- centos-mirror-tools/download_mirror.sh | 107 +++++++++++-- 4 files changed, 229 insertions(+), 46 deletions(-) diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 83d040b1..1c5a6060 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -31,6 +31,11 @@ for ff in $all; do if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name else + if [ -e "$save_path/$_name" ]; then + echo "Already have $save_path/$_name" + continue + fi + echo "remote path: $url_prefix/$_name" echo "local path: $save_path/$_name" if wget $url_prefix/$_name; then diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 303bf94e..327f2143 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -11,6 +11,7 @@ usage() { echo "Options:" echo " -n: Do not use sudo when performing operations" echo " -c: Use an alternate yum.conf rather than the system file" + echo " -x: Clean log files only, do not run." echo " rpm_list: a list of RPM files to be downloaded." echo " match_level: value could be L1, L2 or L3:" echo " L1: use name, major version and minor version:" @@ -19,6 +20,12 @@ usage() { echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" echo " L3: use name:" echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " K1: Use Koji rather than yum repos as a source." + echo " Koji has a longer retention period than epel mirrors." + echo "" + echo "Returns: 0 = All files downloaded successfully" + echo " 1 = Some files could not be downloaded" + echo " 2 = Bad arguements or other error" echo "" } @@ -34,13 +41,20 @@ get_from() { SUDOCMD="sudo -E" YUMCONFOPT="" +CLEAN_LOGS_ONLY=0 +dl_rc=0 + # Parse option flags -while getopts "c:nh" o; do +while getopts "c:nxh" o; do case "${o}" in n) # No-sudo SUDOCMD="" ;; + x) + # Clean only + CLEAN_LOGS_ONLY=1 + ;; c) # Use an alternate yum.conf YUMCONFOPT="-c $OPTARG" @@ -52,7 +66,7 @@ while getopts "c:nh" o; do ;; *) usage - exit 1 + exit 2 ;; esac done @@ -60,12 +74,12 @@ shift $((OPTIND-1)) if [ $# -lt 2 ]; then usage - exit -1 + exit 2 fi if [ "$1" == "" ]; then echo "Need to supply the rpm file list" - exit -1; + exit 2; else rpms_list=$1 echo "using $rpms_list as the download name lists" @@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS cat /dev/null > $MISSING_RPMS cat /dev/null > $URL_RPMS +if [ $CLEAN_LOGS_ONLY -eq 1 ];then + exit 0 +fi + +# Function to split an rpm filename into parts. +# +# Returns a space seperated list containing: +# +# +split_filename () { + local rpm_filename=$1 + + local RPM="" + local SFILE="" + local ARCH="" + local RELEASE="" + local VERSION="" + local NAME="" + + RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev) + ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev) + RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev) + NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev) + + if [[ $NAME = *":"* ]]; then + EPOCH=$(echo $NAME | cut -d':' -f-1) + NAME=$(echo $NAME | cut -d':' -f2-) + fi + + echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH" +} + +# Function to predict the URL where a rpm might be found. +# Assumes the rpm was compile for EPEL by fedora's koji. +koji_url () { + local rpm_filename=$1 + + local arr=( $(split_filename $rpm_filename) ) + + local n=${arr[0]} + local v=${arr[1]} + local r=${arr[2]} + local a=${arr[3]} + local e=${arr[4]} + + echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm" +} + # Function to download different types of RPMs in different ways download () { - _file=$1 - _level=$2 - _list=$(cat $_file) - _from=$(get_from $_file) + local _file=$1 + local _level=$2 + + local _list=$(cat $_file) + local _from=$(get_from $_file) + local _type="" + + local rc=0 + local download_cmd="" + local download_url_cmd="" + local rpm_name="" + local rpm_url="" + local SFILE="" + echo "now the rpm will come from: $_from" for ff in $_list; do + download_cmd="" + download_url_cmd="" _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev) # Decide if the list will be downloaded using yumdownloader or wget if [[ $ff != *"#"* ]]; then rpm_name=$ff - # Cut the rpm name for the specified level (L1, L2 or L3) - if [ $_level == "L1" ]; then + if [ $_level == "K1" ]; then SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` - elif [ $match_level == "L2" ];then - SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + rpm_url=$(koji_url $rpm_name) + download_cmd="wget $rpm_url)" + download_url_cmd="echo $rpm_url)" else - SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` - fi - echo " ------ using $SFILE to search $rpm_name ------" - # Yumdownloader with the appropriate flag for src, noarch or x86_64 - if [ "$_type" == "src" ];then - download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" - else - download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" - download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + if [ $_level == "L1" ]; then + SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev` + elif [ $match_level == "L2" ];then + SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev` + else + SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev` + fi + echo " ------ using $SFILE to search $rpm_name ------" + # Yumdownloader with the appropriate flag for src, noarch or x86_64 + if [ "$_type" == "src" ];then + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" + else + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + fi fi else # Buid wget command rpm_name=`echo $ff | cut -d"#" -f1-1` rpm_url=`echo $ff | cut -d"#" -f2-2` download_cmd="wget $rpm_url" + download_url_cmd="echo $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" # Put the RPM in the Binary or Source directory if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -165,15 +250,18 @@ download () { fi echo $rpm_name >> $FOUND_SRPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_SRPMS + rc=1 fi else - echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" + echo "Already have ${MDIR_SRC}/${_type}/$rpm_name" echo $rpm_name >> $FOUND_SRPMS fi else ## noarch or x86_64 if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then echo "Looking for $rpm_name..." + echo "--> run: $download_cmd" if $download_cmd ; then # Success! Record download URL. # Use 'sort --unique' because sometimes @@ -187,7 +275,9 @@ download () { fi echo $rpm_name >> $FOUND_RPMS else + echo "Warning: $rpm_name not found" echo $rpm_name >> $MISSING_RPMS + rc=1 fi else echo "Already have ${MDIR_BIN}/${_type}/$rpm_name" @@ -195,6 +285,8 @@ download () { fi fi done + + return $rc } # Prime the cache @@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache if [ -s "$rpms_list" ];then echo "--> start searching "$rpms_list download $rpms_list $match_level + if [ $? -ne 0 ]; then + dl_rc=1 + fi fi echo "done!!" -exit 0 +exit $dl_rc diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index e0a64331..0658a99b 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + # Remove leading '!' if present + tarball_name="${tarball_name//!/}" + # - For the General category and the Puppet category: # - Packages have a common process: download, decompressed, # change the directory path and compressed. @@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do download_directory=$output_tarball fi + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + # We have 6 packages from the text file starting with the character "!": # they require special handling besides the common process: remove directory, # remove text from some files, clone a git repository, etc. if [[ "$line" =~ ^'!' ]]; then - tarball_name="${tarball_name//!/}" - if [ -e "$output_tarball/$tarball_name" ]; then - echo "Already have $tarball_name" - continue - fi echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then @@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do else echo "$pkg_version : unknown version" fi - rm -f "$tarball_name" + # Don't delete the original MLNX_OFED_LINUX tarball. + # We don't use it, but it will prevent re-downloading this file. + # rm -f "$tarball_name" + rm -rf "MLNX_OFED_SRC-${pkg_version}" rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 963f7661..e74d71fd 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst" rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst" other_downloads="./other_downloads.lst" +# Overall success +success=1 + # Parse out optional -c or -n arguments while getopts "c:ngh" o; do case "${o}" in @@ -71,7 +74,7 @@ need_file(){ for f in $*; do if [ ! -e $f ]; then echo "ERROR: $f does not exist." - exit -1 + exit 1 fi done } @@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo} need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst + #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." @@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi logfile="log_download_3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then - echo "ERROR: something wrong with downloading, please check the log!!" +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" logfile="log_download_centos3rdparties_L1.txt" -$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile +retcode=${PIPESTATUS[0]} +if [ $retcode -ne 0 ];then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}." + echo " Please check the log at $(pwd)/logs/$logfile !" + echo "" + success=0 +fi if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo fi + echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" logfile="log_download_centos_L1.txt" -$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile +$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} -if [ $retcode -ne 0 ]; then + +K1_logfile="log_download_rpms_from_centos_K1.txt" +if [ $retcode -ne 1 ]; then + # K1 step not needed. Clear any K1 logs from previous download attempts. + $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile +fi + +if [ $retcode -eq 0 ]; then echo "finish 1st round of RPM downloading successfully!" +elif [ $retcode -eq 1 ]; then + echo "finish 1st round of RPM downloading with missing files!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then - missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` + + echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." + $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile + retcode=${PIPESTATUS[0]} + if [ $retcode -eq 0 ]; then + echo "finish 2nd round of RPM downloading successfully!" + elif [ $retcode -eq 1 ]; then + echo "finish 2nd round of RPM downloading with missing files!" + if [ -e "./output/rpms_missing_K1.txt" ]; then + echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !" + fi + fi + + # Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent + # false reporting of missing files. + grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp + mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt + + + missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: -------RPMs missing: $missing_num ---------------" + retcode=1 fi fi if [ -e "./output/centos_srpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then - echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" + echo "ERROR: --------- SRPMs missing: $missing_num ---------------" + retcode=1 fi fi -else - echo "finish 1st round with failures!" +fi + +if [ $retcode -ne 0 ]; then + echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}." + echo " Please check the logs at $(pwd)/logs/$logfile" + echo " and $(pwd)/logs/$K1_logfile !" + echo "" + success=0 fi ## verify all RPMs SRPMs we download for the GPG keys @@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` let total_line=$line1+$line2+$line3 -echo "We expect to download $total_line RPMs." +echo "We expected to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` echo "There are $num_of_downloaded_rpms RPMs in output directory." if [ "$total_line" != "$num_of_downloaded_rpms" ]; then @@ -162,16 +213,33 @@ fi echo "step #3: start downloading other files ..." -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" +else + echo "step #3: finished with errors" + echo "ERROR: Something wrong with downloading from ${other_downloads}." + echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !" + echo "" + success=0 fi + # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -${tarball_downloader} ${tarball_downloader_extra_args} +${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt +retcode=${PIPESTATUS[0]} +if [ $retcode -eq 0 ];then + echo "step #4: done successfully" +else + echo "step #4: finished with errors" + echo "ERROR: Something wrong with downloading tarballs." + echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !" + echo "" + success=0 +fi echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" @@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\"" echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img" echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz" + +echo "" +if [ $success -ne 1 ]; then + echo "Warning: Not all download steps succeeded. You are likely missing files." + exit 1 +fi + +echo "Success" +exit 0 From e3712f2741f71856c44e6e30713cbd477cf2baa2 Mon Sep 17 00:00:00 2001 From: Jason McKenna Date: Tue, 21 Aug 2018 09:14:00 -0400 Subject: [PATCH 15/27] Update populate_downloads to use latest libibverbs An update to stx-integ (https://review.openstack.org/#/c/591936/) upreved the version of libibverbs that the build was expecting. This change uprevs the version which is placed in downloads/ by populate_downloads.sh to match the expected version. Change-Id: I5ab727f373fe4c73b73135e6860beb99792537d6 Signed-off-by: jmckenna Story: 2003508 Task: 24786 --- toCOPY/populate_downloads.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toCOPY/populate_downloads.sh b/toCOPY/populate_downloads.sh index 1df08e10..782e836e 100755 --- a/toCOPY/populate_downloads.sh +++ b/toCOPY/populate_downloads.sh @@ -22,7 +22,7 @@ fi mirror_dir=$1 tarball_lst=${MY_REPO_ROOT_DIR}/stx-tools/centos-mirror-tools/tarball-dl.lst downloads_dir=${MY_REPO}/stx/downloads -extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm rdma-core-43mlnx1-1.43302.src.rpm" +extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.3.2.1.6.43302.src.rpm rdma-core-43mlnx1-1.43302.src.rpm" mkdir -p ${MY_REPO}/stx/downloads From d7be75dcdbaa1912da59c3250f2fb7884ec665fb Mon Sep 17 00:00:00 2001 From: Jason McKenna Date: Tue, 21 Aug 2018 09:19:23 -0400 Subject: [PATCH 16/27] Pass optional args to RPM download scripts There are a few optional arguments, include -n and -c, which should be passed to the RPM download script (dl_rpms.sh). Currently, they are only passed in some calls to dl_rpms.sh. This change adds the args to remaining calls. Change-Id: I661f20fa8ed81e461fcf4e25518beb66402e45ff Signed-off-by: Jason McKenna Story: https://storyboard.openstack.org/#!/story/2003506 Task: 24784 --- centos-mirror-tools/download_mirror.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index e74d71fd..3e739bab 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -130,13 +130,13 @@ fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" logfile="log_download_centos_L1.txt" -$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile retcode=${PIPESTATUS[0]} K1_logfile="log_download_rpms_from_centos_K1.txt" if [ $retcode -ne 1 ]; then # K1 step not needed. Clear any K1 logs from previous download attempts. - $rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile + $rpm_downloader ${rpm_downloader_extra_args} -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile fi if [ $retcode -eq 0 ]; then @@ -146,7 +146,7 @@ elif [ $retcode -eq 1 ]; then if [ -e "./output/centos_rpms_missing_L1.txt" ]; then echo "start 2nd round of downloading Binary RPMs with K1 match criteria..." - $rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile + $rpm_downloader ${rpm_downloader_extra_args} ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ]; then echo "finish 2nd round of RPM downloading successfully!" From 329dcfddf926a6556c6d4c606e321a6cb8257e07 Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Tue, 21 Aug 2018 23:34:27 -0500 Subject: [PATCH 17/27] Update python-pyngus to newer version. The package version was updated in upstream repositories. Change-Id: I1e53a09ebffe83f01acb01a039c37fa66fa2f500 Signed-off-by: Erich Cordoba --- centos-mirror-tools/rpms_centos.lst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 188082ab..b2e3bd34 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -1188,7 +1188,7 @@ python2-pycadf-2.6.0-1.el7.noarch.rpm python2-pycodestyle-2.0.0-5.el7.noarch.rpm python2-pygments-2.2.0-7.el7.noarch.rpm python2-PyMySQL-0.7.11-1.el7.noarch.rpm -python2-pyngus-2.2.3-1.el7.noarch.rpm +python2-pyngus-2.2.4-1.el7.noarch.rpm python2-pyOpenSSL-16.2.0-3.el7.noarch.rpm python2-pyparsing-2.1.10-3.el7.noarch.rpm python2-pyroute2-0.4.19-1.el7.noarch.rpm From 556a6e6d4d3ee5a49f9d43c7df71d1eb45649433 Mon Sep 17 00:00:00 2001 From: Paul-Emile Element Date: Wed, 22 Aug 2018 09:51:25 -0400 Subject: [PATCH 18/27] Add linux screen package to controllers This is an enhancement request to add the screen package to the system Added the screen package version screen-4.1.0-0.25.20120314git3c2946.el7.x86_64.rpm from centos 7.5 Story: 2003061 Task: 23100 Change-Id: I355d517ba0d0392d40fe78991798ddf6e5d16fde Signed-off-by: Paul-Emile Element --- centos-mirror-tools/rpms_centos.lst | 1 + 1 file changed, 1 insertion(+) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 188082ab..33b3e8d6 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -1511,6 +1511,7 @@ sanlock-3.5.0-1.el7.src.rpm sazanami-fonts-common-0.20040629-22.el7.noarch.rpm sazanami-gothic-fonts-0.20040629-22.el7.noarch.rpm sazanami-mincho-fonts-0.20040629-22.el7.noarch.rpm +screen-4.1.0-0.25.20120314git3c2946.el7.x86_64.rpm scrub-2.5.2-7.el7.x86_64.rpm scsi-target-utils-1.0.55-4.el7.src.rpm SDL-1.2.15-14.el7.x86_64.rpm From e6edeb9ed795fe542a78806a17c7ffc184a55aa7 Mon Sep 17 00:00:00 2001 From: David Sullivan Date: Wed, 22 Aug 2018 11:21:47 -0400 Subject: [PATCH 19/27] Add openstack queens repo for kubernetes We are moving to python2-kubernetes-4.0.0-1. In order to download this we need to make the openstack queens repo available for the download tools. Change-Id: I7ba052501facad4a7e6f376b2e91eaa2420e9b9f Story: 2002845 Task: 22793 Signed-off-by: David Sullivan --- .../StarlingX_CentOS-OpenStack-queens.repo | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo new file mode 100644 index 00000000..54c5fc43 --- /dev/null +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo @@ -0,0 +1,41 @@ +# CentOS-OpenStack-queens.repo +# +# Please see http://wiki.centos.org/SpecialInterestGroup/Cloud for more +# information + +[centos-openstack-queens] +name=CentOS-7 - OpenStack queens +baseurl=http://mirror.centos.org/$contentdir/$releasever/cloud/$basearch/openstack-queens/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[centos-openstack-queens-test] +name=CentOS-7 - OpenStack queens Testing +baseurl=https://buildlogs.centos.org/centos/7/cloud/$basearch/openstack-queens/ +gpgcheck=0 +enabled=0 +exclude=sip,PyQt4 + +[centos-openstack-queens-debuginfo] +name=CentOS-7 - OpenStack queens - Debug +baseurl=http://debuginfo.centos.org/centos/7/cloud/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[centos-openstack-queens-source] +name=CentOS-7 - OpenStack queens - Source +baseurl=http://vault.centos.org/centos/7/cloud/Source/openstack-queens/ +gpgcheck=1 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud +exclude=sip,PyQt4 + +[rdo-trunk-queens-tested] +name=OpenStack queens Trunk Tested +baseurl=https://trunk.rdoproject.org/centos7-queens/current-passed-ci/ +gpgcheck=0 +enabled=0 From e7a1653855a4eadc720bd4742a5fb4e4e4f564ad Mon Sep 17 00:00:00 2001 From: Jack Ding Date: Sun, 12 Aug 2018 22:03:44 -0400 Subject: [PATCH 20/27] Add openstack-swift pkgs and dependencies Story: 2003518 Task: 24809 Change-Id: I9ecc5181d8e22ee5c106b70f585db8cfd04487ce Signed-off-by: Jack Ding --- centos-mirror-tools/rpms_centos.lst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 33b3e8d6..cd6b58f8 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -570,6 +570,7 @@ libedit-3.0-12.20121213cvs.el7.x86_64.rpm libedit-devel-3.0-12.20121213cvs.el7.x86_64.rpm libepoxy-1.3.1-1.el7.x86_64.rpm libepoxy-devel-1.3.1-1.el7.x86_64.rpm +liberasurecode-1.5.0-1.el7.x86_64.rpm liberation-fonts-common-1.07.2-16.el7.noarch.rpm liberation-mono-fonts-1.07.2-16.el7.noarch.rpm liberation-narrow-fonts-1.07.2-16.el7.noarch.rpm @@ -880,6 +881,11 @@ openstack-aodh-3.0.4-1.el7.src.rpm openstack-macros-2018.1.0-0.noarch.rpm openstack-nova-cert-14.0.8-1.el7.noarch.rpm openstack-panko-3.0.0-1.el7.src.rpm +openstack-swift-account-2.15.1-1.el7.noarch.rpm +openstack-swift-container-2.15.1-1.el7.noarch.rpm +openstack-swift-doc-2.15.1-1.el7.noarch.rpm +openstack-swift-object-2.15.1-1.el7.noarch.rpm +openstack-swift-proxy-2.15.1-1.el7.noarch.rpm openvswitch-2.9.0-3.el7.src.rpm orc-0.4.26-1.el7.x86_64.rpm osinfo-db-tools-1.1.0-1.el7.x86_64.rpm @@ -1273,6 +1279,7 @@ python-betamax-0.7.0-1.el7.noarch.rpm python-Bottleneck-0.7.0-1.el7.x86_64.rpm python-bson-3.0.3-1.el7.x86_64.rpm python-cachetools-1.1.6-2.el7.noarch.rpm +python-ceilometermiddleware-1.1.0-1.el7.noarch.rpm python-cffi-1.6.0-5.el7.x86_64.rpm python-characteristic-14.3.0-4.el7.noarch.rpm python-chardet-2.2.1-1.el7_1.noarch.rpm @@ -1297,6 +1304,7 @@ python-django-compressor-2.0-1.el7.noarch.rpm python-django-nose-1.4.3-1.el7.noarch.rpm python-django-openstack-auth-3.5.0-1.el7.src.rpm python-django-pyscss-2.0.2-1.el7.noarch.rpm +python-dns-1.12.0-4.20150617git465785f.el7.noarch.rpm python-docker-pycreds-1.10.6-1.el7.noarch.rpm python-docutils-0.11-0.2.20130715svn7687.el7.noarch.rpm python-dogpile-cache-0.6.2-1.el7.noarch.rpm @@ -1392,6 +1400,7 @@ python-py-1.4.32-1.el7.noarch.rpm python-pycadf-common-2.6.0-1.el7.noarch.rpm python-pycparser-2.14-1.el7.noarch.rpm python-pycurl-7.19.0-19.el7.x86_64.rpm +python-pyeclib-1.5.0-1.el7.x86_64.rpm python-pyelftools-0.22-0.5.git20130619.a1d9681.el7.noarch.rpm python-pymongo-3.0.3-1.el7.x86_64.rpm python-pytest-cov-2.5.1-1.el7.noarch.rpm @@ -1419,6 +1428,8 @@ python-sqlparse-0.1.18-5.el7.noarch.rpm python-srpm-macros-3-22.el7.noarch.rpm python-subprocess32-3.2.6-4.el7.x86_64.rpm python-subunit-1.0.0-1.el7.noarch.rpm +python-swift-2.15.1-1.el7.noarch.rpm +python-swift-tests-2.15.1-1.el7.noarch.rpm python-sysv_ipc-0.4.2-11.el7.x86_64.rpm python-tables-3.2.0-1.el7.x86_64.rpm python-tablib-0.10.0-1.el7.noarch.rpm From 854492f364b586caed9951534121f6204019eab5 Mon Sep 17 00:00:00 2001 From: David Sullivan Date: Thu, 23 Aug 2018 14:08:46 -0400 Subject: [PATCH 21/27] Remove contentdir var from openstack queens repo The openstack queens repo includes a contentdir var. This variable may not be present in all systems. For our purposes this var should be centos, so we will use that in place of the variable in the repo file. See https://review.openstack.org/#/c/568180/ and https://lists.centos.org/pipermail/centos-devel/2018-March/016542.html Change-Id: Ie9314d0dfb20e6372ab6ca44ee38a67bfb85dd7c Story: 2002845 Task: 22793 Signed-off-by: David Sullivan --- .../yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo index 54c5fc43..3d74243f 100644 --- a/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo +++ b/centos-mirror-tools/yum.repos.d/StarlingX_CentOS-OpenStack-queens.repo @@ -5,7 +5,7 @@ [centos-openstack-queens] name=CentOS-7 - OpenStack queens -baseurl=http://mirror.centos.org/$contentdir/$releasever/cloud/$basearch/openstack-queens/ +baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud From 8823d787402c1ccbda7fc38ca25c0e22227e69d1 Mon Sep 17 00:00:00 2001 From: David Sullivan Date: Wed, 22 Aug 2018 11:50:16 -0400 Subject: [PATCH 22/27] Add required packages for kuberentes api upversion We are moving to 4.0 for the kubernetes api. That requires the following packages: python2-dateutil python2-google-auth python2-requests-oauthlib Change-Id: I514ada0563ac16dd77a33979a4c021e86c59ada9 Story: 2002845 Task: 22793 Signed-off-by: David Sullivan --- centos-mirror-tools/rpms_centos.lst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 3671a0af..58d2335a 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -1096,6 +1096,7 @@ python2-cursive-0.1.2-1.el7.noarch.rpm python2-cycler-0.10.0-2.el7.noarch.rpm python2-Cython-0.25.2-3.el7.x86_64.rpm python2-daiquiri-1.2.1-1.el7.noarch.rpm +python2-dateutil-2.6.1-1.el7.noarch.rpm python2-ddt-1.1.3-1.el7.noarch.rpm python2-debtcollector-1.17.0-1.el7.noarch.rpm python2-deprecation-1.0-3.el7.noarch.rpm @@ -1116,6 +1117,7 @@ python2-gabbi-1.33.0-1.el7.noarch.rpm python2-gevent-1.1.2-2.el7.x86_64.rpm python2-gflags-2.0-5.el7.noarch.rpm python2-gnocchiclient-3.3.1-1.el7.noarch.rpm +python2-google-auth-1.3.0-1.el7.noarch.rpm python2-greenlet-0.4.9-1.el7.x86_64.rpm python2-hacking-0.13.0-1.el7.noarch.rpm python2-idna-2.5-1.el7.noarch.rpm @@ -1131,7 +1133,7 @@ python2-jsonschema-2.5.1-3.el7.noarch.rpm python2-keystoneauth1-3.1.0-1.el7.noarch.rpm python2-keystoneclient-3.13.0-1.el7.noarch.rpm python2-kombu-4.0.2-5.el7.noarch.rpm -python2-kubernetes-1.0.0-0.3.b3.el7.noarch.rpm +python2-kubernetes-4.0.0-1.el7.noarch.rpm python2-lz4-0.9.0-1.el7.x86_64.rpm python2-marathon-0.8.8-1.el7.noarch.rpm python2-markupsafe-0.23-16.el7.x86_64.rpm @@ -1204,6 +1206,7 @@ python2-pytest-3.0.6-2.el7.noarch.rpm python2-qpid-proton-0.22.0-1.el7.x86_64.rpm python2-rcssmin-1.0.6-2.el7.x86_64.rpm python2-reno-2.5.0-1.el7.noarch.rpm +python2-requests-oauthlib-0.8.0-5.el7.noarch.rpm python2-requestsexceptions-1.3.0-1.el7.noarch.rpm python2-retryz-0.1.8-1.el7.noarch.rpm python2-rfc3986-0.3.1-1.el7.noarch.rpm @@ -1294,6 +1297,7 @@ python-croniter-0.3.4-2.el7.noarch.rpm python-d2to1-0.2.11-1.el7.noarch.rpm python-daemon-1.6-4.el7.noarch.rpm python-dateutil-2.4.2-1.el7.noarch.rpm +python-dateutil-2.6.1-1.el7.src.rpm python-decorator-3.4.0-3.el7.noarch.rpm python-deltarpm-3.6-3.el7.x86_64.rpm python-django-1.8.14-1.el7.noarch.rpm @@ -1321,6 +1325,7 @@ python-flake8-2.4.1-2.el7.noarch.rpm python-flask-0.10.1-4.el7.noarch.rpm python-freezegun-0.3.8-2.el7.noarch.rpm python-glance-store-0.22.0-1.el7.src.rpm +python-google-auth-1.3.0-1.el7.src.rpm python-heatclient-1.11.0-1.el7.src.rpm python-httplib2-0.9.2-1.el7.noarch.rpm python-iniparse-0.4-9.el7.noarch.rpm @@ -1341,6 +1346,7 @@ python-keystoneclient-3.13.0-1.el7.src.rpm python-keystonemiddleware-4.17.0-1.el7.src.rpm python-kitchen-1.1.1-5.el7.noarch.rpm python-kmod-0.9-4.el7.x86_64.rpm +python-kubernetes-4.0.0-1.el7.src.rpm python-ldap-2.4.15-2.el7.x86_64.rpm python-ldappool-1.0-4.el7.noarch.rpm python-lesscpy-0.9j-4.el7.noarch.rpm From 56be9d1b3c8f109606bf4630e563382ba28b457b Mon Sep 17 00:00:00 2001 From: Jason McKenna Date: Fri, 24 Aug 2018 11:07:56 -0400 Subject: [PATCH 23/27] Uprev qpid-proton to satify requirements Package python2-pyngus uprev'd, but new version requires updated qpid-proton packages. This change adds those packages. Tested by doing full build with new packages in repo, and product now passes compile stage. Change-Id: I73e60e8f68499c1b621b5b9d24f13a3b7797288f Signed-off-by: Jason McKenna Closes-Bug: 1788872 --- centos-mirror-tools/rpms_centos.lst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/centos-mirror-tools/rpms_centos.lst b/centos-mirror-tools/rpms_centos.lst index 3671a0af..9209c477 100644 --- a/centos-mirror-tools/rpms_centos.lst +++ b/centos-mirror-tools/rpms_centos.lst @@ -1201,7 +1201,7 @@ python2-pyroute2-0.4.19-1.el7.noarch.rpm python2-pysaml2-3.0.2-2.el7.noarch.rpm python2-pysnmp-4.3.2-3.el7.noarch.rpm python2-pytest-3.0.6-2.el7.noarch.rpm -python2-qpid-proton-0.22.0-1.el7.x86_64.rpm +python2-qpid-proton-0.24.0-1.el7.x86_64.rpm python2-rcssmin-1.0.6-2.el7.x86_64.rpm python2-reno-2.5.0-1.el7.noarch.rpm python2-requestsexceptions-1.3.0-1.el7.noarch.rpm @@ -1474,7 +1474,7 @@ pytz-2016.10-2.el7.noarch.rpm pyxattr-0.5.1-5.el7.x86_64.rpm PyYAML-3.10-11.el7.x86_64.rpm qdox-1.12.1-10.el7.noarch.rpm -qpid-proton-c-0.22.0-1.el7.x86_64.rpm +qpid-proton-c-0.24.0-1.el7.x86_64.rpm qrencode-devel-3.4.1-3.el7.x86_64.rpm qrencode-libs-3.4.1-3.el7.x86_64.rpm qt-4.8.5-13.el7.x86_64.rpm From 8b9a61d38cc7021005d5c35ae49383916f400d78 Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Wed, 25 Jul 2018 14:21:58 -0500 Subject: [PATCH 24/27] Add mirror-check.sh to verify changes in upstream packages. The mirror downloader uses a set of CentOS repositories to get all the packages on specific versions, detailed on the .lst files. If there are failures regarding new package versions we won't know until the end of the download process. This is something, that can take a couple of hours. This script uses the yum cache to verify if the package is present in the current database. Therefore it is possible to verify if the packages are available in the same version without the need of download them. This speeds up the time to verify the mirror to about 5 minutes. This script doesn't consider (yet) the 3rd party packages, those needs to be verified in a different way. Change-Id: I4472d9b6409c9498dcb749c9dee1b5ce43a06025 Signed-off-by: Erich Cordoba --- centos-mirror-tools/mirror-check.sh | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 centos-mirror-tools/mirror-check.sh diff --git a/centos-mirror-tools/mirror-check.sh b/centos-mirror-tools/mirror-check.sh new file mode 100755 index 00000000..6a8c4776 --- /dev/null +++ b/centos-mirror-tools/mirror-check.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This script checks if the required packages in the .lst file list is +# actually downloadable. Sometimes, the release number in upstream is +# changed and that causes a mismatch in the build requirements. +# We can find this problems in an early stage without the need to +# download all the packages. +# +# The yum cache contains this information, more specific the primary_db +# files, so iterating over the content of .lst, parse the name of the +# package and get the information on what is available to download +# should be enough to know the status of the mirror. +# +# If a package is not found then the script will try to get the avai- +# lable version and log that into the error log. By this way we get +# notified on what changed in the external repositories. +# +# How to run: +# This script is intended to be run inside the downloader container. +# It needs that all the CentOS repositories are well setup. +# +# ./mirror-check.sh +# +# And you should see the checking in progress. + +_print_msg() { echo -en "$(date -u +"%Y-%m-%d %H-%M-%S") ==> $1"; } +info() { _print_msg "INFO: $1\n"; } +info_c() { _print_msg "INFO: $1"; } +warning() { _print_msg "WARN: $1\n"; } +error() { _print_msg "ERROR: $1\n"; } + +RPMS_CENTOS_LIST="rpms_centos.lst" +RPMS_3RD_PARTY_LIST="rpms_centos3rdparties.lst" +ERROR_LOG_FILE="mirror-check-failures.log" +truncate -s 0 $ERROR_LOG_FILE +retcode=0 +extra_opts="" + +usage() { + echo "$0 [-c ]" + echo "" + echo "Options:" + echo " -c: Use an alternate yum.conf rather than the system file (option passed" + echo " on to subscripts when appropriate)" + echo "" +} + +get_rpm_name() { + _rpm_file_name=$1 + rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'-' -f3- | rev) + echo "$rpm_name" +} + +get_rpm_full_name() { + _rpm_file_name=$1 + rpm_name=$(echo "$_rpm_file_name" | rev | cut -d'.' -f2- | rev) + echo "$rpm_name" +} + +get_rpm_arch() { + arch=$(echo "$1" | rev | cut -d'.' -f2 | rev) + echo "$arch" +} + +get_repoquery_info() { + _arch=$1 + _package_name=$2 + if [ "$_arch" == "x86_64" ]; then + # To filter out the i686 packages + repoquery_opts="--archlist=x86_64" + elif [ "$_arch" == "src" ]; then + repoquery_opts="--archlist=src" + else + repoquery_opts= + fi + repoquery $extra_opts -C --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' \ + $repoquery_opts "$_package_name" +} + +_check_rpms() { + p=$1 + full_name=$(get_rpm_full_name "$p") + rpm_name=$(get_rpm_name "$p") + arch=$(get_rpm_arch "$p") + info_c "Checking $full_name... " + _repoquery=$(get_repoquery_info "$arch" "$full_name") + if [ -z "$_repoquery" ]; then + echo -e "FAILED!" + available_pkgs=$(get_repoquery_info "$arch" "$rpm_name") + echo -e "Package $full_name not found, available $available_pkgs" >> $ERROR_LOG_FILE + retcode=1 + else + if [ "$full_name" == "$_repoquery" ]; then + echo -e "OK" + else + echo -e "FAILED!" + retcode=1 + echo -e "Required $full_name but found $_repoquery" >> $ERROR_LOG_FILE + fi + fi +} + +check_rpms() { + _rpms_list=$1 + for p in $_rpms_list; do + _check_rpms "$p" + done +} + +while getopts "c:" opt; do + case $opt in + c) + extra_opts="-c ${OPTARG}" + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1 + ;; + esac +done + +info "Getting yum cache" +if ! yum $extra_opts makecache; then + error "There was a problem getting yum cache" + exit 1 +fi + +for rpm_list in "$RPMS_CENTOS_LIST" "$RPMS_3RD_PARTY_LIST"; do + info "Reading $rpm_list..." + for arch in "src" "noarch" "x86_64"; do + info "Getting info for $arch packages..." + rpms=$(echo "$(grep -F "$arch.rpm" < $rpm_list)") + check_rpms "$rpms" + done +done + +if [ $retcode -ne 0 ]; then + error "Failures found, error log:" + error "==========================" + cat $ERROR_LOG_FILE +fi + +exit $retcode From 2f10f6f34d3497f8592a934cc05178135c7d2e43 Mon Sep 17 00:00:00 2001 From: Abraham Arce Date: Tue, 24 Jul 2018 10:16:37 -0500 Subject: [PATCH 25/27] Add virtual environemnt deployment scripts This adds two sets of deployments scripts to aid in setting up virtual environments for testing StarligX, for libvirt/qemu and VirtualBox. This is the first exposure of an internal project and is not fully error-proofed, it should be examined carefully to understand what the scripts will do to your system before running them. The first set of fixes for this to make it less instrusive and work in a shared server follow in https://review.openstack.org/#/c/597643. There are also a couple of bashate errors fixed since that is now a voting job... [NOTE(dtroyer): I converted the original commit message into a README because that's information that needs to be with the scripts.] Needed-by: https://review.openstack.org/#/c/597643 Change-Id: I0a6a148720b7a239380fd48f7ffdab272472e664 Co-authored-by: Lianhao Lu Co-authored-by: Brian Avery Co-authored-by: Nakul Dahiwade Co-authored-by: Yan Chen Co-authored-by: Ruijing Guo Co-authored-by: Shuicheng Lin Co-authored-by: Felipe de Jesus Ruiz Garcia Co-authored-by: Jose Perez Carranza Co-authored-by: Eddie Ramirez Co-authored-by: Kailun Qin Co-authored-by: Forrest Zhao Co-authored-by: Kailun Qin Signed-off-by: Abraham Arce Signed-off-by: Dean Troyer --- centos-mirror-tools/dl_rpms.sh | 7 +- deployment/README.rst | 141 ++++++++++++++++ deployment/libvirt/compute.xml | 115 +++++++++++++ deployment/libvirt/controller.xml | 130 ++++++++++++++ deployment/libvirt/controller_allinone.xml | 130 ++++++++++++++ deployment/libvirt/destroy_allinone.sh | 29 ++++ .../libvirt/destroy_standard_controller.sh | 46 +++++ deployment/libvirt/install_packages.sh | 24 +++ deployment/libvirt/setup_allinone.sh | 78 +++++++++ .../libvirt/setup_standard_controller.sh | 93 +++++++++++ deployment/provision/simplex_stage_1.sh | 121 ++++++++++++++ deployment/provision/simplex_stage_2.sh | 56 +++++++ deployment/virtualbox/all_in_one.conf | 26 +++ deployment/virtualbox/serial_vm.sh | 3 + deployment/virtualbox/setup_vm.sh | 158 ++++++++++++++++++ .../virtualbox/standard_controller.conf | 20 +++ deployment/virtualbox/start_vm.sh | 19 +++ deployment/virtualbox/stop_vm.sh | 5 + tox.ini | 5 +- 19 files changed, 1203 insertions(+), 3 deletions(-) create mode 100644 deployment/README.rst create mode 100644 deployment/libvirt/compute.xml create mode 100644 deployment/libvirt/controller.xml create mode 100644 deployment/libvirt/controller_allinone.xml create mode 100755 deployment/libvirt/destroy_allinone.sh create mode 100755 deployment/libvirt/destroy_standard_controller.sh create mode 100755 deployment/libvirt/install_packages.sh create mode 100755 deployment/libvirt/setup_allinone.sh create mode 100755 deployment/libvirt/setup_standard_controller.sh create mode 100755 deployment/provision/simplex_stage_1.sh create mode 100755 deployment/provision/simplex_stage_2.sh create mode 100644 deployment/virtualbox/all_in_one.conf create mode 100755 deployment/virtualbox/serial_vm.sh create mode 100755 deployment/virtualbox/setup_vm.sh create mode 100644 deployment/virtualbox/standard_controller.conf create mode 100755 deployment/virtualbox/start_vm.sh create mode 100755 deployment/virtualbox/stop_vm.sh diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 327f2143..e4784fdd 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -180,8 +180,8 @@ download () { local _file=$1 local _level=$2 - local _list=$(cat $_file) - local _from=$(get_from $_file) + local _list + local _from local _type="" local rc=0 @@ -191,6 +191,9 @@ download () { local rpm_url="" local SFILE="" + _list=$(cat $_file) + _from=$(get_from $_file) + echo "now the rpm will come from: $_from" for ff in $_list; do download_cmd="" diff --git a/deployment/README.rst b/deployment/README.rst new file mode 100644 index 00000000..9bcb7a14 --- /dev/null +++ b/deployment/README.rst @@ -0,0 +1,141 @@ +StarlingX Deployment in Virtualized Environments +================================================ + +A StarlingX system can be installed in a variety of platforms with the following +deployment options: + +- Standard Controller + + - Dedicated Storage + - Controller Storage + +- All-in-one + + - Duplex + - Simplex + +Deployment options uses a variety of configurations based on 3 node identities: + +- Controller +- Storage +- Compute + +Standard Controller :: Dedicated Storage +---------------------------------------- + +The software installation workflow for an initial Ceph-backed block +storage on dedicated storage nodes is: + +- Controller-0 Installation and Provisioning +- Controller-1 / Compute Host / Storage Host Installation +- Controller-1 Provisioning +- Provider Network Configuration +- Compute Host Provisioning +- Storage Host Provisioning + +Standard Controller :: Controller Storage +----------------------------------------- + +The software installation workflow for an initial LVM-backed block +storage on controller nodes is: + +- Controller-0 Installation +- Controller-0 and System Provisioning +- Controller-1 / Compute Host Installation +- Controller-1 Provisioning +- Compute Host Provisioning + +All-in-one :: Duplex +-------------------- + +The software installation workflow for two combined controller / compute +nodes is: + +- Controller-0 Installation and Provisioning +- Controller-1 Installation and Provisioning + +All-in-one :: Simplex +--------------------- + +The software installation workflow for a single combined controller / compute +node is: + +- Controller-0 Installation and Provisioning + +Virtualization Environments +--------------------------- + +The available virtualization products where StarlingX has been deployed +are: + +- VirtualBox +- Libvirt/QEMU + +Directory Structure +------------------- + +Deployment directory hosts a total of 3 directories and 18 files:: + + $ tree -L 3 deployment/ + deployment/ + ├── libvirt + │   ├── compute.xml + │   ├── controller_allinone.xml + │   ├── controller.xml + │   ├── destroy_allinone.sh + │   ├── destroy_standard_controller.sh + │   ├── install_packages.sh + │   ├── setup_allinone.sh + │   └── setup_standard_controller.sh + ├── provision + │   ├── simplex_stage_1.sh + │   └── simplex_stage_2.sh + └── virtualbox + ├── all_in_one.conf + ├── serial_vm.sh + ├── setup_vm.sh + ├── standard_controller.conf + ├── start_vm.sh + └── stop_vm.sh + +Directory: libvirt +~~~~~~~~~~~~~~~~~~ + +Deployment under Libvirt/QEMU uses a set of xml files to define the node +identity: + +- Controller All-in-one +- Controller +- Compute + +These nodes are used to create the virtual machines and the network interfaces +to setup the StarlingX system: + +- Setup All-in-one + + - 2 Controllers + +- Setup Standard Controller + + - 2 Controllers + - 2 Computes + +Directory: virtualbox +~~~~~~~~~~~~~~~~~~~~~ + +Deployment under VirtualBox uses a set of configuration files to define the +StarlingX system: + +- All-in-one Configuration +- Standard Controller Configuration + +These configurations files are used to create the virtual machines and the +network interfaces from a single script: + +- Setup VM + +Directory: provision +~~~~~~~~~~~~~~~~~~~~ + +A set of scripts are provided to automate the provisioning of data interfaces and +local storage resources for the compute function for StarlingX Duplex or Simplex. diff --git a/deployment/libvirt/compute.xml b/deployment/libvirt/compute.xml new file mode 100644 index 00000000..9e7c699a --- /dev/null +++ b/deployment/libvirt/compute.xml @@ -0,0 +1,115 @@ + + NAME + 16777216 + 16777216 + 4 + + /machine + + + hvm + + + + + + + + + + + + + + destroy + restart + destroy + + /usr/bin/qemu-system-x86_64 + + + + + + + +
+ + + + + + + +
+ + + +
+ + + + + + +
+ + + + + + +
+ + + + + + + +
+ + + + + + +
+ + + + + + +
+ + + + + + + + + + + + + + + + +