From 9ef066dbb9c8a89518606231216f2a62db91e0e9 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 16 Oct 2018 16:24:04 -0500 Subject: [PATCH] Add projects.yaml support The projects.yaml lists all of the "official" StarlingX projects (the ones actually owned by StarlingX and not upstream forks). It is in YAML format to be machine-readable and useful for a number of tools. The first such use is to generate project-specific pages in the governance documentation listing the projects, tech leads and project leads. The current content is basically taken from the existing wiki pages and is lacking in certain areas. Please review it carefully. projects.py is based on the OpenStack governance teams.py. Introduce hacking and switch to flake8 for Python linting, and fix the pep8 errors in members.py. Change-Id: I514de8d574df8e60bb488974f2796e2c41010363 Signed-off-by: Dean Troyer --- .gitignore | 4 + doc/requirements.txt | 1 + doc/source/_exts/members.py | 6 +- doc/source/_exts/projects.py | 157 ++++++++++++++++++++++ doc/source/conf.py | 7 + reference/tsc/index.rst | 5 + reference/tsc/projects.yaml | 153 +++++++++++++++++++++ reference/tsc/projects/index.rst | 16 +++ test-requirements.txt | 1 + tools/check_projects_yaml_alphabetized.sh | 24 ++++ tox.ini | 5 +- 11 files changed, 374 insertions(+), 5 deletions(-) create mode 100644 doc/source/_exts/projects.py create mode 100644 reference/tsc/projects.yaml create mode 100644 reference/tsc/projects/index.rst create mode 100755 tools/check_projects_yaml_alphabetized.sh diff --git a/.gitignore b/.gitignore index 894a44c..e7e7d2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Only pay attention to the project index +reference/tsc/projects/*.rst +!reference/tsc/projects/index.rst + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/doc/requirements.txt b/doc/requirements.txt index 0d2c6fb..4b5d8d8 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ openstack-doc-tools>=1.6.0 openstackdocstheme>=1.19.0 # Apache-2.0 sphinx>=1.6.2 +yamlordereddictloader diff --git a/doc/source/_exts/members.py b/doc/source/_exts/members.py index f07ade2..339a49c 100644 --- a/doc/source/_exts/members.py +++ b/doc/source/_exts/members.py @@ -15,8 +15,9 @@ import re from docutils import nodes -from docutils.parsers.rst.directives.tables import Table from docutils.parsers.rst import directives +from docutils.parsers.rst.directives.tables import Table +from docutils.utils import SystemMessagePropagation from sphinx.util import logging LOG = logging.getLogger(__name__) @@ -64,7 +65,6 @@ class MembersTable(Table): def run(self): env = self.state.document.settings.env app = env.app - config = app.config # The required argument to the directive is the name of the # file to parse. @@ -90,7 +90,7 @@ class MembersTable(Table): 'Error processing memberstable directive:\n%s' % err, nodes.literal_block(self.block_text, self.block_text), line=self.lineno, - ) + ) return [error] # Now find the real path to the file, relative to where we are. diff --git a/doc/source/_exts/projects.py b/doc/source/_exts/projects.py new file mode 100644 index 0000000..195dd4b --- /dev/null +++ b/doc/source/_exts/projects.py @@ -0,0 +1,157 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Report on the current list of projects""" + +import copy +from docutils import nodes +from docutils.parsers import rst +from docutils import statemachine +import os +from sphinx.util import logging +from sphinx.util.nodes import nested_parse_with_titles +import yaml +import yamlordereddictloader + + +LOG = logging.getLogger(__name__) + +IRC_LOG_URL_BASE = 'http://eavesdrop.openstack.org/irclogs/%23' + +_projects_yaml = {} + + +def _get_project_data(): + """Return a copy of the project data.""" + return copy.deepcopy(_projects_yaml) + + +def _load_project_file(filename='reference/tsc/projects.yaml'): + with open(filename, 'r', encoding='utf-8') as f: + return yaml.load( + f.read(), + Loader=yamlordereddictloader.Loader, + ) + + +def _project_to_rst(name, info): + + if 'service' in info: + title = "{0} ({1})".format(name.title(), info['service']) + elif name == 'I18n': + title = name + else: + title = name.title() + + yield '.. _project-%s:' % _slugify(name) + yield '' + yield '=' * len(title) + yield title + yield '=' * len(title) + yield '' + yield ':Home Page: ' + info.get('url', '') + tl = info.get('tl', {'name': '', 'irc': '', 'email': ''}) + yield ':Technical Lead: %(name)s (``%(irc)s``) <%(email)s>' % tl + pl = info.get('pl', {'name': '', 'irc': '', 'email': ''}) + yield ':Project Lead: %(name)s (``%(irc)s``) <%(email)s>' % pl + irc_channel = info.get('irc-channel') + if irc_channel: + yield ':IRC Channel: `#%s <%s%s>`__' % ( + irc_channel, IRC_LOG_URL_BASE, irc_channel) + service = info.get('service') + if service: + yield ':Service: ' + service + yield '' + mission = info.get('mission', '').rstrip() + if mission: + yield "Mission" + yield '-------' + yield '' + yield mission + yield '' + yield 'Deliverables' + yield '------------' + yield '' + deliverables = info.get('deliverables', []) + if deliverables: + for repo_name, deliverable in deliverables.items(): + yield repo_name + yield '~' * len(repo_name) + yield '' + yield ':Repositories: ' + ', '.join( + ':repo:`%s`' % repo + for repo in deliverable.get('repos', []) + ) + yield '' + tags = deliverable.get('tags', []) + if tags: + yield ':Tags:' + yield '' + for tag in tags: + yield ' - :ref:`tag-%s`' % tag + yield '' + else: + yield 'None' + yield '' + + +def _slugify(name): + """Convert name to slug form for references.""" + return name.lower().replace(' ', '-') + + +def _write_project_pages(app): + all_projects = _get_project_data() + files = [] + for project, info in all_projects.items(): + LOG.info("project: %s" % project) + slug = _slugify(project) + filename = 'reference/tsc/projects/%s.rst' % slug + LOG.info('generating project page for %s' % project) + with open(filename, 'w', encoding='utf-8') as f: + f.write('\n'.join(_project_to_rst(project, info))) + files.append(filename) + return files + + +class ProjectListDirective(rst.Directive): + + has_content = False + + def run(self): + all_projects = _get_project_data() + + # Build the view of the data to be parsed for rendering. + result = statemachine.ViewList() + for project_name in sorted(all_projects.keys()): + project_info = all_projects[project_name] + for line in _project_to_rst(project_name, project_info): + result.append(line, '<' + __name__ + '>') + + # Parse what we have into a new section. + node = nodes.section() + node.document = self.state.document + nested_parse_with_titles(self.state, result, node) + + return node.children + + +def setup(app): + global _projects_yaml + + LOG.info('loading projects extension') + app.add_directive('projectlist', ProjectListDirective) + + filename = os.path.abspath('reference/tsc/projects.yaml') + LOG.info('reading %s' % filename) + _projects_yaml = _load_project_file(filename) + _write_project_pages(app) diff --git a/doc/source/conf.py b/doc/source/conf.py index a4a53ec..2a60305 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -43,8 +43,15 @@ release = u'0.1' extensions = [ 'members', 'openstackdocstheme', + 'projects', + 'sphinx.ext.extlinks', ] +# Define shorthand roles for making links to common destinations. +extlinks = { + 'repo': ('https://git.startlingx.io/cgit/%s', ''), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/reference/tsc/index.rst b/reference/tsc/index.rst index c8b9d6d..71d129c 100644 --- a/reference/tsc/index.rst +++ b/reference/tsc/index.rst @@ -17,6 +17,11 @@ documentation the links below are directly there. * `The Four Opens `__ * `StarlingX Charter `__ +.. toctree:: + :maxdepth: 1 + + projects/index + Current Members =============== diff --git a/reference/tsc/projects.yaml b/reference/tsc/projects.yaml new file mode 100644 index 0000000..ab1379a --- /dev/null +++ b/reference/tsc/projects.yaml @@ -0,0 +1,153 @@ +--- +config: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: John Kung + irc: + email: John.Kung@windriver.com + irc-channel: starlingx + service: Configuration + mission: > + Configure all the things + url: https://docs.starlingx.io/stx-config + deliverables: + config: + repos: + - stx-config + +distcloud: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: Bart Wensley + irc: + email: Barton.Wensley@windriver.com + irc-channel: starlingx + service: Distributed Cloud + mission: > + Spreading out the goodness + url: https://docs.starlingx.io/stx-distcloud + deliverables: + fault: + repos: + - stx-distcloud + - stx-distcloud-client + +fault: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: Tao Liu + irc: + email: Tao.Liu@windriver.com + irc-channel: starlingx + service: Fault Management + mission: > + Manage the faults + url: https://docs.starlingx.io/stx-fault + deliverables: + fault: + repos: + - stx-fault + +gui: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: Tyler Smith + irc: + email: Tyler.Smith@windriver.com + irc-channel: starlingx + service: Dashboard Plugin + mission: > + Web GUI + url: https://docs.starlingx.io/stx-gui + deliverables: + fault: + repos: + - stx-gui + +ha: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: Bin Quan + irc: + email: Bin.Qian@windriver.com + irc-channel: starlingx + service: High Availability + mission: > + Stayin' Alive + url: https://docs.starlingx.io/stx-ha + deliverables: + fault: + repos: + - stx-ha + +metal: + pl: + name: Ken Young + irc: + email: Ken.Young@windriver.com + tl: + name: Eric MacDonald + irc: + email: Eric.MacDonald@windriver.com + irc-channel: starlingx + service: Bare Metal Management + mission: > + Tin Cans 'R Us' + url: https://docs.starlingx.io/stx-metal + deliverables: + fault: + repos: + - stx-metal + +nfv: + pl: + name: Frank Miller + irc: + email: Frank.Miller@windriver.com + tl: + name: Bart Wensley + irc: + email: Barton.Wensley@windriver.com + irc-channel: starlingx + service: Network Function Virtualization + mission: > + Don't just stand there, do something! + url: https://docs.starlingx.io/stx-nfv + deliverables: + fault: + repos: + - stx-nfv + +update: + pl: + name: Dariush Eslimi + irc: + email: Dariush.Eslimi@windriver.com + tl: + name: Don Penney + irc: + email: Don.Penney@windriver.com + irc-channel: starlingx + service: Software Update + mission: > + Keeping up with the neighbors + url: https://docs.starlingx.io/stx-update + deliverables: + fault: + repos: + - stx-update diff --git a/reference/tsc/projects/index.rst b/reference/tsc/projects/index.rst new file mode 100644 index 0000000..e6cbe91 --- /dev/null +++ b/reference/tsc/projects/index.rst @@ -0,0 +1,16 @@ +================== +StarlingX Projects +================== + +The projects of StarlingX + +.. _projects: + +Projects +-------- + +.. toctree:: + :glob: + :titlesonly: + + * diff --git a/test-requirements.txt b/test-requirements.txt index 266f880..221759a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ +hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 PyYAML>=3.1.0 yamllint>=0.5.2 diff --git a/tools/check_projects_yaml_alphabetized.sh b/tools/check_projects_yaml_alphabetized.sh new file mode 100755 index 0000000..0e5dd15 --- /dev/null +++ b/tools/check_projects_yaml_alphabetized.sh @@ -0,0 +1,24 @@ +#!/bin/bash -xe + +# Checks that reference/projects.yaml alphabetized and prints list of +# projects that should be sorted. + +export TMPDIR=$(mktemp -d tox.XXXXXX) +trap "rm -rf $TMPDIR" EXIT + +pushd $TMPDIR +PROJECTS_LIST=$OLDPWD/reference/tsc/projects.yaml + +grep '^[a-zA-Z0-9]' $PROJECTS_LIST > projects_list + +LC_ALL=C sort --ignore-case projects_list -o projects_list.sorted + +if ! diff projects_list projects_list.sorted > projects_list.diff; then + echo "The following projects should be alphabetized: " + cat projects_list.diff | grep -e '> ' + exit 1 +else + echo "Projects alphabetized" +fi + +popd diff --git a/tox.ini b/tox.ini index 8eb9e98..17b955e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,13 +27,14 @@ commands = \( -name .tox -prune \) \ -o -type f -name '*.yaml' \ -print0 | xargs -0 yamllint" + {toxinidir}/tools/check_projects_yaml_alphabetized.sh + flake8 [testenv:pep8] basepython = python3 usedevelop = False skip_install = True -deps = pep8 -commands = pep8 +commands = flake8 [testenv:venv] basepython = python3