add a validation step to check README.rst parsing for PyPI

PyPI is now more strict about requiring README.rst files to be
parseable. Failures to parse the readme block uploading new versions
of packages. To detect these errors in advance, we should validate the
README formatting as part of validating a release.

Change-Id: I416befcf3e47818680018394fa63de8983351587
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

openstack_releases/cmds/validate.py

@@ -570,6 +570,49 @@ def validate_tarball_base(deliv, context):
                         sdist, expected))
 
 
+@applies_to_released
+def validate_pypi_readme(deliv, context):
+    "Does the README look right for PyPI?"
+
+    # Check out the repositories to the hash for the latest release on
+    # the branch so we can find the setup.py and README.rst there (if
+    # it exists), in case that has been removed from master after a
+    # project is retired. This also ensures we get the right name for
+    # the branch, in case the sdist name changes over time.
+    latest_release = deliv.releases[-1]
+    for project in latest_release.projects:
+        gitutils.safe_clone_repo(
+            context.workdir, project.repo.name, project.hash, context)
+
+    for repo in deliv.repos:
+
+        job_templates = context.zuul_projects.get(repo.name, {}).get(
+            'templates', [])
+        LOG.debug('{} has job templates {}'.format(repo.name, job_templates))
+
+        # Look for jobs that appear to be talking about publishing to
+        # PyPI. There are variations.
+        pypi_jobs = [
+            j
+            for j in job_templates
+            if 'pypi' in j
+        ]
+
+        if not pypi_jobs:
+            print('rule only applies to repos publishing to PyPI')
+            continue
+
+        LOG.debug('{} publishes to PyPI via {}'.format(repo.name, pypi_jobs))
+
+        try:
+            pythonutils.check_readme_format(context.workdir, repo.name)
+        except Exception as err:
+            context.error('README check for {} failed: {}'.format(
+                repo.name, err))
+        else:
+            print('OK')
+
+
 @applies_to_released
 def validate_pypi_permissions(deliv, context):
     "Do we have permission to upload to PyPI?"
@@ -1511,6 +1554,7 @@ def main():
             validate_model,
             validate_release_type,
             validate_pypi_permissions,
+            validate_pypi_readme,
             validate_gitreview,
             validate_release_sha_exists,
             validate_existing_tags,

openstack_releases/pythonutils.py

@@ -59,6 +59,22 @@ def get_sdist_name(workdir, repo):
     return name
 
 
+def check_readme_format(workdir, repo):
+    "Verify that the README format looks OK."
+    dest = os.path.join(workdir, repo)
+    setup_path = os.path.join(dest, 'setup.py')
+    if not os.path.exists(setup_path):
+        LOG.debug('did not find %s, maybe %s is not a python project',
+                  setup_path, repo)
+        return None
+    # NOTE(dhellmann): This relies on validate being run via tox so
+    # that python3 is present and the docutils package is installed.
+    processutils.check_call(
+        ['python3', 'setup.py', 'check', '--restructuredtext', '--strict'],
+        cwd=dest,
+    )
+
+
 def get_pypi_info(dist_name):
     "Return PyPI information for the distribution."
     canonical_name = packaging_utils.canonicalize_name(dist_name)