provide separate release types for server and non-server deliverables

Change the 'std' release type to 'python-server' and add a
'python-pypi' release type for deliverables that are published to
PyPI.

Separate the release job validation from the validation of release
version numbers and other settings to make the logic clearer.

Add a new function to determine the release type for a project, either
by checking the explicit value or guessing.

Update the unit tests that relied on 'std'.

Remove a unit test that tested a code path that has been removed.

Change-Id: I704ec75fec61ecb6ee379239a5fa8612cb01b426
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

README.rst

@@ -397,10 +397,14 @@ The top level of a deliverable file is a mapping with keys:
 ``release-type``
   This (optional) key sets the level of validation for the versions numbers.
 
-  ``std``
+  ``python-server``
     Default: Enforces 3 digit semver version numbers in releases and allows
     for common alpha, beta and dev releases.  This should be appropriate for
-    most OpenStack release requirements.
+    most OpenStack component release requirements.
+
+  ``python-pypi``
+    Like ``python-server`` but requires the jobs to publish the component
+    to the Python Package Index (PyPI).
 
   ``xstatic``
     Allows a more flexible versioning in line with xstatic package guidelines

deliverables/_independent/ldappool.yaml

@@ -1,7 +1,6 @@
 ---
 launchpad: ldappool
 team: keystone
-release-type: std
 type: library
 releases:
   - version: 2.0.0

deliverables/_independent/os-api-ref.yaml

@@ -2,7 +2,6 @@
 launchpad: openstack-doc-tools
 team: Documentation
 type: library
-release-type: std
 releases:
   - version: 0.1.0
     projects:

deliverables/_independent/os-testr.yaml

@@ -1,7 +1,6 @@
 ---
 launchpad: os-testr
 team: Quality Assurance
-release-type: std
 type: library
 include-pypi-link: yes
 releases:

deliverables/_independent/os-traits.yaml

@@ -1,7 +1,6 @@
 ---
 launchpad: nova
 team: nova
-release-type: std
 type: library
 releases:
   - version: 0.1.0

deliverables/_independent/sqlalchemy-migrate.yaml

@@ -1,7 +1,6 @@
 ---
 launchpad: sqlalchemy-migrate
 team: sqlalchemy-migrate
-release-type: std
 type: library
 releases:
   - version: 0.6.1

deliverables/_independent/whereto.yaml

@@ -1,7 +1,7 @@
 ---
 launchpad: whereto
 team: whereto
-release-type: std
+release-type: python-pypi
 type: other
 releases:
   - version: 0.1.0

deliverables/queens/release-test.yaml

@@ -2,6 +2,7 @@
 launchpad: nova
 team: 'Release Management'
 type: other
+release-type: python-pypi
 branches:
   - name: stable/queens
     location: 0.10.2

deliverables/queens/tripleo-common.yaml

@@ -1,6 +1,7 @@
 ---
 launchpad: tripleo-common
 release-model: cycle-trailing
+release-type: python-pypi
 team: tripleo
 type: other
 releases:

openstack_releases/cmds/validate.py

@@ -348,6 +348,66 @@ def validate_gitreview(deliverable_info, workdir, mk_warning, mk_error):
                 mk_error('%s has no .gitreview file' % (project['repo'],))
 
 
+def get_release_type(deliverable_info, project, workdir):
+    """Return tuple with release type and boolean indicating whether it
+    was explicitly set.
+
+    """
+    if 'release-type' in deliverable_info:
+        return (deliverable_info['release-type'], True)
+
+    if deliverable_info.get('type') == 'library':
+        return ('python-pypi', False)
+
+    if puppetutils.looks_like_a_module(workdir, project['repo']):
+        return ('puppet', False)
+
+    if npmutils.looks_like_a_module(workdir, project['repo']):
+        return ('nodejs', False)
+
+    return ('python-server', False)
+
+
+def validate_release_type(deliverable_info,
+                          zuul_projects,
+                          series_name,
+                          workdir,
+                          mk_warning,
+                          mk_error):
+    """Apply validation rules for the deliverable based on 'release-type'.
+    """
+
+    link_mode = deliverable_info.get('artifact-link-mode', 'tarball')
+    if link_mode == 'none':
+        print('link-mode is "none", skipping release-type checks')
+        return
+
+    if not deliverable_info.get('releases'):
+        print('no releases listed, skipping release-type checks')
+        return
+
+    for release in deliverable_info.get('releases', []):
+        for project in release['projects']:
+
+            print('checking release-type for {}'.format(project['repo']))
+
+            release_type, was_explicit = get_release_type(
+                deliverable_info, project, workdir,
+            )
+            if was_explicit:
+                print('found explicit release-type {!r}'.format(
+                    release_type))
+            else:
+                print('release-type not given, '
+                      'guessing {!r}'.format(release_type))
+
+            project_config.require_release_jobs_for_repo(
+                deliverable_info, zuul_projects,
+                project['repo'],
+                release_type, mk_warning, mk_error,
+            )
+
+
 def validate_releases(deliverable_info, zuul_projects,
                       series_name,
                       workdir,
@@ -481,44 +541,21 @@ def validate_releases(deliverable_info, zuul_projects,
                               (prev_version, ', '.join(sorted(prev_projects))))
                     else:
 
-                        # We change this default to be more
+                        release_type, was_explicit = get_release_type(
-                        # language-specific before testing the release
+                            deliverable_info, project, workdir,
-                        # jobs.
-                        default_release_type = 'std'
-
-                        # If we have an explicit release type, we can
-                        # bypass some of the checks for languages
-                        # other than python.
-                        explicit_release_type = deliverable_info.get(
-                            'release-type',
                         )
-                        if explicit_release_type:
+                        if was_explicit:
                             print('found explicit release-type {!r}'.format(
-                                explicit_release_type))
+                                release_type))
                         else:
                             print('release-type not given, '
-                                  'determining automatically')
+                                  'guessing {!r}'.format(release_type))
-
-                        is_puppet = (
-                            explicit_release_type == 'puppet' or
-                            ((not explicit_release_type) and
-                             puppetutils.looks_like_a_module(workdir,
-                                                             project['repo']))
-                        )
-
-                        is_nodejs = (
-                            explicit_release_type == 'nodejs' or
-                            ((not explicit_release_type) and
-                             npmutils.looks_like_a_module(workdir,
-                                                          project['repo']))
-                        )
 
                         # If this is a puppet module, ensure
                         # that the tag and metadata file
                         # match.
-                        if is_puppet:
+                        if release_type == 'puppet':
                             print('applying puppet version rules')
-                            default_release_type = 'puppet'
                             puppet_ver = puppetutils.get_version(
                                 workdir, project['repo'])
                             if puppet_ver != release['version']:
@@ -534,9 +571,8 @@ def validate_releases(deliverable_info, zuul_projects,
                         # If this is a npm module, ensure
                         # that the tag and metadata file
                         # match.
-                        if is_nodejs:
+                        if release_type == 'nodejs':
                             print('applying nodejs version rules')
-                            default_release_type = 'nodejs'
                             npm_ver = npmutils.get_version(
                                 workdir, project['repo'])
                             if npm_ver != release['version']:
@@ -549,18 +585,6 @@ def validate_releases(deliverable_info, zuul_projects,
                                     )
                                 )
 
-                        # Check for release jobs (if we ship a tarball)
-                        release_type = deliverable_info.get(
-                            'release-type',
-                            default_release_type,
-                        )
-                        if link_mode != 'none':
-                            project_config.require_release_jobs_for_repo(
-                                deliverable_info, zuul_projects,
-                                project['repo'],
-                                release_type, mk_warning, mk_error,
-                            )
-
                         for e in versionutils.validate_version(
                                 release['version'],
                                 release_type=release_type,
@@ -971,6 +995,14 @@ def main():
         validate_release_notes(deliverable_info, mk_warning, mk_error)
         validate_type(deliverable_info, mk_warning, mk_error)
         validate_model(deliverable_info, series_name, mk_warning, mk_error)
+        validate_release_type(
+            deliverable_info,
+            zuul_projects,
+            series_name,
+            workdir,
+            mk_warning,
+            mk_error,
+        )
         validate_gitreview(deliverable_info, workdir, mk_warning, mk_error)
         validate_releases(
             deliverable_info,

openstack_releases/project_config.py

@@ -81,10 +81,12 @@ def get_zuul_project_data(url=ZUUL_PROJECTS_URL):
 
 # Which jobs are needed for which release types.
 _RELEASE_JOBS_FOR_TYPE = {
-    'std': [
+    'python-server': [
-        'publish-to-pypi',
         'release-openstack-server',
     ],
+    'python-pypi': [
+        'publish-to-pypi',
+    ],
     'neutron': [
         'publish-to-pypi-neutron',
     ],
@@ -140,7 +142,7 @@ def require_release_jobs_for_repo(deliverable_info, zuul_projects, repo,
         # jobs, because we want projects to use the templates.
         expected_jobs = _RELEASE_JOBS_FOR_TYPE.get(
             release_type,
-            _RELEASE_JOBS_FOR_TYPE['std'],
+            _RELEASE_JOBS_FOR_TYPE['python-server'],
         )
         if expected_jobs:
             found_jobs = [

openstack_releases/schema.yaml

@@ -35,7 +35,8 @@ properties:
     type: "object"
   release-type:
     type: "string"
-    enum: [ "std", "xstatic", "fuel", "nodejs", "puppet", "neutron", "horizon" ]
+    enum: [ "python-server", "python-pypi", "xstatic", "fuel",
+            "nodejs", "puppet", "neutron", "horizon" ]
   stable-branch-type:
     type: "string"
     enum: [ "std", "tagless", "upstream" ]

openstack_releases/tests/test_project_config.py

@@ -103,7 +103,7 @@ class TestReleaseJobsStandard(base.BaseTestCase):
             deliverable_info,
             zuul_projects,
             'openstack/releases',
-            'std',
+            'python-pypi',
             warnings.append,
             errors.append,
         )
@@ -128,7 +128,7 @@ class TestReleaseJobsStandard(base.BaseTestCase):
             deliverable_info,
             zuul_projects,
             'openstack/releases',
-            'std',
+            'python-pypi',
             warnings.append,
             errors.append,
         )

openstack_releases/tests/test_validate.py

@@ -417,33 +417,6 @@ class TestValidateReleases(base.BaseTestCase):
         self.tmpdir = self.useFixture(fixtures.TempDir()).path
         gitutils.clone_repo(self.tmpdir, 'openstack/release-test')
 
-    @mock.patch('openstack_releases.project_config.require_release_jobs_for_repo')
-    def test_check_release_jobs(self, check_jobs):
-        deliverable_info = {
-            'releases': [
-                {'version': '99.5.0',
-                 'projects': [
-                     {'repo': 'openstack/release-test',
-                      'hash': '218c9c82f168f1db681b27842b5a829428c6b5e1',
-                      'tarball-base': 'openstack-release-test'},
-                 ]}
-            ],
-        }
-        warnings = []
-        errors = []
-        validate.validate_releases(
-            deliverable_info,
-            {'validate-projects-by-name': {}},
-            'queens',
-            self.tmpdir,
-            warnings.append,
-            errors.append,
-        )
-        print(warnings, errors)
-        self.assertEqual(0, len(warnings))
-        self.assertEqual(0, len(errors))
-        check_jobs.assert_called_once()
-
     def test_invalid_hash(self):
         deliverable_info = {
             'artifact-link-mode': 'none',
@@ -806,6 +779,113 @@ class TestValidateReleases(base.BaseTestCase):
         self.assertEqual(1, len(errors))
 
 
+class TestGetReleaseType(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestGetReleaseType, self).setUp()
+        self.tmpdir = self.useFixture(fixtures.TempDir()).path
+
+    def test_explicit(self):
+        deliverable_info = {
+            'artifact-link-mode': 'none',
+            'release-type': 'explicitly-set',
+            'releases': [
+                {'version': '99.1.0',
+                 'projects': [
+                     {'repo': 'openstack/puppet-watcher',
+                      'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'},
+                 ]}
+            ],
+        }
+        release_type, explicit = validate.get_release_type(
+            deliverable_info,
+            deliverable_info['releases'][0]['projects'][0],
+            self.tmpdir,
+        )
+        self.assertEqual(('explicitly-set', True), (release_type, explicit))
+
+    def test_library(self):
+        deliverable_info = {
+            'artifact-link-mode': 'none',
+            'type': 'library',
+            'releases': [
+                {'version': '99.1.0',
+                 'projects': [
+                     {'repo': 'openstack/puppet-watcher',
+                      'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'},
+                 ]}
+            ],
+        }
+        release_type, explicit = validate.get_release_type(
+            deliverable_info,
+            deliverable_info['releases'][0]['projects'][0],
+            self.tmpdir,
+        )
+        self.assertEqual(('python-pypi', False), (release_type, explicit))
+
+    @mock.patch('openstack_releases.puppetutils.looks_like_a_module')
+    def test_puppet(self, llam):
+        llam.return_value = True
+        deliverable_info = {
+            'artifact-link-mode': 'none',
+            'releases': [
+                {'version': '99.1.0',
+                 'projects': [
+                     {'repo': 'openstack/puppet-watcher',
+                      'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'},
+                 ]}
+            ],
+        }
+        release_type, explicit = validate.get_release_type(
+            deliverable_info,
+            deliverable_info['releases'][0]['projects'][0],
+            self.tmpdir,
+        )
+        self.assertEqual(('puppet', False), (release_type, explicit))
+
+    @mock.patch('openstack_releases.npmutils.looks_like_a_module')
+    def test_nodejs(self, llam):
+        llam.return_value = True
+        deliverable_info = {
+            'artifact-link-mode': 'none',
+            'releases': [
+                {'version': '99.1.0',
+                 'projects': [
+                     {'repo': 'openstack/puppet-watcher',
+                      'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'},
+                 ]}
+            ],
+        }
+        release_type, explicit = validate.get_release_type(
+            deliverable_info,
+            deliverable_info['releases'][0]['projects'][0],
+            self.tmpdir,
+        )
+        self.assertEqual(('nodejs', False), (release_type, explicit))
+
+    @mock.patch('openstack_releases.puppetutils.looks_like_a_module')
+    @mock.patch('openstack_releases.npmutils.looks_like_a_module')
+    def test_python_server(self, nllam, pllam):
+        pllam.return_value = False
+        nllam.return_value = False
+        deliverable_info = {
+            'artifact-link-mode': 'none',
+            'releases': [
+                {'version': '99.1.0',
+                 'projects': [
+                     {'repo': 'openstack/puppet-watcher',
+                      'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'},
+                 ]}
+            ],
+        }
+        release_type, explicit = validate.get_release_type(
+            deliverable_info,
+            deliverable_info['releases'][0]['projects'][0],
+            self.tmpdir,
+        )
+        self.assertEqual(('python-server', False), (release_type, explicit))
+
+
 class TestPuppetUtils(base.BaseTestCase):
 
     def setUp(self):

openstack_releases/tests/test_versionutils.py

@@ -19,15 +19,15 @@ from openstack_releases import versionutils
 
 class TestValidateVersion(base.BaseTestCase):
 
-    def test_valid_std(self):
+    def test_valid_python_server(self):
         errors = list(versionutils.validate_version('1.2.3'))
         self.assertEqual(0, len(errors))
 
-    def test_invalid_std(self):
+    def test_invalid_python_server(self):
         errors = list(versionutils.validate_version('1.2.3.4'))
         self.assertEqual(1, len(errors))
 
-    def test_empty_std(self):
+    def test_empty_python_server(self):
         errors = list(versionutils.validate_version(''))
         self.assertEqual(1, len(errors))
 

openstack_releases/versionutils.py

@@ -26,22 +26,23 @@ import pbr.version
 #  3. canonicalise: The function used to canonicalise the *Version object.
 #                   Used to verify that the version string is already in the
 #                   canonical form
-_VALIDATORS = {'std': (pbr.version.SemanticVersion.from_pip_string,
+_VALIDATORS = {'python-server': (pbr.version.SemanticVersion.from_pip_string,
-                       ValueError,
+                                 ValueError,
-                       lambda x: x.release_string()),
+                                 lambda x: x.release_string()),
                'xstatic': (packaging.version.Version,
                            packaging.version.InvalidVersion,
                            lambda x: str(x)),
                }
-_VALIDATORS['fuel'] = _VALIDATORS['std']
+_VALIDATORS['fuel'] = _VALIDATORS['python-server']
-_VALIDATORS['openstack-manuals'] = _VALIDATORS['std']
+_VALIDATORS['openstack-manuals'] = _VALIDATORS['python-server']
-_VALIDATORS['puppet'] = _VALIDATORS['std']
+_VALIDATORS['puppet'] = _VALIDATORS['python-server']
-_VALIDATORS['nodejs'] = _VALIDATORS['std']
+_VALIDATORS['nodejs'] = _VALIDATORS['python-server']
-_VALIDATORS['neutron'] = _VALIDATORS['std']
+_VALIDATORS['neutron'] = _VALIDATORS['python-server']
-_VALIDATORS['horizon'] = _VALIDATORS['std']
+_VALIDATORS['horizon'] = _VALIDATORS['python-server']
+_VALIDATORS['python-pypi'] = _VALIDATORS['python-server']
 
 
-def validate_version(versionstr, release_type='std', pre_ok=True):
+def validate_version(versionstr, release_type='python-server', pre_ok=True):
     """Given a version string, yield error messages if it is "bad"
 
     Apply our SemVer rules to version strings and report all issues.
@@ -54,8 +55,8 @@ def validate_version(versionstr, release_type='std', pre_ok=True):
                       'model does not allow for it' % versionstr)
 
     if release_type not in _VALIDATORS:
-        yield 'Release Type %r not valid using \'std\' instead' % release_type
+        yield 'Release Type %r not valid using \'python-server\' instead' % release_type
-        release_type = 'std'
+        release_type = 'python-server'
 
     constructor, exception, canonicalise = _VALIDATORS[release_type]
     try:
@@ -70,7 +71,7 @@ def validate_version(versionstr, release_type='std', pre_ok=True):
                 (versionstr, canonical)
 
 
-def canonical_version(versionstr, release_type='std'):
+def canonical_version(versionstr, release_type='python-server'):
     """Given a version string verify it is in the canonical form."""
     errors = list(validate_version(versionstr, release_type))
     if errors: