diff --git a/README.rst b/README.rst index 93c9dc5939..8a2b677921 100644 --- a/README.rst +++ b/README.rst @@ -397,10 +397,14 @@ The top level of a deliverable file is a mapping with keys: ``release-type`` This (optional) key sets the level of validation for the versions numbers. - ``std`` + ``python-server`` Default: Enforces 3 digit semver version numbers in releases and allows for common alpha, beta and dev releases. This should be appropriate for - most OpenStack release requirements. + most OpenStack component release requirements. + + ``python-pypi`` + Like ``python-server`` but requires the jobs to publish the component + to the Python Package Index (PyPI). ``xstatic`` Allows a more flexible versioning in line with xstatic package guidelines diff --git a/deliverables/_independent/ldappool.yaml b/deliverables/_independent/ldappool.yaml index a467a45699..3d9d41ea1e 100644 --- a/deliverables/_independent/ldappool.yaml +++ b/deliverables/_independent/ldappool.yaml @@ -1,7 +1,6 @@ --- launchpad: ldappool team: keystone -release-type: std type: library releases: - version: 2.0.0 diff --git a/deliverables/_independent/os-api-ref.yaml b/deliverables/_independent/os-api-ref.yaml index 73e08a6a11..f621bf815e 100644 --- a/deliverables/_independent/os-api-ref.yaml +++ b/deliverables/_independent/os-api-ref.yaml @@ -2,7 +2,6 @@ launchpad: openstack-doc-tools team: Documentation type: library -release-type: std releases: - version: 0.1.0 projects: diff --git a/deliverables/_independent/os-testr.yaml b/deliverables/_independent/os-testr.yaml index 080cf6532c..55803d20d7 100644 --- a/deliverables/_independent/os-testr.yaml +++ b/deliverables/_independent/os-testr.yaml @@ -1,7 +1,6 @@ --- launchpad: os-testr team: Quality Assurance -release-type: std type: library include-pypi-link: yes releases: diff --git a/deliverables/_independent/os-traits.yaml b/deliverables/_independent/os-traits.yaml index 79f9dbffef..687938ce03 100644 --- a/deliverables/_independent/os-traits.yaml +++ b/deliverables/_independent/os-traits.yaml @@ -1,7 +1,6 @@ --- launchpad: nova team: nova -release-type: std type: library releases: - version: 0.1.0 diff --git a/deliverables/_independent/sqlalchemy-migrate.yaml b/deliverables/_independent/sqlalchemy-migrate.yaml index 102862cfe4..66ec132a06 100644 --- a/deliverables/_independent/sqlalchemy-migrate.yaml +++ b/deliverables/_independent/sqlalchemy-migrate.yaml @@ -1,7 +1,6 @@ --- launchpad: sqlalchemy-migrate team: sqlalchemy-migrate -release-type: std type: library releases: - version: 0.6.1 diff --git a/deliverables/_independent/whereto.yaml b/deliverables/_independent/whereto.yaml index 686cebadb2..acb6aee82a 100644 --- a/deliverables/_independent/whereto.yaml +++ b/deliverables/_independent/whereto.yaml @@ -1,7 +1,7 @@ --- launchpad: whereto team: whereto -release-type: std +release-type: python-pypi type: other releases: - version: 0.1.0 diff --git a/deliverables/queens/release-test.yaml b/deliverables/queens/release-test.yaml index 251ba7fe64..ab6fda266c 100644 --- a/deliverables/queens/release-test.yaml +++ b/deliverables/queens/release-test.yaml @@ -2,6 +2,7 @@ launchpad: nova team: 'Release Management' type: other +release-type: python-pypi branches: - name: stable/queens location: 0.10.2 diff --git a/deliverables/queens/tripleo-common.yaml b/deliverables/queens/tripleo-common.yaml index b7beb51ae7..41077b5c79 100644 --- a/deliverables/queens/tripleo-common.yaml +++ b/deliverables/queens/tripleo-common.yaml @@ -1,6 +1,7 @@ --- launchpad: tripleo-common release-model: cycle-trailing +release-type: python-pypi team: tripleo type: other releases: diff --git a/openstack_releases/cmds/validate.py b/openstack_releases/cmds/validate.py index 80a32e55cf..d583113672 100644 --- a/openstack_releases/cmds/validate.py +++ b/openstack_releases/cmds/validate.py @@ -348,6 +348,66 @@ def validate_gitreview(deliverable_info, workdir, mk_warning, mk_error): mk_error('%s has no .gitreview file' % (project['repo'],)) +def get_release_type(deliverable_info, project, workdir): + """Return tuple with release type and boolean indicating whether it + was explicitly set. + + """ + if 'release-type' in deliverable_info: + return (deliverable_info['release-type'], True) + + if deliverable_info.get('type') == 'library': + return ('python-pypi', False) + + if puppetutils.looks_like_a_module(workdir, project['repo']): + return ('puppet', False) + + if npmutils.looks_like_a_module(workdir, project['repo']): + return ('nodejs', False) + + return ('python-server', False) + + +def validate_release_type(deliverable_info, + zuul_projects, + series_name, + workdir, + mk_warning, + mk_error): + """Apply validation rules for the deliverable based on 'release-type'. + """ + + link_mode = deliverable_info.get('artifact-link-mode', 'tarball') + if link_mode == 'none': + print('link-mode is "none", skipping release-type checks') + return + + if not deliverable_info.get('releases'): + print('no releases listed, skipping release-type checks') + return + + for release in deliverable_info.get('releases', []): + for project in release['projects']: + + print('checking release-type for {}'.format(project['repo'])) + + release_type, was_explicit = get_release_type( + deliverable_info, project, workdir, + ) + if was_explicit: + print('found explicit release-type {!r}'.format( + release_type)) + else: + print('release-type not given, ' + 'guessing {!r}'.format(release_type)) + + project_config.require_release_jobs_for_repo( + deliverable_info, zuul_projects, + project['repo'], + release_type, mk_warning, mk_error, + ) + + def validate_releases(deliverable_info, zuul_projects, series_name, workdir, @@ -481,44 +541,21 @@ def validate_releases(deliverable_info, zuul_projects, (prev_version, ', '.join(sorted(prev_projects)))) else: - # We change this default to be more - # language-specific before testing the release - # jobs. - default_release_type = 'std' - - # If we have an explicit release type, we can - # bypass some of the checks for languages - # other than python. - explicit_release_type = deliverable_info.get( - 'release-type', + release_type, was_explicit = get_release_type( + deliverable_info, project, workdir, ) - if explicit_release_type: + if was_explicit: print('found explicit release-type {!r}'.format( - explicit_release_type)) + release_type)) else: print('release-type not given, ' - 'determining automatically') - - is_puppet = ( - explicit_release_type == 'puppet' or - ((not explicit_release_type) and - puppetutils.looks_like_a_module(workdir, - project['repo'])) - ) - - is_nodejs = ( - explicit_release_type == 'nodejs' or - ((not explicit_release_type) and - npmutils.looks_like_a_module(workdir, - project['repo'])) - ) + 'guessing {!r}'.format(release_type)) # If this is a puppet module, ensure # that the tag and metadata file # match. - if is_puppet: + if release_type == 'puppet': print('applying puppet version rules') - default_release_type = 'puppet' puppet_ver = puppetutils.get_version( workdir, project['repo']) if puppet_ver != release['version']: @@ -534,9 +571,8 @@ def validate_releases(deliverable_info, zuul_projects, # If this is a npm module, ensure # that the tag and metadata file # match. - if is_nodejs: + if release_type == 'nodejs': print('applying nodejs version rules') - default_release_type = 'nodejs' npm_ver = npmutils.get_version( workdir, project['repo']) if npm_ver != release['version']: @@ -549,18 +585,6 @@ def validate_releases(deliverable_info, zuul_projects, ) ) - # Check for release jobs (if we ship a tarball) - release_type = deliverable_info.get( - 'release-type', - default_release_type, - ) - if link_mode != 'none': - project_config.require_release_jobs_for_repo( - deliverable_info, zuul_projects, - project['repo'], - release_type, mk_warning, mk_error, - ) - for e in versionutils.validate_version( release['version'], release_type=release_type, @@ -971,6 +995,14 @@ def main(): validate_release_notes(deliverable_info, mk_warning, mk_error) validate_type(deliverable_info, mk_warning, mk_error) validate_model(deliverable_info, series_name, mk_warning, mk_error) + validate_release_type( + deliverable_info, + zuul_projects, + series_name, + workdir, + mk_warning, + mk_error, + ) validate_gitreview(deliverable_info, workdir, mk_warning, mk_error) validate_releases( deliverable_info, diff --git a/openstack_releases/project_config.py b/openstack_releases/project_config.py index ccbe5f9daf..d4a7e2c194 100644 --- a/openstack_releases/project_config.py +++ b/openstack_releases/project_config.py @@ -81,10 +81,12 @@ def get_zuul_project_data(url=ZUUL_PROJECTS_URL): # Which jobs are needed for which release types. _RELEASE_JOBS_FOR_TYPE = { - 'std': [ - 'publish-to-pypi', + 'python-server': [ 'release-openstack-server', ], + 'python-pypi': [ + 'publish-to-pypi', + ], 'neutron': [ 'publish-to-pypi-neutron', ], @@ -140,7 +142,7 @@ def require_release_jobs_for_repo(deliverable_info, zuul_projects, repo, # jobs, because we want projects to use the templates. expected_jobs = _RELEASE_JOBS_FOR_TYPE.get( release_type, - _RELEASE_JOBS_FOR_TYPE['std'], + _RELEASE_JOBS_FOR_TYPE['python-server'], ) if expected_jobs: found_jobs = [ diff --git a/openstack_releases/schema.yaml b/openstack_releases/schema.yaml index 28c269eaca..22edaaa0e2 100644 --- a/openstack_releases/schema.yaml +++ b/openstack_releases/schema.yaml @@ -35,7 +35,8 @@ properties: type: "object" release-type: type: "string" - enum: [ "std", "xstatic", "fuel", "nodejs", "puppet", "neutron", "horizon" ] + enum: [ "python-server", "python-pypi", "xstatic", "fuel", + "nodejs", "puppet", "neutron", "horizon" ] stable-branch-type: type: "string" enum: [ "std", "tagless", "upstream" ] diff --git a/openstack_releases/tests/test_project_config.py b/openstack_releases/tests/test_project_config.py index e768da5830..e9b8aa94c1 100644 --- a/openstack_releases/tests/test_project_config.py +++ b/openstack_releases/tests/test_project_config.py @@ -103,7 +103,7 @@ class TestReleaseJobsStandard(base.BaseTestCase): deliverable_info, zuul_projects, 'openstack/releases', - 'std', + 'python-pypi', warnings.append, errors.append, ) @@ -128,7 +128,7 @@ class TestReleaseJobsStandard(base.BaseTestCase): deliverable_info, zuul_projects, 'openstack/releases', - 'std', + 'python-pypi', warnings.append, errors.append, ) diff --git a/openstack_releases/tests/test_validate.py b/openstack_releases/tests/test_validate.py index ed98ec1af2..c12cd73825 100644 --- a/openstack_releases/tests/test_validate.py +++ b/openstack_releases/tests/test_validate.py @@ -417,33 +417,6 @@ class TestValidateReleases(base.BaseTestCase): self.tmpdir = self.useFixture(fixtures.TempDir()).path gitutils.clone_repo(self.tmpdir, 'openstack/release-test') - @mock.patch('openstack_releases.project_config.require_release_jobs_for_repo') - def test_check_release_jobs(self, check_jobs): - deliverable_info = { - 'releases': [ - {'version': '99.5.0', - 'projects': [ - {'repo': 'openstack/release-test', - 'hash': '218c9c82f168f1db681b27842b5a829428c6b5e1', - 'tarball-base': 'openstack-release-test'}, - ]} - ], - } - warnings = [] - errors = [] - validate.validate_releases( - deliverable_info, - {'validate-projects-by-name': {}}, - 'queens', - self.tmpdir, - warnings.append, - errors.append, - ) - print(warnings, errors) - self.assertEqual(0, len(warnings)) - self.assertEqual(0, len(errors)) - check_jobs.assert_called_once() - def test_invalid_hash(self): deliverable_info = { 'artifact-link-mode': 'none', @@ -806,6 +779,113 @@ class TestValidateReleases(base.BaseTestCase): self.assertEqual(1, len(errors)) +class TestGetReleaseType(base.BaseTestCase): + + def setUp(self): + super(TestGetReleaseType, self).setUp() + self.tmpdir = self.useFixture(fixtures.TempDir()).path + + def test_explicit(self): + deliverable_info = { + 'artifact-link-mode': 'none', + 'release-type': 'explicitly-set', + 'releases': [ + {'version': '99.1.0', + 'projects': [ + {'repo': 'openstack/puppet-watcher', + 'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'}, + ]} + ], + } + release_type, explicit = validate.get_release_type( + deliverable_info, + deliverable_info['releases'][0]['projects'][0], + self.tmpdir, + ) + self.assertEqual(('explicitly-set', True), (release_type, explicit)) + + def test_library(self): + deliverable_info = { + 'artifact-link-mode': 'none', + 'type': 'library', + 'releases': [ + {'version': '99.1.0', + 'projects': [ + {'repo': 'openstack/puppet-watcher', + 'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'}, + ]} + ], + } + release_type, explicit = validate.get_release_type( + deliverable_info, + deliverable_info['releases'][0]['projects'][0], + self.tmpdir, + ) + self.assertEqual(('python-pypi', False), (release_type, explicit)) + + @mock.patch('openstack_releases.puppetutils.looks_like_a_module') + def test_puppet(self, llam): + llam.return_value = True + deliverable_info = { + 'artifact-link-mode': 'none', + 'releases': [ + {'version': '99.1.0', + 'projects': [ + {'repo': 'openstack/puppet-watcher', + 'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'}, + ]} + ], + } + release_type, explicit = validate.get_release_type( + deliverable_info, + deliverable_info['releases'][0]['projects'][0], + self.tmpdir, + ) + self.assertEqual(('puppet', False), (release_type, explicit)) + + @mock.patch('openstack_releases.npmutils.looks_like_a_module') + def test_nodejs(self, llam): + llam.return_value = True + deliverable_info = { + 'artifact-link-mode': 'none', + 'releases': [ + {'version': '99.1.0', + 'projects': [ + {'repo': 'openstack/puppet-watcher', + 'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'}, + ]} + ], + } + release_type, explicit = validate.get_release_type( + deliverable_info, + deliverable_info['releases'][0]['projects'][0], + self.tmpdir, + ) + self.assertEqual(('nodejs', False), (release_type, explicit)) + + @mock.patch('openstack_releases.puppetutils.looks_like_a_module') + @mock.patch('openstack_releases.npmutils.looks_like_a_module') + def test_python_server(self, nllam, pllam): + pllam.return_value = False + nllam.return_value = False + deliverable_info = { + 'artifact-link-mode': 'none', + 'releases': [ + {'version': '99.1.0', + 'projects': [ + {'repo': 'openstack/puppet-watcher', + 'hash': '1e7baef27139f69a83e1fe28686bb72ee7e1d6fa'}, + ]} + ], + } + release_type, explicit = validate.get_release_type( + deliverable_info, + deliverable_info['releases'][0]['projects'][0], + self.tmpdir, + ) + self.assertEqual(('python-server', False), (release_type, explicit)) + + class TestPuppetUtils(base.BaseTestCase): def setUp(self): diff --git a/openstack_releases/tests/test_versionutils.py b/openstack_releases/tests/test_versionutils.py index 73bea3e4b5..2082dadccf 100644 --- a/openstack_releases/tests/test_versionutils.py +++ b/openstack_releases/tests/test_versionutils.py @@ -19,15 +19,15 @@ from openstack_releases import versionutils class TestValidateVersion(base.BaseTestCase): - def test_valid_std(self): + def test_valid_python_server(self): errors = list(versionutils.validate_version('1.2.3')) self.assertEqual(0, len(errors)) - def test_invalid_std(self): + def test_invalid_python_server(self): errors = list(versionutils.validate_version('1.2.3.4')) self.assertEqual(1, len(errors)) - def test_empty_std(self): + def test_empty_python_server(self): errors = list(versionutils.validate_version('')) self.assertEqual(1, len(errors)) diff --git a/openstack_releases/versionutils.py b/openstack_releases/versionutils.py index ce859a83ef..6eea3d5ea8 100644 --- a/openstack_releases/versionutils.py +++ b/openstack_releases/versionutils.py @@ -26,22 +26,23 @@ import pbr.version # 3. canonicalise: The function used to canonicalise the *Version object. # Used to verify that the version string is already in the # canonical form -_VALIDATORS = {'std': (pbr.version.SemanticVersion.from_pip_string, - ValueError, - lambda x: x.release_string()), +_VALIDATORS = {'python-server': (pbr.version.SemanticVersion.from_pip_string, + ValueError, + lambda x: x.release_string()), 'xstatic': (packaging.version.Version, packaging.version.InvalidVersion, lambda x: str(x)), } -_VALIDATORS['fuel'] = _VALIDATORS['std'] -_VALIDATORS['openstack-manuals'] = _VALIDATORS['std'] -_VALIDATORS['puppet'] = _VALIDATORS['std'] -_VALIDATORS['nodejs'] = _VALIDATORS['std'] -_VALIDATORS['neutron'] = _VALIDATORS['std'] -_VALIDATORS['horizon'] = _VALIDATORS['std'] +_VALIDATORS['fuel'] = _VALIDATORS['python-server'] +_VALIDATORS['openstack-manuals'] = _VALIDATORS['python-server'] +_VALIDATORS['puppet'] = _VALIDATORS['python-server'] +_VALIDATORS['nodejs'] = _VALIDATORS['python-server'] +_VALIDATORS['neutron'] = _VALIDATORS['python-server'] +_VALIDATORS['horizon'] = _VALIDATORS['python-server'] +_VALIDATORS['python-pypi'] = _VALIDATORS['python-server'] -def validate_version(versionstr, release_type='std', pre_ok=True): +def validate_version(versionstr, release_type='python-server', pre_ok=True): """Given a version string, yield error messages if it is "bad" Apply our SemVer rules to version strings and report all issues. @@ -54,8 +55,8 @@ def validate_version(versionstr, release_type='std', pre_ok=True): 'model does not allow for it' % versionstr) if release_type not in _VALIDATORS: - yield 'Release Type %r not valid using \'std\' instead' % release_type - release_type = 'std' + yield 'Release Type %r not valid using \'python-server\' instead' % release_type + release_type = 'python-server' constructor, exception, canonicalise = _VALIDATORS[release_type] try: @@ -70,7 +71,7 @@ def validate_version(versionstr, release_type='std', pre_ok=True): (versionstr, canonical) -def canonical_version(versionstr, release_type='std'): +def canonical_version(versionstr, release_type='python-server'): """Given a version string verify it is in the canonical form.""" errors = list(validate_version(versionstr, release_type)) if errors: