From a1c1d3f2f9841e3384108e201cd2a907c0c2f9c6 Mon Sep 17 00:00:00 2001 From: Igor Kalnitsky Date: Mon, 24 Nov 2014 15:15:09 +0200 Subject: [PATCH] Always send versioned paths to manifests/repos Since Fuel 6.0 we always have versioned folders for puppet manifests and repos, so there no need to send unversioned paths by default. The patch implements it by adding records to release_orchestrator_data for initial releases. Change-Id: Icb7c4449696e4032e6d8e1e939ae621614e2bc18 Closes-Bug: #1395047 --- nailgun/nailgun/api/v1/validators/release.py | 6 ++ nailgun/nailgun/db/sqlalchemy/fixman.py | 14 ++- nailgun/nailgun/fixtures/openstack.yaml | 10 ++ nailgun/nailgun/objects/release.py | 55 ++++------- nailgun/nailgun/settings.yaml | 7 -- nailgun/nailgun/test/base.py | 16 ++++ .../test_orchestrator_serializer.py | 6 +- .../test/unit/test_attributes_plugin.py | 6 +- nailgun/nailgun/test/unit/test_objects.py | 6 +- .../unit/test_release_collection_handlers.py | 20 +++- .../test/unit/test_release_validator.py | 96 +++++++++++++++++++ 11 files changed, 184 insertions(+), 58 deletions(-) create mode 100644 nailgun/nailgun/test/unit/test_release_validator.py diff --git a/nailgun/nailgun/api/v1/validators/release.py b/nailgun/nailgun/api/v1/validators/release.py index 32ba6652f0..fa4cbe41be 100644 --- a/nailgun/nailgun/api/v1/validators/release.py +++ b/nailgun/nailgun/api/v1/validators/release.py @@ -35,6 +35,7 @@ class ReleaseValidator(BasicValidator): "Invalid network data: {0}".format(network), log_message=True ) + if "orchestrator_data" in d: if not isinstance(d["orchestrator_data"], dict): raise errors.InvalidData( @@ -68,6 +69,11 @@ class ReleaseValidator(BasicValidator): "No release operating system specified", log_message=True ) + if "orchestrator_data" not in d: + raise errors.InvalidData( + 'No orchestrator_data specified', log_message=True + ) + if db().query(Release).filter_by( name=d["name"], version=d["version"] diff --git a/nailgun/nailgun/db/sqlalchemy/fixman.py b/nailgun/nailgun/db/sqlalchemy/fixman.py index 1c2db776e4..d6eabc9361 100644 --- a/nailgun/nailgun/db/sqlalchemy/fixman.py +++ b/nailgun/nailgun/db/sqlalchemy/fixman.py @@ -106,8 +106,20 @@ def upload_fixture(fileobj, loader=None): except Exception: break - new_obj = obj['model']() + # NOTE(ikalnitsly): + # In order to add a release to Nailgun we have to fill two tables: + # releases and release_orchestrator_data. By using former fixture + # approach we can't do it, since the fixture is bond to only one + # database model and can't deal with additional logic. Therefore + # we need to use Nailgun's objects which know how to handle it. + # + # TODO(ikalnitsky): + # Rewrite fixture logic - it must be simple and obvious. + if obj['model'] is objects.Release.model: + objects.Release.create(obj['fields']) + continue + new_obj = obj['model']() fk_fields = {} for field, value in obj["fields"].iteritems(): f = getattr(obj['model'], field) diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml index d97661fd35..9cfda5f390 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml @@ -1218,6 +1218,11 @@ uri: "http://{{settings.MASTER_IP}}:8080/targetimages/centos_65_x86_64-boot.img.gz" format: "ext2" container: "gzip" + orchestrator_data: + puppet_manifests_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/" + puppet_modules_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/" + repo_metadata: + "{OPENSTACK_VERSION}": "http://{MASTER_IP}:8080/{OPENSTACK_VERSION}/centos/x86_64" - pk: 2 extend: *base_release fields: @@ -1241,3 +1246,8 @@ uri: "http://{{settings.MASTER_IP}}:8080/targetimages/ubuntu_1204_amd64-boot.img.gz" format: "ext2" container: "gzip" + orchestrator_data: + puppet_manifests_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/manifests/" + puppet_modules_source: "rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/" + repo_metadata: + "{OPENSTACK_VERSION}": "http://{MASTER_IP}:8080/{OPENSTACK_VERSION}/ubuntu/x86_64 precise main" diff --git a/nailgun/nailgun/objects/release.py b/nailgun/nailgun/objects/release.py index 8d29ffb9be..49357279fd 100644 --- a/nailgun/nailgun/objects/release.py +++ b/nailgun/nailgun/objects/release.py @@ -24,7 +24,7 @@ from sqlalchemy import not_ from nailgun import consts -from nailgun.objects.serializers.release import ReleaseSerializer +from nailgun.objects.serializers import release as release_serializer from nailgun.db import db @@ -43,6 +43,9 @@ class ReleaseOrchestratorData(NailgunObject): #: SQLAlchemy model model = models.ReleaseOrchestratorData + #: Serializer for ReleaseOrchestratorData + serializer = release_serializer.ReleaseOrchestratorDataSerializer + #: JSON schema schema = { "$schema": "http://json-schema.org/draft-04/schema#", @@ -84,13 +87,14 @@ class ReleaseOrchestratorData(NailgunObject): release = Release.get_by_uid(rendered_data['release_id']) context = { 'MASTER_IP': settings.MASTER_IP, - 'OPENSTACK_VERSION': release.version, - } + 'OPENSTACK_VERSION': release.version} # render all the paths + repo_metadata = {} for key, value in six.iteritems(rendered_data['repo_metadata']): - rendered_data['repo_metadata'][key] = \ - cls.render_path(value, context) + formatted_key = cls.render_path(key, context) + repo_metadata[formatted_key] = cls.render_path(value, context) + rendered_data['repo_metadata'] = repo_metadata rendered_data['puppet_manifests_source'] = \ cls.render_path(rendered_data.get( @@ -115,7 +119,7 @@ class Release(NailgunObject): model = models.Release #: Serializer for Release - serializer = ReleaseSerializer + serializer = release_serializer.ReleaseSerializer #: Release JSON schema schema = { @@ -220,41 +224,16 @@ class Release(NailgunObject): @classmethod def update_orchestrator_data(cls, instance, orchestrator_data): - for k in ["id", "release_id"]: - orchestrator_data.pop(k, None) - if orchestrator_data: - if instance.orchestrator_data: - ReleaseOrchestratorData.update( - instance.orchestrator_data, orchestrator_data) - else: - orchestrator_data["release_id"] = instance.id - ReleaseOrchestratorData.create(orchestrator_data) + orchestrator_data.pop("id", None) + orchestrator_data["release_id"] = instance.id + + ReleaseOrchestratorData.update( + instance.orchestrator_data, orchestrator_data) @classmethod def get_orchestrator_data_dict(cls, instance): - os = instance.operating_system.lower() - default_orchestrator_data = { - "repo_metadata": { - "nailgun": - settings.DEFAULT_REPO[os].format( - MASTER_IP=settings.MASTER_IP), - }, - "puppet_modules_source": - settings.DEFAULT_PUPPET['modules'].format( - MASTER_IP=settings.MASTER_IP), - "puppet_manifests_source": - settings.DEFAULT_PUPPET['manifests'].format( - MASTER_IP=settings.MASTER_IP), - } - - return { - "repo_metadata": - instance.orchestrator_data.repo_metadata, - "puppet_modules_source": - instance.orchestrator_data.puppet_modules_source, - "puppet_manifests_source": - instance.orchestrator_data.puppet_manifests_source - } if instance.orchestrator_data else default_orchestrator_data + data = instance.orchestrator_data + return ReleaseOrchestratorData.serializer.serialize(data) @classmethod def is_deployable(cls, instance): diff --git a/nailgun/nailgun/settings.yaml b/nailgun/nailgun/settings.yaml index 1b24b0d3dd..8877cb176e 100644 --- a/nailgun/nailgun/settings.yaml +++ b/nailgun/nailgun/settings.yaml @@ -62,13 +62,6 @@ RABBITMQ: fake: "0" hostname: "127.0.0.1" -DEFAULT_PUPPET: - modules: "rsync://{MASTER_IP}:/puppet/modules/" - manifests: "rsync://{MASTER_IP}:/puppet/manifests/" -DEFAULT_REPO: - centos: "http://{MASTER_IP}:8080/centos/x86_64" - ubuntu: "http://{MASTER_IP}:8080/ubuntu/x86_64 precise main" - PLUGINS_PATH: '/var/www/nailgun/plugins' PLUGINS_SLAVES_SCRIPTS_PATH: '/etc/fuel/plugins/{plugin_name}/' PLUGINS_REPO_URL: 'http://{master_ip}:8080/plugins/{plugin_name}/' diff --git a/nailgun/nailgun/test/base.py b/nailgun/nailgun/test/base.py index dc23c25a6b..95b3e30650 100644 --- a/nailgun/nailgun/test/base.py +++ b/nailgun/nailgun/test/base.py @@ -134,6 +134,7 @@ class Environment(object): 'attributes_metadata': self.get_default_attributes_metadata(), 'volumes_metadata': self.get_default_volumes_metadata(), 'roles_metadata': self.get_default_roles_metadata(), + 'orchestrator_data': self.get_default_orchestrator_data(), } if kwargs: release_data.update(kwargs) @@ -469,6 +470,21 @@ class Environment(object): sample_plugin.update(kwargs) return sample_plugin + def get_default_orchestrator_data(self, **kwargs): + orchestrator_data = { + 'puppet_manifests_source': + 'rsync://127.0.0.1:/puppet/2014.2-6.0/manifests/', + + 'puppet_modules_source': + 'rsync://127.0.0.1:/puppet/2014.2-6.0/modules/', + + 'repo_metadata': { + '2014.2-6.0': 'http://127.0.0.1:8080/2014.2-6.0/centos/x86_64' + } + } + orchestrator_data.update(kwargs) + return orchestrator_data + def upload_fixtures(self, fxtr_names): for fxtr_path in self.fxtr_paths_by_names(fxtr_names): with open(fxtr_path, "r") as fxtr_file: diff --git a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py index cd6daa20fd..5be691e973 100644 --- a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py +++ b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py @@ -1529,16 +1529,16 @@ class TestRepoAndPuppetDataSerialization(OrchestratorSerializerTestBase): self.assertEqual( fact['repo_metadata'], { - 'nailgun': 'http://127.0.0.1:8080/centos/x86_64' + '2014.2-6.0': 'http://127.0.0.1:8080/2014.2-6.0/centos/x86_64' } ) self.assertEqual( fact['puppet_modules_source'], - 'rsync://127.0.0.1:/puppet/modules/' + 'rsync://127.0.0.1:/puppet/2014.2-6.0/modules/' ) self.assertEqual( fact['puppet_manifests_source'], - 'rsync://127.0.0.1:/puppet/manifests/' + 'rsync://127.0.0.1:/puppet/2014.2-6.0/manifests/' ) def test_orch_data_w_replaced_deployment_info(self): diff --git a/nailgun/nailgun/test/unit/test_attributes_plugin.py b/nailgun/nailgun/test/unit/test_attributes_plugin.py index 5f76afe252..2417cda138 100644 --- a/nailgun/nailgun/test/unit/test_attributes_plugin.py +++ b/nailgun/nailgun/test/unit/test_attributes_plugin.py @@ -33,8 +33,10 @@ class TestPlugin(base.BaseTestCase): self.plugin = Plugin.create(self.plugin_metadata) self.env.create( cluster_kwargs={'mode': 'multinode'}, - release_kwargs={'version': '2014.2-6.0', - 'operating_system': 'Ubuntu'}) + release_kwargs={ + 'version': '2014.2-6.0', + 'operating_system': 'Ubuntu', + 'orchestrator_data': self.env.get_default_orchestrator_data()}) self.cluster = self.env.clusters[0] self.attr_plugin = attr_plugin.ClusterAttributesPlugin(self.plugin) self.env_config = self.env.get_default_plugin_env_config() diff --git a/nailgun/nailgun/test/unit/test_objects.py b/nailgun/nailgun/test/unit/test_objects.py index 8ef264e733..a1dea2c56e 100644 --- a/nailgun/nailgun/test/unit/test_objects.py +++ b/nailgun/nailgun/test/unit/test_objects.py @@ -687,7 +687,8 @@ class TestReleaseOrchestratorData(BaseIntegrationTest): self.data = { 'release_id': self.release.id, 'repo_metadata': { - '5': 'http://10.20.0.2:8080/{OPENSTACK_VERSION}/centos/x86_64', + '{OPENSTACK_VERSION}': + 'http://10.20.0.2:8080/{OPENSTACK_VERSION}/centos/x86_64', }, 'puppet_manifests_source': 'rsync://10.20.0.2:/puppet/modules/', 'puppet_modules_source': 'rsync://10.20.0.2:/puppet/manifests/', @@ -695,7 +696,8 @@ class TestReleaseOrchestratorData(BaseIntegrationTest): instance = objects.ReleaseOrchestratorData.create(self.data) self.assertEqual(instance.repo_metadata, { - '5': 'http://10.20.0.2:8080/{0}/centos/x86_64'.format( + self.release.version: + 'http://10.20.0.2:8080/{0}/centos/x86_64'.format( self.release.version)}) self.assertEqual( instance.puppet_manifests_source, diff --git a/nailgun/nailgun/test/unit/test_release_collection_handlers.py b/nailgun/nailgun/test/unit/test_release_collection_handlers.py index cdcc7145c9..5316e7a46b 100644 --- a/nailgun/nailgun/test/unit/test_release_collection_handlers.py +++ b/nailgun/nailgun/test/unit/test_release_collection_handlers.py @@ -36,7 +36,9 @@ class TestHandlers(BaseIntegrationTest): params=jsonutils.dumps({ 'name': 'Another test release', 'version': '1.0', - 'operating_system': 'CentOS' + 'operating_system': 'CentOS', + 'orchestrator_data': + self.env.get_default_orchestrator_data(), }), headers=self.default_headers ) @@ -80,7 +82,9 @@ class TestHandlers(BaseIntegrationTest): } ] } - } + }, + 'orchestrator_data': + self.env.get_default_orchestrator_data(), }), headers=self.default_headers ) @@ -109,7 +113,9 @@ class TestHandlers(BaseIntegrationTest): } ] } - } + }, + 'orchestrator_data': + self.env.get_default_orchestrator_data() }), headers=self.default_headers, expect_errors=True @@ -161,7 +167,9 @@ class TestHandlers(BaseIntegrationTest): } ] } - } + }, + 'orchestrator_data': + self.env.get_default_orchestrator_data() }), headers=self.default_headers ) @@ -190,7 +198,9 @@ class TestHandlers(BaseIntegrationTest): } ] } - } + }, + 'orchestrator_data': + self.env.get_default_orchestrator_data(), }), headers=self.default_headers, expect_errors=True diff --git a/nailgun/nailgun/test/unit/test_release_validator.py b/nailgun/nailgun/test/unit/test_release_validator.py new file mode 100644 index 0000000000..e1ca9b8ff7 --- /dev/null +++ b/nailgun/nailgun/test/unit/test_release_validator.py @@ -0,0 +1,96 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from nailgun.openstack.common import jsonutils + +from nailgun.api.v1.validators.release import ReleaseValidator +from nailgun.errors import errors +from nailgun.test.base import BaseTestCase + + +class TestReleaseValidator(BaseTestCase): + + def setUp(self): + super(TestReleaseValidator, self).setUp() + + self.release = { + 'name': 'Test Release', + 'version': '2014.2-6.0', + 'operating_system': 'CentOS', + 'orchestrator_data': { + 'puppet_manifests_source': 'path/to/manifests', + 'puppet_modules_source': 'path/to/modules', + + 'repo_metadata': { + 'repo': 'path/to/repo', }}} + self.validator = ReleaseValidator + + def get_release(self, release): + return jsonutils.dumps(release) + + def test_name_is_mandatory(self): + self.release.pop('name') + + self.assertRaisesRegexp( + errors.InvalidData, + 'No release name specified', + self.validator.validate, + self.get_release(self.release)) + + def test_version_is_mandatory(self): + self.release.pop('version') + + self.assertRaisesRegexp( + errors.InvalidData, + 'No release version specified', + self.validator.validate, + self.get_release(self.release)) + + def test_operating_system_is_mandatory(self): + self.release.pop('operating_system') + + self.assertRaisesRegexp( + errors.InvalidData, + 'No release operating system specified', + self.validator.validate, + self.get_release(self.release)) + + def test_orchestrator_data_is_mandatory(self): + self.release.pop('orchestrator_data') + + self.assertRaisesRegexp( + errors.InvalidData, + 'No orchestrator_data specified', + self.validator.validate, + self.get_release(self.release)) + + def test_orchestrator_data_must_be_a_dict(self): + self.release['orchestrator_data'] = None + + self.assertRaisesRegexp( + errors.InvalidData, + "'orchestrator_data' field must be a dict", + self.validator.validate, + self.get_release(self.release)) + + def test_orchestrator_data_required_keys(self): + self.release['orchestrator_data'] = {} + + self.assertRaisesRegexp( + errors.InvalidData, + "'orchestrator_data' doesn't have all required keys", + self.validator.validate, + self.get_release(self.release)) + + def test_default_are_good(self): + self.validator.validate(self.get_release(self.release))