Allow flavor_id on cluster create

Add flavor_id as an option during cluster create. If not given,
the default is taken from the cluster template.

Add flavor_id in the Cluster object and use that instead
of the one from ClusterTemplate.

Update both magnum and magnum cli documentation to reflect the above changes.

Partial-Bug: #1699245
Change-Id: Ib60c05cce1cf2639ca4740abdd264403033433f9

20 files changed, 101 insertions, 27 deletions

diff --git a/api-ref/source/clusters.inc b/api-ref/source/clusters.inc
index 54fc6dd..b38364d 100644
--- a/api-ref/source/clusters.inc
+++ b/api-ref/source/clusters.inc
@@ -41,6 +41,7 @@ Request
   - keypair: keypair_id
   - master_flavor_id: master_flavor_id
   - labels: labels
+  - flavor_id: flavor_id
 
 .. note::
 
diff --git a/api-ref/source/samples/cluster-create-req.json b/api-ref/source/samples/cluster-create-req.json
index e4c77f3..c5f8d49 100644
--- a/api-ref/source/samples/cluster-create-req.json
+++ b/api-ref/source/samples/cluster-create-req.json
@@ -8,5 +8,6 @@
    "keypair":"my_keypair",
    "master_flavor_id":null,
    "labels":{
-   }
+   },
+   "flavor_id":null
 }
diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst
index f6c97d9..68e1b2f 100644
--- a/doc/source/user/index.rst
+++ b/doc/source/user/index.rst
@@ -200,7 +200,7 @@ They are loosely grouped as: mandatory, infrastructure, COE specific.
 
 --flavor \<flavor\>
   The nova flavor id for booting the node servers.  The default
-  is 'm1.small'.
+  is 'm1.small'. This value can be overridden at cluster creation.
 
 --master-flavor \<master-flavor\>
   The nova flavor id for booting the master or manager servers.  The
diff --git a/magnum/api/attr_validator.py b/magnum/api/attr_validator.py
index ade2a19..a056282 100644
--- a/magnum/api/attr_validator.py
+++ b/magnum/api/attr_validator.py
@@ -188,7 +188,12 @@ def validate_os_resources(context, cluster_template, cluster=None):
     cli = clients.OpenStackClients(context)
 
     for attr, validate_method in validators.items():
-        if attr in cluster_template and cluster_template[attr] is not None:
+        if cluster and attr in cluster and cluster[attr]:
+            if attr != 'labels':
+                validate_method(cli, cluster[attr])
+            else:
+                validate_method(cluster[attr])
+        elif attr in cluster_template and cluster_template[attr] is not None:
             if attr != 'labels':
                 validate_method(cli, cluster_template[attr])
             else:
diff --git a/magnum/api/controllers/v1/bay.py b/magnum/api/controllers/v1/bay.py
index 47aba45..5285943 100755
--- a/magnum/api/controllers/v1/bay.py
+++ b/magnum/api/controllers/v1/bay.py
@@ -98,6 +98,9 @@ class Bay(base.APIBase):
     master_flavor_id = wtypes.StringType(min_length=1, max_length=255)
     """The master flavor of this Bay"""
 
+    flavor_id = wtypes.StringType(min_length=1, max_length=255)
+    """The flavor of this Bay"""
+
     bay_create_timeout = wsme.wsattr(wtypes.IntegerType(minimum=0), default=60)
     """Timeout for creating the bay in minutes. Default to 60 if not set"""
 
@@ -181,7 +184,7 @@ class Bay(base.APIBase):
         if not expand:
             bay.unset_fields_except(['uuid', 'name', 'baymodel_id',
                                      'docker_volume_size', 'labels',
-                                     'master_flavor_id',
+                                     'master_flavor_id', 'flavor_id',
                                      'node_count', 'status',
                                      'bay_create_timeout', 'master_count',
                                      'stack_id'])
@@ -208,6 +211,7 @@ class Bay(base.APIBase):
                      docker_volume_size=1,
                      labels={},
                      master_flavor_id=None,
+                     flavor_id=None,
                      bay_create_timeout=15,
                      stack_id='49dc23f5-ffc9-40c3-9d34-7be7f9e34d63',
                      status=fields.ClusterStatus.CREATE_COMPLETE,
@@ -441,6 +445,10 @@ class BaysController(base.Controller):
         if bay.master_flavor_id == wtypes.Unset or not bay.master_flavor_id:
             bay.master_flavor_id = baymodel.master_flavor_id
 
+        # If flavor_id is not present, use baymodel value
+        if bay.flavor_id == wtypes.Unset or not bay.flavor_id:
+            bay.flavor_id = baymodel.flavor_id
+
         bay_dict = bay.as_dict()
         bay_dict['keypair'] = baymodel.keypair_id
         attr_validator.validate_os_resources(context, baymodel.as_dict(),
diff --git a/magnum/api/controllers/v1/cluster.py b/magnum/api/controllers/v1/cluster.py
index f178238..06a7061 100755
--- a/magnum/api/controllers/v1/cluster.py
+++ b/magnum/api/controllers/v1/cluster.py
@@ -116,6 +116,9 @@ class Cluster(base.APIBase):
     master_flavor_id = wtypes.StringType(min_length=1, max_length=255)
     """The flavor of the master node for this Cluster"""
 
+    flavor_id = wtypes.StringType(min_length=1, max_length=255)
+    """The flavor of this Cluster"""
+
     create_timeout = wsme.wsattr(wtypes.IntegerType(minimum=0), default=60)
     """Timeout for creating the cluster in minutes. Default to 60 if not set"""
 
@@ -169,7 +172,7 @@ class Cluster(base.APIBase):
             cluster.unset_fields_except(['uuid', 'name', 'cluster_template_id',
                                          'keypair', 'docker_volume_size',
                                          'labels', 'node_count', 'status',
-                                         'master_flavor_id',
+                                         'master_flavor_id', 'flavor_id',
                                          'create_timeout', 'master_count',
                                          'stack_id'])
 
@@ -197,6 +200,7 @@ class Cluster(base.APIBase):
                      docker_volume_size=1,
                      labels={},
                      master_flavor_id='m1.small',
+                     flavor_id='m1.small',
                      create_timeout=15,
                      stack_id='49dc23f5-ffc9-40c3-9d34-7be7f9e34d63',
                      status=fields.ClusterStatus.CREATE_COMPLETE,
@@ -421,6 +425,10 @@ class ClustersController(base.Controller):
                 not cluster.master_flavor_id):
             cluster.master_flavor_id = cluster_template.master_flavor_id
 
+        # If flavor_id is not present, use cluster_template value
+        if cluster.flavor_id == wtypes.Unset or not cluster.flavor_id:
+            cluster.flavor_id = cluster_template.flavor_id
+
         cluster_dict = cluster.as_dict()
 
         attr_validator.validate_os_resources(context,
diff --git a/magnum/db/sqlalchemy/alembic/versions/041d9a0f1159_add_flavor_id_to_cluster.py b/magnum/db/sqlalchemy/alembic/versions/041d9a0f1159_add_flavor_id_to_cluster.py
new file mode 100644
index 0000000..478671c
--- /dev/null
+++ b/magnum/db/sqlalchemy/alembic/versions/041d9a0f1159_add_flavor_id_to_cluster.py
@@ -0,0 +1,30 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""add flavor_id to cluster
+
+Revision ID: 041d9a0f1159
+Revises: 04c625aa95ba
+Create Date: 2017-07-31 12:46:00.777841
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '041d9a0f1159'
+down_revision = '04c625aa95ba'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('cluster', sa.Column('flavor_id',
+                                       sa.String(length=255), nullable=True))
diff --git a/magnum/db/sqlalchemy/models.py b/magnum/db/sqlalchemy/models.py
index 986b771..1cb017f 100644
--- a/magnum/db/sqlalchemy/models.py
+++ b/magnum/db/sqlalchemy/models.py
@@ -118,6 +118,7 @@ class Cluster(Base):
     docker_volume_size = Column(Integer())
     labels = Column(JSONEncodedDict)
     master_flavor_id = Column(String(255))
+    flavor_id = Column(String(255))
     stack_id = Column(String(255))
     api_address = Column(String(255))
     node_addresses = Column(JSONEncodedList)
diff --git a/magnum/drivers/heat/k8s_template_def.py b/magnum/drivers/heat/k8s_template_def.py
index 8eaafbf..00912ef 100644
--- a/magnum/drivers/heat/k8s_template_def.py
+++ b/magnum/drivers/heat/k8s_template_def.py
@@ -54,7 +54,7 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
         self.add_parameter('master_flavor',
                            cluster_attr='master_flavor_id')
         self.add_parameter('minion_flavor',
-                           cluster_template_attr='flavor_id')
+                           cluster_attr='flavor_id')
         self.add_parameter('number_of_minions',
                            cluster_attr='node_count')
         self.add_parameter('external_network',
diff --git a/magnum/drivers/heat/swarm_fedora_template_def.py b/magnum/drivers/heat/swarm_fedora_template_def.py
index b5904da..a97af38 100644
--- a/magnum/drivers/heat/swarm_fedora_template_def.py
+++ b/magnum/drivers/heat/swarm_fedora_template_def.py
@@ -50,7 +50,7 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition):
         self.add_parameter('master_flavor',
                            cluster_attr='master_flavor_id')
         self.add_parameter('node_flavor',
-                           cluster_template_attr='flavor_id')
+                           cluster_attr='flavor_id')
         self.add_parameter('docker_volume_size',
                            cluster_attr='docker_volume_size')
         self.add_parameter('volume_driver',
diff --git a/magnum/drivers/heat/swarm_mode_template_def.py b/magnum/drivers/heat/swarm_mode_template_def.py
index 23d8683..33c6320 100644
--- a/magnum/drivers/heat/swarm_mode_template_def.py
+++ b/magnum/drivers/heat/swarm_mode_template_def.py
@@ -63,7 +63,7 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
         self.add_parameter('master_flavor',
                            cluster_attr='master_flavor_id')
         self.add_parameter('node_flavor',
-                           cluster_template_attr='flavor_id')
+                           cluster_attr='flavor_id')
         self.add_parameter('docker_volume_size',
                            cluster_attr='docker_volume_size')
         self.add_parameter('volume_driver',
diff --git a/magnum/drivers/mesos_ubuntu_v1/template_def.py b/magnum/drivers/mesos_ubuntu_v1/template_def.py
index 318f259..5c29231 100644
--- a/magnum/drivers/mesos_ubuntu_v1/template_def.py
+++ b/magnum/drivers/mesos_ubuntu_v1/template_def.py
@@ -33,7 +33,7 @@ class UbuntuMesosTemplateDefinition(template_def.BaseTemplateDefinition):
         self.add_parameter('master_flavor',
                            cluster_attr='master_flavor_id')
         self.add_parameter('slave_flavor',
-                           cluster_template_attr='flavor_id')
+                           cluster_attr='flavor_id')
         self.add_parameter('cluster_name',
                            cluster_attr='name')
         self.add_parameter('volume_driver',
diff --git a/magnum/objects/cluster.py b/magnum/objects/cluster.py
index 07facc9..01f704d 100644
--- a/magnum/objects/cluster.py
+++ b/magnum/objects/cluster.py
@@ -45,8 +45,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
     # Version 1.14: Added 'docker_volume_size' field
     # Version 1.15: Added 'labels' field
     # Version 1.16: Added 'master_flavor_id' field
+    # Version 1.17: Added 'flavor_id' field
 
-    VERSION = '1.16'
+    VERSION = '1.17'
 
     dbapi = dbapi.get_instance()
 
@@ -61,6 +62,7 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
         'docker_volume_size': fields.IntegerField(nullable=True),
         'labels': fields.DictOfStringsField(nullable=True),
         'master_flavor_id': fields.StringField(nullable=True),
+        'flavor_id': fields.StringField(nullable=True),
         'stack_id': fields.StringField(nullable=True),
         'status': m_fields.ClusterStatusField(nullable=True),
         'status_reason': fields.StringField(nullable=True),
diff --git a/magnum/tests/unit/api/controllers/v1/test_cluster.py b/magnum/tests/unit/api/controllers/v1/test_cluster.py
index bec0b9e..fb015a6 100644
--- a/magnum/tests/unit/api/controllers/v1/test_cluster.py
+++ b/magnum/tests/unit/api/controllers/v1/test_cluster.py
@@ -856,6 +856,24 @@ class TestPost(api_base.FunctionalTest):
         # Verify master_flavor_id from ClusterTemplate is used
         self.assertEqual('m1.small', cluster[0].master_flavor_id)
 
+    def test_create_cluster_with_flavor_id(self):
+        bdict = apiutils.cluster_post_data()
+        bdict['flavor_id'] = 'm2.small'
+        response = self.post_json('/clusters', bdict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(202, response.status_int)
+        cluster, timeout = self.mock_cluster_create.call_args
+        self.assertEqual('m2.small', cluster[0].flavor_id)
+
+    def test_create_cluster_without_flavor_id(self):
+        bdict = apiutils.cluster_post_data()
+        response = self.post_json('/clusters', bdict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(202, response.status_int)
+        cluster, timeout = self.mock_cluster_create.call_args
+        # Verify flavor_id from ClusterTemplate is used
+        self.assertEqual('m1.small', cluster[0].flavor_id)
+
 
 class TestDelete(api_base.FunctionalTest):
     def setUp(self):
diff --git a/magnum/tests/unit/api/test_attr_validator.py b/magnum/tests/unit/api/test_attr_validator.py
index d680268..9b62a6d 100644
--- a/magnum/tests/unit/api/test_attr_validator.py
+++ b/magnum/tests/unit/api/test_attr_validator.py
@@ -315,11 +315,18 @@ class TestAttrValidator(base.BaseTestCase):
     @mock.patch('magnum.common.clients.OpenStackClients')
     def test_validate_os_resources_with_cluster(self, mock_os_cli):
         mock_cluster_template = {}
-        mock_cluster = {'keypair': 'test-keypair'}
+        mock_cluster = {
+            'keypair': 'test-keypair', 'labels': {'lab1': 'val1'},
+            'image_id': 'e33f0988-1730-405e-8401-30cbc8535302'
+        }
         mock_keypair = mock.MagicMock()
         mock_keypair.id = 'test-keypair'
+        mock_image = {'name': 'fedora-21-atomic-5',
+                      'id': 'e33f0988-1730-405e-8401-30cbc8535302',
+                      'os_distro': 'fedora-atomic'}
         mock_nova = mock.MagicMock()
         mock_nova.keypairs.get.return_value = mock_keypair
+        mock_nova.images.get.return_value = mock_image
         mock_os_cli = mock.MagicMock()
         mock_os_cli.nova.return_value = mock_nova
         mock_context = mock.MagicMock()
diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
index 6cf4947..c18786b 100644
--- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
@@ -77,6 +77,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'master_count': 1,
             'discovery_url': 'https://discovery.etcd.io/test',
             'docker_volume_size': 20,
+            'flavor_id': 'flavor_id',
             'master_addresses': ['172.17.2.18'],
             'ca_cert_ref': 'http://barbican/v1/containers/xx-xx-xx-xx',
             'magnum_cert_ref': 'http://barbican/v1/containers/xx-xx-xx-xx',
@@ -96,6 +97,7 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'kube_dashboard_enabled': 'True',
                        'docker_volume_type': 'lvmdriver-1'},
             'master_flavor_id': 'master_flavor_id',
+            'flavor_id': 'flavor_id',
         }
         self.context.user_name = 'fake_user'
         self.context.tenant = 'fake_tenant'
@@ -374,6 +376,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'discovery_url': 'https://discovery.etcd.io/test',
             'docker_volume_size': 20,
             'master_flavor': 'master_flavor_id',
+            'minion_flavor': 'flavor_id',
             'external_network': 'external_network_id',
             'flannel_backend': 'vxlan',
             'flannel_network_cidr': '10.101.0.0/16',
@@ -592,21 +595,6 @@ class TestClusterConductorWithK8s(base.TestCase):
     @patch('requests.get')
     @patch('magnum.objects.ClusterTemplate.get_by_uuid')
     @patch('magnum.drivers.common.driver.Driver.get_driver')
-    def test_extract_template_definition_without_minion_flavor(
-            self,
-            mock_driver,
-            mock_objects_cluster_template_get_by_uuid,
-            mock_get):
-        mock_driver.return_value = k8s_dr.Driver()
-        self._test_extract_template_definition(
-            mock_driver,
-            mock_objects_cluster_template_get_by_uuid,
-            mock_get,
-            missing_attr='flavor_id')
-
-    @patch('requests.get')
-    @patch('magnum.objects.ClusterTemplate.get_by_uuid')
-    @patch('magnum.drivers.common.driver.Driver.get_driver')
     def test_extract_template_definition_without_docker_storage_driver(
             self,
             mock_driver,
diff --git a/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py
index ed6edcb..19d04aa 100644
--- a/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_mesos_cluster_conductor.py
@@ -57,6 +57,7 @@ class TestClusterConductorWithMesos(base.TestCase):
             'cluster_template_id': 'xx-xx-xx-xx',
             'keypair': 'keypair_id',
             'master_flavor_id': 'master_flavor_id',
+            'flavor_id': 'flavor_id',
             'name': 'cluster1',
             'stack_id': 'xx-xx-xx-xx',
             'api_address': '172.17.2.3',
@@ -194,6 +195,7 @@ class TestClusterConductorWithMesos(base.TestCase):
             'mesos_slave_image_providers': 'docker',
             'master_flavor': 'master_flavor_id',
             'verify_ca': True,
+            'slave_flavor': 'flavor_id',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
diff --git a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py
index 315c1bd..847ce4f 100644
--- a/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_swarm_cluster_conductor.py
@@ -62,6 +62,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
             'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
             'cluster_template_id': 'xx-xx-xx-xx',
             'keypair': 'keypair_id',
+            'flavor_id': 'flavor_id',
             'docker_volume_size': 20,
             'master_flavor_id': 'master_flavor_id',
             'name': 'cluster1',
@@ -309,6 +310,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
             'docker_volume_size': 20,
             'master_flavor': 'master_flavor_id',
             'verify_ca': True,
+            'node_flavor': 'flavor_id',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
diff --git a/magnum/tests/unit/db/utils.py b/magnum/tests/unit/db/utils.py
index db1e85d..bac51b1 100644
--- a/magnum/tests/unit/db/utils.py
+++ b/magnum/tests/unit/db/utils.py
@@ -99,6 +99,7 @@ def get_test_cluster(**kw):
         'docker_volume_size': kw.get('docker_volume_size'),
         'labels': kw.get('labels'),
         'master_flavor_id': kw.get('master_flavor_id', None),
+        'flavor_id': kw.get('flavor_id', None),
     }
 
     # Only add Keystone trusts related attributes on demand since they may
diff --git a/magnum/tests/unit/objects/test_objects.py b/magnum/tests/unit/objects/test_objects.py
index 16d1f5c..a27a3ac 100644
--- a/magnum/tests/unit/objects/test_objects.py
+++ b/magnum/tests/unit/objects/test_objects.py
@@ -355,7 +355,7 @@ class TestObject(test_base.TestCase, _TestObject):
 # For more information on object version testing, read
 # http://docs.openstack.org/developer/magnum/objects.html
 object_data = {
-    'Cluster': '1.16-7a544c5059697c464810470980f81ba1',
+    'Cluster': '1.17-c32c07425ab0042c7370bef2902b4d21',
     'ClusterTemplate': '1.18-7fa94f4fdd027acfb4f022f202afdfb5',
     'Certificate': '1.1-1924dc077daa844f0f9076332ef96815',
     'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',