Merge "Enhance Shaker to support basic heat environment files"

shaker/engine/deploy.py

@@ -302,10 +302,14 @@ class Deployment(object):
             exit(1)
         merged_parameters.update(specification.get('template_parameters', {}))
 
+        env_file = specification.get('env_file', None)
+        if env_file is not None:
+            env_file = self._render_env_template(env_file, base_dir)
+
         self.has_stack = True
         stack_id = heat.create_stack(
             self.openstack_client.heat, self.stack_name, rendered_template,
-            merged_parameters)
+            merged_parameters, env_file)
 
         # get info about deployed objects
         outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id)
@@ -330,6 +334,20 @@ class Deployment(object):
                 return functools.partial(override_ip,
                                          ip_type=override_spec.get('ip'))
 
+    # translate jinja decorations in env files
+    def _render_env_template(self, env_file, base_dir):
+        env_template = utils.read_file(env_file,
+                                       base_dir=base_dir)
+        env_values = {
+            'CONF': cfg.CONF
+        }
+        compiled_env = jinja2.Template(env_template)
+        rendered_env = compiled_env.render(env_values)
+
+        environment = utils.read_yaml(rendered_env)
+
+        return environment
+
     def deploy(self, deployment, base_dir=None, server_endpoint=None):
         agents = {}
 

shaker/engine/utils.py

@@ -136,12 +136,16 @@ def write_file(data, file_name, base_dir=''):
 
 def read_yaml_file(file_name):
     raw = read_file(file_name)
+    return read_yaml(raw)
+
+
+def read_yaml(raw):
     try:
         parsed = yaml.safe_load(raw)
         return parsed
     except Exception as e:
-        LOG.error('Failed to parse file %(file)s in YAML format: %(err)s',
+        LOG.error('Failed to parse input %(yaml)s in YAML format: %(err)s',
-                  dict(file=file_name, err=e))
+                  dict(yaml=raw, err=e))
         raise
 
 

shaker/openstack/clients/heat.py

@@ -22,11 +22,12 @@ from oslo_log import log as logging
 LOG = logging.getLogger(__name__)
 
 
-def create_stack(heat_client, stack_name, template, parameters):
+def create_stack(heat_client, stack_name, template, parameters, environment):
     stack_params = {
         'stack_name': stack_name,
         'template': template,
         'parameters': parameters,
+        'environment': environment,
     }
 
     stack = heat_client.stacks.create(**stack_params)['stack']

shaker/resources/schemas/scenario.yaml

@@ -11,6 +11,8 @@ mapping:
     mapping:
       template:
         type: str
+      env_file:
+        type: str
       agents:
         type: any
       accommodation:

shaker/scenarios/test/env/sample.env

@@ -0,0 +1,3 @@
+parameters:
+  custom_cidr: '192.0.0.0/16'
+  agent_join_timeout: {{ CONF.agent_join_timeout }}

shaker/scenarios/test/l2_with_env.hot

@@ -0,0 +1,112 @@
+heat_template_version: 2013-05-23
+
+description:
+  This Heat template creates a new Neutron network, a router to the external
+  network and plugs instances into this new network. All instances are located
+  in the same L2 domain.
+
+parameters:
+  image:
+    type: string
+    description: Name of image to use for servers
+  flavor:
+    type: string
+    description: Flavor to use for servers
+  external_net:
+    type: string
+    description: ID or name of external network
+  server_endpoint:
+    type: string
+    description: Server endpoint address
+  dns_nameservers:
+    type: comma_delimited_list
+    description: DNS nameservers for the subnet
+  custom_cidr:
+    type: string
+    description: set a cidr from the env file
+  agent_join_timeout:
+    type: string
+    description: shake CONF setting for agent_join_timeout
+
+resources:
+  private_net:
+    type: OS::Neutron::Net
+    properties:
+      name: {{ unique }}_net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network_id: { get_resource: private_net }
+      cidr: { get_param: custom_cidr }
+      dns_nameservers: { get_param: dns_nameservers }
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: { get_param: external_net }
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+  server_security_group:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      rules: [
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: tcp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: udp,
+        port_range_min: 1,
+        port_range_max: 65535},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+  {{ agent.id }}:
+    type: OS::Nova::Server
+    properties:
+      name: {{ agent.id }}
+      image: { get_param: image }
+      flavor: { get_param: flavor }
+      availability_zone: "{{ agent.availability_zone }}"
+      networks:
+        - port: { get_resource: {{ agent.id }}_port }
+      user_data_format: RAW
+      user_data:
+        str_replace:
+          template: |
+            #!/bin/sh
+            echo $AGENT_JOIN_TIMEOUT > /tmp/check_agent_timeout
+
+            screen -dmS shaker-agent-screen shaker-agent --server-endpoint=$SERVER_ENDPOINT --agent-id=$AGENT_ID
+          params:
+            "$SERVER_ENDPOINT": { get_param: server_endpoint }
+            "$AGENT_ID": {{ agent.id }}
+            "$AGENT_JOIN_TIMEOUT": { get_param: agent_join_timeout }
+
+  {{ agent.id }}_port:
+    type: OS::Neutron::Port
+    properties:
+      network_id: { get_resource: private_net }
+      fixed_ips:
+        - subnet_id: { get_resource: private_subnet }
+      security_groups: [{ get_resource: server_security_group }]
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+  {{ agent.id }}_instance_name:
+    value: { get_attr: [ {{ agent.id }}, instance_name ] }
+  {{ agent.id }}_ip:
+    value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+{% endfor %}
+

shaker/scenarios/test/sample_with_env.yaml

@@ -0,0 +1,23 @@
+title: Sample TCP Test with Environment File
+
+description:
+  This test definition demonstrates the use of an environment file.
+
+  In this scenario Shaker launches pairs of instances in the same tenant
+  network. Every instance is hosted on a separate compute node, 1 compute node
+  is utilized. The traffic goes within the tenant network (L2 domain)
+
+deployment:
+  template: l2_with_env.hot
+  env_file: env/sample.env
+
+  accommodation: [pair, compute_nodes: 1]
+
+execution:
+  tests:
+  -
+    title: tcp
+    class: iperf3
+
+    sla:
+    - "[type == 'agent'] >> (stats.bandwidth.avg > 5000)"

shaker/tests/test_deploy.py

@@ -17,12 +17,15 @@ import collections
 import copy
 import itertools
 import mock
+import os
+import re
 import testtools
 
 from oslo_config import cfg
 from oslo_config import fixture as config_fixture_pkg
 from shaker.engine import config
 from shaker.engine import deploy
+from shaker.engine import utils
 from shaker.openstack.clients import nova
 from shaker.tests import fakes
 
@@ -562,6 +565,70 @@ class TestDeploy(testtools.TestCase):
         self.assertRaises(deploy.DeploymentException,
                           deployment.deploy, {'template': 'foo'})
 
+    @mock.patch('shaker.openstack.clients.heat.get_stack_outputs')
+    @mock.patch('shaker.openstack.clients.heat.create_stack')
+    @mock.patch('shaker.openstack.clients.openstack.OpenStackClient')
+    @mock.patch('shaker.engine.deploy.Deployment._get_compute_nodes')
+    def test_deploy_from_hot_with_env_file(self, nova_nodes_mock,
+                                           openstack_mock, create_stack_mock,
+                                           stack_output_mock):
+        test_file = 'shaker/scenarios/test/sample_with_env.yaml'
+        absolute_path = utils.resolve_relative_path(test_file)
+        scenario = utils.read_yaml_file(absolute_path)
+
+        heat_id = 'shaker_abcdefg'
+
+        server_endpoint = "127.0.0.01"
+        base_dir = os.path.dirname(absolute_path)
+
+        deployment = deploy.Deployment()
+        deployment.stack_name = heat_id
+        deployment.external_net = 'test-external_net'
+        deployment.image_name = 'test-image'
+        deployment.flavor_name = 'test-flavor'
+        deployment.dns_nameservers = '8.8.8.8'
+        deployment.openstack_client = openstack_mock
+
+        # read the env file to determine what cidr is set to
+        # minus the last digit
+        env_file = utils.read_file(scenario['deployment']['env_file'],
+                                   base_dir)
+        cidr = re.findall(r'[0-9]+(?:\.[0-9]+){2}', env_file)[0]
+
+        nova_nodes_mock.return_value = [{'host': 'host-1', 'zone': 'nova'}]
+
+        create_stack_mock.create_stack.return_value = heat_id
+
+        heat_outputs = {
+            heat_id + '_master_0_instance_name': 'instance-0000052f',
+            heat_id + '_master_0_ip': '192.0.0.3',
+            heat_id + '_slave_0_ip': '192.0.0.4',
+            heat_id + '_slave_0_instance_name': 'instance-0000052c'}
+
+        stack_output_mock.return_value = heat_outputs
+
+        expected = {
+            'shaker_abcdefg_master_0': {'availability_zone': 'nova:host-1',
+                                        'id': 'shaker_abcdefg_master_0',
+                                        'ip': cidr + '.3',
+                                        'mode': 'master',
+                                        'node': 'host-1',
+                                        'slave_id': 'shaker_abcdefg_slave_0',
+                                        'zone': 'nova'},
+            'shaker_abcdefg_slave_0': {'availability_zone': 'nova:host-1',
+                                       'id': 'shaker_abcdefg_slave_0',
+                                       'ip': cidr + '.4',
+                                       'master_id': 'shaker_abcdefg_master_0',
+                                       'mode': 'slave',
+                                       'node': 'host-1',
+                                       'zone': 'nova'}}
+
+        agents = deployment._deploy_from_hot(scenario['deployment'],
+                                             server_endpoint,
+                                             base_dir=base_dir)
+
+        self.assertEqual(expected, agents)
+
     @mock.patch('shaker.openstack.clients.openstack.OpenStackClient')
     def test_get_compute_nodes_flavor_no_extra_specs(self,
                                                      nova_client_mock):