diff --git a/shaker/engine/deploy.py b/shaker/engine/deploy.py index a4ca56d..95aab41 100644 --- a/shaker/engine/deploy.py +++ b/shaker/engine/deploy.py @@ -302,10 +302,14 @@ class Deployment(object): exit(1) merged_parameters.update(specification.get('template_parameters', {})) + env_file = specification.get('env_file', None) + if env_file is not None: + env_file = self._render_env_template(env_file, base_dir) + self.has_stack = True stack_id = heat.create_stack( self.openstack_client.heat, self.stack_name, rendered_template, - merged_parameters) + merged_parameters, env_file) # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id) @@ -330,6 +334,20 @@ class Deployment(object): return functools.partial(override_ip, ip_type=override_spec.get('ip')) + # translate jinja decorations in env files + def _render_env_template(self, env_file, base_dir): + env_template = utils.read_file(env_file, + base_dir=base_dir) + env_values = { + 'CONF': cfg.CONF + } + compiled_env = jinja2.Template(env_template) + rendered_env = compiled_env.render(env_values) + + environment = utils.read_yaml(rendered_env) + + return environment + def deploy(self, deployment, base_dir=None, server_endpoint=None): agents = {} diff --git a/shaker/engine/utils.py b/shaker/engine/utils.py index 0b226c5..4645355 100644 --- a/shaker/engine/utils.py +++ b/shaker/engine/utils.py @@ -136,12 +136,16 @@ def write_file(data, file_name, base_dir=''): def read_yaml_file(file_name): raw = read_file(file_name) + return read_yaml(raw) + + +def read_yaml(raw): try: parsed = yaml.safe_load(raw) return parsed except Exception as e: - LOG.error('Failed to parse file %(file)s in YAML format: %(err)s', - dict(file=file_name, err=e)) + LOG.error('Failed to parse input %(yaml)s in YAML format: %(err)s', + dict(yaml=raw, err=e)) raise diff --git a/shaker/openstack/clients/heat.py b/shaker/openstack/clients/heat.py index cb7d9d4..71fb581 100644 --- a/shaker/openstack/clients/heat.py +++ b/shaker/openstack/clients/heat.py @@ -22,11 +22,12 @@ from oslo_log import log as logging LOG = logging.getLogger(__name__) -def create_stack(heat_client, stack_name, template, parameters): +def create_stack(heat_client, stack_name, template, parameters, environment): stack_params = { 'stack_name': stack_name, 'template': template, 'parameters': parameters, + 'environment': environment, } stack = heat_client.stacks.create(**stack_params)['stack'] diff --git a/shaker/resources/schemas/scenario.yaml b/shaker/resources/schemas/scenario.yaml index 4630721..856a59f 100644 --- a/shaker/resources/schemas/scenario.yaml +++ b/shaker/resources/schemas/scenario.yaml @@ -11,6 +11,8 @@ mapping: mapping: template: type: str + env_file: + type: str agents: type: any accommodation: diff --git a/shaker/scenarios/test/env/sample.env b/shaker/scenarios/test/env/sample.env new file mode 100644 index 0000000..32fa189 --- /dev/null +++ b/shaker/scenarios/test/env/sample.env @@ -0,0 +1,3 @@ +parameters: + custom_cidr: '192.0.0.0/16' + agent_join_timeout: {{ CONF.agent_join_timeout }} \ No newline at end of file diff --git a/shaker/scenarios/test/l2_with_env.hot b/shaker/scenarios/test/l2_with_env.hot new file mode 100644 index 0000000..f7b4333 --- /dev/null +++ b/shaker/scenarios/test/l2_with_env.hot @@ -0,0 +1,112 @@ +heat_template_version: 2013-05-23 + +description: + This Heat template creates a new Neutron network, a router to the external + network and plugs instances into this new network. All instances are located + in the same L2 domain. + +parameters: + image: + type: string + description: Name of image to use for servers + flavor: + type: string + description: Flavor to use for servers + external_net: + type: string + description: ID or name of external network + server_endpoint: + type: string + description: Server endpoint address + dns_nameservers: + type: comma_delimited_list + description: DNS nameservers for the subnet + custom_cidr: + type: string + description: set a cidr from the env file + agent_join_timeout: + type: string + description: shake CONF setting for agent_join_timeout + +resources: + private_net: + type: OS::Neutron::Net + properties: + name: {{ unique }}_net + + private_subnet: + type: OS::Neutron::Subnet + properties: + network_id: { get_resource: private_net } + cidr: { get_param: custom_cidr } + dns_nameservers: { get_param: dns_nameservers } + + router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: { get_param: external_net } + + router_interface: + type: OS::Neutron::RouterInterface + properties: + router_id: { get_resource: router } + subnet_id: { get_resource: private_subnet } + + server_security_group: + type: OS::Neutron::SecurityGroup + properties: + rules: [ + {remote_ip_prefix: 0.0.0.0/0, + protocol: tcp, + port_range_min: 1, + port_range_max: 65535}, + {remote_ip_prefix: 0.0.0.0/0, + protocol: udp, + port_range_min: 1, + port_range_max: 65535}, + {remote_ip_prefix: 0.0.0.0/0, + protocol: icmp}] + +{% for agent in agents.values() %} + + {{ agent.id }}: + type: OS::Nova::Server + properties: + name: {{ agent.id }} + image: { get_param: image } + flavor: { get_param: flavor } + availability_zone: "{{ agent.availability_zone }}" + networks: + - port: { get_resource: {{ agent.id }}_port } + user_data_format: RAW + user_data: + str_replace: + template: | + #!/bin/sh + echo $AGENT_JOIN_TIMEOUT > /tmp/check_agent_timeout + + screen -dmS shaker-agent-screen shaker-agent --server-endpoint=$SERVER_ENDPOINT --agent-id=$AGENT_ID + params: + "$SERVER_ENDPOINT": { get_param: server_endpoint } + "$AGENT_ID": {{ agent.id }} + "$AGENT_JOIN_TIMEOUT": { get_param: agent_join_timeout } + + {{ agent.id }}_port: + type: OS::Neutron::Port + properties: + network_id: { get_resource: private_net } + fixed_ips: + - subnet_id: { get_resource: private_subnet } + security_groups: [{ get_resource: server_security_group }] + +{% endfor %} + +outputs: +{% for agent in agents.values() %} + {{ agent.id }}_instance_name: + value: { get_attr: [ {{ agent.id }}, instance_name ] } + {{ agent.id }}_ip: + value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] } +{% endfor %} + diff --git a/shaker/scenarios/test/sample_with_env.yaml b/shaker/scenarios/test/sample_with_env.yaml new file mode 100644 index 0000000..b78a9e6 --- /dev/null +++ b/shaker/scenarios/test/sample_with_env.yaml @@ -0,0 +1,23 @@ +title: Sample TCP Test with Environment File + +description: + This test definition demonstrates the use of an environment file. + + In this scenario Shaker launches pairs of instances in the same tenant + network. Every instance is hosted on a separate compute node, 1 compute node + is utilized. The traffic goes within the tenant network (L2 domain) + +deployment: + template: l2_with_env.hot + env_file: env/sample.env + + accommodation: [pair, compute_nodes: 1] + +execution: + tests: + - + title: tcp + class: iperf3 + + sla: + - "[type == 'agent'] >> (stats.bandwidth.avg > 5000)" diff --git a/shaker/tests/test_deploy.py b/shaker/tests/test_deploy.py index 29e8dbd..0c05b55 100644 --- a/shaker/tests/test_deploy.py +++ b/shaker/tests/test_deploy.py @@ -17,12 +17,15 @@ import collections import copy import itertools import mock +import os +import re import testtools from oslo_config import cfg from oslo_config import fixture as config_fixture_pkg from shaker.engine import config from shaker.engine import deploy +from shaker.engine import utils from shaker.openstack.clients import nova from shaker.tests import fakes @@ -562,6 +565,70 @@ class TestDeploy(testtools.TestCase): self.assertRaises(deploy.DeploymentException, deployment.deploy, {'template': 'foo'}) + @mock.patch('shaker.openstack.clients.heat.get_stack_outputs') + @mock.patch('shaker.openstack.clients.heat.create_stack') + @mock.patch('shaker.openstack.clients.openstack.OpenStackClient') + @mock.patch('shaker.engine.deploy.Deployment._get_compute_nodes') + def test_deploy_from_hot_with_env_file(self, nova_nodes_mock, + openstack_mock, create_stack_mock, + stack_output_mock): + test_file = 'shaker/scenarios/test/sample_with_env.yaml' + absolute_path = utils.resolve_relative_path(test_file) + scenario = utils.read_yaml_file(absolute_path) + + heat_id = 'shaker_abcdefg' + + server_endpoint = "127.0.0.01" + base_dir = os.path.dirname(absolute_path) + + deployment = deploy.Deployment() + deployment.stack_name = heat_id + deployment.external_net = 'test-external_net' + deployment.image_name = 'test-image' + deployment.flavor_name = 'test-flavor' + deployment.dns_nameservers = '8.8.8.8' + deployment.openstack_client = openstack_mock + + # read the env file to determine what cidr is set to + # minus the last digit + env_file = utils.read_file(scenario['deployment']['env_file'], + base_dir) + cidr = re.findall(r'[0-9]+(?:\.[0-9]+){2}', env_file)[0] + + nova_nodes_mock.return_value = [{'host': 'host-1', 'zone': 'nova'}] + + create_stack_mock.create_stack.return_value = heat_id + + heat_outputs = { + heat_id + '_master_0_instance_name': 'instance-0000052f', + heat_id + '_master_0_ip': '192.0.0.3', + heat_id + '_slave_0_ip': '192.0.0.4', + heat_id + '_slave_0_instance_name': 'instance-0000052c'} + + stack_output_mock.return_value = heat_outputs + + expected = { + 'shaker_abcdefg_master_0': {'availability_zone': 'nova:host-1', + 'id': 'shaker_abcdefg_master_0', + 'ip': cidr + '.3', + 'mode': 'master', + 'node': 'host-1', + 'slave_id': 'shaker_abcdefg_slave_0', + 'zone': 'nova'}, + 'shaker_abcdefg_slave_0': {'availability_zone': 'nova:host-1', + 'id': 'shaker_abcdefg_slave_0', + 'ip': cidr + '.4', + 'master_id': 'shaker_abcdefg_master_0', + 'mode': 'slave', + 'node': 'host-1', + 'zone': 'nova'}} + + agents = deployment._deploy_from_hot(scenario['deployment'], + server_endpoint, + base_dir=base_dir) + + self.assertEqual(expected, agents) + @mock.patch('shaker.openstack.clients.openstack.OpenStackClient') def test_get_compute_nodes_flavor_no_extra_specs(self, nova_client_mock):