Add Rally benchmark tool to the system tests

Add possibility to use Rally tool during system tests:
 - allow installation of Rally on master node in
   Docker container;
 - creation of Rally Deployments and Tasks from system
   tests;
 - running Rally benchmark tests against environments
   deployed by system tests;
 - fetching and comparing benchmarks results.

Closes-bug: #1269862

Change-Id: I2ab7a0b13eecf4ddef3b14cea46b785bcabca5a1

fuelweb_test/helpers/rally.py

@@ -0,0 +1,413 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import os
+
+from proboscis.asserts import assert_equal
+from proboscis.asserts import assert_true
+
+from devops.helpers.helpers import wait
+from fuelweb_test import logger
+
+
+class RallyEngine(object):
+    def __init__(self,
+                 admin_remote,
+                 container_repo,
+                 proxy_url=None,
+                 user_id=0,
+                 dir_for_home='/var/rally_home',
+                 home_bind_path='/home/rally'):
+        self.admin_remote = admin_remote
+        self.container_repo = container_repo
+        self.repository_tag = 'latest'
+        self.proxy_url = proxy_url or ""
+        self.user_id = user_id
+        self.dir_for_home = dir_for_home
+        self.home_bind_path = home_bind_path
+        self.setup()
+
+    def image_exists(self, tag='latest'):
+        cmd = "docker images | awk 'NR > 1{print $1\" \"$2}'"
+        logger.debug('Checking Docker images...')
+        result = self.admin_remote.execute(cmd)
+        logger.debug(result)
+        existing_images = [line.strip().split() for line in result['stdout']]
+        return [self.container_repo, tag] in existing_images
+
+    def pull_image(self):
+        #TODO(apanchenko): add possibility to load image from local path or
+        #remote link provided in settings, in order to speed up downloading
+        cmd = 'docker pull {0}'.format(self.container_repo)
+        logger.debug('Downloading Rally repository/image from registry...')
+        result = self.admin_remote.execute(cmd)
+        logger.debug(result)
+        return self.image_exists()
+
+    def run_container_command(self, command, in_background=False):
+        command = str(command).replace(r"'", r"'\''")
+        options = ''
+        if in_background:
+            options = '{0} -d'.format(options)
+        cmd = ("docker run {options} --user {user_id} --net=\"host\"  -e "
+               "\"http_proxy={proxy_url}\" -v {dir_for_home}:{home_bind_path} "
+               "{container_repo}:{tag} /bin/bash -c '{command}'".format(
+                   options=options,
+                   user_id=self.user_id,
+                   proxy_url=self.proxy_url,
+                   dir_for_home=self.dir_for_home,
+                   home_bind_path=self.home_bind_path,
+                   container_repo=self.container_repo,
+                   tag=self.repository_tag,
+                   command=command))
+        logger.debug('Executing command "{0}" in Rally container {1}..'.format(
+            cmd, self.container_repo))
+        result = self.admin_remote.execute(cmd)
+        logger.debug(result)
+        return result
+
+    def setup_utils(self):
+        utils = ['gawk', 'vim', 'curl']
+        cmd = ('unset http_proxy; apt-get update; '
+               'apt-get install -y {0}'.format(' '.join(utils)))
+        logger.debug('Installing utils "{0}" to the Rally container...'.format(
+            utils))
+        result = self.run_container_command(cmd)
+        assert_equal(result['exit_code'], 0,
+                     'Utils installation failed in Rally container: '
+                     '{0}'.format(result))
+
+    def create_database(self):
+        check_rally_db_cmd = 'test -s .rally.sqlite'
+        result = self.run_container_command(check_rally_db_cmd)
+        if result['exit_code'] == 0:
+            return
+        logger.debug('Recreating Database for Rally...')
+        create_rally_db_cmd = 'rally-manage db recreate'
+        result = self.run_container_command(create_rally_db_cmd)
+        assert_equal(result['exit_code'], 0,
+                     'Rally Database creation failed: {0}!'.format(result))
+        result = self.run_container_command(check_rally_db_cmd)
+        assert_equal(result['exit_code'], 0, 'Failed to create Database for '
+                                             'Rally: {0} !'.format(result))
+
+    def prepare_image(self):
+        self.create_database()
+        self.setup_utils()
+        last_container_cmd = "docker ps -lq"
+        result = self.admin_remote.execute(last_container_cmd)
+        assert_equal(result['exit_code'], 0,
+                     "Unable to get last container ID: {0}!".format(result))
+        last_container = ''.join([line.strip() for line in result['stdout']])
+        commit_cmd = 'docker commit {0} {1}:ready'.format(last_container,
+                                                          self.container_repo)
+        result = self.admin_remote.execute(commit_cmd)
+        assert_equal(result['exit_code'], 0,
+                     'Commit to Docker image "{0}" failed: {1}.'.format(
+                         self.container_repo, result))
+        return self.image_exists(tag='ready')
+
+    def setup_bash_alias(self):
+        alias_name = 'rally_docker'
+        check_alias_cmd = '. /root/.bashrc && alias {0}'.format(alias_name)
+        result = self.admin_remote.execute(check_alias_cmd)
+        if result['exit_code'] == 0:
+            return
+        logger.debug('Creating bash alias for Rally inside container...')
+        create_alias_cmd = ("alias {alias_name}='docker run --user {user_id} "
+                            "--net=\"host\"  -e \"http_proxy={proxy_url}\" -t "
+                            "-i -v {dir_for_home}:{home_bind_path}  "
+                            "{container_repo}:{tag} rally'".format(
+                                alias_name=alias_name,
+                                user_id=self.user_id,
+                                proxy_url=self.proxy_url,
+                                dir_for_home=self.dir_for_home,
+                                home_bind_path=self.home_bind_path,
+                                container_repo=self.container_repo,
+                                tag=self.repository_tag))
+        result = self.admin_remote.execute('echo "{0}">> /root/.bashrc'.format(
+            create_alias_cmd))
+        assert_equal(result['exit_code'], 0,
+                     "Alias creation for running Rally from container failed: "
+                     "{0}.".format(result))
+        result = self.admin_remote.execute(check_alias_cmd)
+        assert_equal(result['exit_code'], 0,
+                     "Alias creation for running Rally from container failed: "
+                     "{0}.".format(result))
+
+    def setup(self):
+        if not self.image_exists():
+            assert_true(self.pull_image(),
+                        "Docker image for Rally not found!")
+        if not self.image_exists(tag='ready'):
+            assert_true(self.prepare_image(),
+                        "Docker image for Rally is not ready!")
+        self.repository_tag = 'ready'
+        self.setup_bash_alias()
+
+    def list_deployments(self):
+        cmd = (r"rally deployment list | awk -F "
+               r"'[[:space:]]*\\\\|[[:space:]]*' '/\ydeploy\y/{print $2}'")
+        result = self.run_container_command(cmd)
+        logger.debug('Rally deployments list: {0}'.format(result))
+        return [line.strip() for line in result['stdout']]
+
+    def show_deployment(self, deployment_uuid):
+        cmd = ("rally deployment show {0} | awk -F "
+               "'[[:space:]]*\\\\|[[:space:]]*' '/\w/{{print $2\",\"$3\",\"$4"
+               "\",\"$5\",\"$6\",\"$7\",\"$8}}'").format(deployment_uuid)
+        result = self.run_container_command(cmd)
+        assert_equal(len(result['stdout']), 2,
+                     "Command 'rally deployment show' returned unexpected "
+                     "value: expected 2 lines, got {0}: ".format(result))
+        keys = [k for k in result['stdout'][0].strip().split(',') if k != '']
+        values = [v for v in result['stdout'][1].strip().split(',') if v != '']
+        return {keys[i]: values[i] for i in range(0, len(keys))}
+
+    def list_tasks(self):
+        cmd = "rally task list --uuids-only"
+        result = self.run_container_command(cmd)
+        logger.debug('Rally tasks list: {0}'.format(result))
+        return [line.strip() for line in result['stdout']]
+
+    def get_task_status(self, task_uuid):
+        cmd = "rally task status {0}".format(task_uuid)
+        result = self.run_container_command(cmd)
+        assert_equal(result['exit_code'], 0,
+                     "Getting Rally task status failed: {0}".format(result))
+        task_status = ''.join(result['stdout']).strip().split()[-1]
+        logger.debug('Rally task "{0}" has status "{1}".'.format(task_uuid,
+                                                                 task_status))
+        return task_status
+
+
+class RallyDeployment(object):
+    def __init__(self, rally_engine, cluster_vip, username, password, tenant,
+                 key_port=5000, proxy_url=''):
+        self.rally_engine = rally_engine
+        self.cluster_vip = cluster_vip
+        self.username = username
+        self.password = password
+        self.tenant_name = tenant
+        self.keystone_port = str(key_port)
+        self.proxy_url = proxy_url
+        self.auth_url = "http://{0}:{1}/v2.0/".format(self.cluster_vip,
+                                                      self.keystone_port)
+        self.set_proxy = not self.is_proxy_set
+        self._uuid = None
+        self.create_deployment()
+
+    @property
+    def uuid(self):
+        if self._uuid is None:
+            for d_uuid in self.rally_engine.list_deployments():
+                deployment = self.rally_engine.show_deployment(d_uuid)
+                logger.debug("Deployment info: {0}".format(deployment))
+                if self.auth_url in deployment['auth_url'] and \
+                    self.username == deployment['username'] and \
+                        self.tenant_name == deployment['tenant_name']:
+                    self._uuid = d_uuid
+                    break
+        return self._uuid
+
+    @property
+    def is_proxy_set(self):
+        cmd = '[ "${{http_proxy}}" == "{0}" ]'.format(self.proxy_url)
+        return self.rally_engine.run_container_command(cmd)['exit_code'] == 0
+
+    @property
+    def is_deployment_exist(self):
+        if self.uuid is not None:
+                return True
+        return False
+
+    def create_deployment(self):
+        if self.is_deployment_exist:
+            return
+        cmd = ('export OS_USERNAME={0} OS_PASSWORD={1} OS_TENANT_NAME={2} '
+               'OS_AUTH_URL="{3}"; rally deployment create --name "{4}"'
+               ' --fromenv').format(self.username, self.password,
+                                    self.tenant_name, self.auth_url,
+                                    self.cluster_vip)
+        result = self.rally_engine.run_container_command(cmd)
+        assert_true(self.is_deployment_exist,
+                    'Rally deployment creation failed: {0}'.format(result))
+        logger.debug('Rally deployment created: {0}'.format(result))
+        assert_true(self.check_deployment(),
+                    "Rally deployment check failed.")
+
+    def check_deployment(self, deployment_uuid=''):
+        cmd = 'rally deployment check {0}'.format(deployment_uuid)
+        result = self.rally_engine.run_container_command(cmd)
+        if result['exit_code'] == 0:
+            return True
+        else:
+            logger.error('Rally deployment check failed: {0}'.format(result))
+            return False
+
+
+class RallyTask(object):
+    def __init__(self, rally_deployment, test_type):
+        self.deployment = rally_deployment
+        self.engine = self.deployment.rally_engine
+        self.test_type = test_type
+        self.uuid = None
+        self._status = None
+
+    @property
+    def status(self):
+        if self.uuid is None:
+            self._status = None
+        else:
+            self._status = self.engine.get_task_status(self.uuid)
+        return self._status
+
+    def prepare_scenario(self):
+        scenario_file = '{0}/fuelweb_test/rally/screnarios/{1}.json'.format(
+            os.environ.get("WORKSPACE", "./"), self.test_type)
+        remote_path = '{0}/{1}.json'.format(self.engine.dir_for_home,
+                                            self.test_type)
+        self.engine.admin_remote.upload(scenario_file, remote_path)
+        result = self.engine.admin_remote.execute('test -f {0}'.format(
+            remote_path))
+        assert_equal(result['exit_code'], 0,
+                     "Scenario upload filed: {0}".format(result))
+        return '{0}.json'.format(self.test_type)
+
+    def start(self):
+        scenario = self.prepare_scenario()
+        temp_file = '{0}_results.tmp.txt'.format(scenario)
+        cmd = 'rally task start {0} &> {1}'.format(scenario, temp_file)
+        result = self.engine.run_container_command(cmd, in_background=True)
+        logger.debug('Started Rally task: {0}'.format(result))
+        cmd = ("awk 'BEGIN{{retval=1}};/^Using task:/{{print $NF; retval=0}};"
+               "END {{exit retval}}' {0}").format(temp_file)
+        wait(lambda: self.engine.run_container_command(cmd)['exit_code'] == 0,
+             timeout=30)
+        result = self.engine.run_container_command(cmd)
+        task_uuid = ''.join(result['stdout']).strip()
+        assert_true(task_uuid in self.engine.list_tasks(),
+                    "Rally task creation failed: {0}".format(result))
+        self.uuid = task_uuid
+
+    def get_results(self):
+        if self.status == 'finished':
+            cmd = 'rally task results {0}'.format(self.uuid)
+            result = self.engine.run_container_command(cmd)
+            assert_equal(result['exit_code'], 0,
+                         "Getting task results failed: {0}".format(result))
+            logger.debug("Rally task {0} result: {1}".format(self.uuid,
+                                                             result))
+            return ''.join(result['stdout'])
+
+
+class RallyResult(object):
+    def __init__(self, json_results):
+        self.values = {
+            'full_duration': 0.00,
+            'load_duration': 0.00,
+            'errors': 0
+        }
+        self.raw_data = []
+        self.parse_raw_results(json_results)
+
+    def parse_raw_results(self, raw_results):
+        data = json.loads(raw_results)
+        assert_equal(len(data), 1,
+                     "Current implementation of RallyResult class doesn't "
+                     "support results with length greater than '1'!")
+        self.raw_data = data[0]
+        self.values['full_duration'] = data[0]['full_duration']
+        self.values['load_duration'] = data[0]['load_duration']
+        self.values['errors'] = sum([len(result['error'])
+                                     for result in data[0]['result']])
+
+    @staticmethod
+    def compare(first_result, second_result, deviation=0.1):
+        """
+        Compare benchmark results
+        :param first_result: RallyResult
+        :param second_result: RallyResult
+        :param deviation: float
+        :return: bool
+        """
+        message = ''
+        equal = True
+        for val in first_result.values.keys():
+            logger.debug('Comparing {2}: {0} and {1}'.format(
+                first_result.values[val], second_result.values[val],
+                val
+            ))
+            if first_result.values[val] == 0 or second_result.values[val] == 0:
+                if first_result.values[val] != second_result.values[val]:
+                    message += "Values of '{0}' are: {1} and {2}. ".format(
+                        val,
+                        first_result.values[val],
+                        second_result.values[val])
+                    equal = False
+                continue
+            diff = abs(
+                first_result.values[val] / second_result.values[val] - 1)
+            if diff > deviation:
+                message += "Values of '{0}' are: {1} and {2}. ".format(
+                    val, first_result.values[val], second_result.values[val])
+                equal = False
+        if not equal:
+            logger.info("Rally benchmark results aren't equal: {0}".format(
+                message))
+        return equal
+
+    def show(self):
+        return json.dumps(self.raw_data)
+
+
+class RallyBenchmarkTest(object):
+    def __init__(self, container_repo, environment, cluster_id,
+                 test_type):
+        self.admin_remote = environment.d_env.get_admin_remote()
+        self.cluster_vip = environment.fuel_web.get_mgmt_vip(cluster_id)
+        self.cluster_credentials = \
+            environment.fuel_web.get_cluster_credentials(cluster_id)
+        self.proxy_url = environment.fuel_web.get_alive_proxy(cluster_id)
+        logger.debug('Rally proxy URL is: {0}'.format(self.proxy_url))
+        self.container_repo = container_repo
+        self.home_dir = 'rally-{0}'.format(cluster_id)
+        self.test_type = test_type
+        self.engine = RallyEngine(
+            admin_remote=self.admin_remote,
+            container_repo=self.container_repo,
+            proxy_url=self.proxy_url,
+            dir_for_home='/var/{0}/'.format(self.home_dir)
+        )
+        self.deployment = RallyDeployment(
+            rally_engine=self.engine,
+            cluster_vip=self.cluster_vip,
+            username=self.cluster_credentials['username'],
+            password=self.cluster_credentials['password'],
+            tenant=self.cluster_credentials['tenant'],
+            proxy_url=self.proxy_url
+        )
+        self.current_task = None
+
+    def run(self, timeout=60 * 10):
+        self.current_task = RallyTask(self.deployment, self.test_type)
+        logger.info('Starting Rally benchmark test...')
+        self.current_task.start()
+        assert_equal(self.current_task.status, 'running',
+                     'Rally task was started, but it is not running, status: '
+                     '{0}'.format(self.current_task.status))
+        wait(lambda: self.current_task.status == 'finished', timeout=timeout)
+        logger.info('Rally benchmark test is finished.')
+        return RallyResult(json_results=self.current_task.get_results())

fuelweb_test/models/fuel_web_client.py

@@ -940,8 +940,8 @@ class FuelWebClient(object):
         a roles
 
         :type cluster_id: Int
-        :type roles: List
-            :rtype: List
+        :type roles: list
+            :rtype: list
         """
         nodes = self.client.list_cluster_nodes(cluster_id=cluster_id)
         return [n for n in nodes if set(roles) <= set(n['roles'])]
@@ -1981,6 +1981,9 @@ class FuelWebClient(object):
         return self.client.get_networks(
             cluster_id)['management_vrouter_vip']
 
+    def get_mgmt_vip(self, cluster_id):
+        return self.client.get_networks(cluster_id)['management_vip']
+
     @logwrap
     def get_controller_with_running_service(self, slave, service_name):
         ret = self.get_pacemaker_status(slave.name)
@@ -2108,3 +2111,41 @@ class FuelWebClient(object):
         assert_is_not_none(task,
                            'Got empty result after running deployment tasks!')
         self.assert_task_success(task, timeout)
+
+    @logwrap
+    def get_alive_proxy(self, cluster_id, port='8888'):
+        online_controllers = [node for node in
+                              self.get_nailgun_cluster_nodes_by_roles(
+                                  cluster_id,
+                                  roles=['controller', ]) if node['online']]
+
+        admin_remote = self.environment.d_env.get_admin_remote()
+        check_proxy_cmd = ('[[ $(curl -s -w "%{{http_code}}" '
+                           '{0} -o /dev/null) -eq 200 ]]')
+
+        for controller in online_controllers:
+            proxy_url = 'http://{0}:{1}/'.format(controller['ip'], port)
+            logger.debug('Trying to connect to {0} from master node...'.format(
+                proxy_url))
+            if admin_remote.execute(
+                    check_proxy_cmd.format(proxy_url))['exit_code'] == 0:
+                return proxy_url
+
+        assert_true(len(online_controllers) > 0,
+                    'There are no online controllers available '
+                    'to provide HTTP proxy!')
+
+        assert_false(len(online_controllers) == 0,
+                     'There are online controllers available ({0}), '
+                     'but no HTTP proxy is accessible from master '
+                     'node'.format(online_controllers))
+
+    @logwrap
+    def get_cluster_credentials(self, cluster_id):
+        attributes = self.client.get_cluster_attributes(cluster_id)
+        username = attributes['editable']['access']['user']['value']
+        password = attributes['editable']['access']['password']['value']
+        tenant = attributes['editable']['access']['tenant']['value']
+        return {'username': username,
+                'password': password,
+                'tenant': tenant}

fuelweb_test/rally/screnarios/nova.json

@@ -0,0 +1,26 @@
+{
+    "NovaServers.boot_and_delete_server": [
+        {
+            "args": {
+                "flavor": {
+                    "name": "m1.micro"
+                },
+                "image": {
+                    "name": "TestVM"
+                },
+                "force_delete": false
+            },
+            "runner": {
+                "type": "constant",
+                "times": 30,
+                "concurrency": 3
+            },
+            "context": {
+                "users": {
+                    "tenants": 3,
+                    "users_per_tenant": 2
+                }
+            }
+        }
+    ]
+}

fuelweb_test/rally/screnarios/scenarios.yaml

@@ -0,0 +1,16 @@
+categories:
+  - undefined:
+     tags:
+     scenarios:
+       nova
+ - nova:
+     tags:
+       nova
+     scenarions:
+       nova
+ - neutron:
+     tags:
+       neutron
+     scenarios:
+       neutron
+       nova

fuelweb_test/settings.py

@@ -446,3 +446,7 @@ EMC_POOL_NAME = os.environ.get('EMC_POOL_NAME', '')
 
 ALWAYS_CREATE_DIAGNOSTIC_SNAPSHOT = get_var_as_bool(
     'ALWAYS_CREATE_DIAGNOSTIC_SNAPSHOT', False)
+
+RALLY_DOCKER_REPO = os.environ.get('RALLY_DOCKER_REPO', 'rallyforge/rally')
+RALLY_CONTAINER_NAME = os.environ.get('RALLY_CONTAINER_NAME', 'rally')
+RALLY_TAGS = os.environ.get('RALLY_TAGS', 'nova').split(',')