Optimize Rally imports to reduce start time

* Load plugins only if they are required
  This reduce a lot starting time of Rally and will
  allows us to do online bash completition

  As well it cleans up project structure a lot, e.g.
  all plugins are loaded in single place +
  we don't have strange code in cliutils

* Replace rally.ui.utils to not import mako
  and remove all code on module level

* Make Rally DB lazy inited

* Import plugins during test run

----

The best way to test changes in start up time is to
compare "time rally version" e.g.:

before this patch:
$ time rally version
0.0.5
real	0m0.397s
user	0m0.299s
sys	0m0.089s

after this patch:
0.0.5

real	0m0.281s
user	0m0.200s
sys	0m0.077s

Change-Id: Ibec2e6da66a1304730e801de307df7a1da68d51f
This commit is contained in:
Boris Pavlovic 2015-06-28 19:08:03 -07:00
parent 044d6f34b1
commit f2f57e99ad
12 changed files with 148 additions and 101 deletions

View File

@ -519,9 +519,6 @@ def run(argv, categories):
return(1)
try:
discover.load_plugins("/opt/rally/plugins/")
discover.load_plugins(os.path.expanduser("~/.rally/plugins/"))
discover.import_modules_from_package("rally.plugins")
for path in CONF.plugin_paths or []:
discover.load_plugins(path)

View File

@ -34,6 +34,7 @@ from rally import db
from rally import exceptions
from rally import objects
from rally import osclients
from rally import plugins
class DeploymentCommands(object):
@ -49,6 +50,7 @@ class DeploymentCommands(object):
@cliutils.args("--no-use", action="store_false", dest="do_use",
help="Don\'t set new deployment as default for"
" future operations")
@plugins.ensure_plugins_are_loaded
def create(self, name, fromenv=False, filename=None, do_use=False):
"""Create new deployment.
@ -130,6 +132,7 @@ class DeploymentCommands(object):
@cliutils.args("--deployment", dest="deployment", type=str,
required=False, help="UUID or name of a deployment.")
@envutils.with_default_deployment()
@plugins.ensure_plugins_are_loaded
def recreate(self, deployment=None):
"""Destroy and create an existing deployment.
@ -143,6 +146,7 @@ class DeploymentCommands(object):
@cliutils.args("--deployment", dest="deployment", type=str,
required=False, help="UUID or name of a deployment.")
@envutils.with_default_deployment()
@plugins.ensure_plugins_are_loaded
def destroy(self, deployment=None):
"""Destroy existing deployment.

View File

@ -57,6 +57,7 @@ from rally.common import utils
from rally.deployment import engine
from rally.deployment.serverprovider import provider
from rally import exceptions
from rally import plugins
from rally.task.scenarios import base as scenario_base
from rally.task import sla
@ -79,6 +80,7 @@ class InfoCommands(object):
"""
@cliutils.args("--query", dest="query", type=str, help="Search query.")
@plugins.ensure_plugins_are_loaded
def find(self, query):
"""Search for an entity that matches the query and print info about it.
@ -100,6 +102,7 @@ class InfoCommands(object):
"\n\t".join(substitutions))
return 1
@plugins.ensure_plugins_are_loaded
def list(self):
"""List main entities in Rally for which rally info find works.
@ -110,6 +113,7 @@ class InfoCommands(object):
self.DeploymentEngines()
self.ServerProviders()
@plugins.ensure_plugins_are_loaded
def BenchmarkScenarios(self):
"""Get information about benchmark scenarios available in Rally."""
def scenarios_filter(scenario_cls):
@ -142,6 +146,7 @@ class InfoCommands(object):
" $ rally info find <ScenarioGroupName>\n\n")
print(info)
@plugins.ensure_plugins_are_loaded
def SLA(self):
"""Get information about SLA available in Rally."""
sla_descrs = self._get_descriptions(sla.SLA)
@ -173,6 +178,7 @@ class InfoCommands(object):
" $ rally info find <sla_check_name>\n")
print(info)
@plugins.ensure_plugins_are_loaded
def DeploymentEngines(self):
"""Get information about deploy engines available in Rally."""
engines = self._get_descriptions(engine.EngineFactory)
@ -206,6 +212,7 @@ class InfoCommands(object):
" $ rally info find <DeploymentEngineName>\n")
print(info)
@plugins.ensure_plugins_are_loaded
def ServerProviders(self):
"""Get information about server providers available in Rally."""
providers = self._get_descriptions(provider.ProviderFactory)

View File

@ -37,6 +37,7 @@ from rally import consts
from rally import db
from rally import exceptions
from rally import objects
from rally import plugins
from rally.task.processing import plot
from rally.task.processing import utils
@ -132,6 +133,7 @@ class TaskCommands(object):
"json/yaml). These args are used to render input "
"task that is jinja2 template.")
@envutils.with_default_deployment(cli_arg_name="deployment")
@plugins.ensure_plugins_are_loaded
def validate(self, task, deployment=None, task_args=None,
task_args_file=None):
"""Validate a task configuration file.
@ -178,6 +180,7 @@ class TaskCommands(object):
help="Abort the execution of a benchmark scenario when"
"any SLA check for it fails")
@envutils.with_default_deployment(cli_arg_name="deployment")
@plugins.ensure_plugins_are_loaded
def start(self, task, deployment=None, task_args=None, task_args_file=None,
tag=None, do_use=False, abort_on_sla_failure=False):
"""Start benchmark task.

View File

@ -50,24 +50,33 @@ CONF = cfg.CONF
db_options.set_defaults(CONF, connection="sqlite:////tmp/rally.sqlite",
sqlite_db="rally.sqlite")
_BACKEND_MAPPING = {"sqlalchemy": "rally.db.sqlalchemy.api"}
IMPL = db_api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)
IMPL = None
def get_impl():
global IMPL
if not IMPL:
_BACKEND_MAPPING = {"sqlalchemy": "rally.db.sqlalchemy.api"}
IMPL = db_api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)
return IMPL
def db_cleanup():
"""Recreate engine."""
IMPL.db_cleanup()
get_impl().db_cleanup()
def db_create():
"""Initialize DB. This method will drop existing database."""
IMPL.db_create()
get_impl().db_create()
def db_drop():
"""Drop DB. This method drop existing database."""
IMPL.db_drop()
get_impl().db_drop()
def task_get(uuid):
@ -77,12 +86,12 @@ def task_get(uuid):
:raises: :class:`rally.exceptions.TaskNotFound` if the task does not exist.
:returns: task dict with data on the task.
"""
return IMPL.task_get(uuid)
return get_impl().task_get(uuid)
def task_get_detailed_last():
"""Returns the most recently created task."""
return IMPL.task_get_detailed_last()
return get_impl().task_get_detailed_last()
def task_get_detailed(uuid):
@ -91,7 +100,7 @@ def task_get_detailed(uuid):
:param uuid: UUID of the task.
:returns: task dict with data on the task and its results.
"""
return IMPL.task_get_detailed(uuid)
return get_impl().task_get_detailed(uuid)
def task_create(values):
@ -100,7 +109,7 @@ def task_create(values):
:param values: dict with record values.
:returns: task dict with data on the task.
"""
return IMPL.task_create(values)
return get_impl().task_create(values)
def task_update(uuid, values):
@ -111,7 +120,7 @@ def task_update(uuid, values):
:raises: :class:`rally.exceptions.TaskNotFound` if the task does not exist.
:returns: new updated task dict with data on the task.
"""
return IMPL.task_update(uuid, values)
return get_impl().task_update(uuid, values)
def task_list(status=None, deployment=None):
@ -124,7 +133,7 @@ def task_list(status=None, deployment=None):
returned.
:returns: A list of dicts with data on the tasks.
"""
return IMPL.task_list(status=status, deployment=deployment)
return get_impl().task_list(status=status, deployment=deployment)
def task_delete(uuid, status=None):
@ -139,7 +148,7 @@ def task_delete(uuid, status=None):
:raises: :class:`rally.exceptions.TaskInvalidStatus` if the status
of the task does not equal to the status argument.
"""
return IMPL.task_delete(uuid, status=status)
return get_impl().task_delete(uuid, status=status)
def task_result_get_all_by_uuid(task_uuid):
@ -148,7 +157,7 @@ def task_result_get_all_by_uuid(task_uuid):
:param task_uuid: string with UUID of Task instance.
:returns: list instances of TaskResult.
"""
return IMPL.task_result_get_all_by_uuid(task_uuid)
return get_impl().task_result_get_all_by_uuid(task_uuid)
def task_result_create(task_uuid, key, data):
@ -159,7 +168,7 @@ def task_result_create(task_uuid, key, data):
:param data: data expected to update in task result.
:returns: TaskResult instance appended.
"""
return IMPL.task_result_create(task_uuid, key, data)
return get_impl().task_result_create(task_uuid, key, data)
def deployment_create(values):
@ -168,7 +177,7 @@ def deployment_create(values):
:param values: dict with record values on the deployment.
:returns: a dict with data on the deployment.
"""
return IMPL.deployment_create(values)
return get_impl().deployment_create(values)
def deployment_delete(uuid):
@ -180,7 +189,7 @@ def deployment_delete(uuid):
:raises: :class:`rally.exceptions.DeploymentIsBusy` if the resource is
not enough.
"""
return IMPL.deployment_delete(uuid)
return get_impl().deployment_delete(uuid)
def deployment_get(deployment):
@ -191,7 +200,7 @@ def deployment_get(deployment):
does not exist.
:returns: a dict with data on the deployment.
"""
return IMPL.deployment_get(deployment)
return get_impl().deployment_get(deployment)
def deployment_update(uuid, values):
@ -203,7 +212,7 @@ def deployment_update(uuid, values):
does not exist.
:returns: a dict with data on the deployment.
"""
return IMPL.deployment_update(uuid, values)
return get_impl().deployment_update(uuid, values)
def deployment_list(status=None, parent_uuid=None, name=None):
@ -215,8 +224,8 @@ def deployment_list(status=None, parent_uuid=None, name=None):
:param name: Name of deployment
:returns: a list of dicts with data on the deployments.
"""
return IMPL.deployment_list(status=status, parent_uuid=parent_uuid,
name=name)
return get_impl().deployment_list(status=status, parent_uuid=parent_uuid,
name=name)
def resource_create(values):
@ -225,7 +234,7 @@ def resource_create(values):
:param values: a dict with data on the resource.
:returns: a dict with updated data on the resource.
"""
return IMPL.resource_create(values)
return get_impl().resource_create(values)
def resource_get_all(deployment_uuid, provider_name=None, type=None):
@ -237,9 +246,9 @@ def resource_get_all(deployment_uuid, provider_name=None, type=None):
:param type: filter by type, if is None, then return all types
:returns: a list of dicts with data on a resource
"""
return IMPL.resource_get_all(deployment_uuid,
provider_name=provider_name,
type=type)
return get_impl().resource_get_all(deployment_uuid,
provider_name=provider_name,
type=type)
def resource_delete(id):
@ -249,7 +258,7 @@ def resource_delete(id):
:raises: :class:`rally.exceptions.ResourceNotFound` if the resource
does not exist.
"""
return IMPL.resource_delete(id)
return get_impl().resource_delete(id)
def verification_create(deployment_uuid):
@ -258,7 +267,7 @@ def verification_create(deployment_uuid):
:param deployment_uuid: UUID of the deployment.
:returns: a dict with verification data.
"""
return IMPL.verification_create(deployment_uuid)
return get_impl().verification_create(deployment_uuid)
def verification_get(verification_uuid):
@ -269,7 +278,7 @@ def verification_get(verification_uuid):
does not exist.
:returns: a dict with verification data.
"""
return IMPL.verification_get(verification_uuid)
return get_impl().verification_get(verification_uuid)
def verification_delete(verification_uuid):
@ -279,7 +288,7 @@ def verification_delete(verification_uuid):
:raises: :class:`rally.exceptions.NotFoundException` if verification
does not exist.
"""
return IMPL.verification_delete(verification_uuid)
return get_impl().verification_delete(verification_uuid)
def verification_update(uuid, values):
@ -291,7 +300,7 @@ def verification_update(uuid, values):
does not exist.
:returns: new updated task dict with data on the task.
"""
return IMPL.verification_update(uuid, values)
return get_impl().verification_update(uuid, values)
def verification_list(status=None):
@ -300,7 +309,7 @@ def verification_list(status=None):
:param status: Verification status to filter the returned list on.
:returns: A list of dicts with data on the verifications.
"""
return IMPL.verification_list(status=status)
return get_impl().verification_list(status=status)
def verification_result_get(verification_uuid):
@ -309,7 +318,7 @@ def verification_result_get(verification_uuid):
:param verification_uuid: string with UUID of Verification instance.
:returns: dict instance of VerificationResult.
"""
return IMPL.verification_result_get(verification_uuid)
return get_impl().verification_result_get(verification_uuid)
def verification_result_create(verification_uuid, values):
@ -319,7 +328,7 @@ def verification_result_create(verification_uuid, values):
:param values: dict with record values.
:returns: TaskResult instance appended.
"""
return IMPL.verification_result_create(verification_uuid, values)
return get_impl().verification_result_create(verification_uuid, values)
def register_worker(values):
@ -333,7 +342,7 @@ def register_worker(values):
:returns: A worker.
:raises: WorkerAlreadyRegistered
"""
return IMPL.register_worker(values)
return get_impl().register_worker(values)
def get_worker(hostname):
@ -343,7 +352,7 @@ def get_worker(hostname):
:returns: A worker.
:raises: WorkerNotFound
"""
return IMPL.get_worker(hostname)
return get_impl().get_worker(hostname)
def unregister_worker(hostname):
@ -352,7 +361,7 @@ def unregister_worker(hostname):
:param hostname: The hostname of the worker service.
:raises: WorkerNotFound
"""
IMPL.unregister_worker(hostname)
get_impl().unregister_worker(hostname)
def update_worker(hostname):
@ -361,4 +370,4 @@ def update_worker(hostname):
:param hostname: The hostname of this worker service.
:raises: WorkerNotFound
"""
IMPL.update_worker(hostname)
get_impl().update_worker(hostname)

View File

@ -1,21 +0,0 @@
# Copyright 2013: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally.deployment.engine import * # noqa
from rally.common.plugin import discover
discover.import_modules_from_package("rally.deployment.engines")
discover.import_modules_from_package("rally.deployment.serverprovider")

View File

@ -101,7 +101,7 @@ class EngineFactory(plugin.Plugin):
"does not exist.") %
{"uuid": deployment["uuid"], "name": name})
deployment.update_status(consts.DeployStatus.DEPLOY_FAILED)
raise exceptions.PluginNotFound(engine_name=name)
raise exceptions.PluginNotFound(name=name)
@abc.abstractmethod
def deploy(self):

View File

@ -0,0 +1,43 @@
# Copyright 2015: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import decorator
from rally.common.plugin import discover
PLUGINS_LOADED = False
def load():
global PLUGINS_LOADED
if not PLUGINS_LOADED:
discover.import_modules_from_package("rally.deployment.engines")
discover.import_modules_from_package("rally.deployment.serverprovider")
discover.import_modules_from_package("rally.plugins")
discover.load_plugins("/opt/rally/plugins/")
discover.load_plugins(os.path.expanduser("~/.rally/plugins/"))
PLUGINS_LOADED = True
@decorator.decorator
def ensure_plugins_are_loaded(f, *args, **kwargs):
load()
return f(*args, **kwargs)

View File

@ -15,39 +15,38 @@
from __future__ import print_function
import os.path
import re
import sys
import mako.exceptions
import mako.lookup
import mako.template
templates_dir = os.path.join(os.path.dirname(__file__), "templates")
lookup_dirs = [templates_dir,
os.path.abspath(os.path.join(templates_dir, "..", "..", ".."))]
lookup = mako.lookup.TemplateLookup(directories=lookup_dirs)
def get_template(template_path):
import mako.lookup
templates_dir = os.path.join(os.path.dirname(__file__), "templates")
lookup_dirs = [
templates_dir,
os.path.abspath(os.path.join(templates_dir, "..", "..", ".."))
]
lookup = mako.lookup.TemplateLookup(directories=lookup_dirs)
return lookup.get_template(template_path)
def main(*args):
if len(args) < 2 or args[0] != "render":
exit("Usage: \n\t"
"utils.py render <lookup/path/to/template.mako> "
"<key-1>=<value-1> <key-2>=<value-2>\n"
"where key-1,value-1 and key-2,value-2 are key pairs of template")
try:
render_kwargs = dict([arg.split("=") for arg in args[2:]])
if (len(args) < 2 or args[0] != "render"
or not all(re.match("^[^=]+=[^=]+$", arg) for arg in args[2:])):
raise ValueError(
"Usage: \n\t"
"utils.py render <lookup/path/to/template.mako> "
"<key-1>=<value-1> <key-2>=<value-2>\n\n\t"
"Where key-1,value-1 and key-2,value-2 are key pairs of template"
)
print(get_template(sys.argv[2]).render(**render_kwargs))
except mako.exceptions.TopLevelLookupException as e:
exit(e)
render_kwargs = dict([arg.split("=") for arg in args[2:]])
print(get_template(args[1]).render(**render_kwargs))
if __name__ == "__main__":
args = sys.argv[1:]
main(*args)
main(*sys.argv[1:])

View File

@ -13,14 +13,14 @@
import os
import mako.template
__description__ = "List differences between two verification runs"
__title__ = "Verification Comparison"
__version__ = "0.1"
def create_report(results):
import mako.template
template_kw = {
"heading": {
"title": __title__,

View File

@ -21,6 +21,7 @@ from oslotest import base
from oslotest import mockpatch
from rally import db
from rally import plugins
from tests.unit import fakes
@ -41,6 +42,7 @@ class TestCase(base.BaseTestCase):
def setUp(self):
super(TestCase, self).setUp()
self.addCleanup(mock.patch.stopall)
plugins.load()
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = atomic_actions.get(name)

View File

@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import mako
import mock
from rally.ui import utils
@ -22,17 +22,21 @@ from tests.unit import test
class PlotTestCase(test.TestCase):
def test_lookup(self):
self.assertIsInstance(utils.lookup, utils.mako.lookup.TemplateLookup)
self.assertIsInstance(utils.lookup.get_template("/base.mako"),
utils.mako.lookup.Template)
self.assertRaises(
utils.mako.lookup.exceptions.TopLevelLookupException,
utils.lookup.get_template, "absent_template")
def test_get_template(self):
self.assertIsInstance(utils.get_template("task/report.mako"),
mako.template.Template)
@mock.patch("rally.ui.utils.lookup")
def test_get_template(self, mock_lookup):
mock_lookup.get_template.return_value = "foo_template"
template = utils.get_template("foo_path")
self.assertEqual(template, "foo_template")
mock_lookup.get_template.assert_called_once_with("foo_path")
@mock.patch("rally.ui.utils.get_template")
def test_main(self, mock_get_template):
utils.main("render", "somepath", "a=1", "b=2")
mock_get_template.assert_called_once_with("somepath")
mock_get_template.return_value.render.assert_called_once_with(
a="1", b="2"
)
def test_main_bad_input(self):
self.assertRaises(ValueError, utils.main)
self.assertRaises(ValueError, utils.main, "not_a_render")
self.assertRaises(ValueError, utils.main, "render")
self.assertRaises(ValueError, utils.main, "render", "path", "a 1")