From 387dc356e3792f5cf24145bf381c0c30268a9e02 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Wed, 4 Feb 2015 12:23:08 +0200 Subject: [PATCH] Remove all non-incubated oslo code Rally repository includes openstack.common module with modules from oslo-incubator(modules are listed in openstack-common.conf file). All those modules can be splitted by 4 categories: 1. logging modules 2. aas related modules 3. cliutils 4. config Modules from first category were graduated to separate lib, so we can remove them and use `oslo.log`. Modules from second category are not really used and will be not used, because Rally-as-a-Service will use flask microframework instead of pecan(modules from oslo-incubator are designed for pecan). Since oslo core team did not plan to graduate `cliutils` module from oslo-incubator, so there are no reasons to sync this module with oslo, we can copy needed functions and maintain them in our repo. Additional to rally.openstack dir, modules from oslo incubator are located in tools dir. tools/config modules works pretty bad, opts from rally.osclients always lost during automated config generator. `oslo.config` lib provides a better way to autogenerate config sample of Rally. It requires new entry point, which returns a list with all opts(look at 'rally.common.opts', 'setup.cfg' for more details). Since all opts are used in one place, their names were unified. Also config was updated(via `tox -egenconfig`). tools/rally.bash_completion file is moved to etc dir, because etc dir corresponds better and it gave us ability to remove whole tools dir Closes-Bug: #1378960 Change-Id: Ic127269c367275d3adcfc9f40d9144fce8295391 --- {tools => etc}/rally.bash_completion | 0 etc/rally/rally.conf.sample | 637 ++++++++---------- openstack-common.conf | 5 +- rally/aas/__init__.py | 2 + rally/aas/rest/__init__.py | 35 - rally/aas/rest/app.py | 47 -- rally/aas/rest/controllers/__init__.py | 0 rally/aas/rest/controllers/root.py | 47 -- rally/aas/rest/controllers/v1/__init__.py | 16 - rally/aas/rest/controllers/v1/root.py | 35 - rally/aas/rest/types.py | 64 -- rally/benchmark/context/users.py | 4 +- rally/benchmark/scenarios/cinder/utils.py | 4 +- rally/benchmark/scenarios/glance/utils.py | 4 +- rally/benchmark/scenarios/heat/utils.py | 6 +- rally/benchmark/scenarios/nova/utils.py | 6 +- rally/benchmark/scenarios/sahara/utils.py | 4 +- rally/cmd/api.py | 53 -- rally/cmd/cliutils.py | 96 ++- rally/cmd/commands/deployment.py | 10 +- rally/cmd/commands/show.py | 31 +- rally/cmd/commands/task.py | 24 +- rally/cmd/commands/verify.py | 9 +- rally/common/log.py | 31 +- rally/common/opts.py | 44 ++ rally/exceptions.py | 4 +- rally/openstack/__init__.py | 0 rally/openstack/common/__init__.py | 0 rally/openstack/common/_i18n.py | 40 -- rally/openstack/common/cliutils.py | 271 -------- rally/openstack/common/eventlet_backdoor.py | 145 ---- rally/openstack/common/fileutils.py | 146 ---- rally/openstack/common/local.py | 45 -- rally/openstack/common/lockutils.py | 326 --------- rally/openstack/common/log.py | 705 -------------------- rally/openstack/common/loopingcall.py | 147 ---- rally/openstack/common/periodic_task.py | 206 ------ rally/openstack/common/service.py | 504 -------------- rally/openstack/common/systemd.py | 106 --- rally/openstack/common/threadgroup.py | 147 ---- rally/osclients.py | 6 +- rally/verification/tempest/config.py | 4 +- requirements.txt | 9 +- setup.cfg | 5 +- tests/hacking/README.rst | 2 +- tests/hacking/checks.py | 4 +- tests/unit/aas/__init__.py | 0 tests/unit/aas/rest/__init__.py | 0 tests/unit/aas/rest/controllers/__init__.py | 0 tests/unit/aas/rest/test_types.py | 59 -- tests/unit/cmd/commands/test_deployment.py | 8 +- tests/unit/cmd/commands/test_show.py | 10 +- tests/unit/cmd/commands/test_task.py | 4 +- tests/unit/cmd/commands/test_verify.py | 6 +- tests/unit/cmd/test_cliutils.py | 413 +++++++++++- tests/unit/test_hacking.py | 4 +- tests/unit/test_log.py | 30 +- tests/unit/test_resources.py | 6 +- tools/config/check_uptodate.sh | 25 - tools/config/generate_sample.sh | 119 ---- tools/config/oslo.config.generator.rc | 4 - tox.ini | 9 +- 62 files changed, 962 insertions(+), 3771 deletions(-) rename {tools => etc}/rally.bash_completion (100%) delete mode 100644 rally/aas/rest/__init__.py delete mode 100644 rally/aas/rest/app.py delete mode 100644 rally/aas/rest/controllers/__init__.py delete mode 100644 rally/aas/rest/controllers/root.py delete mode 100644 rally/aas/rest/controllers/v1/__init__.py delete mode 100644 rally/aas/rest/controllers/v1/root.py delete mode 100644 rally/aas/rest/types.py delete mode 100644 rally/cmd/api.py create mode 100644 rally/common/opts.py delete mode 100644 rally/openstack/__init__.py delete mode 100644 rally/openstack/common/__init__.py delete mode 100644 rally/openstack/common/_i18n.py delete mode 100644 rally/openstack/common/cliutils.py delete mode 100644 rally/openstack/common/eventlet_backdoor.py delete mode 100644 rally/openstack/common/fileutils.py delete mode 100644 rally/openstack/common/local.py delete mode 100644 rally/openstack/common/lockutils.py delete mode 100644 rally/openstack/common/log.py delete mode 100644 rally/openstack/common/loopingcall.py delete mode 100644 rally/openstack/common/periodic_task.py delete mode 100644 rally/openstack/common/service.py delete mode 100644 rally/openstack/common/systemd.py delete mode 100644 rally/openstack/common/threadgroup.py delete mode 100644 tests/unit/aas/__init__.py delete mode 100644 tests/unit/aas/rest/__init__.py delete mode 100644 tests/unit/aas/rest/controllers/__init__.py delete mode 100644 tests/unit/aas/rest/test_types.py delete mode 100755 tools/config/check_uptodate.sh delete mode 100755 tools/config/generate_sample.sh delete mode 100644 tools/config/oslo.config.generator.rc diff --git a/tools/rally.bash_completion b/etc/rally.bash_completion similarity index 100% rename from tools/rally.bash_completion rename to etc/rally.bash_completion diff --git a/etc/rally/rally.conf.sample b/etc/rally/rally.conf.sample index 2bdd62298e..a8a5cc6378 100644 --- a/etc/rally/rally.conf.sample +++ b/etc/rally/rally.conf.sample @@ -1,542 +1,477 @@ [DEFAULT] # -# Options defined in rally.exceptions +# From oslo.log # -# make exception message format errors fatal (boolean value) -#fatal_exception_format_errors=false +# Print debugging output (set logging level to DEBUG instead of +# default WARNING level). (boolean value) +#debug = false +# Print more verbose output (set logging level to INFO instead of +# default WARNING level). (boolean value) +#verbose = false -# -# Options defined in rally.log -# - -# Print debugging output only for Rally. Off-site components -# stay quiet. (boolean value) -#rally_debug=false - - -# -# Options defined in rally.openstack.common.eventlet_backdoor -# - -# Enable eventlet backdoor. Acceptable values are 0, , -# and :, where 0 results in listening on a random -# tcp port number; results in listening on the -# specified port number (and not enabling backdoor if that -# port is in use); and : results in listening on -# the smallest unused port number within the specified range -# of port numbers. The chosen port is displayed in the -# service's log file. (string value) -#backdoor_port= - - -# -# Options defined in rally.openstack.common.lockutils -# - -# Whether to disable inter-process locks (boolean value) -#disable_process_locking=false - -# Directory to use for lock files. (string value) -#lock_path= - - -# -# Options defined in rally.openstack.common.log -# - -# Print debugging output (set logging level to DEBUG instead -# of default WARNING level). (boolean value) -#debug=false - -# Print more verbose output (set logging level to INFO instead -# of default WARNING level). (boolean value) -#verbose=false - -# Log output to standard error. (boolean value) -#use_stderr=true - -# Format string to use for log messages with context. (string -# value) -#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s - -# Format string to use for log messages without context. +# The name of a logging configuration file. This file is appended to +# any existing logging configuration files. For details about logging +# configuration files, see the Python logging module documentation. # (string value) -#logging_default_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s - -# Data to append to log format when level is DEBUG. (string -# value) -#logging_debug_format_suffix=%(funcName)s %(pathname)s:%(lineno)d - -# Prefix each line of exception output with this format. -# (string value) -#logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s - -# List of logger=LEVEL pairs. (list value) -#default_log_levels=amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN - -# Enables or disables publication of error events. (boolean -# value) -#publish_errors=false - -# Enables or disables fatal status of deprecations. (boolean -# value) -#fatal_deprecations=false - -# The format for an instance that is passed with the log -# message. (string value) -#instance_format="[instance: %(uuid)s] " - -# The format for an instance UUID that is passed with the log -# message. (string value) -#instance_uuid_format="[instance: %(uuid)s] " - -# The name of a logging configuration file. This file is -# appended to any existing logging configuration files. For -# details about logging configuration files, see the Python -# logging module documentation. (string value) # Deprecated group/name - [DEFAULT]/log_config -#log_config_append= +#log_config_append = -# DEPRECATED. A logging.Formatter log message format string -# which may use any of the available logging.LogRecord -# attributes. This option is deprecated. Please use -# logging_context_format_string and +# DEPRECATED. A logging.Formatter log message format string which may +# use any of the available logging.LogRecord attributes. This option +# is deprecated. Please use logging_context_format_string and # logging_default_format_string instead. (string value) -#log_format= +#log_format = -# Format string for %%(asctime)s in log records. Default: -# %(default)s . (string value) -#log_date_format=%Y-%m-%d %H:%M:%S +# Format string for %%(asctime)s in log records. Default: %(default)s +# . (string value) +#log_date_format = %Y-%m-%d %H:%M:%S -# (Optional) Name of log file to output to. If no default is -# set, logging will go to stdout. (string value) +# (Optional) Name of log file to output to. If no default is set, +# logging will go to stdout. (string value) # Deprecated group/name - [DEFAULT]/logfile -#log_file= +#log_file = -# (Optional) The base directory used for relative --log-file -# paths. (string value) +# (Optional) The base directory used for relative --log-file paths. +# (string value) # Deprecated group/name - [DEFAULT]/logdir -#log_dir= +#log_dir = -# Use syslog for logging. Existing syslog format is DEPRECATED -# during I, and will change in J to honor RFC5424. (boolean -# value) -#use_syslog=false +# Use syslog for logging. Existing syslog format is DEPRECATED during +# I, and will change in J to honor RFC5424. (boolean value) +#use_syslog = false -# (Optional) Enables or disables syslog rfc5424 format for -# logging. If enabled, prefixes the MSG part of the syslog -# message with APP-NAME (RFC5424). The format without the APP- -# NAME is deprecated in I, and will be removed in J. (boolean -# value) -#use_syslog_rfc_format=false +# (Optional) Enables or disables syslog rfc5424 format for logging. If +# enabled, prefixes the MSG part of the syslog message with APP-NAME +# (RFC5424). The format without the APP-NAME is deprecated in I, and +# will be removed in J. (boolean value) +#use_syslog_rfc_format = false # Syslog facility to receive log lines. (string value) -#syslog_log_facility=LOG_USER +#syslog_log_facility = LOG_USER +# Log output to standard error. (boolean value) +#use_stderr = true + +# Format string to use for log messages with context. (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages without context. (string +# value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Data to append to log format when level is DEBUG. (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. (string +# value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s + +# List of logger=LEVEL pairs. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + +# The format for an instance that is passed with the log message. +# (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. +# (string value) +#instance_uuid_format = "[instance: %(uuid)s] " # -# Options defined in rally.openstack.common.periodic_task +# From rally # -# Some periodic tasks can be run in a separate process. Should -# we run them here? (boolean value) -#run_external_periodic_tasks=true +# Print debugging output only for Rally. Off-site components stay +# quiet. (boolean value) +#rally_debug = false -# -# Options defined in rally.osclients -# +# make exception message format errors fatal (boolean value) +#fatal_exception_format_errors = false -# HTTP timeout for any of OpenStack service in seconds -#openstack_client_http_timeout=180.0 +# HTTP timeout for any of OpenStack service in seconds (floating point +# value) +#openstack_client_http_timeout = 180.0 -# Use SSL for all OpenStack API interfaces -#https_insecure=False +# Use SSL for all OpenStack API interfaces (boolean value) +#https_insecure = false + +# Path to CA server cetrificate for SSL (string value) +#https_cacert = -# Path to CA server cetrificate for SSL -#https_cacert= [benchmark] # -# Options defined in rally.benchmark.scenarios.cinder.utils +# From rally # -# Time to sleep after creating a resource before polling for -# it status (floating point value) -#cinder_volume_create_prepoll_delay=2.0 +# Time to sleep after creating a resource before polling for it status +# (floating point value) +#cinder_volume_create_prepoll_delay = 2.0 -# Time to wait for cinder volume to be created. (floating +# Time to wait for cinder volume to be created. (floating point value) +#cinder_volume_create_timeout = 600.0 + +# Interval between checks when waiting for volume creation. (floating # point value) -#cinder_volume_create_timeout=600.0 +#cinder_volume_create_poll_interval = 2.0 -# Interval between checks when waiting for volume creation. -# (floating point value) -#cinder_volume_create_poll_interval=2.0 +# Time to wait for cinder volume to be deleted. (floating point value) +#cinder_volume_delete_timeout = 600.0 -# Time to wait for cinder volume to be deleted. (floating +# Interval between checks when waiting for volume deletion. (floating # point value) -#cinder_volume_delete_timeout=600.0 +#cinder_volume_delete_poll_interval = 2.0 -# Interval between checks when waiting for volume deletion. +# Time to sleep after creating a resource before polling for it status # (floating point value) -#cinder_volume_delete_poll_interval=2.0 +#glance_image_create_prepoll_delay = 2.0 +# Time to wait for glance image to be created. (floating point value) +#glance_image_create_timeout = 120.0 -# -# Options defined in rally.benchmark.scenarios.glance.utils -# +# Interval between checks when waiting for image creation. (floating +# point value) +#glance_image_create_poll_interval = 1.0 -# Time to sleep after creating a resource before polling for -# it status (floating point value) -#glance_image_create_prepoll_delay=2.0 +# Time to wait for glance image to be deleted. (floating point value) +#glance_image_delete_timeout = 120.0 -# Time to wait for glance image to be created. (floating point +# Interval between checks when waiting for image deletion. (floating +# point value) +#glance_image_delete_poll_interval = 1.0 + +# Time to sleep after creating a resource before polling for it status +# (floating point value) +#heat_stack_create_prepoll_delay = 2.0 + +# Time to wait for heat stack to be created. (floating point value) +#heat_stack_create_timeout = 3600.0 + +# Interval between checks when waiting for stack creation. (floating +# point value) +#heat_stack_create_poll_interval = 1.0 + +# Time to wait for heat stack to be deleted. (floating point value) +#heat_stack_delete_timeout = 3600.0 + +# Interval between checks when waiting for stack deletion. (floating +# point value) +#heat_stack_delete_poll_interval = 1.0 + +# Time to sleep after updating a resource before polling for it status +# (floating point value) +#heat_stack_update_prepoll_delay = 2.0 + +# Time to wait for stack to be updated (floating point value) +#heat_stack_update_timeout = 3600.0 + +# Interval between checks when waiting for stack update. (floating +# point value) +#heat_stack_update_poll_interval = 1.0 + +# Time to sleep after start before polling for status (floating point # value) -#glance_image_create_timeout=120.0 - -# Interval between checks when waiting for image creation. -# (floating point value) -#glance_image_create_poll_interval=1.0 - -# Time to wait for glance image to be deleted. (floating point -# value) -#glance_image_delete_timeout=120.0 - -# Interval between checks when waiting for image deletion. -# (floating point value) -#glance_image_delete_poll_interval=1.0 - - -# -# Options defined in rally.benchmark.scenarios.heat.utils -# - -# Time to sleep after creating a resource before polling for -# it status (floating point value) -#heat_stack_create_prepoll_delay=2.0 - -# Time to wait for heat stack to be created. (floating point -# value) -#heat_stack_create_timeout=3600.0 - -# Interval between checks when waiting for stack creation. -# (floating point value) -#heat_stack_create_poll_interval=1.0 - -# Time to wait for heat stack to be deleted. (floating point -# value) -#heat_stack_delete_timeout=3600.0 - -# Interval between checks when waiting for stack deletion. -# (floating point value) -#heat_stack_delete_poll_interval=1.0 - - -# -# Options defined in rally.benchmark.scenarios.nova.utils -# - -# Time to sleep after start before polling for status -# (floating point value) -#nova_server_start_prepoll_delay=0.0 +#nova_server_start_prepoll_delay = 0.0 # Server start timeout (floating point value) -#nova_server_start_timeout=300.0 +#nova_server_start_timeout = 300.0 # Server start poll interval (floating point value) -#nova_server_start_poll_interval=1.0 +#nova_server_start_poll_interval = 1.0 -# Time to sleep after stop before polling for status (floating -# point value) -#nova_server_stop_prepoll_delay=0.0 +# Time to sleep after stop before polling for status (floating point +# value) +#nova_server_stop_prepoll_delay = 0.0 # Server stop timeout (floating point value) -#nova_server_stop_timeout=300.0 +#nova_server_stop_timeout = 300.0 # Server stop poll interval (floating point value) -#nova_server_stop_poll_interval=2.0 +#nova_server_stop_poll_interval = 2.0 -# Time to sleep after boot before polling for status (floating -# point value) -#nova_server_boot_prepoll_delay=1.0 +# Time to sleep after boot before polling for status (floating point +# value) +#nova_server_boot_prepoll_delay = 1.0 # Server boot timeout (floating point value) -#nova_server_boot_timeout=300.0 +#nova_server_boot_timeout = 300.0 # Server boot poll interval (floating point value) -#nova_server_boot_poll_interval=1.0 +#nova_server_boot_poll_interval = 1.0 -# Time to sleep after delete before polling for status -# (floating point value) -#nova_server_delete_prepoll_delay=2.0 +# Time to sleep after delete before polling for status (floating point +# value) +#nova_server_delete_prepoll_delay = 2.0 # Server delete timeout (floating point value) -#nova_server_delete_timeout=300.0 +#nova_server_delete_timeout = 300.0 # Server delete poll interval (floating point value) -#nova_server_delete_poll_interval=2.0 +#nova_server_delete_poll_interval = 2.0 -# Time to sleep after reboot before polling for status -# (floating point value) -#nova_server_reboot_prepoll_delay=2.0 +# Time to sleep after reboot before polling for status (floating point +# value) +#nova_server_reboot_prepoll_delay = 2.0 # Server reboot timeout (floating point value) -#nova_server_reboot_timeout=300.0 +#nova_server_reboot_timeout = 300.0 # Server reboot poll interval (floating point value) -#nova_server_reboot_poll_interval=2.0 +#nova_server_reboot_poll_interval = 2.0 -# Time to sleep after rescue before polling for status -# (floating point value) -#nova_server_rescue_prepoll_delay=2.0 +# Time to sleep after rescue before polling for status (floating point +# value) +#nova_server_rescue_prepoll_delay = 2.0 # Server rescue timeout (floating point value) -#nova_server_rescue_timeout=300.0 +#nova_server_rescue_timeout = 300.0 # Server rescue poll interval (floating point value) -#nova_server_rescue_poll_interval=2.0 +#nova_server_rescue_poll_interval = 2.0 -# Time to sleep after unrescue before polling for status -# (floating point value) -#nova_server_unrescue_prepoll_delay=2.0 +# Time to sleep after unrescue before polling for status (floating +# point value) +#nova_server_unrescue_prepoll_delay = 2.0 # Server unrescue timeout (floating point value) -#nova_server_unrescue_timeout=300.0 +#nova_server_unrescue_timeout = 300.0 # Server unrescue poll interval (floating point value) -#nova_server_unrescue_poll_interval=2.0 +#nova_server_unrescue_poll_interval = 2.0 -# Time to sleep after suspend before polling for status -# (floating point value) -#nova_server_suspend_prepoll_delay=2.0 +# Time to sleep after suspend before polling for status (floating +# point value) +#nova_server_suspend_prepoll_delay = 2.0 # Server suspend timeout (floating point value) -#nova_server_suspend_timeout=300.0 +#nova_server_suspend_timeout = 300.0 # Server suspend poll interval (floating point value) -#nova_server_suspend_poll_interval=2.0 +#nova_server_suspend_poll_interval = 2.0 -# Time to sleep after image_create before polling for status -# (floating point value) -#nova_server_image_create_prepoll_delay=0.0 +# Time to sleep after image_create before polling for status (floating +# point value) +#nova_server_image_create_prepoll_delay = 0.0 # Server image_create timeout (floating point value) -#nova_server_image_create_timeout=300.0 +#nova_server_image_create_timeout = 300.0 # Server image_create poll interval (floating point value) -#nova_server_image_create_poll_interval=2.0 +#nova_server_image_create_poll_interval = 2.0 -# Time to sleep after image_delete before polling for status -# (floating point value) -#nova_server_image_delete_prepoll_delay=0.0 +# Time to sleep after image_delete before polling for status (floating +# point value) +#nova_server_image_delete_prepoll_delay = 0.0 # Server image_delete timeout (floating point value) -#nova_server_image_delete_timeout=300.0 +#nova_server_image_delete_timeout = 300.0 # Server image_delete poll interval (floating point value) -#nova_server_image_delete_poll_interval=2.0 +#nova_server_image_delete_poll_interval = 2.0 -# Time to sleep after resize before polling for status -# (floating point value) -#nova_server_resize_prepoll_delay=2.0 +# Time to sleep after resize before polling for status (floating point +# value) +#nova_server_resize_prepoll_delay = 2.0 # Server resize timeout (floating point value) -#nova_server_resize_timeout=400.0 +#nova_server_resize_timeout = 400.0 # Server resize poll interval (floating point value) -#nova_server_resize_poll_interval=5.0 +#nova_server_resize_poll_interval = 5.0 # Time to sleep after resize_confirm before polling for status # (floating point value) -#nova_server_resize_confirm_prepoll_delay=0.0 +#nova_server_resize_confirm_prepoll_delay = 0.0 # Server resize_confirm timeout (floating point value) -#nova_server_resize_confirm_timeout=200.0 +#nova_server_resize_confirm_timeout = 200.0 # Server resize_confirm poll interval (floating point value) -#nova_server_resize_confirm_poll_interval=2.0 +#nova_server_resize_confirm_poll_interval = 2.0 # Time to sleep after resize_revert before polling for status # (floating point value) -#nova_server_resize_revert_prepoll_delay=0.0 +#nova_server_resize_revert_prepoll_delay = 0.0 # Server resize_revert timeout (floating point value) -#nova_server_resize_revert_timeout=200.0 +#nova_server_resize_revert_timeout = 200.0 # Server resize_revert poll interval (floating point value) -#nova_server_resize_revert_poll_interval=2.0 +#nova_server_resize_revert_poll_interval = 2.0 +# Time to sleep after live_migrate before polling for status (floating +# point value) +#nova_server_live_migrate_prepoll_delay = 1.0 -# -# Options defined in rally.benchmark.scenarios.sahara.utils -# +# Server live_migrate timeout (floating point value) +#nova_server_live_migrate_timeout = 400.0 -# A timeout in seconds for a cluster create operation (integer -# value) -#cluster_create_timeout=600 +# Server live_migrate poll interval (floating point value) +#nova_server_live_migrate_poll_interval = 2.0 + +# Time to sleep after migrate before polling for status (floating +# point value) +#nova_server_migrate_prepoll_delay = 1.0 + +# Server migrate timeout (floating point value) +#nova_server_migrate_timeout = 400.0 + +# Server migrate poll interval (floating point value) +#nova_server_migrate_poll_interval = 2.0 + +# A timeout in seconds for a cluster create operation (integer value) +#cluster_create_timeout = 600 # Cluster status polling interval in seconds (integer value) -#cluster_check_interval=5 +#cluster_check_interval = 5 -# A timeout in seconds for a cluster create operation (integer -# value) -#job_execution_timeout=600 +# A timeout in seconds for a cluster create operation (integer value) +#job_execution_timeout = 600 # Cluster status polling interval in seconds (integer value) -#job_check_interval=5 +#job_check_interval = 5 [database] # -# Options defined in oslo.db +# From oslo.db # # The file name to use with SQLite. (string value) -#sqlite_db=oslo.sqlite +# Deprecated group/name - [DEFAULT]/sqlite_db +#sqlite_db = oslo.sqlite # If True, SQLite uses synchronous mode. (boolean value) -#sqlite_synchronous=true +# Deprecated group/name - [DEFAULT]/sqlite_synchronous +#sqlite_synchronous = true # The back end to use for the database. (string value) # Deprecated group/name - [DEFAULT]/db_backend -#backend=sqlalchemy +#backend = sqlalchemy -# The SQLAlchemy connection string to use to connect to the -# database. (string value) +# The SQLAlchemy connection string to use to connect to the database. +# (string value) # Deprecated group/name - [DEFAULT]/sql_connection # Deprecated group/name - [DATABASE]/sql_connection # Deprecated group/name - [sql]/connection -#connection= +#connection = -# The SQLAlchemy connection string to use to connect to the -# slave database. (string value) -#slave_connection= +# The SQLAlchemy connection string to use to connect to the slave +# database. (string value) +#slave_connection = -# The SQL mode to be used for MySQL sessions. This option, -# including the default, overrides any server-set SQL mode. To -# use whatever SQL mode is set by the server configuration, -# set this to no value. Example: mysql_sql_mode= (string -# value) -#mysql_sql_mode=TRADITIONAL +# The SQL mode to be used for MySQL sessions. This option, including +# the default, overrides any server-set SQL mode. To use whatever SQL +# mode is set by the server configuration, set this to no value. +# Example: mysql_sql_mode= (string value) +#mysql_sql_mode = TRADITIONAL -# Timeout before idle SQL connections are reaped. (integer -# value) +# Timeout before idle SQL connections are reaped. (integer value) # Deprecated group/name - [DEFAULT]/sql_idle_timeout # Deprecated group/name - [DATABASE]/sql_idle_timeout # Deprecated group/name - [sql]/idle_timeout -#idle_timeout=3600 +#idle_timeout = 3600 -# Minimum number of SQL connections to keep open in a pool. -# (integer value) +# Minimum number of SQL connections to keep open in a pool. (integer +# value) # Deprecated group/name - [DEFAULT]/sql_min_pool_size # Deprecated group/name - [DATABASE]/sql_min_pool_size -#min_pool_size=1 +#min_pool_size = 1 -# Maximum number of SQL connections to keep open in a pool. -# (integer value) +# Maximum number of SQL connections to keep open in a pool. (integer +# value) # Deprecated group/name - [DEFAULT]/sql_max_pool_size # Deprecated group/name - [DATABASE]/sql_max_pool_size -#max_pool_size= +#max_pool_size = -# Maximum db connection retries during startup. Set to -1 to -# specify an infinite retry count. (integer value) +# Maximum number of database connection retries during startup. Set to +# -1 to specify an infinite retry count. (integer value) # Deprecated group/name - [DEFAULT]/sql_max_retries # Deprecated group/name - [DATABASE]/sql_max_retries -#max_retries=10 +#max_retries = 10 -# Interval between retries of opening a SQL connection. -# (integer value) +# Interval between retries of opening a SQL connection. (integer +# value) # Deprecated group/name - [DEFAULT]/sql_retry_interval # Deprecated group/name - [DATABASE]/reconnect_interval -#retry_interval=10 +#retry_interval = 10 -# If set, use this value for max_overflow with SQLAlchemy. -# (integer value) +# If set, use this value for max_overflow with SQLAlchemy. (integer +# value) # Deprecated group/name - [DEFAULT]/sql_max_overflow # Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow -#max_overflow= +#max_overflow = -# Verbosity of SQL debugging information: 0=None, -# 100=Everything. (integer value) -# Deprecated group/name - [DEFAULT]/sql_connection_debug -#connection_debug=0 - -# Add Python stack traces to SQL as comment strings. (boolean -# value) -# Deprecated group/name - [DEFAULT]/sql_connection_trace -#connection_trace=false - -# If set, use this value for pool_timeout with SQLAlchemy. +# Verbosity of SQL debugging information: 0=None, 100=Everything. # (integer value) -# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout -#pool_timeout= +# Deprecated group/name - [DEFAULT]/sql_connection_debug +#connection_debug = 0 -# Enable the experimental use of database reconnect on -# connection lost. (boolean value) -#use_db_reconnect=false +# Add Python stack traces to SQL as comment strings. (boolean value) +# Deprecated group/name - [DEFAULT]/sql_connection_trace +#connection_trace = false + +# If set, use this value for pool_timeout with SQLAlchemy. (integer +# value) +# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout +#pool_timeout = + +# Enable the experimental use of database reconnect on connection +# lost. (boolean value) +#use_db_reconnect = false # Seconds between database connection retries. (integer value) -#db_retry_interval=1 +#db_retry_interval = 1 -# If True, increases the interval between database connection -# retries up to db_max_retry_interval. (boolean value) -#db_inc_retry_interval=true +# If True, increases the interval between database connection retries +# up to db_max_retry_interval. (boolean value) +#db_inc_retry_interval = true # If db_inc_retry_interval is set, the maximum seconds between # database connection retries. (integer value) -#db_max_retry_interval=10 +#db_max_retry_interval = 10 -# Maximum database connection retries before error is raised. -# Set to -1 to specify an infinite retry count. (integer -# value) -#db_max_retries=20 +# Maximum database connection retries before error is raised. Set to +# -1 to specify an infinite retry count. (integer value) +#db_max_retries = 20 [image] # -# Options defined in rally.verification.verifiers.tempest.config +# From rally # # Version of cirros image (string value) -#cirros_version=0.3.2 +#cirros_version = 0.3.2 # Cirros image name (string value) -#cirros_image=cirros-0.3.2-x86_64-disk.img - - -[rest] - -# -# Options defined in rally.aas.rest -# - -# The port for the Rally API server (integer value) -#port=8877 - -# The listen IP for the Rally API server (string value) -#host=0.0.0.0 +#cirros_image = cirros-0.3.2-x86_64-disk.img [users_context] # -# Options defined in rally.benchmark.context.users +# From rally # -# How many concurrent threads use for serving users context -# (integer value) -#resource_management_workers=30 - -# ID of domain in which projects will be created. (string +# How many concurrent threads use for serving users context (integer # value) -#project_domain=default +#resource_management_workers = 30 + +# ID of domain in which projects will be created. (string value) +#project_domain = default # ID of domain in which users will be created. (string value) -#user_domain=default - - +#user_domain = default diff --git a/openstack-common.conf b/openstack-common.conf index 5aebd0956c..199a6ae044 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,10 +1,7 @@ [DEFAULT] # The list of modules to copy from oslo-incubator.git -module=cliutils -module=log -module=service -module=periodic_task + # The base module to hold the copy of openstack.common base=rally diff --git a/rally/aas/__init__.py b/rally/aas/__init__.py index e69de29bb2..82a80586b6 100644 --- a/rally/aas/__init__.py +++ b/rally/aas/__init__.py @@ -0,0 +1,2 @@ +# FIXME(andreykurilin): implement Rally-as-a-Service +pass diff --git a/rally/aas/rest/__init__.py b/rally/aas/rest/__init__.py deleted file mode 100644 index c700719567..0000000000 --- a/rally/aas/rest/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - - -REST_SERVICE_OPTS = [ - cfg.IntOpt("port", - default=8877, - help="The port for the Rally API server", - ), - cfg.StrOpt("host", - default="0.0.0.0", - help="The listen IP for the Rally API server", - ), -] -REST_OPT_GROUP = cfg.OptGroup(name="rest", - title="Options for the openstack-rally-api " - "service") - -CONF = cfg.CONF -CONF.register_group(REST_OPT_GROUP) -CONF.register_opts(REST_SERVICE_OPTS, REST_OPT_GROUP) diff --git a/rally/aas/rest/app.py b/rally/aas/rest/app.py deleted file mode 100644 index 7db89ea747..0000000000 --- a/rally/aas/rest/app.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pecan - -from rally.common import log as logging - - -def setup_app(config): - """Initialize Pecan application. - - This is a generic interface method of an application. - - :param config: An instance of :class:`pecan.Config`. - :returns: A normal WSGI application, an instance of - :class:`pecan.Pecan`. - """ - app = pecan.Pecan(config.app.root, debug=logging.is_debug()) - return app - - -def make_app(): - """Load Pecan application.""" - config = { - "app": { - "root": "rally.aas.rest.controllers.root.RootController", - "modules": ["rally.aas.rest"], - "debug": logging.is_debug(), - }, - "wsme": { - "debug": logging.is_debug(), - }, - } - app = pecan.load_app(config) - return app diff --git a/rally/aas/rest/controllers/__init__.py b/rally/aas/rest/controllers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/rally/aas/rest/controllers/root.py b/rally/aas/rest/controllers/root.py deleted file mode 100644 index 32985681e6..0000000000 --- a/rally/aas/rest/controllers/root.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from pecan import rest -from wsme import types as wtypes -import wsmeext.pecan as wsme_pecan - -from rally.aas.rest.controllers import v1 -from rally.aas.rest import types - - -class Root(wtypes.Base): - - name = wtypes.text - description = wtypes.text - versions = [types.Version] - - @classmethod - def convert(self, name, description, versions): - root = Root(name=name, description=description) - root.versions = [v.get()["result"] for v in versions] - return root - - -class RootController(rest.RestController): - - v1 = v1.Controller() - - @wsme_pecan.wsexpose(Root) - def get(self): - name = "OpenStack Rally API" - description = ("Rally is a Benchmark-as-a-Service project for " - "OpenStack.") - root = Root.convert(name, description, [self.v1]) - return root diff --git a/rally/aas/rest/controllers/v1/__init__.py b/rally/aas/rest/controllers/v1/__init__.py deleted file mode 100644 index 1dedae1938..0000000000 --- a/rally/aas/rest/controllers/v1/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from rally.aas.rest.controllers.v1.root import Controller # noqa diff --git a/rally/aas/rest/controllers/v1/root.py b/rally/aas/rest/controllers/v1/root.py deleted file mode 100644 index 3fd88695b1..0000000000 --- a/rally/aas/rest/controllers/v1/root.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from pecan import rest -import wsmeext.pecan as wsme_pecan - -from rally.aas.rest import types - - -class Version(types.Version): - @classmethod - def convert(cls): - v = super(Version, cls).convert("v1", "CURRENT", - updated_at="2014-01-07T00:00:00Z") - return v - - -class Controller(rest.RestController): - """Version 1 API Controller Root.""" - - @wsme_pecan.wsexpose(Version) - def get(self): - return Version.convert() diff --git a/rally/aas/rest/types.py b/rally/aas/rest/types.py deleted file mode 100644 index 07c049ba61..0000000000 --- a/rally/aas/rest/types.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pecan -from wsme import types as wtypes - - -class Link(wtypes.Base): - """A link representation.""" - - href = wtypes.text - rel = wtypes.text - type = wtypes.text - - @classmethod - def make_link(cls, rel, url, resource, type=wtypes.Unset): - href = "{url}/{resource}".format(url=url, resource=resource) - return cls(href=href, rel=rel, type=type) - - -class MediaType(wtypes.Base): - """A media type representation.""" - - base = wtypes.text - type = wtypes.text - - def __init__(self, base, type): - self.base = base - self.type = type - - -class Version(wtypes.Base): - """A version type representations.""" - - id = wtypes.text - status = wtypes.text - updated_at = wtypes.text - media_types = [MediaType] - links = [Link] - - @classmethod - def convert(cls, id, status, updated_at=None, media_types=None, - links=None): - v = Version(id=id, status=status, updated_at=updated_at) - if media_types is None: - mime_type = "application/vnd.openstack.rally.%s+json" % id - media_types = [MediaType("application/json", mime_type)] - v.media_types = media_types - if links is None: - links = [Link.make_link("self", pecan.request.host_url, id)] - v.links = links - return v diff --git a/rally/benchmark/context/users.py b/rally/benchmark/context/users.py index f79d21cd47..9506242191 100644 --- a/rally/benchmark/context/users.py +++ b/rally/benchmark/context/users.py @@ -33,7 +33,7 @@ from rally import osclients LOG = logging.getLogger(__name__) -context_opts = [ +USER_CONTEXT_OPTS = [ cfg.IntOpt("resource_management_workers", default=30, help="How many concurrent threads use for serving users " @@ -47,7 +47,7 @@ context_opts = [ ] CONF = cfg.CONF -CONF.register_opts(context_opts, +CONF.register_opts(USER_CONTEXT_OPTS, group=cfg.OptGroup(name="users_context", title="benchmark context options")) diff --git a/rally/benchmark/scenarios/cinder/utils.py b/rally/benchmark/scenarios/cinder/utils.py index df2429671d..e280dcf28f 100644 --- a/rally/benchmark/scenarios/cinder/utils.py +++ b/rally/benchmark/scenarios/cinder/utils.py @@ -22,7 +22,7 @@ from rally.benchmark.scenarios import base from rally.benchmark import utils as bench_utils -cinder_benchmark_opts = [ +CINDER_BENCHMARK_OPTS = [ cfg.FloatOpt("cinder_volume_create_prepoll_delay", default=2.0, help="Time to sleep after creating a resource before" @@ -45,7 +45,7 @@ cinder_benchmark_opts = [ CONF = cfg.CONF benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options") -CONF.register_opts(cinder_benchmark_opts, group=benchmark_group) +CONF.register_opts(CINDER_BENCHMARK_OPTS, group=benchmark_group) class CinderScenario(base.Scenario): diff --git a/rally/benchmark/scenarios/glance/utils.py b/rally/benchmark/scenarios/glance/utils.py index 4ad7293ee6..c054222200 100644 --- a/rally/benchmark/scenarios/glance/utils.py +++ b/rally/benchmark/scenarios/glance/utils.py @@ -22,7 +22,7 @@ from rally.benchmark.scenarios import base from rally.benchmark import utils as bench_utils -glance_benchmark_opts = [ +GLANCE_BENCHMARK_OPTS = [ cfg.FloatOpt("glance_image_create_prepoll_delay", default=2.0, help="Time to sleep after creating a resource before " @@ -46,7 +46,7 @@ glance_benchmark_opts = [ CONF = cfg.CONF benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options") -CONF.register_opts(glance_benchmark_opts, group=benchmark_group) +CONF.register_opts(GLANCE_BENCHMARK_OPTS, group=benchmark_group) class GlanceScenario(base.Scenario): diff --git a/rally/benchmark/scenarios/heat/utils.py b/rally/benchmark/scenarios/heat/utils.py index 29b731cb6f..15b99b53a1 100644 --- a/rally/benchmark/scenarios/heat/utils.py +++ b/rally/benchmark/scenarios/heat/utils.py @@ -21,7 +21,7 @@ from rally.benchmark.scenarios import base from rally.benchmark import utils as bench_utils -heat_benchmark_opts = [ +HEAT_BENCHMARK_OPTS = [ cfg.FloatOpt("heat_stack_create_prepoll_delay", default=2.0, help="Time to sleep after creating a resource before " @@ -56,7 +56,7 @@ heat_benchmark_opts = [ CONF = cfg.CONF benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options") -CONF.register_opts(heat_benchmark_opts, group=benchmark_group) +CONF.register_opts(HEAT_BENCHMARK_OPTS, group=benchmark_group) class HeatScenario(base.Scenario): @@ -148,4 +148,4 @@ class HeatScenario(base.Scenario): stack, update_resource=bench_utils.get_from_manager(), timeout=CONF.benchmark.heat_stack_delete_timeout, - check_interval=CONF.benchmark.heat_stack_delete_poll_interval) \ No newline at end of file + check_interval=CONF.benchmark.heat_stack_delete_poll_interval) diff --git a/rally/benchmark/scenarios/nova/utils.py b/rally/benchmark/scenarios/nova/utils.py index 921d64b867..8e532dbdb5 100644 --- a/rally/benchmark/scenarios/nova/utils.py +++ b/rally/benchmark/scenarios/nova/utils.py @@ -24,7 +24,7 @@ from rally.benchmark import utils as bench_utils from rally import exceptions -nova_benchmark_opts = [] +NOVA_BENCHMARK_OPTS = [] option_names_and_defaults = [ # action, prepoll delay, timeout, poll interval ("start", 0, 300, 1), @@ -45,7 +45,7 @@ option_names_and_defaults = [ ] for action, prepoll, timeout, poll in option_names_and_defaults: - nova_benchmark_opts.extend([ + NOVA_BENCHMARK_OPTS.extend([ cfg.FloatOpt( "nova_server_%s_prepoll_delay" % action, default=float(prepoll), @@ -67,7 +67,7 @@ CONF = cfg.CONF benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options") CONF.register_group(benchmark_group) -CONF.register_opts(nova_benchmark_opts, group=benchmark_group) +CONF.register_opts(NOVA_BENCHMARK_OPTS, group=benchmark_group) class NovaScenario(base.Scenario): diff --git a/rally/benchmark/scenarios/sahara/utils.py b/rally/benchmark/scenarios/sahara/utils.py index aade542887..4219943701 100644 --- a/rally/benchmark/scenarios/sahara/utils.py +++ b/rally/benchmark/scenarios/sahara/utils.py @@ -28,7 +28,7 @@ from rally import exceptions LOG = logging.getLogger(__name__) CONF = cfg.CONF -TIMEOUT_OPTS = [ +SAHARA_TIMEOUT_OPTS = [ cfg.IntOpt("cluster_create_timeout", default=600, help="A timeout in seconds for a cluster create operation"), cfg.IntOpt("cluster_check_interval", default=5, @@ -40,7 +40,7 @@ TIMEOUT_OPTS = [ ] benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options") -CONF.register_opts(TIMEOUT_OPTS, group=benchmark_group) +CONF.register_opts(SAHARA_TIMEOUT_OPTS, group=benchmark_group) class SaharaScenario(base.Scenario): diff --git a/rally/cmd/api.py b/rally/cmd/api.py deleted file mode 100644 index 1f5719dde4..0000000000 --- a/rally/cmd/api.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2014: Mirantis Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" The Rally Service API. """ - -import os -import sys -from wsgiref import simple_server - -from oslo_config import cfg - -from rally.aas.rest import app as rally_app -from rally.common.i18n import _ -from rally.common import log as logging - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -def main(): - # Initialize configuration and logging. - CONF(sys.argv[1:], project="rally") - logging.setup("rally") - # Prepare application and bind to the service socket. - host = CONF.rest.host - port = CONF.rest.port - app = rally_app.make_app() - server = simple_server.make_server(host, port, app) - # Start application. - LOG.info(_("Starting server in PID %s") % os.getpid()) - LOG.info(_("Configuration:")) - CONF.log_opt_values(LOG, logging.INFO) - try: - server.serve_forever() - except KeyboardInterrupt: - pass - - -if __name__ == "__main__": - main() diff --git a/rally/cmd/cliutils.py b/rally/cmd/cliutils.py index fa8014cf7f..5b2a243edc 100644 --- a/rally/cmd/cliutils.py +++ b/rally/cmd/cliutils.py @@ -23,6 +23,8 @@ import warnings import jsonschema from oslo_config import cfg +from oslo_utils import encodeutils +import prettytable import six from rally.common.i18n import _ @@ -30,7 +32,6 @@ from rally.common import log as logging from rally.common import utils from rally.common import version from rally import exceptions -from rally.openstack.common import cliutils CONF = cfg.CONF @@ -41,6 +42,95 @@ LOG = logging.getLogger(__name__) MARGIN = 3 +class MissingArgs(Exception): + """Supplied arguments are not sufficient for calling a function.""" + def __init__(self, missing): + self.missing = missing + msg = _("Missing arguments: %s") % ", ".join(missing) + super(MissingArgs, self).__init__(msg) + + +def validate_args(fn, *args, **kwargs): + """Check that the supplied args are sufficient for calling a function. + + >>> validate_args(lambda a: None) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): a + >>> validate_args(lambda a, b, c, d: None, 0, c=1) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): b, d + + :param fn: the function to check + :param arg: the positional arguments supplied + :param kwargs: the keyword arguments supplied + """ + argspec = inspect.getargspec(fn) + + num_defaults = len(argspec.defaults or []) + required_args = argspec.args[:len(argspec.args) - num_defaults] + + def isbound(method): + return getattr(method, "__self__", None) is not None + + if isbound(fn): + required_args.pop(0) + + missing = [arg for arg in required_args if arg not in kwargs] + missing = missing[len(args):] + if missing: + raise MissingArgs(missing) + + +def print_list(objs, fields, formatters=None, sortby_index=0, + mixed_case_fields=None, field_labels=None): + """Print a list or objects as a table, one row per object. + + :param objs: iterable of :class:`Resource` + :param fields: attributes that correspond to columns, in order + :param formatters: `dict` of callables for field formatting + :param sortby_index: index of the field for sorting table rows + :param mixed_case_fields: fields corresponding to object attributes that + have mixed case names (e.g., 'serverId') + :param field_labels: Labels to use in the heading of the table, default to + fields. + """ + formatters = formatters or {} + mixed_case_fields = mixed_case_fields or [] + field_labels = field_labels or fields + if len(field_labels) != len(fields): + raise ValueError(_("Field labels list %(labels)s has different number " + "of elements than fields list %(fields)s"), + {'labels': field_labels, 'fields': fields}) + + if sortby_index is None: + kwargs = {} + else: + kwargs = {"sortby": field_labels[sortby_index]} + pt = prettytable.PrettyTable(field_labels) + pt.align = "l" + + for o in objs: + row = [] + for field in fields: + if field in formatters: + row.append(formatters[field](o)) + else: + if field in mixed_case_fields: + field_name = field.replace(" ", "_") + else: + field_name = field.lower().replace(" ", "_") + data = getattr(o, field_name, "") + row.append(data) + pt.add_row(row) + + if six.PY3: + print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode()) + else: + print(encodeutils.safe_encode(pt.get_string(**kwargs))) + + def make_header(text, size=80, symbol="-"): """Unified way to make header message to CLI. @@ -302,8 +392,8 @@ def run(argv, categories): # call the action with the remaining arguments # check arguments try: - cliutils.validate_args(fn, *fn_args, **fn_kwargs) - except cliutils.MissingArgs as e: + validate_args(fn, *fn_args, **fn_kwargs) + except MissingArgs as e: # NOTE(mikal): this isn't the most helpful error message ever. It is # long, and tells you a lot of things you probably don't want to know # if you just got a single arg wrong. diff --git a/rally/cmd/commands/deployment.py b/rally/cmd/commands/deployment.py index bc01a49ee5..611128bfde 100644 --- a/rally/cmd/commands/deployment.py +++ b/rally/cmd/commands/deployment.py @@ -33,7 +33,6 @@ from rally.common import utils from rally import db from rally import exceptions from rally import objects -from rally.openstack.common import cliutils as common_cliutils from rally import osclients @@ -171,9 +170,8 @@ class DeploymentCommands(object): r = [str(t[column]) for column in headers[:-1]] r.append("" if t["uuid"] != current_deployment else "*") table_rows.append(utils.Struct(**dict(zip(headers, r)))) - common_cliutils.print_list(table_rows, headers, - sortby_index=headers.index( - "created_at")) + cliutils.print_list(table_rows, headers, + sortby_index=headers.index("created_at")) else: print(_("There are no deployments. " "To create a new deployment, use:" @@ -222,7 +220,7 @@ class DeploymentCommands(object): for ep in endpoints: data = [ep.get(m, "") for m in headers] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list(table_rows, headers) + cliutils.print_list(table_rows, headers) @cliutils.deprecated_args( "--uuid", dest="deployment", type=str, @@ -255,7 +253,7 @@ class DeploymentCommands(object): print(_("Authentication Issues: %s.") % sys.exc_info()[1]) return(1) - common_cliutils.print_list(table_rows, headers) + cliutils.print_list(table_rows, headers) @cliutils.args("--deployment", type=str, dest="deployment", help="UUID or name of the deployment") diff --git a/rally/cmd/commands/show.py b/rally/cmd/commands/show.py index 71b86632aa..fddcc5a0ed 100644 --- a/rally/cmd/commands/show.py +++ b/rally/cmd/commands/show.py @@ -24,7 +24,6 @@ from rally.common import utils from rally import db from rally import exceptions from rally import objects -from rally.openstack.common import cliutils as common_cliutils from rally import osclients @@ -70,10 +69,10 @@ class ShowCommands(object): data = [image.id, image.name, image.size] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list(table_rows, - fields=headers, - formatters=formatters, - mixed_case_fields=mixed_case_fields) + cliutils.print_list(table_rows, + fields=headers, + formatters=formatters, + mixed_case_fields=mixed_case_fields) except exceptions.InvalidArgumentsException as e: print(_("Authentication Issues: %s") % e) @@ -107,10 +106,10 @@ class ShowCommands(object): flavor.ram, flavor.swap, flavor.disk] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list(table_rows, - fields=headers, - formatters=formatters, - mixed_case_fields=mixed_case_fields) + cliutils.print_list(table_rows, + fields=headers, + formatters=formatters, + mixed_case_fields=mixed_case_fields) except exceptions.InvalidArgumentsException as e: print(_("Authentication Issues: %s") % e) @@ -136,9 +135,9 @@ class ShowCommands(object): data = [network.id, network.label, network.cidr] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list(table_rows, - fields=headers, - mixed_case_fields=mixed_case_fields) + cliutils.print_list(table_rows, + fields=headers, + mixed_case_fields=mixed_case_fields) except exceptions.InvalidArgumentsException as e: print(_("Authentication Issues: %s") % e) return(1) @@ -164,7 +163,7 @@ class ShowCommands(object): secgroup.description] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list( + cliutils.print_list( table_rows, fields=headers, mixed_case_fields=mixed_case_fields) @@ -192,9 +191,9 @@ class ShowCommands(object): for keypair in nova_client.keypairs.list(): data = [keypair.name, keypair.fingerprint] table_rows.append(utils.Struct(**dict(zip(headers, data)))) - common_cliutils.print_list(table_rows, - fields=headers, - mixed_case_fields=mixed_case_fields) + cliutils.print_list(table_rows, + fields=headers, + mixed_case_fields=mixed_case_fields) except exceptions.InvalidArgumentsException as e: print(_("Authentication Issues: %s") % e) diff --git a/rally/cmd/commands/task.py b/rally/cmd/commands/task.py index 3cf54d3e92..f022f588ad 100644 --- a/rally/cmd/commands/task.py +++ b/rally/cmd/commands/task.py @@ -39,7 +39,6 @@ from rally import consts from rally import db from rally import exceptions from rally import objects -from rally.openstack.common import cliutils as common_cliutils class FailedToLoadTask(exceptions.RallyException): @@ -287,10 +286,9 @@ class TaskCommands(object): if r["atomic_actions"]: for action in atomic_actions: dlist.append(r["atomic_actions"].get(action) or 0) - table_rows.append(rutils.Struct(**dict(zip(headers, dlist)))) - common_cliutils.print_list(table_rows, - fields=headers, - formatters=formatters) + cliutils.print_list(table_rows, + fields=headers, + formatters=formatters) print() task = db.task_get_detailed(task_id) @@ -355,8 +353,8 @@ class TaskCommands(object): "0.0%", len(raw)] table_rows.append(rutils.Struct(**dict(zip(table_cols, data)))) - common_cliutils.print_list(table_rows, fields=table_cols, - formatters=formatters) + cliutils.print_list(table_rows, fields=table_cols, + formatters=formatters) if iterations_data: _print_iterations_data(raw) @@ -396,9 +394,9 @@ class TaskCommands(object): row = [str(key)] + ["n/a"] * 5 table_rows.append(rutils.Struct(**dict(zip(headers, row)))) print("\nScenario Specific Results\n") - common_cliutils.print_list(table_rows, - fields=headers, - formatters=formatters) + cliutils.print_list(table_rows, + fields=headers, + formatters=formatters) for result in raw: errors = result["scenario_output"].get("errors") @@ -480,7 +478,7 @@ class TaskCommands(object): x["duration"] = x["updated_at"] - x["created_at"] if task_list: - common_cliutils.print_list( + cliutils.print_list( task_list, headers, sortby_index=headers.index("created_at")) else: @@ -620,8 +618,8 @@ class TaskCommands(object): if tojson: print(json.dumps(data)) else: - common_cliutils.print_list(data, ("benchmark", "pos", "criterion", - "status", "detail")) + cliutils.print_list(data, ("benchmark", "pos", "criterion", + "status", "detail")) return failed_criteria @cliutils.args("--task", type=str, dest="task", required=False, diff --git a/rally/cmd/commands/verify.py b/rally/cmd/commands/verify.py index 778b628ff2..902e6026aa 100644 --- a/rally/cmd/commands/verify.py +++ b/rally/cmd/commands/verify.py @@ -30,7 +30,6 @@ from rally import consts from rally import db from rally import exceptions from rally import objects -from rally.openstack.common import cliutils as common_cliutils from rally.verification.tempest import diff from rally.verification.tempest import json2html @@ -96,8 +95,8 @@ class VerifyCommands(object): el["duration"] = el["updated_at"] - el["created_at"] if verifications: - common_cliutils.print_list(verifications, fields, - sortby_index=fields.index("Created at")) + cliutils.print_list(verifications, fields, + sortby_index=fields.index("Created at")) else: print(_("There are no results from verifier. To run a verifier, " "use:\nrally verify start")) @@ -172,14 +171,14 @@ class VerifyCommands(object): print ("Total results of verification:\n") total_fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", "Created at", "Status"] - common_cliutils.print_list([verification], fields=total_fields) + cliutils.print_list([verification], fields=total_fields) print ("\nTests:\n") fields = ["name", "time", "status"] values = [objects.Verification(test) for test in six.itervalues(tests.data["test_cases"])] - common_cliutils.print_list(values, fields, sortby_index=sortby_index) + cliutils.print_list(values, fields, sortby_index=sortby_index) if detailed: for test in six.itervalues(tests.data["test_cases"]): diff --git a/rally/common/log.py b/rally/common/log.py index e4ecfc85f3..205e392185 100644 --- a/rally/common/log.py +++ b/rally/common/log.py @@ -16,17 +16,19 @@ import logging from oslo_config import cfg - -from rally.openstack.common import log as oslogging +from oslo_log import handlers +from oslo_log import log as oslogging -common_cli_opts = [cfg.BoolOpt("rally-debug", - default=False, - help="Print debugging output only for Rally. " - "Off-site components stay quiet.")] +DEBUG_OPTS = [cfg.BoolOpt( + "rally-debug", + default=False, + help="Print debugging output only for Rally. " + "Off-site components stay quiet.")] CONF = cfg.CONF -CONF.register_cli_opts(common_cli_opts) +CONF.register_cli_opts(DEBUG_OPTS) +oslogging.register_options(CONF) logging.RDEBUG = logging.DEBUG + 1 logging.addLevelName(logging.RDEBUG, "RALLYDEBUG") @@ -43,25 +45,26 @@ WARNING = logging.WARNING def setup(product_name, version="unknown"): - dbg_color = oslogging.ColorHandler.LEVEL_COLORS[logging.DEBUG] - oslogging.ColorHandler.LEVEL_COLORS[logging.RDEBUG] = dbg_color + dbg_color = handlers.ColorHandler.LEVEL_COLORS[logging.DEBUG] + handlers.ColorHandler.LEVEL_COLORS[logging.RDEBUG] = dbg_color - oslogging.setup(product_name, version) + oslogging.setup(CONF, product_name, version) if CONF.rally_debug: - oslogging.getLogger(None).logger.setLevel(logging.RDEBUG) + oslogging.getLogger( + project=product_name).logger.setLevel(logging.RDEBUG) def getLogger(name="unknown", version="unknown"): if name not in oslogging._loggers: oslogging._loggers[name] = RallyContextAdapter(logging.getLogger(name), - name, - version) + {"project": "rally", + "version": version}) return oslogging._loggers[name] -class RallyContextAdapter(oslogging.ContextAdapter): +class RallyContextAdapter(oslogging.KeywordArgumentAdapter): def debug(self, msg, *args, **kwargs): self.log(logging.RDEBUG, msg, *args, **kwargs) diff --git a/rally/common/opts.py b/rally/common/opts.py new file mode 100644 index 0000000000..1532ebc4c7 --- /dev/null +++ b/rally/common/opts.py @@ -0,0 +1,44 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from rally.benchmark.context import users +from rally.benchmark.scenarios.cinder import utils as cinder_utils +from rally.benchmark.scenarios.glance import utils as glance_utils +from rally.benchmark.scenarios.heat import utils as heat_utils +from rally.benchmark.scenarios.nova import utils as nova_utils +from rally.benchmark.scenarios.sahara import utils as sahara_utils +from rally.common import log +from rally import exceptions +from rally import osclients +from rally.verification.tempest import config as tempest_conf + + +def list_opts(): + return [ + ("DEFAULT", + itertools.chain(log.DEBUG_OPTS, + exceptions.EXC_LOG_OPTS, + osclients.OSCLIENTS_OPTS)), + ("benchmark", + itertools.chain(cinder_utils.CINDER_BENCHMARK_OPTS, + glance_utils.GLANCE_BENCHMARK_OPTS, + heat_utils.HEAT_BENCHMARK_OPTS, + nova_utils.NOVA_BENCHMARK_OPTS, + sahara_utils.SAHARA_TIMEOUT_OPTS)), + ("image", + itertools.chain(tempest_conf.IMAGE_OPTS)), + ("users_context", itertools.chain(users.USER_CONTEXT_OPTS)) + ] diff --git a/rally/exceptions.py b/rally/exceptions.py index 5a59013dc5..0641b669e0 100644 --- a/rally/exceptions.py +++ b/rally/exceptions.py @@ -23,14 +23,14 @@ from rally.common import log as logging LOG = logging.getLogger(__name__) -exc_log_opts = [ +EXC_LOG_OPTS = [ cfg.BoolOpt("fatal_exception_format_errors", default=False, help="make exception message format errors fatal"), ] CONF = cfg.CONF -CONF.register_opts(exc_log_opts) +CONF.register_opts(EXC_LOG_OPTS) class RallyException(Exception): diff --git a/rally/openstack/__init__.py b/rally/openstack/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/rally/openstack/common/__init__.py b/rally/openstack/common/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/rally/openstack/common/_i18n.py b/rally/openstack/common/_i18n.py deleted file mode 100644 index 109f8bcaf7..0000000000 --- a/rally/openstack/common/_i18n.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""oslo.i18n integration module. - -See http://docs.openstack.org/developer/oslo.i18n/usage.html - -""" - -import oslo.i18n - - -# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the -# application name when this module is synced into the separate -# repository. It is OK to have more than one translation function -# using the same domain, since there will still only be one message -# catalog. -_translators = oslo.i18n.TranslatorFactory(domain='rally') - -# The primary translation function using the well-known name "_" -_ = _translators.primary - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = _translators.log_info -_LW = _translators.log_warning -_LE = _translators.log_error -_LC = _translators.log_critical diff --git a/rally/openstack/common/cliutils.py b/rally/openstack/common/cliutils.py deleted file mode 100644 index 5968a38a27..0000000000 --- a/rally/openstack/common/cliutils.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# W0603: Using the global statement -# W0621: Redefining name %s from outer scope -# pylint: disable=W0603,W0621 - -from __future__ import print_function - -import getpass -import inspect -import os -import sys -import textwrap - -from oslo.utils import encodeutils -from oslo.utils import strutils -import prettytable -import six -from six import moves - -from rally.openstack.common._i18n import _ - - -class MissingArgs(Exception): - """Supplied arguments are not sufficient for calling a function.""" - def __init__(self, missing): - self.missing = missing - msg = _("Missing arguments: %s") % ", ".join(missing) - super(MissingArgs, self).__init__(msg) - - -def validate_args(fn, *args, **kwargs): - """Check that the supplied args are sufficient for calling a function. - - >>> validate_args(lambda a: None) - Traceback (most recent call last): - ... - MissingArgs: Missing argument(s): a - >>> validate_args(lambda a, b, c, d: None, 0, c=1) - Traceback (most recent call last): - ... - MissingArgs: Missing argument(s): b, d - - :param fn: the function to check - :param arg: the positional arguments supplied - :param kwargs: the keyword arguments supplied - """ - argspec = inspect.getargspec(fn) - - num_defaults = len(argspec.defaults or []) - required_args = argspec.args[:len(argspec.args) - num_defaults] - - def isbound(method): - return getattr(method, '__self__', None) is not None - - if isbound(fn): - required_args.pop(0) - - missing = [arg for arg in required_args if arg not in kwargs] - missing = missing[len(args):] - if missing: - raise MissingArgs(missing) - - -def arg(*args, **kwargs): - """Decorator for CLI args. - - Example: - - >>> @arg("name", help="Name of the new entity") - ... def entity_create(args): - ... pass - """ - def _decorator(func): - add_arg(func, *args, **kwargs) - return func - return _decorator - - -def env(*args, **kwargs): - """Returns the first environment variable set. - - If all are empty, defaults to '' or keyword arg `default`. - """ - for arg in args: - value = os.environ.get(arg) - if value: - return value - return kwargs.get('default', '') - - -def add_arg(func, *args, **kwargs): - """Bind CLI arguments to a shell.py `do_foo` function.""" - - if not hasattr(func, 'arguments'): - func.arguments = [] - - # NOTE(sirp): avoid dups that can occur when the module is shared across - # tests. - if (args, kwargs) not in func.arguments: - # Because of the semantics of decorator composition if we just append - # to the options list positional options will appear to be backwards. - func.arguments.insert(0, (args, kwargs)) - - -def unauthenticated(func): - """Adds 'unauthenticated' attribute to decorated function. - - Usage: - - >>> @unauthenticated - ... def mymethod(f): - ... pass - """ - func.unauthenticated = True - return func - - -def isunauthenticated(func): - """Checks if the function does not require authentication. - - Mark such functions with the `@unauthenticated` decorator. - - :returns: bool - """ - return getattr(func, 'unauthenticated', False) - - -def print_list(objs, fields, formatters=None, sortby_index=0, - mixed_case_fields=None, field_labels=None): - """Print a list or objects as a table, one row per object. - - :param objs: iterable of :class:`Resource` - :param fields: attributes that correspond to columns, in order - :param formatters: `dict` of callables for field formatting - :param sortby_index: index of the field for sorting table rows - :param mixed_case_fields: fields corresponding to object attributes that - have mixed case names (e.g., 'serverId') - :param field_labels: Labels to use in the heading of the table, default to - fields. - """ - formatters = formatters or {} - mixed_case_fields = mixed_case_fields or [] - field_labels = field_labels or fields - if len(field_labels) != len(fields): - raise ValueError(_("Field labels list %(labels)s has different number " - "of elements than fields list %(fields)s"), - {'labels': field_labels, 'fields': fields}) - - if sortby_index is None: - kwargs = {} - else: - kwargs = {'sortby': field_labels[sortby_index]} - pt = prettytable.PrettyTable(field_labels) - pt.align = 'l' - - for o in objs: - row = [] - for field in fields: - if field in formatters: - row.append(formatters[field](o)) - else: - if field in mixed_case_fields: - field_name = field.replace(' ', '_') - else: - field_name = field.lower().replace(' ', '_') - data = getattr(o, field_name, '') - row.append(data) - pt.add_row(row) - - if six.PY3: - print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode()) - else: - print(encodeutils.safe_encode(pt.get_string(**kwargs))) - - -def print_dict(dct, dict_property="Property", wrap=0): - """Print a `dict` as a table of two columns. - - :param dct: `dict` to print - :param dict_property: name of the first column - :param wrap: wrapping for the second column - """ - pt = prettytable.PrettyTable([dict_property, 'Value']) - pt.align = 'l' - for k, v in six.iteritems(dct): - # convert dict to str to check length - if isinstance(v, dict): - v = six.text_type(v) - if wrap > 0: - v = textwrap.fill(six.text_type(v), wrap) - # if value has a newline, add in multiple rows - # e.g. fault with stacktrace - if v and isinstance(v, six.string_types) and r'\n' in v: - lines = v.strip().split(r'\n') - col1 = k - for line in lines: - pt.add_row([col1, line]) - col1 = '' - else: - pt.add_row([k, v]) - - if six.PY3: - print(encodeutils.safe_encode(pt.get_string()).decode()) - else: - print(encodeutils.safe_encode(pt.get_string())) - - -def get_password(max_password_prompts=3): - """Read password from TTY.""" - verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD")) - pw = None - if hasattr(sys.stdin, "isatty") and sys.stdin.isatty(): - # Check for Ctrl-D - try: - for __ in moves.range(max_password_prompts): - pw1 = getpass.getpass("OS Password: ") - if verify: - pw2 = getpass.getpass("Please verify: ") - else: - pw2 = pw1 - if pw1 == pw2 and pw1: - pw = pw1 - break - except EOFError: - pass - return pw - - -def service_type(stype): - """Adds 'service_type' attribute to decorated function. - - Usage: - - .. code-block:: python - - @service_type('volume') - def mymethod(f): - ... - """ - def inner(f): - f.service_type = stype - return f - return inner - - -def get_service_type(f): - """Retrieves service type from function.""" - return getattr(f, 'service_type', None) - - -def pretty_choice_list(l): - return ', '.join("'%s'" % i for i in l) - - -def exit(msg=''): - if msg: - print (msg, file=sys.stderr) - sys.exit(1) diff --git a/rally/openstack/common/eventlet_backdoor.py b/rally/openstack/common/eventlet_backdoor.py deleted file mode 100644 index 4f134f27db..0000000000 --- a/rally/openstack/common/eventlet_backdoor.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) 2012 OpenStack Foundation. -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import print_function - -import errno -import gc -import os -import pprint -import socket -import sys -import traceback - -import eventlet -import eventlet.backdoor -import greenlet -from oslo.config import cfg - -from rally.openstack.common._i18n import _LI -from rally.openstack.common import log as logging - -help_for_backdoor_port = ( - "Acceptable values are 0, , and :, where 0 results " - "in listening on a random tcp port number; results in listening " - "on the specified port number (and not enabling backdoor if that port " - "is in use); and : results in listening on the smallest " - "unused port number within the specified range of port numbers. The " - "chosen port is displayed in the service's log file.") -eventlet_backdoor_opts = [ - cfg.StrOpt('backdoor_port', - help="Enable eventlet backdoor. %s" % help_for_backdoor_port) -] - -CONF = cfg.CONF -CONF.register_opts(eventlet_backdoor_opts) -LOG = logging.getLogger(__name__) - - -class EventletBackdoorConfigValueError(Exception): - def __init__(self, port_range, help_msg, ex): - msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. ' - '%(help)s' % - {'range': port_range, 'ex': ex, 'help': help_msg}) - super(EventletBackdoorConfigValueError, self).__init__(msg) - self.port_range = port_range - - -def _dont_use_this(): - print("Don't use this, just disconnect instead") - - -def _find_objects(t): - return [o for o in gc.get_objects() if isinstance(o, t)] - - -def _print_greenthreads(): - for i, gt in enumerate(_find_objects(greenlet.greenlet)): - print(i, gt) - traceback.print_stack(gt.gr_frame) - print() - - -def _print_nativethreads(): - for threadId, stack in sys._current_frames().items(): - print(threadId) - traceback.print_stack(stack) - print() - - -def _parse_port_range(port_range): - if ':' not in port_range: - start, end = port_range, port_range - else: - start, end = port_range.split(':', 1) - try: - start, end = int(start), int(end) - if end < start: - raise ValueError - return start, end - except ValueError as ex: - raise EventletBackdoorConfigValueError(port_range, ex, - help_for_backdoor_port) - - -def _listen(host, start_port, end_port, listen_func): - try_port = start_port - while True: - try: - return listen_func((host, try_port)) - except socket.error as exc: - if (exc.errno != errno.EADDRINUSE or - try_port >= end_port): - raise - try_port += 1 - - -def initialize_if_enabled(): - backdoor_locals = { - 'exit': _dont_use_this, # So we don't exit the entire process - 'quit': _dont_use_this, # So we don't exit the entire process - 'fo': _find_objects, - 'pgt': _print_greenthreads, - 'pnt': _print_nativethreads, - } - - if CONF.backdoor_port is None: - return None - - start_port, end_port = _parse_port_range(str(CONF.backdoor_port)) - - # NOTE(johannes): The standard sys.displayhook will print the value of - # the last expression and set it to __builtin__._, which overwrites - # the __builtin__._ that gettext sets. Let's switch to using pprint - # since it won't interact poorly with gettext, and it's easier to - # read the output too. - def displayhook(val): - if val is not None: - pprint.pprint(val) - sys.displayhook = displayhook - - sock = _listen('localhost', start_port, end_port, eventlet.listen) - - # In the case of backdoor port being zero, a port number is assigned by - # listen(). In any case, pull the port number out here. - port = sock.getsockname()[1] - LOG.info( - _LI('Eventlet backdoor listening on %(port)s for process %(pid)d') % - {'port': port, 'pid': os.getpid()} - ) - eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, - locals=backdoor_locals) - return port diff --git a/rally/openstack/common/fileutils.py b/rally/openstack/common/fileutils.py deleted file mode 100644 index ec26eaf9cd..0000000000 --- a/rally/openstack/common/fileutils.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import logging -import os -import tempfile - -from oslo.utils import excutils - -LOG = logging.getLogger(__name__) - -_FILE_CACHE = {} - - -def ensure_tree(path): - """Create a directory (and any ancestor directories required) - - :param path: Directory to create - """ - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST: - if not os.path.isdir(path): - raise - else: - raise - - -def read_cached_file(filename, force_reload=False): - """Read from a file if it has been modified. - - :param force_reload: Whether to reload the file. - :returns: A tuple with a boolean specifying if the data is fresh - or not. - """ - global _FILE_CACHE - - if force_reload: - delete_cached_file(filename) - - reloaded = False - mtime = os.path.getmtime(filename) - cache_info = _FILE_CACHE.setdefault(filename, {}) - - if not cache_info or mtime > cache_info.get('mtime', 0): - LOG.debug("Reloading cached file %s" % filename) - with open(filename) as fap: - cache_info['data'] = fap.read() - cache_info['mtime'] = mtime - reloaded = True - return (reloaded, cache_info['data']) - - -def delete_cached_file(filename): - """Delete cached file if present. - - :param filename: filename to delete - """ - global _FILE_CACHE - - if filename in _FILE_CACHE: - del _FILE_CACHE[filename] - - -def delete_if_exists(path, remove=os.unlink): - """Delete a file, but ignore file not found error. - - :param path: File to delete - :param remove: Optional function to remove passed path - """ - - try: - remove(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - - -@contextlib.contextmanager -def remove_path_on_error(path, remove=delete_if_exists): - """Protect code that wants to operate on PATH atomically. - Any exception will cause PATH to be removed. - - :param path: File to work with - :param remove: Optional function to remove passed path - """ - - try: - yield - except Exception: - with excutils.save_and_reraise_exception(): - remove(path) - - -def file_open(*args, **kwargs): - """Open file - - see built-in open() documentation for more details - - Note: The reason this is kept in a separate module is to easily - be able to provide a stub module that doesn't alter system - state at all (for unit tests) - """ - return open(*args, **kwargs) - - -def write_to_tempfile(content, path=None, suffix='', prefix='tmp'): - """Create temporary file or use existing file. - - This util is needed for creating temporary file with - specified content, suffix and prefix. If path is not None, - it will be used for writing content. If the path doesn't - exist it'll be created. - - :param content: content for temporary file. - :param path: same as parameter 'dir' for mkstemp - :param suffix: same as parameter 'suffix' for mkstemp - :param prefix: same as parameter 'prefix' for mkstemp - - For example: it can be used in database tests for creating - configuration files. - """ - if path: - ensure_tree(path) - - (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) - try: - os.write(fd, content) - finally: - os.close(fd) - return path diff --git a/rally/openstack/common/local.py b/rally/openstack/common/local.py deleted file mode 100644 index 0819d5b97c..0000000000 --- a/rally/openstack/common/local.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Local storage of variables using weak references""" - -import threading -import weakref - - -class WeakLocal(threading.local): - def __getattribute__(self, attr): - rval = super(WeakLocal, self).__getattribute__(attr) - if rval: - # NOTE(mikal): this bit is confusing. What is stored is a weak - # reference, not the value itself. We therefore need to lookup - # the weak reference and return the inner value here. - rval = rval() - return rval - - def __setattr__(self, attr, value): - value = weakref.ref(value) - return super(WeakLocal, self).__setattr__(attr, value) - - -# NOTE(mikal): the name "store" should be deprecated in the future -store = WeakLocal() - -# A "weak" store uses weak references and allows an object to fall out of scope -# when it falls out of scope in the code that uses the thread local storage. A -# "strong" store will hold a reference to the object so that it never falls out -# of scope. -weak_store = WeakLocal() -strong_store = threading.local() diff --git a/rally/openstack/common/lockutils.py b/rally/openstack/common/lockutils.py deleted file mode 100644 index e72bb827b9..0000000000 --- a/rally/openstack/common/lockutils.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import functools -import logging -import os -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import weakref - -from oslo.config import cfg - -from rally.openstack.common import fileutils -from rally.openstack.common._i18n import _, _LE, _LI - - -LOG = logging.getLogger(__name__) - - -util_opts = [ - cfg.BoolOpt('disable_process_locking', default=False, - help='Enables or disables inter-process locks.'), - cfg.StrOpt('lock_path', - default=os.environ.get("RALLY_LOCK_PATH"), - help='Directory to use for lock files.') -] - - -CONF = cfg.CONF -CONF.register_opts(util_opts) - - -def set_defaults(lock_path): - cfg.set_defaults(util_opts, lock_path=lock_path) - - -class _FileLock(object): - """Lock implementation which allows multiple locks, working around - issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does - not require any cleanup. Since the lock is always held on a file - descriptor rather than outside of the process, the lock gets dropped - automatically if the process crashes, even if __exit__ is not executed. - - There are no guarantees regarding usage by multiple green threads in a - single process here. This lock works only between processes. Exclusive - access between local threads should be achieved using the semaphores - in the @synchronized decorator. - - Note these locks are released when the descriptor is closed, so it's not - safe to close the file descriptor while another green thread holds the - lock. Just opening and closing the lock file can break synchronisation, - so lock files must be accessed only using this abstraction. - """ - - def __init__(self, name): - self.lockfile = None - self.fname = name - - def acquire(self): - basedir = os.path.dirname(self.fname) - - if not os.path.exists(basedir): - fileutils.ensure_tree(basedir) - LOG.info(_LI('Created lock path: %s'), basedir) - - self.lockfile = open(self.fname, 'w') - - while True: - try: - # Using non-blocking locks since green threads are not - # patched to deal with blocking locking calls. - # Also upon reading the MSDN docs for locking(), it seems - # to have a laughable 10 attempts "blocking" mechanism. - self.trylock() - LOG.debug('Got file lock "%s"', self.fname) - return True - except IOError as e: - if e.errno in (errno.EACCES, errno.EAGAIN): - # external locks synchronise things like iptables - # updates - give it some time to prevent busy spinning - time.sleep(0.01) - else: - raise threading.ThreadError(_("Unable to acquire lock on" - " `%(filename)s` due to" - " %(exception)s") % - {'filename': self.fname, - 'exception': e}) - - def __enter__(self): - self.acquire() - return self - - def release(self): - try: - self.unlock() - self.lockfile.close() - LOG.debug('Released file lock "%s"', self.fname) - except IOError: - LOG.exception(_LE("Could not release the acquired lock `%s`"), - self.fname) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - return os.path.exists(self.fname) - - def trylock(self): - raise NotImplementedError() - - def unlock(self): - raise NotImplementedError() - - -class _WindowsLock(_FileLock): - def trylock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) - - def unlock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) - - -class _FcntlLock(_FileLock): - def trylock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - def unlock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_UN) - - -if os.name == 'nt': - import msvcrt - InterProcessLock = _WindowsLock -else: - import fcntl - InterProcessLock = _FcntlLock - -_semaphores = weakref.WeakValueDictionary() -_semaphores_lock = threading.Lock() - - -def _get_lock_path(name, lock_file_prefix, lock_path=None): - # NOTE(mikal): the lock name cannot contain directory - # separators - name = name.replace(os.sep, '_') - if lock_file_prefix: - sep = '' if lock_file_prefix.endswith('-') else '-' - name = '%s%s%s' % (lock_file_prefix, sep, name) - - local_lock_path = lock_path or CONF.lock_path - - if not local_lock_path: - raise cfg.RequiredOptError('lock_path') - - return os.path.join(local_lock_path, name) - - -def external_lock(name, lock_file_prefix=None, lock_path=None): - LOG.debug('Attempting to grab external lock "%(lock)s"', - {'lock': name}) - - lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) - - return InterProcessLock(lock_file_path) - - -def remove_external_lock_file(name, lock_file_prefix=None): - """Remove an external lock file when it's not used anymore - This will be helpful when we have a lot of lock files - """ - with internal_lock(name): - lock_file_path = _get_lock_path(name, lock_file_prefix) - try: - os.remove(lock_file_path) - except OSError: - LOG.info(_LI('Failed to remove file %(file)s'), - {'file': lock_file_path}) - - -def internal_lock(name): - with _semaphores_lock: - try: - sem = _semaphores[name] - LOG.debug('Using existing semaphore "%s"', name) - except KeyError: - sem = threading.Semaphore() - _semaphores[name] = sem - LOG.debug('Created new semaphore "%s"', name) - - return sem - - -@contextlib.contextmanager -def lock(name, lock_file_prefix=None, external=False, lock_path=None): - """Context based lock - - This function yields a `threading.Semaphore` instance (if we don't use - eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is - True, in which case, it'll yield an InterProcessLock instance. - - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. - - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. - """ - int_lock = internal_lock(name) - with int_lock: - LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name}) - try: - if external and not CONF.disable_process_locking: - ext_lock = external_lock(name, lock_file_prefix, lock_path) - with ext_lock: - yield ext_lock - else: - yield int_lock - finally: - LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name}) - - -def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): - """Synchronization decorator. - - Decorating a method like so:: - - @synchronized('mylock') - def foo(self, *args): - ... - - ensures that only one thread will execute the foo method at a time. - - Different methods can share the same lock:: - - @synchronized('mylock') - def foo(self, *args): - ... - - @synchronized('mylock') - def bar(self, *args): - ... - - This way only one of either foo or bar can be executing at a time. - """ - - def wrap(f): - @functools.wraps(f) - def inner(*args, **kwargs): - try: - with lock(name, lock_file_prefix, external, lock_path): - LOG.debug('Got semaphore / lock "%(function)s"', - {'function': f.__name__}) - return f(*args, **kwargs) - finally: - LOG.debug('Semaphore / lock released "%(function)s"', - {'function': f.__name__}) - return inner - return wrap - - -def synchronized_with_prefix(lock_file_prefix): - """Partial object generator for the synchronization decorator. - - Redefine @synchronized in each project like so:: - - (in nova/utils.py) - from nova.openstack.common import lockutils - - synchronized = lockutils.synchronized_with_prefix('nova-') - - - (in nova/foo.py) - from nova import utils - - @utils.synchronized('mylock') - def bar(self, *args): - ... - - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. - """ - - return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) - - -def main(argv): - """Create a dir for locks and pass it to command from arguments - - If you run this: - python -m openstack.common.lockutils python setup.py testr - - a temporary directory will be created for all your locks and passed to all - your tests in an environment variable. The temporary dir will be deleted - afterwards and the return value will be preserved. - """ - - lock_dir = tempfile.mkdtemp() - os.environ["RALLY_LOCK_PATH"] = lock_dir - try: - ret_val = subprocess.call(argv[1:]) - finally: - shutil.rmtree(lock_dir, ignore_errors=True) - return ret_val - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/rally/openstack/common/log.py b/rally/openstack/common/log.py deleted file mode 100644 index 21e03582a9..0000000000 --- a/rally/openstack/common/log.py +++ /dev/null @@ -1,705 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""OpenStack logging handler. - -This module adds to logging functionality by adding the option to specify -a context object when calling the various log methods. If the context object -is not specified, default formatting is used. Additionally, an instance uuid -may be passed as part of the log message, which is intended to make it easier -for admins to find messages related to a specific instance. - -It also allows setting of formatting information through conf. - -""" - -import inspect -import itertools -import logging -import logging.config -import logging.handlers -import os -import socket -import sys -import traceback - -from oslo.config import cfg -from oslo.serialization import jsonutils -from oslo.utils import importutils -import six -from six import moves - -_PY26 = sys.version_info[0:2] == (2, 6) - -from rally.openstack.common._i18n import _ -from rally.openstack.common import local - - -_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" - - -common_cli_opts = [ - cfg.BoolOpt('debug', - short='d', - default=False, - help='Print debugging output (set logging level to ' - 'DEBUG instead of default WARNING level).'), - cfg.BoolOpt('verbose', - short='v', - default=False, - help='Print more verbose output (set logging level to ' - 'INFO instead of default WARNING level).'), -] - -logging_cli_opts = [ - cfg.StrOpt('log-config-append', - metavar='PATH', - deprecated_name='log-config', - help='The name of a logging configuration file. This file ' - 'is appended to any existing logging configuration ' - 'files. For details about logging configuration files, ' - 'see the Python logging module documentation.'), - cfg.StrOpt('log-format', - metavar='FORMAT', - help='DEPRECATED. ' - 'A logging.Formatter log message format string which may ' - 'use any of the available logging.LogRecord attributes. ' - 'This option is deprecated. Please use ' - 'logging_context_format_string and ' - 'logging_default_format_string instead.'), - cfg.StrOpt('log-date-format', - default=_DEFAULT_LOG_DATE_FORMAT, - metavar='DATE_FORMAT', - help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s .'), - cfg.StrOpt('log-file', - metavar='PATH', - deprecated_name='logfile', - help='(Optional) Name of log file to output to. ' - 'If no default is set, logging will go to stdout.'), - cfg.StrOpt('log-dir', - deprecated_name='logdir', - help='(Optional) The base directory used for relative ' - '--log-file paths.'), - cfg.BoolOpt('use-syslog', - default=False, - help='Use syslog for logging. ' - 'Existing syslog format is DEPRECATED during I, ' - 'and will change in J to honor RFC5424.'), - cfg.BoolOpt('use-syslog-rfc-format', - # TODO(bogdando) remove or use True after existing - # syslog format deprecation in J - default=False, - help='(Optional) Enables or disables syslog rfc5424 format ' - 'for logging. If enabled, prefixes the MSG part of the ' - 'syslog message with APP-NAME (RFC5424). The ' - 'format without the APP-NAME is deprecated in I, ' - 'and will be removed in J.'), - cfg.StrOpt('syslog-log-facility', - default='LOG_USER', - help='Syslog facility to receive log lines.') -] - -generic_log_opts = [ - cfg.BoolOpt('use_stderr', - default=True, - help='Log output to standard error.') -] - -DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', - 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', - 'oslo.messaging=INFO', 'iso8601=WARN', - 'requests.packages.urllib3.connectionpool=WARN', - 'urllib3.connectionpool=WARN', 'websocket=WARN', - "keystonemiddleware=WARN", "routes.middleware=WARN", - "stevedore=WARN"] - -log_opts = [ - cfg.StrOpt('logging_context_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [%(request_id)s %(user_identity)s] ' - '%(instance)s%(message)s', - help='Format string to use for log messages with context.'), - cfg.StrOpt('logging_default_format_string', - default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' - '%(name)s [-] %(instance)s%(message)s', - help='Format string to use for log messages without context.'), - cfg.StrOpt('logging_debug_format_suffix', - default='%(funcName)s %(pathname)s:%(lineno)d', - help='Data to append to log format when level is DEBUG.'), - cfg.StrOpt('logging_exception_prefix', - default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' - '%(instance)s', - help='Prefix each line of exception output with this format.'), - cfg.ListOpt('default_log_levels', - default=DEFAULT_LOG_LEVELS, - help='List of logger=LEVEL pairs.'), - cfg.BoolOpt('publish_errors', - default=False, - help='Enables or disables publication of error events.'), - cfg.BoolOpt('fatal_deprecations', - default=False, - help='Enables or disables fatal status of deprecations.'), - - # NOTE(mikal): there are two options here because sometimes we are handed - # a full instance (and could include more information), and other times we - # are just handed a UUID for the instance. - cfg.StrOpt('instance_format', - default='[instance: %(uuid)s] ', - help='The format for an instance that is passed with the log ' - 'message.'), - cfg.StrOpt('instance_uuid_format', - default='[instance: %(uuid)s] ', - help='The format for an instance UUID that is passed with the ' - 'log message.'), -] - -CONF = cfg.CONF -CONF.register_cli_opts(common_cli_opts) -CONF.register_cli_opts(logging_cli_opts) -CONF.register_opts(generic_log_opts) -CONF.register_opts(log_opts) - -# our new audit level -# NOTE(jkoelker) Since we synthesized an audit level, make the logging -# module aware of it so it acts like other levels. -logging.AUDIT = logging.INFO + 1 -logging.addLevelName(logging.AUDIT, 'AUDIT') - - -try: - NullHandler = logging.NullHandler -except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - - -def _dictify_context(context): - if context is None: - return None - if not isinstance(context, dict) and getattr(context, 'to_dict', None): - context = context.to_dict() - return context - - -def _get_binary_name(): - return os.path.basename(inspect.stack()[-1][1]) - - -def _get_log_file_path(binary=None): - logfile = CONF.log_file - logdir = CONF.log_dir - - if logfile and not logdir: - return logfile - - if logfile and logdir: - return os.path.join(logdir, logfile) - - if logdir: - binary = binary or _get_binary_name() - return '%s.log' % (os.path.join(logdir, binary),) - - return None - - -class BaseLoggerAdapter(logging.LoggerAdapter): - - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) - - def isEnabledFor(self, level): - if _PY26: - # This method was added in python 2.7 (and it does the exact - # same logic, so we need to do the exact same logic so that - # python 2.6 has this capability as well). - return self.logger.isEnabledFor(level) - else: - return super(BaseLoggerAdapter, self).isEnabledFor(level) - - -class LazyAdapter(BaseLoggerAdapter): - def __init__(self, name='unknown', version='unknown'): - self._logger = None - self.extra = {} - self.name = name - self.version = version - - @property - def logger(self): - if not self._logger: - self._logger = getLogger(self.name, self.version) - if six.PY3: - # In Python 3, the code fails because the 'manager' attribute - # cannot be found when using a LoggerAdapter as the - # underlying logger. Work around this issue. - self._logger.manager = self._logger.logger.manager - return self._logger - - -class ContextAdapter(BaseLoggerAdapter): - warn = logging.LoggerAdapter.warning - - def __init__(self, logger, project_name, version_string): - self.logger = logger - self.project = project_name - self.version = version_string - self._deprecated_messages_sent = dict() - - @property - def handlers(self): - return self.logger.handlers - - def deprecated(self, msg, *args, **kwargs): - """Call this method when a deprecated feature is used. - - If the system is configured for fatal deprecations then the message - is logged at the 'critical' level and :class:`DeprecatedConfig` will - be raised. - - Otherwise, the message will be logged (once) at the 'warn' level. - - :raises: :class:`DeprecatedConfig` if the system is configured for - fatal deprecations. - - """ - stdmsg = _("Deprecated: %s") % msg - if CONF.fatal_deprecations: - self.critical(stdmsg, *args, **kwargs) - raise DeprecatedConfig(msg=stdmsg) - - # Using a list because a tuple with dict can't be stored in a set. - sent_args = self._deprecated_messages_sent.setdefault(msg, list()) - - if args in sent_args: - # Already logged this message, so don't log it again. - return - - sent_args.append(args) - self.warn(stdmsg, *args, **kwargs) - - def process(self, msg, kwargs): - # NOTE(jecarey): If msg is not unicode, coerce it into unicode - # before it can get to the python logging and - # possibly cause string encoding trouble - if not isinstance(msg, six.text_type): - msg = six.text_type(msg) - - if 'extra' not in kwargs: - kwargs['extra'] = {} - extra = kwargs['extra'] - - context = kwargs.pop('context', None) - if not context: - context = getattr(local.store, 'context', None) - if context: - extra.update(_dictify_context(context)) - - instance = kwargs.pop('instance', None) - instance_uuid = (extra.get('instance_uuid') or - kwargs.pop('instance_uuid', None)) - instance_extra = '' - if instance: - instance_extra = CONF.instance_format % instance - elif instance_uuid: - instance_extra = (CONF.instance_uuid_format - % {'uuid': instance_uuid}) - extra['instance'] = instance_extra - - extra.setdefault('user_identity', kwargs.pop('user_identity', None)) - - extra['project'] = self.project - extra['version'] = self.version - extra['extra'] = extra.copy() - return msg, kwargs - - -class JSONFormatter(logging.Formatter): - def __init__(self, fmt=None, datefmt=None): - # NOTE(jkoelker) we ignore the fmt argument, but its still there - # since logging.config.fileConfig passes it. - self.datefmt = datefmt - - def formatException(self, ei, strip_newlines=True): - lines = traceback.format_exception(*ei) - if strip_newlines: - lines = [moves.filter( - lambda x: x, - line.rstrip().splitlines()) for line in lines] - lines = list(itertools.chain(*lines)) - return lines - - def format(self, record): - message = {'message': record.getMessage(), - 'asctime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'msg': record.msg, - 'args': record.args, - 'levelname': record.levelname, - 'levelno': record.levelno, - 'pathname': record.pathname, - 'filename': record.filename, - 'module': record.module, - 'lineno': record.lineno, - 'funcname': record.funcName, - 'created': record.created, - 'msecs': record.msecs, - 'relative_created': record.relativeCreated, - 'thread': record.thread, - 'thread_name': record.threadName, - 'process_name': record.processName, - 'process': record.process, - 'traceback': None} - - if hasattr(record, 'extra'): - message['extra'] = record.extra - - if record.exc_info: - message['traceback'] = self.formatException(record.exc_info) - - return jsonutils.dumps(message) - - -def _create_logging_excepthook(product_name): - def logging_excepthook(exc_type, value, tb): - extra = {'exc_info': (exc_type, value, tb)} - getLogger(product_name).critical( - "".join(traceback.format_exception_only(exc_type, value)), - **extra) - return logging_excepthook - - -class LogConfigError(Exception): - - message = _('Error loading logging config %(log_config)s: %(err_msg)s') - - def __init__(self, log_config, err_msg): - self.log_config = log_config - self.err_msg = err_msg - - def __str__(self): - return self.message % dict(log_config=self.log_config, - err_msg=self.err_msg) - - -def _load_log_config(log_config_append): - try: - logging.config.fileConfig(log_config_append, - disable_existing_loggers=False) - except (moves.configparser.Error, KeyError) as exc: - raise LogConfigError(log_config_append, six.text_type(exc)) - - -def setup(product_name, version='unknown'): - """Setup logging.""" - if CONF.log_config_append: - _load_log_config(CONF.log_config_append) - else: - _setup_logging_from_conf(product_name, version) - sys.excepthook = _create_logging_excepthook(product_name) - - -def set_defaults(logging_context_format_string=None, - default_log_levels=None): - # Just in case the caller is not setting the - # default_log_level. This is insurance because - # we introduced the default_log_level parameter - # later in a backwards in-compatible change - if default_log_levels is not None: - cfg.set_defaults( - log_opts, - default_log_levels=default_log_levels) - if logging_context_format_string is not None: - cfg.set_defaults( - log_opts, - logging_context_format_string=logging_context_format_string) - - -def _find_facility_from_conf(): - facility_names = logging.handlers.SysLogHandler.facility_names - facility = getattr(logging.handlers.SysLogHandler, - CONF.syslog_log_facility, - None) - - if facility is None and CONF.syslog_log_facility in facility_names: - facility = facility_names.get(CONF.syslog_log_facility) - - if facility is None: - valid_facilities = facility_names.keys() - consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', - 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', - 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', - 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', - 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] - valid_facilities.extend(consts) - raise TypeError(_('syslog facility must be one of: %s') % - ', '.join("'%s'" % fac - for fac in valid_facilities)) - - return facility - - -class RFCSysLogHandler(logging.handlers.SysLogHandler): - def __init__(self, *args, **kwargs): - self.binary_name = _get_binary_name() - # Do not use super() unless type(logging.handlers.SysLogHandler) - # is 'type' (Python 2.7). - # Use old style calls, if the type is 'classobj' (Python 2.6) - logging.handlers.SysLogHandler.__init__(self, *args, **kwargs) - - def format(self, record): - # Do not use super() unless type(logging.handlers.SysLogHandler) - # is 'type' (Python 2.7). - # Use old style calls, if the type is 'classobj' (Python 2.6) - msg = logging.handlers.SysLogHandler.format(self, record) - msg = self.binary_name + ' ' + msg - return msg - - -def _setup_logging_from_conf(project, version): - log_root = getLogger(None).logger - for handler in log_root.handlers: - log_root.removeHandler(handler) - - logpath = _get_log_file_path() - if logpath: - filelog = logging.handlers.WatchedFileHandler(logpath) - log_root.addHandler(filelog) - - if CONF.use_stderr: - streamlog = ColorHandler() - log_root.addHandler(streamlog) - - elif not logpath: - # pass sys.stdout as a positional argument - # python2.6 calls the argument strm, in 2.7 it's stream - streamlog = logging.StreamHandler(sys.stdout) - log_root.addHandler(streamlog) - - if CONF.publish_errors: - handler = importutils.import_object( - "oslo.messaging.notify.log_handler.PublishErrorsHandler", - logging.ERROR) - log_root.addHandler(handler) - - datefmt = CONF.log_date_format - for handler in log_root.handlers: - # NOTE(alaski): CONF.log_format overrides everything currently. This - # should be deprecated in favor of context aware formatting. - if CONF.log_format: - handler.setFormatter(logging.Formatter(fmt=CONF.log_format, - datefmt=datefmt)) - log_root.info('Deprecated: log_format is now deprecated and will ' - 'be removed in the next release') - else: - handler.setFormatter(ContextFormatter(project=project, - version=version, - datefmt=datefmt)) - - if CONF.debug: - log_root.setLevel(logging.DEBUG) - elif CONF.verbose: - log_root.setLevel(logging.INFO) - else: - log_root.setLevel(logging.WARNING) - - for pair in CONF.default_log_levels: - mod, _sep, level_name = pair.partition('=') - logger = logging.getLogger(mod) - # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name - # to integer code. - if sys.version_info < (2, 7): - level = logging.getLevelName(level_name) - logger.setLevel(level) - else: - logger.setLevel(level_name) - - if CONF.use_syslog: - try: - facility = _find_facility_from_conf() - # TODO(bogdando) use the format provided by RFCSysLogHandler - # after existing syslog format deprecation in J - if CONF.use_syslog_rfc_format: - syslog = RFCSysLogHandler(facility=facility) - else: - syslog = logging.handlers.SysLogHandler(facility=facility) - log_root.addHandler(syslog) - except socket.error: - log_root.error('Unable to add syslog handler. Verify that syslog ' - 'is running.') - - -_loggers = {} - - -def getLogger(name='unknown', version='unknown'): - if name not in _loggers: - _loggers[name] = ContextAdapter(logging.getLogger(name), - name, - version) - return _loggers[name] - - -def getLazyLogger(name='unknown', version='unknown'): - """Returns lazy logger. - - Creates a pass-through logger that does not create the real logger - until it is really needed and delegates all calls to the real logger - once it is created. - """ - return LazyAdapter(name, version) - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.INFO): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg.rstrip()) - - -class ContextFormatter(logging.Formatter): - """A context.RequestContext aware formatter configured through flags. - - The flags used to set format strings are: logging_context_format_string - and logging_default_format_string. You can also specify - logging_debug_format_suffix to append extra formatting if the log level is - debug. - - For information about what variables are available for the formatter see: - http://docs.python.org/library/logging.html#formatter - - If available, uses the context value stored in TLS - local.store.context - - """ - - def __init__(self, *args, **kwargs): - """Initialize ContextFormatter instance - - Takes additional keyword arguments which can be used in the message - format string. - - :keyword project: project name - :type project: string - :keyword version: project version - :type version: string - - """ - - self.project = kwargs.pop('project', 'unknown') - self.version = kwargs.pop('version', 'unknown') - - logging.Formatter.__init__(self, *args, **kwargs) - - def format(self, record): - """Uses contextstring if request_id is set, otherwise default.""" - - # NOTE(jecarey): If msg is not unicode, coerce it into unicode - # before it can get to the python logging and - # possibly cause string encoding trouble - if not isinstance(record.msg, six.text_type): - record.msg = six.text_type(record.msg) - - # store project info - record.project = self.project - record.version = self.version - - # store request info - context = getattr(local.store, 'context', None) - if context: - d = _dictify_context(context) - for k, v in d.items(): - setattr(record, k, v) - - # NOTE(sdague): default the fancier formatting params - # to an empty string so we don't throw an exception if - # they get used - for key in ('instance', 'color', 'user_identity'): - if key not in record.__dict__: - record.__dict__[key] = '' - - if record.__dict__.get('request_id'): - fmt = CONF.logging_context_format_string - else: - fmt = CONF.logging_default_format_string - - if (record.levelno == logging.DEBUG and - CONF.logging_debug_format_suffix): - fmt += " " + CONF.logging_debug_format_suffix - - if sys.version_info < (3, 2): - self._fmt = fmt - else: - self._style = logging.PercentStyle(fmt) - self._fmt = self._style._fmt - # Cache this on the record, Logger will respect our formatted copy - if record.exc_info: - record.exc_text = self.formatException(record.exc_info, record) - return logging.Formatter.format(self, record) - - def formatException(self, exc_info, record=None): - """Format exception output with CONF.logging_exception_prefix.""" - if not record: - return logging.Formatter.formatException(self, exc_info) - - stringbuffer = moves.StringIO() - traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], - None, stringbuffer) - lines = stringbuffer.getvalue().split('\n') - stringbuffer.close() - - if CONF.logging_exception_prefix.find('%(asctime)') != -1: - record.asctime = self.formatTime(record, self.datefmt) - - formatted_lines = [] - for line in lines: - pl = CONF.logging_exception_prefix % record.__dict__ - fl = '%s%s' % (pl, line) - formatted_lines.append(fl) - return '\n'.join(formatted_lines) - - -class ColorHandler(logging.StreamHandler): - LEVEL_COLORS = { - logging.DEBUG: '\033[00;32m', # GREEN - logging.INFO: '\033[00;36m', # CYAN - logging.AUDIT: '\033[01;36m', # BOLD CYAN - logging.WARN: '\033[01;33m', # BOLD YELLOW - logging.ERROR: '\033[01;31m', # BOLD RED - logging.CRITICAL: '\033[01;31m', # BOLD RED - } - - def format(self, record): - record.color = self.LEVEL_COLORS[record.levelno] - return logging.StreamHandler.format(self, record) - - -class DeprecatedConfig(Exception): - message = _("Fatal call to deprecated config: %(msg)s") - - def __init__(self, msg): - super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/rally/openstack/common/loopingcall.py b/rally/openstack/common/loopingcall.py deleted file mode 100644 index a579a4df11..0000000000 --- a/rally/openstack/common/loopingcall.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys -import time - -from eventlet import event -from eventlet import greenthread - -from rally.openstack.common._i18n import _LE, _LW -from rally.openstack.common import log as logging - -LOG = logging.getLogger(__name__) - -# NOTE(zyluo): This lambda function was declared to avoid mocking collisions -# with time.time() called in the standard logging module -# during unittests. -_ts = lambda: time.time() - - -class LoopingCallDone(Exception): - """Exception to break out and stop a LoopingCallBase. - - The poll-function passed to LoopingCallBase can raise this exception to - break out of the loop normally. This is somewhat analogous to - StopIteration. - - An optional return-value can be included as the argument to the exception; - this return-value will be returned by LoopingCallBase.wait() - - """ - - def __init__(self, retvalue=True): - """:param retvalue: Value that LoopingCallBase.wait() should return.""" - self.retvalue = retvalue - - -class LoopingCallBase(object): - def __init__(self, f=None, *args, **kw): - self.args = args - self.kw = kw - self.f = f - self._running = False - self.done = None - - def stop(self): - self._running = False - - def wait(self): - return self.done.wait() - - -class FixedIntervalLoopingCall(LoopingCallBase): - """A fixed interval looping call.""" - - def start(self, interval, initial_delay=None): - self._running = True - done = event.Event() - - def _inner(): - if initial_delay: - greenthread.sleep(initial_delay) - - try: - while self._running: - start = _ts() - self.f(*self.args, **self.kw) - end = _ts() - if not self._running: - break - delay = end - start - interval - if delay > 0: - LOG.warn(_LW('task %(func_name)s run outlasted ' - 'interval by %(delay).2f sec'), - {'func_name': repr(self.f), 'delay': delay}) - greenthread.sleep(-delay if delay < 0 else 0) - except LoopingCallDone as e: - self.stop() - done.send(e.retvalue) - except Exception: - LOG.exception(_LE('in fixed duration looping call')) - done.send_exception(*sys.exc_info()) - return - else: - done.send(True) - - self.done = done - - greenthread.spawn_n(_inner) - return self.done - - -class DynamicLoopingCall(LoopingCallBase): - """A looping call which sleeps until the next known event. - - The function called should return how long to sleep for before being - called again. - """ - - def start(self, initial_delay=None, periodic_interval_max=None): - self._running = True - done = event.Event() - - def _inner(): - if initial_delay: - greenthread.sleep(initial_delay) - - try: - while self._running: - idle = self.f(*self.args, **self.kw) - if not self._running: - break - - if periodic_interval_max is not None: - idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call %(func_name)s sleeping ' - 'for %(idle).02f seconds', - {'func_name': repr(self.f), 'idle': idle}) - greenthread.sleep(idle) - except LoopingCallDone as e: - self.stop() - done.send(e.retvalue) - except Exception: - LOG.exception(_LE('in dynamic looping call')) - done.send_exception(*sys.exc_info()) - return - else: - done.send(True) - - self.done = done - - greenthread.spawn(_inner) - return self.done diff --git a/rally/openstack/common/periodic_task.py b/rally/openstack/common/periodic_task.py deleted file mode 100644 index 30bf4666fd..0000000000 --- a/rally/openstack/common/periodic_task.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import random -import time - -from oslo.config import cfg -import six - -from rally.openstack.common._i18n import _, _LE, _LI -from rally.openstack.common import log as logging - - -periodic_opts = [ - cfg.BoolOpt('run_external_periodic_tasks', - default=True, - help='Some periodic tasks can be run in a separate process. ' - 'Should we run them here?'), -] - -CONF = cfg.CONF -CONF.register_opts(periodic_opts) - -LOG = logging.getLogger(__name__) - -DEFAULT_INTERVAL = 60.0 - - -class InvalidPeriodicTaskArg(Exception): - message = _("Unexpected argument for periodic task creation: %(arg)s.") - - -def periodic_task(*args, **kwargs): - """Decorator to indicate that a method is a periodic task. - - This decorator can be used in two ways: - - 1. Without arguments '@periodic_task', this will be run on the default - interval of 60 seconds. - - 2. With arguments: - @periodic_task(spacing=N [, run_immediately=[True|False]]) - this will be run on approximately every N seconds. If this number is - negative the periodic task will be disabled. If the run_immediately - argument is provided and has a value of 'True', the first run of the - task will be shortly after task scheduler starts. If - run_immediately is omitted or set to 'False', the first time the - task runs will be approximately N seconds after the task scheduler - starts. - """ - def decorator(f): - # Test for old style invocation - if 'ticks_between_runs' in kwargs: - raise InvalidPeriodicTaskArg(arg='ticks_between_runs') - - # Control if run at all - f._periodic_task = True - f._periodic_external_ok = kwargs.pop('external_process_ok', False) - if f._periodic_external_ok and not CONF.run_external_periodic_tasks: - f._periodic_enabled = False - else: - f._periodic_enabled = kwargs.pop('enabled', True) - - # Control frequency - f._periodic_spacing = kwargs.pop('spacing', 0) - f._periodic_immediate = kwargs.pop('run_immediately', False) - if f._periodic_immediate: - f._periodic_last_run = None - else: - f._periodic_last_run = time.time() - return f - - # NOTE(sirp): The `if` is necessary to allow the decorator to be used with - # and without parenthesis. - # - # In the 'with-parenthesis' case (with kwargs present), this function needs - # to return a decorator function since the interpreter will invoke it like: - # - # periodic_task(*args, **kwargs)(f) - # - # In the 'without-parenthesis' case, the original function will be passed - # in as the first argument, like: - # - # periodic_task(f) - if kwargs: - return decorator - else: - return decorator(args[0]) - - -class _PeriodicTasksMeta(type): - def __init__(cls, names, bases, dict_): - """Metaclass that allows us to collect decorated periodic tasks.""" - super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_) - - # NOTE(sirp): if the attribute is not present then we must be the base - # class, so, go ahead an initialize it. If the attribute is present, - # then we're a subclass so make a copy of it so we don't step on our - # parent's toes. - try: - cls._periodic_tasks = cls._periodic_tasks[:] - except AttributeError: - cls._periodic_tasks = [] - - try: - cls._periodic_spacing = cls._periodic_spacing.copy() - except AttributeError: - cls._periodic_spacing = {} - - for value in cls.__dict__.values(): - if getattr(value, '_periodic_task', False): - task = value - name = task.__name__ - - if task._periodic_spacing < 0: - LOG.info(_LI('Skipping periodic task %(task)s because ' - 'its interval is negative'), - {'task': name}) - continue - if not task._periodic_enabled: - LOG.info(_LI('Skipping periodic task %(task)s because ' - 'it is disabled'), - {'task': name}) - continue - - # A periodic spacing of zero indicates that this task should - # be run on the default interval to avoid running too - # frequently. - if task._periodic_spacing == 0: - task._periodic_spacing = DEFAULT_INTERVAL - - cls._periodic_tasks.append((name, task)) - cls._periodic_spacing[name] = task._periodic_spacing - - -def _nearest_boundary(last_run, spacing): - """Find nearest boundary which is in the past, which is a multiple of the - spacing with the last run as an offset. - - Eg if last run was 10 and spacing was 7, the new last run could be: 17, 24, - 31, 38... - - 0% to 5% of the spacing value will be added to this value to ensure tasks - do not synchronize. This jitter is rounded to the nearest second, this - means that spacings smaller than 20 seconds will not have jitter. - """ - current_time = time.time() - if last_run is None: - return current_time - delta = current_time - last_run - offset = delta % spacing - # Add up to 5% jitter - jitter = int(spacing * (random.random() / 20)) - return current_time - offset + jitter - - -@six.add_metaclass(_PeriodicTasksMeta) -class PeriodicTasks(object): - def __init__(self): - super(PeriodicTasks, self).__init__() - self._periodic_last_run = {} - for name, task in self._periodic_tasks: - self._periodic_last_run[name] = task._periodic_last_run - - def run_periodic_tasks(self, context, raise_on_error=False): - """Tasks to be run at a periodic interval.""" - idle_for = DEFAULT_INTERVAL - for task_name, task in self._periodic_tasks: - full_task_name = '.'.join([self.__class__.__name__, task_name]) - - spacing = self._periodic_spacing[task_name] - last_run = self._periodic_last_run[task_name] - - # Check if due, if not skip - idle_for = min(idle_for, spacing) - if last_run is not None: - delta = last_run + spacing - time.time() - if delta > 0: - idle_for = min(idle_for, delta) - continue - - LOG.debug("Running periodic task %(full_task_name)s", - {"full_task_name": full_task_name}) - self._periodic_last_run[task_name] = _nearest_boundary( - last_run, spacing) - - try: - task(self, context) - except Exception as e: - if raise_on_error: - raise - LOG.exception(_LE("Error during %(full_task_name)s: %(e)s"), - {"full_task_name": full_task_name, "e": e}) - time.sleep(0) - - return idle_for diff --git a/rally/openstack/common/service.py b/rally/openstack/common/service.py deleted file mode 100644 index 7b9fd874c9..0000000000 --- a/rally/openstack/common/service.py +++ /dev/null @@ -1,504 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2011 Justin Santa Barbara -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Generic Node base class for all workers that run on hosts.""" - -import errno -import logging as std_logging -import os -import random -import signal -import sys -import time - -try: - # Importing just the symbol here because the io module does not - # exist in Python 2.6. - from io import UnsupportedOperation # noqa -except ImportError: - # Python 2.6 - UnsupportedOperation = None - -import eventlet -from eventlet import event -from oslo.config import cfg - -from rally.openstack.common import eventlet_backdoor -from rally.openstack.common._i18n import _LE, _LI, _LW -from rally.openstack.common import log as logging -from rally.openstack.common import systemd -from rally.openstack.common import threadgroup - - -CONF = cfg.CONF -LOG = logging.getLogger(__name__) - - -def _sighup_supported(): - return hasattr(signal, 'SIGHUP') - - -def _is_daemon(): - # The process group for a foreground process will match the - # process group of the controlling terminal. If those values do - # not match, or ioctl() fails on the stdout file handle, we assume - # the process is running in the background as a daemon. - # http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics - try: - is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno()) - except OSError as err: - if err.errno == errno.ENOTTY: - # Assume we are a daemon because there is no terminal. - is_daemon = True - else: - raise - except UnsupportedOperation: - # Could not get the fileno for stdout, so we must be a daemon. - is_daemon = True - return is_daemon - - -def _is_sighup_and_daemon(signo): - if not (_sighup_supported() and signo == signal.SIGHUP): - # Avoid checking if we are a daemon, because the signal isn't - # SIGHUP. - return False - return _is_daemon() - - -def _signo_to_signame(signo): - signals = {signal.SIGTERM: 'SIGTERM', - signal.SIGINT: 'SIGINT'} - if _sighup_supported(): - signals[signal.SIGHUP] = 'SIGHUP' - return signals[signo] - - -def _set_signals_handler(handler): - signal.signal(signal.SIGTERM, handler) - signal.signal(signal.SIGINT, handler) - if _sighup_supported(): - signal.signal(signal.SIGHUP, handler) - - -class Launcher(object): - """Launch one or more services and wait for them to complete.""" - - def __init__(self): - """Initialize the service launcher. - - :returns: None - - """ - self.services = Services() - self.backdoor_port = eventlet_backdoor.initialize_if_enabled() - - def launch_service(self, service): - """Load and start the given service. - - :param service: The service you would like to start. - :returns: None - - """ - service.backdoor_port = self.backdoor_port - self.services.add(service) - - def stop(self): - """Stop all services which are currently running. - - :returns: None - - """ - self.services.stop() - - def wait(self): - """Waits until all services have been stopped, and then returns. - - :returns: None - - """ - self.services.wait() - - def restart(self): - """Reload config files and restart service. - - :returns: None - - """ - cfg.CONF.reload_config_files() - self.services.restart() - - -class SignalExit(SystemExit): - def __init__(self, signo, exccode=1): - super(SignalExit, self).__init__(exccode) - self.signo = signo - - -class ServiceLauncher(Launcher): - def _handle_signal(self, signo, frame): - # Allow the process to be killed again and die from natural causes - _set_signals_handler(signal.SIG_DFL) - raise SignalExit(signo) - - def handle_signal(self): - _set_signals_handler(self._handle_signal) - - def _wait_for_exit_or_signal(self, ready_callback=None): - status = None - signo = 0 - - LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) - - try: - if ready_callback: - ready_callback() - super(ServiceLauncher, self).wait() - except SignalExit as exc: - signame = _signo_to_signame(exc.signo) - LOG.info(_LI('Caught %s, exiting'), signame) - status = exc.code - signo = exc.signo - except SystemExit as exc: - status = exc.code - finally: - self.stop() - - return status, signo - - def wait(self, ready_callback=None): - systemd.notify_once() - while True: - self.handle_signal() - status, signo = self._wait_for_exit_or_signal(ready_callback) - if not _is_sighup_and_daemon(signo): - return status - self.restart() - - -class ServiceWrapper(object): - def __init__(self, service, workers): - self.service = service - self.workers = workers - self.children = set() - self.forktimes = [] - - -class ProcessLauncher(object): - def __init__(self, wait_interval=0.01): - """Constructor. - - :param wait_interval: The interval to sleep for between checks - of child process exit. - """ - self.children = {} - self.sigcaught = None - self.running = True - self.wait_interval = wait_interval - rfd, self.writepipe = os.pipe() - self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') - self.handle_signal() - - def handle_signal(self): - _set_signals_handler(self._handle_signal) - - def _handle_signal(self, signo, frame): - self.sigcaught = signo - self.running = False - - # Allow the process to be killed again and die from natural causes - _set_signals_handler(signal.SIG_DFL) - - def _pipe_watcher(self): - # This will block until the write end is closed when the parent - # dies unexpectedly - self.readpipe.read() - - LOG.info(_LI('Parent process has died unexpectedly, exiting')) - - sys.exit(1) - - def _child_process_handle_signal(self): - # Setup child signal handlers differently - def _sigterm(*args): - signal.signal(signal.SIGTERM, signal.SIG_DFL) - raise SignalExit(signal.SIGTERM) - - def _sighup(*args): - signal.signal(signal.SIGHUP, signal.SIG_DFL) - raise SignalExit(signal.SIGHUP) - - signal.signal(signal.SIGTERM, _sigterm) - if _sighup_supported(): - signal.signal(signal.SIGHUP, _sighup) - # Block SIGINT and let the parent send us a SIGTERM - signal.signal(signal.SIGINT, signal.SIG_IGN) - - def _child_wait_for_exit_or_signal(self, launcher): - status = 0 - signo = 0 - - # NOTE(johannes): All exceptions are caught to ensure this - # doesn't fallback into the loop spawning children. It would - # be bad for a child to spawn more children. - try: - launcher.wait() - except SignalExit as exc: - signame = _signo_to_signame(exc.signo) - LOG.info(_LI('Child caught %s, exiting'), signame) - status = exc.code - signo = exc.signo - except SystemExit as exc: - status = exc.code - except BaseException: - LOG.exception(_LE('Unhandled exception')) - status = 2 - finally: - launcher.stop() - - return status, signo - - def _child_process(self, service): - self._child_process_handle_signal() - - # Reopen the eventlet hub to make sure we don't share an epoll - # fd with parent and/or siblings, which would be bad - eventlet.hubs.use_hub() - - # Close write to ensure only parent has it open - os.close(self.writepipe) - # Create greenthread to watch for parent to close pipe - eventlet.spawn_n(self._pipe_watcher) - - # Reseed random number generator - random.seed() - - launcher = Launcher() - launcher.launch_service(service) - return launcher - - def _start_child(self, wrap): - if len(wrap.forktimes) > wrap.workers: - # Limit ourselves to one process a second (over the period of - # number of workers * 1 second). This will allow workers to - # start up quickly but ensure we don't fork off children that - # die instantly too quickly. - if time.time() - wrap.forktimes[0] < wrap.workers: - LOG.info(_LI('Forking too fast, sleeping')) - time.sleep(1) - - wrap.forktimes.pop(0) - - wrap.forktimes.append(time.time()) - - pid = os.fork() - if pid == 0: - launcher = self._child_process(wrap.service) - while True: - self._child_process_handle_signal() - status, signo = self._child_wait_for_exit_or_signal(launcher) - if not _is_sighup_and_daemon(signo): - break - launcher.restart() - - os._exit(status) - - LOG.info(_LI('Started child %d'), pid) - - wrap.children.add(pid) - self.children[pid] = wrap - - return pid - - def launch_service(self, service, workers=1): - wrap = ServiceWrapper(service, workers) - - LOG.info(_LI('Starting %d workers'), wrap.workers) - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - def _wait_child(self): - try: - # Don't block if no child processes have exited - pid, status = os.waitpid(0, os.WNOHANG) - if not pid: - return None - except OSError as exc: - if exc.errno not in (errno.EINTR, errno.ECHILD): - raise - return None - - if os.WIFSIGNALED(status): - sig = os.WTERMSIG(status) - LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'), - dict(pid=pid, sig=sig)) - else: - code = os.WEXITSTATUS(status) - LOG.info(_LI('Child %(pid)s exited with status %(code)d'), - dict(pid=pid, code=code)) - - if pid not in self.children: - LOG.warning(_LW('pid %d not in child list'), pid) - return None - - wrap = self.children.pop(pid) - wrap.children.remove(pid) - return wrap - - def _respawn_children(self): - while self.running: - wrap = self._wait_child() - if not wrap: - # Yield to other threads if no children have exited - # Sleep for a short time to avoid excessive CPU usage - # (see bug #1095346) - eventlet.greenthread.sleep(self.wait_interval) - continue - while self.running and len(wrap.children) < wrap.workers: - self._start_child(wrap) - - def wait(self): - """Loop waiting on children to die and respawning as necessary.""" - - systemd.notify_once() - LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) - - try: - while True: - self.handle_signal() - self._respawn_children() - # No signal means that stop was called. Don't clean up here. - if not self.sigcaught: - return - - signame = _signo_to_signame(self.sigcaught) - LOG.info(_LI('Caught %s, stopping children'), signame) - if not _is_sighup_and_daemon(self.sigcaught): - break - - for pid in self.children: - os.kill(pid, signal.SIGHUP) - self.running = True - self.sigcaught = None - except eventlet.greenlet.GreenletExit: - LOG.info(_LI("Wait called after thread killed. Cleaning up.")) - - self.stop() - - def stop(self): - """Terminate child processes and wait on each.""" - self.running = False - for pid in self.children: - try: - os.kill(pid, signal.SIGTERM) - except OSError as exc: - if exc.errno != errno.ESRCH: - raise - - # Wait for children to die - if self.children: - LOG.info(_LI('Waiting on %d children to exit'), len(self.children)) - while self.children: - self._wait_child() - - -class Service(object): - """Service object for binaries running on hosts.""" - - def __init__(self, threads=1000): - self.tg = threadgroup.ThreadGroup(threads) - - # signal that the service is done shutting itself down: - self._done = event.Event() - - def reset(self): - # NOTE(Fengqian): docs for Event.reset() recommend against using it - self._done = event.Event() - - def start(self): - pass - - def stop(self): - self.tg.stop() - self.tg.wait() - # Signal that service cleanup is done: - if not self._done.ready(): - self._done.send() - - def wait(self): - self._done.wait() - - -class Services(object): - - def __init__(self): - self.services = [] - self.tg = threadgroup.ThreadGroup() - self.done = event.Event() - - def add(self, service): - self.services.append(service) - self.tg.add_thread(self.run_service, service, self.done) - - def stop(self): - # wait for graceful shutdown of services: - for service in self.services: - service.stop() - service.wait() - - # Each service has performed cleanup, now signal that the run_service - # wrapper threads can now die: - if not self.done.ready(): - self.done.send() - - # reap threads: - self.tg.stop() - - def wait(self): - self.tg.wait() - - def restart(self): - self.stop() - self.done = event.Event() - for restart_service in self.services: - restart_service.reset() - self.tg.add_thread(self.run_service, restart_service, self.done) - - @staticmethod - def run_service(service, done): - """Service start wrapper. - - :param service: service to run - :param done: event to wait on until a shutdown is triggered - :returns: None - - """ - service.start() - done.wait() - - -def launch(service, workers=1): - if workers is None or workers == 1: - launcher = ServiceLauncher() - launcher.launch_service(service) - else: - launcher = ProcessLauncher() - launcher.launch_service(service, workers=workers) - - return launcher diff --git a/rally/openstack/common/systemd.py b/rally/openstack/common/systemd.py deleted file mode 100644 index a4d9b814c0..0000000000 --- a/rally/openstack/common/systemd.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2012-2014 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Helper module for systemd service readiness notification. -""" - -import os -import socket -import sys - -from rally.openstack.common import log as logging - - -LOG = logging.getLogger(__name__) - - -def _abstractify(socket_name): - if socket_name.startswith('@'): - # abstract namespace socket - socket_name = '\0%s' % socket_name[1:] - return socket_name - - -def _sd_notify(unset_env, msg): - notify_socket = os.getenv('NOTIFY_SOCKET') - if notify_socket: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - try: - sock.connect(_abstractify(notify_socket)) - sock.sendall(msg) - if unset_env: - del os.environ['NOTIFY_SOCKET'] - except EnvironmentError: - LOG.debug("Systemd notification failed", exc_info=True) - finally: - sock.close() - - -def notify(): - """Send notification to Systemd that service is ready. - - For details see - http://www.freedesktop.org/software/systemd/man/sd_notify.html - """ - _sd_notify(False, 'READY=1') - - -def notify_once(): - """Send notification once to Systemd that service is ready. - - Systemd sets NOTIFY_SOCKET environment variable with the name of the - socket listening for notifications from services. - This method removes the NOTIFY_SOCKET environment variable to ensure - notification is sent only once. - """ - _sd_notify(True, 'READY=1') - - -def onready(notify_socket, timeout): - """Wait for systemd style notification on the socket. - - :param notify_socket: local socket address - :type notify_socket: string - :param timeout: socket timeout - :type timeout: float - :returns: 0 service ready - 1 service not ready - 2 timeout occurred - """ - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - sock.settimeout(timeout) - sock.bind(_abstractify(notify_socket)) - try: - msg = sock.recv(512) - except socket.timeout: - return 2 - finally: - sock.close() - if 'READY=1' in msg: - return 0 - else: - return 1 - - -if __name__ == '__main__': - # simple CLI for testing - if len(sys.argv) == 1: - notify() - elif len(sys.argv) >= 2: - timeout = float(sys.argv[1]) - notify_socket = os.getenv('NOTIFY_SOCKET') - if notify_socket: - retval = onready(notify_socket, timeout) - sys.exit(retval) diff --git a/rally/openstack/common/threadgroup.py b/rally/openstack/common/threadgroup.py deleted file mode 100644 index ead256966d..0000000000 --- a/rally/openstack/common/threadgroup.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright 2012 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import threading - -import eventlet -from eventlet import greenpool - -from rally.openstack.common import log as logging -from rally.openstack.common import loopingcall - - -LOG = logging.getLogger(__name__) - - -def _thread_done(gt, *args, **kwargs): - """Callback function to be passed to GreenThread.link() when we spawn() - Calls the :class:`ThreadGroup` to notify if. - - """ - kwargs['group'].thread_done(kwargs['thread']) - - -class Thread(object): - """Wrapper around a greenthread, that holds a reference to the - :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when - it has done so it can be removed from the threads list. - """ - def __init__(self, thread, group): - self.thread = thread - self.thread.link(_thread_done, group=group, thread=self) - - def stop(self): - self.thread.kill() - - def wait(self): - return self.thread.wait() - - def link(self, func, *args, **kwargs): - self.thread.link(func, *args, **kwargs) - - -class ThreadGroup(object): - """The point of the ThreadGroup class is to: - - * keep track of timers and greenthreads (making it easier to stop them - when need be). - * provide an easy API to add timers. - """ - def __init__(self, thread_pool_size=10): - self.pool = greenpool.GreenPool(thread_pool_size) - self.threads = [] - self.timers = [] - - def add_dynamic_timer(self, callback, initial_delay=None, - periodic_interval_max=None, *args, **kwargs): - timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs) - timer.start(initial_delay=initial_delay, - periodic_interval_max=periodic_interval_max) - self.timers.append(timer) - - def add_timer(self, interval, callback, initial_delay=None, - *args, **kwargs): - pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs) - pulse.start(interval=interval, - initial_delay=initial_delay) - self.timers.append(pulse) - - def add_thread(self, callback, *args, **kwargs): - gt = self.pool.spawn(callback, *args, **kwargs) - th = Thread(gt, self) - self.threads.append(th) - return th - - def thread_done(self, thread): - self.threads.remove(thread) - - def _stop_threads(self): - current = threading.current_thread() - - # Iterate over a copy of self.threads so thread_done doesn't - # modify the list while we're iterating - for x in self.threads[:]: - if x is current: - # don't kill the current thread. - continue - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - - def stop_timers(self): - for x in self.timers: - try: - x.stop() - except Exception as ex: - LOG.exception(ex) - self.timers = [] - - def stop(self, graceful=False): - """stop function has the option of graceful=True/False. - - * In case of graceful=True, wait for all threads to be finished. - Never kill threads. - * In case of graceful=False, kill threads immediately. - """ - self.stop_timers() - if graceful: - # In case of graceful=True, wait for all threads to be - # finished, never kill threads - self.wait() - else: - # In case of graceful=False(Default), kill threads - # immediately - self._stop_threads() - - def wait(self): - for x in self.timers: - try: - x.wait() - except eventlet.greenlet.GreenletExit: - pass - except Exception as ex: - LOG.exception(ex) - current = threading.current_thread() - - # Iterate over a copy of self.threads so thread_done doesn't - # modify the list while we're iterating - for x in self.threads[:]: - if x is current: - continue - try: - x.wait() - except eventlet.greenlet.GreenletExit: - pass - except Exception as ex: - LOG.exception(ex) diff --git a/rally/osclients.py b/rally/osclients.py index 01f8477182..04078818d5 100644 --- a/rally/osclients.py +++ b/rally/osclients.py @@ -36,14 +36,16 @@ from rally import exceptions CONF = cfg.CONF -CONF.register_opts([ + +OSCLIENTS_OPTS = [ cfg.FloatOpt("openstack_client_http_timeout", default=180.0, help="HTTP timeout for any of OpenStack service in seconds"), cfg.BoolOpt("https_insecure", default=False, help="Use SSL for all OpenStack API interfaces"), cfg.StrOpt("https_cacert", default=None, help="Path to CA server cetrificate for SSL") -]) +] +CONF.register_opts(OSCLIENTS_OPTS) # NOTE(boris-42): super dirty hack to fix nova python client 2.17 thread safe diff --git a/rally/verification/tempest/config.py b/rally/verification/tempest/config.py index abba15f5b0..b8a20bb0e8 100644 --- a/rally/verification/tempest/config.py +++ b/rally/verification/tempest/config.py @@ -34,7 +34,7 @@ from rally import osclients LOG = logging.getLogger(__name__) -image_opts = [ +IMAGE_OPTS = [ cfg.StrOpt("cirros_version", default="0.3.2", help="Version of cirros image"), @@ -43,7 +43,7 @@ image_opts = [ help="Cirros image name"), ] CONF = cfg.CONF -CONF.register_opts(image_opts, "image") +CONF.register_opts(IMAGE_OPTS, "image") class TempestConfigCreationFailure(exceptions.RallyException): diff --git a/requirements.txt b/requirements.txt index 0fecc34e08..9805743228 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,14 +8,14 @@ iso8601>=0.1.9 Jinja2>=2.6 # BSD License (3 clause) jsonschema>=2.0.0,<3.0.0 netaddr>=0.7.12 -oslo.config>=1.6.0 # Apache-2.0 -oslo.db>=1.4.1 # Apache-2.0 -oslo.i18n>=1.3.0 # Apache-2.0 +oslo.config>=1.6.0 # Apache-2.0 +oslo.db>=1.4.1 # Apache-2.0 +oslo.i18n>=1.3.0 # Apache-2.0 +oslo.log>=0.1.0 # Apache-2.0 oslo.serialization>=1.2.0 # Apache-2.0 oslo.utils>=1.2.0 # Apache-2.0 paramiko>=1.13.0 pbr>=0.6,!=0.7,<1.0 -pecan>=0.8.0 PrettyTable>=0.7,<0.8 PyYAML>=3.1.0 psycopg2 @@ -36,4 +36,3 @@ requests>=2.2.0,!=2.4.0 SQLAlchemy>=0.9.7,<=0.9.99 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 six>=1.9.0 -WSME>=0.6 diff --git a/setup.cfg b/setup.cfg index 013722b445..1aaa7da213 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,13 +24,14 @@ packages = data_files = /etc/bash_completion.d = - tools/rally.bash_completion + etc/rally.bash_completion [entry_points] console_scripts = rally = rally.cmd.main:main - rally-api = rally.cmd.api:main rally-manage = rally.cmd.manage:main +oslo.config.opts = + rally = rally.common.opts:list_opts [global] setup-hooks = diff --git a/tests/hacking/README.rst b/tests/hacking/README.rst index 1f3ba92bb4..1102b8b9e8 100644 --- a/tests/hacking/README.rst +++ b/tests/hacking/README.rst @@ -12,7 +12,7 @@ Rally Specific Commandments * [N302] - Ensure that nonexistent "assert_called" is not used * [N303] - Ensure that nonexistent "assert_called_once" is not used * [N310-N314] - Reserved for rules related to logging - * [N310] - Ensure that ``rally.common.log`` is used instead of ``rally.openstack.common.log`` + * [N310] - Ensure that ``rally.common.log`` is used as logging module * [N311] - Validate that debug level logs are not translated * [N312] - Validate correctness of debug on check. * [N32x] - Reserved for rules related to assert* methods diff --git a/tests/hacking/checks.py b/tests/hacking/checks.py index 842aa3d2d2..252cc44879 100644 --- a/tests/hacking/checks.py +++ b/tests/hacking/checks.py @@ -120,8 +120,8 @@ def check_import_of_logging(logical_line, filename): excluded_files = ["./rally/common/log.py", "./tests/unit/test_log.py"] - forbidden_imports = ["from rally.openstack.common import log", - "import rally.openstack.common.log", + forbidden_imports = ["from oslo_log", + "import oslo_log", "import logging"] if filename not in excluded_files: diff --git a/tests/unit/aas/__init__.py b/tests/unit/aas/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/aas/rest/__init__.py b/tests/unit/aas/rest/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/aas/rest/controllers/__init__.py b/tests/unit/aas/rest/controllers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/aas/rest/test_types.py b/tests/unit/aas/rest/test_types.py deleted file mode 100644 index 723a347774..0000000000 --- a/tests/unit/aas/rest/test_types.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2014 Kylin Cloud -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Test for rest types.""" - -from __future__ import print_function - -from rally.aas.rest import types -from tests.unit import test - - -class TestLink(test.TestCase): - - def test_make_link(self): - url = "http://localhost:8877" - rel = "version" - link = types.Link.make_link(rel, url, "fake") - self.assertEqual("http://localhost:8877/fake", link.href) - self.assertEqual(rel, link.rel) - - -class TestMediaType(test.TestCase): - - def test_init(self): - base = "application/json" - _type = "application/vnd.openstack.rally.v1+json" - mt = types.MediaType(base, _type) - self.assertEqual(base, mt.base) - self.assertEqual(_type, mt.type) - - -class TestVersion(test.TestCase): - - def test_convert(self): - id = "v1" - status = "active" - updated_at = "2014-01-07T00:00:00Z" - link = types.Link.make_link("version", "http://localhost:8877", "fake") - version = types.Version.convert(id, status, updated_at=updated_at, - links=[link]) - self.assertEqual(id, version.id) - self.assertEqual(status, version.status) - self.assertEqual(updated_at, version.updated_at) - self.assertEqual("application/json", version.media_types[0].base) - self.assertEqual("application/vnd.openstack.rally.v1+json", - version.media_types[0].type) - self.assertEqual([link], version.links) diff --git a/tests/unit/cmd/commands/test_deployment.py b/tests/unit/cmd/commands/test_deployment.py index f532514985..36ee89d779 100644 --- a/tests/unit/cmd/commands/test_deployment.py +++ b/tests/unit/cmd/commands/test_deployment.py @@ -100,7 +100,7 @@ class DeploymentCommandsTestCase(test.TestCase): self.assertRaises(exceptions.InvalidArgumentsException, self.deployment.destroy, None) - @mock.patch("rally.cmd.commands.deployment.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.deployment.cliutils.print_list") @mock.patch("rally.cmd.commands.deployment.utils.Struct") @mock.patch("rally.cmd.commands.deployment.envutils.get_global") @mock.patch("rally.cmd.commands.deployment.db.deployment_list") @@ -128,7 +128,7 @@ class DeploymentCommandsTestCase(test.TestCase): sortby_index=headers.index( "created_at")) - @mock.patch("rally.cmd.commands.deployment.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.deployment.cliutils.print_list") @mock.patch("rally.cmd.commands.deployment.utils.Struct") @mock.patch("rally.cmd.commands.deployment.envutils.get_global") @mock.patch("rally.cmd.commands.deployment.db.deployment_list") @@ -171,7 +171,7 @@ class DeploymentCommandsTestCase(test.TestCase): self.assertRaises(exceptions.InvalidArgumentsException, self.deployment.config, None) - @mock.patch("rally.cmd.commands.deployment.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.deployment.cliutils.print_list") @mock.patch("rally.cmd.commands.deployment.utils.Struct") @mock.patch("rally.cmd.commands.deployment.db.deployment_get") def test_show(self, mock_deployment, mock_struct, mock_print_list): @@ -206,4 +206,4 @@ class DeploymentCommandsTestCase(test.TestCase): @mock.patch("rally.cmd.commands.use.UseCommands.deployment") def test_use(self, mock_use_deployment): self.deployment.use("fake_id") - mock_use_deployment.assert_called_once_with("fake_id") \ No newline at end of file + mock_use_deployment.assert_called_once_with("fake_id") diff --git a/tests/unit/cmd/commands/test_show.py b/tests/unit/cmd/commands/test_show.py index d0b742b3cb..c40b9b2f6d 100644 --- a/tests/unit/cmd/commands/test_show.py +++ b/tests/unit/cmd/commands/test_show.py @@ -34,7 +34,7 @@ class ShowCommandsTestCase(test.TestCase): self.fake_glance_client = fakes.FakeGlanceClient() self.fake_nova_client = fakes.FakeNovaClient() - @mock.patch("rally.cmd.commands.show.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.show.cliutils.print_list") @mock.patch("rally.cmd.commands.show.cliutils.pretty_float_formatter") @mock.patch("rally.cmd.commands.show.utils.Struct") @mock.patch("rally.cmd.commands.show.osclients.Clients.glance") @@ -62,7 +62,7 @@ class ShowCommandsTestCase(test.TestCase): formatters=fake_formatters, mixed_case_fields=mixed_case_fields) - @mock.patch("rally.cmd.commands.show.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.show.cliutils.print_list") @mock.patch("rally.cmd.commands.show.cliutils.pretty_float_formatter") @mock.patch("rally.cmd.commands.show.utils.Struct") @mock.patch("rally.cmd.commands.show.osclients.Clients.nova") @@ -94,7 +94,7 @@ class ShowCommandsTestCase(test.TestCase): formatters=fake_formatters, mixed_case_fields=mixed_case_fields) - @mock.patch("rally.cmd.commands.show.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.show.cliutils.print_list") @mock.patch("rally.cmd.commands.show.utils.Struct") @mock.patch("rally.cmd.commands.show.osclients.Clients.nova") @mock.patch("rally.cmd.commands.show.db.deployment_get") @@ -120,7 +120,7 @@ class ShowCommandsTestCase(test.TestCase): fields=headers, mixed_case_fields=mixed_case_fields) - @mock.patch("rally.cmd.commands.show.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.show.cliutils.print_list") @mock.patch("rally.cmd.commands.show.utils.Struct") @mock.patch("rally.cmd.commands.show.osclients.Clients.nova") @mock.patch("rally.cmd.commands.show.db.deployment_get") @@ -145,7 +145,7 @@ class ShowCommandsTestCase(test.TestCase): fields=headers, mixed_case_fields=mixed_case_fields) - @mock.patch("rally.cmd.commands.show.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.show.cliutils.print_list") @mock.patch("rally.cmd.commands.show.utils.Struct") @mock.patch("rally.cmd.commands.show.osclients.Clients.nova") @mock.patch("rally.cmd.commands.show.db.deployment_get") diff --git a/tests/unit/cmd/commands/test_task.py b/tests/unit/cmd/commands/test_task.py index 33a2a728ef..e958fc029a 100644 --- a/tests/unit/cmd/commands/test_task.py +++ b/tests/unit/cmd/commands/test_task.py @@ -479,7 +479,7 @@ class TaskCommandsTestCase(test.TestCase): out="/tmp/tmp.hsml") self.assertEqual(ret, 1) - @mock.patch("rally.cmd.commands.task.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.task.cliutils.print_list") @mock.patch("rally.cmd.commands.task.envutils.get_global", return_value="123456789") @mock.patch("rally.cmd.commands.task.objects.Task.list", @@ -542,7 +542,7 @@ class TaskCommandsTestCase(test.TestCase): in task_uuids] self.assertTrue(mock_api.Task.delete.mock_calls == expected_calls) - @mock.patch("rally.cmd.commands.task.common_cliutils.print_list") + @mock.patch("rally.cmd.commands.task.cliutils.print_list") @mock.patch("rally.cmd.commands.task.objects.Task.get") def test_sla_check(self, mock_task_get, mock_print_list): data = [{"key": {"name": "fake_name", diff --git a/tests/unit/cmd/commands/test_verify.py b/tests/unit/cmd/commands/test_verify.py index ed16af1336..c22e3337ec 100644 --- a/tests/unit/cmd/commands/test_verify.py +++ b/tests/unit/cmd/commands/test_verify.py @@ -94,7 +94,7 @@ class VerifyCommandsTestCase(test.TestCase): consts.TempestTestsAPI) self.assertFalse(mock_verify.called) - @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.cmd.cliutils.print_list") @mock.patch("rally.db.verification_list") def test_list(self, mock_db_verification_list, mock_print_list): fields = ["UUID", "Deployment UUID", "Set name", "Tests", "Failures", @@ -113,7 +113,7 @@ class VerifyCommandsTestCase(test.TestCase): sortby_index=fields.index( "Created at")) - @mock.patch("rally.openstack.common.cliutils.print_list") + @mock.patch("rally.cmd.cliutils.print_list") @mock.patch("rally.db.verification_get") @mock.patch("rally.db.verification_result_get") @mock.patch("rally.objects.Verification") @@ -304,4 +304,4 @@ class VerifyCommandsTestCase(test.TestCase): @mock.patch("rally.cmd.commands.use.UseCommands.verification") def test_use(self, mock_use_verification): self.verify.use("fake_id") - mock_use_verification.assert_called_once_with("fake_id") \ No newline at end of file + mock_use_verification.assert_called_once_with("fake_id") diff --git a/tests/unit/cmd/test_cliutils.py b/tests/unit/cmd/test_cliutils.py index 63f355b510..a27fcf2318 100644 --- a/tests/unit/cmd/test_cliutils.py +++ b/tests/unit/cmd/test_cliutils.py @@ -25,7 +25,6 @@ from rally.cmd.commands import task from rally.cmd.commands import use from rally.cmd.commands import verify from rally import exceptions -from rally.openstack.common import cliutils as common_cliutils from tests.unit import test CONF = cfg.CONF @@ -158,8 +157,8 @@ class CliUtilsTestCase(test.TestCase): self.assertTrue(mock_task_get.called) self.assertEqual(ret, 1) - @mock.patch("rally.openstack.common.cliutils.validate_args", - side_effect=common_cliutils.MissingArgs("missing")) + @mock.patch("rally.common.cliutils.validate_args", + side_effect=cliutils.MissingArgs("missing")) def test_run_show_fails(self, mock_validate_args): ret = cliutils.run(["rally", "show", "keypairs"], self.categories) self.assertTrue(mock_validate_args.called) @@ -175,3 +174,411 @@ class CliUtilsTestCase(test.TestCase): ret = cliutils.run(["rally", "failure", "failed_to_open_file"], {"failure": FailuresCommands}) self.assertEqual(1, ret) + + +class ValidateArgsTest(test.TestCase): + + def test_lambda_no_args(self): + cliutils.validate_args(lambda: None) + + def _test_lambda_with_args(self, *args, **kwargs): + cliutils.validate_args(lambda x, y: None, *args, **kwargs) + + def test_lambda_positional_args(self): + self._test_lambda_with_args(1, 2) + + def test_lambda_kwargs(self): + self._test_lambda_with_args(x=1, y=2) + + def test_lambda_mixed_kwargs(self): + self._test_lambda_with_args(1, y=2) + + def test_lambda_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_args) + + def test_lambda_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_args, 1) + + def test_lambda_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_args, y=2) + + def _test_lambda_with_default(self, *args, **kwargs): + cliutils.validate_args(lambda x, y, z=3: None, *args, **kwargs) + + def test_lambda_positional_args_with_default(self): + self._test_lambda_with_default(1, 2) + + def test_lambda_kwargs_with_default(self): + self._test_lambda_with_default(x=1, y=2) + + def test_lambda_mixed_kwargs_with_default(self): + self._test_lambda_with_default(1, y=2) + + def test_lambda_positional_args_all_with_default(self): + self._test_lambda_with_default(1, 2, 3) + + def test_lambda_kwargs_all_with_default(self): + self._test_lambda_with_default(x=1, y=2, z=3) + + def test_lambda_mixed_kwargs_all_with_default(self): + self._test_lambda_with_default(1, y=2, z=3) + + def test_lambda_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_default) + + def test_lambda_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_default, 1) + + def test_lambda_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_default, y=2) + + def test_lambda_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_lambda_with_default, y=2, z=3) + + def test_function_no_args(self): + def func(): + pass + cliutils.validate_args(func) + + def _test_function_with_args(self, *args, **kwargs): + def func(x, y): + pass + cliutils.validate_args(func, *args, **kwargs) + + def test_function_positional_args(self): + self._test_function_with_args(1, 2) + + def test_function_kwargs(self): + self._test_function_with_args(x=1, y=2) + + def test_function_mixed_kwargs(self): + self._test_function_with_args(1, y=2) + + def test_function_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_args) + + def test_function_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_args, 1) + + def test_function_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_args, y=2) + + def _test_function_with_default(self, *args, **kwargs): + def func(x, y, z=3): + pass + cliutils.validate_args(func, *args, **kwargs) + + def test_function_positional_args_with_default(self): + self._test_function_with_default(1, 2) + + def test_function_kwargs_with_default(self): + self._test_function_with_default(x=1, y=2) + + def test_function_mixed_kwargs_with_default(self): + self._test_function_with_default(1, y=2) + + def test_function_positional_args_all_with_default(self): + self._test_function_with_default(1, 2, 3) + + def test_function_kwargs_all_with_default(self): + self._test_function_with_default(x=1, y=2, z=3) + + def test_function_mixed_kwargs_all_with_default(self): + self._test_function_with_default(1, y=2, z=3) + + def test_function_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_default) + + def test_function_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_default, 1) + + def test_function_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_default, y=2) + + def test_function_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_function_with_default, y=2, z=3) + + def test_bound_method_no_args(self): + class Foo(object): + def bar(self): + pass + cliutils.validate_args(Foo().bar) + + def _test_bound_method_with_args(self, *args, **kwargs): + class Foo(object): + def bar(self, x, y): + pass + cliutils.validate_args(Foo().bar, *args, **kwargs) + + def test_bound_method_positional_args(self): + self._test_bound_method_with_args(1, 2) + + def test_bound_method_kwargs(self): + self._test_bound_method_with_args(x=1, y=2) + + def test_bound_method_mixed_kwargs(self): + self._test_bound_method_with_args(1, y=2) + + def test_bound_method_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_args) + + def test_bound_method_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_args, 1) + + def test_bound_method_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_args, y=2) + + def _test_bound_method_with_default(self, *args, **kwargs): + class Foo(object): + def bar(self, x, y, z=3): + pass + cliutils.validate_args(Foo().bar, *args, **kwargs) + + def test_bound_method_positional_args_with_default(self): + self._test_bound_method_with_default(1, 2) + + def test_bound_method_kwargs_with_default(self): + self._test_bound_method_with_default(x=1, y=2) + + def test_bound_method_mixed_kwargs_with_default(self): + self._test_bound_method_with_default(1, y=2) + + def test_bound_method_positional_args_all_with_default(self): + self._test_bound_method_with_default(1, 2, 3) + + def test_bound_method_kwargs_all_with_default(self): + self._test_bound_method_with_default(x=1, y=2, z=3) + + def test_bound_method_mixed_kwargs_all_with_default(self): + self._test_bound_method_with_default(1, y=2, z=3) + + def test_bound_method_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_default) + + def test_bound_method_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_default, 1) + + def test_bound_method_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_default, y=2) + + def test_bound_method_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_bound_method_with_default, y=2, z=3) + + def test_unbound_method_no_args(self): + class Foo(object): + def bar(self): + pass + cliutils.validate_args(Foo.bar, Foo()) + + def _test_unbound_method_with_args(self, *args, **kwargs): + class Foo(object): + def bar(self, x, y): + pass + cliutils.validate_args(Foo.bar, Foo(), *args, **kwargs) + + def test_unbound_method_positional_args(self): + self._test_unbound_method_with_args(1, 2) + + def test_unbound_method_kwargs(self): + self._test_unbound_method_with_args(x=1, y=2) + + def test_unbound_method_mixed_kwargs(self): + self._test_unbound_method_with_args(1, y=2) + + def test_unbound_method_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_args) + + def test_unbound_method_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_args, 1) + + def test_unbound_method_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_args, y=2) + + def _test_unbound_method_with_default(self, *args, **kwargs): + class Foo(object): + def bar(self, x, y, z=3): + pass + cliutils.validate_args(Foo.bar, Foo(), *args, **kwargs) + + def test_unbound_method_positional_args_with_default(self): + self._test_unbound_method_with_default(1, 2) + + def test_unbound_method_kwargs_with_default(self): + self._test_unbound_method_with_default(x=1, y=2) + + def test_unbound_method_mixed_kwargs_with_default(self): + self._test_unbound_method_with_default(1, y=2) + + def test_unbound_method_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_default) + + def test_unbound_method_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_default, 1) + + def test_unbound_method_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_default, y=2) + + def test_unbound_method_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_unbound_method_with_default, y=2, z=3) + + def test_class_method_no_args(self): + class Foo(object): + @classmethod + def bar(cls): + pass + cliutils.validate_args(Foo.bar) + + def _test_class_method_with_args(self, *args, **kwargs): + class Foo(object): + @classmethod + def bar(cls, x, y): + pass + cliutils.validate_args(Foo.bar, *args, **kwargs) + + def test_class_method_positional_args(self): + self._test_class_method_with_args(1, 2) + + def test_class_method_kwargs(self): + self._test_class_method_with_args(x=1, y=2) + + def test_class_method_mixed_kwargs(self): + self._test_class_method_with_args(1, y=2) + + def test_class_method_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_args) + + def test_class_method_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_args, 1) + + def test_class_method_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_args, y=2) + + def _test_class_method_with_default(self, *args, **kwargs): + class Foo(object): + @classmethod + def bar(cls, x, y, z=3): + pass + cliutils.validate_args(Foo.bar, *args, **kwargs) + + def test_class_method_positional_args_with_default(self): + self._test_class_method_with_default(1, 2) + + def test_class_method_kwargs_with_default(self): + self._test_class_method_with_default(x=1, y=2) + + def test_class_method_mixed_kwargs_with_default(self): + self._test_class_method_with_default(1, y=2) + + def test_class_method_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_default) + + def test_class_method_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_default, 1) + + def test_class_method_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_default, y=2) + + def test_class_method_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_class_method_with_default, y=2, z=3) + + def test_static_method_no_args(self): + class Foo(object): + @staticmethod + def bar(): + pass + cliutils.validate_args(Foo.bar) + + def _test_static_method_with_args(self, *args, **kwargs): + class Foo(object): + @staticmethod + def bar(x, y): + pass + cliutils.validate_args(Foo.bar, *args, **kwargs) + + def test_static_method_positional_args(self): + self._test_static_method_with_args(1, 2) + + def test_static_method_kwargs(self): + self._test_static_method_with_args(x=1, y=2) + + def test_static_method_mixed_kwargs(self): + self._test_static_method_with_args(1, y=2) + + def test_static_method_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_args) + + def test_static_method_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_args, 1) + + def test_static_method_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_args, y=2) + + def _test_static_method_with_default(self, *args, **kwargs): + class Foo(object): + @staticmethod + def bar(x, y, z=3): + pass + cliutils.validate_args(Foo.bar, *args, **kwargs) + + def test_static_method_positional_args_with_default(self): + self._test_static_method_with_default(1, 2) + + def test_static_method_kwargs_with_default(self): + self._test_static_method_with_default(x=1, y=2) + + def test_static_method_mixed_kwargs_with_default(self): + self._test_static_method_with_default(1, y=2) + + def test_static_method_with_default_missing_args1(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_default) + + def test_static_method_with_default_missing_args2(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_default, 1) + + def test_static_method_with_default_missing_args3(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_default, y=2) + + def test_static_method_with_default_missing_args4(self): + self.assertRaises(cliutils.MissingArgs, + self._test_static_method_with_default, y=2, z=3) diff --git a/tests/unit/test_hacking.py b/tests/unit/test_hacking.py index 650580586b..b94c2c57a1 100644 --- a/tests/unit/test_hacking.py +++ b/tests/unit/test_hacking.py @@ -62,8 +62,8 @@ class HackingTestCase(test.TestCase): self.assertTrue(actual_msg.startswith("N303")) def test_check_wrong_logging_import(self): - bad_imports = ["from rally.openstack.common import log", - "import rally.openstack.common.log", + bad_imports = ["from oslo_log import log", + "import oslo_log", "import logging"] good_imports = ["from rally.common import log", "from rally.common.log", diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index 30af0834eb..01d8784596 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -24,23 +24,24 @@ from tests.unit import test class LogTestCase(test.TestCase): @mock.patch("rally.common.log.CONF") + @mock.patch("rally.common.log.handlers") @mock.patch("rally.common.log.oslogging") - def test_setup(self, mock_oslogger, mock_conf): + def test_setup(self, mock_oslogger, mock_handlers, mock_conf): proj = "fakep" version = "fakev" - mock_oslogger.ColorHandler.LEVEL_COLORS = { + mock_handlers.ColorHandler.LEVEL_COLORS = { logging.DEBUG: "debug_color"} mock_conf.rally_debug = True log.setup(proj, version) - self.assertIn(logging.RDEBUG, mock_oslogger.ColorHandler.LEVEL_COLORS) + self.assertIn(logging.RDEBUG, mock_handlers.ColorHandler.LEVEL_COLORS) self.assertEqual( - mock_oslogger.ColorHandler.LEVEL_COLORS[logging.DEBUG], - mock_oslogger.ColorHandler.LEVEL_COLORS[logging.RDEBUG]) + mock_handlers.ColorHandler.LEVEL_COLORS[logging.DEBUG], + mock_handlers.ColorHandler.LEVEL_COLORS[logging.RDEBUG]) - mock_oslogger.setup.assert_called_once_with(proj, version) + mock_oslogger.setup.assert_called_once_with(mock_conf, proj, version) mock_oslogger.getLogger(None).logger.setLevel.assert_called_once_with( logging.RDEBUG) @@ -49,27 +50,28 @@ class LogTestCase(test.TestCase): @mock.patch("rally.common.log.oslogging") def test_getLogger(self, mock_oslogger, mock_radapter, mock_pylogging): - proj = "fake" + name = "fake" vers = "fake" mock_oslogger._loggers = dict() - returned_logger = log.getLogger(proj, vers) + returned_logger = log.getLogger(name, vers) - self.assertIn(proj, mock_oslogger._loggers) - mock_radapter.assert_called_once_with(mock_pylogging.getLogger(proj), - proj, vers) - self.assertEqual(mock_oslogger._loggers[proj], returned_logger) + self.assertIn(name, mock_oslogger._loggers) + mock_radapter.assert_called_once_with( + mock_pylogging.getLogger(name), + {"project": "rally", "version": vers}) + self.assertEqual(mock_oslogger._loggers[name], returned_logger) class LogRallyContaxtAdapter(test.TestCase): @mock.patch("rally.common.log.logging") - @mock.patch("rally.common.log.oslogging.ContextAdapter") + @mock.patch("rally.common.log.oslogging.KeywordArgumentAdapter") def test_debug(self, mock_oslo_adapter, mock_logging): mock_logging.RDEBUG = 123 fake_msg = "fake message" - radapter = log.RallyContextAdapter(mock.MagicMock(), "fakep", "fakev") + radapter = log.RallyContextAdapter(mock.MagicMock(), "fakep") radapter.log = mock.MagicMock() radapter.debug(fake_msg) diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index a82d0b57f2..0da7d94580 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -22,7 +22,7 @@ import rally from rally.cmd import cliutils from tests.unit import test -RES_PATH = os.path.join(os.path.dirname(rally.__file__), os.pardir, "tools") +RES_PATH = os.path.join(os.path.dirname(rally.__file__), os.pardir, "etc") class BashCompletionTestCase(test.TestCase): @@ -39,5 +39,5 @@ class BashCompletionTestCase(test.TestCase): self.fail("bash completion script is outdated. " "New script is located at %s " "You may fix this by executing " - "`mv %s tools/rally.bash_completion`" % (new_filename, - new_filename)) + "`mv %s etc/rally.bash_completion`" % (new_filename, + new_filename)) diff --git a/tools/config/check_uptodate.sh b/tools/config/check_uptodate.sh deleted file mode 100755 index b3c6c6335a..0000000000 --- a/tools/config/check_uptodate.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -PROJECT_NAME=${PROJECT_NAME:-rally} -CFGFILE_NAME=${PROJECT_NAME}.conf.sample - -if [ -e etc/${PROJECT_NAME}/${CFGFILE_NAME} ]; then - CFGFILE=etc/${PROJECT_NAME}/${CFGFILE_NAME} -elif [ -e etc/${CFGFILE_NAME} ]; then - CFGFILE=etc/${CFGFILE_NAME} -else - echo "${0##*/}: can not find config file" - exit 1 -fi - -TEMPDIR=`mktemp -d /tmp/${PROJECT_NAME}.XXXXXX` -trap "rm -rf $TEMPDIR" EXIT - -tools/config/generate_sample.sh -b ./ -p ${PROJECT_NAME} -o ${TEMPDIR} - -if ! diff -u ${TEMPDIR}/${CFGFILE_NAME} ${CFGFILE} -then - echo "${0##*/}: ${PROJECT_NAME}.conf.sample is not up to date." - echo "${0##*/}: Please run ${0%%${0##*/}}generate_sample.sh." - exit 1 -fi diff --git a/tools/config/generate_sample.sh b/tools/config/generate_sample.sh deleted file mode 100755 index 130b0475f7..0000000000 --- a/tools/config/generate_sample.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash - -print_hint() { - echo "Try \`${0##*/} --help' for more information." >&2 -} - -PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:m:l:o: \ - --long help,base-dir:,package-name:,output-dir:,module:,library: -- "$@") - -if [ $? != 0 ] ; then print_hint ; exit 1 ; fi - -eval set -- "$PARSED_OPTIONS" - -while true; do - case "$1" in - -h|--help) - echo "${0##*/} [options]" - echo "" - echo "options:" - echo "-h, --help show brief help" - echo "-b, --base-dir=DIR project base directory" - echo "-p, --package-name=NAME project package name" - echo "-o, --output-dir=DIR file output directory" - echo "-m, --module=MOD extra python module to interrogate for options" - echo "-l, --library=LIB extra library that registers options for discovery" - exit 0 - ;; - -b|--base-dir) - shift - BASEDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -p|--package-name) - shift - PACKAGENAME=`echo $1` - shift - ;; - -o|--output-dir) - shift - OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` - shift - ;; - -m|--module) - shift - MODULES="$MODULES -m $1" - shift - ;; - -l|--library) - shift - LIBRARIES="$LIBRARIES -l $1" - shift - ;; - --) - break - ;; - esac -done - -BASEDIR=${BASEDIR:-`pwd`} -if ! [ -d $BASEDIR ] -then - echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 -elif [[ $BASEDIR != /* ]] -then - BASEDIR=$(cd "$BASEDIR" && pwd) -fi - -PACKAGENAME=${PACKAGENAME:-$(python setup.py --name)} -TARGETDIR=$BASEDIR/$PACKAGENAME -if ! [ -d $TARGETDIR ] -then - echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 -fi - -OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} -# NOTE(bnemec): Some projects put their sample config in etc/, -# some in etc/$PACKAGENAME/ -if [ -d $OUTPUTDIR/$PACKAGENAME ] -then - OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME -elif ! [ -d $OUTPUTDIR ] -then - echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 - exit 1 -fi - -BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` -find $TARGETDIR -type f -name "*.pyc" -delete -FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ - -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) - -RC_FILE="`dirname $0`/oslo.config.generator.rc" -if test -r "$RC_FILE" -then - source "$RC_FILE" -fi - -for mod in ${RALLY_CONFIG_GENERATOR_EXTRA_MODULES}; do - MODULES="$MODULES -m $mod" -done - -for lib in ${RALLY_CONFIG_GENERATOR_EXTRA_LIBRARIES}; do - LIBRARIES="$LIBRARIES -l $lib" -done - -export EVENTLET_NO_GREENDNS=yes - -OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) -[ "$OS_VARS" ] && eval "unset \$OS_VARS" -DEFAULT_MODULEPATH=rally.openstack.common.config.generator -MODULEPATH=${MODULEPATH:-$DEFAULT_MODULEPATH} -OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample -python -m $MODULEPATH $MODULES $LIBRARIES $FILES > $OUTPUTFILE - -# Hook to allow projects to append custom config file snippets -CONCAT_FILES=$(ls $BASEDIR/tools/config/*.conf.sample 2>/dev/null) -for CONCAT_FILE in $CONCAT_FILES; do - cat $CONCAT_FILE >> $OUTPUTFILE -done diff --git a/tools/config/oslo.config.generator.rc b/tools/config/oslo.config.generator.rc deleted file mode 100644 index 702c8444cc..0000000000 --- a/tools/config/oslo.config.generator.rc +++ /dev/null @@ -1,4 +0,0 @@ -# Environmental Variables that affect the automatic sample config generation. -# Additions to any of these variables are space delimited. See the "generate_sample.sh" -# script for the variables that can be used. -RALLY_CONFIG_GENERATOR_EXTRA_LIBRARIES='oslo.db' diff --git a/tox.ini b/tox.ini index 1fa53121eb..c755e6dc78 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,13 @@ commands = python setup.py testr --coverage --testr-args='{posargs}' changedir = doc/source commands = make html +[testenv:genconfig] +commands = + oslo-config-generator --output-file etc/rally/rally.conf.sample \ + --namespace rally \ + --namespace oslo.db \ + --namespace oslo.log + [tox:jenkins] downloadcache = ~/cache/pip @@ -41,7 +48,7 @@ downloadcache = ~/cache/pip ignore = E126,H703 show-source = true builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,*rally/openstack*,setup.py +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py [hacking] import_exceptions = rally.common.i18n