diff --git a/.gitignore b/.gitignore index 9cc41b7a7..215d3f3cc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ ChangeLog .tox .coverage *.egg-info/ +.eggs *.egg build/ doc/build/ diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 000000000..d41da69aa --- /dev/null +++ b/bandit.yaml @@ -0,0 +1,362 @@ +# optional: after how many files to update progress +#show_progress_every: 100 + +# optional: plugins directory name +#plugins_dir: plugins + +# optional: plugins discovery name pattern +plugin_name_pattern: '*.py' + +# optional: terminal escape sequences to display colors +#output_colors: +# DEFAULT: \033[0m +# HEADER: \033[95m +# LOW: \033[94m +# MEDIUM: \033[93m +# HIGH: \033[91m + +# optional: log format string +#log_format: "[%(module)s]\t%(levelname)s\t%(message)s" + +# globs of files which should be analyzed +include: + - '*.py' + - '*.pyw' + +# a list of strings, which if found in the path will cause files to be excluded +# for example /tests/ - to remove all files in tests directory +exclude_dirs: + - '/tests/' + +profiles: + oslo.messaging: + include: + - any_other_function_with_shell_equals_true + # Some occurrences in the olso.messaging code, but not much to do + # to get rid of these warnings, so just skip this. + # - assert_used + - blacklist_calls + - blacklist_import_func + - blacklist_imports + - exec_used + - execute_with_run_as_root_equals_true + - hardcoded_bind_all_interfaces + - hardcoded_password_string + - hardcoded_password_funcarg + - hardcoded_password_default + - hardcoded_sql_expressions + - hardcoded_tmp_directory + - jinja2_autoescape_false + - linux_commands_wildcard_injection + - paramiko_calls + - password_config_option_not_marked_secret + - request_with_no_cert_validation + - set_bad_file_permissions + - subprocess_popen_with_shell_equals_true + - subprocess_without_shell_equals_true + - start_process_with_a_shell + - start_process_with_no_shell + - start_process_with_partial_path + - ssl_with_bad_defaults + - ssl_with_bad_version + - ssl_with_no_version + # This might be nice to have, but we currently ignore a lot of + # exceptions during the cleanup phases, so this throws a lot + # false positives. + # - try_except_pass + - use_of_mako_templates + - weak_cryptographic_key + + XSS: + include: + - jinja2_autoescape_false + - use_of_mako_templates + + ShellInjection: + include: + - subprocess_popen_with_shell_equals_true + - subprocess_without_shell_equals_true + - any_other_function_with_shell_equals_true + - start_process_with_a_shell + - start_process_with_no_shell + - start_process_with_partial_path + exclude: + + SqlInjection: + include: + - hardcoded_sql_expressions + +blacklist_calls: + bad_name_sets: + - pickle: + qualnames: + - pickle.loads + - pickle.load + - pickle.Unpickler + - cPickle.loads + - cPickle.load + - cPickle.Unpickler + message: > + Pickle library appears to be in use, possible security issue. + - marshal: + qualnames: [marshal.load, marshal.loads] + message: > + Deserialization with the marshal module is possibly dangerous. + - md5: + qualnames: + - hashlib.md5 + - Crypto.Hash.MD2.new + - Crypto.Hash.MD4.new + - Crypto.Hash.MD5.new + - cryptography.hazmat.primitives.hashes.MD5 + message: Use of insecure MD2, MD4, or MD5 hash function. + - ciphers: + qualnames: + - Crypto.Cipher.ARC2.new + - Crypto.Cipher.ARC4.new + - Crypto.Cipher.Blowfish.new + - Crypto.Cipher.DES.new + - Crypto.Cipher.XOR.new + - cryptography.hazmat.primitives.ciphers.algorithms.ARC4 + - cryptography.hazmat.primitives.ciphers.algorithms.Blowfish + - cryptography.hazmat.primitives.ciphers.algorithms.IDEA + message: > + Use of insecure cipher {func}. Replace with a known secure + cipher such as AES. + level: HIGH + - cipher_modes: + qualnames: + - cryptography.hazmat.primitives.ciphers.modes.ECB + message: Use of insecure cipher mode {func}. + - mktemp_q: + qualnames: [tempfile.mktemp] + message: Use of insecure and deprecated function (mktemp). + - eval: + qualnames: [eval] + message: > + Use of possibly insecure function - consider using safer + ast.literal_eval. + - mark_safe: + names: [mark_safe] + message: > + Use of mark_safe() may expose cross-site scripting + vulnerabilities and should be reviewed. + - httpsconnection: + qualnames: [httplib.HTTPSConnection] + message: > + Use of HTTPSConnection does not provide security, see + https://wiki.openstack.org/wiki/OSSN/OSSN-0033 + - yaml_load: + qualnames: [yaml.load] + message: > + Use of unsafe yaml load. Allows instantiation of arbitrary + objects. Consider yaml.safe_load(). + - urllib_urlopen: + qualnames: + - urllib.urlopen + - urllib.urlretrieve + - urllib.URLopener + - urllib.FancyURLopener + - urllib2.urlopen + - urllib2.Request + message: > + Audit url open for permitted schemes. Allowing use of file:/ or + custom schemes is often unexpected. + - telnetlib: + qualnames: + - telnetlib.* + message: > + Telnet-related funtions are being called. Telnet is considered + insecure. Use SSH or some other encrypted protocol. + level: HIGH + # Most of this is based off of Christian Heimes' work on defusedxml: + # https://pypi.python.org/pypi/defusedxml/#defusedxml-sax + - xml_bad_cElementTree: + qualnames: + - xml.etree.cElementTree.parse + - xml.etree.cElementTree.iterparse + - xml.etree.cElementTree.fromstring + - xml.etree.cElementTree.XMLParser + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_ElementTree: + qualnames: + - xml.etree.ElementTree.parse + - xml.etree.ElementTree.iterparse + - xml.etree.ElementTree.fromstring + - xml.etree.ElementTree.XMLParser + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_expatreader: + qualnames: [xml.sax.expatreader.create_parser] + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_expatbuilder: + qualnames: + - xml.dom.expatbuilder.parse + - xml.dom.expatbuilder.parseString + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_sax: + qualnames: + - xml.sax.parse + - xml.sax.parseString + - xml.sax.make_parser + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_minidom: + qualnames: + - xml.dom.minidom.parse + - xml.dom.minidom.parseString + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_pulldom: + qualnames: + - xml.dom.pulldom.parse + - xml.dom.pulldom.parseString + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + - xml_bad_etree: + qualnames: + - lxml.etree.parse + - lxml.etree.fromstring + - lxml.etree.RestrictedElement + - lxml.etree.GlobalParserTLS + - lxml.etree.getDefaultParser + - lxml.etree.check_docinfo + message: > + Using {func} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {func} with its defusedxml + equivalent function. + + +shell_injection: + # Start a process using the subprocess module, or one of its wrappers. + subprocess: + - subprocess.Popen + - subprocess.call + - subprocess.check_call + - subprocess.check_output + - utils.execute + - utils.execute_with_timeout + # Start a process with a function vulnerable to shell injection. + shell: + - os.system + - os.popen + - os.popen2 + - os.popen3 + - os.popen4 + - popen2.popen2 + - popen2.popen3 + - popen2.popen4 + - popen2.Popen3 + - popen2.Popen4 + - commands.getoutput + - commands.getstatusoutput + # Start a process with a function that is not vulnerable to shell injection. + no_shell: + - os.execl + - os.execle + - os.execlp + - os.execlpe + - os.execv + - os.execve + - os.execvp + - os.execvpe + - os.spawnl + - os.spawnle + - os.spawnlp + - os.spawnlpe + - os.spawnv + - os.spawnve + - os.spawnvp + - os.spawnvpe + - os.startfile + +blacklist_imports: + bad_import_sets: + - telnet: + imports: [telnetlib] + level: HIGH + message: > + A telnet-related module is being imported. Telnet is + considered insecure. Use SSH or some other encrypted protocol. + - info_libs: + imports: [pickle, cPickle, subprocess, Crypto] + level: LOW + message: > + Consider possible security implications associated with + {module} module. + + # Most of this is based off of Christian Heimes' work on defusedxml: + # https://pypi.python.org/pypi/defusedxml/#defusedxml-sax + + - xml_libs: + imports: + - xml.etree.cElementTree + - xml.etree.ElementTree + - xml.sax.expatreader + - xml.sax + - xml.dom.expatbuilder + - xml.dom.minidom + - xml.dom.pulldom + - lxml.etree + - lxml + message: > + Using {module} to parse untrusted XML data is known to be + vulnerable to XML attacks. Replace {module} with the equivalent + defusedxml package. + level: LOW + - xml_libs_high: + imports: [xmlrpclib] + message: > + Using {module} to parse untrusted XML data is known to be + vulnerable to XML attacks. Use defused.xmlrpc.monkey_patch() + function to monkey-patch xmlrpclib and mitigate XML + vulnerabilities. + level: HIGH + +hardcoded_tmp_directory: + tmp_dirs: [/tmp, /var/tmp, /dev/shm] + +hardcoded_password: + # Support for full path, relative path and special "%(site_data_dir)s" + # substitution (/usr/{local}/share) + word_list: "%(site_data_dir)s/wordlist/default-passwords" + +ssl_with_bad_version: + bad_protocol_versions: + - PROTOCOL_SSLv2 + - SSLv2_METHOD + - SSLv23_METHOD + - PROTOCOL_SSLv3 # strict option + - PROTOCOL_TLSv1 # strict option + - SSLv3_METHOD # strict option + - TLSv1_METHOD # strict option + +password_config_option_not_marked_secret: + function_names: + - oslo.config.cfg.StrOpt + - oslo_config.cfg.StrOpt + +execute_with_run_as_root_equals_true: + function_names: + - ceilometer.utils.execute + - cinder.utils.execute + - neutron.agent.linux.utils.execute + - nova.utils.execute + - nova.utils.trycmd diff --git a/doc/source/FAQ.rst b/doc/source/FAQ.rst index ebb6fe3c2..2a67bba8c 100644 --- a/doc/source/FAQ.rst +++ b/doc/source/FAQ.rst @@ -6,7 +6,8 @@ I don't need notifications on the message bus. How do I disable them? ===================================================================== Notification messages can be disabled using the ``noop`` notify -driver. Set ``notification_driver = noop`` in your configuration file. +driver. Set ``driver = noop`` in your configuration file under the +[oslo_messaging_notifications] section. Why does the notification publisher create queues, too? Shouldn't the subscriber do that? ========================================================================================= @@ -26,9 +27,9 @@ notification "level". The default topic is ``notifications``, so an info-level notification is published to the topic ``notifications.info``. A subscriber queue of the same name is created automatically for each of these topics. To change the queue names, -change the notification topic using the ``notification_topics`` -configuration option. The option accepts a list of values, so it is -possible to publish to multiple topics. +change the notification topic using the ``topics`` +configuration option in ``[oslo_messaging_notifications]``. The option +accepts a list of values, so it is possible to publish to multiple topics. What are the other choices of notification drivers available? ============================================================= diff --git a/doc/source/index.rst b/doc/source/index.rst index 5f45af872..9a6873cbd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -23,6 +23,7 @@ Contents opts conffixture drivers + supported-messaging-drivers AMQP1.0 zmq_driver FAQ diff --git a/doc/source/supported-messaging-drivers.rst b/doc/source/supported-messaging-drivers.rst new file mode 100644 index 000000000..75c7a844f --- /dev/null +++ b/doc/source/supported-messaging-drivers.rst @@ -0,0 +1,60 @@ +============================= + Supported Messaging Drivers +============================= + +RabbitMQ may not be sufficient for the entire community as the community +grows. Pluggability is still something we should maintain, but we should +have a very high standard for drivers that are shipped and documented +as being supported. + +This document defines a very clear policy as to the requirements +for drivers to be carried in oslo.messaging and thus supported by the +OpenStack community as a whole. We will deprecate any drivers that do not +meet the requirements, and announce said deprecations in any appropriate +channels to give users time to signal their needs. Deprecation will last +for two release cycles before removing the code. We will also review and +update documentation to annotate which drivers are supported and which +are deprecated given these policies + +Policy +------ + +Testing +~~~~~~~ + +* Must have unit and/or functional test coverage of at least 60% as + reported by coverage report. Unit tests must be run for all versions + of python oslo.messaging currently gates on. + +* Must have integration testing including at least 3 popular oslo.messaging + dependents, preferrably at the minimum a devstack-gate job with Nova, + Cinder, and Neutron. + +* All testing above must be voting in the gate of oslo.messaging. + +Documentation +~~~~~~~~~~~~~ + +* Must have a reasonable amount of documentation including documentation + in the official OpenStack deployment guide. + +Support +~~~~~~~ + +* Must have at least two individuals from the community commited to + triaging and fixing bugs, and responding to test failures in a timely + manner. + +Prospective Drivers +~~~~~~~~~~~~~~~~~~~ + +* Drivers that intend to meet the requirements above, but that do not yet + meet them will be given one full release cycle, or 6 months, whichever + is longer, to comply before being marked for deprecation. Their use, + however, will not be supported by the community. This will prevent a + chicken and egg problem for new drivers. + +.. note:: + + This work is licensed under a Creative Commons Attribution 3.0 Unported License. + http://creativecommons.org/licenses/by/3.0/legalcode diff --git a/doc/source/transport.rst b/doc/source/transport.rst index 547198aa8..3449e9b7d 100644 --- a/doc/source/transport.rst +++ b/doc/source/transport.rst @@ -25,6 +25,4 @@ different 3rd party libraries that don't ensure that. In certain cases, with some drivers, it does work: * rabbit: works only if no connection have already been established. -* qpid: doesn't work (The qpid library has a global state that uses - file descriptors that can't be reset) * amqp1: works diff --git a/doc/source/zmq_driver.rst b/doc/source/zmq_driver.rst index eff2bf3ce..da4d0abc5 100644 --- a/doc/source/zmq_driver.rst +++ b/doc/source/zmq_driver.rst @@ -8,9 +8,9 @@ ZeroMQ Driver Deployment Guide Introduction ============ -0MQ (also known as ZeroMQ or zmq) looks like an embeddable -networking library but acts like a concurrency framework. It gives -you sockets that carry atomic messages across various transports +0MQ (also known as ZeroMQ or zmq) is embeddable networking library +but acts like a concurrency framework. It gives you sockets +that carry atomic messages across various transports like in-process, inter-process, TCP, and multicast. You can connect sockets N-to-N with patterns like fan-out, pub-sub, task distribution, and request-reply. It's fast enough to be the fabric for clustered @@ -45,7 +45,7 @@ Juno release, as almost all the core projects in OpenStack have switched to oslo_messaging, ZeroMQ can be the only RPC driver across the OpenStack cluster. This document provides deployment information for this driver in oslo_messaging. -Other than AMQP-based drivers, like RabbitMQ or Qpid, ZeroMQ doesn't have +Other than AMQP-based drivers, like RabbitMQ, ZeroMQ doesn't have any central brokers in oslo.messaging, instead, each host (running OpenStack services) is both ZeroMQ client and server. As a result, each host needs to listen to a certain TCP port for incoming connections and directly connect @@ -96,8 +96,9 @@ must be set to the hostname of the current node. rpc_backend = zmq rpc_zmq_host = {hostname} + Match Making (mandatory) -------------------------- +------------------------ The ZeroMQ driver implements a matching capability to discover hosts available for communication when sending to a bare topic. This allows broker-less @@ -105,35 +106,20 @@ communications. The MatchMaker is pluggable and it provides two different MatchMaker classes. -MatchMakerLocalhost: default matchmaker driver for all-in-one scenario (messages +DummyMatchMaker: default matchmaker driver for all-in-one scenario (messages are sent to itself). -MatchMakerRing: loads a static hash table from a JSON file, sends messages to -a certain host via directed topics or cycles hosts per bare topic and supports -broker-less fanout messaging. On fanout messages returns an array of directed -topics (messages are sent to all destinations). - -MatchMakerRedis: loads the hash table from a remote Redis server, supports +RedisMatchMaker: loads the hash table from a remote Redis server, supports dynamic host/topic registrations, host expiration, and hooks for consuming applications to acknowledge or neg-acknowledge topic.host service availability. To set the MatchMaker class, use option 'rpc_zmq_matchmaker' in [DEFAULT]. - rpc_zmq_matchmaker = local - or - rpc_zmq_matchmaker = ring + rpc_zmq_matchmaker = dummy or rpc_zmq_matchmaker = redis -To specify the ring file for MatchMakerRing, use option 'ringfile' in -[matchmaker_ring]. - -For example:: - - [matchmaker_ring] - ringfile = /etc/oslo/oslo_matchmaker_ring.json - -To specify the Redis server for MatchMakerRedis, use options in +To specify the Redis server for RedisMatchMaker, use options in [matchmaker_redis] of each project. [matchmaker_redis] @@ -141,47 +127,36 @@ To specify the Redis server for MatchMakerRedis, use options in port = 6379 password = None + MatchMaker Data Source (mandatory) ------------------------------------ +---------------------------------- MatchMaker data source is stored in files or Redis server discussed in the previous section. How to make up the database is the key issue for making ZeroMQ driver work. -If deploying the MatchMakerRing, a ring file is required. The format of the ring -file should contain a hash where each key is a base topic and the values are -hostname arrays to be sent to. - -For example:: - - /etc/oslo/oslo_matchmaker_ring.json - { - "scheduler": ["host1", "host2"], - "conductor": ["host1", "host2"], - } - -The AMQP-based methods like RabbitMQ and Qpid don't require any knowledge -about the source and destination of any topic. However, ZeroMQ driver -with MatchMakerRing does. The challenging task is that you should learn -and get all the (K, V) pairs from each OpenStack project to make up the -matchmaker ring file. - -If deploying the MatchMakerRedis, a Redis server is required. Each (K, V) pair +If deploying the RedisMatchMaker, a Redis server is required. Each (K, V) pair stored in Redis is that the key is a base topic and the corresponding values are hostname arrays to be sent to. -Message Receivers (mandatory) -------------------------------- -Each machine running OpenStack services, or sending RPC messages, must run the -'oslo-messaging-zmq-receiver' daemon. This receives replies to call requests and -routes responses via IPC to blocked callers. +Proxy to avoid blocking (optional) +---------------------------------- -The way that deploy the receiver process is to run it under a new user 'oslo' -and give all openstack daemons access via group membership of 'oslo' - this -supports using /var/run/openstack as a shared IPC directory for all openstack -processes, allowing different services to be hosted on the same server, served -by a single oslo-messaging-zmq-receiver process. +Each machine running OpenStack services, or sending RPC messages, may run the +'oslo-messaging-zmq-broker' daemon. This is needed to avoid blocking +if a listener (server) appears after the sender (client). + +Running the local broker (proxy) or not is defined by the option 'zmq_use_broker' +(True by default). This option can be set in [DEFAULT] section. + +For example:: + + zmq_use_broker = False + + +In case of using the broker all publishers (clients) talk to servers over +the local broker connecting to it via IPC transport. The IPC runtime directory, 'rpc_zmq_ipc_dir', can be set in [DEFAULT] section. @@ -191,28 +166,14 @@ For example:: The parameters for the script oslo-messaging-zmq-receiver should be:: - oslo-messaging-zmq-receiver + oslo-messaging-zmq-broker --config-file /etc/oslo/zeromq.conf - --log-file /var/log/oslo/zmq-receiver.log + --log-file /var/log/oslo/zmq-broker.log You can specify ZeroMQ options in /etc/oslo/zeromq.conf if necessary. -Thread Pool (optional) ------------------------ - -Each service will launch threads for incoming requests. These threads are -maintained via a pool, the maximum number of threads is limited by -rpc_thread_pool_size. The default value is 1024. (This is a common RPC -configuration variable, also applicable to Kombu and Qpid) - -This configuration can be set in [DEFAULT] section. - -For example:: - - rpc_thread_pool_size = 1024 - Listening Address (optional) ------------------------------- +---------------------------- All services bind to an IP address or Ethernet adapter. By default, all services bind to '*', effectively binding to 0.0.0.0. This may be changed with the option @@ -224,18 +185,40 @@ For example:: rpc_zmq_bind_address = * +Currently zmq driver uses dynamic port binding mechanism, which means that +each listener will allocate port of a random number. Ports range is controlled +by two options 'rpc_zmq_min_port' and 'rpc_zmq_max_port'. Change them to +restrict current service's port binding range. 'rpc_zmq_bind_port_retries' +controls number of retries before 'ports range exceeded' failure. + +For example:: + + rpc_zmq_min_port = 9050 + rpc_zmq_max_port = 10050 + rpc_zmq_bind_port_retries = 100 + + DevStack Support ---------------- ZeroMQ driver has been supported by DevStack. The configuration is as follows:: - ENABLED_SERVICES+=,-rabbit,-qpid,zeromq + ENABLED_SERVICES+=,-rabbit,zeromq ZEROMQ_MATCHMAKER=redis +In local.conf [localrc] section need to enable zmq plugin which lives in +`devstack-plugin-zmq`_ repository. + +For example:: + + enable_plugin zmq https://github.com/openstack/devstack-plugin-zmq.git + +.. _devstack-plugin-zmq: https://github.com/openstack/devstack-plugin-zmq.git + + Current Status ---------------- +-------------- The current development status of ZeroMQ driver is shown in `wiki`_. .. _wiki: https://wiki.openstack.org/ZeroMQ - diff --git a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-error.po b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-error.po index 1fce8b65e..2a8f94dba 100644 --- a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-error.po +++ b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-error.po @@ -8,19 +8,18 @@ # OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging 2.5.1.dev7\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-09-16 18:44+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" "PO-Revision-Date: 2015-08-27 12:47+0000\n" "Last-Translator: Andi Chandler \n" -"Language-Team: English (United Kingdom) (http://www.transifex.com/openstack/" -"oslomessaging/language/en_GB/)\n" +"Language-Team: English (United Kingdom)\n" "Language: en-GB\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Generated-By: Babel 2.0\n" "X-Generator: Zanata 3.7.1\n" #, python-format diff --git a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-info.po b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-info.po index 9280d3220..6a9211d95 100644 --- a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-info.po +++ b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-info.po @@ -8,19 +8,18 @@ # OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging 2.5.1.dev7\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-09-16 18:44+0000\n" +"POT-Creation-Date: 2015-10-23 06:26+0000\n" "PO-Revision-Date: 2015-08-27 12:47+0000\n" "Last-Translator: Andi Chandler \n" -"Language-Team: English (United Kingdom) (http://www.transifex.com/openstack/" -"oslomessaging/language/en_GB/)\n" +"Language-Team: English (United Kingdom)\n" "Language: en-GB\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Generated-By: Babel 2.0\n" "X-Generator: Zanata 3.7.1\n" #, python-format diff --git a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-warning.po b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-warning.po index 61e720610..ca64daf3a 100644 --- a/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-warning.po +++ b/oslo.messaging/locale/en_GB/LC_MESSAGES/oslo.messaging-log-warning.po @@ -8,28 +8,24 @@ # OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging 2.5.1.dev7\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-09-16 18:44+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" "PO-Revision-Date: 2015-08-27 12:55+0000\n" "Last-Translator: Andi Chandler \n" -"Language-Team: English (United Kingdom) (http://www.transifex.com/openstack/" -"oslomessaging/language/en_GB/)\n" +"Language-Team: English (United Kingdom)\n" "Language: en-GB\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Generated-By: Babel 2.0\n" "X-Generator: Zanata 3.7.1\n" #, python-format msgid "Failed to load any notifiers for %s" msgstr "Failed to load any notifiers for %s" -msgid "start/stop/wait must be called in the same thread" -msgstr "start/stop/wait must be called in the same thread" - msgid "" "wait() should be called after stop() as it waits for existing messages to " "finish processing" diff --git a/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-error.po b/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-error.po index a0ff82857..18978068a 100644 --- a/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-error.po +++ b/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-error.po @@ -6,21 +6,22 @@ # Translators: # Adriana Chisco Landazábal , 2015 # Miriam Godinez , 2015 +# OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-09-08 06:18+0000\n" -"PO-Revision-Date: 2015-09-07 22:46+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" +"PO-Revision-Date: 2015-09-07 10:46+0000\n" "Last-Translator: Miriam Godinez \n" -"Language-Team: Spanish (http://www.transifex.com/openstack/oslomessaging/" -"language/es/)\n" +"Language-Team: Spanish\n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Generated-By: Babel 2.0\n" +"X-Generator: Zanata 3.7.1\n" #, python-format msgid "An exception occurred processing the API call: %s " diff --git a/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-warning.po b/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-warning.po deleted file mode 100644 index f7970147c..000000000 --- a/oslo.messaging/locale/es/LC_MESSAGES/oslo.messaging-log-warning.po +++ /dev/null @@ -1,26 +0,0 @@ -# Translations template for oslo.messaging. -# Copyright (C) 2015 ORGANIZATION -# This file is distributed under the same license as the oslo.messaging -# project. -# -# Translators: -# Adriana Chisco Landazábal , 2015 -# Lucía Pradillos , 2015 -msgid "" -msgstr "" -"Project-Id-Version: oslo.messaging\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-08-26 06:33+0000\n" -"PO-Revision-Date: 2015-08-26 03:46+0000\n" -"Last-Translator: Lucía Pradillos \n" -"Language-Team: Spanish (http://www.transifex.com/openstack/oslomessaging/" -"language/es/)\n" -"Language: es\n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" - -msgid "start/stop/wait must be called in the same thread" -msgstr "empezar/parar/esperar debe ser llamado en el mismo hilo" diff --git a/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-error.po b/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-error.po index d6931b3e2..585e8b228 100644 --- a/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-error.po +++ b/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-error.po @@ -5,21 +5,22 @@ # # Translators: # Maxime COQUEREL , 2014 +# OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-08-04 06:29+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" "PO-Revision-Date: 2014-09-25 08:57+0000\n" "Last-Translator: Maxime COQUEREL \n" -"Language-Team: French (http://www.transifex.com/openstack/oslomessaging/" -"language/fr/)\n" +"Language-Team: French\n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" +"Generated-By: Babel 2.0\n" +"X-Generator: Zanata 3.7.1\n" #, python-format msgid "An exception occurred processing the API call: %s " diff --git a/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-warning.po b/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-warning.po deleted file mode 100644 index c7a5cc23d..000000000 --- a/oslo.messaging/locale/fr/LC_MESSAGES/oslo.messaging-log-warning.po +++ /dev/null @@ -1,26 +0,0 @@ -# Translations template for oslo.messaging. -# Copyright (C) 2015 ORGANIZATION -# This file is distributed under the same license as the oslo.messaging -# project. -# -# Translators: -# Lucas Mascaro , 2015 -# Maxime COQUEREL , 2014 -msgid "" -msgstr "" -"Project-Id-Version: oslo.messaging\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-08-18 06:33+0000\n" -"PO-Revision-Date: 2015-08-17 22:45+0000\n" -"Last-Translator: Lucas Mascaro \n" -"Language-Team: French (http://www.transifex.com/openstack/oslomessaging/" -"language/fr/)\n" -"Language: fr\n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" -"Plural-Forms: nplurals=2; plural=(n > 1);\n" - -msgid "start/stop/wait must be called in the same thread" -msgstr "start/stop/wait doivent être appellés dans le même thread " diff --git a/oslo.messaging/locale/oslo.messaging-log-critical.pot b/oslo.messaging/locale/oslo.messaging-log-critical.pot deleted file mode 100644 index d921c5a9c..000000000 --- a/oslo.messaging/locale/oslo.messaging-log-critical.pot +++ /dev/null @@ -1,20 +0,0 @@ -# Translations template for oslo.messaging. -# Copyright (C) 2015 ORGANIZATION -# This file is distributed under the same license as the oslo.messaging -# project. -# FIRST AUTHOR , 2015. -# -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: oslo.messaging 2.1.0\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-07-29 06:39+0000\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=utf-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" - diff --git a/oslo.messaging/locale/oslo.messaging-log-warning.pot b/oslo.messaging/locale/oslo.messaging-log-warning.pot index de9167049..2dbd51efd 100644 --- a/oslo.messaging/locale/oslo.messaging-log-warning.pot +++ b/oslo.messaging/locale/oslo.messaging-log-warning.pot @@ -7,20 +7,16 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: oslo.messaging 2.4.1.dev1\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-08-26 06:33+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" - -#: oslo_messaging/server.py:145 -msgid "start/stop/wait must be called in the same thread" -msgstr "" +"Generated-By: Babel 2.1.1\n" #: oslo_messaging/server.py:178 msgid "" @@ -28,6 +24,14 @@ msgid "" " finish processing" msgstr "" +#: oslo_messaging/server.py:191 +#, python-format +msgid "" +"wait() should have been called after stop() as wait() waits for existing " +"messages to finish processing, it has been %0.2f seconds and stop() still" +" has not been called" +msgstr "" + #: oslo_messaging/notify/_impl_routing.py:80 #, python-format msgid "Failed to load any notifiers for %s" diff --git a/oslo.messaging/locale/ru/LC_MESSAGES/oslo.messaging-log-error.po b/oslo.messaging/locale/ru/LC_MESSAGES/oslo.messaging-log-error.po index 14a6dc7b6..cf613ada1 100644 --- a/oslo.messaging/locale/ru/LC_MESSAGES/oslo.messaging-log-error.po +++ b/oslo.messaging/locale/ru/LC_MESSAGES/oslo.messaging-log-error.po @@ -5,23 +5,24 @@ # # Translators: # kogamatranslator49 , 2015 +# OpenStack Infra , 2015. #zanata msgid "" msgstr "" -"Project-Id-Version: oslo.messaging\n" +"Project-Id-Version: oslo.messaging 2.7.1.dev15\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2015-08-04 06:29+0000\n" +"POT-Creation-Date: 2015-10-23 06:27+0000\n" "PO-Revision-Date: 2015-07-05 11:39+0000\n" "Last-Translator: kogamatranslator49 \n" -"Language-Team: Russian (http://www.transifex.com/openstack/oslomessaging/" -"language/ru/)\n" +"Language-Team: Russian\n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.0\n" "Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" "%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || (n" "%100>=11 && n%100<=14)? 2 : 3);\n" +"Generated-By: Babel 2.0\n" +"X-Generator: Zanata 3.7.1\n" #, python-format msgid "An exception occurred processing the API call: %s " diff --git a/oslo_messaging/_drivers/amqp.py b/oslo_messaging/_drivers/amqp.py index 55fb9d49f..06a59f846 100644 --- a/oslo_messaging/_drivers/amqp.py +++ b/oslo_messaging/_drivers/amqp.py @@ -19,7 +19,7 @@ Shared code between AMQP based openstack.common.rpc implementations. The code in this module is shared between the rpc implementations based on -AMQP. Specifically, this includes impl_kombu and impl_qpid. impl_carrot also +AMQP. Specifically, this includes impl_kombu. impl_carrot also uses AMQP, but is deprecated and predates this code. """ @@ -31,7 +31,6 @@ from oslo_config import cfg import six from oslo_messaging._drivers import common as rpc_common -from oslo_messaging._drivers import pool deprecated_durable_opts = [ cfg.DeprecatedOpt('amqp_durable_queues', @@ -49,139 +48,11 @@ amqp_opts = [ default=False, deprecated_group='DEFAULT', help='Auto-delete queues in AMQP.'), - cfg.BoolOpt('send_single_reply', - default=False, - help='Send a single AMQP reply to call message. The current ' - 'behaviour since oslo-incubator is to send two AMQP ' - 'replies - first one with the payload, a second one to ' - 'ensure the other have finish to send the payload. We ' - 'are going to remove it in the N release, but we must ' - 'keep backward compatible at the same time. This option ' - 'provides such compatibility - it defaults to False in ' - 'Liberty and can be turned on for early adopters with a ' - 'new installations or for testing. Please note, that ' - 'this option will be removed in the Mitaka release.') ] UNIQUE_ID = '_unique_id' LOG = logging.getLogger(__name__) -# NOTE(sileht): Even if rabbit/qpid have only one Connection class, -# this connection can be used for two purposes: -# * wait and receive amqp messages (only do read stuffs on the socket) -# * send messages to the broker (only do write stuffs on the socket) -# The code inside a connection class is not concurrency safe. -# Using one Connection class instance for doing both, will result -# of eventlet complaining of multiple greenthreads that read/write the -# same fd concurrently... because 'send' and 'listen' run in different -# greenthread. -# So, a connection cannot be shared between thread/greenthread and -# this two variables permit to define the purpose of the connection -# to allow drivers to add special handling if needed (like heatbeat). -# amqp drivers create 3 kind of connections: -# * driver.listen*(): each call create a new 'PURPOSE_LISTEN' connection -# * driver.send*(): a pool of 'PURPOSE_SEND' connections is used -# * driver internally have another 'PURPOSE_LISTEN' connection dedicated -# to wait replies of rpc call -PURPOSE_LISTEN = 'listen' -PURPOSE_SEND = 'send' - - -class ConnectionPool(pool.Pool): - """Class that implements a Pool of Connections.""" - def __init__(self, conf, rpc_conn_pool_size, url, connection_cls): - self.connection_cls = connection_cls - self.conf = conf - self.url = url - super(ConnectionPool, self).__init__(rpc_conn_pool_size) - self.reply_proxy = None - - # TODO(comstud): Timeout connections not used in a while - def create(self, purpose=None): - if purpose is None: - purpose = PURPOSE_SEND - LOG.debug('Pool creating new connection') - return self.connection_cls(self.conf, self.url, purpose) - - def empty(self): - for item in self.iter_free(): - item.close() - - -class ConnectionContext(rpc_common.Connection): - """The class that is actually returned to the create_connection() caller. - - This is essentially a wrapper around Connection that supports 'with'. - It can also return a new Connection, or one from a pool. - - The function will also catch when an instance of this class is to be - deleted. With that we can return Connections to the pool on exceptions - and so forth without making the caller be responsible for catching them. - If possible the function makes sure to return a connection to the pool. - """ - - def __init__(self, connection_pool, purpose): - """Create a new connection, or get one from the pool.""" - self.connection = None - self.connection_pool = connection_pool - pooled = purpose == PURPOSE_SEND - if pooled: - self.connection = connection_pool.get() - else: - # a non-pooled connection is requested, so create a new connection - self.connection = connection_pool.create(purpose) - self.pooled = pooled - self.connection.pooled = pooled - - def __enter__(self): - """When with ConnectionContext() is used, return self.""" - return self - - def _done(self): - """If the connection came from a pool, clean it up and put it back. - If it did not come from a pool, close it. - """ - if self.connection: - if self.pooled: - # Reset the connection so it's ready for the next caller - # to grab from the pool - try: - self.connection.reset() - except Exception: - LOG.exception("Fail to reset the connection, drop it") - try: - self.connection.close() - except Exception: - pass - self.connection = self.connection_pool.create() - finally: - self.connection_pool.put(self.connection) - else: - try: - self.connection.close() - except Exception: - pass - self.connection = None - - def __exit__(self, exc_type, exc_value, tb): - """End of 'with' statement. We're done here.""" - self._done() - - def __del__(self): - """Caller is done with this connection. Make sure we cleaned up.""" - self._done() - - def close(self): - """Caller is done with this connection.""" - self._done() - - def __getattr__(self, key): - """Proxy all other calls to the Connection instance.""" - if self.connection: - return getattr(self.connection, key) - else: - raise rpc_common.InvalidRPCConnectionReuse() - class RpcContext(rpc_common.CommonRpcContext): """Context that supports replying to a rpc.call.""" diff --git a/oslo_messaging/_drivers/amqpdriver.py b/oslo_messaging/_drivers/amqpdriver.py index d3405086a..21cda4373 100644 --- a/oslo_messaging/_drivers/amqpdriver.py +++ b/oslo_messaging/_drivers/amqpdriver.py @@ -17,6 +17,7 @@ __all__ = ['AMQPDriverBase'] import logging import threading +import time import uuid import cachetools @@ -47,45 +48,27 @@ class AMQPIncomingMessage(base.IncomingMessage): self.requeue_callback = message.requeue self._obsolete_reply_queues = obsolete_reply_queues - def _send_reply(self, conn, reply=None, failure=None, - ending=False, log_failure=True): - if (self.reply_q and - not self._obsolete_reply_queues.reply_q_valid(self.reply_q, - self.msg_id)): + def _send_reply(self, conn, reply=None, failure=None, log_failure=True): + if not self._obsolete_reply_queues.reply_q_valid(self.reply_q, + self.msg_id): return if failure: failure = rpc_common.serialize_remote_exception(failure, log_failure) - - msg = {'result': reply, 'failure': failure} - if ending: - msg['ending'] = True - + # NOTE(sileht): ending can be removed in N*, see Listener.wait() + # for more detail. + msg = {'result': reply, 'failure': failure, 'ending': True, + '_msg_id': self.msg_id} rpc_amqp._add_unique_id(msg) unique_id = msg[rpc_amqp.UNIQUE_ID] - # If a reply_q exists, add the msg_id to the reply and pass the - # reply_q to direct_send() to use it as the response queue. - # Otherwise use the msg_id for backward compatibility. - if self.reply_q: - msg['_msg_id'] = self.msg_id - try: - if ending: - LOG.debug("sending reply msg_id: %(msg_id)s " - "reply queue: %(reply_q)s" % { - 'msg_id': self.msg_id, - 'unique_id': unique_id, - 'reply_q': self.reply_q}) - conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg)) - except rpc_amqp.AMQPDestinationNotFound: - self._obsolete_reply_queues.add(self.reply_q, self.msg_id) - else: - # TODO(sileht): look at which version of oslo-incubator rpc - # send need this, but I guess this is older than icehouse - # if this is icehouse, we can drop this at Mitaka - # if this is havana, we can drop this now. - conn.direct_send(self.msg_id, rpc_common.serialize_msg(msg)) + LOG.debug("sending reply msg_id: %(msg_id)s " + "reply queue: %(reply_q)s" % { + 'msg_id': self.msg_id, + 'unique_id': unique_id, + 'reply_q': self.reply_q}) + conn.direct_send(self.reply_q, rpc_common.serialize_msg(msg)) def reply(self, reply=None, failure=None, log_failure=True): if not self.msg_id: @@ -94,19 +77,41 @@ class AMQPIncomingMessage(base.IncomingMessage): return # NOTE(sileht): return without hold the a connection if possible - if (self.reply_q and - not self._obsolete_reply_queues.reply_q_valid(self.reply_q, - self.msg_id)): + if not self._obsolete_reply_queues.reply_q_valid(self.reply_q, + self.msg_id): return - with self.listener.driver._get_connection( - rpc_amqp.PURPOSE_SEND) as conn: - if self.listener.driver.send_single_reply: - self._send_reply(conn, reply, failure, log_failure=log_failure, - ending=True) - else: - self._send_reply(conn, reply, failure, log_failure=log_failure) - self._send_reply(conn, ending=True) + # NOTE(sileht): we read the configuration value from the driver + # to be able to backport this change in previous version that + # still have the qpid driver + duration = self.listener.driver.missing_destination_retry_timeout + timer = rpc_common.DecayingTimer(duration=duration) + timer.start() + + while True: + try: + with self.listener.driver._get_connection( + rpc_common.PURPOSE_SEND) as conn: + self._send_reply(conn, reply, failure, + log_failure=log_failure) + return + except rpc_amqp.AMQPDestinationNotFound: + if timer.check_return() > 0: + LOG.debug(("The reply %(msg_id)s cannot be sent " + "%(reply_q)s reply queue don't exist, " + "retrying...") % { + 'msg_id': self.msg_id, + 'reply_q': self.reply_q}) + time.sleep(0.25) + else: + self._obsolete_reply_queues.add(self.reply_q, self.msg_id) + LOG.info(_LI("The reply %(msg_id)s cannot be sent " + "%(reply_q)s reply queue don't exist after " + "%(duration)s sec abandoning...") % { + 'msg_id': self.msg_id, + 'reply_q': self.reply_q, + 'duration': duration}) + return def acknowledge(self): self.acknowledge_callback() @@ -187,12 +192,8 @@ class AMQPListener(base.Listener): unique_id = self.msg_id_cache.check_duplicate_message(message) - if ctxt.reply_q: - LOG.debug( - "received message msg_id: %(msg_id)s reply to %(queue)s" % { - 'queue': ctxt.reply_q, 'msg_id': ctxt.msg_id}) - else: - LOG.debug("received message unique_id: %s " % unique_id) + LOG.debug("received message msg_id: %(msg_id)s reply to %(queue)s" % { + 'queue': ctxt.reply_q, 'msg_id': ctxt.msg_id}) self.incoming.append(AMQPIncomingMessage(self, ctxt.to_dict(), @@ -202,6 +203,7 @@ class AMQPListener(base.Listener): ctxt.reply_q, self._obsolete_reply_queues)) + @base.batch_poll_helper def poll(self, timeout=None): while not self._stopped.is_set(): if self.incoming: @@ -345,10 +347,10 @@ class ReplyWaiter(object): class AMQPDriverBase(base.BaseDriver): + missing_destination_retry_timeout = 0 def __init__(self, conf, url, connection_pool, - default_exchange=None, allowed_remote_exmods=None, - send_single_reply=False): + default_exchange=None, allowed_remote_exmods=None): super(AMQPDriverBase, self).__init__(conf, url, default_exchange, allowed_remote_exmods) @@ -361,14 +363,12 @@ class AMQPDriverBase(base.BaseDriver): self._reply_q_conn = None self._waiter = None - self.send_single_reply = send_single_reply - def _get_exchange(self, target): return target.exchange or self._default_exchange - def _get_connection(self, purpose=rpc_amqp.PURPOSE_SEND): - return rpc_amqp.ConnectionContext(self._connection_pool, - purpose=purpose) + def _get_connection(self, purpose=rpc_common.PURPOSE_SEND): + return rpc_common.ConnectionContext(self._connection_pool, + purpose=purpose) def _get_reply_q(self): with self._reply_q_lock: @@ -377,7 +377,7 @@ class AMQPDriverBase(base.BaseDriver): reply_q = 'reply_' + uuid.uuid4().hex - conn = self._get_connection(rpc_amqp.PURPOSE_LISTEN) + conn = self._get_connection(rpc_common.PURPOSE_LISTEN) self._waiter = ReplyWaiter(reply_q, conn, self._allowed_remote_exmods) @@ -422,7 +422,7 @@ class AMQPDriverBase(base.BaseDriver): log_msg = "CAST unique_id: %s " % unique_id try: - with self._get_connection(rpc_amqp.PURPOSE_SEND) as conn: + with self._get_connection(rpc_common.PURPOSE_SEND) as conn: if notify: exchange = self._get_exchange(target) log_msg += "NOTIFY exchange '%(exchange)s'" \ @@ -468,7 +468,7 @@ class AMQPDriverBase(base.BaseDriver): envelope=(version == 2.0), notify=True, retry=retry) def listen(self, target): - conn = self._get_connection(rpc_amqp.PURPOSE_LISTEN) + conn = self._get_connection(rpc_common.PURPOSE_LISTEN) listener = AMQPListener(self, conn) @@ -484,7 +484,7 @@ class AMQPDriverBase(base.BaseDriver): return listener def listen_for_notifications(self, targets_and_priorities, pool): - conn = self._get_connection(rpc_amqp.PURPOSE_LISTEN) + conn = self._get_connection(rpc_common.PURPOSE_LISTEN) listener = AMQPListener(self, conn) for target, priority in targets_and_priorities: diff --git a/oslo_messaging/_drivers/base.py b/oslo_messaging/_drivers/base.py index 607821faa..9c2cb87ba 100644 --- a/oslo_messaging/_drivers/base.py +++ b/oslo_messaging/_drivers/base.py @@ -15,9 +15,12 @@ import abc -import six - from oslo_config import cfg +from oslo_utils import timeutils +import six +from six.moves import range as compat_range + + from oslo_messaging import exceptions base_opts = [ @@ -28,6 +31,27 @@ base_opts = [ ] +def batch_poll_helper(func): + """Decorator to poll messages in batch + + This decorator helps driver that polls message one by one, + to returns a list of message. + """ + def wrapper(in_self, timeout=None, prefetch_size=1): + incomings = [] + watch = timeutils.StopWatch(duration=timeout) + with watch: + for __ in compat_range(prefetch_size): + msg = func(in_self, timeout=watch.leftover(return_none=True)) + if msg is not None: + incomings.append(msg) + else: + # timeout reached or listener stopped + break + return incomings + return wrapper + + class TransportDriverError(exceptions.MessagingException): """Base class for transport driver specific exceptions.""" @@ -61,8 +85,9 @@ class Listener(object): self.driver = driver @abc.abstractmethod - def poll(self, timeout=None): - """Blocking until a message is pending and return IncomingMessage. + def poll(self, timeout=None, prefetch_size=1): + """Blocking until 'prefetch_size' message is pending and return + [IncomingMessage]. Return None after timeout seconds if timeout is set and no message is ending or if the listener have been stopped. """ diff --git a/oslo_messaging/_drivers/common.py b/oslo_messaging/_drivers/common.py index 78bdd9239..85d814da6 100644 --- a/oslo_messaging/_drivers/common.py +++ b/oslo_messaging/_drivers/common.py @@ -109,7 +109,7 @@ class Timeout(RPCException): :param info: Extra info to convey to the user :param topic: The topic that the rpc call was sent to - :param rpc_method_name: The name of the rpc method being + :param method: The name of the rpc method being called """ self.info = info @@ -348,3 +348,99 @@ class DecayingTimer(object): if left <= 0 and timeout_callback is not None: timeout_callback(*args, **kwargs) return left if maximum is None else min(left, maximum) + + +# NOTE(sileht): Even if rabbit has only one Connection class, +# this connection can be used for two purposes: +# * wait and receive amqp messages (only do read stuffs on the socket) +# * send messages to the broker (only do write stuffs on the socket) +# The code inside a connection class is not concurrency safe. +# Using one Connection class instance for doing both, will result +# of eventlet complaining of multiple greenthreads that read/write the +# same fd concurrently... because 'send' and 'listen' run in different +# greenthread. +# So, a connection cannot be shared between thread/greenthread and +# this two variables permit to define the purpose of the connection +# to allow drivers to add special handling if needed (like heatbeat). +# amqp drivers create 3 kind of connections: +# * driver.listen*(): each call create a new 'PURPOSE_LISTEN' connection +# * driver.send*(): a pool of 'PURPOSE_SEND' connections is used +# * driver internally have another 'PURPOSE_LISTEN' connection dedicated +# to wait replies of rpc call +PURPOSE_LISTEN = 'listen' +PURPOSE_SEND = 'send' + + +class ConnectionContext(Connection): + """The class that is actually returned to the create_connection() caller. + + This is essentially a wrapper around Connection that supports 'with'. + It can also return a new Connection, or one from a pool. + + The function will also catch when an instance of this class is to be + deleted. With that we can return Connections to the pool on exceptions + and so forth without making the caller be responsible for catching them. + If possible the function makes sure to return a connection to the pool. + """ + + def __init__(self, connection_pool, purpose): + """Create a new connection, or get one from the pool.""" + self.connection = None + self.connection_pool = connection_pool + pooled = purpose == PURPOSE_SEND + if pooled: + self.connection = connection_pool.get() + else: + # a non-pooled connection is requested, so create a new connection + self.connection = connection_pool.create(purpose) + self.pooled = pooled + self.connection.pooled = pooled + + def __enter__(self): + """When with ConnectionContext() is used, return self.""" + return self + + def _done(self): + """If the connection came from a pool, clean it up and put it back. + If it did not come from a pool, close it. + """ + if self.connection: + if self.pooled: + # Reset the connection so it's ready for the next caller + # to grab from the pool + try: + self.connection.reset() + except Exception: + LOG.exception("Fail to reset the connection, drop it") + try: + self.connection.close() + except Exception: + pass + self.connection = self.connection_pool.create() + finally: + self.connection_pool.put(self.connection) + else: + try: + self.connection.close() + except Exception: + pass + self.connection = None + + def __exit__(self, exc_type, exc_value, tb): + """End of 'with' statement. We're done here.""" + self._done() + + def __del__(self): + """Caller is done with this connection. Make sure we cleaned up.""" + self._done() + + def close(self): + """Caller is done with this connection.""" + self._done() + + def __getattr__(self, key): + """Proxy all other calls to the Connection instance.""" + if self.connection: + return getattr(self.connection, key) + else: + raise InvalidRPCConnectionReuse() diff --git a/oslo_messaging/_drivers/impl_fake.py b/oslo_messaging/_drivers/impl_fake.py index 36365e91c..4dea4df79 100644 --- a/oslo_messaging/_drivers/impl_fake.py +++ b/oslo_messaging/_drivers/impl_fake.py @@ -54,6 +54,7 @@ class FakeListener(base.Listener): exchange = self._exchange_manager.get_exchange(target.exchange) exchange.ensure_queue(target, pool) + @base.batch_poll_helper def poll(self, timeout=None): if timeout is not None: deadline = time.time() + timeout diff --git a/oslo_messaging/_drivers/impl_kafka.py b/oslo_messaging/_drivers/impl_kafka.py new file mode 100644 index 000000000..ce6452dc0 --- /dev/null +++ b/oslo_messaging/_drivers/impl_kafka.py @@ -0,0 +1,364 @@ +# Copyright (C) 2015 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import threading + +from oslo_messaging._drivers import base +from oslo_messaging._drivers import common as driver_common +from oslo_messaging._drivers import pool as driver_pool +from oslo_messaging._i18n import _LE +from oslo_messaging._i18n import _LW +from oslo_serialization import jsonutils + +import kafka +from kafka.common import KafkaError +from oslo_config import cfg +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) + +PURPOSE_SEND = 'send' +PURPOSE_LISTEN = 'listen' + +kafka_opts = [ + cfg.StrOpt('kafka_default_host', default='localhost', + help='Default Kafka broker Host'), + + cfg.IntOpt('kafka_default_port', default=9092, + help='Default Kafka broker Port'), + + cfg.IntOpt('kafka_max_fetch_bytes', default=1024 * 1024, + help='Max fetch bytes of Kafka consumer'), + + cfg.IntOpt('kafka_consumer_timeout', default=1.0, + help='Default timeout(s) for Kafka consumers'), + + cfg.IntOpt('pool_size', default=10, + help='Pool Size for Kafka Consumers'), +] + +CONF = cfg.CONF + + +def pack_context_with_message(ctxt, msg): + """Pack context into msg.""" + if isinstance(ctxt, dict): + context_d = ctxt + else: + context_d = ctxt.to_dict() + + return {'message': msg, 'context': context_d} + + +def target_to_topic(target): + """Convert target into topic string + + :param target: Message destination target + :type target: oslo_messaging.Target + """ + if target.exchange is None: + return target.topic + return "%s_%s" % (target.exchange, target.topic) + + +class Connection(object): + + def __init__(self, conf, url, purpose): + + driver_conf = conf.oslo_messaging_kafka + + self.conf = conf + self.kafka_client = None + self.producer = None + self.consumer = None + self.fetch_messages_max_bytes = driver_conf.kafka_max_fetch_bytes + self.consumer_timeout = float(driver_conf.kafka_consumer_timeout) + self.url = url + self._parse_url() + # TODO(Support for manual/auto_commit functionality) + # When auto_commit is False, consumer can manually notify + # the completion of the subscription. + # Currently we don't support for non auto commit option + self.auto_commit = True + self._consume_loop_stopped = False + + def _parse_url(self): + driver_conf = self.conf.oslo_messaging_kafka + try: + self.host = self.url.hosts[0].hostname + except (NameError, IndexError): + self.host = driver_conf.kafka_default_host + + try: + self.port = self.url.hosts[0].port + except (NameError, IndexError): + self.port = driver_conf.kafka_default_port + + if self.host is None: + self.host = driver_conf.kafka_default_host + + if self.port is None: + self.port = driver_conf.kafka_default_port + + def notify_send(self, topic, ctxt, msg, retry): + """Send messages to Kafka broker. + + :param topic: String of the topic + :param ctxt: context for the messages + :param msg: messages for publishing + :param retry: the number of retry + """ + message = pack_context_with_message(ctxt, msg) + self._ensure_connection() + self._send_and_retry(message, topic, retry) + + def _send_and_retry(self, message, topic, retry): + current_retry = 0 + if not isinstance(message, str): + message = jsonutils.dumps(message) + while message is not None: + try: + self._send(message, topic) + message = None + except Exception: + LOG.warn(_LW("Failed to publish a message of topic %s"), topic) + current_retry += 1 + if retry is not None and current_retry >= retry: + LOG.exception(_LE("Failed to retry to send data " + "with max retry times")) + message = None + + def _send(self, message, topic): + self.producer.send_messages(topic, message) + + def consume(self, timeout=None): + """recieve messages as many as max_fetch_messages. + + In this functions, there are no while loop to subscribe. + This would be helpful when we wants to control the velocity of + subscription. + """ + duration = (self.consumer_timeout if timeout is None else timeout) + timer = driver_common.DecayingTimer(duration=duration) + timer.start() + + def _raise_timeout(): + LOG.debug('Timed out waiting for Kafka response') + raise driver_common.Timeout() + + poll_timeout = (self.consumer_timeout if timeout is None + else min(timeout, self.consumer_timeout)) + + while True: + if self._consume_loop_stopped: + return + try: + next_timeout = poll_timeout * 1000.0 + # TODO(use configure() method instead) + # Currently KafkaConsumer does not support for + # the case of updating only fetch_max_wait_ms parameter + self.consumer._config['fetch_max_wait_ms'] = next_timeout + messages = list(self.consumer.fetch_messages()) + except Exception as e: + LOG.exception(_LE("Failed to consume messages: %s"), e) + messages = None + + if not messages: + poll_timeout = timer.check_return( + _raise_timeout, maximum=self.consumer_timeout) + continue + + return messages + + def stop_consuming(self): + self._consume_loop_stopped = True + + def reset(self): + """Reset a connection so it can be used again.""" + if self.kafka_client: + self.kafka_client.close() + self.kafka_client = None + if self.producer: + self.producer.stop() + self.producer = None + self.consumer = None + + def close(self): + if self.kafka_client: + self.kafka_client.close() + self.kafka_client = None + if self.producer: + self.producer.stop() + self.consumer = None + + def commit(self): + """Commit is used by subscribers belonging to the same group. + After subscribing messages, commit is called to prevent + the other subscribers which belong to the same group + from re-subscribing the same messages. + + Currently self.auto_commit option is always True, + so we don't need to call this function. + """ + self.consumer.commit() + + def _ensure_connection(self): + if self.kafka_client: + return + try: + self.kafka_client = kafka.KafkaClient( + "%s:%s" % (self.host, str(self.port))) + self.producer = kafka.SimpleProducer(self.kafka_client) + except KafkaError as e: + LOG.exception(_LE("Kafka Connection is not available: %s"), e) + self.kafka_client = None + + def declare_topic_consumer(self, topics, group=None): + self.consumer = kafka.KafkaConsumer( + *topics, group_id=group, + metadata_broker_list=["%s:%s" % (self.host, str(self.port))], + # auto_commit_enable=self.auto_commit, + fetch_message_max_bytes=self.fetch_messages_max_bytes) + + +class OsloKafkaMessage(base.IncomingMessage): + + def __init__(self, listener, ctxt, message): + super(OsloKafkaMessage, self).__init__(listener, ctxt, message) + + def requeue(self): + LOG.warn(_LW("requeue is not supported")) + + def reply(self, reply=None, failure=None, log_failure=True): + LOG.warn(_LW("reply is not supported")) + + +class KafkaListener(base.Listener): + + def __init__(self, driver, conn): + super(KafkaListener, self).__init__(driver) + self._stopped = threading.Event() + self.conn = conn + self.incoming_queue = [] + + @base.batch_poll_helper + def poll(self, timeout=None): + while not self._stopped.is_set(): + if self.incoming_queue: + return self.incoming_queue.pop(0) + try: + messages = self.conn.consume(timeout=timeout) + for msg in messages: + message = msg.value + message = jsonutils.loads(message) + self.incoming_queue.append(OsloKafkaMessage( + listener=self, ctxt=message['context'], + message=message['message'])) + except driver_common.Timeout: + return None + + def stop(self): + self._stopped.set() + self.conn.stop_consuming() + + def cleanup(self): + self.conn.close() + + def commit(self): + # TODO(Support for manually/auto commit functionality) + # It's better to allow users to commit manually and support for + # self.auto_commit = False option. For now, this commit function + # is meaningless since user couldn't call this function and + # auto_commit option is always True. + self.conn.commit() + + +class KafkaDriver(base.BaseDriver): + """Note: Current implementation of this driver is experimental. + We will have functional and/or integrated testing enabled for this driver. + """ + + def __init__(self, conf, url, default_exchange=None, + allowed_remote_exmods=None): + + opt_group = cfg.OptGroup(name='oslo_messaging_kafka', + title='Kafka driver options') + conf.register_group(opt_group) + conf.register_opts(kafka_opts, group=opt_group) + + super(KafkaDriver, self).__init__( + conf, url, default_exchange, allowed_remote_exmods) + + self.connection_pool = driver_pool.ConnectionPool( + self.conf, self.conf.oslo_messaging_kafka.pool_size, + self._url, Connection) + self.listeners = [] + + def cleanup(self): + for c in self.listeners: + c.close() + self.listeners = [] + + def send(self, target, ctxt, message, wait_for_reply=None, timeout=None, + retry=None): + raise NotImplementedError( + 'The RPC implementation for Kafka is not implemented') + + def send_notification(self, target, ctxt, message, version, retry=None): + """Send notification to Kafka brokers + + :param target: Message destination target + :type target: oslo_messaging.Target + :param ctxt: Message context + :type ctxt: dict + :param message: Message payload to pass + :type message: dict + :param version: Messaging API version (currently not used) + :type version: str + :param retry: an optional default kafka consumer retries configuration + None means to retry forever + 0 means no retry + N means N retries + :type retry: int + """ + with self._get_connection(purpose=PURPOSE_SEND) as conn: + conn.notify_send(target_to_topic(target), ctxt, message, retry) + + def listen(self, target): + raise NotImplementedError( + 'The RPC implementation for Kafka is not implemented') + + def listen_for_notifications(self, targets_and_priorities, pool=None): + """Listen to a specified list of targets on Kafka brokers + + :param targets_and_priorities: List of pairs (target, priority) + priority is not used for kafka driver + target.exchange_target.topic is used as + a kafka topic + :type targets_and_priorities: list + :param pool: consumer group of Kafka consumers + :type pool: string + """ + conn = self._get_connection(purpose=PURPOSE_LISTEN) + topics = [] + for target, priority in targets_and_priorities: + topics.append(target_to_topic(target)) + + conn.declare_topic_consumer(topics, pool) + + listener = KafkaListener(self, conn) + return listener + + def _get_connection(self, purpose): + return driver_common.ConnectionContext(self.connection_pool, purpose) diff --git a/oslo_messaging/_drivers/impl_qpid.py b/oslo_messaging/_drivers/impl_qpid.py deleted file mode 100644 index e0e901968..000000000 --- a/oslo_messaging/_drivers/impl_qpid.py +++ /dev/null @@ -1,800 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# Copyright 2011 - 2012, Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import functools -import itertools -import logging -import os -import random -import time -import warnings - -from oslo_config import cfg -from oslo_serialization import jsonutils -from oslo_utils import importutils -from oslo_utils import netutils -import six - -from oslo_messaging._drivers import amqp as rpc_amqp -from oslo_messaging._drivers import amqpdriver -from oslo_messaging._drivers import base -from oslo_messaging._drivers import common as rpc_common -from oslo_messaging._i18n import _ -from oslo_messaging._i18n import _LE -from oslo_messaging._i18n import _LI -from oslo_messaging import exceptions - -qpid_codec = importutils.try_import("qpid.codec010") -qpid_messaging = importutils.try_import("qpid.messaging") -qpid_exceptions = importutils.try_import("qpid.messaging.exceptions") - -LOG = logging.getLogger(__name__) - -qpid_opts = [ - cfg.StrOpt('qpid_hostname', - default='localhost', - deprecated_group='DEFAULT', - help='Qpid broker hostname.'), - cfg.IntOpt('qpid_port', - default=5672, - deprecated_group='DEFAULT', - help='Qpid broker port.'), - cfg.ListOpt('qpid_hosts', - default=['$qpid_hostname:$qpid_port'], - deprecated_group='DEFAULT', - help='Qpid HA cluster host:port pairs.'), - cfg.StrOpt('qpid_username', - default='', - deprecated_group='DEFAULT', - help='Username for Qpid connection.'), - cfg.StrOpt('qpid_password', - default='', - deprecated_group='DEFAULT', - help='Password for Qpid connection.', - secret=True), - cfg.StrOpt('qpid_sasl_mechanisms', - default='', - deprecated_group='DEFAULT', - help='Space separated list of SASL mechanisms to use for ' - 'auth.'), - cfg.IntOpt('qpid_heartbeat', - default=60, - deprecated_group='DEFAULT', - help='Seconds between connection keepalive heartbeats.'), - cfg.StrOpt('qpid_protocol', - default='tcp', - deprecated_group='DEFAULT', - help="Transport to use, either 'tcp' or 'ssl'."), - cfg.BoolOpt('qpid_tcp_nodelay', - default=True, - deprecated_group='DEFAULT', - help='Whether to disable the Nagle algorithm.'), - cfg.IntOpt('qpid_receiver_capacity', - default=1, - deprecated_group='DEFAULT', - help='The number of prefetched messages held by receiver.'), - # NOTE(russellb) If any additional versions are added (beyond 1 and 2), - # this file could probably use some additional refactoring so that the - # differences between each version are split into different classes. - cfg.IntOpt('qpid_topology_version', - default=1, - deprecated_group='DEFAULT', - help="The qpid topology version to use. Version 1 is what " - "was originally used by impl_qpid. Version 2 includes " - "some backwards-incompatible changes that allow broker " - "federation to work. Users should update to version 2 " - "when they are able to take everything down, as it " - "requires a clean break."), -] - -JSON_CONTENT_TYPE = 'application/json; charset=utf8' - - -def raise_invalid_topology_version(conf): - msg = (_("Invalid value for qpid_topology_version: %d") % - conf.qpid_topology_version) - LOG.error(msg) - raise Exception(msg) - - -class QpidMessage(dict): - def __init__(self, session, raw_message): - super(QpidMessage, self).__init__( - rpc_common.deserialize_msg(raw_message.content)) - self._raw_message = raw_message - self._session = session - - def acknowledge(self): - self._session.acknowledge(self._raw_message) - - def requeue(self): - pass - - -class ConsumerBase(object): - """Consumer base class.""" - - def __init__(self, conf, session, callback, node_name, node_opts, - link_name, link_opts): - """Declare a queue on an amqp session. - - 'session' is the amqp session to use - 'callback' is the callback to call when messages are received - 'node_name' is the first part of the Qpid address string, before ';' - 'node_opts' will be applied to the "x-declare" section of "node" - in the address string. - 'link_name' goes into the "name" field of the "link" in the address - string - 'link_opts' will be applied to the "x-declare" section of "link" - in the address string. - """ - self.callback = callback - self.receiver = None - self.rcv_capacity = conf.qpid_receiver_capacity - self.session = None - - if conf.qpid_topology_version == 1: - addr_opts = { - "create": "always", - "node": { - "type": "topic", - "x-declare": { - "durable": True, - "auto-delete": True, - }, - }, - "link": { - "durable": True, - "x-declare": { - "durable": False, - "auto-delete": True, - "exclusive": False, - }, - }, - } - addr_opts["node"]["x-declare"].update(node_opts) - elif conf.qpid_topology_version == 2: - addr_opts = { - "link": { - "x-declare": { - "auto-delete": True, - "exclusive": False, - }, - }, - } - else: - raise_invalid_topology_version(conf) - - addr_opts["link"]["x-declare"].update(link_opts) - if link_name: - addr_opts["link"]["name"] = link_name - - self.address = "%s ; %s" % (node_name, jsonutils.dumps(addr_opts)) - - self.connect(session) - - def connect(self, session): - """Declare the receiver on connect.""" - self._declare_receiver(session) - - def reconnect(self, session): - """Re-declare the receiver after a Qpid reconnect.""" - self._declare_receiver(session) - - def _declare_receiver(self, session): - self.session = session - self.receiver = session.receiver(self.address) - self.receiver.capacity = self.rcv_capacity - - def _unpack_json_msg(self, msg): - """Load the JSON data in msg if msg.content_type indicates that it - is necessary. Put the loaded data back into msg.content and - update msg.content_type appropriately. - - A Qpid Message containing a dict will have a content_type of - 'amqp/map', whereas one containing a string that needs to be converted - back from JSON will have a content_type of JSON_CONTENT_TYPE. - - :param msg: a Qpid Message object - :returns: None - """ - if msg.content_type == JSON_CONTENT_TYPE: - msg.content = jsonutils.loads(msg.content) - msg.content_type = 'amqp/map' - - def consume(self): - """Fetch the message and pass it to the callback object.""" - message = self.receiver.fetch() - try: - self._unpack_json_msg(message) - self.callback(QpidMessage(self.session, message)) - except Exception: - LOG.exception(_LE("Failed to process message... skipping it.")) - self.session.acknowledge(message) - - def get_receiver(self): - return self.receiver - - def get_node_name(self): - return self.address.split(';')[0] - - -class DirectConsumer(ConsumerBase): - """Queue/consumer class for 'direct'.""" - - def __init__(self, conf, session, msg_id, callback): - """Init a 'direct' queue. - - 'session' is the amqp session to use - 'msg_id' is the msg_id to listen on - 'callback' is the callback to call when messages are received - """ - - link_opts = { - "exclusive": True, - "durable": conf.amqp_durable_queues, - } - - if conf.qpid_topology_version == 1: - node_name = "%s/%s" % (msg_id, msg_id) - node_opts = {"type": "direct"} - link_name = msg_id - elif conf.qpid_topology_version == 2: - node_name = "amq.direct/%s" % msg_id - node_opts = {} - link_name = msg_id - else: - raise_invalid_topology_version(conf) - - super(DirectConsumer, self).__init__(conf, session, callback, - node_name, node_opts, link_name, - link_opts) - - -class TopicConsumer(ConsumerBase): - """Consumer class for 'topic'.""" - - def __init__(self, conf, session, topic, callback, exchange_name, - name=None): - """Init a 'topic' queue. - - :param session: the amqp session to use - :param topic: is the topic to listen on - :paramtype topic: str - :param callback: the callback to call when messages are received - :param name: optional queue name, defaults to topic - """ - - link_opts = { - "auto-delete": conf.amqp_auto_delete, - "durable": conf.amqp_durable_queues, - } - - if conf.qpid_topology_version == 1: - node_name = "%s/%s" % (exchange_name, topic) - elif conf.qpid_topology_version == 2: - node_name = "amq.topic/topic/%s/%s" % (exchange_name, topic) - else: - raise_invalid_topology_version(conf) - - super(TopicConsumer, self).__init__(conf, session, callback, node_name, - {}, name or topic, link_opts) - - -class FanoutConsumer(ConsumerBase): - """Consumer class for 'fanout'.""" - - def __init__(self, conf, session, topic, callback): - """Init a 'fanout' queue. - - 'session' is the amqp session to use - 'topic' is the topic to listen on - 'callback' is the callback to call when messages are received - """ - self.conf = conf - - link_opts = {"exclusive": True} - - if conf.qpid_topology_version == 1: - node_name = "%s_fanout" % topic - node_opts = {"durable": False, "type": "fanout"} - elif conf.qpid_topology_version == 2: - node_name = "amq.topic/fanout/%s" % topic - node_opts = {} - else: - raise_invalid_topology_version(conf) - - super(FanoutConsumer, self).__init__(conf, session, callback, - node_name, node_opts, None, - link_opts) - - -class Publisher(object): - """Base Publisher class.""" - - def __init__(self, conf, session, node_name, node_opts=None): - """Init the Publisher class with the exchange_name, routing_key, - and other options - """ - self.sender = None - self.session = session - - if conf.qpid_topology_version == 1: - addr_opts = { - "create": "always", - "node": { - "type": "topic", - "x-declare": { - "durable": False, - # auto-delete isn't implemented for exchanges in qpid, - # but put in here anyway - "auto-delete": True, - }, - }, - } - if node_opts: - addr_opts["node"]["x-declare"].update(node_opts) - - self.address = "%s ; %s" % (node_name, jsonutils.dumps(addr_opts)) - elif conf.qpid_topology_version == 2: - self.address = node_name - else: - raise_invalid_topology_version(conf) - - self.reconnect(session) - - def reconnect(self, session): - """Re-establish the Sender after a reconnection.""" - self.sender = session.sender(self.address) - - def _pack_json_msg(self, msg): - """Qpid cannot serialize dicts containing strings longer than 65535 - characters. This function dumps the message content to a JSON - string, which Qpid is able to handle. - - :param msg: May be either a Qpid Message object or a bare dict. - :returns: A Qpid Message with its content field JSON encoded. - """ - try: - msg.content = jsonutils.dumps(msg.content) - except AttributeError: - # Need to have a Qpid message so we can set the content_type. - msg = qpid_messaging.Message(jsonutils.dumps(msg)) - msg.content_type = JSON_CONTENT_TYPE - return msg - - def send(self, msg): - """Send a message.""" - try: - # Check if Qpid can encode the message - check_msg = msg - if not hasattr(check_msg, 'content_type'): - check_msg = qpid_messaging.Message(msg) - content_type = check_msg.content_type - enc, dec = qpid_messaging.message.get_codec(content_type) - enc(check_msg.content) - except qpid_codec.CodecException: - # This means the message couldn't be serialized as a dict. - msg = self._pack_json_msg(msg) - self.sender.send(msg) - - -class DirectPublisher(Publisher): - """Publisher class for 'direct'.""" - def __init__(self, conf, session, topic): - """Init a 'direct' publisher.""" - - if conf.qpid_topology_version == 1: - node_name = "%s/%s" % (topic, topic) - node_opts = {"type": "direct"} - elif conf.qpid_topology_version == 2: - node_name = "amq.direct/%s" % topic - node_opts = {} - else: - raise_invalid_topology_version(conf) - - super(DirectPublisher, self).__init__(conf, session, node_name, - node_opts) - - -class TopicPublisher(Publisher): - """Publisher class for 'topic'.""" - def __init__(self, conf, session, exchange_name, topic): - """Init a 'topic' publisher. - """ - if conf.qpid_topology_version == 1: - node_name = "%s/%s" % (exchange_name, topic) - elif conf.qpid_topology_version == 2: - node_name = "amq.topic/topic/%s/%s" % (exchange_name, topic) - else: - raise_invalid_topology_version(conf) - - super(TopicPublisher, self).__init__(conf, session, node_name) - - -class FanoutPublisher(Publisher): - """Publisher class for 'fanout'.""" - def __init__(self, conf, session, topic): - """Init a 'fanout' publisher. - """ - - if conf.qpid_topology_version == 1: - node_name = "%s_fanout" % topic - node_opts = {"type": "fanout"} - elif conf.qpid_topology_version == 2: - node_name = "amq.topic/fanout/%s" % topic - node_opts = {} - else: - raise_invalid_topology_version(conf) - - super(FanoutPublisher, self).__init__(conf, session, node_name, - node_opts) - - -class NotifyPublisher(Publisher): - """Publisher class for notifications.""" - def __init__(self, conf, session, exchange_name, topic): - """Init a 'topic' publisher. - """ - node_opts = {"durable": True} - - if conf.qpid_topology_version == 1: - node_name = "%s/%s" % (exchange_name, topic) - elif conf.qpid_topology_version == 2: - node_name = "amq.topic/topic/%s/%s" % (exchange_name, topic) - else: - raise_invalid_topology_version(conf) - - super(NotifyPublisher, self).__init__(conf, session, node_name, - node_opts) - - -class Connection(object): - """Connection object.""" - - pools = {} - - def __init__(self, conf, url, purpose): - if not qpid_messaging: - raise ImportError("Failed to import qpid.messaging") - - self.connection = None - self.session = None - self.consumers = {} - self.conf = conf - self.driver_conf = conf.oslo_messaging_qpid - - self._consume_loop_stopped = False - - self.brokers_params = [] - if url.hosts: - for host in url.hosts: - params = { - 'username': host.username or '', - 'password': host.password or '', - } - if host.port is not None: - params['host'] = '%s:%d' % (host.hostname, host.port) - else: - params['host'] = host.hostname - self.brokers_params.append(params) - else: - # Old configuration format - for adr in self.driver_conf.qpid_hosts: - hostname, port = netutils.parse_host_port( - adr, default_port=5672) - - if ':' in hostname: - hostname = '[' + hostname + ']' - - params = { - 'host': '%s:%d' % (hostname, port), - 'username': self.driver_conf.qpid_username, - 'password': self.driver_conf.qpid_password, - } - self.brokers_params.append(params) - - random.shuffle(self.brokers_params) - self.brokers = itertools.cycle(self.brokers_params) - - self._initial_pid = os.getpid() - self.reconnect() - - def _connect(self, broker): - # Create the connection - this does not open the connection - self.connection = qpid_messaging.Connection(broker['host']) - - # Check if flags are set and if so set them for the connection - # before we call open - self.connection.username = broker['username'] - self.connection.password = broker['password'] - - self.connection.sasl_mechanisms = self.driver_conf.qpid_sasl_mechanisms - # Reconnection is done by self.reconnect() - self.connection.reconnect = False - self.connection.heartbeat = self.driver_conf.qpid_heartbeat - self.connection.transport = self.driver_conf.qpid_protocol - self.connection.tcp_nodelay = self.driver_conf.qpid_tcp_nodelay - self.connection.open() - - def _register_consumer(self, consumer): - self.consumers[six.text_type(consumer.get_receiver())] = consumer - - def _lookup_consumer(self, receiver): - return self.consumers[six.text_type(receiver)] - - def _disconnect(self): - # Close the session if necessary - if self.connection is not None and self.connection.opened(): - try: - self.connection.close() - except qpid_exceptions.MessagingError: - pass - self.connection = None - - def reconnect(self, retry=None): - """Handles reconnecting and re-establishing sessions and queues. - Will retry up to retry number of times. - retry = None or -1 means to retry forever - retry = 0 means no retry - retry = N means N retries - """ - delay = 1 - attempt = 0 - loop_forever = False - if retry is None or retry < 0: - loop_forever = True - - while True: - self._disconnect() - - attempt += 1 - broker = six.next(self.brokers) - try: - self._connect(broker) - except qpid_exceptions.MessagingError as e: - msg_dict = dict(e=e, - delay=delay, - retry=retry, - broker=broker) - if not loop_forever and attempt > retry: - msg = _('Unable to connect to AMQP server on ' - '%(broker)s after %(retry)d ' - 'tries: %(e)s') % msg_dict - LOG.error(msg) - raise exceptions.MessageDeliveryFailure(msg) - else: - msg = _LE("Unable to connect to AMQP server on " - "%(broker)s: %(e)s. Sleeping %(delay)s seconds") - LOG.error(msg, msg_dict) - time.sleep(delay) - delay = min(delay + 1, 5) - else: - LOG.info(_LI('Connected to AMQP server on %s'), broker['host']) - break - - self.session = self.connection.session() - - if self.consumers: - consumers = self.consumers - self.consumers = {} - - for consumer in six.itervalues(consumers): - consumer.reconnect(self.session) - self._register_consumer(consumer) - - LOG.debug("Re-established AMQP queues") - - def ensure(self, error_callback, method, retry=None): - - current_pid = os.getpid() - if self._initial_pid != current_pid: - # NOTE(sileht): - # to get the same level of fork support that rabbit driver have - # (ie: allow fork before the first connection established) - # we could use the kombu workaround: - # https://github.com/celery/kombu/blob/master/kombu/transport/ - # qpid_patches.py#L67 - LOG.warn("Process forked! " - "This can result in unpredictable behavior. " - "See: http://docs.openstack.org/developer/" - "oslo_messaging/transport.html") - self._initial_pid = current_pid - - while True: - try: - return method() - except (qpid_exceptions.Empty, - qpid_exceptions.MessagingError) as e: - if error_callback: - error_callback(e) - self.reconnect(retry=retry) - - def close(self): - """Close/release this connection.""" - try: - self.connection.close() - except Exception: - # NOTE(dripton) Logging exceptions that happen during cleanup just - # causes confusion; there's really nothing useful we can do with - # them. - pass - self.connection = None - - def reset(self): - """Reset a connection so it can be used again.""" - self.session.close() - self.session = self.connection.session() - self.consumers = {} - - def declare_consumer(self, consumer_cls, topic, callback): - """Create a Consumer using the class that was passed in and - add it to our list of consumers - """ - def _connect_error(exc): - log_info = {'topic': topic, 'err_str': exc} - LOG.error(_LE("Failed to declare consumer for topic '%(topic)s': " - "%(err_str)s"), log_info) - - def _declare_consumer(): - consumer = consumer_cls(self.driver_conf, self.session, topic, - callback) - self._register_consumer(consumer) - return consumer - - return self.ensure(_connect_error, _declare_consumer) - - def consume(self, timeout=None): - """Consume from all queues/consumers.""" - - timer = rpc_common.DecayingTimer(duration=timeout) - timer.start() - - def _raise_timeout(exc): - LOG.debug('Timed out waiting for RPC response: %s', exc) - raise rpc_common.Timeout() - - def _error_callback(exc): - timer.check_return(_raise_timeout, exc) - LOG.exception(_LE('Failed to consume message from queue: %s'), exc) - - def _consume(): - # NOTE(sileht): - # maximum value chosen according the best practice from kombu: - # http://kombu.readthedocs.org/en/latest/reference/kombu.common.html#kombu.common.eventloop - poll_timeout = 1 if timeout is None else min(timeout, 1) - - while True: - if self._consume_loop_stopped: - self._consume_loop_stopped = False - return - - try: - nxt_receiver = self.session.next_receiver( - timeout=poll_timeout) - except qpid_exceptions.Empty as exc: - poll_timeout = timer.check_return(_raise_timeout, exc, - maximum=1) - else: - break - - try: - self._lookup_consumer(nxt_receiver).consume() - except Exception: - LOG.exception(_LE("Error processing message. " - "Skipping it.")) - - self.ensure(_error_callback, _consume) - - def publisher_send(self, cls, topic, msg, retry=None, **kwargs): - """Send to a publisher based on the publisher class.""" - - def _connect_error(exc): - log_info = {'topic': topic, 'err_str': exc} - LOG.exception(_LE("Failed to publish message to topic " - "'%(topic)s': %(err_str)s"), log_info) - - def _publisher_send(): - publisher = cls(self.driver_conf, self.session, topic=topic, - **kwargs) - publisher.send(msg) - - return self.ensure(_connect_error, _publisher_send, retry=retry) - - def declare_direct_consumer(self, topic, callback): - """Create a 'direct' queue. - In nova's use, this is generally a msg_id queue used for - responses for call/multicall - """ - self.declare_consumer(DirectConsumer, topic, callback) - - def declare_topic_consumer(self, exchange_name, topic, callback=None, - queue_name=None): - """Create a 'topic' consumer.""" - self.declare_consumer(functools.partial(TopicConsumer, - name=queue_name, - exchange_name=exchange_name, - ), - topic, callback) - - def declare_fanout_consumer(self, topic, callback): - """Create a 'fanout' consumer.""" - self.declare_consumer(FanoutConsumer, topic, callback) - - def direct_send(self, msg_id, msg): - """Send a 'direct' message.""" - self.publisher_send(DirectPublisher, topic=msg_id, msg=msg) - - def topic_send(self, exchange_name, topic, msg, timeout=None, retry=None): - """Send a 'topic' message.""" - # - # We want to create a message with attributes, for example a TTL. We - # don't really need to keep 'msg' in its JSON format any longer - # so let's create an actual Qpid message here and get some - # value-add on the go. - # - # WARNING: Request timeout happens to be in the same units as - # Qpid's TTL (seconds). If this changes in the future, then this - # will need to be altered accordingly. - # - qpid_message = qpid_messaging.Message(content=msg, ttl=timeout) - self.publisher_send(TopicPublisher, topic=topic, msg=qpid_message, - exchange_name=exchange_name, retry=retry) - - def fanout_send(self, topic, msg, retry=None): - """Send a 'fanout' message.""" - self.publisher_send(FanoutPublisher, topic=topic, msg=msg, retry=retry) - - def notify_send(self, exchange_name, topic, msg, retry=None, **kwargs): - """Send a notify message on a topic.""" - self.publisher_send(NotifyPublisher, topic=topic, msg=msg, - exchange_name=exchange_name, retry=retry) - - def stop_consuming(self): - self._consume_loop_stopped = True - - -class QpidDriver(amqpdriver.AMQPDriverBase): - """qpidd Driver - - .. deprecated:: 1.16 (Liberty) - """ - - def __init__(self, conf, url, - default_exchange=None, allowed_remote_exmods=None): - - warnings.warn(_('The Qpid driver has been deprecated. ' - 'The driver is planned to be removed during the ' - '`Mitaka` development cycle.'), - DeprecationWarning, stacklevel=2) - - opt_group = cfg.OptGroup(name='oslo_messaging_qpid', - title='QPID driver options') - conf.register_group(opt_group) - conf.register_opts(qpid_opts, group=opt_group) - conf.register_opts(rpc_amqp.amqp_opts, group=opt_group) - conf.register_opts(base.base_opts, group=opt_group) - - connection_pool = rpc_amqp.ConnectionPool( - conf, conf.oslo_messaging_qpid.rpc_conn_pool_size, - url, Connection) - - super(QpidDriver, self).__init__( - conf, url, - connection_pool, - default_exchange, - allowed_remote_exmods, - conf.oslo_messaging_qpid.send_single_reply, - ) diff --git a/oslo_messaging/_drivers/impl_rabbit.py b/oslo_messaging/_drivers/impl_rabbit.py index eaa866882..1f75b3349 100644 --- a/oslo_messaging/_drivers/impl_rabbit.py +++ b/oslo_messaging/_drivers/impl_rabbit.py @@ -37,6 +37,7 @@ from oslo_messaging._drivers import amqp as rpc_amqp from oslo_messaging._drivers import amqpdriver from oslo_messaging._drivers import base from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._drivers import pool from oslo_messaging._i18n import _ from oslo_messaging._i18n import _LE from oslo_messaging._i18n import _LI @@ -72,27 +73,28 @@ rabbit_opts = [ deprecated_group='DEFAULT', help='How long to wait before reconnecting in response to an ' 'AMQP consumer cancel notification.'), - cfg.IntOpt('kombu_reconnect_timeout', - # NOTE(dhellmann): We want this to be similar to - # rpc_response_timeout, but we can't use - # "$rpc_response_timeout" as a default because that - # option may not have been defined by the time this - # option is accessed. Instead, document the intent in - # the help text for this option and provide a separate - # literal default value. + cfg.IntOpt('kombu_missing_consumer_retry_timeout', + deprecated_name="kombu_reconnect_timeout", default=60, - help='How long to wait before considering a reconnect ' - 'attempt to have failed. This value should not be ' - 'longer than rpc_response_timeout.'), + help='How long to wait a missing client beforce abandoning to ' + 'send it its replies. This value should not be longer ' + 'than rpc_response_timeout.'), + cfg.StrOpt('kombu_failover_strategy', + choices=('round-robin', 'shuffle'), + default='round-robin', + help='Determines how the next RabbitMQ node is chosen in case ' + 'the one we are currently connected to becomes ' + 'unavailable. Takes effect only if more than one ' + 'RabbitMQ node is provided in config.'), cfg.StrOpt('rabbit_host', default='localhost', deprecated_group='DEFAULT', help='The RabbitMQ broker address where a single node is ' 'used.'), - cfg.IntOpt('rabbit_port', - default=5672, - deprecated_group='DEFAULT', - help='The RabbitMQ broker port where a single node is used.'), + cfg.PortOpt('rabbit_port', + default=5672, + deprecated_group='DEFAULT', + help='The RabbitMQ broker port where a single node is used.'), cfg.ListOpt('rabbit_hosts', default=['$rabbit_host:$rabbit_port'], deprecated_group='DEFAULT', @@ -376,7 +378,9 @@ class Connection(object): self.amqp_durable_queues = driver_conf.amqp_durable_queues self.amqp_auto_delete = driver_conf.amqp_auto_delete self.rabbit_use_ssl = driver_conf.rabbit_use_ssl - self.kombu_reconnect_timeout = driver_conf.kombu_reconnect_timeout + self.kombu_missing_consumer_retry_timeout = \ + driver_conf.kombu_missing_consumer_retry_timeout + self.kombu_failover_strategy = driver_conf.kombu_failover_strategy if self.rabbit_use_ssl: self.kombu_ssl_version = driver_conf.kombu_ssl_version @@ -448,7 +452,7 @@ class Connection(object): # NOTE(sileht): if purpose is PURPOSE_LISTEN # we don't need the lock because we don't # have a heartbeat thread - if purpose == rpc_amqp.PURPOSE_SEND: + if purpose == rpc_common.PURPOSE_SEND: self._connection_lock = ConnectionLock() else: self._connection_lock = DummyConnectionLock() @@ -456,8 +460,8 @@ class Connection(object): self.connection = kombu.connection.Connection( self._url, ssl=self._fetch_ssl_params(), login_method=self.login_method, - failover_strategy="shuffle", heartbeat=self.heartbeat_timeout_threshold, + failover_strategy=self.kombu_failover_strategy, transport_options={ 'confirm_publish': True, 'on_blocked': self._on_connection_blocked, @@ -465,8 +469,8 @@ class Connection(object): }, ) - LOG.info(_LI('Connecting to AMQP server on %(hostname)s:%(port)s'), - self.connection.info()) + LOG.debug('Connecting to AMQP server on %(hostname)s:%(port)s', + self.connection.info()) # NOTE(sileht): kombu recommend to run heartbeat_check every # seconds, but we use a lock around the kombu connection @@ -488,11 +492,12 @@ class Connection(object): # the consume code does the heartbeat stuff # we don't need a thread self._heartbeat_thread = None - if purpose == rpc_amqp.PURPOSE_SEND: + if purpose == rpc_common.PURPOSE_SEND: self._heartbeat_start() - LOG.info(_LI('Connected to AMQP server on %(hostname)s:%(port)s'), - self.connection.info()) + LOG.debug('Connected to AMQP server on %(hostname)s:%(port)s ' + 'via [%(transport)s] client', + self.connection.info()) # NOTE(sileht): value chosen according the best practice from kombu # http://kombu.readthedocs.org/en/latest/reference/kombu.common.html#kombu.common.eventloop @@ -576,7 +581,10 @@ class Connection(object): LOG.info(_LI("The broker has unblocked the connection")) def ensure_connection(self): - self.ensure(method=lambda: True) + # NOTE(sileht): we reset the channel and ensure + # the kombu underlying connection works + self._set_current_channel(None) + self.ensure(method=lambda: self.connection.connection) def ensure(self, method, retry=None, recoverable_error_callback=None, error_callback=None, @@ -649,7 +657,7 @@ class Connection(object): consumer.declare(self) LOG.info(_LI('Reconnected to AMQP server on ' - '%(hostname)s:%(port)s'), + '%(hostname)s:%(port)s via [%(transport)s] client'), self.connection.info()) def execute_method(channel): @@ -695,6 +703,10 @@ class Connection(object): 'tries: %(err_str)s') % info LOG.error(msg) raise exceptions.MessageDeliveryFailure(msg) + except rpc_amqp.AMQPDestinationNotFound: + # NOTE(sileht): we must reraise this without + # trigger error_callback + raise except Exception as exc: error_callback and error_callback(exc) raise @@ -727,7 +739,6 @@ class Connection(object): for tag, consumer in enumerate(self._consumers): consumer.cancel(tag=tag) except recoverable_errors: - self._set_current_channel(None) self.ensure_connection() self._consumers = [] @@ -848,7 +859,8 @@ class Connection(object): raise rpc_common.Timeout() def _recoverable_error_callback(exc): - self._new_consumers = self._consumers + if not isinstance(exc, rpc_common.Timeout): + self._new_consumers = self._consumers timer.check_return(_raise_timeout, exc) def _error_callback(exc): @@ -1033,32 +1045,20 @@ class Connection(object): self._publish(exchange, msg, routing_key=routing_key, timeout=timeout) - def _publish_and_retry_on_missing_exchange(self, exchange, msg, - routing_key=None, timeout=None): - """Publisher that retry if the exchange is missing. - """ - + def _publish_and_raises_on_missing_exchange(self, exchange, msg, + routing_key=None, + timeout=None): + """Publisher that raises exception if exchange is missing.""" if not exchange.passive: raise RuntimeError("_publish_and_retry_on_missing_exchange() must " "be called with an passive exchange.") - # TODO(sileht): use @retrying - # NOTE(sileht): no need to wait the application expect a response - # before timeout is exshauted - duration = ( - timeout if timeout is not None - else self.kombu_reconnect_timeout - ) - - timer = rpc_common.DecayingTimer(duration=duration) - timer.start() - - while True: - try: - self._publish(exchange, msg, routing_key=routing_key, - timeout=timeout) - return - except self.connection.channel_errors as exc: + try: + self._publish(exchange, msg, routing_key=routing_key, + timeout=timeout) + return + except self.connection.channel_errors as exc: + if exc.code == 404: # NOTE(noelbk/sileht): # If rabbit dies, the consumer can be disconnected before the # publisher sends, and if the consumer hasn't declared the @@ -1067,24 +1067,9 @@ class Connection(object): # So we set passive=True to the publisher exchange and catch # the 404 kombu ChannelError and retry until the exchange # appears - if exc.code == 404 and timer.check_return() > 0: - LOG.info(_LI("The exchange %(exchange)s to send to " - "%(routing_key)s doesn't exist yet, " - "retrying...") % { - 'exchange': exchange.name, - 'routing_key': routing_key}) - time.sleep(0.25) - continue - elif exc.code == 404: - msg = _("The exchange %(exchange)s to send to " - "%(routing_key)s still doesn't exist after " - "%(duration)s sec abandoning...") % { - 'duration': duration, - 'exchange': exchange.name, - 'routing_key': routing_key} - LOG.info(msg) - raise rpc_amqp.AMQPDestinationNotFound(msg) - raise + raise rpc_amqp.AMQPDestinationNotFound( + "exchange %s doesn't exists" % exchange.name) + raise def direct_send(self, msg_id, msg): """Send a 'direct' message.""" @@ -1094,7 +1079,7 @@ class Connection(object): auto_delete=True, passive=True) - self._ensure_publishing(self._publish_and_retry_on_missing_exchange, + self._ensure_publishing(self._publish_and_raises_on_missing_exchange, exchange, msg, routing_key=msg_id) def topic_send(self, exchange_name, topic, msg, timeout=None, retry=None): @@ -1150,7 +1135,10 @@ class RabbitDriver(amqpdriver.AMQPDriverBase): conf.register_opts(rpc_amqp.amqp_opts, group=opt_group) conf.register_opts(base.base_opts, group=opt_group) - connection_pool = rpc_amqp.ConnectionPool( + self.missing_destination_retry_timeout = ( + conf.oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout) + + connection_pool = pool.ConnectionPool( conf, conf.oslo_messaging_rabbit.rpc_conn_pool_size, url, Connection) @@ -1158,8 +1146,7 @@ class RabbitDriver(amqpdriver.AMQPDriverBase): conf, url, connection_pool, default_exchange, - allowed_remote_exmods, - conf.oslo_messaging_rabbit.send_single_reply, + allowed_remote_exmods ) def require_features(self, requeue=True): diff --git a/oslo_messaging/_drivers/impl_zmq.py b/oslo_messaging/_drivers/impl_zmq.py index f91c34166..f8bf1378e 100644 --- a/oslo_messaging/_drivers/impl_zmq.py +++ b/oslo_messaging/_drivers/impl_zmq.py @@ -13,6 +13,7 @@ # under the License. import logging +import os import pprint import socket import threading @@ -23,8 +24,11 @@ from stevedore import driver from oslo_messaging._drivers import base from oslo_messaging._drivers import common as rpc_common from oslo_messaging._drivers.zmq_driver.client import zmq_client +from oslo_messaging._drivers.zmq_driver.client import zmq_client_light from oslo_messaging._drivers.zmq_driver.server import zmq_server +from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._executors import impl_pooledexecutor # FIXME(markmc) +from oslo_messaging._i18n import _LE pformat = pprint.pformat @@ -78,8 +82,23 @@ zmq_opts = [ 'Poll raises timeout exception when timeout expired.'), cfg.BoolOpt('zmq_use_broker', - default=True, - help='Shows whether zmq-messaging uses broker or not.') + default=False, + help='Configures zmq-messaging to use broker or not.'), + + cfg.PortOpt('rpc_zmq_min_port', + default=49152, + help='Minimal port number for random ports range.'), + + cfg.IntOpt('rpc_zmq_max_port', + min=1, + max=65536, + default=65536, + help='Maximal port number for random ports range.'), + + cfg.IntOpt('rpc_zmq_bind_port_retries', + default=100, + help='Number of retries to find free port number before ' + 'fail with ZMQBindError.') ] @@ -91,6 +110,7 @@ class LazyDriverItem(object): self.item_class = item_cls self.args = args self.kwargs = kwargs + self.process_id = os.getpid() def get(self): # NOTE(ozamiatin): Lazy initialization. @@ -99,11 +119,12 @@ class LazyDriverItem(object): # __init__, but 'fork' extensively used by services # breaks all things. - if self.item is not None: + if self.item is not None and os.getpid() == self.process_id: return self.item self._lock.acquire() - if self.item is None: + if self.item is None or os.getpid() != self.process_id: + self.process_id = os.getpid() self.item = self.item_class(*self.args, **self.kwargs) self._lock.release() return self.item @@ -143,6 +164,10 @@ class ZmqDriver(base.BaseDriver): :param allowed_remote_exmods: remote exception passing options :type allowed_remote_exmods: list """ + zmq = zmq_async.import_zmq() + if zmq is None: + raise ImportError(_LE("ZeroMQ is not available!")) + conf.register_opts(zmq_opts) conf.register_opts(impl_pooledexecutor._pool_opts) conf.register_opts(base.base_opts) @@ -160,12 +185,15 @@ class ZmqDriver(base.BaseDriver): self.notify_server = LazyDriverItem( zmq_server.ZmqServer, self, self.conf, self.matchmaker) + client_cls = zmq_client_light.ZmqClientLight \ + if conf.zmq_use_broker else zmq_client.ZmqClient + self.client = LazyDriverItem( - zmq_client.ZmqClient, self.conf, self.matchmaker, + client_cls, self.conf, self.matchmaker, self.allowed_remote_exmods) self.notifier = LazyDriverItem( - zmq_client.ZmqClient, self.conf, self.matchmaker, + client_cls, self.conf, self.matchmaker, self.allowed_remote_exmods) super(ZmqDriver, self).__init__(conf, url, default_exchange, @@ -229,7 +257,7 @@ class ZmqDriver(base.BaseDriver): :param target: Message destination target :type target: oslo_messaging.Target """ - server = self.server.get() + server = zmq_server.ZmqServer(self, self.conf, self.matchmaker) server.listen(target) return server diff --git a/oslo_messaging/_drivers/pool.py b/oslo_messaging/_drivers/pool.py index e689d678a..699ce5c10 100644 --- a/oslo_messaging/_drivers/pool.py +++ b/oslo_messaging/_drivers/pool.py @@ -17,8 +17,13 @@ import abc import collections import threading +from oslo_log import log as logging import six +from oslo_messaging._drivers import common + +LOG = logging.getLogger(__name__) + @six.add_metaclass(abc.ABCMeta) class Pool(object): @@ -86,3 +91,24 @@ class Pool(object): @abc.abstractmethod def create(self): """Construct a new item.""" + + +class ConnectionPool(Pool): + """Class that implements a Pool of Connections.""" + def __init__(self, conf, rpc_conn_pool_size, url, connection_cls): + self.connection_cls = connection_cls + self.conf = conf + self.url = url + super(ConnectionPool, self).__init__(rpc_conn_pool_size) + self.reply_proxy = None + + # TODO(comstud): Timeout connections not used in a while + def create(self, purpose=None): + if purpose is None: + purpose = common.PURPOSE_SEND + LOG.debug('Pool creating new connection') + return self.connection_cls(self.conf, self.url, purpose) + + def empty(self): + for item in self.iter_free(): + item.close() diff --git a/oslo_messaging/_drivers/protocols/amqp/driver.py b/oslo_messaging/_drivers/protocols/amqp/driver.py index cf1c9127b..32abf435f 100644 --- a/oslo_messaging/_drivers/protocols/amqp/driver.py +++ b/oslo_messaging/_drivers/protocols/amqp/driver.py @@ -117,8 +117,12 @@ class ProtonListener(base.Listener): super(ProtonListener, self).__init__(driver) self.incoming = moves.queue.Queue() - def poll(self): - message = self.incoming.get() + @base.batch_poll_helper + def poll(self, timeout=None): + try: + message = self.incoming.get(True, timeout) + except moves.queue.Empty: + return request, ctxt = unmarshal_request(message) LOG.debug("Returning incoming message") return ProtonIncomingMessage(self, ctxt, request, message) @@ -180,7 +184,7 @@ class ProtonDriver(base.BaseDriver): """Send a message to the given target.""" # TODO(kgiusti) need to add support for retry if retry is not None: - raise NotImplementedError('"retry" not implemented by' + raise NotImplementedError('"retry" not implemented by ' 'this transport driver') request = marshal_request(message, ctxt, envelope) @@ -210,7 +214,7 @@ class ProtonDriver(base.BaseDriver): """Send a notification message to the given target.""" # TODO(kgiusti) need to add support for retry if retry is not None: - raise NotImplementedError('"retry" not implemented by' + raise NotImplementedError('"retry" not implemented by ' 'this transport driver') return self.send(target, ctxt, message, envelope=(version == 2.0)) @@ -226,7 +230,7 @@ class ProtonDriver(base.BaseDriver): def listen_for_notifications(self, targets_and_priorities, pool): LOG.debug("Listen for notifications %s", targets_and_priorities) if pool: - raise NotImplementedError('"pool" not implemented by' + raise NotImplementedError('"pool" not implemented by ' 'this transport driver') listener = ProtonListener(self) for target, priority in targets_and_priorities: diff --git a/oslo_messaging/_drivers/protocols/amqp/drivertasks.py b/oslo_messaging/_drivers/protocols/amqp/drivertasks.py index 5d9e2ed4d..385241334 100644 --- a/oslo_messaging/_drivers/protocols/amqp/drivertasks.py +++ b/oslo_messaging/_drivers/protocols/amqp/drivertasks.py @@ -41,7 +41,7 @@ class SendTask(controller.Task): """Wait for the send to complete, and, optionally, a reply message from the remote. Will raise MessagingTimeout if the send does not complete or no reply is received within timeout seconds. If the request has - failed for any other reason, a MessagingException is raised." + failed for any other reason, a MessagingException is raised. """ try: result = self._results_queue.get(timeout=timeout) diff --git a/oslo_messaging/_drivers/protocols/amqp/opts.py b/oslo_messaging/_drivers/protocols/amqp/opts.py index cba1fd339..5c69c966c 100644 --- a/oslo_messaging/_drivers/protocols/amqp/opts.py +++ b/oslo_messaging/_drivers/protocols/amqp/opts.py @@ -64,6 +64,7 @@ amqp1_opts = [ cfg.StrOpt('ssl_key_password', default=None, deprecated_group='amqp1', + secret=True, help='Password for decrypting ssl_key_file (if encrypted)'), cfg.BoolOpt('allow_insecure_clients', @@ -94,5 +95,6 @@ amqp1_opts = [ cfg.StrOpt('password', default='', deprecated_group='amqp1', + secret=True, help='Password for message broker authentication') ] diff --git a/oslo_messaging/_drivers/zmq_driver/broker/zmq_broker.py b/oslo_messaging/_drivers/zmq_driver/broker/zmq_broker.py index 5f20b807d..8351e2ef9 100644 --- a/oslo_messaging/_drivers/zmq_driver/broker/zmq_broker.py +++ b/oslo_messaging/_drivers/zmq_driver/broker/zmq_broker.py @@ -16,7 +16,6 @@ import logging import os from oslo_utils import excutils -import six from stevedore import driver from oslo_messaging._drivers.zmq_driver.broker import zmq_queue_proxy @@ -51,11 +50,8 @@ class ZmqBroker(object): ).driver(self.conf) self.context = zmq.Context() - self.queue = six.moves.queue.Queue() - self.proxies = [zmq_queue_proxy.OutgoingQueueProxy( - conf, self.context, self.queue, self.matchmaker), - zmq_queue_proxy.IncomingQueueProxy( - conf, self.context, self.queue) + self.proxies = [zmq_queue_proxy.UniversalQueueProxy( + conf, self.context, self.matchmaker) ] def _create_ipc_dirs(self): diff --git a/oslo_messaging/_drivers/zmq_driver/broker/zmq_queue_proxy.py b/oslo_messaging/_drivers/zmq_driver/broker/zmq_queue_proxy.py index 11114d008..eb752bed7 100644 --- a/oslo_messaging/_drivers/zmq_driver/broker/zmq_queue_proxy.py +++ b/oslo_messaging/_drivers/zmq_driver/broker/zmq_queue_proxy.py @@ -14,65 +14,69 @@ import logging -import six - from oslo_messaging._drivers.zmq_driver.broker import zmq_base_proxy -from oslo_messaging._drivers.zmq_driver.client.publishers\ - import zmq_dealer_publisher +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ + import zmq_dealer_publisher_proxy from oslo_messaging._drivers.zmq_driver import zmq_address from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names from oslo_messaging._i18n import _LI zmq = zmq_async.import_zmq(zmq_concurrency='native') LOG = logging.getLogger(__name__) -class OutgoingQueueProxy(zmq_base_proxy.BaseProxy): +class UniversalQueueProxy(zmq_base_proxy.BaseProxy): + + def __init__(self, conf, context, matchmaker): + super(UniversalQueueProxy, self).__init__(conf, context) + self.poller = zmq_async.get_poller(zmq_concurrency='native') + + self.router_socket = context.socket(zmq.ROUTER) + self.router_socket.bind(zmq_address.get_broker_address(conf)) + + self.poller.register(self.router_socket, self._receive_in_request) + LOG.info(_LI("Polling at universal proxy")) - def __init__(self, conf, context, queue, matchmaker): - super(OutgoingQueueProxy, self).__init__(conf, context) - self.queue = queue self.matchmaker = matchmaker - self.publisher = zmq_dealer_publisher.DealerPublisher( - conf, matchmaker) - LOG.info(_LI("Polling at outgoing proxy ...")) + reply_receiver = zmq_dealer_publisher_proxy.ReplyReceiver(self.poller) + self.publisher = zmq_dealer_publisher_proxy.DealerPublisherProxy( + conf, matchmaker, reply_receiver) def run(self): - try: - request = self.queue.get(timeout=self.conf.rpc_poll_timeout) - LOG.info(_LI("Redirecting request %s to TCP publisher ...") - % request) - self.publisher.send_request(request) - except six.moves.queue.Empty: + message, socket = self.poller.poll(self.conf.rpc_poll_timeout) + if message is None: return + if socket == self.router_socket: + self._redirect_in_request(message) + else: + self._redirect_reply(message) -class IncomingQueueProxy(zmq_base_proxy.BaseProxy): + def _redirect_in_request(self, request): + LOG.debug("-> Redirecting request %s to TCP publisher" + % request) + self.publisher.send_request(request) - def __init__(self, conf, context, queue): - super(IncomingQueueProxy, self).__init__(conf, context) - self.poller = zmq_async.get_poller( - zmq_concurrency='native') - - self.queue = queue - - self.socket = context.socket(zmq.ROUTER) - self.socket.bind(zmq_address.get_broker_address(conf)) - self.poller.register(self.socket, self.receive_request) - LOG.info(_LI("Polling at incoming proxy ...")) - - def run(self): - request, socket = self.poller.poll(self.conf.rpc_poll_timeout) - if request is None: + def _redirect_reply(self, reply): + LOG.debug("Reply proxy %s" % reply) + if reply[zmq_names.IDX_REPLY_TYPE] == zmq_names.ACK_TYPE: + LOG.debug("Acknowledge dropped %s" % reply) return - LOG.info(_LI("Received request and queue it: %s") % str(request)) + LOG.debug("<- Redirecting reply to ROUTER: reply: %s" + % reply[zmq_names.IDX_REPLY_BODY:]) - self.queue.put(request) + self.router_socket.send_multipart(reply[zmq_names.IDX_REPLY_BODY:]) - def receive_request(self, socket): + def _receive_in_request(self, socket): reply_id = socket.recv() assert reply_id is not None, "Valid id expected" empty = socket.recv() assert empty == b'', "Empty delimiter expected" - return socket.recv_pyobj() + envelope = socket.recv_pyobj() + if envelope[zmq_names.FIELD_MSG_TYPE] == zmq_names.CALL_TYPE: + envelope[zmq_names.FIELD_REPLY_ID] = reply_id + payload = socket.recv_multipart() + payload.insert(0, envelope) + return payload diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/__init__.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_call_publisher.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_call_publisher.py new file mode 100644 index 000000000..0c4e7536d --- /dev/null +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_call_publisher.py @@ -0,0 +1,194 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import threading + +from concurrent import futures +import futurist + +import oslo_messaging +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._drivers.zmq_driver.client.publishers\ + import zmq_publisher_base +from oslo_messaging._drivers.zmq_driver import zmq_address +from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names +from oslo_messaging._drivers.zmq_driver import zmq_socket + +LOG = logging.getLogger(__name__) + +zmq = zmq_async.import_zmq() + + +class DealerCallPublisher(zmq_publisher_base.PublisherBase): + """Thread-safe CALL publisher + + Used as faster and thread-safe publisher for CALL + instead of ReqPublisher. + """ + + def __init__(self, conf, matchmaker): + super(DealerCallPublisher, self).__init__(conf) + self.matchmaker = matchmaker + self.reply_waiter = ReplyWaiter(conf) + self.sender = RequestSender(conf, matchmaker, self.reply_waiter) \ + if not conf.zmq_use_broker else \ + RequestSenderLight(conf, matchmaker, self.reply_waiter) + + def send_request(self, request): + reply_future = self.sender.send_request(request) + try: + reply = reply_future.result(timeout=request.timeout) + except futures.TimeoutError: + raise oslo_messaging.MessagingTimeout( + "Timeout %s seconds was reached" % request.timeout) + finally: + self.reply_waiter.untrack_id(request.message_id) + + LOG.debug("Received reply %s" % reply) + if reply[zmq_names.FIELD_FAILURE]: + raise rpc_common.deserialize_remote_exception( + reply[zmq_names.FIELD_FAILURE], + request.allowed_remote_exmods) + else: + return reply[zmq_names.FIELD_REPLY] + + +class RequestSender(zmq_publisher_base.PublisherMultisend): + + def __init__(self, conf, matchmaker, reply_waiter): + super(RequestSender, self).__init__(conf, matchmaker, zmq.DEALER) + self.reply_waiter = reply_waiter + self.queue, self.empty_except = zmq_async.get_queue() + self.executor = zmq_async.get_executor(self.run_loop) + self.executor.execute() + + def send_request(self, request): + reply_future = futurist.Future() + self.reply_waiter.track_reply(reply_future, request.message_id) + self.queue.put(request) + return reply_future + + def _do_send_request(self, socket, request): + socket.send(b'', zmq.SNDMORE) + socket.send_pyobj(request) + + LOG.debug("Sending message_id %(message)s to a target %(target)s" + % {"message": request.message_id, + "target": request.target}) + + def _check_hosts_connections(self, target, listener_type): + if str(target) in self.outbound_sockets: + socket = self.outbound_sockets[str(target)] + else: + hosts = self.matchmaker.get_hosts( + target, listener_type) + socket = zmq_socket.ZmqSocket(self.zmq_context, self.socket_type) + self.outbound_sockets[str(target)] = socket + + for host in hosts: + self._connect_to_host(socket, host, target) + + return socket + + def run_loop(self): + try: + request = self.queue.get(timeout=self.conf.rpc_poll_timeout) + except self.empty_except: + return + + socket = self._check_hosts_connections( + request.target, zmq_names.socket_type_str(zmq.ROUTER)) + + self._do_send_request(socket, request) + self.reply_waiter.poll_socket(socket) + + +class RequestSenderLight(RequestSender): + """This class used with proxy. + + Simplified address matching because there is only + one proxy IPC address. + """ + + def __init__(self, conf, matchmaker, reply_waiter): + if not conf.zmq_use_broker: + raise rpc_common.RPCException("RequestSenderLight needs a proxy!") + + super(RequestSenderLight, self).__init__( + conf, matchmaker, reply_waiter) + + self.socket = None + + def _check_hosts_connections(self, target, listener_type): + if self.socket is None: + self.socket = zmq_socket.ZmqSocket(self.zmq_context, + self.socket_type) + self.outbound_sockets[str(target)] = self.socket + address = zmq_address.get_broker_address(self.conf) + self._connect_to_address(self.socket, address, target) + return self.socket + + def _do_send_request(self, socket, request): + LOG.debug("Sending %(type)s message_id %(message)s" + " to a target %(target)s" + % {"type": request.msg_type, + "message": request.message_id, + "target": request.target}) + + envelope = request.create_envelope() + + socket.send(b'', zmq.SNDMORE) + socket.send_pyobj(envelope, zmq.SNDMORE) + socket.send_pyobj(request) + + +class ReplyWaiter(object): + + def __init__(self, conf): + self.conf = conf + self.replies = {} + self.poller = zmq_async.get_poller() + self.executor = zmq_async.get_executor(self.run_loop) + self.executor.execute() + self._lock = threading.Lock() + + def track_reply(self, reply_future, message_id): + self._lock.acquire() + self.replies[message_id] = reply_future + self._lock.release() + + def untrack_id(self, message_id): + self._lock.acquire() + self.replies.pop(message_id) + self._lock.release() + + def poll_socket(self, socket): + + def _receive_method(socket): + empty = socket.recv() + assert empty == b'', "Empty expected!" + reply = socket.recv_pyobj() + LOG.debug("Received reply %s" % reply) + return reply + + self.poller.register(socket, recv_method=_receive_method) + + def run_loop(self): + reply, socket = self.poller.poll( + timeout=self.conf.rpc_poll_timeout) + if reply is not None: + call_future = self.replies[reply[zmq_names.FIELD_MSG_ID]] + call_future.set_result(reply) diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_dealer_publisher.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher.py similarity index 84% rename from oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_dealer_publisher.py rename to oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher.py index 2c8fc5ec5..22e09c3fe 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_dealer_publisher.py +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher.py @@ -18,7 +18,7 @@ from oslo_messaging._drivers.zmq_driver.client.publishers\ import zmq_publisher_base from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._drivers.zmq_driver import zmq_names -from oslo_messaging._i18n import _LI, _LW +from oslo_messaging._i18n import _LW LOG = logging.getLogger(__name__) @@ -29,14 +29,13 @@ class DealerPublisher(zmq_publisher_base.PublisherMultisend): def __init__(self, conf, matchmaker): super(DealerPublisher, self).__init__(conf, matchmaker, zmq.DEALER) - self.ack_receiver = AcknowledgementReceiver() def send_request(self, request): - if request.msg_type == zmq_names.CALL_TYPE: - raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) + self._check_request_pattern(request) - dealer_socket, hosts = self._check_hosts_connections(request.target) + dealer_socket = self._check_hosts_connections( + request.target, zmq_names.socket_type_str(zmq.ROUTER)) if not dealer_socket.connections: # NOTE(ozamiatin): Here we can provide @@ -47,29 +46,31 @@ class DealerPublisher(zmq_publisher_base.PublisherMultisend): % request.msg_type) return - self.ack_receiver.track_socket(dealer_socket.handle) - if request.msg_type in zmq_names.MULTISEND_TYPES: for _ in range(dealer_socket.connections_count()): self._send_request(dealer_socket, request) else: self._send_request(dealer_socket, request) + def _check_request_pattern(self, request): + if request.msg_type == zmq_names.CALL_TYPE: + raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) + def _send_request(self, socket, request): socket.send(b'', zmq.SNDMORE) socket.send_pyobj(request) - LOG.info(_LI("Sending message %(message)s to a target %(target)s") - % {"message": request.message, - "target": request.target}) + LOG.debug("Sending message_id %(message)s to a target %(target)s" + % {"message": request.message_id, + "target": request.target}) def cleanup(self): - self.ack_receiver.cleanup() super(DealerPublisher, self).cleanup() class DealerPublisherLight(zmq_publisher_base.PublisherBase): + """Used when publishing to proxy. """ def __init__(self, conf, address): super(DealerPublisherLight, self).__init__(conf) @@ -81,7 +82,10 @@ class DealerPublisherLight(zmq_publisher_base.PublisherBase): if request.msg_type == zmq_names.CALL_TYPE: raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) + envelope = request.create_envelope() + self.socket.send(b'', zmq.SNDMORE) + self.socket.send_pyobj(envelope, zmq.SNDMORE) self.socket.send_pyobj(request) def cleanup(self): @@ -107,8 +111,7 @@ class AcknowledgementReceiver(object): def poll_for_acknowledgements(self): ack_message, socket = self.poller.poll() - LOG.info(_LI("Message %s acknowledged") - % ack_message[zmq_names.FIELD_ID]) + LOG.debug("Message %s acknowledged" % ack_message[zmq_names.FIELD_ID]) def cleanup(self): self.thread.stop() diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher_proxy.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher_proxy.py new file mode 100644 index 000000000..c8ad98345 --- /dev/null +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/dealer/zmq_dealer_publisher_proxy.py @@ -0,0 +1,87 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ + import zmq_dealer_publisher +from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names +from oslo_messaging._i18n import _LI, _LW + +zmq = zmq_async.import_zmq() + +LOG = logging.getLogger(__name__) + + +class DealerPublisherProxy(zmq_dealer_publisher.DealerPublisher): + + def __init__(self, conf, matchmaker, reply_receiver): + super(DealerPublisherProxy, self).__init__(conf, matchmaker) + self.reply_receiver = reply_receiver + + def send_request(self, multipart_message): + + envelope = multipart_message[zmq_names.MULTIPART_IDX_ENVELOPE] + + LOG.debug("Envelope: %s" % envelope) + + target = envelope[zmq_names.FIELD_TARGET] + dealer_socket = self._check_hosts_connections( + target, zmq_names.socket_type_str(zmq.ROUTER)) + + if not dealer_socket.connections: + # NOTE(ozamiatin): Here we can provide + # a queue for keeping messages to send them later + # when some listener appears. However such approach + # being more reliable will consume additional memory. + LOG.warning(_LW("Request %s was dropped because no connection") + % envelope[zmq_names.FIELD_MSG_TYPE]) + return + + self.reply_receiver.track_socket(dealer_socket.handle) + + LOG.debug("Sending message %(message)s to a target %(target)s" + % {"message": envelope[zmq_names.FIELD_MSG_ID], + "target": envelope[zmq_names.FIELD_TARGET]}) + + if envelope[zmq_names.FIELD_MSG_TYPE] in zmq_names.MULTISEND_TYPES: + for _ in range(dealer_socket.connections_count()): + self._send_request(dealer_socket, multipart_message) + else: + self._send_request(dealer_socket, multipart_message) + + def _send_request(self, socket, multipart_message): + + socket.send(b'', zmq.SNDMORE) + socket.send_pyobj( + multipart_message[zmq_names.MULTIPART_IDX_ENVELOPE], + zmq.SNDMORE) + socket.send(multipart_message[zmq_names.MULTIPART_IDX_BODY]) + + +class ReplyReceiver(object): + + def __init__(self, poller): + self.poller = poller + LOG.info(_LI("Reply waiter created in broker")) + + def _receive_reply(self, socket): + return socket.recv_multipart() + + def track_socket(self, socket): + self.poller.register(socket, self._receive_reply) + + def cleanup(self): + self.poller.close() diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_pub_publisher.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_pub_publisher.py index 228724b6c..6ecfb2f77 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_pub_publisher.py +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_pub_publisher.py @@ -18,7 +18,6 @@ from oslo_messaging._drivers.zmq_driver.client.publishers\ import zmq_publisher_base from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._drivers.zmq_driver import zmq_names -from oslo_messaging._i18n import _LI LOG = logging.getLogger(__name__) @@ -35,13 +34,14 @@ class PubPublisher(zmq_publisher_base.PublisherMultisend): if request.msg_type not in zmq_names.NOTIFY_TYPES: raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) - pub_socket, hosts = self._check_hosts_connections(request.target) + pub_socket = self._check_hosts_connections( + request.target, zmq_names.socket_type_str(zmq.SUB)) self._send_request(pub_socket, request) def _send_request(self, socket, request): super(PubPublisher, self)._send_request(socket, request) - LOG.info(_LI("Publishing message %(message)s to a target %(target)s") - % {"message": request.message, - "target": request.target}) + LOG.debug("Publishing message %(message)s to a target %(target)s" + % {"message": request.message, + "target": request.target}) diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_publisher_base.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_publisher_base.py index faee64d25..e2f898550 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_publisher_base.py +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_publisher_base.py @@ -14,6 +14,7 @@ import abc import logging +import uuid import six @@ -89,6 +90,11 @@ class PublisherBase(object): :param request: Message data and destination container object :type request: zmq_request.Request """ + LOG.debug("Sending %(type)s message_id %(message)s to a target " + "%(target)s" + % {"type": request.msg_type, + "message": request.message_id, + "target": request.target}) socket.send_pyobj(request) def cleanup(self): @@ -115,30 +121,33 @@ class PublisherMultisend(PublisherBase): self.socket_type = socket_type self.matchmaker = matchmaker - def _check_hosts_connections(self, target): + def _check_hosts_connections(self, target, listener_type): # TODO(ozamiatin): Place for significant optimization # Matchmaker cache should be implemented - hosts = self.matchmaker.get_hosts(target) if str(target) in self.outbound_sockets: socket = self.outbound_sockets[str(target)] else: + hosts = self.matchmaker.get_hosts(target, listener_type) socket = zmq_socket.ZmqSocket(self.zmq_context, self.socket_type) self.outbound_sockets[str(target)] = socket + for host in hosts: + self._connect_to_host(socket, host, target) - for host in hosts: - self._connect_to_host(socket, host, target) + return socket - return socket, hosts - - def _connect_to_host(self, socket, host, target): - address = zmq_address.get_tcp_direct_address(host) - LOG.info(address) + def _connect_to_address(self, socket, address, target): stype = zmq_names.socket_type_str(self.socket_type) try: LOG.info(_LI("Connecting %(stype)s to %(address)s for %(target)s") % {"stype": stype, "address": address, "target": target}) + + if six.PY3: + socket.setsockopt_string(zmq.IDENTITY, str(uuid.uuid1())) + else: + socket.handle.identity = str(uuid.uuid1()) + socket.connect(address) except zmq.ZMQError as e: errmsg = _LE("Failed connecting %(stype) to %(address)s: %(e)s")\ @@ -146,3 +155,7 @@ class PublisherMultisend(PublisherBase): LOG.error(_LE("Failed connecting %(stype) to %(address)s: %(e)s") % (stype, address, e)) raise rpc_common.RPCException(errmsg) + + def _connect_to_host(self, socket, host, target): + address = zmq_address.get_tcp_direct_address(host) + self._connect_to_address(socket, address, target) diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_push_publisher.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_push_publisher.py index b8fc4fe51..3a38cfd43 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_push_publisher.py +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_push_publisher.py @@ -18,7 +18,7 @@ from oslo_messaging._drivers.zmq_driver.client.publishers\ import zmq_publisher_base from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._drivers.zmq_driver import zmq_names -from oslo_messaging._i18n import _LI, _LW +from oslo_messaging._i18n import _LW LOG = logging.getLogger(__name__) @@ -35,7 +35,8 @@ class PushPublisher(zmq_publisher_base.PublisherMultisend): if request.msg_type == zmq_names.CALL_TYPE: raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) - push_socket, hosts = self._check_hosts_connections(request.target) + push_socket = self._check_hosts_connections( + request.target, zmq_names.socket_type_str(zmq.PULL)) if not push_socket.connections: LOG.warning(_LW("Request %s was dropped because no connection") @@ -52,6 +53,6 @@ class PushPublisher(zmq_publisher_base.PublisherMultisend): super(PushPublisher, self)._send_request(socket, request) - LOG.info(_LI("Publishing message %(message)s to a target %(target)s") - % {"message": request.message, - "target": request.target}) + LOG.debug("Publishing message %(message)s to a target %(target)s" + % {"message": request.message, + "target": request.target}) diff --git a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_req_publisher.py b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_req_publisher.py index d4dbaa9ab..78330f3a3 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_req_publisher.py +++ b/oslo_messaging/_drivers/zmq_driver/client/publishers/zmq_req_publisher.py @@ -14,6 +14,9 @@ import contextlib import logging +import uuid + +import six import oslo_messaging from oslo_messaging._drivers import common as rpc_common @@ -40,24 +43,35 @@ class ReqPublisher(zmq_publisher_base.PublisherBase): if request.msg_type != zmq_names.CALL_TYPE: raise zmq_publisher_base.UnsupportedSendPattern(request.msg_type) - socket = self._connect_to_host(request.target, request.timeout) + socket, connect_address = self._connect_to_host(request.target, + request.timeout) + request.host = connect_address self._send_request(socket, request) return self._receive_reply(socket, request) + def _resolve_host_address(self, target, timeout=0): + host = self.matchmaker.get_single_host( + target, zmq_names.socket_type_str(zmq.ROUTER), timeout) + return zmq_address.get_tcp_direct_address(host) + def _connect_to_host(self, target, timeout=0): try: self.zmq_context = zmq.Context() socket = self.zmq_context.socket(zmq.REQ) - host = self.matchmaker.get_single_host(target, timeout) - connect_address = zmq_address.get_tcp_direct_address(host) + if six.PY3: + socket.setsockopt_string(zmq.IDENTITY, str(uuid.uuid1())) + else: + socket.identity = str(uuid.uuid1()) + + connect_address = self._resolve_host_address(target, timeout) LOG.info(_LI("Connecting REQ to %s") % connect_address) socket.connect(connect_address) self.outbound_sockets[str(target)] = socket - return socket + return socket, connect_address except zmq.ZMQError as e: errmsg = _LE("Error connecting to socket: %s") % str(e) @@ -68,8 +82,11 @@ class ReqPublisher(zmq_publisher_base.PublisherBase): def _receive_reply(socket, request): def _receive_method(socket): - return socket.recv_pyobj() + reply = socket.recv_pyobj() + LOG.debug("Received reply %s" % reply) + return reply + LOG.debug("Start waiting reply") # NOTE(ozamiatin): Check for retry here (no retries now) with contextlib.closing(zmq_async.get_reply_poller()) as poller: poller.register(socket, recv_method=_receive_method) @@ -77,6 +94,7 @@ class ReqPublisher(zmq_publisher_base.PublisherBase): if reply is None: raise oslo_messaging.MessagingTimeout( "Timeout %s seconds was reached" % request.timeout) + LOG.debug("Received reply %s" % reply) if reply[zmq_names.FIELD_FAILURE]: raise rpc_common.deserialize_remote_exception( reply[zmq_names.FIELD_FAILURE], @@ -87,3 +105,26 @@ class ReqPublisher(zmq_publisher_base.PublisherBase): def close(self): # For contextlib compatibility self.cleanup() + + +class ReqPublisherLight(ReqPublisher): + + def __init__(self, conf, matchmaker): + super(ReqPublisherLight, self).__init__(conf, matchmaker) + + def _resolve_host_address(self, target, timeout=0): + return zmq_address.get_broker_address(self.conf) + + def _send_request(self, socket, request): + + LOG.debug("Sending %(type)s message_id %(message)s" + " to a target %(target)s, host:%(host)s" + % {"type": request.msg_type, + "message": request.message_id, + "target": request.target, + "host": request.host}) + + envelope = request.create_envelope() + + socket.send_pyobj(envelope, zmq.SNDMORE) + socket.send_pyobj(request) diff --git a/oslo_messaging/_drivers/zmq_driver/client/zmq_client.py b/oslo_messaging/_drivers/zmq_driver/client/zmq_client.py index 3e7888d5f..c6e895863 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/zmq_client.py +++ b/oslo_messaging/_drivers/zmq_driver/client/zmq_client.py @@ -12,68 +12,33 @@ # License for the specific language governing permissions and limitations # under the License. -import contextlib -from oslo_messaging._drivers.zmq_driver.client.publishers\ +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ + import zmq_dealer_call_publisher +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ import zmq_dealer_publisher -from oslo_messaging._drivers.zmq_driver.client.publishers\ - import zmq_req_publisher -from oslo_messaging._drivers.zmq_driver.client import zmq_request -from oslo_messaging._drivers.zmq_driver import zmq_address +from oslo_messaging._drivers.zmq_driver.client import zmq_client_base from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names zmq = zmq_async.import_zmq() -class ZmqClient(object): +class ZmqClient(zmq_client_base.ZmqClientBase): def __init__(self, conf, matchmaker=None, allowed_remote_exmods=None): - self.conf = conf - self.context = zmq.Context() - self.matchmaker = matchmaker - self.allowed_remote_exmods = allowed_remote_exmods or [] + if conf.zmq_use_broker: + raise rpc_common.RPCException("This client doesn't need proxy!") - self.dealer_publisher = None - if self.conf.zmq_use_broker: - self.dealer_publisher = zmq_dealer_publisher.DealerPublisherLight( - conf, zmq_address.get_broker_address(self.conf)) - else: - self.dealer_publisher = zmq_dealer_publisher.DealerPublisher( - conf, matchmaker) + super(ZmqClient, self).__init__( + conf, matchmaker, allowed_remote_exmods, + publishers={ + zmq_names.CALL_TYPE: + zmq_dealer_call_publisher.DealerCallPublisher( + conf, matchmaker), - def send_call(self, target, context, message, timeout=None, retry=None): - with contextlib.closing(zmq_request.CallRequest( - target, context=context, message=message, - timeout=timeout, retry=retry, - allowed_remote_exmods=self.allowed_remote_exmods)) as request: - with contextlib.closing(zmq_req_publisher.ReqPublisher( - self.conf, self.matchmaker)) as req_publisher: - return req_publisher.send_request(request) - - def send_cast(self, target, context, message, timeout=None, retry=None): - with contextlib.closing(zmq_request.CastRequest( - target, context=context, message=message, - timeout=timeout, retry=retry)) as request: - self.dealer_publisher.send_request(request) - - def send_fanout(self, target, context, message, timeout=None, retry=None): - with contextlib.closing(zmq_request.FanoutRequest( - target, context=context, message=message, - timeout=timeout, retry=retry)) as request: - self.dealer_publisher.send_request(request) - - def send_notify(self, target, context, message, version, retry=None): - with contextlib.closing(zmq_request.NotificationRequest( - target, context, message, version=version, - retry=retry)) as request: - self.dealer_publisher.send_request(request) - - def send_notify_fanout(self, target, context, message, version, - retry=None): - with contextlib.closing(zmq_request.NotificationFanoutRequest( - target, context, message, version=version, - retry=retry)) as request: - self.dealer_publisher.send_request(request) - - def cleanup(self): - self.dealer_publisher.cleanup() + "default": zmq_dealer_publisher.DealerPublisher( + conf, matchmaker) + } + ) diff --git a/oslo_messaging/_drivers/zmq_driver/client/zmq_client_base.py b/oslo_messaging/_drivers/zmq_driver/client/zmq_client_base.py new file mode 100644 index 000000000..aa7cd12d1 --- /dev/null +++ b/oslo_messaging/_drivers/zmq_driver/client/zmq_client_base.py @@ -0,0 +1,77 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import contextlib + +from oslo_messaging._drivers.zmq_driver.client import zmq_request +from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names + +zmq = zmq_async.import_zmq() + + +class ZmqClientBase(object): + + def __init__(self, conf, matchmaker=None, allowed_remote_exmods=None, + publishers=None): + self.conf = conf + self.context = zmq.Context() + self.matchmaker = matchmaker + self.allowed_remote_exmods = allowed_remote_exmods or [] + + self.publishers = publishers + self.call_publisher = publishers.get(zmq_names.CALL_TYPE) \ + or publishers["default"] + self.cast_publisher = publishers.get(zmq_names.CAST_TYPE) \ + or publishers["default"] + self.fanout_publisher = publishers.get(zmq_names.CAST_FANOUT_TYPE) \ + or publishers["default"] + self.notify_publisher = publishers.get(zmq_names.NOTIFY_TYPE) \ + or publishers["default"] + + def send_call(self, target, context, message, timeout=None, retry=None): + with contextlib.closing(zmq_request.CallRequest( + target, context=context, message=message, + timeout=timeout, retry=retry, + allowed_remote_exmods=self.allowed_remote_exmods)) as request: + return self.call_publisher.send_request(request) + + def send_cast(self, target, context, message, timeout=None, retry=None): + with contextlib.closing(zmq_request.CastRequest( + target, context=context, message=message, + timeout=timeout, retry=retry)) as request: + self.cast_publisher.send_request(request) + + def send_fanout(self, target, context, message, timeout=None, retry=None): + with contextlib.closing(zmq_request.FanoutRequest( + target, context=context, message=message, + timeout=timeout, retry=retry)) as request: + self.fanout_publisher.send_request(request) + + def send_notify(self, target, context, message, version, retry=None): + with contextlib.closing(zmq_request.NotificationRequest( + target, context, message, version=version, + retry=retry)) as request: + self.notify_publisher.send_request(request) + + def send_notify_fanout(self, target, context, message, version, + retry=None): + with contextlib.closing(zmq_request.NotificationFanoutRequest( + target, context, message, version=version, + retry=retry)) as request: + self.notify_publisher.send_request(request) + + def cleanup(self): + for publisher in self.publishers.values(): + publisher.cleanup() diff --git a/oslo_messaging/_drivers/zmq_driver/client/zmq_client_light.py b/oslo_messaging/_drivers/zmq_driver/client/zmq_client_light.py new file mode 100644 index 000000000..873911f8d --- /dev/null +++ b/oslo_messaging/_drivers/zmq_driver/client/zmq_client_light.py @@ -0,0 +1,46 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_messaging._drivers import common as rpc_common +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ + import zmq_dealer_call_publisher +from oslo_messaging._drivers.zmq_driver.client.publishers.dealer \ + import zmq_dealer_publisher +from oslo_messaging._drivers.zmq_driver.client import zmq_client_base +from oslo_messaging._drivers.zmq_driver import zmq_address +from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names + +zmq = zmq_async.import_zmq() + + +class ZmqClientLight(zmq_client_base.ZmqClientBase): + + def __init__(self, conf, matchmaker=None, allowed_remote_exmods=None): + if not conf.zmq_use_broker: + raise rpc_common.RPCException( + "This client needs proxy to be configured!") + + super(ZmqClientLight, self).__init__( + conf, matchmaker, allowed_remote_exmods, + publishers={ + zmq_names.CALL_TYPE: + zmq_dealer_call_publisher.DealerCallPublisher( + conf, matchmaker), + + "default": zmq_dealer_publisher.DealerPublisherLight( + conf, zmq_address.get_broker_address(self.conf)) + } + ) diff --git a/oslo_messaging/_drivers/zmq_driver/client/zmq_request.py b/oslo_messaging/_drivers/zmq_driver/client/zmq_request.py index 92d444a33..455b7ba5a 100644 --- a/oslo_messaging/_drivers/zmq_driver/client/zmq_request.py +++ b/oslo_messaging/_drivers/zmq_driver/client/zmq_request.py @@ -63,6 +63,12 @@ class Request(object): self.message = message self.retry = retry self.message_id = str(uuid.uuid1()) + self.proxy_reply_id = None + + def create_envelope(self): + return {'msg_type': self.msg_type, + 'message_id': self.message_id, + 'target': self.target} @abc.abstractproperty def msg_type(self): @@ -86,6 +92,11 @@ class RpcRequest(Request): super(RpcRequest, self).__init__(*args, **kwargs) + def create_envelope(self): + envelope = super(RpcRequest, self).create_envelope() + envelope['timeout'] = self.timeout + return envelope + class CallRequest(RpcRequest): diff --git a/oslo_messaging/_drivers/zmq_driver/matchmaker/base.py b/oslo_messaging/_drivers/zmq_driver/matchmaker/base.py index 8b2365b41..7b9b69d79 100644 --- a/oslo_messaging/_drivers/zmq_driver/matchmaker/base.py +++ b/oslo_messaging/_drivers/zmq_driver/matchmaker/base.py @@ -20,6 +20,7 @@ import retrying import six import oslo_messaging +from oslo_messaging._drivers.zmq_driver import zmq_address from oslo_messaging._i18n import _LI, _LW @@ -35,27 +36,31 @@ class MatchMakerBase(object): self.conf = conf @abc.abstractmethod - def register(self, target, hostname): + def register(self, target, hostname, listener_type): """Register target on nameserver. :param target: the target for host :type target: Target :param hostname: host for the topic in "host:port" format :type hostname: String + :param listener_type: Listener socket type ROUTER, SUB etc. + :type listener_type: String """ @abc.abstractmethod - def unregister(self, target, hostname): + def unregister(self, target, hostname, listener_type): """Unregister target from nameserver. :param target: the target for host :type target: Target :param hostname: host for the topic in "host:port" format :type hostname: String + :param listener_type: Listener socket type ROUTER, SUB etc. + :type listener_type: String """ @abc.abstractmethod - def get_hosts(self, target): + def get_hosts(self, target, listener_type): """Get all hosts from nameserver by target. :param target: the default target for invocations @@ -63,7 +68,7 @@ class MatchMakerBase(object): :returns: a list of "hostname:port" hosts """ - def get_single_host(self, target, timeout=None, retry=0): + def get_single_host(self, target, listener_type, timeout=None, retry=0): """Get a single host by target. :param target: the target for messages @@ -101,7 +106,7 @@ class MatchMakerBase(object): @_retry def _get_single_host(): - hosts = self.get_hosts(target) + hosts = self.get_hosts(target, listener_type) try: if not hosts: err_msg = "No hosts were found for target %s." % target @@ -136,16 +141,16 @@ class DummyMatchMaker(MatchMakerBase): self._cache = collections.defaultdict(list) - def register(self, target, hostname): - key = str(target) + def register(self, target, hostname, listener_type): + key = zmq_address.target_to_key(target, listener_type) if hostname not in self._cache[key]: self._cache[key].append(hostname) - def unregister(self, target, hostname): - key = str(target) + def unregister(self, target, hostname, listener_type): + key = zmq_address.target_to_key(target, listener_type) if hostname in self._cache[key]: self._cache[key].remove(hostname) - def get_hosts(self, target): - key = str(target) + def get_hosts(self, target, listener_type): + key = zmq_address.target_to_key(target, listener_type) return self._cache[key] diff --git a/oslo_messaging/_drivers/zmq_driver/matchmaker/matchmaker_redis.py b/oslo_messaging/_drivers/zmq_driver/matchmaker/matchmaker_redis.py index c8402c6c8..3bbcf321a 100644 --- a/oslo_messaging/_drivers/zmq_driver/matchmaker/matchmaker_redis.py +++ b/oslo_messaging/_drivers/zmq_driver/matchmaker/matchmaker_redis.py @@ -17,6 +17,7 @@ from oslo_config import cfg from oslo_utils import importutils from oslo_messaging._drivers.zmq_driver.matchmaker import base +from oslo_messaging._drivers.zmq_driver import zmq_address redis = importutils.try_import('redis') LOG = logging.getLogger(__name__) @@ -26,9 +27,9 @@ matchmaker_redis_opts = [ cfg.StrOpt('host', default='127.0.0.1', help='Host to locate redis.'), - cfg.IntOpt('port', - default=6379, - help='Use this port to connect to redis host.'), + cfg.PortOpt('port', + default=6379, + help='Use this port to connect to redis host.'), cfg.StrOpt('password', default='', secret=True, @@ -48,34 +49,32 @@ class RedisMatchMaker(base.MatchMakerBase): password=self.conf.matchmaker_redis.password, ) - def _target_to_key(self, target): - attributes = ['topic', 'exchange', 'server'] - prefix = "ZMQ-target" - key = ":".join((getattr(target, attr) or "*") for attr in attributes) - return "%s-%s" % (prefix, key) - - def _get_keys_by_pattern(self, pattern): - return self._redis.keys(pattern) - def _get_hosts_by_key(self, key): return self._redis.lrange(key, 0, -1) - def register(self, target, hostname): - key = self._target_to_key(target) - if hostname not in self._get_hosts_by_key(key): - self._redis.lpush(key, hostname) + def register(self, target, hostname, listener_type): - def unregister(self, target, hostname): - key = self._target_to_key(target) + if target.topic and target.server: + key = zmq_address.target_to_key(target, listener_type) + if hostname not in self._get_hosts_by_key(key): + self._redis.lpush(key, hostname) + + if target.topic: + key = zmq_address.prefix_str(target.topic, listener_type) + if hostname not in self._get_hosts_by_key(key): + self._redis.lpush(key, hostname) + + if target.server: + key = zmq_address.prefix_str(target.server, listener_type) + if hostname not in self._get_hosts_by_key(key): + self._redis.lpush(key, hostname) + + def unregister(self, target, hostname, listener_type): + key = zmq_address.target_to_key(target, listener_type) self._redis.lrem(key, 0, hostname) - def get_hosts(self, target): - pattern = self._target_to_key(target) - if "*" not in pattern: - # pattern have no placeholders, so this is valid key - return self._get_hosts_by_key(pattern) - + def get_hosts(self, target, listener_type): hosts = [] - for key in self._get_keys_by_pattern(pattern): - hosts.extend(self._get_hosts_by_key(key)) + key = zmq_address.target_to_key(target, listener_type) + hosts.extend(self._get_hosts_by_key(key)) return hosts diff --git a/oslo_messaging/_drivers/zmq_driver/poller/threading_poller.py b/oslo_messaging/_drivers/zmq_driver/poller/threading_poller.py index c0a46d981..8167715f1 100644 --- a/oslo_messaging/_drivers/zmq_driver/poller/threading_poller.py +++ b/oslo_messaging/_drivers/zmq_driver/poller/threading_poller.py @@ -38,12 +38,20 @@ class ThreadingPoller(zmq_poller.ZmqPoller): self.recv_methods = {} def register(self, socket, recv_method=None): + LOG.debug("Registering socket") + if socket in self.recv_methods: + return if recv_method is not None: self.recv_methods[socket] = recv_method self.poller.register(socket, zmq.POLLIN) def poll(self, timeout=None): - timeout *= 1000 # zmq poller waits milliseconds + + LOG.debug("Entering poll method") + + if timeout: + timeout *= 1000 # zmq poller waits milliseconds + sockets = None try: diff --git a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_consumer_base.py b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_consumer_base.py index 8bb2461e7..b7532a74a 100644 --- a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_consumer_base.py +++ b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_consumer_base.py @@ -19,8 +19,9 @@ import six from oslo_messaging._drivers import common as rpc_common from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_names from oslo_messaging._drivers.zmq_driver import zmq_socket -from oslo_messaging._i18n import _LE, _LI +from oslo_messaging._i18n import _LE LOG = logging.getLogger(__name__) @@ -43,10 +44,10 @@ class ConsumerBase(object): self.conf, self.context, socket_type) self.sockets.append(socket) self.poller.register(socket, self.receive_message) - LOG.info(_LI("Run %(stype)s consumer on %(addr)s:%(port)d"), - {"stype": socket_type, - "addr": socket.bind_address, - "port": socket.port}) + LOG.debug("Run %(stype)s consumer on %(addr)s:%(port)d", + {"stype": zmq_names.socket_type_str(socket_type), + "addr": socket.bind_address, + "port": socket.port}) return socket except zmq.ZMQError as e: errmsg = _LE("Failed binding to port %(port)d: %(e)s")\ diff --git a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_pull_consumer.py b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_pull_consumer.py index 98ef3a73c..81cf7fde0 100644 --- a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_pull_consumer.py +++ b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_pull_consumer.py @@ -56,9 +56,9 @@ class PullConsumer(zmq_consumer_base.SingleSocketConsumer): assert msg_type is not None, 'Bad format: msg type expected' context = socket.recv_pyobj() message = socket.recv_pyobj() - LOG.info(_LI("Received %(msg_type)s message %(msg)s") - % {"msg_type": msg_type, - "msg": str(message)}) + LOG.debug("Received %(msg_type)s message %(msg)s" + % {"msg_type": msg_type, + "msg": str(message)}) if msg_type in (zmq_names.CAST_TYPES + zmq_names.NOTIFY_TYPES): return PullIncomingMessage(self.server, context, message) diff --git a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_router_consumer.py b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_router_consumer.py index f6016607e..f5885c55a 100644 --- a/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_router_consumer.py +++ b/oslo_messaging/_drivers/zmq_driver/server/consumers/zmq_router_consumer.py @@ -21,7 +21,7 @@ from oslo_messaging._drivers.zmq_driver.server import zmq_incoming_message from oslo_messaging._drivers.zmq_driver import zmq_address from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._drivers.zmq_driver import zmq_names -from oslo_messaging._i18n import _LE, _LI +from oslo_messaging._i18n import _LE LOG = logging.getLogger(__name__) @@ -43,11 +43,7 @@ class RouterIncomingMessage(base.IncomingMessage): """Reply is not needed for non-call messages""" def acknowledge(self): - LOG.info("Sending acknowledge for %s", self.msg_id) - ack_message = {zmq_names.FIELD_ID: self.msg_id} - self.socket.send(self.reply_id, zmq.SNDMORE) - self.socket.send(b'', zmq.SNDMORE) - self.socket.send_pyobj(ack_message) + LOG.debug("Not sending acknowledge for %s", self.msg_id) def requeue(self): """Requeue is not supported""" @@ -61,36 +57,41 @@ class RouterConsumer(zmq_consumer_base.SingleSocketConsumer): self.targets = [] self.host = zmq_address.combine_address(self.conf.rpc_zmq_host, self.port) + LOG.info("[%s] Run ROUTER consumer" % self.host) def listen(self, target): - LOG.info("Listen to target %s on %s:%d" % - (target, self.address, self.port)) + LOG.info("[%s] Listen to target %s" % (self.host, target)) self.targets.append(target) - self.matchmaker.register(target=target, - hostname=self.host) + self.matchmaker.register(target, self.host, + zmq_names.socket_type_str(zmq.ROUTER)) def cleanup(self): super(RouterConsumer, self).cleanup() for target in self.targets: - self.matchmaker.unregister(target, self.host) + self.matchmaker.unregister(target, self.host, + zmq_names.socket_type_str(zmq.ROUTER)) + + def _receive_request(self, socket): + reply_id = socket.recv() + empty = socket.recv() + assert empty == b'', 'Bad format: empty delimiter expected' + request = socket.recv_pyobj() + return request, reply_id def receive_message(self, socket): try: - reply_id = socket.recv() - empty = socket.recv() - assert empty == b'', 'Bad format: empty delimiter expected' - request = socket.recv_pyobj() - - LOG.info(_LI("Received %(msg_type)s message %(msg)s") - % {"msg_type": request.msg_type, - "msg": str(request.message)}) + request, reply_id = self._receive_request(socket) + LOG.debug("[%(host)s] Received %(type)s, %(id)s, %(target)s" + % {"host": self.host, + "type": request.msg_type, + "id": request.message_id, + "target": request.target}) if request.msg_type == zmq_names.CALL_TYPE: return zmq_incoming_message.ZmqIncomingRequest( - self.server, request.context, request.message, socket, - reply_id, self.poller) + self.server, socket, reply_id, request, self.poller) elif request.msg_type in zmq_names.NON_BLOCKING_TYPES: return RouterIncomingMessage( self.server, request.context, request.message, socket, @@ -100,3 +101,20 @@ class RouterConsumer(zmq_consumer_base.SingleSocketConsumer): except zmq.ZMQError as e: LOG.error(_LE("Receiving message failed: %s") % str(e)) + + +class RouterConsumerBroker(RouterConsumer): + + def __init__(self, conf, poller, server): + super(RouterConsumerBroker, self).__init__(conf, poller, server) + + def _receive_request(self, socket): + reply_id = socket.recv() + empty = socket.recv() + assert empty == b'', 'Bad format: empty delimiter expected' + envelope = socket.recv_pyobj() + request = socket.recv_pyobj() + + if zmq_names.FIELD_REPLY_ID in envelope: + request.proxy_reply_id = envelope[zmq_names.FIELD_REPLY_ID] + return request, reply_id diff --git a/oslo_messaging/_drivers/zmq_driver/server/zmq_incoming_message.py b/oslo_messaging/_drivers/zmq_driver/server/zmq_incoming_message.py index f43ec2325..e009d55c9 100644 --- a/oslo_messaging/_drivers/zmq_driver/server/zmq_incoming_message.py +++ b/oslo_messaging/_drivers/zmq_driver/server/zmq_incoming_message.py @@ -28,10 +28,12 @@ zmq = zmq_async.import_zmq() class ZmqIncomingRequest(base.IncomingMessage): - def __init__(self, listener, context, message, socket, rep_id, poller): - super(ZmqIncomingRequest, self).__init__(listener, context, message) + def __init__(self, listener, socket, rep_id, request, poller): + super(ZmqIncomingRequest, self).__init__(listener, request.context, + request.message) self.reply_socket = socket self.reply_id = rep_id + self.request = request self.received = None self.poller = poller @@ -39,15 +41,22 @@ class ZmqIncomingRequest(base.IncomingMessage): if failure is not None: failure = rpc_common.serialize_remote_exception(failure, log_failure) - message_reply = {zmq_names.FIELD_REPLY: reply, + message_reply = {zmq_names.FIELD_TYPE: zmq_names.REPLY_TYPE, + zmq_names.FIELD_REPLY: reply, zmq_names.FIELD_FAILURE: failure, - zmq_names.FIELD_LOG_FAILURE: log_failure} + zmq_names.FIELD_LOG_FAILURE: log_failure, + zmq_names.FIELD_ID: self.request.proxy_reply_id, + zmq_names.FIELD_MSG_ID: self.request.message_id} - LOG.info("Replying %s REP", (str(message_reply))) + LOG.debug("Replying %s", (str(self.request.message_id))) self.received = True self.reply_socket.send(self.reply_id, zmq.SNDMORE) self.reply_socket.send(b'', zmq.SNDMORE) + if self.request.proxy_reply_id: + self.reply_socket.send_string(zmq_names.REPLY_TYPE, zmq.SNDMORE) + self.reply_socket.send(self.request.proxy_reply_id, zmq.SNDMORE) + self.reply_socket.send(b'', zmq.SNDMORE) self.reply_socket.send_pyobj(message_reply) self.poller.resume_polling(self.reply_socket) diff --git a/oslo_messaging/_drivers/zmq_driver/server/zmq_server.py b/oslo_messaging/_drivers/zmq_driver/server/zmq_server.py index afe03b81b..c37aef047 100644 --- a/oslo_messaging/_drivers/zmq_driver/server/zmq_server.py +++ b/oslo_messaging/_drivers/zmq_driver/server/zmq_server.py @@ -31,11 +31,16 @@ class ZmqServer(base.Listener): super(ZmqServer, self).__init__(driver) self.matchmaker = matchmaker self.poller = zmq_async.get_poller() - self.rpc_consumer = zmq_router_consumer.RouterConsumer( - conf, self.poller, self) + if conf.zmq_use_broker: + self.rpc_consumer = zmq_router_consumer.RouterConsumerBroker( + conf, self.poller, self) + else: + self.rpc_consumer = zmq_router_consumer.RouterConsumer( + conf, self.poller, self) self.notify_consumer = self.rpc_consumer self.consumers = [self.rpc_consumer] + @base.batch_poll_helper def poll(self, timeout=None): message, socket = self.poller.poll( timeout or self.conf.rpc_poll_timeout) diff --git a/oslo_messaging/_drivers/zmq_driver/zmq_address.py b/oslo_messaging/_drivers/zmq_driver/zmq_address.py index e8c48291b..397bd1074 100644 --- a/oslo_messaging/_drivers/zmq_driver/zmq_address.py +++ b/oslo_messaging/_drivers/zmq_driver/zmq_address.py @@ -22,8 +22,27 @@ def get_tcp_direct_address(host): def get_tcp_random_address(conf): - return "tcp://*" + return "tcp://%s" % conf.rpc_zmq_bind_address def get_broker_address(conf): return "ipc://%s/zmq-broker" % conf.rpc_zmq_ipc_dir + + +def prefix_str(key, listener_type): + return listener_type + "_" + key + + +def target_to_key(target, listener_type): + + def prefix(key): + return prefix_str(key, listener_type) + + if target.topic and target.server: + attributes = ['topic', 'server'] + key = ".".join(getattr(target, attr) for attr in attributes) + return prefix(key) + if target.topic: + return prefix(target.topic) + if target.server: + return prefix(target.server) diff --git a/oslo_messaging/_drivers/zmq_driver/zmq_async.py b/oslo_messaging/_drivers/zmq_driver/zmq_async.py index 7a993a285..093544118 100644 --- a/oslo_messaging/_drivers/zmq_driver/zmq_async.py +++ b/oslo_messaging/_drivers/zmq_driver/zmq_async.py @@ -30,12 +30,10 @@ def import_zmq(zmq_concurrency='eventlet'): _raise_error_if_invalid_config_value(zmq_concurrency) imported_zmq = importutils.try_import(ZMQ_MODULES[zmq_concurrency], - default='zmq') + default=None) if imported_zmq is None: - errmsg = _LE("ZeroMQ not found!") LOG.error(_LE("ZeroMQ not found!")) - raise ImportError(errmsg) return imported_zmq @@ -80,3 +78,13 @@ def _raise_error_if_invalid_config_value(zmq_concurrency): if zmq_concurrency not in ZMQ_MODULES: errmsg = _('Invalid zmq_concurrency value: %s') raise ValueError(errmsg % zmq_concurrency) + + +def get_queue(zmq_concurrency='eventlet'): + _raise_error_if_invalid_config_value(zmq_concurrency) + if zmq_concurrency == 'eventlet' and _is_eventlet_zmq_available(): + import eventlet + return eventlet.queue.Queue(), eventlet.queue.Empty + else: + import six + return six.moves.queue.Queue(), six.moves.queue.Empty diff --git a/oslo_messaging/_drivers/zmq_driver/zmq_names.py b/oslo_messaging/_drivers/zmq_driver/zmq_names.py index a317456e7..f7401ab21 100644 --- a/oslo_messaging/_drivers/zmq_driver/zmq_names.py +++ b/oslo_messaging/_drivers/zmq_driver/zmq_names.py @@ -17,10 +17,23 @@ from oslo_messaging._drivers.zmq_driver import zmq_async zmq = zmq_async.import_zmq() +FIELD_TYPE = 'type' FIELD_FAILURE = 'failure' FIELD_REPLY = 'reply' FIELD_LOG_FAILURE = 'log_failure' FIELD_ID = 'id' +FIELD_MSG_ID = 'message_id' +FIELD_MSG_TYPE = 'msg_type' +FIELD_REPLY_ID = 'reply_id' +FIELD_TARGET = 'target' + + +IDX_REPLY_TYPE = 1 +IDX_REPLY_BODY = 2 + +MULTIPART_IDX_ENVELOPE = 0 +MULTIPART_IDX_BODY = 1 + CALL_TYPE = 'call' CAST_TYPE = 'cast' @@ -28,6 +41,9 @@ CAST_FANOUT_TYPE = 'cast-f' NOTIFY_TYPE = 'notify' NOTIFY_FANOUT_TYPE = 'notify-f' +REPLY_TYPE = 'reply' +ACK_TYPE = 'ack' + MESSAGE_TYPES = (CALL_TYPE, CAST_TYPE, CAST_FANOUT_TYPE, diff --git a/oslo_messaging/_drivers/zmq_driver/zmq_socket.py b/oslo_messaging/_drivers/zmq_driver/zmq_socket.py index 2a4144c5a..4119e5735 100644 --- a/oslo_messaging/_drivers/zmq_driver/zmq_socket.py +++ b/oslo_messaging/_drivers/zmq_driver/zmq_socket.py @@ -17,6 +17,8 @@ import logging from oslo_messaging._drivers.zmq_driver import zmq_address from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging._drivers.zmq_driver import zmq_names +from oslo_messaging._i18n import _LE +from oslo_messaging import exceptions LOG = logging.getLogger(__name__) @@ -45,6 +47,9 @@ class ZmqSocket(object): def setsockopt(self, *args, **kwargs): self.handle.setsockopt(*args, **kwargs) + def setsockopt_string(self, *args, **kwargs): + self.handle.setsockopt_string(*args, **kwargs) + def send(self, *args, **kwargs): self.handle.send(*args, **kwargs) @@ -57,6 +62,9 @@ class ZmqSocket(object): def send_pyobj(self, *args, **kwargs): self.handle.send_pyobj(*args, **kwargs) + def send_multipart(self, *args, **kwargs): + self.handle.send_multipart(*args, **kwargs) + def recv(self, *args, **kwargs): return self.handle.recv(*args, **kwargs) @@ -69,14 +77,30 @@ class ZmqSocket(object): def recv_pyobj(self, *args, **kwargs): return self.handle.recv_pyobj(*args, **kwargs) + def recv_multipart(self, *args, **kwargs): + return self.handle.recv_multipart(*args, **kwargs) + def close(self, *args, **kwargs): self.handle.close(*args, **kwargs) +class ZmqPortRangeExceededException(exceptions.MessagingException): + """Raised by ZmqRandomPortSocket - wrapping zmq.ZMQBindError""" + + class ZmqRandomPortSocket(ZmqSocket): def __init__(self, conf, context, socket_type): super(ZmqRandomPortSocket, self).__init__(context, socket_type) self.conf = conf self.bind_address = zmq_address.get_tcp_random_address(self.conf) - self.port = self.handle.bind_to_random_port(self.bind_address) + + try: + self.port = self.handle.bind_to_random_port( + self.bind_address, + min_port=conf.rpc_zmq_min_port, + max_port=conf.rpc_zmq_max_port, + max_tries=conf.rpc_zmq_bind_port_retries) + except zmq.ZMQBindError: + LOG.error(_LE("Random ports range exceeded!")) + raise ZmqPortRangeExceededException() diff --git a/oslo_messaging/_executors/impl_blocking.py b/oslo_messaging/_executors/impl_blocking.py index b59818f5c..b788c47f4 100644 --- a/oslo_messaging/_executors/impl_blocking.py +++ b/oslo_messaging/_executors/impl_blocking.py @@ -14,28 +14,57 @@ # under the License. import futurist +import threading from oslo_messaging._executors import impl_pooledexecutor +from oslo_utils import timeutils class FakeBlockingThread(object): + '''A minimal implementation of threading.Thread which does not create a + thread or start executing the target when start() is called. Instead, the + caller must explicitly execute the non-blocking thread.execute() method + after start() has been called. + ''' + def __init__(self, target): self._target = target + self._running = False + self._running_cond = threading.Condition() def start(self): - self._target() + if self._running: + # Not a user error. No need to translate. + raise RuntimeError('FakeBlockingThread already started') - @staticmethod - def join(timeout=None): - pass + with self._running_cond: + self._running = True + self._running_cond.notify_all() - @staticmethod - def stop(): - pass + def join(self, timeout=None): + with timeutils.StopWatch(duration=timeout) as w, self._running_cond: + while self._running: + self._running_cond.wait(w.leftover(return_none=True)) - @staticmethod - def is_alive(): - return False + # Thread.join() does not raise an exception on timeout. It is + # the caller's responsibility to check is_alive(). + if w.expired(): + return + + def is_alive(self): + return self._running + + def execute(self): + if not self._running: + # Not a user error. No need to translate. + raise RuntimeError('FakeBlockingThread not started') + + try: + self._target() + finally: + with self._running_cond: + self._running = False + self._running_cond.notify_all() class BlockingExecutor(impl_pooledexecutor.PooledExecutor): @@ -52,3 +81,22 @@ class BlockingExecutor(impl_pooledexecutor.PooledExecutor): _executor_cls = lambda __, ___: futurist.SynchronousExecutor() _thread_cls = FakeBlockingThread + + def __init__(self, *args, **kwargs): + super(BlockingExecutor, self).__init__(*args, **kwargs) + + def execute(self): + '''Explicitly run the executor in the current context.''' + # NOTE(mdbooth): Splitting start into start and execute for the + # blocking executor closes a potential race. On a non-blocking + # executor, calling start performs some initialisation synchronously + # before starting the executor and returning control to the caller. In + # the non-blocking caller there was no externally visible boundary + # between the completion of initialisation and the start of execution, + # meaning the caller cannot indicate to another thread that + # initialisation is complete. With the split, the start call for the + # blocking executor becomes analogous to the non-blocking case, + # indicating that initialisation is complete. The caller can then + # synchronously call execute. + if self._poller is not None: + self._poller.execute() diff --git a/oslo_messaging/_executors/impl_pooledexecutor.py b/oslo_messaging/_executors/impl_pooledexecutor.py index c0837701c..f442c3ae9 100644 --- a/oslo_messaging/_executors/impl_pooledexecutor.py +++ b/oslo_messaging/_executors/impl_pooledexecutor.py @@ -33,7 +33,7 @@ _pool_opts = [ class PooledExecutor(base.ExecutorBase): - """A message executor which integrates with some async executor. + """A message executor which integrates with some executor. This will create a message thread that polls for messages from a dispatching thread and on reception of an incoming message places the @@ -93,8 +93,11 @@ class PooledExecutor(base.ExecutorBase): @excutils.forever_retry_uncaught_exceptions def _runner(self): while not self._tombstone.is_set(): - incoming = self.listener.poll() - if incoming is None: + incoming = self.listener.poll( + timeout=self.dispatcher.batch_timeout, + prefetch_size=self.dispatcher.batch_size) + + if not incoming: continue callback = self.dispatcher(incoming, self._executor_callback) was_submitted = self._do_submit(callback) diff --git a/oslo_messaging/_utils.py b/oslo_messaging/_utils.py index 1bb20b089..021fea26c 100644 --- a/oslo_messaging/_utils.py +++ b/oslo_messaging/_utils.py @@ -46,57 +46,6 @@ def version_is_compatible(imp_version, version): return True -class DispatcherExecutorContext(object): - """Dispatcher executor context helper - - A dispatcher can have work to do before and after the dispatch of the - request in the main server thread while the dispatcher itself can be - done in its own thread. - - The executor can use the helper like this: - - callback = dispatcher(incoming) - callback.prepare() - thread = MyWhateverThread() - thread.on_done(callback.done) - thread.run(callback.run) - - """ - def __init__(self, incoming, dispatch, executor_callback=None, - post=None): - self._result = None - self._incoming = incoming - self._dispatch = dispatch - self._post = post - self._executor_callback = executor_callback - - def run(self): - """The incoming message dispath itself - - Can be run in an other thread/greenlet/corotine if the executor is - able to do it. - """ - try: - self._result = self._dispatch(self._incoming, - self._executor_callback) - except Exception: - msg = 'The dispatcher method must catches all exceptions' - LOG.exception(msg) - raise RuntimeError(msg) - - def done(self): - """Callback after the incoming message have been dispathed - - Should be ran in the main executor thread/greenlet/corotine - """ - # FIXME(sileht): this is not currently true, this works only because - # the driver connection used for polling write on the wire only to - # ack/requeue message, but what if one day, the driver do something - # else - if self._post is not None: - self._post(self._incoming, self._result) - - def fetch_current_thread_functor(): # Until https://github.com/eventlet/eventlet/issues/172 is resolved # or addressed we have to use complicated workaround to get a object @@ -116,29 +65,6 @@ def fetch_current_thread_functor(): return lambda: threading.current_thread() -class DummyCondition(object): - def acquire(self): - pass - - def notify(self): - pass - - def notify_all(self): - pass - - def wait(self, timeout=None): - pass - - def release(self): - pass - - def __enter__(self): - self.acquire() - - def __exit__(self, type, value, traceback): - self.release() - - class DummyLock(object): def acquire(self): pass diff --git a/oslo_messaging/conffixture.py b/oslo_messaging/conffixture.py index 1312d66cf..0d05a5881 100644 --- a/oslo_messaging/conffixture.py +++ b/oslo_messaging/conffixture.py @@ -18,6 +18,7 @@ __all__ = ['ConfFixture'] import sys import fixtures +from functools import wraps def _import_opts(conf, module, opts, group=None): @@ -50,9 +51,6 @@ class ConfFixture(fixtures.Fixture): _import_opts(self.conf, 'oslo_messaging._drivers.amqp', 'amqp_opts', 'oslo_messaging_rabbit') - _import_opts(self.conf, - 'oslo_messaging._drivers.impl_qpid', 'qpid_opts', - 'oslo_messaging_qpid') _import_opts(self.conf, 'oslo_messaging._drivers.amqp', 'amqp_opts', 'oslo_messaging_qpid') @@ -69,15 +67,63 @@ class ConfFixture(fixtures.Fixture): _import_opts(self.conf, 'oslo_messaging.rpc.client', '_client_opts') _import_opts(self.conf, 'oslo_messaging.transport', '_transport_opts') _import_opts(self.conf, - 'oslo_messaging.notify.notifier', '_notifier_opts') + 'oslo_messaging.notify.notifier', + '_notifier_opts', + 'oslo_messaging_notifications') + + def _setup_decorator(self): + # Support older test cases that still use the set_override + # with the old config key names + def decorator_for_set_override(wrapped_function): + @wraps(wrapped_function) + def _wrapper(*args, **kwargs): + group = 'oslo_messaging_notifications' + if args[0] == 'notification_driver': + args = ('driver', args[1], group) + elif args[0] == 'notification_transport_url': + args = ('transport_url', args[1], group) + elif args[0] == 'notification_topics': + args = ('topics', args[1], group) + return wrapped_function(*args, **kwargs) + _wrapper.wrapped = wrapped_function + return _wrapper + + def decorator_for_clear_override(wrapped_function): + @wraps(wrapped_function) + def _wrapper(*args, **kwargs): + group = 'oslo_messaging_notifications' + if args[0] == 'notification_driver': + args = ('driver', group) + elif args[0] == 'notification_transport_url': + args = ('transport_url', group) + elif args[0] == 'notification_topics': + args = ('topics', group) + return wrapped_function(*args, **kwargs) + _wrapper.wrapped = wrapped_function + return _wrapper + + if not hasattr(self.conf.set_override, 'wrapped'): + self.conf.set_override = decorator_for_set_override( + self.conf.set_override) + if not hasattr(self.conf.clear_override, 'wrapped'): + self.conf.clear_override = decorator_for_clear_override( + self.conf.clear_override) + + def _teardown_decorator(self): + if hasattr(self.conf.set_override, 'wrapped'): + self.conf.set_override = self.conf.set_override.wrapped + if hasattr(self.conf.clear_override, 'wrapped'): + self.conf.clear_override = self.conf.clear_override.wrapped def setUp(self): super(ConfFixture, self).setUp() + self._setup_decorator() + self.addCleanup(self._teardown_decorator) self.addCleanup(self.conf.reset) @property def transport_driver(self): - """The transport driver - for example 'rabbit', 'qpid' or 'fake'.""" + """The transport driver - for example 'rabbit', 'amqp' or 'fake'.""" return self.conf.rpc_backend @transport_driver.setter diff --git a/oslo_messaging/dispatcher.py b/oslo_messaging/dispatcher.py new file mode 100644 index 000000000..780724416 --- /dev/null +++ b/oslo_messaging/dispatcher.py @@ -0,0 +1,111 @@ + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import logging + +import six + +__all__ = [ + "DispatcherBase", + "DispatcherExecutorContext" +] + +LOG = logging.getLogger(__name__) + + +class DispatcherExecutorContext(object): + """Dispatcher executor context helper + + A dispatcher can have work to do before and after the dispatch of the + request in the main server thread while the dispatcher itself can be + done in its own thread. + + The executor can use the helper like this: + + callback = dispatcher(incoming) + callback.prepare() + thread = MyWhateverThread() + thread.on_done(callback.done) + thread.run(callback.run) + + """ + def __init__(self, incoming, dispatch, executor_callback=None, + post=None): + self._result = None + self._incoming = incoming + self._dispatch = dispatch + self._post = post + self._executor_callback = executor_callback + + def run(self): + """The incoming message dispath itself + + Can be run in an other thread/greenlet/corotine if the executor is + able to do it. + """ + try: + self._result = self._dispatch(self._incoming, + self._executor_callback) + except Exception: + msg = 'The dispatcher method must catches all exceptions' + LOG.exception(msg) + raise RuntimeError(msg) + + def done(self): + """Callback after the incoming message have been dispathed + + Should be ran in the main executor thread/greenlet/corotine + """ + # FIXME(sileht): this is not currently true, this works only because + # the driver connection used for polling write on the wire only to + # ack/requeue message, but what if one day, the driver do something + # else + if self._post is not None: + self._post(self._incoming, self._result) + + +@six.add_metaclass(abc.ABCMeta) +class DispatcherBase(object): + "Base class for dispatcher" + + batch_size = 1 + "Number of messages to wait before calling endpoints callacks" + + batch_timeout = None + "Number of seconds to wait before calling endpoints callacks" + + @abc.abstractmethod + def _listen(self, transport): + """Initiate the driver Listener + + Usualy the driver Listener is start with the transport helper methods: + + * transport._listen() + * transport._listen_for_notifications() + + :param transport: the transport object + :type transport: oslo_messaging.transport.Transport + :returns: a driver Listener object + :rtype: oslo_messaging._drivers.base.Listener + """ + + @abc.abstractmethod + def __call__(self, incoming, executor_callback=None): + """Called by the executor to get the DispatcherExecutorContext + + :param incoming: list of messages + :type incoming: oslo_messging._drivers.base.IncomingMessage + :returns: DispatcherExecutorContext + :rtype: DispatcherExecutorContext + """ diff --git a/oslo_messaging/notify/__init__.py b/oslo_messaging/notify/__init__.py index dd5304d46..912e63398 100644 --- a/oslo_messaging/notify/__init__.py +++ b/oslo_messaging/notify/__init__.py @@ -15,7 +15,9 @@ __all__ = ['Notifier', 'LoggingNotificationHandler', + 'get_notification_transport', 'get_notification_listener', + 'get_batch_notification_listener', 'NotificationResult', 'NotificationFilter', 'PublishErrorsHandler', diff --git a/oslo_messaging/notify/_impl_log.py b/oslo_messaging/notify/_impl_log.py index 40833e96b..fa6e1f3cf 100644 --- a/oslo_messaging/notify/_impl_log.py +++ b/oslo_messaging/notify/_impl_log.py @@ -16,6 +16,7 @@ # under the License. import logging +import warnings from oslo_serialization import jsonutils from oslo_utils import strutils @@ -40,3 +41,7 @@ class LogDriver(notifier.Driver): method = getattr(logger, priority.lower(), None) if method: method(strutils.mask_password(jsonutils.dumps(message))) + else: + warnings.warn('Unable to log message as notify cannot find a ' + 'logger with the priority specified ' + '%s' % priority.lower()) diff --git a/oslo_messaging/notify/_impl_messaging.py b/oslo_messaging/notify/_impl_messaging.py new file mode 100644 index 000000000..9f7c57113 --- /dev/null +++ b/oslo_messaging/notify/_impl_messaging.py @@ -0,0 +1,24 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from debtcollector import removals + +from oslo_messaging.notify.messaging import * # noqa + + +# NOTE(mriedem): removal depends on how we can cap requirements in +# stable/liberty such that neutron does not try to load this +removals.removed_module(__name__, + oslo_messaging.notify.messaging.__name__, + removal_version='?') diff --git a/oslo_messaging/notify/_impl_routing.py b/oslo_messaging/notify/_impl_routing.py index bf07e673e..3e89f8770 100644 --- a/oslo_messaging/notify/_impl_routing.py +++ b/oslo_messaging/notify/_impl_routing.py @@ -27,11 +27,13 @@ from oslo_messaging.notify import notifier LOG = logging.getLogger(__name__) -router_config = cfg.StrOpt('routing_notifier_config', default='', +router_config = cfg.StrOpt('routing_config', default='', + deprecated_group='DEFAULT', + deprecated_name='routing_notifier_config', help='RoutingNotifier configuration file location.') CONF = cfg.CONF -CONF.register_opt(router_config) +CONF.register_opt(router_config, group='oslo_messaging_notifications') class RoutingDriver(notifier.Driver): @@ -56,12 +58,12 @@ class RoutingDriver(notifier.Driver): """One-time load of notifier config file.""" self.routing_groups = {} self.used_drivers = set() - filename = CONF.routing_notifier_config + filename = CONF.oslo_messaging_notifications.routing_config if not filename: return # Infer which drivers are used from the config file. - self.routing_groups = yaml.load( + self.routing_groups = yaml.safe_load( self._get_notifier_config_file(filename)) if not self.routing_groups: self.routing_groups = {} # In case we got None from load() diff --git a/oslo_messaging/notify/dispatcher.py b/oslo_messaging/notify/dispatcher.py index 46d53035e..72287678d 100644 --- a/oslo_messaging/notify/dispatcher.py +++ b/oslo_messaging/notify/dispatcher.py @@ -16,9 +16,10 @@ import itertools import logging -import sys -from oslo_messaging import _utils as utils +import six + +from oslo_messaging import dispatcher from oslo_messaging import localcontext from oslo_messaging import serializer as msg_serializer @@ -33,17 +34,7 @@ class NotificationResult(object): REQUEUE = 'requeue' -class NotificationDispatcher(object): - """A message dispatcher which understands Notification messages. - - A MessageHandlingServer is constructed by passing a callable dispatcher - which is invoked with context and message dictionaries each time a message - is received. - - NotifcationDispatcher is one such dispatcher which pass a raw notification - message to the endpoints - """ - +class _NotificationDispatcherBase(dispatcher.DispatcherBase): def __init__(self, targets, endpoints, serializer, allow_requeue, pool=None): self.targets = targets @@ -65,21 +56,25 @@ class NotificationDispatcher(object): priorities)) def _listen(self, transport): + transport._require_driver_features(requeue=self.allow_requeue) return transport._listen_for_notifications(self._targets_priorities, pool=self.pool) def __call__(self, incoming, executor_callback=None): - return utils.DispatcherExecutorContext( + return dispatcher.DispatcherExecutorContext( incoming, self._dispatch_and_handle_error, executor_callback=executor_callback, post=self._post_dispatch) - @staticmethod - def _post_dispatch(incoming, result): - if result == NotificationResult.HANDLED: - incoming.acknowledge() - else: - incoming.requeue() + def _post_dispatch(self, incoming, requeues): + for m in incoming: + try: + if requeues and m in requeues: + m.requeue() + else: + m.acknowledge() + except Exception: + LOG.error("Fail to ack/requeue message", exc_info=True) def _dispatch_and_handle_error(self, incoming, executor_callback): """Dispatch a notification message to the appropriate endpoint method. @@ -88,24 +83,59 @@ class NotificationDispatcher(object): :type ctxt: IncomingMessage """ try: - return self._dispatch(incoming.ctxt, incoming.message, - executor_callback) + return self._dispatch(incoming, executor_callback) except Exception: - # sys.exc_info() is deleted by LOG.exception(). - exc_info = sys.exc_info() - LOG.error('Exception during message handling', - exc_info=exc_info) - return NotificationResult.HANDLED + LOG.error('Exception during message handling', exc_info=True) - def _dispatch(self, ctxt, message, executor_callback=None): - """Dispatch an RPC message to the appropriate endpoint method. - - :param ctxt: the request context - :type ctxt: dict - :param message: the message payload - :type message: dict + def _dispatch(self, incoming, executor_callback=None): + """Dispatch notification messages to the appropriate endpoint method. """ - ctxt = self.serializer.deserialize_context(ctxt) + + messages_grouped = itertools.groupby(( + self._extract_user_message(m) + for m in incoming), lambda x: x[0]) + + requeues = set() + for priority, messages in messages_grouped: + __, raw_messages, messages = six.moves.zip(*messages) + raw_messages = list(raw_messages) + messages = list(messages) + if priority not in PRIORITIES: + LOG.warning('Unknown priority "%s"', priority) + continue + for screen, callback in self._callbacks_by_priority.get(priority, + []): + if screen: + filtered_messages = [message for message in messages + if screen.match( + message["ctxt"], + message["publisher_id"], + message["event_type"], + message["metadata"], + message["payload"])] + else: + filtered_messages = messages + + if not filtered_messages: + continue + + ret = self._exec_callback(executor_callback, callback, + filtered_messages) + if self.allow_requeue and ret == NotificationResult.REQUEUE: + requeues.update(raw_messages) + break + return requeues + + def _exec_callback(self, executor_callback, callback, *args): + if executor_callback: + ret = executor_callback(callback, *args) + else: + ret = callback(*args) + return NotificationResult.HANDLED if ret is None else ret + + def _extract_user_message(self, incoming): + ctxt = self.serializer.deserialize_context(incoming.ctxt) + message = incoming.message publisher_id = message.get('publisher_id') event_type = message.get('event_type') @@ -114,28 +144,50 @@ class NotificationDispatcher(object): 'timestamp': message.get('timestamp') } priority = message.get('priority', '').lower() - if priority not in PRIORITIES: - LOG.warning('Unknown priority "%s"', priority) - return - payload = self.serializer.deserialize_entity(ctxt, message.get('payload')) + return priority, incoming, dict(ctxt=ctxt, + publisher_id=publisher_id, + event_type=event_type, + payload=payload, + metadata=metadata) - for screen, callback in self._callbacks_by_priority.get(priority, []): - if screen and not screen.match(ctxt, publisher_id, event_type, - metadata, payload): - continue - localcontext._set_local_context(ctxt) - try: - if executor_callback: - ret = executor_callback(callback, ctxt, publisher_id, - event_type, payload, metadata) - else: - ret = callback(ctxt, publisher_id, event_type, payload, - metadata) - ret = NotificationResult.HANDLED if ret is None else ret - if self.allow_requeue and ret == NotificationResult.REQUEUE: - return ret - finally: - localcontext._clear_local_context() - return NotificationResult.HANDLED + +class NotificationDispatcher(_NotificationDispatcherBase): + """A message dispatcher which understands Notification messages. + + A MessageHandlingServer is constructed by passing a callable dispatcher + which is invoked with context and message dictionaries each time a message + is received. + """ + def _exec_callback(self, executor_callback, callback, messages): + localcontext._set_local_context( + messages[0]["ctxt"]) + try: + return super(NotificationDispatcher, self)._exec_callback( + executor_callback, callback, + messages[0]["ctxt"], + messages[0]["publisher_id"], + messages[0]["event_type"], + messages[0]["payload"], + messages[0]["metadata"]) + finally: + localcontext._clear_local_context() + + +class BatchNotificationDispatcher(_NotificationDispatcherBase): + """A message dispatcher which understands Notification messages. + + A MessageHandlingServer is constructed by passing a callable dispatcher + which is invoked with a list of message dictionaries each time 'batch_size' + messages are received or 'batch_timeout' seconds is reached. + """ + + def __init__(self, targets, endpoints, serializer, allow_requeue, + pool=None, batch_size=None, batch_timeout=None): + super(BatchNotificationDispatcher, self).__init__(targets, endpoints, + serializer, + allow_requeue, + pool) + self.batch_size = batch_size + self.batch_timeout = batch_timeout diff --git a/oslo_messaging/notify/listener.py b/oslo_messaging/notify/listener.py index 3460ef962..8855d5875 100644 --- a/oslo_messaging/notify/listener.py +++ b/oslo_messaging/notify/listener.py @@ -19,12 +19,13 @@ contain a set of methods. Each method corresponds to a notification priority. To create a notification listener, you supply a transport, list of targets and a list of endpoints. -A transport can be obtained simply by calling the get_transport() method:: +A transport can be obtained simply by calling the get_notification_transport() +method:: - transport = messaging.get_transport(conf) + transport = messaging.get_notification_transport(conf) which will load the appropriate transport driver according to the user's -messaging configuration. See get_transport() for more details. +messaging configuration. See get_notification_transport() for more details. The target supplied when creating a notification listener expresses the topic and - optionally - the exchange to listen on. See Target for more details @@ -56,7 +57,7 @@ A simple example of a notification listener with multiple endpoints might be:: def error(self, ctxt, publisher_id, event_type, payload, metadata): do_something(payload) - transport = oslo_messaging.get_transport(cfg.CONF) + transport = oslo_messaging.get_notification_transport(cfg.CONF) targets = [ oslo_messaging.Target(topic='notifications') oslo_messaging.Target(topic='notifications_bis') @@ -136,8 +137,49 @@ def get_notification_listener(transport, targets, endpoints, :type pool: str :raises: NotImplementedError """ - transport._require_driver_features(requeue=allow_requeue) dispatcher = notify_dispatcher.NotificationDispatcher(targets, endpoints, serializer, allow_requeue, pool) return msg_server.MessageHandlingServer(transport, dispatcher, executor) + + +def get_batch_notification_listener(transport, targets, endpoints, + executor='blocking', serializer=None, + allow_requeue=False, pool=None, + batch_size=None, batch_timeout=None): + """Construct a batch notification listener + + The executor parameter controls how incoming messages will be received and + dispatched. By default, the most simple executor is used - the blocking + executor. + + If the eventlet executor is used, the threading and time library need to be + monkeypatched. + + :param transport: the messaging transport + :type transport: Transport + :param targets: the exchanges and topics to listen on + :type targets: list of Target + :param endpoints: a list of endpoint objects + :type endpoints: list + :param executor: name of a message executor - for example + 'eventlet', 'blocking' + :type executor: str + :param serializer: an optional entity serializer + :type serializer: Serializer + :param allow_requeue: whether NotificationResult.REQUEUE support is needed + :type allow_requeue: bool + :param pool: the pool name + :type pool: str + :param batch_size: number of messages to wait before calling + endpoints callacks + :type batch_size: int + :param batch_timeout: number of seconds to wait before calling + endpoints callacks + :type batch_timeout: int + :raises: NotImplementedError + """ + dispatcher = notify_dispatcher.BatchNotificationDispatcher( + targets, endpoints, serializer, allow_requeue, pool, + batch_size, batch_timeout) + return msg_server.MessageHandlingServer(transport, dispatcher, executor) diff --git a/oslo_messaging/notify/log_handler.py b/oslo_messaging/notify/log_handler.py index 2137a8c17..8dc8454b5 100644 --- a/oslo_messaging/notify/log_handler.py +++ b/oslo_messaging/notify/log_handler.py @@ -21,14 +21,15 @@ class LoggingErrorNotificationHandler(logging.Handler): # at runtime. import oslo_messaging logging.Handler.__init__(self, *args, **kwargs) - self._transport = oslo_messaging.get_transport(cfg.CONF) + self._transport = oslo_messaging.get_notification_transport(cfg.CONF) self._notifier = oslo_messaging.Notifier( self._transport, publisher_id='error.publisher') def emit(self, record): + conf = self._transport.conf # NOTE(bnemec): Notifier registers this opt with the transport. - if ('log' in self._transport.conf.notification_driver): + if ('log' in conf.oslo_messaging_notifications.driver): # NOTE(lbragstad): If we detect that log is one of the # notification drivers, then return. This protects from infinite # recursion where something bad happens, it gets logged, the log diff --git a/oslo_messaging/notify/logger.py b/oslo_messaging/notify/logger.py index 3748533b8..b4e48df0f 100644 --- a/oslo_messaging/notify/logger.py +++ b/oslo_messaging/notify/logger.py @@ -19,7 +19,6 @@ import logging from oslo_config import cfg from oslo_messaging.notify import notifier -from oslo_messaging import transport class LoggingNotificationHandler(logging.Handler): @@ -34,7 +33,7 @@ class LoggingNotificationHandler(logging.Handler): [handler_notifier] class=oslo_messaging.LoggingNotificationHandler level=ERROR - args=('qpid:///') + args=('rabbit:///') """ @@ -47,7 +46,7 @@ class LoggingNotificationHandler(logging.Handler): def __init__(self, url, publisher_id=None, driver=None, topic=None, serializer=None): self.notifier = notifier.Notifier( - transport.get_transport(self.CONF, url), + notifier.get_notification_transport(self.CONF, url), publisher_id, driver, topic, serializer() if serializer else None) diff --git a/oslo_messaging/notify/middleware.py b/oslo_messaging/notify/middleware.py index 9c6c34294..60aab278b 100644 --- a/oslo_messaging/notify/middleware.py +++ b/oslo_messaging/notify/middleware.py @@ -22,7 +22,6 @@ import sys import traceback as tb from oslo_config import cfg -from oslo_context import context from oslo_middleware import base import six import webob.dec @@ -59,7 +58,8 @@ class RequestNotifier(base.Middleware): def __init__(self, app, **conf): self.notifier = notify.Notifier( - oslo_messaging.get_transport(cfg.CONF, conf.get('url')), + oslo_messaging.get_notification_transport(cfg.CONF, + conf.get('url')), publisher_id=conf.get('publisher_id', os.path.basename(sys.argv[0]))) self.service_name = conf.get('service_name') @@ -84,7 +84,7 @@ class RequestNotifier(base.Middleware): 'request': self.environ_to_dict(request.environ), } - self.notifier.info(context.get_admin_context(), + self.notifier.info({}, 'http.request', payload) @@ -107,7 +107,7 @@ class RequestNotifier(base.Middleware): 'traceback': tb.format_tb(traceback) } - self.notifier.info(context.get_admin_context(), + self.notifier.info({}, 'http.response', payload) diff --git a/oslo_messaging/notify/notifier.py b/oslo_messaging/notify/notifier.py index cc8a33868..cc4f2eb8f 100644 --- a/oslo_messaging/notify/notifier.py +++ b/oslo_messaging/notify/notifier.py @@ -25,17 +25,30 @@ import six from stevedore import named from oslo_messaging import serializer as msg_serializer +from oslo_messaging import transport as msg_transport _notifier_opts = [ - cfg.MultiStrOpt('notification_driver', + cfg.MultiStrOpt('driver', default=[], + deprecated_name='notification_driver', + deprecated_group='DEFAULT', help='The Drivers(s) to handle sending notifications. ' 'Possible values are messaging, messagingv2, ' 'routing, log, test, noop'), - cfg.ListOpt('notification_topics', + cfg.StrOpt('transport_url', + deprecated_name='notification_transport_url', + deprecated_group='DEFAULT', + help='A URL representing the messaging driver to use for ' + 'notifications. If not set, we fall back to the same ' + 'configuration used for RPC.'), + cfg.ListOpt('topics', default=['notifications', ], - deprecated_name='topics', - deprecated_group='rpc_notifier2', + deprecated_opts=[ + cfg.DeprecatedOpt('topics', + group='rpc_notifier2'), + cfg.DeprecatedOpt('notification_topics', + group='DEFAULT') + ], help='AMQP topic used for OpenStack notifications.'), ] @@ -75,6 +88,16 @@ class Driver(object): pass +def get_notification_transport(conf, url=None, + allowed_remote_exmods=None, aliases=None): + conf.register_opts(_notifier_opts, + group='oslo_messaging_notifications') + if url is None: + url = conf.oslo_messaging_notifications.transport_url + return msg_transport.get_transport(conf, url, + allowed_remote_exmods, aliases) + + class Notifier(object): """Send notification messages. @@ -94,16 +117,18 @@ class Notifier(object): A Notifier object can be instantiated with a transport object and a publisher ID: - notifier = messaging.Notifier(get_transport(CONF), 'compute') + notifier = messaging.Notifier(get_notification_transport(CONF), + 'compute') - and notifications are sent via drivers chosen with the notification_driver - config option and on the topics chosen with the notification_topics config - option. + and notifications are sent via drivers chosen with the driver + config option and on the topics chosen with the topics config + option in [oslo_messaging_notifications] section. Alternatively, a Notifier object can be instantiated with a specific driver or topic:: - notifier = notifier.Notifier(RPC_TRANSPORT, + transport = notifier.get_notification_transport(CONF) + notifier = notifier.Notifier(transport, 'compute.host', driver='messaging', topic='notifications') @@ -138,24 +163,26 @@ class Notifier(object): N means N retries :type retry: int """ - transport.conf.register_opts(_notifier_opts) + conf = transport.conf + conf.register_opts(_notifier_opts, + group='oslo_messaging_notifications') self.transport = transport self.publisher_id = publisher_id self.retry = retry - self._driver_names = ([driver] if driver is not None - else transport.conf.notification_driver) + self._driver_names = ([driver] if driver is not None else + conf.oslo_messaging_notifications.driver) - self._topics = ([topic] if topic is not None - else transport.conf.notification_topics) + self._topics = ([topic] if topic is not None else + conf.oslo_messaging_notifications.topics) self._serializer = serializer or msg_serializer.NoOpSerializer() self._driver_mgr = named.NamedExtensionManager( 'oslo.messaging.notify.drivers', names=self._driver_names, invoke_on_load=True, - invoke_args=[transport.conf], + invoke_args=[conf], invoke_kwds={ 'topics': self._topics, 'transport': self.transport, diff --git a/oslo_messaging/opts.py b/oslo_messaging/opts.py index c5856595d..263c59f68 100644 --- a/oslo_messaging/opts.py +++ b/oslo_messaging/opts.py @@ -22,7 +22,6 @@ import itertools from oslo_messaging._drivers import amqp from oslo_messaging._drivers import base as drivers_base -from oslo_messaging._drivers import impl_qpid from oslo_messaging._drivers import impl_rabbit from oslo_messaging._drivers import impl_zmq from oslo_messaging._drivers.protocols.amqp import opts as amqp_opts @@ -48,8 +47,6 @@ _opts = [ ('oslo_messaging_amqp', amqp_opts.amqp1_opts), ('oslo_messaging_rabbit', list(itertools.chain(amqp.amqp_opts, impl_rabbit.rabbit_opts))), - ('oslo_messaging_qpid', list(itertools.chain(amqp.amqp_opts, - impl_qpid.qpid_opts))) ] diff --git a/oslo_messaging/rpc/client.py b/oslo_messaging/rpc/client.py index 04992288f..7beea841e 100644 --- a/oslo_messaging/rpc/client.py +++ b/oslo_messaging/rpc/client.py @@ -229,7 +229,7 @@ class RPCClient(object): class TestClient(object): def __init__(self, transport): - target = messaging.Target(topic='testtopic', version='2.0') + target = messaging.Target(topic='test', version='2.0') self._client = messaging.RPCClient(transport, target) def test(self, ctxt, arg): @@ -254,7 +254,7 @@ class RPCClient(object): For example:: transport = messaging.get_transport(cfg.CONF) - target = messaging.Target(topic='testtopic', version='2.0') + target = messaging.Target(topic='test', version='2.0') client = messaging.RPCClient(transport, target) client.call(ctxt, 'test', arg=arg) @@ -356,6 +356,10 @@ class RPCClient(object): Similarly, the request context must be a dict unless the client's serializer supports serializing another type. + Note: cast doesn't ensure the remote method to be been executed + on each destination. But ensures that it will be not executed twice + on a destination. + :param ctxt: a request context dict :type ctxt: dict :param method: the method name @@ -392,6 +396,12 @@ class RPCClient(object): allowed_remote_exmods list, then a messaging.RemoteError exception is raised with all details of the remote exception. + Note: call is done 'at-most-once'. In case of we can't known + if the call have been done correctly, because we didn't get the + response on time, MessagingTimeout exception is raised. + The real reason can vary, transport failure, worker + doesn't answer in time or crash, ... + :param ctxt: a request context dict :type ctxt: dict :param method: the method name diff --git a/oslo_messaging/rpc/dispatcher.py b/oslo_messaging/rpc/dispatcher.py index 6913e7afe..5ff0610ad 100644 --- a/oslo_messaging/rpc/dispatcher.py +++ b/oslo_messaging/rpc/dispatcher.py @@ -31,6 +31,7 @@ import six from oslo_messaging._i18n import _LE from oslo_messaging import _utils as utils +from oslo_messaging import dispatcher from oslo_messaging import localcontext from oslo_messaging import serializer as msg_serializer from oslo_messaging import server as msg_server @@ -75,7 +76,7 @@ class UnsupportedVersion(RPCDispatcherError): self.method = method -class RPCDispatcher(object): +class RPCDispatcher(dispatcher.DispatcherBase): """A message dispatcher which understands RPC messages. A MessageHandlingServer is constructed by passing a callable dispatcher @@ -130,9 +131,9 @@ class RPCDispatcher(object): return self.serializer.serialize_entity(ctxt, result) def __call__(self, incoming, executor_callback=None): - incoming.acknowledge() - return utils.DispatcherExecutorContext( - incoming, self._dispatch_and_reply, + incoming[0].acknowledge() + return dispatcher.DispatcherExecutorContext( + incoming[0], self._dispatch_and_reply, executor_callback=executor_callback) def _dispatch_and_reply(self, incoming, executor_callback): @@ -145,7 +146,9 @@ class RPCDispatcher(object): e.exc_info[1]) incoming.reply(failure=e.exc_info, log_failure=False) except Exception as e: - # sys.exc_info() is deleted by LOG.exception(). + # current sys.exc_info() content can be overriden + # by another exception raise by a log handler during + # LOG.exception(). So keep a copy and delete it later. exc_info = sys.exc_info() LOG.error(_LE('Exception during message handling: %s'), e, exc_info=exc_info) diff --git a/oslo_messaging/rpc/server.py b/oslo_messaging/rpc/server.py index 855e3d9a6..74dbede44 100644 --- a/oslo_messaging/rpc/server.py +++ b/oslo_messaging/rpc/server.py @@ -44,6 +44,7 @@ A simple example of an RPC server with multiple endpoints might be:: from oslo_config import cfg import oslo_messaging + import time class ServerControlEndpoint(object): @@ -54,7 +55,7 @@ A simple example of an RPC server with multiple endpoints might be:: self.server = server def stop(self, ctx): - if server: + if self.server: self.server.stop() class TestEndpoint(object): @@ -70,7 +71,14 @@ A simple example of an RPC server with multiple endpoints might be:: ] server = oslo_messaging.get_rpc_server(transport, target, endpoints, executor='blocking') - server.start() + try: + server.start() + while True: + time.sleep(1) + except KeyboardInterrupt: + print("Stopping server") + + server.stop() server.wait() Clients can invoke methods on the server by sending the request to a topic and diff --git a/oslo_messaging/serializer.py b/oslo_messaging/serializer.py index b1761fd83..8b7c0a7a3 100644 --- a/oslo_messaging/serializer.py +++ b/oslo_messaging/serializer.py @@ -19,6 +19,7 @@ __all__ = ['Serializer', 'NoOpSerializer', 'JsonPayloadSerializer', import abc +from debtcollector import removals from oslo_context import context as common_context from oslo_serialization import jsonutils import six @@ -63,6 +64,7 @@ class Serializer(object): """ +@removals.remove(version="2.9", removal_version="3.0") class RequestContextSerializer(Serializer): def __init__(self, base): diff --git a/oslo_messaging/server.py b/oslo_messaging/server.py index 476e899cd..6b4e50a0c 100644 --- a/oslo_messaging/server.py +++ b/oslo_messaging/server.py @@ -23,20 +23,25 @@ __all__ = [ 'ServerListenError', ] +import functools +import inspect import logging import threading +import traceback from oslo_service import service from oslo_utils import timeutils from stevedore import driver from oslo_messaging._drivers import base as driver_base -from oslo_messaging._i18n import _LW -from oslo_messaging import _utils from oslo_messaging import exceptions LOG = logging.getLogger(__name__) +# The default number of seconds of waiting after which we will emit a log +# message +DEFAULT_LOG_AFTER = 30 + class MessagingServerError(exceptions.MessagingException): """Base class for all MessageHandlingServer exceptions.""" @@ -62,7 +67,223 @@ class ServerListenError(MessagingServerError): self.ex = ex -class MessageHandlingServer(service.ServiceBase): +class TaskTimeout(MessagingServerError): + """Raised if we timed out waiting for a task to complete.""" + + +class _OrderedTask(object): + """A task which must be executed in a particular order. + + A caller may wait for this task to complete by calling + `wait_for_completion`. + + A caller may run this task with `run_once`, which will ensure that however + many times the task is called it only runs once. Simultaneous callers will + block until the running task completes, which means that any caller can be + sure that the task has completed after run_once returns. + """ + + INIT = 0 # The task has not yet started + RUNNING = 1 # The task is running somewhere + COMPLETE = 2 # The task has run somewhere + + def __init__(self, name): + """Create a new _OrderedTask. + + :param name: The name of this task. Used in log messages. + """ + super(_OrderedTask, self).__init__() + + self._name = name + self._cond = threading.Condition() + self._state = self.INIT + + def _wait(self, condition, msg, log_after, timeout_timer): + """Wait while condition() is true. Write a log message if condition() + has not become false within `log_after` seconds. Raise TaskTimeout if + timeout_timer expires while waiting. + """ + + log_timer = None + if log_after != 0: + log_timer = timeutils.StopWatch(duration=log_after) + log_timer.start() + + while condition(): + if log_timer is not None and log_timer.expired(): + LOG.warn('Possible hang: %s' % msg) + LOG.debug(''.join(traceback.format_stack())) + # Only log once. After than we wait indefinitely without + # logging. + log_timer = None + + if timeout_timer is not None and timeout_timer.expired(): + raise TaskTimeout(msg) + + timeouts = [] + if log_timer is not None: + timeouts.append(log_timer.leftover()) + if timeout_timer is not None: + timeouts.append(timeout_timer.leftover()) + + wait = None + if timeouts: + wait = min(timeouts) + self._cond.wait(wait) + + @property + def complete(self): + return self._state == self.COMPLETE + + def wait_for_completion(self, caller, log_after, timeout_timer): + """Wait until this task has completed. + + :param caller: The name of the task which is waiting. + :param log_after: Emit a log message if waiting longer than `log_after` + seconds. + :param timeout_timer: Raise TaskTimeout if StopWatch object + `timeout_timer` expires while waiting. + """ + with self._cond: + msg = '%s is waiting for %s to complete' % (caller, self._name) + self._wait(lambda: not self.complete, + msg, log_after, timeout_timer) + + def run_once(self, fn, log_after, timeout_timer): + """Run a task exactly once. If it is currently running in another + thread, wait for it to complete. If it has already run, return + immediately without running it again. + + :param fn: The task to run. It must be a callable taking no arguments. + It may optionally return another callable, which also takes + no arguments, which will be executed after completion has + been signaled to other threads. + :param log_after: Emit a log message if waiting longer than `log_after` + seconds. + :param timeout_timer: Raise TaskTimeout if StopWatch object + `timeout_timer` expires while waiting. + """ + with self._cond: + if self._state == self.INIT: + self._state = self.RUNNING + # Note that nothing waits on RUNNING, so no need to notify + + # We need to release the condition lock before calling out to + # prevent deadlocks. Reacquire it immediately afterwards. + self._cond.release() + try: + post_fn = fn() + finally: + self._cond.acquire() + self._state = self.COMPLETE + self._cond.notify_all() + + if post_fn is not None: + # Release the condition lock before calling out to prevent + # deadlocks. Reacquire it immediately afterwards. + self._cond.release() + try: + post_fn() + finally: + self._cond.acquire() + elif self._state == self.RUNNING: + msg = ('%s is waiting for another thread to complete' + % self._name) + self._wait(lambda: self._state == self.RUNNING, + msg, log_after, timeout_timer) + + +class _OrderedTaskRunner(object): + """Mixin for a class which executes ordered tasks.""" + + def __init__(self, *args, **kwargs): + super(_OrderedTaskRunner, self).__init__(*args, **kwargs) + + # Get a list of methods on this object which have the _ordered + # attribute + self._tasks = [name + for (name, member) in inspect.getmembers(self) + if inspect.ismethod(member) and + getattr(member, '_ordered', False)] + self.reset_states() + + self._reset_lock = threading.Lock() + + def reset_states(self): + # Create new task states for tasks in reset + self._states = {task: _OrderedTask(task) for task in self._tasks} + + @staticmethod + def decorate_ordered(fn, state, after, reset_after): + + @functools.wraps(fn) + def wrapper(self, *args, **kwargs): + # If the reset_after state has already completed, reset state so + # we can run again. + # NOTE(mdbooth): This is ugly and requires external locking to be + # deterministic when using multiple threads. Consider a thread that + # does: server.stop(), server.wait(). If another thread causes a + # reset between stop() and wait(), this will not have the intended + # behaviour. It is safe without external locking, if the caller + # instantiates a new object. + with self._reset_lock: + if (reset_after is not None and + self._states[reset_after].complete): + self.reset_states() + + # Store the states we started with in case the state wraps on us + # while we're sleeping. We must wait and run_once in the same + # epoch. If the epoch ended while we were sleeping, run_once will + # safely do nothing. + states = self._states + + log_after = kwargs.pop('log_after', DEFAULT_LOG_AFTER) + timeout = kwargs.pop('timeout', None) + + timeout_timer = None + if timeout is not None: + timeout_timer = timeutils.StopWatch(duration=timeout) + timeout_timer.start() + + # Wait for the given preceding state to complete + if after is not None: + states[after].wait_for_completion(state, + log_after, timeout_timer) + + # Run this state + states[state].run_once(lambda: fn(self, *args, **kwargs), + log_after, timeout_timer) + return wrapper + + +def ordered(after=None, reset_after=None): + """A method which will be executed as an ordered task. The method will be + called exactly once, however many times it is called. If it is called + multiple times simultaneously it will only be called once, but all callers + will wait until execution is complete. + + If `after` is given, this method will not run until `after` has completed. + + If `reset_after` is given and the target method has completed, allow this + task to run again by resetting all task states. + + :param after: Optionally, the name of another `ordered` method. Wait for + the completion of `after` before executing this method. + :param reset_after: Optionally, the name of another `ordered` method. Reset + all states when calling this method if `reset_after` + has completed. + """ + def _ordered(fn): + # Set an attribute on the method so we can find it later + setattr(fn, '_ordered', True) + state = fn.__name__ + + return _OrderedTaskRunner.decorate_ordered(fn, state, after, + reset_after) + return _ordered + + +class MessageHandlingServer(service.ServiceBase, _OrderedTaskRunner): """Server for handling messages. Connect a transport to a dispatcher that knows how to process the @@ -94,29 +315,20 @@ class MessageHandlingServer(service.ServiceBase): self.dispatcher = dispatcher self.executor = executor - # NOTE(sileht): we use a lock to protect the state change of the - # server, we don't want to call stop until the transport driver - # is fully started. Except for the blocking executor that have - # start() that doesn't return - if self.executor != "blocking": - self._state_cond = threading.Condition() - self._dummy_cond = False - else: - self._state_cond = _utils.DummyCondition() - self._dummy_cond = True - try: mgr = driver.DriverManager('oslo.messaging.executors', self.executor) except RuntimeError as ex: raise ExecutorLoadFailure(self.executor, ex) - else: - self._executor_cls = mgr.driver - self._executor = None - self._running = False + + self._executor_cls = mgr.driver + self._executor_obj = None + + self._started = False super(MessageHandlingServer, self).__init__() + @ordered(reset_after='stop') def start(self): """Start handling incoming messages. @@ -131,21 +343,30 @@ class MessageHandlingServer(service.ServiceBase): choose to dispatch messages in a new thread, coroutine or simply the current thread. """ - if self._executor is not None: - return - with self._state_cond: - if self._executor is not None: - return - try: - listener = self.dispatcher._listen(self.transport) - except driver_base.TransportDriverError as ex: - raise ServerListenError(self.target, ex) - self._running = True - self._executor = self._executor_cls(self.conf, listener, - self.dispatcher) - self._executor.start() - self._state_cond.notify_all() + # Warn that restarting will be deprecated + if self._started: + LOG.warn('Restarting a MessageHandlingServer is inherently racy. ' + 'It is deprecated, and will become a noop in a future ' + 'release of oslo.messaging. If you need to restart ' + 'MessageHandlingServer you should instantiate a new ' + 'object.') + self._started = True + try: + listener = self.dispatcher._listen(self.transport) + except driver_base.TransportDriverError as ex: + raise ServerListenError(self.target, ex) + executor = self._executor_cls(self.conf, listener, self.dispatcher) + executor.start() + self._executor_obj = executor + + if self.executor == 'blocking': + # N.B. This will be executed unlocked and unordered, so + # we can't rely on the value of self._executor_obj when this runs. + # We explicitly pass the local variable. + return lambda: executor.execute() + + @ordered(after='start') def stop(self): """Stop handling incoming messages. @@ -154,12 +375,9 @@ class MessageHandlingServer(service.ServiceBase): some messages, and underlying driver resources associated to this server are still in use. See 'wait' for more details. """ - with self._state_cond: - if self._executor is not None: - self._running = False - self._executor.stop() - self._state_cond.notify_all() + self._executor_obj.stop() + @ordered(after='stop') def wait(self): """Wait for message processing to complete. @@ -170,37 +388,12 @@ class MessageHandlingServer(service.ServiceBase): Once it's finished, the underlying driver resources associated to this server are released (like closing useless network connections). """ - with self._state_cond: - if self._running: - LOG.warn(_LW("wait() should be called after stop() as it " - "waits for existing messages to finish " - "processing")) - w = timeutils.StopWatch() - w.start() - while self._running: - # NOTE(harlowja): 1.0 seconds was mostly chosen at - # random, but it seems like a reasonable value to - # use to avoid spamming the logs with to much - # information. - self._state_cond.wait(1.0) - if self._running and not self._dummy_cond: - LOG.warn( - _LW("wait() should have been called" - " after stop() as wait() waits for existing" - " messages to finish processing, it has" - " been %0.2f seconds and stop() still has" - " not been called"), w.elapsed()) - executor = self._executor - self._executor = None - if executor is not None: - # We are the lucky calling thread to wait on the executor to - # actually finish. - try: - executor.wait() - finally: - # Close listener connection after processing all messages - executor.listener.cleanup() - executor = None + try: + self._executor_obj.wait() + finally: + # Close listener connection after processing all messages + self._executor_obj.listener.cleanup() + self._executor_obj = None def reset(self): """Reset service. diff --git a/oslo_messaging/tests/drivers/test_impl_kafka.py b/oslo_messaging/tests/drivers/test_impl_kafka.py new file mode 100644 index 000000000..dcbab0a88 --- /dev/null +++ b/oslo_messaging/tests/drivers/test_impl_kafka.py @@ -0,0 +1,288 @@ +# Copyright (C) 2015 Cisco Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json +import kafka +from kafka.common import KafkaError +import mock +import testscenarios +from testtools.testcase import unittest +import time + +import oslo_messaging +from oslo_messaging._drivers import common as driver_common +from oslo_messaging._drivers import impl_kafka as kafka_driver +from oslo_messaging.tests import utils as test_utils + +load_tests = testscenarios.load_tests_apply_scenarios + +KAFKA_BROKER = 'localhost:9092' +KAFKA_BROKER_URL = 'kafka://localhost:9092' + + +def _is_kafka_service_running(): + """Checks whether the Kafka service is running or not""" + kafka_running = True + try: + broker = KAFKA_BROKER + kafka.KafkaClient(broker) + except KafkaError: + # Kafka service is not running. + kafka_running = False + return kafka_running + + +class TestKafkaDriverLoad(test_utils.BaseTestCase): + + def setUp(self): + super(TestKafkaDriverLoad, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + + def test_driver_load(self): + transport = oslo_messaging.get_transport(self.conf) + self.assertIsInstance(transport._driver, kafka_driver.KafkaDriver) + + +class TestKafkaTransportURL(test_utils.BaseTestCase): + + scenarios = [ + ('none', dict(url=None, + expected=[dict(host='localhost', port=9092)])), + ('empty', dict(url='kafka:///', + expected=[dict(host='localhost', port=9092)])), + ('host', dict(url='kafka://127.0.0.1', + expected=[dict(host='127.0.0.1', port=9092)])), + ('port', dict(url='kafka://localhost:1234', + expected=[dict(host='localhost', port=1234)])), + ] + + def setUp(self): + super(TestKafkaTransportURL, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + + def test_transport_url(self): + transport = oslo_messaging.get_transport(self.conf, self.url) + self.addCleanup(transport.cleanup) + driver = transport._driver + + conn = driver._get_connection(kafka_driver.PURPOSE_SEND) + self.assertEqual(self.expected[0]['host'], conn.host) + self.assertEqual(self.expected[0]['port'], conn.port) + + +class TestKafkaDriver(test_utils.BaseTestCase): + """Unit Test cases to test the kafka driver + """ + + def setUp(self): + super(TestKafkaDriver, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + def test_send(self): + target = oslo_messaging.Target(topic="topic_test") + self.assertRaises(NotImplementedError, + self.driver.send, target, {}, {}) + + def test_send_notification(self): + target = oslo_messaging.Target(topic="topic_test") + + with mock.patch.object( + kafka_driver.Connection, 'notify_send') as fake_send: + self.driver.send_notification(target, {}, {}, None) + self.assertEquals(1, len(fake_send.mock_calls)) + + def test_listen(self): + target = oslo_messaging.Target(topic="topic_test") + self.assertRaises(NotImplementedError, self.driver.listen, target) + + +class TestKafkaConnection(test_utils.BaseTestCase): + + def setUp(self): + super(TestKafkaConnection, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_send') + def test_notify(self, fake_send, fake_ensure_connection): + conn = self.driver._get_connection(kafka_driver.PURPOSE_SEND) + conn.notify_send("fake_topic", {"fake_ctxt": "fake_param"}, + {"fake_text": "fake_message_1"}, 10) + self.assertEqual(1, len(fake_send.mock_calls)) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_send') + def test_notify_with_retry(self, fake_send, fake_ensure_connection): + conn = self.driver._get_connection(kafka_driver.PURPOSE_SEND) + fake_send.side_effect = KafkaError("fake_exception") + conn.notify_send("fake_topic", {"fake_ctxt": "fake_param"}, + {"fake_text": "fake_message_2"}, 10) + self.assertEqual(10, len(fake_send.mock_calls)) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_parse_url') + def test_consume(self, fake_parse_url, fake_ensure_connection): + fake_message = { + "context": {"fake": "fake_context_1"}, + "message": {"fake": "fake_message_1"}} + + conn = kafka_driver.Connection( + self.conf, '', kafka_driver.PURPOSE_LISTEN) + + conn.consumer = mock.MagicMock() + conn.consumer.fetch_messages = mock.MagicMock( + return_value=iter([json.dumps(fake_message)])) + + self.assertEqual(fake_message, json.loads(conn.consume()[0])) + self.assertEqual(1, len(conn.consumer.fetch_messages.mock_calls)) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_parse_url') + def test_consume_timeout(self, fake_parse_url, fake_ensure_connection): + deadline = time.time() + 3 + conn = kafka_driver.Connection( + self.conf, '', kafka_driver.PURPOSE_LISTEN) + + conn.consumer = mock.MagicMock() + conn.consumer.fetch_messages = mock.MagicMock(return_value=iter([])) + + self.assertRaises(driver_common.Timeout, conn.consume, timeout=3) + self.assertEqual(0, int(deadline - time.time())) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_parse_url') + def test_consume_with_default_timeout( + self, fake_parse_url, fake_ensure_connection): + deadline = time.time() + 1 + conn = kafka_driver.Connection( + self.conf, '', kafka_driver.PURPOSE_LISTEN) + + conn.consumer = mock.MagicMock() + conn.consumer.fetch_messages = mock.MagicMock(return_value=iter([])) + + self.assertRaises(driver_common.Timeout, conn.consume) + self.assertEqual(0, int(deadline - time.time())) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, '_parse_url') + def test_consume_timeout_without_consumers( + self, fake_parse_url, fake_ensure_connection): + deadline = time.time() + 3 + conn = kafka_driver.Connection( + self.conf, '', kafka_driver.PURPOSE_LISTEN) + conn.consumer = mock.MagicMock(return_value=None) + + self.assertRaises(driver_common.Timeout, conn.consume, timeout=3) + self.assertEqual(0, int(deadline - time.time())) + + +class TestKafkaListener(test_utils.BaseTestCase): + + def setUp(self): + super(TestKafkaListener, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + transport = oslo_messaging.get_transport(self.conf) + self.driver = transport._driver + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, 'declare_topic_consumer') + def test_create_listener(self, fake_consumer, fake_ensure_connection): + fake_target = oslo_messaging.Target(topic='fake_topic') + fake_targets_and_priorities = [(fake_target, 'info')] + listener = self.driver.listen_for_notifications( + fake_targets_and_priorities) + self.assertEqual(1, len(fake_consumer.mock_calls)) + + @mock.patch.object(kafka_driver.Connection, '_ensure_connection') + @mock.patch.object(kafka_driver.Connection, 'declare_topic_consumer') + def test_stop_listener(self, fake_consumer, fake_client): + fake_target = oslo_messaging.Target(topic='fake_topic') + fake_targets_and_priorities = [(fake_target, 'info')] + listener = self.driver.listen_for_notifications( + fake_targets_and_priorities) + listener.conn.consume = mock.MagicMock() + listener.conn.consume.return_value = ( + iter([kafka.common.KafkaMessage( + topic='fake_topic', partition=0, offset=0, + key=None, value='{"message": {"fake": "fake_message_1"},' + '"context": {"fake": "fake_context_1"}}')])) + listener.poll() + self.assertEqual(1, len(listener.conn.consume.mock_calls)) + listener.conn.stop_consuming = mock.MagicMock() + listener.stop() + fake_response = listener.poll() + self.assertEqual(1, len(listener.conn.consume.mock_calls)) + self.assertEqual([], fake_response) + + +class TestWithRealKafkaBroker(test_utils.BaseTestCase): + + def setUp(self): + super(TestWithRealKafkaBroker, self).setUp() + self.messaging_conf.transport_driver = 'kafka' + transport = oslo_messaging.get_transport(self.conf, KAFKA_BROKER_URL) + self.driver = transport._driver + + @unittest.skipUnless( + _is_kafka_service_running(), "Kafka service is not available") + def test_send_and_recieve_message(self): + target = oslo_messaging.Target( + topic="fake_topic", exchange='fake_exchange') + targets_and_priorities = [(target, 'fake_info')] + + listener = self.driver.listen_for_notifications( + targets_and_priorities) + fake_context = {"fake_context_key": "fake_context_value"} + fake_message = {"fake_message_key": "fake_message_value"} + self.driver.send_notification( + target, fake_context, fake_message, None) + + received_message = listener.poll()[0] + self.assertEqual(fake_context, received_message.ctxt) + self.assertEqual(fake_message, received_message.message) + + @unittest.skipUnless( + _is_kafka_service_running(), "Kafka service is not available") + def test_send_and_recieve_message_without_exchange(self): + target = oslo_messaging.Target(topic="fake_no_exchange_topic") + targets_and_priorities = [(target, 'fake_info')] + + listener = self.driver.listen_for_notifications( + targets_and_priorities) + fake_context = {"fake_context_key": "fake_context_value"} + fake_message = {"fake_message_key": "fake_message_value"} + self.driver.send_notification( + target, fake_context, fake_message, None) + + received_message = listener.poll()[0] + self.assertEqual(fake_context, received_message.ctxt) + self.assertEqual(fake_message, received_message.message) + + @unittest.skipUnless( + _is_kafka_service_running(), "Kafka service is not available") + def test_recieve_message_from_empty_topic_with_timeout(self): + target = oslo_messaging.Target( + topic="fake_empty_topic", exchange='fake_empty_exchange') + targets_and_priorities = [(target, 'fake_info')] + + listener = self.driver.listen_for_notifications( + targets_and_priorities) + + deadline = time.time() + 3 + received_message = listener.poll(timeout=3) + self.assertEqual(0, int(deadline - time.time())) + self.assertEqual(None, received_message) diff --git a/oslo_messaging/tests/drivers/test_impl_qpid.py b/oslo_messaging/tests/drivers/test_impl_qpid.py deleted file mode 100644 index 2eb0bb244..000000000 --- a/oslo_messaging/tests/drivers/test_impl_qpid.py +++ /dev/null @@ -1,850 +0,0 @@ -# Copyright (C) 2014 eNovance SAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import operator -import random -import threading -import time - -try: - import qpid -except ImportError: - qpid = None -from six.moves import _thread -import testscenarios -import testtools - -import oslo_messaging -from oslo_messaging._drivers import amqp -from oslo_messaging._drivers import impl_qpid as qpid_driver -from oslo_messaging.tests import utils as test_utils -from six.moves import mock - - -load_tests = testscenarios.load_tests_apply_scenarios - -QPID_BROKER = 'localhost:5672' - - -class TestQpidDriverLoad(test_utils.BaseTestCase): - - def setUp(self): - super(TestQpidDriverLoad, self).setUp() - self.messaging_conf.transport_driver = 'qpid' - - def test_driver_load(self): - transport = oslo_messaging.get_transport(self.conf) - self.assertIsInstance(transport._driver, qpid_driver.QpidDriver) - - -def _is_qpidd_service_running(): - - """this function checks if the qpid service is running or not.""" - - qpid_running = True - try: - broker = QPID_BROKER - connection = qpid.messaging.Connection(broker) - connection.open() - except Exception: - # qpid service is not running. - qpid_running = False - else: - connection.close() - - return qpid_running - - -class _QpidBaseTestCase(test_utils.BaseTestCase): - - @testtools.skipIf(qpid is None, "qpid not available") - def setUp(self): - super(_QpidBaseTestCase, self).setUp() - self.messaging_conf.transport_driver = 'qpid' - self.fake_qpid = not _is_qpidd_service_running() - - if self.fake_qpid: - self.session_receive = get_fake_qpid_session() - self.session_send = get_fake_qpid_session() - else: - self.broker = QPID_BROKER - # create connection from the qpid.messaging - # connection for the Consumer. - self.con_receive = qpid.messaging.Connection(self.broker) - self.con_receive.open() - # session to receive the messages - self.session_receive = self.con_receive.session() - - # connection for sending the message - self.con_send = qpid.messaging.Connection(self.broker) - self.con_send.open() - # session to send the messages - self.session_send = self.con_send.session() - - # list to store the expected messages and - # the actual received messages - self._expected = [] - self._messages = [] - self.initialized = True - - def tearDown(self): - super(_QpidBaseTestCase, self).tearDown() - - if self.initialized: - if self.fake_qpid: - _fake_session.flush_exchanges() - else: - self.con_receive.close() - self.con_send.close() - - -class TestQpidTransportURL(_QpidBaseTestCase): - - scenarios = [ - ('none', dict(url=None, - expected=[dict(host='localhost:5672', - username='', - password='')])), - ('empty', - dict(url='qpid:///', - expected=[dict(host='localhost:5672', - username='', - password='')])), - ('localhost', - dict(url='qpid://localhost/', - expected=[dict(host='localhost', - username='', - password='')])), - ('no_creds', - dict(url='qpid://host/', - expected=[dict(host='host', - username='', - password='')])), - ('no_port', - dict(url='qpid://user:password@host/', - expected=[dict(host='host', - username='user', - password='password')])), - ('full_url', - dict(url='qpid://user:password@host:10/', - expected=[dict(host='host:10', - username='user', - password='password')])), - ('full_two_url', - dict(url='qpid://user:password@host:10,' - 'user2:password2@host2:12/', - expected=[dict(host='host:10', - username='user', - password='password'), - dict(host='host2:12', - username='user2', - password='password2') - ] - )), - - ] - - @mock.patch.object(qpid_driver.Connection, 'reconnect') - def test_transport_url(self, *args): - transport = oslo_messaging.get_transport(self.conf, self.url) - self.addCleanup(transport.cleanup) - driver = transport._driver - - brokers_params = driver._get_connection().brokers_params - self.assertEqual(sorted(self.expected, - key=operator.itemgetter('host')), - sorted(brokers_params, - key=operator.itemgetter('host'))) - - -class TestQpidInvalidTopologyVersion(_QpidBaseTestCase): - """Unit test cases to test invalid qpid topology version.""" - - scenarios = [ - ('direct', dict(consumer_cls=qpid_driver.DirectConsumer, - consumer_kwargs={}, - publisher_cls=qpid_driver.DirectPublisher, - publisher_kwargs={})), - ('topic', dict(consumer_cls=qpid_driver.TopicConsumer, - consumer_kwargs={'exchange_name': 'openstack'}, - publisher_cls=qpid_driver.TopicPublisher, - publisher_kwargs={'exchange_name': 'openstack'})), - ('fanout', dict(consumer_cls=qpid_driver.FanoutConsumer, - consumer_kwargs={}, - publisher_cls=qpid_driver.FanoutPublisher, - publisher_kwargs={})), - ] - - def setUp(self): - super(TestQpidInvalidTopologyVersion, self).setUp() - self.config(qpid_topology_version=-1, - group='oslo_messaging_qpid') - - def test_invalid_topology_version(self): - def consumer_callback(msg): - pass - - msgid_or_topic = 'test' - - # not using self.assertRaises because - # 1. qpid driver raises Exception(msg) for invalid topology version - # 2. flake8 - H202 assertRaises Exception too broad - exception_msg = ("Invalid value for qpid_topology_version: %d" % - self.conf.oslo_messaging_qpid.qpid_topology_version) - recvd_exc_msg = '' - - try: - self.consumer_cls(self.conf.oslo_messaging_qpid, - self.session_receive, - msgid_or_topic, - consumer_callback, - **self.consumer_kwargs) - except Exception as e: - recvd_exc_msg = e.message - - self.assertEqual(exception_msg, recvd_exc_msg) - - recvd_exc_msg = '' - try: - self.publisher_cls(self.conf.oslo_messaging_qpid, - self.session_send, - topic=msgid_or_topic, - **self.publisher_kwargs) - except Exception as e: - recvd_exc_msg = e.message - - self.assertEqual(exception_msg, recvd_exc_msg) - - -class TestQpidDirectConsumerPublisher(_QpidBaseTestCase): - """Unit test cases to test DirectConsumer and Direct Publisher.""" - - _n_qpid_topology = [ - ('v1', dict(qpid_topology=1)), - ('v2', dict(qpid_topology=2)), - ] - - _n_msgs = [ - ('single', dict(no_msgs=1)), - ('multiple', dict(no_msgs=10)), - ] - - @classmethod - def generate_scenarios(cls): - cls.scenarios = testscenarios.multiply_scenarios(cls._n_qpid_topology, - cls._n_msgs) - - def consumer_callback(self, msg): - # This function will be called by the DirectConsumer - # when any message is received. - # Append the received message into the messages list - # so that the received messages can be validated - # with the expected messages - if isinstance(msg, dict): - self._messages.append(msg['content']) - else: - self._messages.append(msg) - - def test_qpid_direct_consumer_producer(self): - self.msgid = str(random.randint(1, 100)) - - # create a DirectConsumer and DirectPublisher class objects - self.dir_cons = qpid_driver.DirectConsumer( - self.conf.oslo_messaging_qpid, - self.session_receive, - self.msgid, - self.consumer_callback) - self.dir_pub = qpid_driver.DirectPublisher( - self.conf.oslo_messaging_qpid, - self.session_send, - self.msgid) - - def try_send_msg(no_msgs): - for i in range(no_msgs): - self._expected.append(str(i)) - snd_msg = {'content_type': 'text/plain', 'content': str(i)} - self.dir_pub.send(snd_msg) - - def try_receive_msg(no_msgs): - for i in range(no_msgs): - self.dir_cons.consume() - - thread1 = threading.Thread(target=try_receive_msg, - args=(self.no_msgs,)) - thread2 = threading.Thread(target=try_send_msg, - args=(self.no_msgs,)) - - thread1.start() - thread2.start() - thread1.join() - thread2.join() - - self.assertEqual(self.no_msgs, len(self._messages)) - self.assertEqual(self._expected, self._messages) - - -TestQpidDirectConsumerPublisher.generate_scenarios() - - -class TestQpidTopicAndFanout(_QpidBaseTestCase): - """Unit Test cases to test TopicConsumer and - TopicPublisher classes of the qpid driver - and FanoutConsumer and FanoutPublisher classes - of the qpid driver - """ - - _n_qpid_topology = [ - ('v1', dict(qpid_topology=1)), - ('v2', dict(qpid_topology=2)), - ] - - _n_msgs = [ - ('single', dict(no_msgs=1)), - ('multiple', dict(no_msgs=10)), - ] - - _n_senders = [ - ('single', dict(no_senders=1)), - ('multiple', dict(no_senders=10)), - ] - - _n_receivers = [ - ('single', dict(no_receivers=1)), - ] - _exchange_class = [ - ('topic', dict(consumer_cls=qpid_driver.TopicConsumer, - consumer_kwargs={'exchange_name': 'openstack'}, - publisher_cls=qpid_driver.TopicPublisher, - publisher_kwargs={'exchange_name': 'openstack'}, - topic='topictest.test', - receive_topic='topictest.test')), - ('fanout', dict(consumer_cls=qpid_driver.FanoutConsumer, - consumer_kwargs={}, - publisher_cls=qpid_driver.FanoutPublisher, - publisher_kwargs={}, - topic='fanouttest', - receive_topic='fanouttest')), - ] - - @classmethod - def generate_scenarios(cls): - cls.scenarios = testscenarios.multiply_scenarios(cls._n_qpid_topology, - cls._n_msgs, - cls._n_senders, - cls._n_receivers, - cls._exchange_class) - - def setUp(self): - super(TestQpidTopicAndFanout, self).setUp() - - # to store the expected messages and the - # actual received messages - # - # NOTE(dhellmann): These are dicts, where the base class uses - # lists. - self._expected = {} - self._messages = {} - - self._senders = [] - self._receivers = [] - - self._sender_threads = [] - self._receiver_threads = [] - - def consumer_callback(self, msg): - """callback function called by the ConsumerBase class of - qpid driver. - Message will be received in the format x-y - where x is the sender id and y is the msg number of the sender - extract the sender id 'x' and store the msg 'x-y' with 'x' as - the key - """ - - if isinstance(msg, dict): - msgcontent = msg['content'] - else: - msgcontent = msg - - splitmsg = msgcontent.split('-') - key = _thread.get_ident() - - if key not in self._messages: - self._messages[key] = dict() - - tdict = self._messages[key] - - if splitmsg[0] not in tdict: - tdict[splitmsg[0]] = [] - - tdict[splitmsg[0]].append(msgcontent) - - def _try_send_msg(self, sender_id, no_msgs): - for i in range(no_msgs): - sendmsg = '%s-%s' % (str(sender_id), str(i)) - key = str(sender_id) - # Store the message in the self._expected for each sender. - # This will be used later to - # validate the test by comparing it with the - # received messages by all the receivers - if key not in self._expected: - self._expected[key] = [] - self._expected[key].append(sendmsg) - send_dict = {'content_type': 'text/plain', 'content': sendmsg} - self._senders[sender_id].send(send_dict) - - def _try_receive_msg(self, receiver_id, no_msgs): - for i in range(self.no_senders * no_msgs): - no_of_attempts = 0 - - # ConsumerBase.consume blocks indefinitely until a message - # is received. - # So qpid_receiver.available() is called before calling - # ConsumerBase.consume() so that we are not - # blocked indefinitely - qpid_receiver = self._receivers[receiver_id].get_receiver() - while no_of_attempts < 50: - if qpid_receiver.available() > 0: - self._receivers[receiver_id].consume() - break - no_of_attempts += 1 - time.sleep(0.05) - - def test_qpid_topic_and_fanout(self): - for receiver_id in range(self.no_receivers): - consumer = self.consumer_cls(self.conf.oslo_messaging_qpid, - self.session_receive, - self.receive_topic, - self.consumer_callback, - **self.consumer_kwargs) - self._receivers.append(consumer) - - # create receivers threads - thread = threading.Thread(target=self._try_receive_msg, - args=(receiver_id, self.no_msgs,)) - self._receiver_threads.append(thread) - - for sender_id in range(self.no_senders): - publisher = self.publisher_cls(self.conf.oslo_messaging_qpid, - self.session_send, - topic=self.topic, - **self.publisher_kwargs) - self._senders.append(publisher) - - # create sender threads - thread = threading.Thread(target=self._try_send_msg, - args=(sender_id, self.no_msgs,)) - self._sender_threads.append(thread) - - for thread in self._receiver_threads: - thread.start() - - for thread in self._sender_threads: - thread.start() - - for thread in self._receiver_threads: - thread.join() - - for thread in self._sender_threads: - thread.join() - - # Each receiver should receive all the messages sent by - # the sender(s). - # So, Iterate through each of the receiver items in - # self._messages and compare with the expected messages - # messages. - - self.assertEqual(self.no_senders, len(self._expected)) - self.assertEqual(self.no_receivers, len(self._messages)) - - for key, messages in self._messages.iteritems(): - self.assertEqual(self._expected, messages) - -TestQpidTopicAndFanout.generate_scenarios() - - -class AddressNodeMatcher(object): - def __init__(self, node): - self.node = node - - def __eq__(self, address): - return address.split(';')[0].strip() == self.node - - -class TestDriverInterface(_QpidBaseTestCase): - """Unit Test cases to test the amqpdriver with qpid - """ - - def setUp(self): - super(TestDriverInterface, self).setUp() - self.config(qpid_topology_version=2, - group='oslo_messaging_qpid') - transport = oslo_messaging.get_transport(self.conf) - self.driver = transport._driver - - original_get_connection = self.driver._get_connection - p = mock.patch.object(self.driver, '_get_connection', - side_effect=lambda pooled=True: - original_get_connection(False)) - p.start() - self.addCleanup(p.stop) - - def test_listen_and_direct_send(self): - target = oslo_messaging.Target(exchange="exchange_test", - topic="topic_test", - server="server_test") - - with mock.patch('qpid.messaging.Connection') as conn_cls: - conn = conn_cls.return_value - session = conn.session.return_value - session.receiver.side_effect = [mock.Mock(), mock.Mock(), - mock.Mock()] - - listener = self.driver.listen(target) - listener.conn.direct_send("msg_id", {}) - - self.assertEqual(3, len(listener.conn.consumers)) - - expected_calls = [ - mock.call(AddressNodeMatcher( - 'amq.topic/topic/exchange_test/topic_test')), - mock.call(AddressNodeMatcher( - 'amq.topic/topic/exchange_test/topic_test.server_test')), - mock.call(AddressNodeMatcher('amq.topic/fanout/topic_test')), - ] - session.receiver.assert_has_calls(expected_calls) - session.sender.assert_called_with( - AddressNodeMatcher("amq.direct/msg_id")) - - def test_send(self): - target = oslo_messaging.Target(exchange="exchange_test", - topic="topic_test", - server="server_test") - with mock.patch('qpid.messaging.Connection') as conn_cls: - conn = conn_cls.return_value - session = conn.session.return_value - - self.driver.send(target, {}, {}) - session.sender.assert_called_with(AddressNodeMatcher( - "amq.topic/topic/exchange_test/topic_test.server_test")) - - def test_send_notification(self): - target = oslo_messaging.Target(exchange="exchange_test", - topic="topic_test.info") - with mock.patch('qpid.messaging.Connection') as conn_cls: - conn = conn_cls.return_value - session = conn.session.return_value - - self.driver.send_notification(target, {}, {}, "2.0") - session.sender.assert_called_with(AddressNodeMatcher( - "amq.topic/topic/exchange_test/topic_test.info")) - - -class TestQpidReconnectOrder(test_utils.BaseTestCase): - """Unit Test cases to test reconnection - """ - - @testtools.skipIf(qpid is None, "qpid not available") - def test_reconnect_order(self): - brokers = ['host1', 'host2', 'host3', 'host4', 'host5'] - brokers_count = len(brokers) - - self.config(qpid_hosts=brokers, - group='oslo_messaging_qpid') - - with mock.patch('qpid.messaging.Connection') as conn_mock: - # starting from the first broker in the list - url = oslo_messaging.TransportURL.parse(self.conf, None) - connection = qpid_driver.Connection(self.conf, url, - amqp.PURPOSE_SEND) - - # reconnect will advance to the next broker, one broker per - # attempt, and then wrap to the start of the list once the end is - # reached - for _ in range(brokers_count): - connection.reconnect() - - expected = [] - for broker in brokers: - expected.extend([mock.call("%s:5672" % broker), - mock.call().open(), - mock.call().session(), - mock.call().opened(), - mock.call().opened().__nonzero__(), - mock.call().close()]) - - conn_mock.assert_has_calls(expected, any_order=True) - - -def synchronized(func): - func.__lock__ = threading.Lock() - - def synced_func(*args, **kws): - with func.__lock__: - return func(*args, **kws) - - return synced_func - - -class FakeQpidMsgManager(object): - def __init__(self): - self._exchanges = {} - - @synchronized - def add_exchange(self, exchange): - if exchange not in self._exchanges: - self._exchanges[exchange] = {'msgs': [], 'consumers': {}} - - @synchronized - def add_exchange_consumer(self, exchange, consumer_id): - exchange_info = self._exchanges[exchange] - cons_dict = exchange_info['consumers'] - cons_dict[consumer_id] = 0 - - @synchronized - def add_exchange_msg(self, exchange, msg): - exchange_info = self._exchanges[exchange] - exchange_info['msgs'].append(msg) - - def get_exchange_msg(self, exchange, index): - exchange_info = self._exchanges[exchange] - return exchange_info['msgs'][index] - - def get_no_exch_msgs(self, exchange): - exchange_info = self._exchanges[exchange] - return len(exchange_info['msgs']) - - def get_exch_cons_index(self, exchange, consumer_id): - exchange_info = self._exchanges[exchange] - cons_dict = exchange_info['consumers'] - return cons_dict[consumer_id] - - @synchronized - def inc_consumer_index(self, exchange, consumer_id): - exchange_info = self._exchanges[exchange] - cons_dict = exchange_info['consumers'] - cons_dict[consumer_id] += 1 - -_fake_qpid_msg_manager = FakeQpidMsgManager() - - -class FakeQpidSessionSender(object): - def __init__(self, session, id, target, options): - self.session = session - self.id = id - self.target = target - self.options = options - - @synchronized - def send(self, object, sync=True, timeout=None): - _fake_qpid_msg_manager.add_exchange_msg(self.target, object) - - def close(self, timeout=None): - pass - - -class FakeQpidSessionReceiver(object): - - def __init__(self, session, id, source, options): - self.session = session - self.id = id - self.source = source - self.options = options - - @synchronized - def fetch(self, timeout=None): - if timeout is None: - # if timeout is not given, take a default time out - # of 30 seconds to avoid indefinite loop - _timeout = 30 - else: - _timeout = timeout - - deadline = time.time() + _timeout - while time.time() <= deadline: - index = _fake_qpid_msg_manager.get_exch_cons_index(self.source, - self.id) - try: - msg = _fake_qpid_msg_manager.get_exchange_msg(self.source, - index) - except IndexError: - pass - else: - _fake_qpid_msg_manager.inc_consumer_index(self.source, - self.id) - return qpid.messaging.Message(msg) - time.sleep(0.050) - - if timeout is None: - raise Exception('timed out waiting for reply') - - def close(self, timeout=None): - pass - - @synchronized - def available(self): - no_msgs = _fake_qpid_msg_manager.get_no_exch_msgs(self.source) - index = _fake_qpid_msg_manager.get_exch_cons_index(self.source, - self.id) - if no_msgs == 0 or index >= no_msgs: - return 0 - else: - return no_msgs - index - - -class FakeQpidSession(object): - - def __init__(self, connection=None, name=None, transactional=None): - self.connection = connection - self.name = name - self.transactional = transactional - self._receivers = {} - self.conf = None - self.url = None - self._senders = {} - self._sender_id = 0 - self._receiver_id = 0 - - @synchronized - def sender(self, target, **options): - exchange_key = self._extract_exchange_key(target) - _fake_qpid_msg_manager.add_exchange(exchange_key) - - sendobj = FakeQpidSessionSender(self, self._sender_id, - exchange_key, options) - self._senders[self._sender_id] = sendobj - self._sender_id = self._sender_id + 1 - return sendobj - - @synchronized - def receiver(self, source, **options): - exchange_key = self._extract_exchange_key(source) - _fake_qpid_msg_manager.add_exchange(exchange_key) - recvobj = FakeQpidSessionReceiver(self, self._receiver_id, - exchange_key, options) - self._receivers[self._receiver_id] = recvobj - _fake_qpid_msg_manager.add_exchange_consumer(exchange_key, - self._receiver_id) - self._receiver_id += 1 - return recvobj - - def acknowledge(self, message=None, disposition=None, sync=True): - pass - - @synchronized - def flush_exchanges(self): - _fake_qpid_msg_manager._exchanges = {} - - def _extract_exchange_key(self, exchange_msg): - """This function extracts a unique key for the exchange. - This key is used in the dictionary as a 'key' for - this exchange. - Eg. if the exchange_msg (for qpid topology version 1) - is 33/33 ; {"node": {"x-declare": {"auto-delete": true, .... - then 33 is returned as the key. - Eg 2. For topology v2, if the - exchange_msg is - amq.direct/44 ; {"link": {"x-dec....... - then 44 is returned - """ - # first check for ';' - semicolon_split = exchange_msg.split(';') - - # split the first item of semicolon_split with '/' - slash_split = semicolon_split[0].split('/') - # return the last element of the list as the key - key = slash_split[-1] - return key.strip() - - def close(self): - pass - -_fake_session = FakeQpidSession() - - -def get_fake_qpid_session(): - return _fake_session - - -class QPidHATestCase(test_utils.BaseTestCase): - - @testtools.skipIf(qpid is None, "qpid not available") - def setUp(self): - super(QPidHATestCase, self).setUp() - self.brokers = ['host1', 'host2', 'host3', 'host4', 'host5'] - - self.config(qpid_hosts=self.brokers, - qpid_username=None, - qpid_password=None, - group='oslo_messaging_qpid') - - hostname_sets = set() - self.info = {'attempt': 0, - 'fail': False} - - def _connect(myself, broker): - # do as little work that is enough to pass connection attempt - myself.connection = mock.Mock() - hostname = broker['host'] - self.assertNotIn(hostname, hostname_sets) - hostname_sets.add(hostname) - - self.info['attempt'] += 1 - if self.info['fail']: - raise qpid.messaging.exceptions.ConnectionError - - # just make sure connection instantiation does not fail with an - # exception - self.stubs.Set(qpid_driver.Connection, '_connect', _connect) - - # starting from the first broker in the list - url = oslo_messaging.TransportURL.parse(self.conf, None) - self.connection = qpid_driver.Connection(self.conf, url, - amqp.PURPOSE_SEND) - self.addCleanup(self.connection.close) - - self.info.update({'attempt': 0, - 'fail': True}) - hostname_sets.clear() - - def test_reconnect_order(self): - self.assertRaises(oslo_messaging.MessageDeliveryFailure, - self.connection.reconnect, - retry=len(self.brokers) - 1) - self.assertEqual(len(self.brokers), self.info['attempt']) - - def test_ensure_four_retries(self): - mock_callback = mock.Mock( - side_effect=qpid.messaging.exceptions.ConnectionError) - self.assertRaises(oslo_messaging.MessageDeliveryFailure, - self.connection.ensure, None, mock_callback, - retry=4) - self.assertEqual(5, self.info['attempt']) - self.assertEqual(1, mock_callback.call_count) - - def test_ensure_one_retry(self): - mock_callback = mock.Mock( - side_effect=qpid.messaging.exceptions.ConnectionError) - self.assertRaises(oslo_messaging.MessageDeliveryFailure, - self.connection.ensure, None, mock_callback, - retry=1) - self.assertEqual(2, self.info['attempt']) - self.assertEqual(1, mock_callback.call_count) - - def test_ensure_no_retry(self): - mock_callback = mock.Mock( - side_effect=qpid.messaging.exceptions.ConnectionError) - self.assertRaises(oslo_messaging.MessageDeliveryFailure, - self.connection.ensure, None, mock_callback, - retry=0) - self.assertEqual(1, self.info['attempt']) - self.assertEqual(1, mock_callback.call_count) diff --git a/oslo_messaging/tests/drivers/test_impl_rabbit.py b/oslo_messaging/tests/drivers/test_impl_rabbit.py index 915715450..c2efb27a4 100644 --- a/oslo_messaging/tests/drivers/test_impl_rabbit.py +++ b/oslo_messaging/tests/drivers/test_impl_rabbit.py @@ -28,7 +28,6 @@ from oslotest import mockpatch import testscenarios import oslo_messaging -from oslo_messaging._drivers import amqp from oslo_messaging._drivers import amqpdriver from oslo_messaging._drivers import common as driver_common from oslo_messaging._drivers import impl_rabbit as rabbit_driver @@ -91,10 +90,12 @@ class TestHeartbeat(test_utils.BaseTestCase): if not heartbeat_side_effect: self.assertEqual(1, fake_ensure_connection.call_count) - self.assertEqual(2, fake_logger.info.call_count) + self.assertEqual(2, fake_logger.debug.call_count) + self.assertEqual(0, fake_logger.info.call_count) else: self.assertEqual(2, fake_ensure_connection.call_count) - self.assertEqual(3, fake_logger.info.call_count) + self.assertEqual(2, fake_logger.debug.call_count) + self.assertEqual(1, fake_logger.info.call_count) self.assertIn(mock.call(info, mock.ANY), fake_logger.info.mock_calls) @@ -167,7 +168,7 @@ class TestRabbitDriverLoadSSL(test_utils.BaseTestCase): 'on_blocked': mock.ANY, 'on_unblocked': mock.ANY}, ssl=self.expected, login_method='AMQPLAIN', - heartbeat=60, failover_strategy="shuffle") + heartbeat=60, failover_strategy='round-robin') class TestRabbitPublisher(test_utils.BaseTestCase): @@ -175,7 +176,7 @@ class TestRabbitPublisher(test_utils.BaseTestCase): def test_send_with_timeout(self, fake_publish): transport = oslo_messaging.get_transport(self.conf, 'kombu+memory:////') - with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn: + with transport._driver._get_connection(driver_common.PURPOSE_SEND) as pool_conn: conn = pool_conn.connection conn._publish(mock.Mock(), 'msg', routing_key='routing_key', timeout=1) @@ -185,7 +186,7 @@ class TestRabbitPublisher(test_utils.BaseTestCase): def test_send_no_timeout(self, fake_publish): transport = oslo_messaging.get_transport(self.conf, 'kombu+memory:////') - with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn: + with transport._driver._get_connection(driver_common.PURPOSE_SEND) as pool_conn: conn = pool_conn.connection conn._publish(mock.Mock(), 'msg', routing_key='routing_key') fake_publish.assert_called_with('msg', expiration=None) @@ -205,7 +206,7 @@ class TestRabbitPublisher(test_utils.BaseTestCase): type='topic', passive=False) - with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn: + with transport._driver._get_connection(driver_common.PURPOSE_SEND) as pool_conn: conn = pool_conn.connection exc = conn.connection.channel_errors[0] @@ -238,7 +239,7 @@ class TestRabbitConsume(test_utils.BaseTestCase): 'kombu+memory:////') self.addCleanup(transport.cleanup) deadline = time.time() + 6 - with transport._driver._get_connection(amqp.PURPOSE_LISTEN) as conn: + with transport._driver._get_connection(driver_common.PURPOSE_LISTEN) as conn: self.assertRaises(driver_common.Timeout, conn.consume, timeout=3) @@ -257,7 +258,7 @@ class TestRabbitConsume(test_utils.BaseTestCase): transport = oslo_messaging.get_transport(self.conf, 'kombu+memory:////') self.addCleanup(transport.cleanup) - with transport._driver._get_connection(amqp.PURPOSE_LISTEN) as conn: + with transport._driver._get_connection(driver_common.PURPOSE_LISTEN) as conn: channel = conn.connection.channel with mock.patch('kombu.connection.Connection.connected', new_callable=mock.PropertyMock, @@ -361,11 +362,6 @@ class TestSendReceive(test_utils.BaseTestCase): ('timeout', dict(timeout=0.01)), # FIXME(markmc): timeout=0 is broken? ] - _reply_ending = [ - ('old_behavior', dict(send_single_reply=False)), - ('new_behavior', dict(send_single_reply=True)), - ] - @classmethod def generate_scenarios(cls): cls.scenarios = testscenarios.multiply_scenarios(cls._n_senders, @@ -373,16 +369,13 @@ class TestSendReceive(test_utils.BaseTestCase): cls._reply, cls._reply_fail, cls._failure, - cls._timeout, - cls._reply_ending) + cls._timeout) def test_send_receive(self): - self.config(kombu_reconnect_timeout=0.5, + self.config(kombu_missing_consumer_retry_timeout=0.5, group="oslo_messaging_rabbit") self.config(heartbeat_timeout_threshold=0, group="oslo_messaging_rabbit") - self.config(send_single_reply=self.send_single_reply, - group="oslo_messaging_rabbit") transport = oslo_messaging.get_transport(self.conf, 'kombu+memory:////') self.addCleanup(transport.cleanup) @@ -430,7 +423,7 @@ class TestSendReceive(test_utils.BaseTestCase): for i in range(len(senders)): senders[i].start() - received = listener.poll() + received = listener.poll()[0] self.assertIsNotNone(received) self.assertEqual(self.ctxt, received.ctxt) self.assertEqual({'tx_id': i}, received.message) @@ -472,10 +465,10 @@ class TestSendReceive(test_utils.BaseTestCase): if self.reply_failure_404: # NOTE(sileht) all reply fail, first take - # kombu_reconnect_timeout seconds to fail + # kombu_missing_consumer_retry_timeout seconds to fail # next immediately fail dt = time.time() - start - timeout = self.conf.oslo_messaging_rabbit.kombu_reconnect_timeout + timeout = self.conf.oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout self.assertTrue(timeout <= dt < (timeout + 0.100), dt) self.assertEqual(len(senders), len(replies)) @@ -508,7 +501,7 @@ class TestPollAsync(test_utils.BaseTestCase): target = oslo_messaging.Target(topic='testtopic') listener = driver.listen(target) received = listener.poll(timeout=0.050) - self.assertIsNone(received) + self.assertEqual([], received) class TestRacyWaitForReply(test_utils.BaseTestCase): @@ -568,13 +561,13 @@ class TestRacyWaitForReply(test_utils.BaseTestCase): senders[0].start() notify_condition.wait() - msgs.append(listener.poll()) + msgs.extend(listener.poll()) self.assertEqual({'tx_id': 0}, msgs[-1].message) # Start the second guy, receive his message senders[1].start() - msgs.append(listener.poll()) + msgs.extend(listener.poll()) self.assertEqual({'tx_id': 1}, msgs[-1].message) # Reply to both in order, making the second thread queue @@ -588,7 +581,7 @@ class TestRacyWaitForReply(test_utils.BaseTestCase): # Start the 3rd guy, receive his message senders[2].start() - msgs.append(listener.poll()) + msgs.extend(listener.poll()) self.assertEqual({'tx_id': 2}, msgs[-1].message) # Verify the _send_reply was not invoked by driver: @@ -869,7 +862,7 @@ class TestReplyWireFormat(test_utils.BaseTestCase): producer.publish(msg) - received = listener.poll() + received = listener.poll()[0] self.assertIsNotNone(received) self.assertEqual(self.expected_ctxt, received.ctxt) self.assertEqual(self.expected, received.message) @@ -894,13 +887,15 @@ class RpcKombuHATestCase(test_utils.BaseTestCase): self.useFixture(mockpatch.Patch( 'kombu.connection.Connection.connect', side_effect=self.kombu_connect)) + self.useFixture(mockpatch.Patch( + 'kombu.connection.Connection.connection')) self.useFixture(mockpatch.Patch( 'kombu.connection.Connection.channel')) # starting from the first broker in the list url = oslo_messaging.TransportURL.parse(self.conf, None) self.connection = rabbit_driver.Connection(self.conf, url, - amqp.PURPOSE_SEND) + driver_common.PURPOSE_SEND) self.addCleanup(self.connection.close) def test_ensure_four_retry(self): diff --git a/oslo_messaging/tests/drivers/zmq/matchmaker/test_impl_matchmaker.py b/oslo_messaging/tests/drivers/zmq/matchmaker/test_impl_matchmaker.py index ba5f1f399..5751e5ba3 100644 --- a/oslo_messaging/tests/drivers/zmq/matchmaker/test_impl_matchmaker.py +++ b/oslo_messaging/tests/drivers/zmq/matchmaker/test_impl_matchmaker.py @@ -62,47 +62,47 @@ class TestImplMatchmaker(test_utils.BaseTestCase): self.host2 = b"test_host2" def test_register(self): - self.test_matcher.register(self.target, self.host1) + self.test_matcher.register(self.target, self.host1, "test") - self.assertEqual(self.test_matcher.get_hosts(self.target), + self.assertEqual(self.test_matcher.get_hosts(self.target, "test"), [self.host1]) - self.assertEqual(self.test_matcher.get_single_host(self.target), + self.assertEqual(self.test_matcher.get_single_host(self.target, "test"), self.host1) def test_register_two_hosts(self): - self.test_matcher.register(self.target, self.host1) - self.test_matcher.register(self.target, self.host2) + self.test_matcher.register(self.target, self.host1, "test") + self.test_matcher.register(self.target, self.host2, "test") - self.assertItemsEqual(self.test_matcher.get_hosts(self.target), + self.assertItemsEqual(self.test_matcher.get_hosts(self.target, "test"), [self.host1, self.host2]) - self.assertIn(self.test_matcher.get_single_host(self.target), + self.assertIn(self.test_matcher.get_single_host(self.target, "test"), [self.host1, self.host2]) def test_register_unsibscribe(self): - self.test_matcher.register(self.target, self.host1) - self.test_matcher.register(self.target, self.host2) + self.test_matcher.register(self.target, self.host1, "test") + self.test_matcher.register(self.target, self.host2, "test") - self.test_matcher.unregister(self.target, self.host2) + self.test_matcher.unregister(self.target, self.host2, "test") - self.assertItemsEqual(self.test_matcher.get_hosts(self.target), + self.assertItemsEqual(self.test_matcher.get_hosts(self.target, "test"), [self.host1]) - self.assertNotIn(self.test_matcher.get_single_host(self.target), + self.assertNotIn(self.test_matcher.get_single_host(self.target, "test"), [self.host2]) def test_register_two_same_hosts(self): - self.test_matcher.register(self.target, self.host1) - self.test_matcher.register(self.target, self.host1) + self.test_matcher.register(self.target, self.host1, "test") + self.test_matcher.register(self.target, self.host1, "test") - self.assertEqual(self.test_matcher.get_hosts(self.target), + self.assertEqual(self.test_matcher.get_hosts(self.target, "test"), [self.host1]) - self.assertEqual(self.test_matcher.get_single_host(self.target), + self.assertEqual(self.test_matcher.get_single_host(self.target, "test"), self.host1) def test_get_hosts_wrong_topic(self): target = oslo_messaging.Target(topic="no_such_topic") - self.assertEqual(self.test_matcher.get_hosts(target), []) + self.assertEqual(self.test_matcher.get_hosts(target, "test"), []) def test_get_single_host_wrong_topic(self): target = oslo_messaging.Target(topic="no_such_topic") self.assertRaises(oslo_messaging.InvalidTarget, - self.test_matcher.get_single_host, target) + self.test_matcher.get_single_host, target, "test") diff --git a/oslo_messaging/tests/drivers/zmq/test_impl_zmq.py b/oslo_messaging/tests/drivers/zmq/test_impl_zmq.py index c40007523..1d710d3a9 100644 --- a/oslo_messaging/tests/drivers/zmq/test_impl_zmq.py +++ b/oslo_messaging/tests/drivers/zmq/test_impl_zmq.py @@ -21,6 +21,7 @@ import testtools import oslo_messaging from oslo_messaging._drivers import impl_zmq from oslo_messaging._drivers.zmq_driver import zmq_async +from oslo_messaging._drivers.zmq_driver import zmq_socket from oslo_messaging._i18n import _ from oslo_messaging.tests import utils as test_utils @@ -51,7 +52,8 @@ class TestServerListener(object): def _run(self): try: message = self.listener.poll() - if message is not None: + if message: + message = message[0] message.acknowledge() self._received.set() self.message = message @@ -90,6 +92,33 @@ class ZmqBaseTestCase(test_utils.BaseTestCase): self.addCleanup(stopRpc(self.__dict__)) +class ZmqTestPortsRange(ZmqBaseTestCase): + + @testtools.skipIf(zmq is None, "zmq not available") + def setUp(self): + super(ZmqTestPortsRange, self).setUp() + + # Set config values + kwargs = {'rpc_zmq_min_port': 5555, + 'rpc_zmq_max_port': 5560} + self.config(**kwargs) + + def test_ports_range(self): + listeners = [] + + for i in range(10): + try: + target = oslo_messaging.Target(topic='testtopic_'+str(i)) + new_listener = self.driver.listen(target) + listeners.append(new_listener) + except zmq_socket.ZmqPortRangeExceededException: + pass + + self.assertLessEqual(len(listeners), 5) + + for l in listeners: + l.cleanup() + class TestConfZmqDriverLoad(test_utils.BaseTestCase): @testtools.skipIf(zmq is None, "zmq not available") @@ -196,6 +225,7 @@ class TestZmqBasics(ZmqBaseTestCase): class TestPoller(test_utils.BaseTestCase): + @testtools.skipIf(zmq is None, "zmq not available") def setUp(self): super(TestPoller, self).setUp() self.poller = zmq_async.get_poller() diff --git a/oslo_messaging/tests/drivers/zmq/test_zmq_async.py b/oslo_messaging/tests/drivers/zmq/test_zmq_async.py index 28e091a0e..ccfae3348 100644 --- a/oslo_messaging/tests/drivers/zmq/test_zmq_async.py +++ b/oslo_messaging/tests/drivers/zmq/test_zmq_async.py @@ -11,15 +11,19 @@ # under the License. import mock +import testtools from oslo_messaging._drivers.zmq_driver.poller import green_poller from oslo_messaging._drivers.zmq_driver.poller import threading_poller from oslo_messaging._drivers.zmq_driver import zmq_async from oslo_messaging.tests import utils as test_utils +zmq = zmq_async.import_zmq() + class TestImportZmq(test_utils.BaseTestCase): + @testtools.skipIf(zmq is None, "zmq not available") def setUp(self): super(TestImportZmq, self).setUp() @@ -29,12 +33,12 @@ class TestImportZmq(test_utils.BaseTestCase): zmq_async.importutils.try_import.return_value = 'mock zmq module' self.assertEqual('mock zmq module', zmq_async.import_zmq('native')) - mock_try_import.assert_called_with('zmq', default='zmq') + mock_try_import.assert_called_with('zmq', default=None) zmq_async.importutils.try_import.return_value = 'mock eventlet module' self.assertEqual('mock eventlet module', zmq_async.import_zmq('eventlet')) - mock_try_import.assert_called_with('eventlet.green.zmq', default='zmq') + mock_try_import.assert_called_with('eventlet.green.zmq', default=None) def test_when_no_args_then_default_zmq_module_is_loaded(self): mock_try_import = mock.Mock() @@ -42,14 +46,7 @@ class TestImportZmq(test_utils.BaseTestCase): zmq_async.import_zmq() - mock_try_import.assert_called_with('eventlet.green.zmq', default='zmq') - - def test_when_import_fails_then_raise_ImportError(self): - zmq_async.importutils.try_import = mock.Mock() - zmq_async.importutils.try_import.return_value = None - - with self.assertRaisesRegexp(ImportError, "ZeroMQ not found!"): - zmq_async.import_zmq('native') + mock_try_import.assert_called_with('eventlet.green.zmq', default=None) def test_invalid_config_value_raise_ValueError(self): invalid_opt = 'x' @@ -61,6 +58,7 @@ class TestImportZmq(test_utils.BaseTestCase): class TestGetPoller(test_utils.BaseTestCase): + @testtools.skipIf(zmq is None, "zmq not available") def setUp(self): super(TestGetPoller, self).setUp() @@ -100,6 +98,7 @@ class TestGetPoller(test_utils.BaseTestCase): class TestGetReplyPoller(test_utils.BaseTestCase): + @testtools.skipIf(zmq is None, "zmq not available") def setUp(self): super(TestGetReplyPoller, self).setUp() @@ -134,6 +133,7 @@ class TestGetReplyPoller(test_utils.BaseTestCase): class TestGetExecutor(test_utils.BaseTestCase): + @testtools.skipIf(zmq is None, "zmq not available") def setUp(self): super(TestGetExecutor, self).setUp() diff --git a/oslo_messaging/tests/executors/test_executor.py b/oslo_messaging/tests/executors/test_executor.py index 007d3ac6a..fb91c11be 100644 --- a/oslo_messaging/tests/executors/test_executor.py +++ b/oslo_messaging/tests/executors/test_executor.py @@ -44,7 +44,7 @@ try: except ImportError: impl_eventlet = None from oslo_messaging._executors import impl_thread -from oslo_messaging import _utils as utils +from oslo_messaging import dispatcher as dispatcher_base from oslo_messaging.tests import utils as test_utils from six.moves import mock @@ -81,6 +81,12 @@ class TestExecutor(test_utils.BaseTestCase): aioeventlet_class = None is_aioeventlet = (self.executor == aioeventlet_class) + if impl_blocking is not None: + blocking_class = impl_blocking.BlockingExecutor + else: + blocking_class = None + is_blocking = (self.executor == blocking_class) + if is_aioeventlet: policy = aioeventlet.EventLoopPolicy() trollius.set_event_loop_policy(policy) @@ -110,8 +116,15 @@ class TestExecutor(test_utils.BaseTestCase): endpoint = mock.MagicMock(return_value=simple_coroutine('result')) event = eventlet.event.Event() - else: + elif is_blocking: + def run_executor(executor): + executor.start() + executor.execute() + executor.wait() + endpoint = mock.MagicMock(return_value='result') + event = None + else: def run_executor(executor): executor.start() executor.wait() @@ -119,11 +132,14 @@ class TestExecutor(test_utils.BaseTestCase): endpoint = mock.MagicMock(return_value='result') event = None - class Dispatcher(object): + class Dispatcher(dispatcher_base.DispatcherBase): def __init__(self, endpoint): self.endpoint = endpoint self.result = "not set" + def _listen(self, transport): + pass + def callback(self, incoming, executor_callback): if executor_callback is None: result = self.endpoint(incoming.ctxt, @@ -138,9 +154,8 @@ class TestExecutor(test_utils.BaseTestCase): return result def __call__(self, incoming, executor_callback=None): - return utils.DispatcherExecutorContext(incoming, - self.callback, - executor_callback) + return dispatcher_base.DispatcherExecutorContext( + incoming[0], self.callback, executor_callback) return Dispatcher(endpoint), endpoint, event, run_executor @@ -150,7 +165,7 @@ class TestExecutor(test_utils.BaseTestCase): executor = self.executor(self.conf, listener, dispatcher) incoming_message = mock.MagicMock(ctxt={}, message={'payload': 'data'}) - def fake_poll(timeout=None): + def fake_poll(timeout=None, prefetch_size=1): time.sleep(0.1) if listener.poll.call_count == 10: if event is not None: @@ -178,9 +193,9 @@ class TestExecutor(test_utils.BaseTestCase): executor = self.executor(self.conf, listener, dispatcher) incoming_message = mock.MagicMock(ctxt={}, message={'payload': 'data'}) - def fake_poll(timeout=None): + def fake_poll(timeout=None, prefetch_size=1): if listener.poll.call_count == 1: - return incoming_message + return [incoming_message] if event is not None: event.wait() executor.stop() diff --git a/oslo_messaging/tests/functional/gate/post_test_hook.sh b/oslo_messaging/tests/functional/gate/post_test_hook.sh index 276129cdd..23ee6ab48 100755 --- a/oslo_messaging/tests/functional/gate/post_test_hook.sh +++ b/oslo_messaging/tests/functional/gate/post_test_hook.sh @@ -46,10 +46,6 @@ case $RPC_BACKEND in sudo apt-get update -y sudo apt-get install -y redis-server python-redis ;; - qpid) - sudo apt-get update -y - sudo apt-get install -y qpidd sasl2-bin - ;; amqp1) sudo yum install -y qpid-cpp-server qpid-proton-c-devel python-qpid-proton cyrus-sasl-lib cyrus-sasl-plain ;; diff --git a/oslo_messaging/tests/functional/notify/test_logger.py b/oslo_messaging/tests/functional/notify/test_logger.py index ad30e8850..a7f580bc3 100644 --- a/oslo_messaging/tests/functional/notify/test_logger.py +++ b/oslo_messaging/tests/functional/notify/test_logger.py @@ -51,8 +51,9 @@ class LoggingNotificationHandlerTestCase(utils.SkipIfNoTransportURL): # NOTE(gtt): Using different topic to make tests run in parallel topic = 'test_logging_%s_driver_%s' % (self.priority, self.driver) - self.conf.notification_driver = [self.driver] - self.conf.notification_topics = [topic] + self.config(driver=[self.driver], + topics=[topic], + group='oslo_messaging_notifications') listener = self.useFixture( utils.NotificationFixture(self.conf, self.url, [topic])) diff --git a/oslo_messaging/tests/functional/test_functional.py b/oslo_messaging/tests/functional/test_functional.py index 5e50a399a..8ee758dfc 100644 --- a/oslo_messaging/tests/functional/test_functional.py +++ b/oslo_messaging/tests/functional/test_functional.py @@ -16,6 +16,7 @@ import uuid import concurrent.futures from oslo_config import cfg +import six.moves from testtools import matchers import oslo_messaging @@ -27,8 +28,8 @@ class CallTestCase(utils.SkipIfNoTransportURL): def setUp(self): super(CallTestCase, self).setUp(conf=cfg.ConfigOpts()) - self.conf.prog="test_prog" - self.conf.project="test_project" + self.conf.prog = "test_prog" + self.conf.project = "test_project" self.config(heartbeat_timeout_threshold=0, group='oslo_messaging_rabbit') @@ -324,3 +325,18 @@ class NotifyTestCase(utils.SkipIfNoTransportURL): self.assertEqual(expected[1], actual[0]) self.assertEqual(expected[2], actual[1]) self.assertEqual(expected[3], actual[2]) + + def test_simple_batch(self): + listener = self.useFixture( + utils.BatchNotificationFixture(self.conf, self.url, + ['test_simple_batch'], + batch_size=100, batch_timeout=2)) + notifier = listener.notifier('abc') + + for i in six.moves.range(0, 205): + notifier.info({}, 'test%s' % i, 'Hello World!') + events = listener.get_events(timeout=3) + self.assertEqual(3, len(events), events) + self.assertEqual(100, len(events[0][1])) + self.assertEqual(100, len(events[1][1])) + self.assertEqual(5, len(events[2][1])) diff --git a/oslo_messaging/tests/functional/utils.py b/oslo_messaging/tests/functional/utils.py index 99794ed37..2f9e7b9f7 100644 --- a/oslo_messaging/tests/functional/utils.py +++ b/oslo_messaging/tests/functional/utils.py @@ -293,13 +293,14 @@ class SkipIfNoTransportURL(test_utils.BaseTestCase): class NotificationFixture(fixtures.Fixture): - def __init__(self, conf, url, topics): + def __init__(self, conf, url, topics, batch=None): super(NotificationFixture, self).__init__() self.conf = conf self.url = url self.topics = topics self.events = moves.queue.Queue() self.name = str(id(self)) + self.batch = batch def setUp(self): super(NotificationFixture, self).setUp() @@ -307,10 +308,7 @@ class NotificationFixture(fixtures.Fixture): # add a special topic for internal notifications targets.append(oslo_messaging.Target(topic=self.name)) transport = self.useFixture(TransportFixture(self.conf, self.url)) - self.server = oslo_messaging.get_notification_listener( - transport.transport, - targets, - [self], 'eventlet') + self.server = self._get_server(transport, targets) self._ctrl = self.notifier('internal', topic=self.name) self._start() transport.wait() @@ -319,6 +317,12 @@ class NotificationFixture(fixtures.Fixture): self._stop() super(NotificationFixture, self).cleanUp() + def _get_server(self, transport, targets): + return oslo_messaging.get_notification_listener( + transport.transport, + targets, + [self], 'eventlet') + def _start(self): self.thread = test_utils.ServerThreadHelper(self.server) self.thread.start() @@ -366,3 +370,39 @@ class NotificationFixture(fixtures.Fixture): except moves.queue.Empty: pass return results + + +class BatchNotificationFixture(NotificationFixture): + def __init__(self, conf, url, topics, batch_size=5, batch_timeout=2): + super(BatchNotificationFixture, self).__init__(conf, url, topics) + self.batch_size = batch_size + self.batch_timeout = batch_timeout + + def _get_server(self, transport, targets): + return oslo_messaging.get_batch_notification_listener( + transport.transport, + targets, + [self], 'eventlet', + batch_timeout=self.batch_timeout, + batch_size=self.batch_size) + + def debug(self, messages): + self.events.put(['debug', messages]) + + def audit(self, messages): + self.events.put(['audit', messages]) + + def info(self, messages): + self.events.put(['info', messages]) + + def warn(self, messages): + self.events.put(['warn', messages]) + + def error(self, messages): + self.events.put(['error', messages]) + + def critical(self, messages): + self.events.put(['critical', messages]) + + def sample(self, messages): + pass # Just used for internal shutdown control diff --git a/oslo_messaging/tests/notify/test_dispatcher.py b/oslo_messaging/tests/notify/test_dispatcher.py index f0da90d89..2442d535e 100644 --- a/oslo_messaging/tests/notify/test_dispatcher.py +++ b/oslo_messaging/tests/notify/test_dispatcher.py @@ -107,7 +107,7 @@ class TestDispatcher(test_utils.BaseTestCase): sorted(dispatcher._targets_priorities)) incoming = mock.Mock(ctxt={}, message=msg) - callback = dispatcher(incoming) + callback = dispatcher([incoming]) callback.run() callback.done() @@ -144,7 +144,7 @@ class TestDispatcher(test_utils.BaseTestCase): msg['priority'] = 'what???' dispatcher = notify_dispatcher.NotificationDispatcher( [mock.Mock()], [mock.Mock()], None, allow_requeue=True, pool=None) - callback = dispatcher(mock.Mock(ctxt={}, message=msg)) + callback = dispatcher([mock.Mock(ctxt={}, message=msg)]) callback.run() callback.done() mylog.warning.assert_called_once_with('Unknown priority "%s"', @@ -246,7 +246,7 @@ class TestDispatcherFilter(test_utils.BaseTestCase): 'timestamp': '2014-03-03 18:21:04.369234', 'message_id': '99863dda-97f0-443a-a0c1-6ed317b7fd45'} incoming = mock.Mock(ctxt=self.context, message=message) - callback = dispatcher(incoming) + callback = dispatcher([incoming]) callback.run() callback.done() diff --git a/oslo_messaging/tests/notify/test_impl_messaging.py b/oslo_messaging/tests/notify/test_impl_messaging.py new file mode 100644 index 000000000..d2a9a2ee0 --- /dev/null +++ b/oslo_messaging/tests/notify/test_impl_messaging.py @@ -0,0 +1,30 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslo_messaging.tests import utils as test_utils + + +class TestDeprecationWarning(test_utils.BaseTestCase): + + @mock.patch('warnings.warn') + def test_impl_messaging_deprecation_warning(self, mock_warn): + # Tests that we get a deprecation warning when loading a messaging + # driver out of oslo_messaging.notify._impl_messaging. + from oslo_messaging.notify import _impl_messaging as messaging + driver = messaging.MessagingV2Driver( + conf={}, topics=['notifications'], transport='rpc') + # Make sure we got a deprecation warning by loading from the alias + self.assertEqual(1, mock_warn.call_count) diff --git a/oslo_messaging/tests/notify/test_listener.py b/oslo_messaging/tests/notify/test_listener.py index 1644936ad..3e534aaac 100644 --- a/oslo_messaging/tests/notify/test_listener.py +++ b/oslo_messaging/tests/notify/test_listener.py @@ -21,7 +21,9 @@ import testscenarios import oslo_messaging from oslo_messaging.notify import dispatcher +from oslo_messaging.notify import notifier as msg_notifier from oslo_messaging.tests import utils as test_utils +import six from six.moves import mock load_tests = testscenarios.load_tests_apply_scenarios @@ -53,16 +55,18 @@ class ListenerSetupMixin(object): def __init__(self): self._received_msgs = 0 self.threads = [] - self.lock = threading.Lock() + self.lock = threading.Condition() - def info(self, ctxt, publisher_id, event_type, payload, metadata): + def info(self, *args, **kwargs): # NOTE(sileht): this run into an other thread with self.lock: self._received_msgs += 1 + self.lock.notify_all() def wait_for_messages(self, expect_messages): - while self._received_msgs < expect_messages: - time.sleep(0.01) + with self.lock: + while self._received_msgs < expect_messages: + self.lock.wait() def stop(self): for thread in self.threads: @@ -83,7 +87,7 @@ class ListenerSetupMixin(object): self.trackers = {} def _setup_listener(self, transport, endpoints, - targets=None, pool=None): + targets=None, pool=None, batch=False): if pool is None: tracker_name = '__default__' @@ -95,9 +99,15 @@ class ListenerSetupMixin(object): tracker = self.trackers.setdefault( tracker_name, self.ThreadTracker()) - listener = oslo_messaging.get_notification_listener( - transport, targets=targets, endpoints=[tracker] + endpoints, - allow_requeue=True, pool=pool, executor='eventlet') + if batch: + listener = oslo_messaging.get_batch_notification_listener( + transport, targets=targets, endpoints=[tracker] + endpoints, + allow_requeue=True, pool=pool, executor='eventlet', + batch_size=batch[0], batch_timeout=batch[1]) + else: + listener = oslo_messaging.get_notification_listener( + transport, targets=targets, endpoints=[tracker] + endpoints, + allow_requeue=True, pool=pool, executor='eventlet') thread = RestartableServerThread(listener) tracker.start(thread) @@ -124,7 +134,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): ListenerSetupMixin.setUp(self) def test_constructor(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') target = oslo_messaging.Target(topic='foo') endpoints = [object()] @@ -139,7 +150,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): self.assertEqual('blocking', listener.executor) def test_no_target_topic(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') listener = oslo_messaging.get_notification_listener( transport, @@ -153,7 +165,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): self.assertTrue(False) def test_unknown_executor(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') try: oslo_messaging.get_notification_listener(transport, [], [], @@ -164,9 +177,86 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): else: self.assertTrue(False) - def test_one_topic(self): + def test_batch_timeout(self): transport = oslo_messaging.get_transport(self.conf, url='fake:') + endpoint = mock.Mock() + endpoint.info.return_value = None + listener_thread = self._setup_listener(transport, [endpoint], + batch=(5, 1)) + + notifier = self._setup_notifier(transport) + for i in six.moves.range(12): + notifier.info({}, 'an_event.start', 'test message') + + self.wait_for_messages(3) + self.assertFalse(listener_thread.stop()) + + messages = [dict(ctxt={}, + publisher_id='testpublisher', + event_type='an_event.start', + payload='test message', + metadata={'message_id': mock.ANY, + 'timestamp': mock.ANY})] + + endpoint.info.assert_has_calls([mock.call(messages * 5), + mock.call(messages * 5), + mock.call(messages * 2)]) + + def test_batch_size(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint = mock.Mock() + endpoint.info.return_value = None + listener_thread = self._setup_listener(transport, [endpoint], + batch=(5, None)) + + notifier = self._setup_notifier(transport) + for i in six.moves.range(10): + notifier.info({}, 'an_event.start', 'test message') + + self.wait_for_messages(2) + self.assertFalse(listener_thread.stop()) + + messages = [dict(ctxt={}, + publisher_id='testpublisher', + event_type='an_event.start', + payload='test message', + metadata={'message_id': mock.ANY, + 'timestamp': mock.ANY})] + + endpoint.info.assert_has_calls([mock.call(messages * 5), + mock.call(messages * 5)]) + + def test_batch_size_exception_path(self): + transport = oslo_messaging.get_transport(self.conf, url='fake:') + + endpoint = mock.Mock() + endpoint.info.side_effect = [None, Exception('boom!')] + listener_thread = self._setup_listener(transport, [endpoint], + batch=(5, None)) + + notifier = self._setup_notifier(transport) + for i in six.moves.range(10): + notifier.info({}, 'an_event.start', 'test message') + + self.wait_for_messages(2) + self.assertFalse(listener_thread.stop()) + + messages = [dict(ctxt={}, + publisher_id='testpublisher', + event_type='an_event.start', + payload='test message', + metadata={'message_id': mock.ANY, + 'timestamp': mock.ANY})] + + endpoint.info.assert_has_calls([mock.call(messages * 5)]) + + + def test_one_topic(self): + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') + endpoint = mock.Mock() endpoint.info.return_value = None listener_thread = self._setup_listener(transport, [endpoint]) @@ -182,7 +272,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): {'message_id': mock.ANY, 'timestamp': mock.ANY}) def test_two_topics(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint = mock.Mock() endpoint.info.return_value = None @@ -208,7 +299,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): any_order=True) def test_two_exchanges(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint = mock.Mock() endpoint.info.return_value = None @@ -252,7 +344,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): any_order=True) def test_two_endpoints(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint1 = mock.Mock() endpoint1.info.return_value = None @@ -277,7 +370,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): 'message_id': mock.ANY}) def test_requeue(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint = mock.Mock() endpoint.info = mock.Mock() @@ -301,7 +395,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): {'timestamp': mock.ANY, 'message_id': mock.ANY})]) def test_two_pools(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint1 = mock.Mock() endpoint1.info.return_value = None @@ -334,7 +429,8 @@ class TestNotifyListener(test_utils.BaseTestCase, ListenerSetupMixin): mocked_endpoint_call(1)]) def test_two_pools_three_listener(self): - transport = oslo_messaging.get_transport(self.conf, url='fake:') + transport = msg_notifier.get_notification_transport( + self.conf, url='fake:') endpoint1 = mock.Mock() endpoint1.info.return_value = None diff --git a/oslo_messaging/tests/notify/test_log_handler.py b/oslo_messaging/tests/notify/test_log_handler.py index dfc214aed..1851321dd 100644 --- a/oslo_messaging/tests/notify/test_log_handler.py +++ b/oslo_messaging/tests/notify/test_log_handler.py @@ -28,7 +28,8 @@ class PublishErrorsHandlerTestCase(test_utils.BaseTestCase): def test_emit_cfg_log_notifier_in_notifier_drivers(self): drivers = ['messaging', 'log'] - self.config(notification_driver=drivers) + self.config(driver=drivers, + group='oslo_messaging_notifications') self.stub_flg = True transport = test_notifier._FakeTransport(self.conf) diff --git a/oslo_messaging/tests/notify/test_logger.py b/oslo_messaging/tests/notify/test_logger.py index f0ea8e8b8..12d1cd100 100644 --- a/oslo_messaging/tests/notify/test_logger.py +++ b/oslo_messaging/tests/notify/test_logger.py @@ -49,7 +49,8 @@ class TestLogNotifier(test_utils.BaseTestCase): def setUp(self): super(TestLogNotifier, self).setUp() self.addCleanup(oslo_messaging.notify._impl_test.reset) - self.config(notification_driver=['test']) + self.config(driver=['test'], + group='oslo_messaging_notifications') # NOTE(jamespage) disable thread information logging for testing # as this causes test failures when zmq tests monkey_patch via # eventlet diff --git a/oslo_messaging/tests/notify/test_notifier.py b/oslo_messaging/tests/notify/test_notifier.py index 557b9bb25..0b2da7fbb 100644 --- a/oslo_messaging/tests/notify/test_notifier.py +++ b/oslo_messaging/tests/notify/test_notifier.py @@ -156,8 +156,9 @@ class TestMessagingNotifier(test_utils.BaseTestCase): if self.v2: drivers.append('messagingv2') - self.config(notification_driver=drivers, - notification_topics=self.topics) + self.config(driver=drivers, + topics=self.topics, + group='oslo_messaging_notifications') transport = _FakeTransport(self.conf) @@ -269,7 +270,8 @@ class TestLogNotifier(test_utils.BaseTestCase): @mock.patch('oslo_utils.timeutils.utcnow') def test_notifier(self, mock_utcnow): - self.config(notification_driver=['log']) + self.config(driver=['log'], + group='oslo_messaging_notifications') transport = _FakeTransport(self.conf) @@ -338,7 +340,8 @@ class TestLogNotifier(test_utils.BaseTestCase): class TestRoutingNotifier(test_utils.BaseTestCase): def setUp(self): super(TestRoutingNotifier, self).setUp() - self.config(notification_driver=['routing']) + self.config(driver=['routing'], + group='oslo_messaging_notifications') transport = _FakeTransport(self.conf) self.notifier = oslo_messaging.Notifier(transport) @@ -360,13 +363,14 @@ class TestRoutingNotifier(test_utils.BaseTestCase): self.assertTrue(self.router._should_load_plugin(ext)) def test_load_notifiers_no_config(self): - # default routing_notifier_config="" + # default routing_config="" self.router._load_notifiers() self.assertEqual({}, self.router.routing_groups) self.assertEqual(0, len(self.router.used_drivers)) def test_load_notifiers_no_extensions(self): - self.config(routing_notifier_config="routing_notifier.yaml") + self.config(routing_config="routing_notifier.yaml", + group='oslo_messaging_notifications') routing_config = r"" config_file = mock.MagicMock() config_file.return_value = routing_config @@ -382,7 +386,8 @@ class TestRoutingNotifier(test_utils.BaseTestCase): self.assertEqual({}, self.router.routing_groups) def test_load_notifiers_config(self): - self.config(routing_notifier_config="routing_notifier.yaml") + self.config(routing_config="routing_notifier.yaml", + group='oslo_messaging_notifications') routing_config = r""" group_1: rpc : foo @@ -412,7 +417,7 @@ group_1: - blah.zoo.* - zip """ - groups = yaml.load(config) + groups = yaml.safe_load(config) group = groups['group_1'] # No matching event ... @@ -443,7 +448,7 @@ group_1: - info - error """ - groups = yaml.load(config) + groups = yaml.safe_load(config) group = groups['group_1'] # No matching priority @@ -476,7 +481,7 @@ group_1: accepted_events: - foo.* """ - groups = yaml.load(config) + groups = yaml.safe_load(config) group = groups['group_1'] # Valid event, but no matching priority @@ -519,7 +524,8 @@ group_1: sorted(pm.map.call_args[0][6])) def test_notify_filtered(self): - self.config(routing_notifier_config="routing_notifier.yaml") + self.config(routing_config="routing_notifier.yaml", + group='oslo_messaging_notifications') routing_config = r""" group_1: rpc: diff --git a/oslo_messaging/tests/rpc/test_dispatcher.py b/oslo_messaging/tests/rpc/test_dispatcher.py index f81be0b9c..672733a05 100644 --- a/oslo_messaging/tests/rpc/test_dispatcher.py +++ b/oslo_messaging/tests/rpc/test_dispatcher.py @@ -133,7 +133,7 @@ class TestDispatcher(test_utils.BaseTestCase): incoming = mock.Mock(ctxt=self.ctxt, message=self.msg) incoming.reply.side_effect = check_reply - callback = dispatcher(incoming) + callback = dispatcher([incoming]) callback.run() callback.done() diff --git a/oslo_messaging/tests/rpc/test_server.py b/oslo_messaging/tests/rpc/test_server.py index b1f8961c5..846ea86e2 100644 --- a/oslo_messaging/tests/rpc/test_server.py +++ b/oslo_messaging/tests/rpc/test_server.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import eventlet +import time import threading from oslo_config import cfg @@ -20,6 +22,7 @@ import testscenarios import mock import oslo_messaging +from oslo_messaging import server as server_module from oslo_messaging.tests import utils as test_utils load_tests = testscenarios.load_tests_apply_scenarios @@ -27,22 +30,38 @@ load_tests = testscenarios.load_tests_apply_scenarios class ServerSetupMixin(object): - class Server(object): + class Server(threading.Thread): def __init__(self, transport, topic, server, endpoint, serializer): + self.controller = ServerSetupMixin.ServerController() target = oslo_messaging.Target(topic=topic, server=server) - self._server = oslo_messaging.get_rpc_server(transport, - target, - [endpoint, self], - serializer=serializer) + self.server = oslo_messaging.get_rpc_server(transport, + target, + [endpoint, + self.controller], + serializer=serializer) + + super(ServerSetupMixin.Server, self).__init__() + self.daemon = True + + def wait(self): + # Wait for the executor to process the stop message, indicating all + # test messages have been processed + self.controller.stopped.wait() + + # Check start() does nothing with a running server + self.server.start() + self.server.stop() + self.server.wait() + + def run(self): + self.server.start() + + class ServerController(object): + def __init__(self): + self.stopped = threading.Event() def stop(self, ctxt): - # Check start() does nothing with a running server - self._server.start() - self._server.stop() - self._server.wait() - - def start(self): - self._server.start() + self.stopped.set() class TestSerializer(object): @@ -72,13 +91,14 @@ class ServerSetupMixin(object): thread.daemon = True thread.start() - return thread + return server - def _stop_server(self, client, server_thread, topic=None): + def _stop_server(self, client, server, topic=None): if topic is not None: client = client.prepare(topic=topic) client.cast({}, 'stop') - server_thread.join(timeout=30) + server.wait() + def _setup_client(self, transport, topic='testtopic'): return oslo_messaging.RPCClient(transport, @@ -117,17 +137,26 @@ class TestRPCServer(test_utils.BaseTestCase, ServerSetupMixin): endpoints = [object()] serializer = object() + class MagicMockIgnoreArgs(mock.MagicMock): + '''A MagicMock which can never misinterpret the arguments passed to + it during construction.''' + + def __init__(self, *args, **kwargs): + super(MagicMockIgnoreArgs, self).__init__() + server = oslo_messaging.get_rpc_server(transport, target, endpoints, serializer=serializer) # Mocking executor - server._executor = mock.Mock() + server._executor_cls = MagicMockIgnoreArgs # Here assigning executor's listener object to listener variable # before calling wait method, because in wait method we are # setting executor to None. - listener = server._executor.listener + server.start() + listener = server._executor_obj.listener + server.stop() # call server wait method server.wait() - self.assertIsNone(server._executor) + self.assertIsNone(server._executor_obj) self.assertEqual(1, listener.cleanup.call_count) def test_no_target_server(self): @@ -502,3 +531,302 @@ class TestMultipleServers(test_utils.BaseTestCase, ServerSetupMixin): TestMultipleServers.generate_scenarios() + +class TestServerLocking(test_utils.BaseTestCase): + def setUp(self): + super(TestServerLocking, self).setUp(conf=cfg.ConfigOpts()) + + def _logmethod(name): + def method(self): + with self._lock: + self._calls.append(name) + return method + + executors = [] + class FakeExecutor(object): + def __init__(self, *args, **kwargs): + self._lock = threading.Lock() + self._calls = [] + self.listener = mock.MagicMock() + executors.append(self) + + start = _logmethod('start') + stop = _logmethod('stop') + wait = _logmethod('wait') + execute = _logmethod('execute') + self.executors = executors + + self.server = oslo_messaging.MessageHandlingServer(mock.Mock(), + mock.Mock()) + self.server._executor_cls = FakeExecutor + + def test_start_stop_wait(self): + # Test a simple execution of start, stop, wait in order + + thread = eventlet.spawn(self.server.start) + self.server.stop() + self.server.wait() + + self.assertEqual(len(self.executors), 1) + executor = self.executors[0] + self.assertEqual(executor._calls, + ['start', 'execute', 'stop', 'wait']) + self.assertTrue(executor.listener.cleanup.called) + + def test_reversed_order(self): + # Test that if we call wait, stop, start, these will be correctly + # reordered + + wait = eventlet.spawn(self.server.wait) + # This is non-deterministic, but there's not a great deal we can do + # about that + eventlet.sleep(0) + + stop = eventlet.spawn(self.server.stop) + eventlet.sleep(0) + + start = eventlet.spawn(self.server.start) + + self.server.wait() + + self.assertEqual(len(self.executors), 1) + executor = self.executors[0] + self.assertEqual(executor._calls, + ['start', 'execute', 'stop', 'wait']) + + def test_wait_for_running_task(self): + # Test that if 2 threads call a method simultaneously, both will wait, + # but only 1 will call the underlying executor method. + + start_event = threading.Event() + finish_event = threading.Event() + + running_event = threading.Event() + done_event = threading.Event() + + runner = [None] + class SteppingFakeExecutor(self.server._executor_cls): + def start(self): + # Tell the test which thread won the race + runner[0] = eventlet.getcurrent() + running_event.set() + + start_event.wait() + super(SteppingFakeExecutor, self).start() + done_event.set() + + finish_event.wait() + self.server._executor_cls = SteppingFakeExecutor + + start1 = eventlet.spawn(self.server.start) + start2 = eventlet.spawn(self.server.start) + + # Wait until one of the threads starts running + running_event.wait() + runner = runner[0] + waiter = start2 if runner == start1 else start2 + + waiter_finished = threading.Event() + waiter.link(lambda _: waiter_finished.set()) + + # At this point, runner is running start(), and waiter() is waiting for + # it to complete. runner has not yet logged anything. + self.assertEqual(1, len(self.executors)) + executor = self.executors[0] + + self.assertEqual(executor._calls, []) + self.assertFalse(waiter_finished.is_set()) + + # Let the runner log the call + start_event.set() + done_event.wait() + + # We haven't signalled completion yet, so execute shouldn't have run + self.assertEqual(executor._calls, ['start']) + self.assertFalse(waiter_finished.is_set()) + + # Let the runner complete + finish_event.set() + waiter.wait() + runner.wait() + + # Check that both threads have finished, start was only called once, + # and execute ran + self.assertTrue(waiter_finished.is_set()) + self.assertEqual(executor._calls, ['start', 'execute']) + + def test_start_stop_wait_stop_wait(self): + # Test that we behave correctly when calling stop/wait more than once. + # Subsequent calls should be noops. + + self.server.start() + self.server.stop() + self.server.wait() + self.server.stop() + self.server.wait() + + self.assertEqual(len(self.executors), 1) + executor = self.executors[0] + self.assertEqual(executor._calls, + ['start', 'execute', 'stop', 'wait']) + self.assertTrue(executor.listener.cleanup.called) + + def test_state_wrapping(self): + # Test that we behave correctly if a thread waits, and the server state + # has wrapped when it it next scheduled + + # Ensure that if 2 threads wait for the completion of 'start', the + # first will wait until complete_event is signalled, but the second + # will continue + complete_event = threading.Event() + complete_waiting_callback = threading.Event() + + start_state = self.server._states['start'] + old_wait_for_completion = start_state.wait_for_completion + waited = [False] + def new_wait_for_completion(*args, **kwargs): + if not waited[0]: + waited[0] = True + complete_waiting_callback.set() + complete_event.wait() + old_wait_for_completion(*args, **kwargs) + start_state.wait_for_completion = new_wait_for_completion + + # thread1 will wait for start to complete until we signal it + thread1 = eventlet.spawn(self.server.stop) + thread1_finished = threading.Event() + thread1.link(lambda _: thread1_finished.set()) + + self.server.start() + complete_waiting_callback.wait() + + # The server should have started, but stop should not have been called + self.assertEqual(1, len(self.executors)) + self.assertEqual(self.executors[0]._calls, ['start', 'execute']) + self.assertFalse(thread1_finished.is_set()) + + self.server.stop() + self.server.wait() + + # We should have gone through all the states, and thread1 should still + # be waiting + self.assertEqual(1, len(self.executors)) + self.assertEqual(self.executors[0]._calls, ['start', 'execute', + 'stop', 'wait']) + self.assertFalse(thread1_finished.is_set()) + + # Start again + self.server.start() + + # We should now record 2 executors + self.assertEqual(2, len(self.executors)) + self.assertEqual(self.executors[0]._calls, ['start', 'execute', + 'stop', 'wait']) + self.assertEqual(self.executors[1]._calls, ['start', 'execute']) + self.assertFalse(thread1_finished.is_set()) + + # Allow thread1 to complete + complete_event.set() + thread1_finished.wait() + + # thread1 should now have finished, and stop should not have been + # called again on either the first or second executor + self.assertEqual(2, len(self.executors)) + self.assertEqual(self.executors[0]._calls, ['start', 'execute', + 'stop', 'wait']) + self.assertEqual(self.executors[1]._calls, ['start', 'execute']) + self.assertTrue(thread1_finished.is_set()) + + @mock.patch.object(server_module, 'DEFAULT_LOG_AFTER', 1) + @mock.patch.object(server_module, 'LOG') + def test_logging(self, mock_log): + # Test that we generate a log message if we wait longer than + # DEFAULT_LOG_AFTER + + log_event = threading.Event() + mock_log.warn.side_effect = lambda _: log_event.set() + + # Call stop without calling start. We should log a wait after 1 second + thread = eventlet.spawn(self.server.stop) + log_event.wait() + + # Redundant given that we already waited, but it's nice to assert + self.assertTrue(mock_log.warn.called) + thread.kill() + + @mock.patch.object(server_module, 'LOG') + def test_logging_explicit_wait(self, mock_log): + # Test that we generate a log message if we wait longer than + # the number of seconds passed to log_after + + log_event = threading.Event() + mock_log.warn.side_effect = lambda _: log_event.set() + + # Call stop without calling start. We should log a wait after 1 second + thread = eventlet.spawn(self.server.stop, log_after=1) + log_event.wait() + + # Redundant given that we already waited, but it's nice to assert + self.assertTrue(mock_log.warn.called) + thread.kill() + + @mock.patch.object(server_module, 'LOG') + def test_logging_with_timeout(self, mock_log): + # Test that we log a message after log_after seconds if we've also + # specified an absolute timeout + + log_event = threading.Event() + mock_log.warn.side_effect = lambda _: log_event.set() + + # Call stop without calling start. We should log a wait after 1 second + thread = eventlet.spawn(self.server.stop, log_after=1, timeout=2) + log_event.wait() + + # Redundant given that we already waited, but it's nice to assert + self.assertTrue(mock_log.warn.called) + thread.kill() + + def test_timeout_wait(self): + # Test that we will eventually timeout when passing the timeout option + # if a preceding condition is not satisfied. + + self.assertRaises(server_module.TaskTimeout, + self.server.stop, timeout=1) + + def test_timeout_running(self): + # Test that we will eventually timeout if we're waiting for another + # thread to complete this task + + # Start the server, which will also instantiate an executor + self.server.start() + + stop_called = threading.Event() + + # Patch the executor's stop method to be very slow + def slow_stop(): + stop_called.set() + eventlet.sleep(10) + self.executors[0].stop = slow_stop + + # Call stop in a new thread + thread = eventlet.spawn(self.server.stop) + + # Wait until the thread is in the slow stop method + stop_called.wait() + + # Call stop again in the main thread with a timeout + self.assertRaises(server_module.TaskTimeout, + self.server.stop, timeout=1) + thread.kill() + + @mock.patch.object(server_module, 'LOG') + def test_log_after_zero(self, mock_log): + # Test that we do not log a message after DEFAULT_LOG_AFTER if the + # caller gave log_after=1 + + # Call stop without calling start. + self.assertRaises(server_module.TaskTimeout, + self.server.stop, log_after=0, timeout=2) + + # We timed out. Ensure we didn't log anything. + self.assertFalse(mock_log.warn.called) diff --git a/oslo_messaging/tests/test_amqp_driver.py b/oslo_messaging/tests/test_amqp_driver.py index 4e3b750fe..909bc599d 100644 --- a/oslo_messaging/tests/test_amqp_driver.py +++ b/oslo_messaging/tests/test_amqp_driver.py @@ -38,6 +38,11 @@ pyngus = importutils.try_import("pyngus") if pyngus: from oslo_messaging._drivers.protocols.amqp import driver as amqp_driver +# The Cyrus-based SASL tests can only be run if the installed version of proton +# has been built with Cyrus SASL support. +_proton = importutils.try_import("proton") +CYRUS_ENABLED = (pyngus and pyngus.VERSION >= (2, 0, 0) and _proton + and getattr(_proton.SASL, "extended", lambda: False)()) LOG = logging.getLogger(__name__) @@ -55,7 +60,7 @@ class _ListenerThread(threading.Thread): def run(self): LOG.debug("Listener started") while self.msg_count > 0: - in_msg = self.listener.poll() + in_msg = self.listener.poll()[0] self.messages.put(in_msg) self.msg_count -= 1 if in_msg.message.get('method') == 'echo': @@ -354,8 +359,7 @@ class TestAuthentication(test_utils.BaseTestCase): driver.cleanup() -@testtools.skipUnless(pyngus and pyngus.VERSION >= (2, 0, 0), - "pyngus module not present") +@testtools.skipUnless(CYRUS_ENABLED, "Cyrus SASL not supported") class TestCyrusAuthentication(test_utils.BaseTestCase): """Test the driver's Cyrus SASL integration""" diff --git a/oslo_messaging/tests/test_fixture.py b/oslo_messaging/tests/test_fixture.py new file mode 100644 index 000000000..2ad1117d6 --- /dev/null +++ b/oslo_messaging/tests/test_fixture.py @@ -0,0 +1,82 @@ +# Copyright 2015 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from oslo_messaging import conffixture +from oslo_messaging.tests import utils as test_utils + + +class TestConfFixture(test_utils.BaseTestCase): + + def test_fixture_wraps_set_override(self): + conf = self.messaging_conf.conf + self.assertIsNotNone(conf.set_override.wrapped) + self.messaging_conf._teardown_decorator() + self.assertFalse(hasattr(conf.set_override, 'wrapped')) + + def test_fixture_wraps_clear_override(self): + conf = self.messaging_conf.conf + self.assertIsNotNone(conf.clear_override.wrapped) + self.messaging_conf._teardown_decorator() + self.assertFalse(hasattr(conf.clear_override, 'wrapped')) + + def test_fixture_setup_teardown_decorator(self): + conf = cfg.ConfigOpts() + self.assertFalse(hasattr(conf.set_override, 'wrapped')) + self.assertFalse(hasattr(conf.clear_override, 'wrapped')) + fixture = conffixture.ConfFixture(conf) + self.assertFalse(hasattr(conf.set_override, 'wrapped')) + self.assertFalse(hasattr(conf.clear_override, 'wrapped')) + self.useFixture(fixture) + self.assertTrue(hasattr(conf.set_override, 'wrapped')) + self.assertTrue(hasattr(conf.clear_override, 'wrapped')) + fixture._teardown_decorator() + self.assertFalse(hasattr(conf.set_override, 'wrapped')) + self.assertFalse(hasattr(conf.clear_override, 'wrapped')) + + def test_fixture_properties(self): + conf = self.messaging_conf.conf + self.messaging_conf.transport_driver = 'fake' + self.assertEqual('fake', + self.messaging_conf.transport_driver) + self.assertEqual('fake', + conf.rpc_backend) + + def test_old_notifications_config_override(self): + conf = self.messaging_conf.conf + conf.set_override( + "notification_driver", "messaging") + conf.set_override( + "notification_transport_url", "http://xyz") + conf.set_override( + "notification_topics", ['topic1']) + + self.assertEqual("messaging", + conf.oslo_messaging_notifications.driver) + self.assertEqual("http://xyz", + conf.oslo_messaging_notifications.transport_url) + self.assertEqual(['topic1'], + conf.oslo_messaging_notifications.topics) + + conf.clear_override("notification_driver") + conf.clear_override("notification_transport_url") + conf.clear_override("notification_topics") + + self.assertEqual([], + conf.oslo_messaging_notifications.driver) + self.assertEqual(None, + conf.oslo_messaging_notifications.transport_url) + self.assertEqual(['notifications'], + conf.oslo_messaging_notifications.topics) \ No newline at end of file diff --git a/oslo_messaging/tests/test_opts.py b/oslo_messaging/tests/test_opts.py index e16145b94..931ded80f 100644 --- a/oslo_messaging/tests/test_opts.py +++ b/oslo_messaging/tests/test_opts.py @@ -32,14 +32,13 @@ class OptsTestCase(test_utils.BaseTestCase): super(OptsTestCase, self).setUp() def _test_list_opts(self, result): - self.assertEqual(5, len(result)) + self.assertEqual(4, len(result)) groups = [g for (g, l) in result] self.assertIn(None, groups) self.assertIn('matchmaker_redis', groups) self.assertIn('oslo_messaging_amqp', groups) self.assertIn('oslo_messaging_rabbit', groups) - self.assertIn('oslo_messaging_qpid', groups) opt_names = [o.name for (g, l) in result for o in l] self.assertIn('rpc_backend', opt_names) diff --git a/oslo_messaging/tests/utils.py b/oslo_messaging/tests/utils.py index 8ea89c5ed..eacfaedc8 100644 --- a/oslo_messaging/tests/utils.py +++ b/oslo_messaging/tests/utils.py @@ -66,7 +66,6 @@ class ServerThreadHelper(threading.Thread): self.daemon = True self._server = server self._stop_event = threading.Event() - self._wait_event = threading.Event() def run(self): self._server.start() @@ -75,7 +74,6 @@ class ServerThreadHelper(threading.Thread): self._server.start() self._server.stop() self._server.wait() - self._wait_event.set() def stop(self): self._stop_event.set() diff --git a/oslo_messaging/transport.py b/oslo_messaging/transport.py index 5a8c3891a..144d1a7f3 100644 --- a/oslo_messaging/transport.py +++ b/oslo_messaging/transport.py @@ -43,7 +43,7 @@ _transport_opts = [ cfg.StrOpt('rpc_backend', default='rabbit', help='The messaging driver to use, defaults to rabbit. Other ' - 'drivers include qpid and zmq.'), + 'drivers include amqp and zmq.'), cfg.StrOpt('control_exchange', default='openstack', help='The default exchange under which topics are scoped. May ' @@ -232,7 +232,7 @@ class TransportURL(object): :param conf: a ConfigOpts instance :type conf: oslo.config.cfg.ConfigOpts - :param transport: a transport name for example 'rabbit' or 'qpid' + :param transport: a transport name for example 'rabbit' :type transport: str :param virtual_host: a virtual host path for example '/' :type virtual_host: str @@ -381,7 +381,6 @@ class TransportURL(object): hosts = [] - username = password = '' for host in url.netloc.split(','): if not host: continue diff --git a/requirements.txt b/requirements.txt index fa33c2437..4799d76d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,19 +5,21 @@ pbr>=1.6 futurist>=0.1.2 # Apache-2.0 -oslo.config>=2.3.0 # Apache-2.0 +oslo.config>=2.7.0 # Apache-2.0 oslo.context>=0.2.0 # Apache-2.0 -oslo.log>=1.8.0 # Apache-2.0 -oslo.utils>=2.4.0 # Apache-2.0 -oslo.serialization>=1.4.0 # Apache-2.0 -oslo.service>=0.10.0 # Apache-2.0 +oslo.log>=1.12.0 # Apache-2.0 +oslo.utils!=3.1.0,>=2.8.0 # Apache-2.0 +oslo.serialization>=1.10.0 # Apache-2.0 +oslo.service>=1.0.0 # Apache-2.0 oslo.i18n>=1.5.0 # Apache-2.0 stevedore>=1.5.0 # Apache-2.0 +debtcollector>=0.3.0 # Apache-2.0 # for jsonutils six>=1.9.0 cachetools>=1.0.0 # MIT License + # FIXME(markmc): remove this when the drivers no longer # import eventlet @@ -37,7 +39,7 @@ pika>=0.10.0 pika-pool>=0.1.3 # middleware -oslo.middleware>=2.8.0 # Apache-2.0 +oslo.middleware>=3.0.0 # Apache-2.0 # needed by the aioeventlet executor aioeventlet>=0.4 diff --git a/setup.cfg b/setup.cfg index d5b4b08bc..d09e6e685 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,10 +26,12 @@ console_scripts = oslo.messaging.drivers = rabbit = oslo_messaging._drivers.impl_rabbit:RabbitDriver - qpid = oslo_messaging._drivers.impl_qpid:QpidDriver zmq = oslo_messaging._drivers.impl_zmq:ZmqDriver amqp = oslo_messaging._drivers.protocols.amqp.driver:ProtonDriver + # This driver is supporting for only notification usage + kafka = oslo_messaging._drivers.impl_kafka:KafkaDriver + # To avoid confusion kombu = oslo_messaging._drivers.impl_rabbit:RabbitDriver diff --git a/test-requirements.txt b/test-requirements.txt index 693f069b1..1387e1a6b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -15,15 +15,15 @@ testscenarios>=0.4 testtools>=1.4.0 oslotest>=1.10.0 # Apache-2.0 -# for test_qpid -qpid-python;python_version=='2.7' - # for test_matchmaker_redis redis>=2.10.0 # for test_impl_zmq pyzmq>=14.3.1 # LGPL+BSD +# for test_impl_kafka +kafka-python>=0.9.2 # Apache-2.0 + # when we can require tox>= 1.4, this can go into tox.ini: # [testenv:cover] # deps = {[testenv]deps} coverage @@ -31,8 +31,11 @@ coverage>=3.6 # this is required for the docs build jobs sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 -oslosphinx>=2.5.0 # Apache-2.0 +oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 # AMQP 1.0 support depends on the Qpid Proton AMQP 1.0 # development libraries. pyngus>=2.0.0 # Apache-2.0 + +# Bandit security code scanner +bandit>=0.13.2 diff --git a/tools/messages_length.yaml b/tools/messages_length.yaml new file mode 100644 index 000000000..092ed513c --- /dev/null +++ b/tools/messages_length.yaml @@ -0,0 +1,13 @@ +# The numbers below present the length of the messages (in string equivalent) +# that were sent through the MQ backend (RabbitMQ) during the +# boot_and_delete_server Rally scenario run (50 times, concurrency equal to 3). +# The information was gathered via adding log to the _send method of +# AMQPDriverBase class after all lines related to the msg object modifications. + +# Message length was gathered to introduce real-like message generator for +# simulator.py oslo.messaging tool, that could introduce traffic closer to the +# real control plane load and estimate both message length and size (in bytes) +# going through the MQ layer. + +test_data: + string_lengths: 806, 992, 992, 1116, 1116, 1191, 1595, 1199, 1043, 1210, 1220, 1191, 1123, 1624, 2583, 1153, 4412, 1642, 1210, 1590, 1500, 1500, 1500, 1500, 1500, 1500, 6386, 6368, 6386, 6368, 6386, 11292, 2136, 5407, 6368, 11292, 2136, 5407, 2116, 2116, 11292, 2136, 5398, 5407, 4357, 5431, 2116, 2116, 5398, 4407, 5431, 2116, 2116, 5398, 4457, 5431, 4387, 2627, 4387, 2094, 2038, 2627, 2094, 2038, 5438, 4387, 5438, 2310, 2310, 2627, 2094, 2496, 2038, 5451, 2310, 5438, 2496, 2496, 2240, 2099, 2240, 1500, 2099, 2626, 5451, 2240, 2626, 1555, 1555, 1702, 1500, 5451, 1702, 2450, 2450, 1570, 1155, 4539, 1570, 4539, 1641, 2099, 1641, 2626, 1555, 1702, 2450, 1570, 3518, 5710, 1641, 2226, 2643, 3382, 6671, 3518, 2531, 2226, 2643, 2124, 3382, 5500, 3518, 2531, 2226, 2643, 965, 2124, 3382, 5500, 6858, 2531, 1177, 965, 2124, 5687, 1177, 965, 1575, 1500, 1500, 2549, 7745, 1575, 5687, 7688, 2183, 1177, 2549, 965, 6574, 7688, 2183, 7270, 2128, 7270, 2128, 1575, 6535, 2549, 6574, 6480, 2643, 2584, 6535, 1220, 2644, 7688, 2183, 1500, 1676, 2611, 1500, 6480, 2611, 2643, 1624, 2241, 1153, 4696, 7270, 2128, 2584, 2644, 1590, 2611, 2611, 1555, 2241, 1555, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4480, 6536, 2298, 2608, 1855, 1880, 2175, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4504, 5431, 4434, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 5687, 1177, 965, 1575, 2549, 6574, 7688, 2183, 1500, 7270, 2128, 1500, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 1575, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4532, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4532, 5431, 4434, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 5687, 1177, 965, 1575, 1575, 2549, 6574, 7688, 2183, 1500, 7270, 2128, 1500, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4532, 6536, 2298, 2608, 1855, 1880, 2175, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4582, 5431, 4484, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 2099, 2626, 5451, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 1575, 5687, 1177, 965, 1575, 2549, 6574, 7688, 2183, 7270, 1500, 2128, 1500, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4582, 6536, 2298, 2608, 1855, 1880, 2175, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4582, 5431, 4484, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 5687, 1177, 965, 1575, 1575, 2549, 6574, 7688, 2183, 7270, 2128, 1500, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1500, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4582, 6536, 2298, 2608, 1855, 1880, 2175, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4582, 5431, 4484, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 5687, 1177, 965, 1575, 1575, 2549, 6574, 7688, 2183, 7270, 2128, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 1500, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4582, 6536, 2298, 2608, 1855, 1880, 1500, 2175, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4582, 5431, 4484, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 5687, 1177, 965, 1575, 1575, 2549, 6574, 7688, 2183, 7270, 2128, 6516, 2300, 6516, 5839, 6156, 6512, 1597, 1500, 1026, 1676, 1500, 6516, 4505, 1220, 2300, 6516, 1624, 6535, 1153, 4668, 5839, 2228, 6156, 1590, 6480, 2643, 6512, 2228, 2584, 1611, 2644, 1102, 1701, 2611, 4354, 2449, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 1575, 2582, 2398, 6386, 2226, 6368, 2093, 3420, 6576, 2142, 4452, 11292, 2136, 6536, 5407, 6386, 6368, 2298, 2116, 2116, 2608, 5398, 1855, 1880, 2175, 4526, 5431, 11292, 2136, 5407, 4456, 2627, 2094, 2038, 2116, 2310, 2496, 5438, 2116, 2240, 5398, 5451, 4604, 5431, 2099, 2626, 1555, 4506, 2627, 1702, 2094, 2038, 5438, 2310, 2450, 2496, 4539, 2240, 1641, 2099, 1500, 1570, 6386, 2626, 5451, 1555, 6368, 1500, 1702, 2450, 11292, 2136, 1570, 5407, 3518, 2116, 2116, 5398, 4539, 2226, 1641, 4604, 2643, 5431, 3382, 3518, 5500, 4506, 2531, 2627, 2094, 2038, 5438, 2226, 2310, 2124, 2643, 3382, 5451, 2496, 5500, 2240, 2531, 2099, 2626, 1555, 5687, 2124, 1177, 1702, 965, 2450, 1570, 4539, 1641, 1575, 3518, 2226, 2643, 3382, 5500, 1575, 5687, 2531, 1177, 965, 6574, 2549, 2124, 1500, 1500, 7688, 2183, 7270, 2128, 1575, 5687, 1177, 2549, 6574, 965, 6535, 7688, 2183, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1500, 1702, 1500, 2450, 1570, 3308, 2043, 3518, 7270, 2128, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 1575, 2549, 6574, 4604, 6535, 6536, 7688, 2183, 2298, 6480, 2643, 2608, 1855, 1880, 2175, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 1500, 1500, 7270, 2128, 2582, 2398, 2226, 2093, 3420, 6576, 1575, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4604, 5431, 2142, 4604, 6535, 6536, 4506, 2627, 2094, 2038, 2298, 6480, 2643, 2310, 5438, 2608, 2496, 1855, 1880, 2175, 2584, 2240, 2644, 2099, 2626, 5451, 2611, 1555, 2611, 2241, 1702, 2450, 1555, 1570, 1702, 2450, 1570, 3308, 2043, 3518, 4539, 1641, 3518, 2582, 2398, 6386, 2226, 6368, 2093, 3420, 6576, 2226, 2643, 3382, 5500, 2142, 4604, 11292, 2136, 6536, 5407, 2531, 2116, 2116, 2124, 5398, 2298, 2608, 1855, 1880, 2175, 4604, 5431, 5687, 1177, 4506, 965, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 1575, 1500, 4539, 1641, 1500, 1575, 2549, 6574, 3518, 7688, 2183, 2226, 2643, 3382, 5500, 2531, 2124, 7270, 2128, 6386, 6368, 11292, 2136, 5407, 5687, 1177, 2116, 2116, 5398, 965, 4604, 6535, 5431, 6480, 2643, 4506, 2584, 2627, 2094, 2644, 2038, 5438, 2611, 2310, 2611, 5451, 2496, 2241, 2240, 1575, 1555, 1702, 2450, 2099, 1570, 2626, 3308, 1555, 2043, 3518, 1702, 4539, 1575, 2450, 1641, 1570, 2549, 1500, 6574, 1500, 1220, 2582, 2398, 2226, 2093, 7688, 2183, 3420, 1624, 6576, 1676, 3518, 1153, 4717, 2142, 1590, 4501, 2226, 6536, 1611, 2643, 7270, 2128, 1102, 1701, 3382, 5500, 2449, 2298, 2608, 1855, 2531, 1880, 2175, 2124, 6535, 6480, 2643, 2584, 5687, 2644, 1177, 2611, 965, 2611, 2241, 1555, 1702, 2450, 6386, 6368, 1570, 3308, 2043, 3518, 11292, 2136, 5407, 2116, 2582, 2116, 2398, 5398, 2226, 2093, 4551, 3420, 6576, 5431, 1575, 1500, 6574, 1500, 4481, 2549, 1575, 2627, 2142, 2094, 2038, 5438, 2310, 2496, 4579, 6536, 2240, 2099, 7688, 2183, 2626, 5451, 1555, 2298, 1702, 2450, 1570, 2608, 1855, 1880, 2175, 7270, 2128, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 5687, 2241, 1177, 965, 1555, 6386, 6368, 1702, 2450, 1570, 11292, 2136, 3308, 5407, 2043, 3518, 2116, 2116, 5398, 2582, 4579, 2398, 5431, 2226, 2093, 3420, 4481, 1500, 6576, 2627, 2094, 2038, 5438, 1500, 2142, 2310, 1575, 1575, 2496, 2240, 6574, 2099, 4579, 2626, 1555, 2549, 5451, 1702, 6536, 2450, 1570, 7688, 2183, 2298, 2608, 1855, 1880, 2175, 3518, 5710, 2226, 1641, 2643, 3382, 6671, 7270, 2128, 2531, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 2124, 4629, 5431, 6535, 4531, 2627, 2094, 2038, 2310, 6480, 2643, 2496, 5438, 6858, 2584, 1177, 2240, 965, 2644, 1500, 2611, 5451, 2611, 2241, 2099, 1500, 2626, 1555, 1555, 1702, 2450, 1702, 1575, 1570, 2450, 4539, 1570, 1641, 3308, 2043, 3518, 1575, 3518, 2549, 7745, 2582, 2398, 2226, 2643, 2226, 7688, 2093, 2183, 3382, 3420, 5500, 6576, 2531, 2124, 2142, 4629, 6536, 2298, 2608, 7270, 2128, 1855, 1880, 2175, 5687, 1177, 965, 6535, 6480, 2643, 2584, 2644, 6386, 6368, 2611, 2611, 2241, 11292, 2136, 5407, 1555, 1500, 1702, 2116, 2116, 1500, 5398, 2450, 1570, 3308, 4629, 2043, 5431, 3518, 1575, 4531, 2549, 2627, 2094, 2038, 5438, 6574, 2582, 2310, 2496, 2398, 5451, 2240, 7688, 2183, 2226, 1575, 2093, 3420, 2099, 2626, 1555, 6576, 1702, 2450, 2142, 1570, 4629, 6536, 4539, 1641, 2298, 2608, 1855, 1880, 2175, 7270, 2128, 3518, 2226, 2643, 3382, 5500, 1500, 2531, 1500, 2124, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 5687, 6386, 1177, 1555, 6368, 965, 1702, 2450, 11292, 1570, 2136, 3308, 5407, 2043, 3518, 2116, 2116, 5398, 1575, 2582, 4679, 2398, 2226, 5431, 2093, 3420, 6576, 4581, 2627, 2094, 2038, 2310, 1575, 2496, 2549, 2142, 5438, 6574, 2240, 4679, 6536, 7688, 2183, 5451, 2099, 2626, 2298, 1555, 2608, 1855, 1880, 2175, 1702, 2450, 1570, 7270, 4539, 1500, 2128, 1641, 1500, 1597, 1066, 3518, 2226, 2643, 3382, 5500, 1220, 2531, 1624, 2124, 1153, 1676, 4818, 6386, 6535, 6368, 1624, 6480, 2643, 2584, 1611, 2644, 5687, 2611, 11292, 2136, 2611, 2241, 1177, 965, 1102, 1701, 5407, 2449, 1555, 1575, 1702, 2116, 2450, 2116, 1570, 5398, 3308, 2043, 3518, 4602, 5431, 2582, 2398, 4532, 2226, 2627, 2094, 2038, 2093, 5438, 2310, 3420, 2496, 6576, 1575, 2240, 5451, 2549, 2142, 6574, 4630, 6536, 2099, 2626, 1500, 7688, 2183, 1500, 4539, 1555, 2298, 1641, 2608, 1702, 1855, 1880, 2175, 2450, 1570, 7270, 2128, 3518, 2226, 2643, 3382, 5500, 2531, 6386, 6368, 6535, 2124, 6480, 2643, 11292, 2136, 2584, 5407, 2644, 2611, 2611, 2241, 2116, 2116, 5687, 5398, 1177, 1555, 965, 1575, 1702, 2450, 4630, 1570, 3308, 5431, 2043, 3518, 4532, 2627, 2094, 2038, 5438, 2310, 2496, 2582, 2398, 2240, 5451, 2226, 2093, 1500, 2099, 3420, 6576, 2626, 1500, 1555, 1575, 6574, 2549, 2142, 1702, 4630, 4539, 2450, 1641, 6536, 1570, 7688, 2183, 2298, 2608, 1855, 1880, 2175, 7270, 2128, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 5687, 2241, 1177, 965, 1555, 1702, 6386, 2450, 6368, 1570, 3308, 2043, 1575, 1500, 3518, 11292, 2136, 5407, 1500, 2582, 2116, 2398, 2116, 2226, 5398, 2093, 3420, 6576, 4680, 5431, 2142, 4680, 6536, 4582, 1575, 2627, 2094, 2038, 5438, 6574, 2549, 2310, 5451, 2496, 2298, 2240, 2608, 1855, 1880, 2175, 7688, 2183, 2099, 2626, 1555, 1702, 2450, 1570, 4539, 1641, 3518, 2226, 2643, 3382, 5500, 7270, 2128, 2531, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4680, 5431, 4582, 1500, 2627, 2094, 2038, 2310, 2124, 2496, 5438, 1500, 2240, 5451, 6535, 2099, 2626, 1555, 5687, 1177, 1702, 965, 6480, 2643, 2450, 2584, 1570, 2644, 2611, 1575, 4539, 2611, 1641, 2241, 1555, 1702, 3518, 2450, 1570, 3308, 1575, 2043, 3518, 2226, 2549, 2643, 6574, 3382, 5500, 2531, 7688, 2183, 2582, 2398, 2124, 2226, 2093, 3420, 6576, 2142, 4680, 6536, 5687, 1177, 2298, 965, 2608, 1855, 1880, 2175, 7270, 2128, 1500, 1500, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4680, 5431, 4582, 1575, 2627, 2094, 2038, 5438, 2549, 6574, 2310, 2496, 5451, 6535, 1575, 2240, 6480, 2643, 2099, 2626, 7688, 2183, 2584, 1555, 2644, 1702, 2611, 2611, 2450, 1570, 2241, 4539, 1641, 1555, 7270, 2128, 1712, 1702, 1154, 2450, 1570, 3308, 2043, 1500, 3518, 3518, 1500, 2582, 2398, 1220, 2226, 2226, 2643, 2093, 1624, 3420, 6576, 3382, 1153, 5500, 6535, 2531, 2124, 4768, 1624, 2142, 1676, 4552, 6480, 6536, 2643, 2584, 2644, 2611, 2298, 2611, 2608, 1855, 1880, 2241, 2175, 5687, 1177, 965, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4552, 1575, 1575, 6536, 6386, 2549, 6368, 6574, 1500, 2298, 1500, 7688, 2183, 2608, 11292, 1855, 1880, 2175, 2136, 5407, 2116, 2116, 5398, 4552, 5431, 7270, 4482, 2128, 2627, 2094, 2038, 2310, 5438, 2496, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 6386, 6368, 6535, 4539, 1641, 11292, 2136, 5407, 6480, 2643, 1575, 2584, 3518, 2644, 2611, 2611, 2116, 2116, 2241, 5398, 2226, 2643, 1555, 1702, 3382, 5500, 4580, 2450, 1570, 5431, 3308, 2043, 2531, 3518, 4482, 2124, 2627, 2094, 2038, 2310, 2496, 5438, 2582, 5451, 2240, 2398, 2226, 5687, 2093, 2099, 3420, 2626, 1177, 1555, 6576, 965, 1702, 2450, 1570, 2142, 4580, 4539, 6536, 1641, 1500, 2298, 1500, 2608, 1855, 1880, 2175, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 1575, 2549, 6574, 5687, 7688, 2183, 1177, 965, 7270, 2128, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4630, 1575, 5431, 1500, 1575, 4532, 1500, 2627, 2094, 2038, 5438, 2310, 2496, 2549, 6574, 6535, 2240, 7688, 2183, 2099, 2626, 5451, 6480, 2643, 1555, 2584, 2644, 1702, 2611, 2450, 1570, 2611, 7270, 2241, 2128, 1555, 1702, 4539, 1641, 2450, 1570, 3308, 2043, 3518, 3518, 6535, 6480, 2643, 2582, 2226, 2398, 2226, 2584, 2644, 2643, 2611, 2093, 2611, 3382, 3420, 2241, 5500, 6576, 1500, 1500, 2531, 1555, 2142, 4630, 6536, 2124, 1702, 2450, 1570, 2298, 5687, 2608, 1855, 1880, 2175, 3308, 2043, 1177, 965, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4630, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 1575, 6386, 6368, 2549, 6574, 11292, 2136, 7688, 2183, 5407, 2116, 2116, 5398, 4630, 5431, 4532, 2627, 2094, 2038, 2310, 5438, 7270, 2496, 2128, 1500, 1500, 2240, 2099, 5451, 2626, 1555, 6386, 6368, 1702, 2450, 1570, 11292, 1575, 2136, 5407, 4539, 2116, 1641, 2116, 5398, 6535, 3518, 6480, 2643, 4630, 5431, 2226, 2643, 2584, 2644, 2611, 3382, 2611, 2241, 5500, 1555, 4532, 2627, 2094, 2038, 2531, 1702, 2310, 2450, 1570, 2496, 2124, 3308, 5438, 2240, 2043, 3518, 2099, 5451, 2626, 1555, 1702, 2582, 2398, 5687, 2450, 2226, 1570, 1177, 965, 2093, 3420, 6576, 2142, 4630, 4539, 6536, 1641, 1500, 3518, 1500, 2298, 2608, 1855, 1880, 2175, 2226, 2643, 1220, 3382, 5500, 1575, 1676, 2531, 2549, 6574, 1624, 2124, 7688, 2183, 1153, 4741, 1590, 1611, 5687, 1102, 1701, 1177, 965, 2449, 1597, 1066, 7270, 2128, 1575, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 4525, 5431, 4455, 2627, 2094, 2038, 5438, 2310, 2496, 1500, 2240, 5451, 1500, 2099, 2626, 1555, 1702, 2450, 1570, 1575, 4539, 1641, 2549, 6574, 6535, 3518, 7688, 2183, 6480, 2643, 2584, 2644, 2226, 2611, 2643, 2611, 3382, 2241, 5500, 1555, 2531, 7270, 2124, 2128, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4553, 6536, 1500, 1500, 2298, 2608, 1855, 1880, 2175, 6535, 5687, 1177, 965, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 1575, 1575, 6574, 6386, 2549, 2142, 6368, 4553, 11292, 2136, 6536, 5407, 7688, 2183, 2116, 2298, 2116, 5398, 2608, 1855, 1880, 2175, 1500, 1500, 7270, 2128, 4553, 5431, 4455, 6386, 6368, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 6535, 5451, 11292, 2136, 6480, 2643, 5407, 2584, 2099, 2116, 2626, 2644, 1555, 2116, 2611, 5398, 1702, 2611, 1575, 2450, 2241, 4539, 4553, 1570, 1555, 1641, 5431, 1702, 2450, 4455, 1570, 2627, 2094, 2038, 3308, 5438, 2310, 2043, 2496, 3518, 2240, 3518, 5451, 2099, 2626, 2226, 2643, 2582, 2398, 3382, 1555, 5500, 2226, 1702, 2093, 2531, 2450, 3420, 1570, 6576, 2124, 4539, 1641, 2142, 4553, 6536, 2298, 3518, 1500, 2608, 1855, 1880, 2175, 1500, 2226, 2643, 3382, 5500, 5687, 2531, 1177, 965, 2124, 6386, 6368, 11292, 2136, 5407, 1575, 5687, 2549, 6574, 1177, 2116, 965, 2116, 7688, 2183, 5398, 4553, 5431, 1575, 4455, 2627, 2094, 2038, 5438, 2310, 2496, 1500, 7270, 1500, 2128, 2240, 5451, 2099, 2626, 1555, 1702, 2450, 1570, 1575, 2549, 4539, 6574, 1641, 6535, 3518, 7688, 2183, 6480, 2643, 2584, 2226, 2644, 2643, 2611, 3382, 2611, 5500, 2241, 1555, 2531, 1702, 2450, 2124, 1570, 7270, 2128, 3308, 2043, 3518, 2582, 1500, 2398, 2226, 1500, 2093, 5687, 3420, 1177, 6576, 2142, 4553, 965, 6536, 6535, 2298, 2608, 6480, 1855, 2643, 1880, 2175, 2584, 2644, 2611, 1220, 2611, 2241, 1555, 1702, 2450, 1570, 1676, 3308, 2043, 3518, 1575, 2582, 2398, 1624, 2226, 2549, 6574, 2093, 3420, 1153, 6386, 6576, 7688, 6368, 2183, 1575, 4767, 1624, 11292, 2136, 5407, 2142, 4551, 1611, 7270, 2128, 1102, 1701, 1500, 2449, 1500, 6536, 2116, 2116, 5398, 2298, 2608, 1855, 1880, 2175, 4551, 5431, 4481, 2627, 2094, 2038, 5438, 2310, 2496, 5451, 6535, 2240, 2099, 6480, 2643, 2626, 1555, 2584, 2644, 1702, 4539, 2611, 6386, 1641, 2450, 2611, 6368, 1570, 2241, 1555, 1575, 1702, 11292, 2450, 1570, 2136, 5407, 3308, 2043, 3518, 2116, 3518, 2116, 5398, 4579, 2582, 2226, 5431, 2398, 2643, 2226, 2093, 3382, 3420, 5500, 4481, 6576, 2627, 2094, 2038, 5438, 2531, 2310, 2496, 5451, 2142, 2124, 4579, 2240, 6536, 2099, 2626, 1555, 2298, 2608, 1702, 1855, 1880, 2175, 2450, 1570, 4539, 1641, 5687, 1500, 1177, 965, 1500, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 1575, 2549, 6574, 7688, 2183, 5687, 1177, 965, 6386, 6368, 11292, 2136, 1575, 5407, 2116, 2116, 5398, 1500, 1500, 4579, 7270, 2128, 5431, 4481, 1575, 2627, 2094, 2038, 5438, 2549, 2310, 6574, 2496, 6535, 5451, 2240, 7688, 2183, 2099, 6457, 2643, 2626, 1555, 2584, 4539, 2644, 2611, 1641, 1702, 7270, 2128, 2611, 2450, 2241, 1570, 1555, 1500, 1500, 1702, 2450, 1570, 3308, 2043, 3518, 3518, 6535, 2582, 2398, 2226, 2643, 6480, 2643, 3382, 2226, 5500, 2584, 2644, 2093, 3420, 2611, 6553, 2531, 2611, 2124, 2241, 2142, 4579, 1555, 6513, 1702, 2298, 2450, 1570, 2608, 1855, 1880, 2175, 3308, 2043, 3518, 5687, 1177, 965, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4579, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 1575, 2549, 6574, 6386, 6368, 7688, 2183, 11292, 2136, 5407, 1500, 2116, 1500, 2116, 5398, 4579, 5431, 4481, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 5451, 2099, 2626, 7270, 2128, 1555, 1575, 1702, 2450, 4539, 1570, 6386, 1641, 6368, 11292, 2136, 6535, 5407, 6480, 2643, 2116, 2116, 3518, 2584, 5398, 2644, 2611, 2226, 2643, 4629, 2611, 5431, 3382, 2241, 5500, 4531, 1555, 2531, 2627, 2094, 2038, 1702, 2310, 5438, 2450, 2496, 2124, 1570, 3308, 2240, 2043, 3518, 5451, 2099, 1500, 2626, 1500, 1555, 5687, 1702, 1177, 2450, 2582, 965, 1570, 2398, 2226, 2093, 3420, 6576, 4539, 1641, 2142, 4629, 6536, 3518, 2298, 2608, 1855, 1880, 2175, 2226, 2643, 3382, 5500, 1575, 1220, 2531, 1676, 2549, 6574, 2124, 1624, 7688, 2183, 1153, 4769, 1624, 1611, 1102, 1701, 5687, 2449, 1177, 1597, 965, 1066, 7270, 2128, 1500, 6386, 1500, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 1575, 4553, 5431, 4483, 2627, 2094, 2038, 5438, 2310, 2496, 2240, 2099, 2626, 1555, 1702, 2450, 1570, 1575, 5451, 6535, 6574, 2549, 6480, 2643, 3518, 2584, 2644, 7688, 2183, 2226, 2611, 2643, 2611, 5710, 2241, 3382, 1641, 1555, 6671, 1702, 2450, 1570, 3308, 2531, 2043, 3518, 2124, 1500, 2582, 1500, 2398, 2226, 2093, 3420, 7270, 2128, 6576, 2142, 6858, 4581, 1177, 6536, 2298, 965, 2608, 6535, 1855, 1880, 2175, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 1575, 2142, 4581, 2549, 7745, 6536, 1575, 2298, 2608, 7688, 2183, 1855, 1880, 2175, 6386, 6368, 1500, 1500, 11292, 2136, 5407, 7270, 2128, 2116, 2116, 5398, 4631, 6386, 6368, 5431, 11292, 2136, 5407, 4533, 2627, 2094, 2038, 2310, 2496, 2116, 5438, 2116, 5398, 2240, 2099, 6535, 2626, 6480, 2643, 5451, 2584, 2644, 4631, 1555, 5431, 2611, 4533, 2627, 2094, 2038, 1702, 2310, 2496, 2611, 2241, 2450, 1570, 2240, 5438, 2099, 2626, 1555, 5451, 1555, 1702, 4539, 1641, 1702, 2450, 2450, 1570, 1570, 3518, 3308, 2043, 3518, 2226, 1575, 2643, 4539, 3382, 5500, 2582, 2398, 3518, 2226, 1641, 2226, 2093, 3420, 2643, 6576, 2531, 3382, 2124, 5500, 2142, 4631, 6536, 2531, 2298, 2608, 1855, 1880, 2175, 2124, 5687, 1177, 965, 1500, 1500, 1575, 5687, 1177, 2549, 965, 6574, 7688, 2183, 7270, 2128, 1575, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 1575, 4631, 1500, 2549, 1500, 5431, 6574, 6535, 4533, 2627, 2094, 2038, 7688, 2183, 2310, 6480, 2643, 2496, 5438, 2240, 2584, 2099, 2626, 2644, 2611, 5451, 1555, 2611, 1702, 2241, 2450, 1570, 1555, 1702, 2450, 1570, 7270, 3308, 2128, 4539, 2043, 3518, 1641, 3518, 2582, 2226, 2398, 2643, 2226, 2093, 3382, 3420, 5500, 6576, 2531, 2142, 4631, 2124, 6536, 6535, 2298, 2608, 6480, 1855, 2643, 1880, 2175, 2584, 2644, 2611, 2611, 2241, 5687, 1177, 1555, 965, 1702, 2450, 1570, 3308, 2043, 3518, 1500, 1500, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4631, 6536, 2298, 1575, 2608, 1855, 1880, 2175, 6574, 1575, 1676, 7688, 2183, 1220, 2549, 1624, 1153, 4691, 6386, 6368, 1590, 1611, 7270, 2128, 1102, 1701, 11292, 2136, 2449, 5407, 1500, 1500, 2116, 2116, 5398, 4549, 5431, 6535, 6386, 6480, 6368, 2643, 4479, 2627, 2094, 2038, 2584, 2644, 5438, 1575, 2310, 5451, 2496, 2611, 2240, 2099, 2611, 2241, 2626, 11292, 2136, 1555, 5407, 1702, 2450, 1555, 1702, 2116, 1570, 2116, 2450, 5398, 4539, 1570, 1641, 4577, 3308, 5431, 2043, 3518, 3518, 4479, 2226, 2627, 2094, 2038, 5438, 2643, 2310, 3382, 5500, 2496, 2582, 5451, 2240, 2398, 2099, 2531, 2626, 1555, 2226, 2093, 1702, 2124, 3420, 2450, 1570, 6576, 2142, 4577, 6536, 4539, 1641, 2298, 5687, 2608, 1855, 1880, 2175, 1177, 965, 3518, 2226, 2643, 3382, 5500, 2531, 2124, 1500, 1500, 1575, 5687, 2549, 1177, 6574, 965, 7688, 2183, 6386, 6368, 1575, 7270, 2128, 11292, 2136, 5407, 2116, 2116, 5398, 4577, 5431, 4479, 1575, 2627, 2094, 2038, 5438, 2549, 2310, 6574, 2496, 6535, 1500, 5451, 1500, 2240, 6480, 2643, 7688, 2183, 2584, 2099, 2644, 2626, 1555, 2611, 2611, 1702, 4539, 2450, 2241, 1570, 1641, 1555, 1702, 2450, 1570, 3308, 7270, 2043, 2128, 3518, 3518, 2582, 2398, 2226, 2226, 2643, 2093, 3382, 3420, 5500, 6576, 2142, 2531, 4577, 6536, 6535, 6480, 2643, 2124, 2584, 2644, 2298, 2608, 2611, 1855, 1880, 2175, 2611, 2241, 1555, 1702, 2450, 1570, 5687, 3308, 1177, 2043, 965, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 1500, 2142, 1500, 4577, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 2549, 6574, 7688, 2183, 1575, 6386, 6368, 11292, 2136, 5407, 2116, 2116, 5398, 7270, 2128, 4627, 5431, 4529, 2627, 2094, 2038, 5438, 2310, 2496, 6386, 6368, 6535, 11292, 2136, 5407, 2240, 2099, 5451, 2626, 6480, 2643, 1555, 2584, 2116, 2644, 1702, 2611, 2116, 2450, 5398, 2611, 1570, 2241, 4539, 4627, 1641, 1555, 1500, 5431, 1500, 1702, 2450, 4529, 1570, 2627, 2094, 3518, 2038, 5438, 3308, 2310, 2043, 3518, 2226, 2496, 2643, 3382, 5451, 1575, 2240, 5500, 2582, 2398, 2226, 2099, 2626, 2093, 3420, 1555, 2531, 6576, 2124, 1702, 4539, 2450, 2142, 1570, 1641, 4627, 6536, 2298, 2608, 1855, 1880, 2175, 5687, 1177, 965, 3518, 2226, 2643, 3382, 5500, 2531, 1575, 2124, 2549, 6574, 6386, 7688, 2183, 6368, 1568, 5687, 1177, 11292, 965, 2136, 5407, 1500, 1500, 2116, 2116, 5398, 7270, 2128, 1712, 1575, 4627, 1154, 5431, 4529, 2627, 2094, 2038, 2310, 5438, 2496, 2240, 5451, 1676, 2099, 2626, 1555, 1220, 1702, 2450, 1575, 1570, 2549, 6574, 6535, 1624, 4539, 7688, 2183, 1641, 1500, 1500, 6480, 2643, 3518, 1153, 2584, 2644, 2226, 4817, 2611, 2643, 2611, 1590, 3382, 2241, 5500, 1624, 1555, 2559, 2561, 2559, 2531, 1702, 2124, 7270, 2579, 2579, 2450, 1611, 1570, 2128, 3308, 1102, 1701, 2449, 2043, 3518, 1597, 1106, 2582, 5687, 2398, 2226, 1177, 2093, 3420, 6576, 965, 6535, 2142, 4601, 6536, 6480, 2643, 2584, 2644, 2298, 1500, 2608, 1500, 2611, 1855, 1880, 2175, 2611, 2241, 1555, 1702, 2450, 1570, 1575, 3308, 2043, 3518, 1575, 2549, 6574, 2582, 2398, 2226, 7688, 2093, 2183, 3420, 6576, 2142, 4601, 6536, 2298, 6386, 2608, 6368, 1855, 1880, 2175, 7270, 2128, 11292, 2136, 5407, 2116, 2116, 5398, 4601, 5431, 4531, 2627, 2094, 2038, 2310, 5438, 2496, 2240, 1500, 5451, 1500, 6535, 2099, 2626, 1555, 6480, 2643, 2584, 1702, 2644, 2450, 2611, 1570, 2611, 2241, 1555, 4539, 1641, 1702, 2450, 1570, 3308, 2043, 3518, 3518, 2582, 2226, 2398, 2643, 2226, 2093, 3382, 3420, 5500, 6576, 2531, 2142, 4629, 2124, 6536, 2298, 2608, 1855, 1880, 2175, 5687, 1177, 965, 1575, 1575, 2549, 6574, 7688, 2183, 7270, 2128, 6535, 6480, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 1500, 1500, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4629, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 7291, 2128, 6534, 6479, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 2093, 3420, 6576, 2142, 4629, 7291, 2128, 6536, 2298, 6534, 2608, 1855, 1880, 2175, 6479, 2643, 2584, 2644, 2611, 2611, 2241, 1555, 1702, 2450, 1570, 3308, 2043, 3518, 2582, 2398, 2226, 1500, 2093, 3420, 1500, 6576, 2142, 4629, 6536, 2298, 2608, 1855, 1880, 2175, 1575, 1500, 1500, 1220, 1624, 1153, 4412, 1676, 1590, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1676, 1220, 1624, 1153, 4412, 1597, 908, 1590, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1676, 1220, 1624, 1153, 1500, 4412, 1500, 1590, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1500, 1597, 908, 1500, 1500, 1676, 1220, 1624, 1153, 4412, 1590, 1500, 1500, 1500, 1500, 1500, 1500 \ No newline at end of file diff --git a/tools/simulator.py b/tools/simulator.py index 8b8fc0bbc..c3f94158f 100755 --- a/tools/simulator.py +++ b/tools/simulator.py @@ -13,14 +13,19 @@ import eventlet eventlet.monkey_patch() -import os - import argparse +import collections import datetime import logging +import os +import random +import string import sys import threading import time +import yaml + +from scipy.stats import rv_discrete from oslo_config import cfg import oslo_messaging as messaging @@ -28,6 +33,8 @@ from oslo_messaging import notify # noqa from oslo_messaging import rpc # noqa LOG = logging.getLogger() +RANDOM_VARIABLE = None +CURRENT_PID = None USAGE = """ Usage: ./simulator.py [-h] [--url URL] [-d DEBUG]\ {notify-server,notify-client,rpc-server,rpc-client} ... @@ -40,6 +47,29 @@ Usage example: --exit-wait 15000 -p 64 -m 64""" +def init_random_generator(): + data = [] + with open('./messages_length.yaml') as m_file: + content = yaml.load(m_file) + data += [int(n) for n in content[ + 'test_data']['string_lengths'].split(', ')] + + ranges = collections.defaultdict(int) + for msg_length in data: + range_start = (msg_length / 500) * 500 + 1 + ranges[range_start] += 1 + + ranges_start = sorted(ranges.keys()) + total_count = len(data) + ranges_dist = [] + for r in ranges_start: + r_dist = float(ranges[r]) / total_count + ranges_dist.append(r_dist) + + random_var = rv_discrete(values=(ranges_start, ranges_dist)) + return random_var + + class LoggingNoParsingFilter(logging.Filter): def filter(self, record): msg = record.getMessage() @@ -49,14 +79,34 @@ class LoggingNoParsingFilter(logging.Filter): return True -class NotifyEndpoint(object): - def __init__(self): +class Monitor(object): + def __init__(self, show_stats=False, *args, **kwargs): + self._count = self._prev_count = 0 + self.show_stats = show_stats + if self.show_stats: + self._monitor() + + def _monitor(self): + threading.Timer(1.0, self._monitor).start() + print ("%d msg was received per second" + % (self._count - self._prev_count)) + self._prev_count = self._count + + def info(self, *args, **kwargs): + self._count += 1 + + +class NotifyEndpoint(Monitor): + def __init__(self, *args, **kwargs): + super(NotifyEndpoint, self).__init__(*args, **kwargs) self.cache = [] def info(self, ctxt, publisher_id, event_type, payload, metadata): + super(NotifyEndpoint, self).info(ctxt, publisher_id, event_type, + payload, metadata) LOG.info('msg rcv') LOG.info("%s %s %s %s" % (ctxt, publisher_id, event_type, payload)) - if payload not in self.cache: + if not self.show_stats and payload not in self.cache: LOG.info('requeue msg') self.cache.append(payload) for i in range(15): @@ -67,8 +117,8 @@ class NotifyEndpoint(object): return messaging.NotificationResult.HANDLED -def notify_server(transport): - endpoints = [NotifyEndpoint()] +def notify_server(transport, show_stats): + endpoints = [NotifyEndpoint(show_stats)] target = messaging.Target(topic='n-t1') server = notify.get_notification_listener(transport, [target], endpoints, executor='eventlet') @@ -76,8 +126,41 @@ def notify_server(transport): server.wait() -class RpcEndpoint(object): - def __init__(self, wait_before_answer): +class BatchNotifyEndpoint(Monitor): + def __init__(self, *args, **kwargs): + super(BatchNotifyEndpoint, self).__init__(*args, **kwargs) + self.cache = [] + + def info(self, messages): + super(BatchNotifyEndpoint, self).info(messages) + self._count += len(messages) - 1 + + LOG.info('msg rcv') + LOG.info("%s" % messages) + if not self.show_stats and messages not in self.cache: + LOG.info('requeue msg') + self.cache.append(messages) + for i in range(15): + eventlet.sleep(1) + return messaging.NotificationResult.REQUEUE + else: + LOG.info('ack msg') + return messaging.NotificationResult.HANDLED + + +def batch_notify_server(transport, show_stats): + endpoints = [BatchNotifyEndpoint(show_stats)] + target = messaging.Target(topic='n-t1') + server = notify.get_batch_notification_listener( + transport, [target], + endpoints, executor='eventlet', + batch_size=1000, batch_time=5) + server.start() + server.wait() + + +class RpcEndpoint(Monitor): + def __init__(self, wait_before_answer, show_stats): self.count = None self.wait_before_answer = wait_before_answer @@ -96,27 +179,8 @@ class RpcEndpoint(object): return "OK: %s" % message -class RpcEndpointMonitor(RpcEndpoint): - def __init__(self, *args, **kwargs): - super(RpcEndpointMonitor, self).__init__(*args, **kwargs) - - self._count = self._prev_count = 0 - self._monitor() - - def _monitor(self): - threading.Timer(1.0, self._monitor).start() - print ("%d msg was received per second" - % (self._count - self._prev_count)) - self._prev_count = self._count - - def info(self, *args, **kwargs): - self._count += 1 - super(RpcEndpointMonitor, self).info(*args, **kwargs) - - def rpc_server(transport, target, wait_before_answer, executor, show_stats): - endpoint_cls = RpcEndpointMonitor if show_stats else RpcEndpoint - endpoints = [endpoint_cls(wait_before_answer)] + endpoints = [RpcEndpoint(wait_before_answer, show_stats)] server = rpc.get_rpc_server(transport, target, endpoints, executor=executor) server.start() @@ -136,9 +200,19 @@ def send_msg(_id, transport, target, messages, wait_after_msg, timeout, client = client.prepare(timeout=timeout) rpc_method = _rpc_cast if is_cast else _rpc_call - for i in range(0, messages): - msg = "test message %d" % i - LOG.info("SEND: %s" % msg) + ranges = RANDOM_VARIABLE.rvs(size=messages) + i = 0 + for range_start in ranges: + length = random.randint(range_start, range_start + 497) + msg = ''.join(random.choice(string.lowercase) for x in range(length)) \ + + ' ' + str(i) + i += 1 + # temporary file to log approximate bytes size of messages + with open('./oslo_%s_%s.log' % (target.topic, CURRENT_PID), 'a+') as f: + # 37 additional bytes for Python String object size canculation. + # In fact we may ignore these bytes, and estimate the data flow + # via number of symbols + f.write(str(length + 37) + '\n') rpc_method(client, msg) if wait_after_msg > 0: time.sleep(wait_after_msg) @@ -197,10 +271,18 @@ def main(): parser.add_argument('-d', '--debug', dest='debug', type=bool, default=False, help="Turn on DEBUG logging level instead of WARN") + parser.add_argument('-tp', '--topic', dest='topic', + default="profiler_topic", + help="Topic to publish/receive messages to/from.") subparsers = parser.add_subparsers(dest='mode', help='notify/rpc server/client mode') server = subparsers.add_parser('notify-server') + server.add_argument('--show-stats', dest='show_stats', + type=bool, default=True) + server = subparsers.add_parser('batch-notify-server') + server.add_argument('--show-stats', dest='show_stats', + type=bool, default=True) client = subparsers.add_parser('notify-client') client.add_argument('-p', dest='threads', type=int, default=1, help='number of client threads') @@ -238,16 +320,20 @@ def main(): _setup_logging(is_debug=args.debug) + if args.mode in ['rpc-server', 'rpc-client']: + transport = messaging.get_transport(cfg.CONF, url=args.url) + else: + transport = messaging.get_notification_transport(cfg.CONF, + url=args.url) + cfg.CONF.oslo_messaging_notifications.topics = "notif" + cfg.CONF.oslo_messaging_notifications.driver = "messaging" + target = messaging.Target(topic=args.topic, server='profiler_server') + # oslo.config defaults cfg.CONF.heartbeat_interval = 5 - cfg.CONF.notification_topics = "notif" - cfg.CONF.notification_driver = "messaging" cfg.CONF.prog = os.path.basename(__file__) cfg.CONF.project = 'oslo.messaging' - transport = messaging.get_transport(cfg.CONF, url=args.url) - target = messaging.Target(topic='profiler_topic', server='profiler_server') - if args.mode == 'rpc-server': if args.url.startswith('zmq'): cfg.CONF.rpc_zmq_matchmaker = "redis" @@ -255,7 +341,9 @@ def main(): rpc_server(transport, target, args.wait_before_answer, args.executor, args.show_stats) elif args.mode == 'notify-server': - notify_server(transport) + notify_server(transport, args.show_stats) + elif args.mode == 'batch-notify-server': + batch_notify_server(transport, args.show_stats) elif args.mode == 'notify-client': threads_spawner(args.threads, notifier, transport, args.messages, args.wait_after_msg, args.timeout) @@ -266,11 +354,29 @@ def main(): args.is_cast) time_ellapsed = (datetime.datetime.now() - start).total_seconds() msg_count = args.messages * args.threads - print ('%d messages was sent for %s seconds. Bandwight is %s msg/sec' - % (msg_count, time_ellapsed, (msg_count / time_ellapsed))) + log_msg = '%d messages was sent for %s seconds. ' \ + 'Bandwidth is %s msg/sec' % (msg_count, time_ellapsed, + (msg_count / time_ellapsed)) + print (log_msg) + with open('./oslo_res_%s.txt' % args.topic, 'a+') as f: + f.write(log_msg + '\n') + + with open('./oslo_%s_%s.log' % (args.topic, CURRENT_PID), 'a+') as f: + data = f.read() + data = [int(i) for i in data.split()] + data_sum = sum(data) + log_msg = '%s bytes were sent for %s seconds. Bandwidth is %s b/s' % ( + data_sum, time_ellapsed, (data_sum / time_ellapsed)) + print(log_msg) + with open('./oslo_res_%s.txt' % args.topic, 'a+') as f: + f.write(log_msg + '\n') + os.remove('./oslo_%s_%s.log' % (args.topic, CURRENT_PID)) + LOG.info("calls finished, wait %d seconds" % args.exit_wait) time.sleep(args.exit_wait) if __name__ == '__main__': + RANDOM_VARIABLE = init_random_generator() + CURRENT_PID = os.getpid() main() diff --git a/tox.ini b/tox.ini index bb4a69a6a..f38dc7e40 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,17 @@ [tox] -envlist = py34,py27,pep8 +envlist = py34,py27,pep8,bandit [testenv] setenv = VIRTUAL_ENV={envdir} OS_TEST_TIMEOUT=30 -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +passend = OS_* +deps = -r{toxinidir}/test-requirements.txt commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:pep8] commands = flake8 +deps = hacking<0.11,>=0.10.0 [testenv:cover] setenv = VIRTUAL_ENV={envdir} @@ -23,10 +24,6 @@ commands = {posargs} [testenv:docs] commands = python setup.py build_sphinx -[testenv:py27-func-qpid] -setenv = TRANSPORT_URL=qpid://stackqpid:secretqpid@127.0.0.1:65123// -commands = {toxinidir}/setup-test-env-qpid.sh 0-10 python setup.py testr --slowest --testr-args='oslo_messaging.tests.functional' - [testenv:py27-func-rabbit] commands = {toxinidir}/setup-test-env-rabbit.sh python setup.py testr --slowest --testr-args='oslo_messaging.tests.functional' @@ -41,6 +38,10 @@ commands = {toxinidir}/setup-test-env-qpid.sh 1.0 python setup.py testr --slowes [testenv:py27-func-zeromq] commands = {toxinidir}/setup-test-env-zmq.sh python setup.py testr --slowest --testr-args='oslo_messaging.tests.functional' +[testenv:bandit] +deps = -r{toxinidir}/test-requirements.txt +commands = bandit -c bandit.yaml -r oslo_messaging -n5 -p oslo.messaging + [flake8] show-source = True ignore = H237,H402,H405,H904