# Copyright (c) 2010-2013 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import unittest import threading import six from concurrent.futures import as_completed from six.moves.queue import Queue, Empty from time import sleep from swiftclient import multithreading as mt from .utils import CaptureStream class ThreadTestCase(unittest.TestCase): def setUp(self): super(ThreadTestCase, self).setUp() self.got_items = Queue() self.got_args_kwargs = Queue() self.starting_thread_count = threading.active_count() def _func(self, conn, item, *args, **kwargs): self.got_items.put((conn, item)) self.got_args_kwargs.put((args, kwargs)) if item == 'sleep': sleep(1) if item == 'go boom': raise Exception('I went boom!') return 'success' def _create_conn(self): return "This is a connection" def _create_conn_fail(self): raise Exception("This is a failed connection") def assertQueueContains(self, queue, expected_contents): got_contents = [] try: while True: got_contents.append(queue.get(timeout=0.1)) except Empty: pass if isinstance(expected_contents, set): got_contents = set(got_contents) self.assertEqual(expected_contents, got_contents) class TestConnectionThreadPoolExecutor(ThreadTestCase): def setUp(self): super(TestConnectionThreadPoolExecutor, self).setUp() self.input_queue = Queue() self.stored_results = [] def tearDown(self): super(TestConnectionThreadPoolExecutor, self).tearDown() def test_submit_good_connection(self): ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 1) with ctpe as pool: # Try submitting a job that should succeed f = pool.submit(self._func, "succeed") f.result() self.assertQueueContains( self.got_items, [("This is a connection", "succeed")] ) # Now a job that fails try: f = pool.submit(self._func, "go boom") f.result() except Exception as e: self.assertEqual('I went boom!', str(e)) else: self.fail('I never went boom!') # Has the connection been returned to the pool? f = pool.submit(self._func, "succeed") f.result() self.assertQueueContains( self.got_items, [ ("This is a connection", "go boom"), ("This is a connection", "succeed") ] ) def test_submit_bad_connection(self): ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn_fail, 1) with ctpe as pool: # Now a connection that fails try: f = pool.submit(self._func, "succeed") f.result() except Exception as e: self.assertEqual('This is a failed connection', str(e)) else: self.fail('The connection did not fail') # Make sure we don't lock up on failed connections try: f = pool.submit(self._func, "go boom") f.result() except Exception as e: self.assertEqual('This is a failed connection', str(e)) else: self.fail('The connection did not fail') def test_lazy_connections(self): ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) with ctpe as pool: # Submit multiple jobs sequentially - should only use 1 conn f = pool.submit(self._func, "succeed") f.result() f = pool.submit(self._func, "succeed") f.result() f = pool.submit(self._func, "succeed") f.result() expected_connections = [(0, "This is a connection")] expected_connections.extend([(x, None) for x in range(1, 10)]) self.assertQueueContains( pool._connections, expected_connections ) ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) with ctpe as pool: fs = [] f1 = pool.submit(self._func, "sleep") f2 = pool.submit(self._func, "sleep") f3 = pool.submit(self._func, "sleep") fs.extend([f1, f2, f3]) expected_connections = [ (0, "This is a connection"), (1, "This is a connection"), (2, "This is a connection") ] expected_connections.extend([(x, None) for x in range(3, 10)]) for f in as_completed(fs): f.result() self.assertQueueContains( pool._connections, expected_connections ) class TestOutputManager(unittest.TestCase): def test_instantiation(self): output_manager = mt.OutputManager() self.assertEqual(sys.stdout, output_manager.print_stream) self.assertEqual(sys.stderr, output_manager.error_stream) def test_printers(self): out_stream = CaptureStream(sys.stdout) err_stream = CaptureStream(sys.stderr) starting_thread_count = threading.active_count() with mt.OutputManager( print_stream=out_stream, error_stream=err_stream) as thread_manager: # Sanity-checking these gives power to the previous test which # looked at the default values of thread_manager.print/error_stream self.assertEqual(out_stream, thread_manager.print_stream) self.assertEqual(err_stream, thread_manager.error_stream) # No printing has happened yet, so no new threads self.assertEqual(starting_thread_count, threading.active_count()) thread_manager.print_msg('one-argument') thread_manager.print_msg('one %s, %d fish', 'fish', 88) thread_manager.error('I have %d problems, but a %s is not one', 99, u'\u062A\u062A') thread_manager.print_msg('some\n%s\nover the %r', 'where', u'\u062A\u062A') thread_manager.error('one-error-argument') thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', 3.14159) thread_manager.print_raw( u'some raw bytes: \u062A\u062A'.encode('utf-8')) thread_manager.print_items([ ('key', 'value'), ('object', u'O\u0308bject'), ]) thread_manager.print_raw(b'\xffugly\xffraw') # Now we have a thread for error printing and a thread for # normal print messages self.assertEqual(starting_thread_count + 2, threading.active_count()) # The threads should have been cleaned up self.assertEqual(starting_thread_count, threading.active_count()) if six.PY3: over_the = "over the '\u062a\u062a'\n" else: over_the = "over the u'\\u062a\\u062a'\n" # We write to the CaptureStream so no decoding is performed self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', 'some\n', 'where\n', over_the, u'some raw bytes: \u062a\u062a', ' key: value\n', u' object: O\u0308bject\n' ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) self.assertEqual(''.join([ u'I have 99 problems, but a \u062A\u062A is not one\n', 'one-error-argument\n', 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' ]), err_stream.getvalue().decode('utf8')) self.assertEqual(3, thread_manager.error_count)