From d501960d7e8474927ebd6d6abc52350a0f23275a Mon Sep 17 00:00:00 2001 From: Brian Rosmaita Date: Tue, 8 May 2018 12:54:24 -0400 Subject: [PATCH] Refactor wait_for_scrubber_shutdown function The wait_for_scrubber_shutdown function catches AssertionErrors and retries. Refactor it so that it only retries for a specific error (namely, when glance scrubber is running in daemon mode when the test is being executed) by changing the function so that it returns values instead of relying on test assertions in the function passed to it. Also refactor the tests that call wait_for_scrubber_shutdown so that they check the returned results instead of using assertions in the function they pass to wait_for_scrubber_shutdown. This is a follow-up to https://review.openstack.org/#/c/566681/ Change-Id: I7108179e0d96e09638ff783b029a8216f0938c3b Closes-bug: #1768077 --- .../tests/functional/serial/test_scrubber.py | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/glance/tests/functional/serial/test_scrubber.py b/glance/tests/functional/serial/test_scrubber.py index 91597051a8..d8c50c5d85 100644 --- a/glance/tests/functional/serial/test_scrubber.py +++ b/glance/tests/functional/serial/test_scrubber.py @@ -263,9 +263,10 @@ class TestScrubber(functional.FunctionalTest): exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable cmd = ("%s --config-file %s --restore %s" % (exe_cmd, self.scrubber_daemon.conf_file_name, image['id'])) - exitcode, out, err = execute(cmd, raise_error=False) - self.assertEqual(0, exitcode) - self.wait_for_scrubber_shutdown(_test_content) + return execute(cmd, raise_error=False) + + exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content) + self.assertEqual(0, exitcode) response, content = self._send_http_request(path, 'GET') image = jsonutils.loads(content) @@ -298,11 +299,12 @@ class TestScrubber(functional.FunctionalTest): exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable cmd = ("%s --config-file %s --restore %s" % (exe_cmd, self.scrubber_daemon.conf_file_name, image['id'])) - exitcode, out, err = execute(cmd, raise_error=False) - self.assertEqual(1, exitcode) - self.assertIn('cannot restore the image from active to active ' - '(wanted from_state=pending_delete)', str(err)) - self.wait_for_scrubber_shutdown(_test_content) + return execute(cmd, raise_error=False) + + exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content) + self.assertEqual(1, exitcode) + self.assertIn('cannot restore the image from active to active ' + '(wanted from_state=pending_delete)', str(err)) self.stop_servers() @@ -317,11 +319,11 @@ class TestScrubber(functional.FunctionalTest): exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable cmd = ("%s --config-file %s --restore fake_image_id" % (exe_cmd, scrubber.conf_file_name)) - exitcode, out, err = execute(cmd, raise_error=False) - self.assertEqual(1, exitcode) - self.assertIn('No image found with ID fake_image_id', str(err)) + return execute(cmd, raise_error=False) - self.wait_for_scrubber_shutdown(_test_content) + exitcode, out, err = self.wait_for_scrubber_shutdown(_test_content) + self.assertEqual(1, exitcode) + self.assertIn('No image found with ID fake_image_id', str(err)) def test_scrubber_restore_image_with_daemon_raise_error(self): @@ -347,18 +349,23 @@ class TestScrubber(functional.FunctionalTest): self.stop_server(self.scrubber_daemon) def wait_for_scrubber_shutdown(self, func): - # NOTE: sometimes the glance-scrubber process which is setup by the - # previous test can't be shutdown immediately, we need wait a second to - # make sure it's down. - for _ in range(15): - try: - func() - break - except AssertionError: + # NOTE(wangxiyuan, rosmaita): The image-restore functionality contains + # a check to make sure the scrubber isn't also running in daemon mode + # to prevent a race condition between a delete and a restore. + # Sometimes the glance-scrubber process which is setup by the + # previous test can't be shutdown immediately, so if we get the "daemon + # running" message we sleep and try again. + not_down_msg = 'The glance-scrubber process is running under daemon' + total_wait = 15 + for _ in range(total_wait): + exitcode, out, err = func() + if exitcode == 1 and not_down_msg in str(err): time.sleep(1) continue + return exitcode, out, err else: - self.fail('unexpected error occurred in glance-scrubber') + self.fail('Scrubber did not shut down within {} sec'.format( + total_wait)) def wait_for_scrub(self, image_id): """