From 98b7ef195c18640b0a036e2ab0f03ec0cb29174d Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Mon, 28 Jan 2019 12:56:49 +0000 Subject: [PATCH] Allow glance tests to run on Windows In order to run the unit and functional Glance tests on Windows, we have to: * avoid monkey patching the os module on Windows (which causes Popen to fail) * update sqlite connection URL * avoid os.fork, not available on Windows. * we'll use subprocess.Popen when spinning up http servers. * for the really simple ones defined in the test helpers, we'll just use threads * do not attempt to connect to '0.0.0.0', use '127.0.0.1' instead * some tests aren't properly skipped (xattr ones), so we're covering that as well * skip log rotation test, we can't move in-use files. Log rotation can be performed by the log handler itself. * expect an exception when hitting connection timeouts * avoid installing unavailable test requirements (xattr, pysendfile) * pin the instance creation timestamp. some tests that deal with markers rely on ordering, which can be flipped if the timestamps are identical (can happen in case of resources created one after the other, not sure yet if this happens really fast or the clock isn't accurate enough). * add a few seconds to some timeouts (much needed when running the tests in VMs). blueprint windows-support Change-Id: Ife69f56a3f9f4d81e1e2e47fde4778efd490938f --- glance/registry/client/v2/api.py | 1 - glance/tests/__init__.py | 10 +- glance/tests/functional/__init__.py | 239 +++++++++++++----- glance/tests/functional/db/base.py | 5 + .../tests/functional/serial/test_scrubber.py | 2 + .../tests/functional/test_cache_middleware.py | 26 +- .../functional/test_client_exceptions.py | 1 - glance/tests/functional/test_logging.py | 6 + glance/tests/functional/test_sqlite.py | 2 +- glance/tests/functional/test_wsgi.py | 8 +- glance/tests/functional/v2/test_images.py | 63 ++--- glance/tests/integration/v2/base.py | 7 +- glance/tests/unit/async_/flows/test_import.py | 4 +- glance/tests/unit/base.py | 6 +- glance/tests/unit/common/test_wsgi.py | 5 +- glance/tests/unit/test_image_cache.py | 7 + glance/tests/unit/v2/test_images_resource.py | 28 +- glance/tests/unit/v2/test_registry_client.py | 2 +- glance/tests/utils.py | 35 ++- 19 files changed, 309 insertions(+), 148 deletions(-) diff --git a/glance/registry/client/v2/api.py b/glance/registry/client/v2/api.py index 69a8016823..0a2397ebb9 100644 --- a/glance/registry/client/v2/api.py +++ b/glance/registry/client/v2/api.py @@ -32,7 +32,6 @@ CONF = cfg.CONF _registry_client = 'glance.registry.client' CONF.import_opt('registry_client_protocol', _registry_client) CONF.import_opt('registry_client_key_file', _registry_client) -CONF.import_opt('registry_client_cert_file', _registry_client) CONF.import_opt('registry_client_ca_file', _registry_client) CONF.import_opt('registry_client_insecure', _registry_client) CONF.import_opt('registry_client_timeout', _registry_client) diff --git a/glance/tests/__init__.py b/glance/tests/__init__.py index 724b7e982c..eab8985891 100644 --- a/glance/tests/__init__.py +++ b/glance/tests/__init__.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +import os + import eventlet # NOTE(jokke): As per the eventlet commit # b756447bab51046dfc6f1e0e299cc997ab343701 there's circular import happening @@ -20,7 +22,13 @@ import eventlet # before calling monkey_patch(). This is solved in eventlet 0.22.0 but we # need to address it before that is widely used around. eventlet.hubs.get_hub() -eventlet.patcher.monkey_patch() + +if os.name == 'nt': + # eventlet monkey patching the os module causes subprocess.Popen to fail + # on Windows when using pipes due to missing non-blocking IO support. + eventlet.patcher.monkey_patch(os=False) +else: + eventlet.patcher.monkey_patch() # See http://code.google.com/p/python-nose/issues/detail?id=373 # The code below enables tests to work with i18n _() blocks diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index ac42b1da26..ff19b93f75 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -21,6 +21,7 @@ and Registry server, grabbing the logs of each, cleaning up pidfiles, and spinning down the servers. """ +import abc import atexit import datetime import errno @@ -28,12 +29,16 @@ import os import platform import shutil import signal +import six import socket +import subprocess import sys import tempfile import time import fixtures +from os_win import utilsfactory as os_win_utilsfactory +from oslo_config import cfg from oslo_serialization import jsonutils # NOTE(jokke): simplified transition to py3, behaves like py2 xrange from six.moves import range @@ -48,8 +53,18 @@ from glance.tests import utils as test_utils execute, get_unused_port = test_utils.execute, test_utils.get_unused_port tracecmd_osmap = {'Linux': 'strace', 'FreeBSD': 'truss'} +if os.name == 'nt': + SQLITE_CONN_TEMPLATE = 'sqlite:///%s/tests.sqlite' +else: + SQLITE_CONN_TEMPLATE = 'sqlite:////%s/tests.sqlite' -class Server(object): + +CONF = cfg.CONF +CONF.import_opt('registry_host', 'glance.registry') + + +@six.add_metaclass(abc.ABCMeta) +class BaseServer(object): """ Class used to easily manage starting and stopping a server during functional test runs. @@ -131,6 +146,78 @@ class Server(object): return self.conf_file_name, overridden + @abc.abstractmethod + def start(self, expect_exit=True, expected_exitcode=0, **kwargs): + pass + + @abc.abstractmethod + def stop(self): + pass + + def reload(self, expect_exit=True, expected_exitcode=0, **kwargs): + """ + Start and stop the service to reload + + Any kwargs passed to this method will override the configuration + value in the conf file used in starting the servers. + """ + self.stop() + return self.start(expect_exit=expect_exit, + expected_exitcode=expected_exitcode, **kwargs) + + def create_database(self): + """Create database if required for this server""" + if self.needs_database: + conf_dir = os.path.join(self.test_dir, 'etc') + utils.safe_mkdirs(conf_dir) + conf_filepath = os.path.join(conf_dir, 'glance-manage.conf') + + with open(conf_filepath, 'w') as conf_file: + conf_file.write('[DEFAULT]\n') + conf_file.write('sql_connection = %s' % self.sql_connection) + conf_file.flush() + + glance_db_env = 'GLANCE_DB_TEST_SQLITE_FILE' + if glance_db_env in os.environ: + # use the empty db created and cached as a tempfile + # instead of spending the time creating a new one + db_location = os.environ[glance_db_env] + shutil.copyfile(db_location, "%s/tests.sqlite" % self.test_dir) + else: + cmd = ('%s -m glance.cmd.manage --config-file %s db sync' % + (sys.executable, conf_filepath)) + execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env, + expect_exit=True) + + # copy the clean db to a temp location so that it + # can be reused for future tests + (osf, db_location) = tempfile.mkstemp() + os.close(osf) + shutil.copyfile('%s/tests.sqlite' % self.test_dir, db_location) + os.environ[glance_db_env] = db_location + + # cleanup the temp file when the test suite is + # complete + def _delete_cached_db(): + try: + os.remove(os.environ[glance_db_env]) + except Exception: + glance_tests.logger.exception( + "Error cleaning up the file %s" % + os.environ[glance_db_env]) + atexit.register(_delete_cached_db) + + def dump_log(self): + if not self.log_file: + return "log_file not set for {name}".format(name=self.server_name) + elif not os.path.exists(self.log_file): + return "{log_file} for {name} did not exist".format( + log_file=self.log_file, name=self.server_name) + with open(self.log_file, 'r') as fptr: + return fptr.read().strip() + + +class PosixServer(BaseServer): def start(self, expect_exit=True, expected_exitcode=0, **kwargs): """ Starts the server. @@ -190,61 +277,6 @@ class Server(object): self.sock = None return (rc, '', '') - def reload(self, expect_exit=True, expected_exitcode=0, **kwargs): - """ - Start and stop the service to reload - - Any kwargs passed to this method will override the configuration - value in the conf file used in starting the servers. - """ - self.stop() - return self.start(expect_exit=expect_exit, - expected_exitcode=expected_exitcode, **kwargs) - - def create_database(self): - """Create database if required for this server""" - if self.needs_database: - conf_dir = os.path.join(self.test_dir, 'etc') - utils.safe_mkdirs(conf_dir) - conf_filepath = os.path.join(conf_dir, 'glance-manage.conf') - - with open(conf_filepath, 'w') as conf_file: - conf_file.write('[DEFAULT]\n') - conf_file.write('sql_connection = %s' % self.sql_connection) - conf_file.flush() - - glance_db_env = 'GLANCE_DB_TEST_SQLITE_FILE' - if glance_db_env in os.environ: - # use the empty db created and cached as a tempfile - # instead of spending the time creating a new one - db_location = os.environ[glance_db_env] - os.system('cp %s %s/tests.sqlite' - % (db_location, self.test_dir)) - else: - cmd = ('%s -m glance.cmd.manage --config-file %s db sync' % - (sys.executable, conf_filepath)) - execute(cmd, no_venv=self.no_venv, exec_env=self.exec_env, - expect_exit=True) - - # copy the clean db to a temp location so that it - # can be reused for future tests - (osf, db_location) = tempfile.mkstemp() - os.close(osf) - os.system('cp %s/tests.sqlite %s' - % (self.test_dir, db_location)) - os.environ[glance_db_env] = db_location - - # cleanup the temp file when the test suite is - # complete - def _delete_cached_db(): - try: - os.remove(os.environ[glance_db_env]) - except Exception: - glance_tests.logger.exception( - "Error cleaning up the file %s" % - os.environ[glance_db_env]) - atexit.register(_delete_cached_db) - def stop(self): """ Spin down the server. @@ -257,14 +289,80 @@ class Server(object): rc = test_utils.wait_for_fork(self.process_pid, raise_error=False) return (rc, '', '') - def dump_log(self): - if not self.log_file: - return "log_file not set for {name}".format(name=self.server_name) - elif not os.path.exists(self.log_file): - return "{log_file} for {name} did not exist".format( - log_file=self.log_file, name=self.server_name) - with open(self.log_file, 'r') as fptr: - return fptr.read().strip() + +class Win32Server(BaseServer): + def __init__(self, *args, **kwargs): + super(Win32Server, self).__init__(*args, **kwargs) + + self._processutils = os_win_utilsfactory.get_processutils() + + def start(self, expect_exit=True, expected_exitcode=0, **kwargs): + """ + Starts the server. + + Any kwargs passed to this method will override the configuration + value in the conf file used in starting the servers. + """ + + # Ensure the configuration file is written + self.write_conf(**kwargs) + + self.create_database() + + cmd = ("%(server_module)s --config-file %(conf_file_name)s" + % {"server_module": self.server_module, + "conf_file_name": self.conf_file_name}) + cmd = "%s -m %s" % (sys.executable, cmd) + + # Passing socket objects on Windows is a bit more cumbersome. + # We don't really have to do it. + if self.sock: + self.sock.close() + self.sock = None + + self.process = subprocess.Popen( + cmd, + env=self.exec_env) + self.process_pid = self.process.pid + + try: + self.job_handle = self._processutils.kill_process_on_job_close( + self.process_pid) + except Exception: + # Could not associate child process with a job, killing it. + self.process.kill() + raise + + self.stop_kill = not expect_exit + if self.pid_file: + pf = open(self.pid_file, 'w') + pf.write('%d\n' % self.process_pid) + pf.close() + + rc = 0 + if expect_exit: + self.process.communicate() + rc = self.process.returncode + + return (rc, '', '') + + def stop(self): + """ + Spin down the server. + """ + if not self.process_pid: + raise Exception('Server "%s" process not running.' + % self.server_name) + + if self.stop_kill: + self.process.terminate() + return (0, '', '') + + +if os.name == 'nt': + Server = Win32Server +else: + Server = PosixServer class ApiServer(Server): @@ -305,7 +403,7 @@ class ApiServer(Server): self.disable_path = None self.needs_database = True - default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir + default_sql_connection = SQLITE_CONN_TEMPLATE % self.test_dir self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION', default_sql_connection) self.data_api = kwargs.get("data_api", @@ -488,7 +586,7 @@ class ApiServerForMultipleBackend(Server): self.disable_path = None self.needs_database = True - default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir + default_sql_connection = SQLITE_CONN_TEMPLATE % self.test_dir self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION', default_sql_connection) self.data_api = kwargs.get("data_api", @@ -646,7 +744,7 @@ class RegistryServer(Server): self.server_module = 'glance.cmd.%s' % self.server_name self.needs_database = True - default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir + default_sql_connection = SQLITE_CONN_TEMPLATE % self.test_dir self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION', default_sql_connection) @@ -732,7 +830,7 @@ class ScrubberDaemon(Server): self.metadata_encryption_key = "012345678901234567890123456789ab" self.lock_path = self.test_dir - default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir + default_sql_connection = SQLITE_CONN_TEMPLATE % self.test_dir self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION', default_sql_connection) self.policy_file = policy_file @@ -790,6 +888,11 @@ class FunctionalTest(test_utils.BaseTestCase): # False in the test SetUps that do not require Scrubber to run. self.include_scrubber = True + # The clients will try to connect to this address. Let's make sure + # we're not using the default '0.0.0.0' + self.config(bind_host='127.0.0.1', + registry_host='127.0.0.1') + self.tracecmd = tracecmd_osmap.get(platform.system()) conf_dir = os.path.join(self.test_dir, 'etc') diff --git a/glance/tests/functional/db/base.py b/glance/tests/functional/db/base.py index 6d8a4dd892..d74db9a247 100644 --- a/glance/tests/functional/db/base.py +++ b/glance/tests/functional/db/base.py @@ -17,6 +17,7 @@ import copy import datetime +import time import uuid import mock @@ -1340,6 +1341,10 @@ class DriverTests(object): 'deleted': False} self.assertEqual(expected, member) + # The clock may not be very accurate, for which reason we may + # get identical timestamps. + time.sleep(0.01) + member = self.db_api.image_member_update(self.context, member_id, {'status': 'accepted'}) diff --git a/glance/tests/functional/serial/test_scrubber.py b/glance/tests/functional/serial/test_scrubber.py index f00504bf61..4f0a5ae116 100644 --- a/glance/tests/functional/serial/test_scrubber.py +++ b/glance/tests/functional/serial/test_scrubber.py @@ -346,6 +346,8 @@ class TestScrubber(functional.FunctionalTest): def test_scrubber_restore_image_with_daemon_running(self): self.cleanup() self.scrubber_daemon.start(daemon=True) + # Give the scrubber some time to start. + time.sleep(5) exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable cmd = ("%s --restore fake_image_id" % exe_cmd) diff --git a/glance/tests/functional/test_cache_middleware.py b/glance/tests/functional/test_cache_middleware.py index 178115ab3f..f6ef309c55 100644 --- a/glance/tests/functional/test_cache_middleware.py +++ b/glance/tests/functional/test_cache_middleware.py @@ -45,7 +45,7 @@ class BaseCacheMiddlewareTest(object): self.start_servers(**self.__dict__.copy()) # Add an image and verify success - path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port) + path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() headers = {'content-type': 'application/json'} image_entity = { @@ -61,7 +61,7 @@ class BaseCacheMiddlewareTest(object): data = jsonutils.loads(content) image_id = data['id'] - path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port, image_id) headers = {'content-type': 'application/octet-stream'} image_data = "*" * FIVE_KB @@ -87,7 +87,7 @@ class BaseCacheMiddlewareTest(object): # Now, we delete the image from the server and verify that # the image cache no longer contains the deleted image - path = "http://%s:%d/v2/images/%s" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') @@ -107,7 +107,7 @@ class BaseCacheMiddlewareTest(object): self.start_servers(**self.__dict__.copy()) # Add an image and verify success - path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port) + path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() headers = {'content-type': 'application/json'} image_entity = { @@ -123,7 +123,7 @@ class BaseCacheMiddlewareTest(object): data = jsonutils.loads(content) image_id = data['id'] - path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port, image_id) headers = {'content-type': 'application/octet-stream'} image_data = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -187,7 +187,7 @@ class BaseCacheMiddlewareTest(object): self.start_servers(**self.__dict__.copy()) # Add an image and verify success - path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port) + path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() headers = {'content-type': 'application/json'} image_entity = { @@ -203,7 +203,7 @@ class BaseCacheMiddlewareTest(object): data = jsonutils.loads(content) image_id = data['id'] - path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port, image_id) headers = {'content-type': 'application/octet-stream'} image_data = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -283,7 +283,7 @@ class BaseCacheMiddlewareTest(object): self.start_servers(**self.__dict__.copy()) # Add an image and verify success - path = "http://%s:%d/v2/images" % ("0.0.0.0", self.api_port) + path = "http://%s:%d/v2/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() headers = {'content-type': 'application/json'} image_entity = { @@ -299,7 +299,7 @@ class BaseCacheMiddlewareTest(object): data = jsonutils.loads(content) image_id = data['id'] - path = "http://%s:%d/v2/images/%s/file" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s/file" % ("127.0.0.1", self.api_port, image_id) headers = {'content-type': 'application/octet-stream'} image_data = "*" * FIVE_KB @@ -324,7 +324,7 @@ class BaseCacheMiddlewareTest(object): # Now, we delete the image from the server and verify that # the image cache no longer contains the deleted image - path = "http://%s:%d/v2/images/%s" % ("0.0.0.0", self.api_port, + path = "http://%s:%d/v2/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') @@ -347,7 +347,7 @@ class TestImageCacheXattr(functional.FunctionalTest, filesystem) """ if getattr(self, 'disabled', False): - return + raise self.skipException('Test disabled.') if not getattr(self, 'inited', False): try: @@ -356,7 +356,7 @@ class TestImageCacheXattr(functional.FunctionalTest, self.inited = True self.disabled = True self.disabled_message = ("python-xattr not installed.") - return + raise self.skipException(self.disabled_message) self.inited = True self.disabled = False @@ -370,7 +370,7 @@ class TestImageCacheXattr(functional.FunctionalTest, self.inited = True self.disabled = True self.disabled_message = ("filesystem does not support xattr") - return + raise self.skipException(self.disabled_message) def tearDown(self): super(TestImageCacheXattr, self).tearDown() diff --git a/glance/tests/functional/test_client_exceptions.py b/glance/tests/functional/test_client_exceptions.py index 72489b9fc8..0b8b5ed06d 100644 --- a/glance/tests/functional/test_client_exceptions.py +++ b/glance/tests/functional/test_client_exceptions.py @@ -28,7 +28,6 @@ from glance.common import wsgi from glance.tests import functional from glance.tests import utils - eventlet.patcher.monkey_patch(socket=True) diff --git a/glance/tests/functional/test_logging.py b/glance/tests/functional/test_logging.py index 135d560182..842b3a7950 100644 --- a/glance/tests/functional/test_logging.py +++ b/glance/tests/functional/test_logging.py @@ -85,6 +85,12 @@ class TestLogging(functional.FunctionalTest): """ Test that we notice when our log file has been rotated """ + + # Moving in-use files is not supported on Windows. + # The log handler itself may be configured to rotate files. + if os.name == 'nt': + raise self.skipException("Unsupported platform.") + self.cleanup() self.start_servers() diff --git a/glance/tests/functional/test_sqlite.py b/glance/tests/functional/test_sqlite.py index 8957573bf1..524a1426c5 100644 --- a/glance/tests/functional/test_sqlite.py +++ b/glance/tests/functional/test_sqlite.py @@ -32,7 +32,7 @@ class TestSqlite(functional.FunctionalTest): self.cleanup() self.start_servers(**self.__dict__.copy()) - cmd = "sqlite3 tests.sqlite '.schema'" + cmd = 'sqlite3 tests.sqlite ".schema"' exitcode, out, err = execute(cmd, raise_error=True) self.assertNotIn('BIGINT', out) diff --git a/glance/tests/functional/test_wsgi.py b/glance/tests/functional/test_wsgi.py index e375e3b8ae..6c6b887859 100644 --- a/glance/tests/functional/test_wsgi.py +++ b/glance/tests/functional/test_wsgi.py @@ -15,6 +15,7 @@ """Tests for `glance.wsgi`.""" +import os import socket import time @@ -52,4 +53,9 @@ class TestWSGIServer(testtools.TestCase): # Should succeed - no timeout self.assertIn(greetings, get_request()) # Should fail - connection timed out so we get nothing from the server - self.assertFalse(get_request(delay=1.1)) + if os.name == 'nt': + self.assertRaises(ConnectionAbortedError, + get_request, + delay=1.1) + else: + self.assertFalse(get_request(delay=1.1)) diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index 7d6d933989..cb566924b1 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -14,8 +14,6 @@ # under the License. import hashlib -import os -import signal import uuid from oslo_serialization import jsonutils @@ -48,16 +46,17 @@ class TestImages(functional.FunctionalTest): for i in range(3): ret = test_utils.start_http_server("foo_image_id%d" % i, "foo_image%d" % i) - setattr(self, 'http_server%d_pid' % i, ret[0]) - setattr(self, 'http_port%d' % i, ret[1]) + setattr(self, 'http_server%d' % i, ret[1]) + setattr(self, 'http_port%d' % i, ret[2]) self.api_server.use_user_token = True self.api_server.send_identity_credentials = True def tearDown(self): for i in range(3): - pid = getattr(self, 'http_server%d_pid' % i, None) - if pid: - os.kill(pid, signal.SIGKILL) + httpd = getattr(self, 'http_server%d' % i, None) + if httpd: + httpd.shutdown() + httpd.server_close() super(TestImages, self).tearDown() @@ -219,7 +218,7 @@ class TestImages(functional.FunctionalTest): func_utils.wait_for_status(request_path=path, request_headers=self._headers(), status='active', - max_sec=2, + max_sec=10, delay_sec=0.2) expect_c = six.text_type(hashlib.md5(image_data).hexdigest()) expect_h = six.text_type(hashlib.sha512(image_data).hexdigest()) @@ -343,7 +342,7 @@ class TestImages(functional.FunctionalTest): }) # Start http server locally - pid, port = test_utils.start_standalone_http_server() + thread, httpd, port = test_utils.start_standalone_http_server() image_data_uri = 'http://localhost:%s/' % port data = jsonutils.dumps({'method': { @@ -373,7 +372,8 @@ class TestImages(functional.FunctionalTest): status='active') # kill the local http server - os.kill(pid, signal.SIGKILL) + httpd.shutdown() + httpd.server_close() # Deleting image should work path = self._url('/v2/images/%s' % image_id) @@ -1609,8 +1609,8 @@ class TestImages(functional.FunctionalTest): path = self._url('/v2/images/%s' % image_id) media_type = 'application/openstack-images-v2.1-json-patch' headers = self._headers({'content-type': media_type}) - http_server_pid, http_port = test_utils.start_http_server(image_id, - "image-1") + thread, httpd, http_port = test_utils.start_http_server(image_id, + "image-1") values = [{'url': 'http://127.0.0.1:%s/image-1' % http_port, 'metadata': {'idx': '0'}}] doc = [{'op': 'replace', @@ -1627,7 +1627,8 @@ class TestImages(functional.FunctionalTest): self.assertEqual(http.OK, response.status_code) # Stop http server used to update image location - os.kill(http_server_pid, signal.SIGKILL) + httpd.shutdown() + httpd.server_close() # Download an image should raise HTTPServiceUnavailable path = self._url('/v2/images/%s/file' % image_id) @@ -3895,14 +3896,15 @@ class TestImageLocationSelectionStrategy(functional.FunctionalTest): for i in range(3): ret = test_utils.start_http_server("foo_image_id%d" % i, "foo_image%d" % i) - setattr(self, 'http_server%d_pid' % i, ret[0]) - setattr(self, 'http_port%d' % i, ret[1]) + setattr(self, 'http_server%d' % i, ret[1]) + setattr(self, 'http_port%d' % i, ret[2]) def tearDown(self): for i in range(3): - pid = getattr(self, 'http_server%d_pid' % i, None) - if pid: - os.kill(pid, signal.SIGKILL) + httpd = getattr(self, 'http_server%d' % i, None) + if httpd: + httpd.shutdown() + httpd.server_close() super(TestImageLocationSelectionStrategy, self).tearDown() @@ -4453,14 +4455,15 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): for i in range(3): ret = test_utils.start_http_server("foo_image_id%d" % i, "foo_image%d" % i) - setattr(self, 'http_server%d_pid' % i, ret[0]) - setattr(self, 'http_port%d' % i, ret[1]) + setattr(self, 'http_server%d' % i, ret[1]) + setattr(self, 'http_port%d' % i, ret[2]) def tearDown(self): for i in range(3): - pid = getattr(self, 'http_server%d_pid' % i, None) - if pid: - os.kill(pid, signal.SIGKILL) + httpd = getattr(self, 'http_server%d' % i, None) + if httpd: + httpd.shutdown() + httpd.server_close() super(TestImagesMultipleBackend, self).tearDown() @@ -4605,7 +4608,7 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): func_utils.wait_for_status(request_path=path, request_headers=self._headers(), status='active', - max_sec=2, + max_sec=15, delay_sec=0.2) expect_c = six.text_type(hashlib.md5(image_data).hexdigest()) expect_h = six.text_type(hashlib.sha512(image_data).hexdigest()) @@ -4766,7 +4769,7 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): func_utils.wait_for_status(request_path=path, request_headers=self._headers(), status='active', - max_sec=2, + max_sec=15, delay_sec=0.2) expect_c = six.text_type(hashlib.md5(image_data).hexdigest()) expect_h = six.text_type(hashlib.sha512(image_data).hexdigest()) @@ -4909,7 +4912,7 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): }) # Start http server locally - pid, port = test_utils.start_standalone_http_server() + thread, httpd, port = test_utils.start_standalone_http_server() image_data_uri = 'http://localhost:%s/' % port data = jsonutils.dumps({'method': { @@ -4939,7 +4942,8 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): status='active') # kill the local http server - os.kill(pid, signal.SIGKILL) + httpd.shutdown() + httpd.server_close() # Ensure image is created in default backend path = self._url('/v2/images/%s' % image_id) response = requests.get(path, headers=self._headers()) @@ -5069,7 +5073,7 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): }) # Start http server locally - pid, port = test_utils.start_standalone_http_server() + thread, httpd, port = test_utils.start_standalone_http_server() image_data_uri = 'http://localhost:%s/' % port data = jsonutils.dumps({'method': { @@ -5099,7 +5103,8 @@ class TestImagesMultipleBackend(functional.MultipleBackendFunctionalTest): status='active') # kill the local http server - os.kill(pid, signal.SIGKILL) + httpd.shutdown() + httpd.server_close() # Ensure image is created in different backend path = self._url('/v2/images/%s' % image_id) diff --git a/glance/tests/integration/v2/base.py b/glance/tests/integration/v2/base.py index 22771024a7..6bcd0c88f1 100644 --- a/glance/tests/integration/v2/base.py +++ b/glance/tests/integration/v2/base.py @@ -15,6 +15,7 @@ import atexit import os.path +import shutil import tempfile import fixtures @@ -163,8 +164,7 @@ class ApiTest(test_utils.BaseTestCase): # use the empty db created and cached as a tempfile # instead of spending the time creating a new one db_location = os.environ[glance_db_env] - test_utils.execute('cp %s %s/tests.sqlite' - % (db_location, self.test_dir)) + shutil.copyfile(db_location, "%s/tests.sqlite" % self.test_dir) else: test_utils.db_sync() @@ -172,8 +172,7 @@ class ApiTest(test_utils.BaseTestCase): # can be reused for future tests (osf, db_location) = tempfile.mkstemp() os.close(osf) - test_utils.execute('cp %s/tests.sqlite %s' - % (self.test_dir, db_location)) + shutil.copyfile('%s/tests.sqlite' % self.test_dir, db_location) os.environ[glance_db_env] = db_location # cleanup the temp file when the test suite is diff --git a/glance/tests/unit/async_/flows/test_import.py b/glance/tests/unit/async_/flows/test_import.py index e3d1a4a150..4998f6e38f 100644 --- a/glance/tests/unit/async_/flows/test_import.py +++ b/glance/tests/unit/async_/flows/test_import.py @@ -135,8 +135,8 @@ class TestImportTask(test_utils.BaseTestCase): self.assertFalse(os.path.exists(tmp_image_path)) self.assertTrue(os.path.exists(image_path)) self.assertEqual(1, len(list(self.image.locations))) - self.assertEqual("file://%s/%s" % (self.test_dir, - self.image.image_id), + self.assertEqual("file://%s%s%s" % (self.test_dir, os.sep, + self.image.image_id), self.image.locations[0]['url']) self._assert_qemu_process_limits(tmock) diff --git a/glance/tests/unit/base.py b/glance/tests/unit/base.py index cc35342a36..de8a24650d 100644 --- a/glance/tests/unit/base.py +++ b/glance/tests/unit/base.py @@ -108,11 +108,9 @@ class IsolatedUnitTest(StoreClearingUnitTest): DEFAULT_REGISTRY_PORT = 9191 DEFAULT_API_PORT = 9292 - if (client.port == DEFAULT_API_PORT and - client.host == '0.0.0.0'): + if client.port == DEFAULT_API_PORT: return stubs.FakeGlanceConnection - elif (client.port == DEFAULT_REGISTRY_PORT and - client.host == '0.0.0.0'): + elif client.port == DEFAULT_REGISTRY_PORT: return stubs.FakeRegistryConnection(registry=self.registry) self.patcher = mock.patch( diff --git a/glance/tests/unit/common/test_wsgi.py b/glance/tests/unit/common/test_wsgi.py index 794873d427..38ec61291b 100644 --- a/glance/tests/unit/common/test_wsgi.py +++ b/glance/tests/unit/common/test_wsgi.py @@ -588,8 +588,11 @@ class ServerTest(test_utils.BaseTestCase): keepalive=False, socket_timeout=900) - def test_number_of_workers(self): + def test_number_of_workers_posix(self): """Ensure the number of workers matches num cpus limited to 8.""" + if os.name == 'nt': + raise self.skipException("Unsupported platform.") + def pid(): i = 1 while True: diff --git a/glance/tests/unit/test_image_cache.py b/glance/tests/unit/test_image_cache.py index 8226d60d47..9890b52245 100644 --- a/glance/tests/unit/test_image_cache.py +++ b/glance/tests/unit/test_image_cache.py @@ -281,6 +281,7 @@ class ImageCacheTestCase(object): self.assertEqual(['0', '1', '2'], self.cache.get_queued_images()) + @skip_if_disabled def test_open_for_write_good(self): """ Test to see if open_for_write works in normal case @@ -300,6 +301,7 @@ class ImageCacheTestCase(object): self.assertFalse(os.path.exists(incomplete_file_path)) self.assertFalse(os.path.exists(invalid_file_path)) + @skip_if_disabled def test_open_for_write_with_exception(self): """ Test to see if open_for_write works in a failure case for each driver @@ -324,6 +326,7 @@ class ImageCacheTestCase(object): self.assertFalse(os.path.exists(incomplete_file_path)) self.assertTrue(os.path.exists(invalid_file_path)) + @skip_if_disabled def test_caching_iterator(self): """ Test to see if the caching iterator interacts properly with the driver @@ -351,6 +354,7 @@ class ImageCacheTestCase(object): self.assertFalse(os.path.exists(incomplete_file_path)) self.assertFalse(os.path.exists(invalid_file_path)) + @skip_if_disabled def test_caching_iterator_handles_backend_failure(self): """ Test that when the backend fails, caching_iter does not continue trying @@ -374,6 +378,7 @@ class ImageCacheTestCase(object): # make sure bad image was not cached self.assertFalse(self.cache.is_cached(image_id)) + @skip_if_disabled def test_caching_iterator_falloffend(self): """ Test to see if the caching iterator interacts properly with the driver @@ -402,6 +407,7 @@ class ImageCacheTestCase(object): self.assertFalse(os.path.exists(incomplete_file_path)) self.assertTrue(os.path.exists(invalid_file_path)) + @skip_if_disabled def test_gate_caching_iter_good_checksum(self): image = b"12345678990abcdefghijklmnop" image_id = 123 @@ -417,6 +423,7 @@ class ImageCacheTestCase(object): # checksum is valid, fake image should be cached: self.assertTrue(cache.is_cached(image_id)) + @skip_if_disabled def test_gate_caching_iter_bad_checksum(self): image = b"12345678990abcdefghijklmnop" image_id = 123 diff --git a/glance/tests/unit/v2/test_images_resource.py b/glance/tests/unit/v2/test_images_resource.py index 45b7951e34..542c103ac4 100644 --- a/glance/tests/unit/v2/test_images_resource.py +++ b/glance/tests/unit/v2/test_images_resource.py @@ -165,7 +165,9 @@ class TestImagesController(base.IsolatedUnitTest): 'metadata': {}, 'status': 'active'}], disk_format='raw', container_format='bare', - status='active'), + status='active', + created_at=DATETIME, + updated_at=DATETIME), _db_fixture(UUID2, owner=TENANT1, checksum=CHKSUM1, os_hash_algo=FAKEHASHALGO, os_hash_value=MULTIHASH2, name='2', size=512, virtual_size=2048, @@ -175,13 +177,19 @@ class TestImagesController(base.IsolatedUnitTest): status='active', tags=['redhat', '64bit', 'power'], properties={'hypervisor_type': 'kvm', 'foo': 'bar', - 'bar': 'foo'}), + 'bar': 'foo'}, + created_at=DATETIME + datetime.timedelta(seconds=1), + updated_at=DATETIME + datetime.timedelta(seconds=1)), _db_fixture(UUID3, owner=TENANT3, checksum=CHKSUM1, os_hash_algo=FAKEHASHALGO, os_hash_value=MULTIHASH2, name='3', size=512, virtual_size=2048, - visibility='public', tags=['windows', '64bit', 'x86']), + visibility='public', tags=['windows', '64bit', 'x86'], + created_at=DATETIME + datetime.timedelta(seconds=2), + updated_at=DATETIME + datetime.timedelta(seconds=2)), _db_fixture(UUID4, owner=TENANT4, name='4', - size=1024, virtual_size=3072), + size=1024, virtual_size=3072, + created_at=DATETIME + datetime.timedelta(seconds=3), + updated_at=DATETIME + datetime.timedelta(seconds=3)), ] [self.db.image_create(None, image) for image in self.images] @@ -4649,7 +4657,8 @@ class TestMultiImagesController(base.MultiIsolatedUnitTest): 'metadata': {}, 'status': 'active'}], disk_format='raw', container_format='bare', - status='active'), + status='active', + created_at=DATETIME), _db_fixture(UUID2, owner=TENANT1, checksum=CHKSUM1, name='2', size=512, virtual_size=2048, visibility='public', @@ -4658,12 +4667,15 @@ class TestMultiImagesController(base.MultiIsolatedUnitTest): status='active', tags=['redhat', '64bit', 'power'], properties={'hypervisor_type': 'kvm', 'foo': 'bar', - 'bar': 'foo'}), + 'bar': 'foo'}, + created_at=DATETIME + datetime.timedelta(seconds=1)), _db_fixture(UUID3, owner=TENANT3, checksum=CHKSUM1, name='3', size=512, virtual_size=2048, - visibility='public', tags=['windows', '64bit', 'x86']), + visibility='public', tags=['windows', '64bit', 'x86'], + created_at=DATETIME + datetime.timedelta(seconds=2)), _db_fixture(UUID4, owner=TENANT4, name='4', - size=1024, virtual_size=3072), + size=1024, virtual_size=3072, + created_at=DATETIME + datetime.timedelta(seconds=3)), ] [self.db.image_create(None, image) for image in self.images] diff --git a/glance/tests/unit/v2/test_registry_client.py b/glance/tests/unit/v2/test_registry_client.py index 2401dd54e1..96d3380563 100644 --- a/glance/tests/unit/v2/test_registry_client.py +++ b/glance/tests/unit/v2/test_registry_client.py @@ -79,7 +79,7 @@ class TestRegistryV2Client(base.IsolatedUnitTest, created_at=uuid2_time)] self.destroy_fixtures() self.create_fixtures() - self.client = rclient.RegistryClient("0.0.0.0") + self.client = rclient.RegistryClient("127.0.0.1") def tearDown(self): """Clear the test environment""" diff --git a/glance/tests/utils.py b/glance/tests/utils.py index 140a7723a9..e8cff70486 100644 --- a/glance/tests/utils.py +++ b/glance/tests/utils.py @@ -22,6 +22,7 @@ import shlex import shutil import socket import subprocess +import threading from alembic import command as alembic_command import fixtures @@ -176,7 +177,11 @@ class depends_on_exe(object): def __call__(self, func): def _runner(*args, **kw): - cmd = 'which %s' % self.exe + if os.name != 'nt': + cmd = 'which %s' % self.exe + else: + cmd = 'where.exe', '%s' % self.exe + exitcode, out, err = execute(cmd, raise_error=False) if exitcode != 0: args[0].disabled_message = 'test requires exe: %s' % self.exe @@ -325,7 +330,11 @@ def execute(cmd, path_ext = [os.path.join(os.getcwd(), 'bin')] # Also jack in the path cmd comes from, if it's absolute - args = shlex.split(cmd) + if os.name != 'nt': + args = shlex.split(cmd) + else: + args = cmd + executable = args[0] if os.path.isabs(executable): path_ext.append(os.path.dirname(executable)) @@ -484,7 +493,7 @@ def start_http_server(image_id, image_data): self.send_response(http.OK) self.send_header('Content-Length', str(len(fixture))) self.end_headers() - self.wfile.write(fixture) + self.wfile.write(six.b(fixture)) return def do_HEAD(self): @@ -510,11 +519,11 @@ def start_http_server(image_id, image_data): httpd = BaseHTTPServer.HTTPServer(server_address, handler_class) port = httpd.socket.getsockname()[1] - pid = os.fork() - if pid == 0: - httpd.serve_forever() - else: - return pid, port + thread = threading.Thread(target=httpd.serve_forever) + thread.daemon = True + thread.start() + + return thread, httpd, port class RegistryAPIMixIn(object): @@ -730,8 +739,8 @@ def start_standalone_http_server(): httpd = BaseHTTPServer.HTTPServer(server_address, handler_class) port = httpd.socket.getsockname()[1] - pid = os.fork() - if pid == 0: - httpd.serve_forever() - else: - return pid, port + thread = threading.Thread(target=httpd.serve_forever) + thread.daemon = True + thread.start() + + return thread, httpd, port