From cd7b36904ef47bbb2f97bc6a9c129007faeecfb2 Mon Sep 17 00:00:00 2001 From: Dmitry Bogun Date: Fri, 9 Dec 2016 17:02:07 +0200 Subject: [PATCH] Refactoring of stub webserver Main goals - avoid "path math". There was an ugly path math in stub webserver start phase. Also there was mostly same path math in lookup of static files served by this stub webserver. Change-Id: Id0c348ceefb257dff0950adcd778edd69e6ad11e --- ramdisk_func_test/environment.py | 33 +-- ramdisk_func_test/webserver/__init__.py | 188 ++++++++++++++++++ .../webserver/{ => data}/stubfile | 0 ramdisk_func_test/webserver/server.py | 158 --------------- setup.py | 11 +- 5 files changed, 218 insertions(+), 172 deletions(-) rename ramdisk_func_test/webserver/{ => data}/stubfile (100%) delete mode 100755 ramdisk_func_test/webserver/server.py diff --git a/ramdisk_func_test/environment.py b/ramdisk_func_test/environment.py index 49441c0..84857d2 100644 --- a/ramdisk_func_test/environment.py +++ b/ramdisk_func_test/environment.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import os import shutil import subprocess @@ -26,7 +27,6 @@ from oslo_config import cfg from ramdisk_func_test import conf from ramdisk_func_test import utils from ramdisk_func_test.base import TemplateEngine -from ramdisk_func_test.base import ABS_PATH from ramdisk_func_test.network import Network from ramdisk_func_test.node import Node @@ -157,13 +157,11 @@ class Environment(object): port = CONF.stub_webserver_port LOG.info("Starting stub webserver (at IP {0} port {1}, path to tenant " "images folder is '{2}')".format(self.network.address, port, - self.tenant_images_dir)) + CONF.tenant_images_dir)) # TODO(max_lobur) make webserver singletone - self.webserver = subprocess.Popen( - ['python', - os.path.join(ABS_PATH, 'webserver/server.py'), - self.network.address, port, self.tenant_images_dir], shell=False) + cmd = ['ramdisk-stub-webserver', self.network.address, str(port)] + self.webserver = subprocess.Popen(cmd, shell=False) def get_url_for_image(self, image_name, source_type): if source_type == 'swift': @@ -208,14 +206,23 @@ class Environment(object): def _teardown_webserver(self): LOG.info("Stopping stub web server ...") - self.webserver.terminate() - - for i in range(0, 15): - if self.webserver.poll() is not None: - LOG.info("Stub web server has stopped.") + try: + self.webserver.terminate() + for i in range(0, 15): + if self.webserver.poll() is not None: + LOG.info("Stub web server has stopped.") + break + time.sleep(1) + else: + LOG.warning( + '15 seconds have passed since sending SIGTERM to the stub ' + 'web server. It is still alive. Send SIGKILL.') + self.webserver.kill() + self.webserver.wait() # collect zombie + except OSError as e: + if e.errno == errno.ESRCH: return - time.sleep(1) - LOG.warning("Cannot terminate web server in 15 sec!") + raise def _delete_workdir(self): LOG.info("Deleting workdir {0}".format(CONF.ramdisk_func_test_workdir)) diff --git a/ramdisk_func_test/webserver/__init__.py b/ramdisk_func_test/webserver/__init__.py index e69de29..3420f0c 100644 --- a/ramdisk_func_test/webserver/__init__.py +++ b/ramdisk_func_test/webserver/__init__.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# +# Copyright 2016 Cray Inc., All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Mock web-server. + +It gets 2 positional parameters: +- host +- port + +For GET requests: +- If URL contains '/fake' at the beginning, mock web-server returns content + of ./stubfile +- If URL is like '/tenant_images/' at the beginning, mock web-server + returns content of file from folder specified in third positional + parameter +- Fof all other cases, it tries to return appropriate file (e.g. + '/tmp/banana.txt' for URL 'http://host:port/tmp/banana.txt') + +For POST requests: +- If URL contains '/v1/nodes//vendor_passthru', mock web-server + creates empty file with 'callback' name at subfolder named by , in + fpa_func_test working dir (and returns 202, with content of ./stubfile) +""" + +import argparse +import os +import SimpleHTTPServer +import SocketServer +import logging +import pkg_resources +import signal +import sys +import re + +from ramdisk_func_test import conf + + +logging.basicConfig(filename='/tmp/mock-web-server.log', + level=logging.DEBUG, + format='%(asctime)s %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p') + +CONF = conf.CONF +CONF.import_opt('tenant_images_dir', 'ramdisk_func_test.environment') +CONF.import_opt('ramdisk_func_test_workdir', 'ramdisk_func_test.utils') +LOG = logging.getLogger(__name__) + +httpd = None + + +class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + def __init__(self, ctx, *args, **kwargs): + self.ctx = ctx + SimpleHTTPServer.SimpleHTTPRequestHandler.__init__( + self, *args, **kwargs) + + def do_GET(self): + LOG.info('Got GET request: %s', self.path) + fake_check = re.match(r'/fake', self.path) + tenant_images_check = re.match(r'/tenant_images/(.*)$', self.path) + + if fake_check is not None: + LOG.info("This is 'fake' request.") + self.path = os.path.join(self.ctx.htdocs, 'stubfile') + elif tenant_images_check is not None: + LOG.info("This is 'tenant-images' request: %s", self.path) + tenant_image = tenant_images_check.group(1) + self.path = os.path.join(self.ctx.images_path, tenant_image) + + return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + + def do_POST(self): + callback_check = re.search( + r'/v1/nodes/([^/]*)/vendor_passthru', self.path) + + if callback_check is not None: + callback_file_path = os.path.join( + CONF.ramdisk_func_test_workdir, callback_check.group(1), + 'callback') + open(callback_file_path, 'a').close() + LOG.info("Got callback: %s", self.path) + + self.path = os.path.join(self.ctx.htdocs, 'stubfile') + return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + + def send_head(self): + """Common code for GET and HEAD commands. + + This sends the response code and MIME headers. + + Return value is either a file object (which has to be copied + to the output file by the caller unless the command was HEAD, + and must be closed by the caller under all circumstances), or + None, in which case the caller has nothing further to do. + + """ + path = self.path + ctype = self.guess_type(path) + try: + # Always read in binary mode. Opening files in text mode may cause + # newline translations, making the actual size of the content + # transmitted *less* than the content-length! + payload = open(path, 'rb') + except IOError: + self.send_error(404, "File not found ({0})".format(path)) + return None + + if self.command == 'POST': + self.send_response(202) + else: + self.send_response(200) + + stat = os.fstat(payload.fileno()) + + self.send_header("Content-type", ctype) + self.send_header("Content-Length", str(stat.st_size)) + self.send_header("Last-Modified", self.date_time_string(stat.st_mtime)) + self.end_headers() + + return payload + + +class HandlerFactory(object): + def __init__(self, ctx, halder_class): + self.ctx = ctx + self.handler_class = halder_class + + def __call__(self, *args, **kwargs): + return self.handler_class(self.ctx, *args, **kwargs) + + +class Context(object): + def __init__(self): + self.images_path = CONF.tenant_images_dir + self.htdocs = pkg_resources.resource_filename(__name__, 'data') + + +def signal_term_handler(signal, frame): + LOG.info("ramdisk-func-test stub web server terminating ...") + try: + httpd.server_close() + except Exception: + LOG.error('Cannot close server!', exc_info=True) + sys.exit(1) + LOG.info("ramdisk-func-test stub web server has terminated.") + sys.exit(0) + + +def main(): + global httpd + + argp = argparse.ArgumentParser() + argp.add_argument('address', help='Bind address') + argp.add_argument('port', help='Bind port', type=int) + + cli = argp.parse_args() + + bind = (cli.address, cli.port) + handler = HandlerFactory(Context(), RequestHandler) + try: + SocketServer.TCPServer.allow_reuse_address = True + httpd = SocketServer.TCPServer(bind, handler) + except Exception: + LOG.error('=' * 80) + LOG.error('Error in webserver start stage', exc_info=True) + sys.exit(1) + + LOG.info('=' * 80) + LOG.info('ramdisk-func-test stub webserver started at %s:%s ' + '(tenant-images path is %s)', + cli.address, cli.port, handler.ctx.images_path) + + signal.signal(signal.SIGTERM, signal_term_handler) + httpd.serve_forever() diff --git a/ramdisk_func_test/webserver/stubfile b/ramdisk_func_test/webserver/data/stubfile similarity index 100% rename from ramdisk_func_test/webserver/stubfile rename to ramdisk_func_test/webserver/data/stubfile diff --git a/ramdisk_func_test/webserver/server.py b/ramdisk_func_test/webserver/server.py deleted file mode 100755 index a5744c2..0000000 --- a/ramdisk_func_test/webserver/server.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Cray Inc., All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import os -import SimpleHTTPServer -import SocketServer -import logging -import signal -import sys -import traceback -import re - -from ramdisk_func_test import conf -from ramdisk_func_test.base import ABS_PATH - - -CONF = conf.CONF -LOG = logging.getLogger(__name__) -logging.basicConfig(filename='/tmp/mock-web-server.log', - level=logging.DEBUG, - format='%(asctime)s %(message)s', - datefmt='%m/%d/%Y %I:%M:%S %p') - - -class MyRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - - path_to_images_folder = None - - @classmethod - def _set_path_to_images_folder(cls, path): - cls.path_to_images_folder = path - - def do_GET(self): - - LOG.info("Got GET request: {0} ".format(self.path)) - fake_check = re.match(r'/fake', self.path) - tenant_images_check = re.match(r'/tenant_images/', self.path) - - if fake_check is not None: - LOG.info("This is 'fake' request.") - self.path = os.path.join(ABS_PATH, 'webserver', 'stubfile') - elif tenant_images_check is not None: - LOG.info("This is 'tenant-images' request: {0} ".format(self.path)) - tenant_images_name = re.match( - r'/tenant_images/(.*)', self.path).group(1) - self.path = os.path.join( - self.path_to_images_folder, tenant_images_name) - - return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) - - def do_POST(self): - - callback_check = re.search( - r'/v1/nodes/([^/]*)/vendor_passthru', self.path) - - if callback_check is not None: - callback_file_path = os.path.join( - CONF.ramdisk_func_test_workdir, callback_check.group(1), - 'callback') - open(callback_file_path, 'a').close() - LOG.info("Got callback: {0} ".format(self.path)) - - self.path = os.path.join(ABS_PATH, 'webserver', 'stubfile') - return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) - - def send_head(self): - """Common code for GET and HEAD commands. - - This sends the response code and MIME headers. - - Return value is either a file object (which has to be copied - to the output file by the caller unless the command was HEAD, - and must be closed by the caller under all circumstances), or - None, in which case the caller has nothing further to do. - - """ - f = None - path = self.path - ctype = self.guess_type(path) - try: - # Always read in binary mode. Opening files in text mode may cause - # newline translations, making the actual size of the content - # transmitted *less* than the content-length! - f = open(path, 'rb') - except IOError: - self.send_error(404, "File not found ({0})".format(path)) - return None - - if self.command == 'POST': - self.send_response(202) - else: - self.send_response(200) - - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - self.end_headers() - return f - - -Handler = MyRequestHandler - -httpd = None - - -def signal_term_handler(s, f): - LOG.info("ramdisk-func-test stub web server terminating ...") - try: - httpd.server_close() - except Exception: - LOG.error("Cannot close server!") - sys.exit(1) - LOG.info("ramdisk-func-test stub web server has terminated.") - sys.exit(0) - -signal.signal(signal.SIGTERM, signal_term_handler) - -if __name__ == "__main__": - - try: - host = sys.argv[1] - port = int(sys.argv[2]) - path_to_images_folder = sys.argv[3] - except IndexError: - LOG.error("Mock web-server cannot get enough valid parameters!") - exit(1) - - Handler._set_path_to_images_folder(path_to_images_folder) - - try: - SocketServer.TCPServer.allow_reuse_address = True - httpd = SocketServer.TCPServer((host, port), Handler) - except Exception: - LOG.error("="*80) - LOG.error("Cannot start: {0}".format(traceback.format_exc())) - exit(1) - - LOG.info("="*80) - LOG.info("ramdisk-func-test stub webserver started at {0}:{1} " - "(tenant-images path is '{2}')".format(host, port, - path_to_images_folder)) - - httpd.serve_forever() diff --git a/setup.py b/setup.py index aca4343..3e4ccff 100644 --- a/setup.py +++ b/setup.py @@ -14,14 +14,20 @@ # under the License. from setuptools import setup +from setuptools import find_packages + setup( name='ramdisk-func-test', version='0.1.0', - packages=['ramdisk_func_test'], + packages=find_packages(), classifiers=[ 'Programming Language :: Python :: 2.7', ], + entry_points={ + 'console_scripts': + 'ramdisk-stub-webserver = ramdisk_func_test.webserver:main' + }, install_requires=[ 'stevedore>=1.3.0,<1.4.0', # Not used. Prevents pip dependency conflict. # This corresponds to openstack global-requirements.txt @@ -31,6 +37,9 @@ setup( 'pyyaml', 'sh', ], + package_data={ + 'ramdisk_func_test.webserver': ['data/*'] + }, url='', license='Apache License, Version 2.0', author='',