diff --git a/README.rst b/README.rst index 2a402809..b77a2341 100644 --- a/README.rst +++ b/README.rst @@ -36,8 +36,10 @@ TOC -------------------- :: - # sudo cp etc/freezer-api.conf /etc/freezer-api.conf - # sudo vi /etc/freezer-api.conf + # sudo cp etc/freezer/freezer-api.conf /etc/freezer/freezer-api.conf + # sudo cp etc/freezer/freezer-paste.ini /etc/freezer/freezer-paste.ini + # sudo vi /etc/freezer/freezer-api.conf + # sudo vi /etc/freezer/freezer-paste.ini 1.4 setup/configure the db @@ -63,7 +65,7 @@ To do that, use the freezer-db-init command: # freezer-db-init [db-host] The url of the db-host is optional and can be automatically guessed from -/etc/freezer-api.conf +/etc/freezer/freezer-api.conf To get information about optional additional parameters: :: diff --git a/config-generator/freezer-api.conf b/config-generator/freezer-api.conf index ea1d26c1..c3ae79bc 100644 --- a/config-generator/freezer-api.conf +++ b/config-generator/freezer-api.conf @@ -1,5 +1,5 @@ [DEFAULT] -output_file = etc/freezer-api.conf.sample +output_file = etc/freezer/freezer-api.conf.sample wrap_width = 79 namespace = "freezer-api" namespace = oslo.log diff --git a/devstack/lib/freezer-api b/devstack/lib/freezer-api index 7dd1e99f..48053e9f 100644 --- a/devstack/lib/freezer-api +++ b/devstack/lib/freezer-api @@ -87,13 +87,22 @@ function configure_freezer_api { [ ! -d $FREEZER_API_LOG_DIR ] && sudo mkdir -m 755 -p $FREEZER_API_LOG_DIR sudo chown $USER $FREEZER_API_LOG_DIR - sudo cp $FREEZER_API_DIR/etc/freezer-api.conf.sample $FREEZER_API_CONF_DIR/freezer-api.conf + sudo cp $FREEZER_API_DIR/etc/freezer/freezer-api.conf.sample $FREEZER_API_CONF_DIR/freezer-api.conf + sudo cp $FREEZER_API_DIR/etc/freezer/freezer-paste.ini $FREEZER_API_CONF_DIR/freezer-paste.ini + # set paste configuration + iniset $FREEZER_API_CONF 'paste_deploy' config_file $FREEZER_API_CONF_DIR/freezer-paste.ini + + # make sure the stack user has the right permissions on the config folder + sudo chown -R $USER $FREEZER_API_CONF_DIR + + #set elasticsearch configuration iniset $FREEZER_API_CONF 'storage' db elasticsearch iniset $FREEZER_API_CONF 'storage' index freezer iniset $FREEZER_API_CONF 'storage' number_of_replicas 0 iniset $FREEZER_API_CONF 'storage' hosts http://$SERVICE_HOST:9200 + # set keystone configuration iniset $FREEZER_API_CONF 'keystone_authtoken' auth_protocol $KEYSTONE_AUTH_PROTOCOL iniset $FREEZER_API_CONF 'keystone_authtoken' auth_host $KEYSTONE_AUTH_HOST iniset $FREEZER_API_CONF 'keystone_authtoken' auth_port $KEYSTONE_AUTH_PORT diff --git a/devstack/settings b/devstack/settings index c108f003..5b9b9976 100644 --- a/devstack/settings +++ b/devstack/settings @@ -18,7 +18,7 @@ # Set up default directories FREEZER_API_DIR=$DEST/freezer-api FREEZER_API_FILES=${FREEZER_API_DIR}/devstack/files -FREEZER_API_CONF_DIR=${FREEZER_API_CONF_DIR:-/etc} +FREEZER_API_CONF_DIR=${FREEZER_API_CONF_DIR:-/etc/freezer} FREEZER_API_CONF=$FREEZER_API_CONF_DIR/freezer-api.conf FREEZER_API_LOG_DIR=$DEST/logs diff --git a/etc/freezer-api.conf.sample b/etc/freezer/freezer-api.conf.sample similarity index 96% rename from etc/freezer-api.conf.sample rename to etc/freezer/freezer-api.conf.sample index 24b1d6e6..e17ef6c2 100644 --- a/etc/freezer-api.conf.sample +++ b/etc/freezer/freezer-api.conf.sample @@ -18,10 +18,11 @@ # If set to true, the logging level will be set to DEBUG instead of the default # INFO level. (boolean value) +# Note: This option can be changed without restarting. #debug = false -# If set to false, the logging level will be set to WARNING instead of the -# default INFO level. (boolean value) +# DEPRECATED: If set to false, the logging level will be set to WARNING instead +# of the default INFO level. (boolean value) # This option is deprecated for removal. # Its value may be silently ignored in the future. #verbose = true @@ -166,8 +167,8 @@ # Determines the frequency at which the list of revoked tokens is retrieved # from the Identity service (in seconds). A high number of revocation events -# combined with a low cache duration may significantly reduce performance. -# (integer value) +# combined with a low cache duration may significantly reduce performance. Only +# valid for PKI tokens. (integer value) #revocation_cache_time = 10 # (Optional) If defined, indicate whether token data should be authenticated or @@ -279,6 +280,17 @@ #auth_section = +[paste_deploy] + +# +# From freezer-api +# + +# Name of the paste configuration file that defines the available pipelines. +# (string value) +#config_file = freezer-paste.ini + + [storage] # diff --git a/etc/freezer/freezer-paste.ini b/etc/freezer/freezer-paste.ini new file mode 100644 index 00000000..e2272f60 --- /dev/null +++ b/etc/freezer/freezer-paste.ini @@ -0,0 +1,23 @@ +[app:freezer_app] +paste.app_factory = freezer_api.service:freezer_app_factory + +[app:service_v1] +use = egg:freezer-api#service_v1 + +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:healthcheck] +paste.filter_factory = oslo_middleware:Healthcheck.factory +backends = disable_by_file +disable_by_file_path = /etc/freezer/healthcheck_disable + +# @todo deprecated and should be removed soon +[filter:HealthApp] +paste.filter_factory = freezer_api.api.common.middleware:HealthApp.factory + +[pipeline:main] +pipeline = HealthApp healthcheck authtoken freezer_app + +[pipeline:unauthenticated_freezer_api] +pipeline = HealthApp Healthcheck freezer_app diff --git a/freezer_api/api/common/middleware.py b/freezer_api/api/common/middleware.py index 9b277a1d..5e9d654b 100644 --- a/freezer_api/api/common/middleware.py +++ b/freezer_api/api/common/middleware.py @@ -18,45 +18,66 @@ limitations under the License. import falcon import json +from oslo_log import log -class HealthApp(object): +LOG = log.getLogger(__name__) + + +class Middleware(object): + """ + WSGI wrapper for all freezer middlewares. Use this will allow to manage + middlewares through paste + """ + + def __init__(self, app): + self.app = app + + def process_request(self, req): + """ + implement this function in your middleware to change the request + if the function return None the request will be handled in the next + level functions + """ + return None + + def process_response(self, resp): + """ + Implement this to modify your response + """ + return None + + @classmethod + def factory(cls, global_conf, **local_conf): + def filter(app): + return cls(app) + return filter + + def __call__(self, req, resp): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.app) + response.req = req + try: + self.process_response(response) + except falcon.HTTPError as e: + LOG.error(e) + + +# @todo this should be removed and oslo.middleware should be used instead +class HealthApp(Middleware): """ Simple WSGI app to support HAProxy polling. If the requested url matches the configured path it replies with a 200 otherwise passes the request to the inner app """ - def __init__(self, app, path): - self.app = app - self.path = path - def __call__(self, environ, start_response): - if environ.get('PATH_INFO') == self.path: + if environ.get('PATH_INFO') == '/v1/health': start_response('200 OK', []) return [] return self.app(environ, start_response) -class SignedHeadersAuth(object): - - def __init__(self, app, auth_app): - self._app = app - self._auth_app = auth_app - - def __call__(self, environ, start_response): - path = environ.get('PATH_INFO') - # NOTE(flwang): The root path of Zaqar service shouldn't require any - # auth. - if path == '/': - return self._app(environ, start_response) - - signature = environ.get('HTTP_URL_SIGNATURE') - - if signature is None or path.startswith('/v1'): - return self._auth_app(environ, start_response) - - return self._app(environ, start_response) - - class RequireJSON(object): def process_request(self, req, resp): diff --git a/freezer_api/cmd/api.py b/freezer_api/cmd/api.py index 78db3a72..accf993e 100644 --- a/freezer_api/cmd/api.py +++ b/freezer_api/cmd/api.py @@ -20,10 +20,11 @@ from __future__ import print_function import falcon import sys -from keystonemiddleware import auth_token from oslo_config import cfg from oslo_log import log -from wsgiref import simple_server +from paste import deploy +from paste import httpserver + from freezer_api.api.common import middleware from freezer_api.api.common import utils @@ -39,19 +40,21 @@ CONF = cfg.CONF _LOG = log.getLogger(__name__) -def get_application(db=None): - config.parse_args() - config.setup_logging() - +def build_app(db=None): + """ + Building routes and forming the root freezer-api app + :param db: instance of elastic search db class if not freezer will try to + initialize this instance itself + :return: wsgi app + """ if not db: db = driver.get_db() - # injecting FreezerContext & hooks - middlware_list = [utils.FuncMiddleware(hook) for hook in - utils.before_hooks()] - middlware_list.append(middleware.JSONTranslator()) - middlware_list.append(middleware.RequireJSON()) - app = falcon.API(middleware=middlware_list) + middleware_list = [utils.FuncMiddleware(hook) for hook in + utils.before_hooks()] + middleware_list.append(middleware.JSONTranslator()) + middleware_list.append(middleware.RequireJSON()) + app = falcon.API(middleware=middleware_list) for exception_class in freezer_api_exc.exception_handlers_catalog: app.add_error_handler(exception_class, exception_class.handle) @@ -64,42 +67,30 @@ def get_application(db=None): for route, resource in endpoints: app.add_route(version_path + route, resource) - if 'keystone_authtoken' in config.CONF: - auth_app = auth_token.AuthProtocol(app, - conf={"oslo-config-config": CONF, - "oslo-config-project": - "freezer-api"}) - else: - _LOG.warning(_i18n._LW("keystone authentication disabled")) - - app = middleware.SignedHeadersAuth(app, auth_app) - app = middleware.HealthApp(app=app, path='/v1/health') - return app def main(): - try: - application = get_application() - except Exception as err: - message = _i18n._('Unable to start server: %s ') % err - print(message) - _LOG.error(message) - sys.exit(1) + # setup opts + config.parse_args() + config.setup_logging() + paste_conf = config.find_paste_config() + # quick simple server for testing purposes or simple scenarios ip = CONF.get('bind_host', '0.0.0.0') port = CONF.get('bind_port', 9090) - httpd = simple_server.make_server(ip, int(port), application) - message = _i18n._('Server listening on %(ip)s:%(port)s' - % {'ip': ip, 'port': port}) - print(message) - _LOG.info(message) try: - httpd.serve_forever() + httpserver.serve( + application=deploy.loadapp('config:%s' % paste_conf, name='main'), + host=ip, + port=port) + message = _i18n._('Server listening on %(ip)s:%(port)s' % + {'ip': ip, 'port': port}) + _LOG.info(message) + print(message) except KeyboardInterrupt: - print(_i18n._("\nThanks, Bye")) + print(_i18n._("Thank You ! \nBye.")) sys.exit(0) - if __name__ == '__main__': main() diff --git a/freezer_api/cmd/db_init.py b/freezer_api/cmd/db_init.py index 6973f57f..7bb9e0bb 100755 --- a/freezer_api/cmd/db_init.py +++ b/freezer_api/cmd/db_init.py @@ -30,7 +30,7 @@ import requests from freezer_api.common import db_mappings -DEFAULT_CONF_PATH = '/etc/freezer-api.conf' +DEFAULT_CONF_PATH = '/etc/freezer/freezer-api.conf' DEFAULT_ES_SERVER_PORT = 9200 DEFAULT_INDEX = 'freezer' DEFAULT_REPLICAS = 1 @@ -307,7 +307,7 @@ def get_db_params(args): """ Extracts the db configuration parameters either from the provided command line arguments or searching in the default freezer-api config - file /etc/freezer-api.conf + file /etc/freezer/freezer-api.conf :param args: argparsed command line arguments :return: (elasticsearch_url, elastichsearch_index, number_of_replicas) diff --git a/freezer_api/cmd/wsgi.py b/freezer_api/cmd/wsgi.py index 1239f7d3..8c5319fc 100644 --- a/freezer_api/cmd/wsgi.py +++ b/freezer_api/cmd/wsgi.py @@ -14,20 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. """ -from freezer_api.cmd.api import get_application -from freezer_api.storage import driver -from oslo_config import cfg -from oslo_log import log +from freezer_api.service import initialize_app -CONF = cfg.CONF -_LOG = log.getLogger(__name__) - -db = None - -try: - db = driver.get_db() -except Exception as err: - message = 'Unable to start server: {0} '.format(err) - _LOG.error(message) - -application = get_application(db) +app = application = initialize_app() diff --git a/freezer_api/common/config.py b/freezer_api/common/config.py index 3c004c89..bf4fbce4 100644 --- a/freezer_api/common/config.py +++ b/freezer_api/common/config.py @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ +import os import sys from oslo_config import cfg @@ -25,6 +26,12 @@ from keystonemiddleware import opts CONF = cfg.CONF +paste_deploy = [ + cfg.StrOpt('config_file', default='freezer-paste.ini', + help='Name of the paste configuration file that defines ' + 'the available pipelines.'), + ] + def api_common_opts(): @@ -46,6 +53,11 @@ def api_common_opts(): def parse_args(): CONF.register_cli_opts(api_common_opts()) driver.register_elk_opts() + # register paste configuration + paste_grp = cfg.OptGroup('paste_deploy', + 'Paste Configuration') + CONF.register_group(paste_grp) + CONF.register_opts(paste_deploy, group=paste_grp) log.register_options(CONF) default_config_files = cfg.find_config_files('freezer', 'freezer-api') CONF(args=sys.argv[1:], @@ -70,10 +82,42 @@ def setup_logging(): log.setup(CONF, 'freezer-api', version=FREEZER_API_VERSION) +def find_paste_config(): + """Find freezer's paste.deploy configuration file. + freezer's paste.deploy configuration file is specified in the + ``[paste_deploy]`` section of the main freezer-api configuration file, + ``freezer-api.conf``. + For example:: + [paste_deploy] + config_file = freezer-paste.ini + :returns: The selected configuration filename + :raises: exception.ConfigFileNotFound + """ + if CONF.paste_deploy.config_file: + paste_config = CONF.paste_deploy.config_file + paste_config_value = paste_config + if not os.path.isabs(paste_config): + paste_config = CONF.find_file(paste_config) + elif CONF.config_file: + paste_config = CONF.config_file[0] + paste_config_value = paste_config + else: + # this provides backwards compatibility for keystone.conf files that + # still have the entire paste configuration included, rather than just + # a [paste_deploy] configuration section referring to an external file + paste_config = CONF.find_file('freezer-api.conf') + paste_config_value = 'freezer-api.conf' + if not paste_config or not os.path.exists(paste_config): + raise Exception('paste configuration file {0} not found !'. + format(paste_config)) + return paste_config + + def list_opts(): _OPTS = { None: api_common_opts(), 'storage': driver.get_elk_opts(), + 'paste_deploy': paste_deploy, opts.auth_token_opts[0][0]: opts.auth_token_opts[0][1] } return _OPTS.items() diff --git a/freezer_api/service.py b/freezer_api/service.py new file mode 100644 index 00000000..49f10ea4 --- /dev/null +++ b/freezer_api/service.py @@ -0,0 +1,35 @@ +""" +(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from freezer_api.cmd.api import build_app +from freezer_api.common import config +from paste import deploy + + +def freezer_app_factory(global_conf, **local_conf): + return build_app() + + +def initialize_app(conf=None, name='main'): + try: + config.parse_args() + config.setup_logging() + except Exception as e: + pass + conf = config.find_paste_config() + app = deploy.loadapp('config:%s' % conf, name=name) + return app + diff --git a/freezer_api/tests/unit/test_driver.py b/freezer_api/tests/unit/test_driver.py index 8e6b61ad..a83abe19 100644 --- a/freezer_api/tests/unit/test_driver.py +++ b/freezer_api/tests/unit/test_driver.py @@ -21,7 +21,7 @@ import unittest from mock import Mock, patch from freezer_api.storage import driver -from freezer_api.cmd.api import get_application +from freezer_api.cmd.api import build_app class TestStorageDriver(unittest.TestCase): diff --git a/freezer_api/tests/unit/test_middleware.py b/freezer_api/tests/unit/test_middleware.py index 46acab1c..2eff4a46 100644 --- a/freezer_api/tests/unit/test_middleware.py +++ b/freezer_api/tests/unit/test_middleware.py @@ -21,12 +21,13 @@ from mock import Mock from freezer_api.api.common import middleware + class TestHealthApp(unittest.TestCase): def test_call_nested_app(self): mock_app = Mock() mock_app.return_value = ['app_body'] - health_app = middleware.HealthApp(mock_app, 'test_path_78908') + health_app = middleware.HealthApp(mock_app) environ = {} start_response = Mock() result = health_app(environ, start_response) @@ -35,8 +36,8 @@ class TestHealthApp(unittest.TestCase): def test_return_200_when_paths_match(self): mock_app = Mock() mock_app.return_value = ['app_body'] - health_app = middleware.HealthApp(mock_app, 'test_path_6789') - environ = {'PATH_INFO': 'test_path_6789'} + health_app = middleware.HealthApp(mock_app) + environ = {'PATH_INFO': '/v1/health'} start_response = Mock() result = health_app(environ, start_response) start_response.assert_called_once_with('200 OK', []) diff --git a/requirements.txt b/requirements.txt index c54dd5f8..1583ad88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,8 @@ elasticsearch>=1.3.0,<2.0 # Apache-2.0 falcon>=0.1.6 # Apache-2.0 jsonschema>=2.0.0,<3.0.0,!=2.5.0 # MIT keystonemiddleware>=4.0.0 # Apache-2.0 +Paste # MIT +PasteDeploy>=1.5.0 # MIT oslo.config>=3.2.0 # Apache-2.0 oslo.context>=2.2.0 # Apache-2.0 oslo.i18n>=1.5.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 61a3f221..59eef9a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,10 @@ console_scripts = freezer-db-init = freezer_api.cmd.db_init:main tempest.test_plugins = freezer_api_tempest_tests = freezer_api.tests.freezer_api_tempest_plugin.plugin:FreezerApiTempestPlugin - +paste.app_factory = + service_v1 = freezer_api.service:freezer_app_factory +wsgi_scripts = + freezer-api-wsgi = freezer_api.service:initialize_app [pytests] where=tests