From 163ea0beaaae66299f5158177a567761194f06af Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Thu, 23 May 2013 01:29:39 -0700 Subject: [PATCH 1/6] fixes the problem of 'Command tools/with_venv.sh pip install --upgrade pip failed.' In the process adopted install_venv_common.py. Note setup.sh is currently commented out. --- tools/install_venv.py | 128 ++++--------------- tools/install_venv_common.py | 233 +++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+), 104 deletions(-) create mode 100644 tools/install_venv_common.py diff --git a/tools/install_venv.py b/tools/install_venv.py index 8ccb2cef..58737958 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -1,10 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. # -# Copyright 2010 OpenStack LLC. +# Copyright 2013 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -26,100 +22,7 @@ import os import subprocess import sys - -ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.venv') -PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') -TEST_REQUIRES = os.path.join(ROOT, 'tools', 'test-requires') - - -def die(message, *args): - print >> sys.stderr, message % args - sys.exit(1) - - -def run_command(cmd, redirect_output=True, check_exit_code=True): - """ - Runs a command in an out-of-process shell, returning the - output of that command. Working directory is ROOT. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=ROOT, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - die('Command "%s" failed.\n%s', ' '.join(cmd), output) - return output - - -HAS_EASY_INSTALL = bool(run_command(['which', 'easy_install'], - check_exit_code=False).strip()) -HAS_VIRTUALENV = bool(run_command(['which', 'virtualenv'], - check_exit_code=False).strip()) - - -def check_dependencies(): - """Make sure virtualenv is in the path.""" - - if not HAS_VIRTUALENV: - print 'not found.' - # Try installing it via easy_install... - if HAS_EASY_INSTALL: - print 'Installing virtualenv via easy_install...', - if not run_command(['which', 'easy_install']): - die('ERROR: virtualenv not found.\n\n' - 'Barbican development requires virtualenv, please install' - ' it using your favorite package management tool') - print 'done.' - print 'done.' - - -def create_virtualenv(venv=VENV): - """ - Creates the virtual environment and installs PIP only into the - virtual environment - """ - print 'Creating venv...', - run_command(['virtualenv', '-q', '--no-site-packages', VENV]) - print 'done.' - print 'Installing pip in virtualenv...', - if not run_command(['tools/with_venv.sh', 'easy_install', - 'pip>1.0']).strip(): - die("Failed to install pip.") - print 'done.' - - -def pip_install(*args): - run_command(['tools/with_venv.sh', - 'pip', 'install', '--upgrade'] + list(args), - redirect_output=False) - - -def install_dependencies(venv=VENV): - print 'Installing dependencies with pip (this can take a while)...' - - pip_install('pip') - - pip_install('-r', PIP_REQUIRES) - pip_install('-r', TEST_REQUIRES) - - # Tell the virtual env how to "import barbican" - py_ver = _detect_python_version(venv) - pthfile = os.path.join(venv, "lib", py_ver, - "site-packages", "barbican.pth") - f = open(pthfile, 'w') - f.write("%s\n" % ROOT) - - -def _detect_python_version(venv): - lib_dir = os.path.join(venv, "lib") - for pathname in os.listdir(lib_dir): - if pathname.startswith('python'): - return pathname - raise Exception('Unable to detect Python version') +import install_venv_common as install_venv def print_help(): @@ -129,8 +32,8 @@ def print_help(): Barbican development uses virtualenv to track and manage Python dependencies while in development and testing. - To activate the Barbican virtualenv for the extent of your current shell - session you can run: + To activate the Barbican virtualenv for the extent of your current shell session + you can run: $ source .venv/bin/activate @@ -144,10 +47,27 @@ def print_help(): print help + def main(argv): - check_dependencies() - create_virtualenv() - install_dependencies() + root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + venv = os.path.join(root, '.venv') + pip_requires = os.path.join(root, 'tools', 'pip-requires') + test_requires = os.path.join(root, 'tools', 'test-requires') + py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) + project = 'Barbican' + project_name = "barbican.pth" + install = install_venv.InstallVenv( +root, venv, pip_requires, test_requires, + py_version, project) + options = install.parse_args(argv) + install.check_python_version() + install.check_dependencies() + install.create_virtualenv(no_site_packages=options.no_site_packages) + install.install_dependencies(venv, project_name) + print 'Installing barbican module in development mode...' + # install.run_command([os.path.join(venv, 'bin/python'), + # 'setup.py', 'develop']) + install.post_process() print_help() if __name__ == '__main__': diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py new file mode 100644 index 00000000..63cda282 --- /dev/null +++ b/tools/install_venv_common.py @@ -0,0 +1,233 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack, LLC +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Synced in from openstack-common +""" + +import argparse +import os +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +VENV = os.path.join(ROOT, '.venv') + +class InstallVenv(object): + + def __init__(self, root, venv, pip_requires, test_requires, py_version, + project): + self.root = root + self.venv = venv + self.pip_requires = pip_requires + self.test_requires = test_requires + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print >> sys.stderr, message % args + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, + redirect_output=True, + check_exit_code=True, + cwd=ROOT, + die_message=None): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + if die_message is None: + die('Command "%s" failed.\n%s', ' '.join(cmd), output) + else: + die(die_message) + return (output, proc.returncode) + + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora(self.root, self.venv, self.pip_requires, + self.test_requires, self.py_version, self.project) + else: + return Distro(self.root, self.venv, self.pip_requires, + self.test_requires, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print 'Creating venv...', + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print 'done.' + print 'Installing pip in venv...', + if not self.run_command(['tools/with_venv.sh', 'easy_install', + 'pip>1.0']).strip(): + self.die("Failed to install pip.") + print 'done.' + else: + print "venv already exists..." + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self, venv, project_name_path): + print 'Installing dependencies with pip (this can take a while)...' + + # First things first, make sure our venv has the latest pip and + # distribute. + # NOTE: we keep pip at version 1.1 since the most recent version causes + # the .venv creation to fail. See: + # https://bugs.launchpad.net/nova/+bug/1047120 + self.pip_install('pip==1.1') + self.pip_install('distribute') + + # Install greenlet by hand - just listing it in the requires file does + # not + # get it installed in the right order + self.pip_install('greenlet') + + self.pip_install('-r', self.pip_requires) + self.pip_install('-r', self.test_requires) + + py = 'python%d.%d' % (sys.version_info[0], sys.version_info[1]) + pthfile = os.path.join(venv, "lib", py, "site-packages", project_name_path) + f = open(pthfile, 'w') + f.write("%s\n" % ROOT) + + def post_process(self): + self.get_distro().post_process() + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:]) + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print 'Installing virtualenv via easy_install...', + if self.run_command(['easy_install', 'virtualenv']): + print 'Succeeded' + return + else: + print 'Failed' + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def yum_install(self, pkg, **kwargs): + print "Attempting to install '%s' via yum" % pkg + self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs) + + def apply_patch(self, originalfile, patchfile): + self.run_command(['patch', originalfile, patchfile]) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.yum_install('python-virtualenv', check_exit_code=False) + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/which_linden/eventlet/issue/89 + """ + + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.yum_install('patch') + + # Apply the eventlet patch + self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, + 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch') From 52f8c73a89867265aa01c23a2e7d512baee5907d Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Thu, 23 May 2013 18:29:24 -0700 Subject: [PATCH 2/6] support for keystone context in request --- barbican/api/middleware/context.py | 139 +++++++ barbican/api/policy.py | 2 +- barbican/common/wsgi.py | 621 ++++++++++++++++++++++++++++ barbican/context.py | 90 ++++ etc/barbican/barbican-api-paste.ini | 22 +- 5 files changed, 867 insertions(+), 7 deletions(-) create mode 100644 barbican/api/middleware/context.py create mode 100644 barbican/common/wsgi.py create mode 100644 barbican/context.py diff --git a/barbican/api/middleware/context.py b/barbican/api/middleware/context.py new file mode 100644 index 00000000..d6d8c813 --- /dev/null +++ b/barbican/api/middleware/context.py @@ -0,0 +1,139 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from oslo.config import cfg +import webob.exc + +from barbican.api import policy +from barbican.common import wsgi +import barbican.context +import barbican.openstack.common.log as logging + + +context_opts = [ + cfg.BoolOpt('owner_is_tenant', default=True, + help=_('When true, this option sets the owner of an image ' + 'to be the tenant. Otherwise, the owner of the ' + ' image will be the authenticated user issuing the ' + 'request.')), + cfg.StrOpt('admin_role', default='admin', + help=_('Role used to identify an authenticated user as ' + 'administrator.')), + cfg.BoolOpt('allow_anonymous_access', default=False, + help=_('Allow unauthenticated users to access the API with ' + 'read-only privileges. This only applies when using ' + 'ContextMiddleware.')), +] + +CONF = cfg.CONF +CONF.register_opts(context_opts) + +LOG = logging.getLogger(__name__) + + +class BaseContextMiddleware(wsgi.Middleware): + def process_response(self, resp): + try: + request_id = resp.request.context.request_id + except AttributeError: + LOG.warn(_('Unable to retrieve request id from context')) + else: + resp.headers['x-openstack-request-id'] = 'req-%s' % request_id + return resp + + +class ContextMiddleware(BaseContextMiddleware): + def __init__(self, app): + self.policy_enforcer = policy.Enforcer() + super(ContextMiddleware, self).__init__(app) + + def process_request(self, req): + """Convert authentication information into a request context + + Generate a barbican.context.RequestContext object from the available + authentication headers and store on the 'context' attribute + of the req object. + + :param req: wsgi request object that will be given the context object + :raises webob.exc.HTTPUnauthorized: when value of the X-Identity-Status + header is not 'Confirmed' and + anonymous access is disallowed + """ + if req.headers.get('X-Identity-Status') == 'Confirmed': + req.context = self._get_authenticated_context(req) + elif CONF.allow_anonymous_access: + req.context = self._get_anonymous_context() + else: + raise webob.exc.HTTPUnauthorized() + + def _get_anonymous_context(self): + kwargs = { + 'user': None, + 'tenant': None, + 'roles': [], + 'is_admin': False, + 'read_only': True, + 'policy_enforcer': self.policy_enforcer, + } + return barbican.context.RequestContext(**kwargs) + + def _get_authenticated_context(self, req): + #NOTE(bcwaldon): X-Roles is a csv string, but we need to parse + # it into a list to be useful + roles_header = req.headers.get('X-Roles', '') + roles = [r.strip().lower() for r in roles_header.split(',')] + + #NOTE(bcwaldon): This header is deprecated in favor of X-Auth-Token + #(mkbhanda) keeping this just-in-case for swift + deprecated_token = req.headers.get('X-Storage-Token') + + service_catalog = None + if req.headers.get('X-Service-Catalog') is not None: + try: + catalog_header = req.headers.get('X-Service-Catalog') + service_catalog = json.loads(catalog_header) + except ValueError: + raise webob.exc.HTTPInternalServerError( + _('Invalid service catalog json.')) + + kwargs = { + 'user': req.headers.get('X-User-Id'), + 'tenant': req.headers.get('X-Tenant-Id'), + 'roles': roles, + 'is_admin': CONF.admin_role.strip().lower() in roles, + 'auth_tok': req.headers.get('X-Auth-Token', deprecated_token), + 'owner_is_tenant': CONF.owner_is_tenant, + 'service_catalog': service_catalog, + 'policy_enforcer': self.policy_enforcer, + } + + return barbican.context.RequestContext(**kwargs) + + +class UnauthenticatedContextMiddleware(BaseContextMiddleware): + def process_request(self, req): + """Create a context without an authorized user.""" + kwargs = { + 'user': None, + 'tenant': None, + 'roles': [], + 'is_admin': True, + } + + req.context = barbican.context.RequestContext(**kwargs) diff --git a/barbican/api/policy.py b/barbican/api/policy.py index 5ba8e0b1..0a8efc7e 100644 --- a/barbican/api/policy.py +++ b/barbican/api/policy.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2011 OpenStack, LLC. +# Copyright (c) 2013 OpenStack, LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/barbican/common/wsgi.py b/barbican/common/wsgi.py new file mode 100644 index 00000000..354caf08 --- /dev/null +++ b/barbican/common/wsgi.py @@ -0,0 +1,621 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utility methods for working with WSGI servers +""" + +import datetime +import errno +import json +import logging +import os +import signal +import sys +import time + +import eventlet +from eventlet.green import socket, ssl +import eventlet.greenio +import eventlet.wsgi +from oslo.config import cfg +# mkbhanda -- if we want this, need to include routes in pip-requires +#import routes +#import routes.middleware +import webob.dec +import webob.exc + +from barbican.common import exception +from barbican.common import utils +import barbican.openstack.common.log as os_logging + + +bind_opts = [ + cfg.StrOpt('bind_host', default='0.0.0.0', + help=_('Address to bind the server. Useful when ' + 'selecting a particular network interface.')), + cfg.IntOpt('bind_port', + help=_('The port on which the server will listen.')), +] + +socket_opts = [ + cfg.IntOpt('backlog', default=4096, + help=_('The backlog value that will be used when creating the ' + 'TCP listener socket.')), + cfg.IntOpt('tcp_keepidle', default=600, + help=_('The value for the socket option TCP_KEEPIDLE. This is' + 'the time in seconds that the connection must be idle ' + 'before TCP starts sending keepalive probes.')), + cfg.StrOpt('ca_file', help=_('CA certificate file to use to verify ' + 'connecting clients.')), + cfg.StrOpt('cert_file', help=_('Certificate file to use when starting API ' + 'server securely.')), + cfg.StrOpt('key_file', help=_('Private key file to use when starting API ' + 'server securely.')), +] + +workers_opt = cfg.IntOpt('workers', default=1, + help=_('The number of child process workers that ' + 'will be created to service API requests.')) + +CONF = cfg.CONF +CONF.register_opts(bind_opts) +CONF.register_opts(socket_opts) +CONF.register_opt(workers_opt) + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.DEBUG): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg.strip("\n")) + + +def get_bind_addr(default_port=None): + """Return the host and port to bind to.""" + return (CONF.bind_host, CONF.bind_port or default_port) + + +def get_socket(default_port): + """ + Bind socket to bind ip:port in conf + + note: Mostly comes from Swift with a few small changes... + + :param default_port: port to bind to if none is specified in conf + + :returns : a socket object as returned from socket.listen or + ssl.wrap_socket if conf specifies cert_file + """ + bind_addr = get_bind_addr(default_port) + + # TODO(jaypipes): eventlet's greened socket module does not actually + # support IPv6 in getaddrinfo(). We need to get around this in the + # future or monitor upstream for a fix + address_family = [ + addr[0] for addr in socket.getaddrinfo(bind_addr[0], + bind_addr[1], + socket.AF_UNSPEC, + socket.SOCK_STREAM) + if addr[0] in (socket.AF_INET, socket.AF_INET6) + ][0] + + cert_file = CONF.cert_file + key_file = CONF.key_file + use_ssl = cert_file or key_file + if use_ssl and (not cert_file or not key_file): + raise RuntimeError(_("When running server in SSL mode, you must " + "specify both a cert_file and key_file " + "option value in your configuration file")) + + def wrap_ssl(sock): + utils.validate_key_cert(key_file, cert_file) + + ssl_kwargs = { + 'server_side': True, + 'certfile': cert_file, + 'keyfile': key_file, + 'cert_reqs': ssl.CERT_NONE, + } + + if CONF.ca_file: + ssl_kwargs['ca_certs'] = CONF.ca_file + ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED + + return ssl.wrap_socket(sock, **ssl_kwargs) + + sock = utils.get_test_suite_socket() + retry_until = time.time() + 30 + + if sock and use_ssl: + sock = wrap_ssl(sock) + while not sock and time.time() < retry_until: + try: + sock = eventlet.listen(bind_addr, + backlog=CONF.backlog, + family=address_family) + if use_ssl: + sock = wrap_ssl(sock) + + except socket.error as err: + if err.args[0] != errno.EADDRINUSE: + raise + eventlet.sleep(0.1) + if not sock: + raise RuntimeError(_("Could not bind to %s:%s after trying for 30 " + "seconds") % bind_addr) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # in my experience, sockets can hang around forever without keepalive + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + # This option isn't available in the OS X version of eventlet + if hasattr(socket, 'TCP_KEEPIDLE'): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, + CONF.tcp_keepidle) + + return sock + + +class Server(object): + """Server class to manage multiple WSGI sockets and applications.""" + + def __init__(self, threads=1000): + self.threads = threads + self.children = [] + self.running = True + + def start(self, application, default_port): + """ + Run a WSGI server with the given application. + + :param application: The application to be run in the WSGI server + :param default_port: Port to bind to if none is specified in conf + """ + pgid = os.getpid() + try: + # NOTE(flaper87): Make sure this process + # runs in its own process group. + os.setpgid(pgid, pgid) + except OSError: + # watch out for failures with + # setpgid fails with EPERM because of fresh session creation + pgid = 0 + + def kill_children(*args): + """Kills the entire process group.""" + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + self.running = False + os.killpg(pgid, signal.SIGTERM) + + def hup(*args): + """ + Shuts down the server, but allows running requests to complete + """ + signal.signal(signal.SIGHUP, signal.SIG_IGN) + self.running = False + + self.application = application + self.sock = get_socket(default_port) + + os.umask(027) # ensure files are created with the correct privileges + self.logger = os_logging.getLogger('eventlet.wsgi.server') + + if CONF.workers == 0: + # Useful for profiling, test, debug etc. + self.pool = self.create_pool() + self.pool.spawn_n(self._single_run, self.application, self.sock) + return + else: + self.logger.info(_("Starting %d workers") % CONF.workers) + signal.signal(signal.SIGTERM, kill_children) + signal.signal(signal.SIGINT, kill_children) + signal.signal(signal.SIGHUP, hup) + while len(self.children) < CONF.workers: + self.run_child() + + def create_pool(self): + return eventlet.GreenPool(size=self.threads) + + def wait_on_children(self): + while self.running: + try: + pid, status = os.wait() + if os.WIFEXITED(status) or os.WIFSIGNALED(status): + self.logger.info(_('Removing dead child %s') % pid) + self.children.remove(pid) + if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0: + self.logger.error(_('Not respawning child %d, cannot ' + 'recover from termination') % pid) + if not self.children: + self.logger.info( + _('All workers have terminated. Exiting')) + self.running = False + else: + self.run_child() + except OSError as err: + if err.errno not in (errno.EINTR, errno.ECHILD): + raise + except KeyboardInterrupt: + self.logger.info(_('Caught keyboard interrupt. Exiting.')) + break + eventlet.greenio.shutdown_safe(self.sock) + self.sock.close() + self.logger.debug(_('Exited')) + + def wait(self): + """Wait until all servers have completed running.""" + try: + if self.children: + self.wait_on_children() + else: + self.pool.waitall() + except KeyboardInterrupt: + pass + + def run_child(self): + pid = os.fork() + if pid == 0: + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + # ignore the interrupt signal to avoid a race whereby + # a child worker receives the signal before the parent + # and is respawned unneccessarily as a result + signal.signal(signal.SIGINT, signal.SIG_IGN) + self.run_server() + self.logger.info(_('Child %d exiting normally') % os.getpid()) + # self.pool.waitall() has been called by run_server, so + # its safe to exit here + sys.exit(0) + else: + self.logger.info(_('Started child %s') % pid) + self.children.append(pid) + + def run_server(self): + """Run a WSGI server.""" + if cfg.CONF.pydev_worker_debug_host: + utils.setup_remote_pydev_debug(cfg.CONF.pydev_worker_debug_host, + cfg.CONF.pydev_worker_debug_port) + + eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" + try: + eventlet.hubs.use_hub('poll') + except Exception: + msg = _("eventlet 'poll' hub is not available on this platform") + raise exception.WorkerCreationFailure(reason=msg) + self.pool = self.create_pool() + try: + eventlet.wsgi.server(self.sock, + self.application, + log=WritableLogger(self.logger), + custom_pool=self.pool) + except socket.error as err: + if err[0] != errno.EINVAL: + raise + self.pool.waitall() + + def _single_run(self, application, sock): + """Start a WSGI server in a new green thread.""" + self.logger.info(_("Starting single process server")) + eventlet.wsgi.server(sock, application, custom_pool=self.pool, + log=WritableLogger(self.logger)) + + +class Middleware(object): + """ + Base WSGI middleware wrapper. These classes require an application to be + initialized that will be called next. By default the middleware will + simply call its wrapped app, or you can override __call__ to customize its + behavior. + """ + + def __init__(self, application): + self.application = application + + @classmethod + def factory(cls, global_conf, **local_conf): + def filter(app): + return cls(app) + return filter + + def process_request(self, req): + """ + Called on each request. + + If this returns None, the next application down the stack will be + executed. If it returns a response then that response will be returned + and execution will stop here. + + """ + return None + + def process_response(self, response): + """Do whatever you'd like to the response.""" + return response + + @webob.dec.wsgify + def __call__(self, req): + response = self.process_request(req) + if response: + return response + response = req.get_response(self.application) + response.request = req + return self.process_response(response) + + +class Debug(Middleware): + """ + Helper class that can be inserted into any WSGI application chain + to get information about the request and response. + """ + + @webob.dec.wsgify + def __call__(self, req): + print ("*" * 40) + " REQUEST ENVIRON" + for key, value in req.environ.items(): + print key, "=", value + print + resp = req.get_response(self.application) + + print ("*" * 40) + " RESPONSE HEADERS" + for (key, value) in resp.headers.iteritems(): + print key, "=", value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """ + Iterator that prints the contents of a wrapper string iterator + when iterated. + """ + print ("*" * 40) + " BODY" + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + + +# class APIMapper(routes.Mapper): +# """ +# Handle route matching when url is '' because routes.Mapper returns +# an error in this case. +# """ + +# def routematch(self, url=None, environ=None): +# if url is "": +# result = self._match("", environ) +# return result[0], result[1] +# return routes.Mapper.routematch(self, url, environ) + + +# class Router(object): +# """ +# WSGI middleware that maps incoming requests to WSGI apps. +# """ + +# def __init__(self, mapper): +# """ +# Create a router for the given routes.Mapper. + +# Each route in `mapper` must specify a 'controller', which is a +# WSGI app to call. You'll probably want to specify an 'action' as +# well and have your controller be a wsgi.Controller, who will route +# the request to the action method. + +# Examples: +# mapper = routes.Mapper() +# sc = ServerController() + +# # Explicit mapping of one route to a controller+action +# mapper.connect(None, "/svrlist", controller=sc, action="list") + +# # Actions are all implicitly defined +# mapper.resource("server", "servers", controller=sc) + +# # Pointing to an arbitrary WSGI app. You can specify the +# # {path_info:.*} parameter so the target app can be handed just that +# # section of the URL. +# mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) +# """ +# mapper.redirect("", "/") +# self.map = mapper +# self._router = routes.middleware.RoutesMiddleware(self._dispatch, +# self.map) + +# @classmethod +# def factory(cls, global_conf, **local_conf): +# return cls(APIMapper()) + +# @webob.dec.wsgify +# def __call__(self, req): +# """ +# Route the incoming request to a controller based on self.map. +# If no match, return a 404. +# """ +# return self._router + +# @staticmethod +# @webob.dec.wsgify +# def _dispatch(req): +# """ +# Called by self._router after matching the incoming request to a route +# and putting the information into req.environ. Either returns 404 +# or the routed WSGI app's response. +# """ +# match = req.environ['wsgiorg.routing_args'][1] +# if not match: +# return webob.exc.HTTPNotFound() +# app = match['controller'] +# return app + + +class Request(webob.Request): + """Add some Openstack API-specific logic to the base webob.Request.""" + + def best_match_content_type(self): + """Determine the requested response content-type.""" + supported = ('application/json',) + bm = self.accept.best_match(supported) + return bm or 'application/json' + + def get_content_type(self, allowed_content_types): + """Determine content type of the request body.""" + if "Content-Type" not in self.headers: + raise exception.InvalidContentType(content_type=None) + + content_type = self.content_type + + if content_type not in allowed_content_types: + raise exception.InvalidContentType(content_type=content_type) + else: + return content_type + + +class JSONRequestDeserializer(object): + def has_body(self, request): + """ + Returns whether a Webob.Request object will possess an entity body. + + :param request: Webob.Request object + """ + if 'transfer-encoding' in request.headers: + return True + elif request.content_length > 0: + return True + + return False + + def from_json(self, datastring): + try: + return json.loads(datastring) + except ValueError: + msg = _('Malformed JSON in request body.') + raise webob.exc.HTTPBadRequest(explanation=msg) + + def default(self, request): + if self.has_body(request): + return {'body': self.from_json(request.body)} + else: + return {} + + +class JSONResponseSerializer(object): + + def to_json(self, data): + def sanitizer(obj): + if isinstance(obj, datetime.datetime): + return obj.isoformat() + if hasattr(obj, "to_dict"): + return obj.to_dict() + return obj + + return json.dumps(data, default=sanitizer) + + def default(self, response, result): + response.content_type = 'application/json' + response.body = self.to_json(result) + + +class Resource(object): + """ + WSGI app that handles (de)serialization and controller dispatch. + + Reads routing information supplied by RoutesMiddleware and calls + the requested action method upon its deserializer, controller, + and serializer. Those three objects may implement any of the basic + controller action methods (create, update, show, index, delete) + along with any that may be specified in the api router. A 'default' + method may also be implemented to be used in place of any + non-implemented actions. Deserializer methods must accept a request + argument and return a dictionary. Controller methods must accept a + request argument. Additionally, they must also accept keyword + arguments that represent the keys returned by the Deserializer. They + may raise a webob.exc exception or return a dict, which will be + serialized by requested content type. + """ + + def __init__(self, controller, deserializer=None, serializer=None): + """ + :param controller: object that implement methods created by routes lib + :param deserializer: object that supports webob request deserialization + through controller-like actions + :param serializer: object that supports webob response serialization + through controller-like actions + """ + self.controller = controller + self.serializer = serializer or JSONResponseSerializer() + self.deserializer = deserializer or JSONRequestDeserializer() + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, request): + """WSGI method that controls (de)serialization and method dispatch.""" + action_args = self.get_action_args(request.environ) + action = action_args.pop('action', None) + + deserialized_request = self.dispatch(self.deserializer, + action, request) + action_args.update(deserialized_request) + + action_result = self.dispatch(self.controller, action, + request, **action_args) + try: + response = webob.Response(request=request) + self.dispatch(self.serializer, action, response, action_result) + return response + + # return unserializable result (typically a webob exc) + except Exception: + return action_result + + def dispatch(self, obj, action, *args, **kwargs): + """Find action-specific method on self and call it.""" + try: + method = getattr(obj, action) + except AttributeError: + method = getattr(obj, 'default') + + return method(*args, **kwargs) + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args diff --git a/barbican/context.py b/barbican/context.py new file mode 100644 index 00000000..f926071e --- /dev/null +++ b/barbican/context.py @@ -0,0 +1,90 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from barbican.api import policy +from barbican.openstack.common import local +from barbican.openstack.common import uuidutils + + +class RequestContext(object): + """ + Stores information about the security context under which the user + accesses the system, as well as additional request information. + """ + + def __init__(self, auth_tok=None, user=None, tenant=None, roles=None, + is_admin=False, read_only=False, show_deleted=False, + owner_is_tenant=True, service_catalog=None, + policy_enforcer=None): + self.auth_tok = auth_tok + self.user = user + self.tenant = tenant + self.roles = roles or [] + self.read_only = read_only + self._show_deleted = show_deleted + self.owner_is_tenant = owner_is_tenant + self.request_id = uuidutils.generate_uuid() + self.service_catalog = service_catalog + self.policy_enforcer = policy_enforcer or policy.Enforcer() + self.is_admin = is_admin + if not self.is_admin: + self.is_admin = \ + self.policy_enforcer.check_is_admin(self) + + if not hasattr(local.store, 'context'): + self.update_store() + + def to_dict(self): + # NOTE(ameade): These keys are named to correspond with the default + # format string for logging the context in openstack common + return { + 'request_id': self.request_id, + + #NOTE(bcwaldon): openstack-common logging expects 'user' + 'user': self.user, + 'user_id': self.user, + + #NOTE(bcwaldon): openstack-common logging expects 'tenant' + 'tenant': self.tenant, + 'tenant_id': self.tenant, + 'project_id': self.tenant, + + 'is_admin': self.is_admin, + 'read_deleted': self.show_deleted, + 'roles': self.roles, + 'auth_token': self.auth_tok, + 'service_catalog': self.service_catalog, + } + + @classmethod + def from_dict(cls, values): + return cls(**values) + + def update_store(self): + local.store.context = self + + @property + def owner(self): + """Return the owner to correlate with key.""" + return self.tenant if self.owner_is_tenant else self.user + + @property + def show_deleted(self): + """Admins can see deleted by default""" + if self._show_deleted or self.is_admin: + return True + return False diff --git a/etc/barbican/barbican-api-paste.ini b/etc/barbican/barbican-api-paste.ini index 63ac4b17..ae94425a 100644 --- a/etc/barbican/barbican-api-paste.ini +++ b/etc/barbican/barbican-api-paste.ini @@ -1,10 +1,13 @@ -# Use this pipeline for Barbican API - DEFAULT -[pipeline:main] -pipeline = simple apiapp - #Use this pipeline for keystone auth +[pipeline:main] #[pipeline:barbican-api-keystone] -#pipeline = keystone_authtoken apiapp +pipeline = keystone_authtoken context apiapp + +# Use this pipeline for Barbican API - DEFAULT no authentication +#[pipeline:main] +#pipeline = unauthenticated-context apiapp +#pipeline = simple apiapp + [app:apiapp] paste.app_factory = barbican.api.app:create_main_app @@ -12,9 +15,16 @@ paste.app_factory = barbican.api.app:create_main_app [filter:simple] paste.filter_factory = barbican.api.middleware.simple:SimpleFilter.factory +[filter:unauthenticated-context] +paste.filter_factory = barbican.api.middleware.context:UnauthenticatedContextMiddleware.factory + [filter:context] paste.filter_factory = barbican.api.middleware.context:ContextMiddleware.factory - + +[filter:keystonecontext] +paste.filter_factory = barbican.api.middleware.context:BarbicanKeystoneContext.factory + + [filter:keystone_authtoken] paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory signing_dir = /tmp/barbican/cache From e73b00271c0282fff852986c370d3747e9325f81 Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Fri, 24 May 2013 00:38:21 -0700 Subject: [PATCH 3/6] creating a request context after token authorization that contains user particulars such as tenant-id, user-id etc --- barbican/api/middleware/context.py | 2 ++ barbican/api/policy.py | 15 ++++++++++++++- barbican/context.py | 2 ++ etc/barbican/barbican-api-paste.ini | 18 ++++++------------ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/barbican/api/middleware/context.py b/barbican/api/middleware/context.py index d6d8c813..a86d1d4b 100644 --- a/barbican/api/middleware/context.py +++ b/barbican/api/middleware/context.py @@ -77,8 +77,10 @@ class ContextMiddleware(BaseContextMiddleware): """ if req.headers.get('X-Identity-Status') == 'Confirmed': req.context = self._get_authenticated_context(req) + LOG.debug("==== Inserted barbican auth request context: %s ====" % (req.context.to_dict())) elif CONF.allow_anonymous_access: req.context = self._get_anonymous_context() + LOG.debug("==== Inserted barbican unauth request context: %s ====" % (req.context.to_dict())) else: raise webob.exc.HTTPUnauthorized() diff --git a/barbican/api/policy.py b/barbican/api/policy.py index 0a8efc7e..993db2c2 100644 --- a/barbican/api/policy.py +++ b/barbican/api/policy.py @@ -40,7 +40,9 @@ CONF.register_opts(policy_opts) DEFAULT_RULES = { - 'default': policy.TrueCheck(), + 'context_is_admin': policy.RoleCheck('role', 'admin'), + 'default': policy.TrueCheck(), + 'manage_re_key': policy.RoleCheck('role', 'admin'), } @@ -141,3 +143,14 @@ class Enforcer(object): :returns: A non-False value if access is allowed. """ return self._check(context, action, target) + + + def check_is_admin(self, context): + """Check if the given context is associated with an admin role, + as defined via the 'context_is_admin' RBAC rule. + + :param context: request context + :returns: A non-False value if context role is admin. + """ + target = context.to_dict() + return self.check(context, 'context_is_admin', target) diff --git a/barbican/context.py b/barbican/context.py index f926071e..db8f933d 100644 --- a/barbican/context.py +++ b/barbican/context.py @@ -36,6 +36,8 @@ class RequestContext(object): self.roles = roles or [] self.read_only = read_only self._show_deleted = show_deleted + # (mkbhanda) possibly domain could be owner + # brings us to the key scope question self.owner_is_tenant = owner_is_tenant self.request_id = uuidutils.generate_uuid() self.service_catalog = service_catalog diff --git a/etc/barbican/barbican-api-paste.ini b/etc/barbican/barbican-api-paste.ini index ae94425a..dd13e12a 100644 --- a/etc/barbican/barbican-api-paste.ini +++ b/etc/barbican/barbican-api-paste.ini @@ -1,13 +1,11 @@ -#Use this pipeline for keystone auth -[pipeline:main] -#[pipeline:barbican-api-keystone] -pipeline = keystone_authtoken context apiapp - # Use this pipeline for Barbican API - DEFAULT no authentication -#[pipeline:main] -#pipeline = unauthenticated-context apiapp -#pipeline = simple apiapp +[pipeline:main] +pipeline = unauthenticated-context apiapp +####pipeline = simple apiapp +#Use this pipeline for keystone auth +[pipeline:barbican-api-keystone] +pipeline = keystone_authtoken context apiapp [app:apiapp] paste.app_factory = barbican.api.app:create_main_app @@ -21,10 +19,6 @@ paste.filter_factory = barbican.api.middleware.context:UnauthenticatedContextMid [filter:context] paste.filter_factory = barbican.api.middleware.context:ContextMiddleware.factory -[filter:keystonecontext] -paste.filter_factory = barbican.api.middleware.context:BarbicanKeystoneContext.factory - - [filter:keystone_authtoken] paste.filter_factory = keystoneclient.middleware.auth_token:filter_factory signing_dir = /tmp/barbican/cache From 752b78d6d540b5aaa38983ae39ab875054389f21 Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Fri, 24 May 2013 18:10:30 -0700 Subject: [PATCH 4/6] using Middleware from api.middleware.__init__, just what we need. Added debug middleware to the __init__ file. Also extended copyright range from 2011-2012. --- barbican/api/middleware/__init__.py | 39 +++++++++++++++++++++++++++++ barbican/api/middleware/context.py | 6 ++--- barbican/context.py | 2 +- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/barbican/api/middleware/__init__.py b/barbican/api/middleware/__init__.py index 84a91e9f..c93ec4a1 100644 --- a/barbican/api/middleware/__init__.py +++ b/barbican/api/middleware/__init__.py @@ -59,3 +59,42 @@ class Middleware(object): response = req.get_response(self.application) response.request = req return self.process_response(response) + + +# Brought over from an OpenStack project +class Debug(Middleware): + """ + Helper class that can be inserted into any WSGI application chain + to get information about the request and response. + """ + + @webob.dec.wsgify + def __call__(self, req): + print ("*" * 40) + " REQUEST ENVIRON" + for key, value in req.environ.items(): + print key, "=", value + print + resp = req.get_response(self.application) + + print ("*" * 40) + " RESPONSE HEADERS" + for (key, value) in resp.headers.iteritems(): + print key, "=", value + print + + resp.app_iter = self.print_generator(resp.app_iter) + + return resp + + @staticmethod + def print_generator(app_iter): + """ + Iterator that prints the contents of a wrapper string iterator + when iterated. + """ + print ("*" * 40) + " BODY" + for part in app_iter: + sys.stdout.write(part) + sys.stdout.flush() + yield part + print + diff --git a/barbican/api/middleware/context.py b/barbican/api/middleware/context.py index a86d1d4b..f9ae780d 100644 --- a/barbican/api/middleware/context.py +++ b/barbican/api/middleware/context.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2013 OpenStack LLC. +# Copyright 2011-2013 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -21,7 +21,7 @@ from oslo.config import cfg import webob.exc from barbican.api import policy -from barbican.common import wsgi +from barbican.api.middleware import Middleware import barbican.context import barbican.openstack.common.log as logging @@ -47,7 +47,7 @@ CONF.register_opts(context_opts) LOG = logging.getLogger(__name__) -class BaseContextMiddleware(wsgi.Middleware): +class BaseContextMiddleware(Middleware): def process_response(self, resp): try: request_id = resp.request.context.request_id diff --git a/barbican/context.py b/barbican/context.py index db8f933d..e7ce7fcc 100644 --- a/barbican/context.py +++ b/barbican/context.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2013 OpenStack LLC. +# Copyright 2011-2013 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may From 8f6a59243b37520a1c7c88d30ebd2d62c33ea450 Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Fri, 24 May 2013 18:19:22 -0700 Subject: [PATCH 5/6] do not need wsgi, using Middleware class from barbican.api.middleware.__init__.py --- barbican/common/wsgi.py | 621 ---------------------------------------- 1 file changed, 621 deletions(-) delete mode 100644 barbican/common/wsgi.py diff --git a/barbican/common/wsgi.py b/barbican/common/wsgi.py deleted file mode 100644 index 354caf08..00000000 --- a/barbican/common/wsgi.py +++ /dev/null @@ -1,621 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Utility methods for working with WSGI servers -""" - -import datetime -import errno -import json -import logging -import os -import signal -import sys -import time - -import eventlet -from eventlet.green import socket, ssl -import eventlet.greenio -import eventlet.wsgi -from oslo.config import cfg -# mkbhanda -- if we want this, need to include routes in pip-requires -#import routes -#import routes.middleware -import webob.dec -import webob.exc - -from barbican.common import exception -from barbican.common import utils -import barbican.openstack.common.log as os_logging - - -bind_opts = [ - cfg.StrOpt('bind_host', default='0.0.0.0', - help=_('Address to bind the server. Useful when ' - 'selecting a particular network interface.')), - cfg.IntOpt('bind_port', - help=_('The port on which the server will listen.')), -] - -socket_opts = [ - cfg.IntOpt('backlog', default=4096, - help=_('The backlog value that will be used when creating the ' - 'TCP listener socket.')), - cfg.IntOpt('tcp_keepidle', default=600, - help=_('The value for the socket option TCP_KEEPIDLE. This is' - 'the time in seconds that the connection must be idle ' - 'before TCP starts sending keepalive probes.')), - cfg.StrOpt('ca_file', help=_('CA certificate file to use to verify ' - 'connecting clients.')), - cfg.StrOpt('cert_file', help=_('Certificate file to use when starting API ' - 'server securely.')), - cfg.StrOpt('key_file', help=_('Private key file to use when starting API ' - 'server securely.')), -] - -workers_opt = cfg.IntOpt('workers', default=1, - help=_('The number of child process workers that ' - 'will be created to service API requests.')) - -CONF = cfg.CONF -CONF.register_opts(bind_opts) -CONF.register_opts(socket_opts) -CONF.register_opt(workers_opt) - - -class WritableLogger(object): - """A thin wrapper that responds to `write` and logs.""" - - def __init__(self, logger, level=logging.DEBUG): - self.logger = logger - self.level = level - - def write(self, msg): - self.logger.log(self.level, msg.strip("\n")) - - -def get_bind_addr(default_port=None): - """Return the host and port to bind to.""" - return (CONF.bind_host, CONF.bind_port or default_port) - - -def get_socket(default_port): - """ - Bind socket to bind ip:port in conf - - note: Mostly comes from Swift with a few small changes... - - :param default_port: port to bind to if none is specified in conf - - :returns : a socket object as returned from socket.listen or - ssl.wrap_socket if conf specifies cert_file - """ - bind_addr = get_bind_addr(default_port) - - # TODO(jaypipes): eventlet's greened socket module does not actually - # support IPv6 in getaddrinfo(). We need to get around this in the - # future or monitor upstream for a fix - address_family = [ - addr[0] for addr in socket.getaddrinfo(bind_addr[0], - bind_addr[1], - socket.AF_UNSPEC, - socket.SOCK_STREAM) - if addr[0] in (socket.AF_INET, socket.AF_INET6) - ][0] - - cert_file = CONF.cert_file - key_file = CONF.key_file - use_ssl = cert_file or key_file - if use_ssl and (not cert_file or not key_file): - raise RuntimeError(_("When running server in SSL mode, you must " - "specify both a cert_file and key_file " - "option value in your configuration file")) - - def wrap_ssl(sock): - utils.validate_key_cert(key_file, cert_file) - - ssl_kwargs = { - 'server_side': True, - 'certfile': cert_file, - 'keyfile': key_file, - 'cert_reqs': ssl.CERT_NONE, - } - - if CONF.ca_file: - ssl_kwargs['ca_certs'] = CONF.ca_file - ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED - - return ssl.wrap_socket(sock, **ssl_kwargs) - - sock = utils.get_test_suite_socket() - retry_until = time.time() + 30 - - if sock and use_ssl: - sock = wrap_ssl(sock) - while not sock and time.time() < retry_until: - try: - sock = eventlet.listen(bind_addr, - backlog=CONF.backlog, - family=address_family) - if use_ssl: - sock = wrap_ssl(sock) - - except socket.error as err: - if err.args[0] != errno.EADDRINUSE: - raise - eventlet.sleep(0.1) - if not sock: - raise RuntimeError(_("Could not bind to %s:%s after trying for 30 " - "seconds") % bind_addr) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - # in my experience, sockets can hang around forever without keepalive - sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - - # This option isn't available in the OS X version of eventlet - if hasattr(socket, 'TCP_KEEPIDLE'): - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, - CONF.tcp_keepidle) - - return sock - - -class Server(object): - """Server class to manage multiple WSGI sockets and applications.""" - - def __init__(self, threads=1000): - self.threads = threads - self.children = [] - self.running = True - - def start(self, application, default_port): - """ - Run a WSGI server with the given application. - - :param application: The application to be run in the WSGI server - :param default_port: Port to bind to if none is specified in conf - """ - pgid = os.getpid() - try: - # NOTE(flaper87): Make sure this process - # runs in its own process group. - os.setpgid(pgid, pgid) - except OSError: - # watch out for failures with - # setpgid fails with EPERM because of fresh session creation - pgid = 0 - - def kill_children(*args): - """Kills the entire process group.""" - signal.signal(signal.SIGTERM, signal.SIG_IGN) - signal.signal(signal.SIGINT, signal.SIG_IGN) - self.running = False - os.killpg(pgid, signal.SIGTERM) - - def hup(*args): - """ - Shuts down the server, but allows running requests to complete - """ - signal.signal(signal.SIGHUP, signal.SIG_IGN) - self.running = False - - self.application = application - self.sock = get_socket(default_port) - - os.umask(027) # ensure files are created with the correct privileges - self.logger = os_logging.getLogger('eventlet.wsgi.server') - - if CONF.workers == 0: - # Useful for profiling, test, debug etc. - self.pool = self.create_pool() - self.pool.spawn_n(self._single_run, self.application, self.sock) - return - else: - self.logger.info(_("Starting %d workers") % CONF.workers) - signal.signal(signal.SIGTERM, kill_children) - signal.signal(signal.SIGINT, kill_children) - signal.signal(signal.SIGHUP, hup) - while len(self.children) < CONF.workers: - self.run_child() - - def create_pool(self): - return eventlet.GreenPool(size=self.threads) - - def wait_on_children(self): - while self.running: - try: - pid, status = os.wait() - if os.WIFEXITED(status) or os.WIFSIGNALED(status): - self.logger.info(_('Removing dead child %s') % pid) - self.children.remove(pid) - if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0: - self.logger.error(_('Not respawning child %d, cannot ' - 'recover from termination') % pid) - if not self.children: - self.logger.info( - _('All workers have terminated. Exiting')) - self.running = False - else: - self.run_child() - except OSError as err: - if err.errno not in (errno.EINTR, errno.ECHILD): - raise - except KeyboardInterrupt: - self.logger.info(_('Caught keyboard interrupt. Exiting.')) - break - eventlet.greenio.shutdown_safe(self.sock) - self.sock.close() - self.logger.debug(_('Exited')) - - def wait(self): - """Wait until all servers have completed running.""" - try: - if self.children: - self.wait_on_children() - else: - self.pool.waitall() - except KeyboardInterrupt: - pass - - def run_child(self): - pid = os.fork() - if pid == 0: - signal.signal(signal.SIGHUP, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - # ignore the interrupt signal to avoid a race whereby - # a child worker receives the signal before the parent - # and is respawned unneccessarily as a result - signal.signal(signal.SIGINT, signal.SIG_IGN) - self.run_server() - self.logger.info(_('Child %d exiting normally') % os.getpid()) - # self.pool.waitall() has been called by run_server, so - # its safe to exit here - sys.exit(0) - else: - self.logger.info(_('Started child %s') % pid) - self.children.append(pid) - - def run_server(self): - """Run a WSGI server.""" - if cfg.CONF.pydev_worker_debug_host: - utils.setup_remote_pydev_debug(cfg.CONF.pydev_worker_debug_host, - cfg.CONF.pydev_worker_debug_port) - - eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0" - try: - eventlet.hubs.use_hub('poll') - except Exception: - msg = _("eventlet 'poll' hub is not available on this platform") - raise exception.WorkerCreationFailure(reason=msg) - self.pool = self.create_pool() - try: - eventlet.wsgi.server(self.sock, - self.application, - log=WritableLogger(self.logger), - custom_pool=self.pool) - except socket.error as err: - if err[0] != errno.EINVAL: - raise - self.pool.waitall() - - def _single_run(self, application, sock): - """Start a WSGI server in a new green thread.""" - self.logger.info(_("Starting single process server")) - eventlet.wsgi.server(sock, application, custom_pool=self.pool, - log=WritableLogger(self.logger)) - - -class Middleware(object): - """ - Base WSGI middleware wrapper. These classes require an application to be - initialized that will be called next. By default the middleware will - simply call its wrapped app, or you can override __call__ to customize its - behavior. - """ - - def __init__(self, application): - self.application = application - - @classmethod - def factory(cls, global_conf, **local_conf): - def filter(app): - return cls(app) - return filter - - def process_request(self, req): - """ - Called on each request. - - If this returns None, the next application down the stack will be - executed. If it returns a response then that response will be returned - and execution will stop here. - - """ - return None - - def process_response(self, response): - """Do whatever you'd like to the response.""" - return response - - @webob.dec.wsgify - def __call__(self, req): - response = self.process_request(req) - if response: - return response - response = req.get_response(self.application) - response.request = req - return self.process_response(response) - - -class Debug(Middleware): - """ - Helper class that can be inserted into any WSGI application chain - to get information about the request and response. - """ - - @webob.dec.wsgify - def __call__(self, req): - print ("*" * 40) + " REQUEST ENVIRON" - for key, value in req.environ.items(): - print key, "=", value - print - resp = req.get_response(self.application) - - print ("*" * 40) + " RESPONSE HEADERS" - for (key, value) in resp.headers.iteritems(): - print key, "=", value - print - - resp.app_iter = self.print_generator(resp.app_iter) - - return resp - - @staticmethod - def print_generator(app_iter): - """ - Iterator that prints the contents of a wrapper string iterator - when iterated. - """ - print ("*" * 40) + " BODY" - for part in app_iter: - sys.stdout.write(part) - sys.stdout.flush() - yield part - print - - -# class APIMapper(routes.Mapper): -# """ -# Handle route matching when url is '' because routes.Mapper returns -# an error in this case. -# """ - -# def routematch(self, url=None, environ=None): -# if url is "": -# result = self._match("", environ) -# return result[0], result[1] -# return routes.Mapper.routematch(self, url, environ) - - -# class Router(object): -# """ -# WSGI middleware that maps incoming requests to WSGI apps. -# """ - -# def __init__(self, mapper): -# """ -# Create a router for the given routes.Mapper. - -# Each route in `mapper` must specify a 'controller', which is a -# WSGI app to call. You'll probably want to specify an 'action' as -# well and have your controller be a wsgi.Controller, who will route -# the request to the action method. - -# Examples: -# mapper = routes.Mapper() -# sc = ServerController() - -# # Explicit mapping of one route to a controller+action -# mapper.connect(None, "/svrlist", controller=sc, action="list") - -# # Actions are all implicitly defined -# mapper.resource("server", "servers", controller=sc) - -# # Pointing to an arbitrary WSGI app. You can specify the -# # {path_info:.*} parameter so the target app can be handed just that -# # section of the URL. -# mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) -# """ -# mapper.redirect("", "/") -# self.map = mapper -# self._router = routes.middleware.RoutesMiddleware(self._dispatch, -# self.map) - -# @classmethod -# def factory(cls, global_conf, **local_conf): -# return cls(APIMapper()) - -# @webob.dec.wsgify -# def __call__(self, req): -# """ -# Route the incoming request to a controller based on self.map. -# If no match, return a 404. -# """ -# return self._router - -# @staticmethod -# @webob.dec.wsgify -# def _dispatch(req): -# """ -# Called by self._router after matching the incoming request to a route -# and putting the information into req.environ. Either returns 404 -# or the routed WSGI app's response. -# """ -# match = req.environ['wsgiorg.routing_args'][1] -# if not match: -# return webob.exc.HTTPNotFound() -# app = match['controller'] -# return app - - -class Request(webob.Request): - """Add some Openstack API-specific logic to the base webob.Request.""" - - def best_match_content_type(self): - """Determine the requested response content-type.""" - supported = ('application/json',) - bm = self.accept.best_match(supported) - return bm or 'application/json' - - def get_content_type(self, allowed_content_types): - """Determine content type of the request body.""" - if "Content-Type" not in self.headers: - raise exception.InvalidContentType(content_type=None) - - content_type = self.content_type - - if content_type not in allowed_content_types: - raise exception.InvalidContentType(content_type=content_type) - else: - return content_type - - -class JSONRequestDeserializer(object): - def has_body(self, request): - """ - Returns whether a Webob.Request object will possess an entity body. - - :param request: Webob.Request object - """ - if 'transfer-encoding' in request.headers: - return True - elif request.content_length > 0: - return True - - return False - - def from_json(self, datastring): - try: - return json.loads(datastring) - except ValueError: - msg = _('Malformed JSON in request body.') - raise webob.exc.HTTPBadRequest(explanation=msg) - - def default(self, request): - if self.has_body(request): - return {'body': self.from_json(request.body)} - else: - return {} - - -class JSONResponseSerializer(object): - - def to_json(self, data): - def sanitizer(obj): - if isinstance(obj, datetime.datetime): - return obj.isoformat() - if hasattr(obj, "to_dict"): - return obj.to_dict() - return obj - - return json.dumps(data, default=sanitizer) - - def default(self, response, result): - response.content_type = 'application/json' - response.body = self.to_json(result) - - -class Resource(object): - """ - WSGI app that handles (de)serialization and controller dispatch. - - Reads routing information supplied by RoutesMiddleware and calls - the requested action method upon its deserializer, controller, - and serializer. Those three objects may implement any of the basic - controller action methods (create, update, show, index, delete) - along with any that may be specified in the api router. A 'default' - method may also be implemented to be used in place of any - non-implemented actions. Deserializer methods must accept a request - argument and return a dictionary. Controller methods must accept a - request argument. Additionally, they must also accept keyword - arguments that represent the keys returned by the Deserializer. They - may raise a webob.exc exception or return a dict, which will be - serialized by requested content type. - """ - - def __init__(self, controller, deserializer=None, serializer=None): - """ - :param controller: object that implement methods created by routes lib - :param deserializer: object that supports webob request deserialization - through controller-like actions - :param serializer: object that supports webob response serialization - through controller-like actions - """ - self.controller = controller - self.serializer = serializer or JSONResponseSerializer() - self.deserializer = deserializer or JSONRequestDeserializer() - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, request): - """WSGI method that controls (de)serialization and method dispatch.""" - action_args = self.get_action_args(request.environ) - action = action_args.pop('action', None) - - deserialized_request = self.dispatch(self.deserializer, - action, request) - action_args.update(deserialized_request) - - action_result = self.dispatch(self.controller, action, - request, **action_args) - try: - response = webob.Response(request=request) - self.dispatch(self.serializer, action, response, action_result) - return response - - # return unserializable result (typically a webob exc) - except Exception: - return action_result - - def dispatch(self, obj, action, *args, **kwargs): - """Find action-specific method on self and call it.""" - try: - method = getattr(obj, action) - except AttributeError: - method = getattr(obj, 'default') - - return method(*args, **kwargs) - - def get_action_args(self, request_environment): - """Parse dictionary created by routes library.""" - try: - args = request_environment['wsgiorg.routing_args'][1].copy() - except Exception: - return {} - - try: - del args['controller'] - except KeyError: - pass - - try: - del args['format'] - except KeyError: - pass - - return args From 3ee5d7e24426634418ce44c8a779853b62c2c97a Mon Sep 17 00:00:00 2001 From: "Malini K. Bhandaru" Date: Fri, 24 May 2013 18:23:03 -0700 Subject: [PATCH 6/6] put back the copyright range to that of the original copied file .. instead of the incremented 2013 --- barbican/api/middleware/context.py | 2 +- barbican/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/barbican/api/middleware/context.py b/barbican/api/middleware/context.py index f9ae780d..3a6d46e2 100644 --- a/barbican/api/middleware/context.py +++ b/barbican/api/middleware/context.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011-2013 OpenStack LLC. +# Copyright 2011-2012 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may diff --git a/barbican/context.py b/barbican/context.py index e7ce7fcc..f7332db5 100644 --- a/barbican/context.py +++ b/barbican/context.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011-2013 OpenStack LLC. +# Copyright 2011-2012 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may