From 76d0164fb009fbd54dd6f1100d55c15943ed63e5 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Mon, 22 Apr 2019 18:04:41 +0200 Subject: [PATCH] Fake unused packages Many packages that are automatically imported when loading cinder modules are only used for normal Cinder operation and are not necessary for cinderlib's execution. One example of this happening is when cinderlib loads a Cinder module to get configuration options but won't execute any of the code present in that module. This patch fakes these unnecessary packages, providing faster load times, reduced footprint, and the possibility for distributions to create a cinderlib package or containers with up to 40% fewer dependencies. Change-Id: If577b9163d4e942b701db4b2a47cd66e6bd17b6f --- .zuul.yaml | 1 + cinderlib/__init__.py | 1 + cinderlib/_fake_packages.py | 170 ++++++++++++++++++++++++++++++++++ cinderlib/persistence/dbms.py | 6 +- doc/source/installation.rst | 51 ++++++++++ playbooks/setup-ceph.yaml | 11 +++ 6 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 cinderlib/_fake_packages.py diff --git a/.zuul.yaml b/.zuul.yaml index 1b26c58..83b5886 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -47,6 +47,7 @@ pre-run: playbooks/setup-lvm.yaml nodeset: centos-7 +# The Ceph job tests cinderlib without unnecessary libraries - job: name: cinderlib-ceph-functional parent: openstack-tox-functional-with-sudo diff --git a/cinderlib/__init__.py b/cinderlib/__init__.py index 5f7cb43..3b9dc3d 100644 --- a/cinderlib/__init__.py +++ b/cinderlib/__init__.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import pkg_resources +from cinderlib import _fake_packages # noqa F401 from cinderlib import cinderlib from cinderlib import objects from cinderlib import serialization diff --git a/cinderlib/_fake_packages.py b/cinderlib/_fake_packages.py new file mode 100644 index 0000000..32a2a20 --- /dev/null +++ b/cinderlib/_fake_packages.py @@ -0,0 +1,170 @@ +# Copyright (c) 2019, Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Fake unnecessary packages + +There are many packages that are automatically imported when loading cinder +modules, and are used for normal Cinder operation, but they are not necessary +for cinderlib's execution. One example of this happening is when cinderlib +loads a module to get configuration options but won't execute any of the code +present in that module. + +This module fakes these packages providing the following benefits: + +- Faster load times +- Reduced memory footprint +- Distributions can create a cinderlib package with fewer dependencies. +""" +from __future__ import absolute_import +try: + # Only present and needed in Python >= 3.4 + from importlib import machinery +except ImportError: + pass +import logging +import sys +import types + +from oslo_config import cfg + + +__all__ = ['faker'] +PACKAGES = [ + 'glanceclient', 'novaclient', 'swiftclient', 'barbicanclient', 'cursive', + 'keystoneauth1', 'keystonemiddleware', 'keystoneclient', 'castellan', + 'oslo_reports', 'oslo_policy', 'oslo_messaging', 'osprofiler', 'paste', + 'oslo_middleware', 'webob', 'pyparsing', 'routes', 'jsonschema', 'os_win', + 'oauth2client', 'oslo_upgradecheck', 'googleapiclient', 'pastedeploy', +] +_DECORATOR_CLASSES = (types.FunctionType, types.MethodType) +LOG = logging.getLogger(__name__) + + +class _FakeObject(object): + """Generic fake object: Iterable, Class, decorator, etc.""" + def __init__(self, *args, **kwargs): + self.__key_value__ = {} + + def __len__(self): + return len(self.__key_value__) + + def __contains__(self, key): + return key in self.__key_value__ + + def __iter__(self): + return iter(self.__key_value__) + + def __mro_entries__(self, bases): + return (self.__class__,) + + def __setitem__(self, key, value): + self.__key_value__[key] = value + + def _new_instance(self, class_name): + attrs = {'__module__': self.__module__ + '.' + self.__class__.__name__} + return type(class_name, (self.__class__,), attrs)() + + # No need to define __class_getitem__, as __getitem__ has the priority + def __getitem__(self, key): + if key in self.__key_value__.get: + return self.__key_value__.get[key] + return self._new_instance(key) + + def __getattr__(self, key): + return self._new_instance(key) + + def __call__(self, *args, **kw): + # If we are a decorator return the method that we are decorating + if args and isinstance(args[0], _DECORATOR_CLASSES): + return args[0] + return self + + def __repr__(self): + return self.__qualname__ + + +class Faker(object): + """Fake Finder and Loader for whole packages.""" + def __init__(self, packages): + self.faked_modules = [] + self.packages = packages + + def _fake_module(self, name): + """Dynamically create a module as close as possible to a real one.""" + LOG.debug('Faking %s', name) + attributes = { + '__doc__': None, + '__name__': name, + '__file__': name, + '__loader__': self, + '__builtins__': __builtins__, + '__package__': name.rsplit('.', 1)[0] if '.' in name else None, + '__repr__': lambda self: self.__name__, + '__getattr__': lambda self, name: ( + type(name, (_FakeObject,), {'__module__': self.__name__})()), + } + + keys = ['__doc__', '__name__', '__file__', '__builtins__', + '__package__'] + + # Path only present at the package level + if '.' not in name: + attributes['__path__'] = [name] + keys.append('__path__') + + # We only want to show some of our attributes + attributes.update(__dict__={k: attributes[k] for k in keys}, + __dir__=lambda self: keys) + + # Create the class and instantiate it + module_class = type(name, (types.ModuleType,), attributes) + self.faked_modules.append(name) + return module_class(name) + + def find_module(self, fullname, path=None): + """Find a module and return a Loader if it's one of ours or None.""" + package = fullname.split('.')[0] + # If it's one of ours, then we are the loader + if package in self.packages: + return self + return None + + def load_module(self, fullname): + """Create a new Fake module if it's not already present.""" + if fullname in sys.modules: + return sys.modules[fullname] + + sys.modules[fullname] = self._fake_module(fullname) + return sys.modules[fullname] + + def find_spec(self, fullname, path=None, target=None): + """Return our spec it it's one of our packages or None.""" + if self.find_module(fullname): + return machinery.ModuleSpec(fullname, + self, + is_package='.' not in fullname) + return None + + def create_module(self, spec): + """Fake a module.""" + return self._fake_module(spec.name) + + +# cinder.quota_utils manually imports keystone_authtoken config group, so we +# create a fake one to avoid failure. +cfg.CONF.register_opts([cfg.StrOpt('fake')], group='keystone_authtoken') + +# Create faker and add it to the list of Finders +faker = Faker(PACKAGES) +sys.meta_path.insert(0, faker) diff --git a/cinderlib/persistence/dbms.py b/cinderlib/persistence/dbms.py index 9feb2d4..9e8938e 100644 --- a/cinderlib/persistence/dbms.py +++ b/cinderlib/persistence/dbms.py @@ -104,9 +104,9 @@ class DBPersistence(persistence_base.PersistenceDriverBase): # This is for Pike if hasattr(sqla_api, '_FACADE'): sqla_api._FACADE = None - # This is for Queens and Rocky (untested) - elif hasattr(sqla_api, 'configure'): - sqla_api.configure(cfg.CONF) + # This is for Queens or later + elif hasattr(sqla_api, 'main_context_manager'): + sqla_api.main_context_manager.configure(**dict(cfg.CONF.database)) def _create_key_value_table(self): models.BASE.metadata.create_all(sqla_api.get_engine(), diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 2015ee8..40d76b0 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -110,5 +110,56 @@ Once you have a copy of the source, you can install it with: $ virtualenv cinder $ python setup.py install + +Dependencies +------------ + +*Cinderlib* has less functionality than Cinder, which results in fewer required +libraries. + +When installing from PyPi or source, we'll get all the dependencies regardless +of whether they are needed by *cinderlib* or not, since the Cinder Python +package specifies all the dependencies. Installing from packages may result in +fewer dependencies, but this will depend on the distribution package itself. + +To increase loading speed, and reduce memory footprint and dependencies, +*cinderlib* fakes all unnecessary packages at runtime if they have not already +been loaded. + +This can be convenient when creating containers, as one can remove unnecessary +packages on the same layer *cinderlib* gets installed to get a smaller +containers. + +If our application uses any of the packages *cinderlib* fakes, we just have to +import them before importing *cinderlib*. This way *cinderlib* will not fake +them. + +The list of top level packages unnecessary for *cinderlib* are: + +- castellan +- cursive +- googleapiclient +- jsonschema +- keystoneauth1 +- keystonemiddleware +- oauth2client +- os-win +- oslo.messaging +- oslo.middleware +- oslo.policy +- oslo.reports +- oslo.upgradecheck +- osprofiler +- paste +- pastedeploy +- pyparsing +- python-barbicanclient +- python-glanceclient +- python-novaclient +- python-swiftclient +- python-keystoneclient +- routes +- webob + .. _Github repo: https://github.com/openstack/cinderlib .. _tarball: https://github.com/openstack/cinderlib/tarball/master diff --git a/playbooks/setup-ceph.yaml b/playbooks/setup-ceph.yaml index 46db2b0..7035227 100644 --- a/playbooks/setup-ceph.yaml +++ b/playbooks/setup-ceph.yaml @@ -30,6 +30,17 @@ name: six state: absent + # Leave pyparsing, as it's needed by tox through the packaging library. + - name: Remove Python packages unnecessary for cinderlib + pip: + name: ['glanceclient', 'novaclient', 'swiftclient', 'barbicanclient', + 'cursive', 'keystoneauth1', 'keystonemiddleware', 'webob', + 'keystoneclient', 'castellan', 'oslo_reports', 'oslo_policy', + 'oslo_messaging', 'osprofiler', 'oauth2client', 'paste', + 'oslo_middleware', 'routes', 'jsonschema', 'os-win', + 'oslo_upgradecheck', 'googleapiclient', 'pastedeploy'] + state: absent + - name: Install ceph-common and epel-release yum: name: ['epel-release', 'ceph-common']