From 5d2dfd94dd086c741c5e50e7d2047769d4e1454f Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 18 Jan 2017 12:17:19 +0000 Subject: [PATCH] Drop apache+mod-fastcgi support The ceph project has focussed on running the RADOS Gateway using the embedded civetweb engine for many cycles now; mod-fastcgi is buggy and no longer provided in Ubuntu as of 17.04, so switch to always running in embedded mode. Existing installs will be upgraded to run in this mode, purging apache related packaging and configuration. Change-Id: I90e6c047d78de2b0ebf8c24bd2f2d6d1bfbd5c5d Closes-Bug: 1657370 --- config.yaml | 20 --------- hooks/ceph_radosgw_context.py | 60 ------------------------- hooks/hooks.py | 42 +++-------------- hooks/utils.py | 26 ----------- templates/ceph.conf | 6 --- templates/ports.conf | 11 ----- templates/rgw | 25 ----------- templates/rgw.conf | 25 ----------- tests/basic_deployment.py | 4 +- unit_tests/test_ceph_radosgw_context.py | 7 --- unit_tests/test_hooks.py | 42 ----------------- 11 files changed, 7 insertions(+), 261 deletions(-) delete mode 100644 templates/ports.conf delete mode 100644 templates/rgw delete mode 100644 templates/rgw.conf diff --git a/config.yaml b/config.yaml index 9abae24b..77768f00 100644 --- a/config.yaml +++ b/config.yaml @@ -75,26 +75,6 @@ options: default: False description: | If set to True, supporting services will log to syslog. - use-ceph-optimised-packages: - type: boolean - default: false - description: | - By default apache2 and libapache2-mod-fastcgi will be installed from the - Ubuntu archives. This option allows for an alternate ceph.com install - source which contains patched versions with added support for HTTP - 100-continue. See the following page for more info: - - http://ceph.com/docs/dumpling/radosgw/manual-install/#continue-support - use-embedded-webserver: - type: boolean - default: false - description: | - Newer versions of the Ceph RADOS Gateway support use of an embedded web - container instead of Apache + mod-fastcgi, avoiding some of the nuances - of using the stock mod-fastcgi packages from Ubuntu. - - Enable this option to disable use of Apache and enable the embedded - web container feature. dns-ha: type: boolean default: False diff --git a/hooks/ceph_radosgw_context.py b/hooks/ceph_radosgw_context.py index 389538cc..3b225afa 100644 --- a/hooks/ceph_radosgw_context.py +++ b/hooks/ceph_radosgw_context.py @@ -16,9 +16,7 @@ import os import re import socket import tempfile -import glob import shutil -import subprocess from charmhelpers.contrib.openstack import context from charmhelpers.contrib.hahelpers.cluster import ( @@ -34,7 +32,6 @@ from charmhelpers.core.hookenv import ( relation_ids, related_units, relation_get, - status_set, ) from charmhelpers.contrib.network.ip import ( format_ipv6_addr, @@ -44,55 +41,6 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.storage.linux.ceph import CephConfContext -def is_apache_24(): - if os.path.exists('/etc/apache2/conf-available'): - return True - else: - return False - - -class ApacheContext(context.OSContextGenerator): - interfaces = ['http'] - service_namespace = 'ceph-radosgw' - - def __call__(self): - ctxt = {} - if config('use-embedded-webserver'): - log("Skipping ApacheContext since we are using the embedded " - "webserver") - return {} - - status_set('maintenance', 'configuring apache') - - src = 'files/www/*' - dst = '/var/www/' - log("Installing www scripts", level=DEBUG) - try: - for x in glob.glob(src): - shutil.copy(x, dst) - except IOError as e: - log("Error copying files from '%s' to '%s': %s" % (src, dst, e), - level=WARNING) - - try: - subprocess.check_call(['a2enmod', 'fastcgi']) - subprocess.check_call(['a2enmod', 'rewrite']) - except subprocess.CalledProcessError as e: - log("Error enabling apache modules - %s" % e, level=WARNING) - - try: - if is_apache_24(): - subprocess.check_call(['a2dissite', '000-default']) - else: - subprocess.check_call(['a2dissite', 'default']) - except subprocess.CalledProcessError as e: - log("Error disabling apache sites - %s" % e, level=WARNING) - - ctxt['hostname'] = socket.gethostname() - ctxt['port'] = determine_api_port(config('port'), singlenode_mode=True) - return ctxt - - class HAProxyContext(context.HAProxyContext): def __call__(self): @@ -220,7 +168,6 @@ class MonContext(context.OSContextGenerator): 'hostname': host, 'old_auth': cmp_pkgrevno('radosgw', "0.51") < 0, 'use_syslog': str(config('use-syslog')).lower(), - 'embedded_webserver': config('use-embedded-webserver'), 'loglevel': config('loglevel'), 'port': port, 'ipv6': config('prefer-ipv6') @@ -232,13 +179,6 @@ class MonContext(context.OSContextGenerator): if all([os.path.isfile(p) for p in paths]): ctxt['cms'] = True - if (config('use-ceph-optimised-packages') and - not config('use-embedded-webserver')): - ctxt['disable_100_continue'] = False - else: - # NOTE: currently only applied if NOT using embedded webserver - ctxt['disable_100_continue'] = True - # NOTE(dosaboy): these sections must correspond to what is supported in # the config template. sections = ['global', 'client.radosgw.gateway'] diff --git a/hooks/hooks.py b/hooks/hooks.py index 8d586e82..5a22e964 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -30,7 +30,6 @@ from charmhelpers.core.hookenv import ( relation_set, log, DEBUG, - WARNING, Hooks, UnregisteredHookError, status_set, ) @@ -39,8 +38,8 @@ from charmhelpers.fetch import ( apt_install, apt_purge, add_source, + filter_installed_packages, ) -from charmhelpers.core.host import lsb_release from charmhelpers.payload.execd import execd_preinstall from charmhelpers.core.host import cmp_pkgrevno from charmhelpers.contrib.network.ip import ( @@ -86,21 +85,6 @@ CONFIGS = register_configs() NSS_DIR = '/var/lib/ceph/nss' -def install_ceph_optimised_packages(): - """Inktank provides patched/optimised packages for HTTP 100-continue - support that does has not yet been ported to upstream. These can - optionally be installed from ceph.com archives. - """ - prolog = "http://gitbuilder.ceph.com/" - epilog = "-x86_64-basic/ref/master" - rel = lsb_release()['DISTRIB_CODENAME'] - fastcgi_source = "%slibapache-mod-fastcgi-deb-%s%s" % (prolog, rel, epilog) - apache_source = "%sapache2-deb-%s%s" % (prolog, rel, epilog) - - for source in [fastcgi_source, apache_source]: - add_source(source, key='6EAEAE2203C3951A') - - PACKAGES = [ 'haproxy', 'libnss3-tools', @@ -119,18 +103,13 @@ APACHE_PACKAGES = [ def install_packages(): - status_set('maintenance', 'Installing apt packages') add_source(config('source'), config('key')) - if (config('use-ceph-optimised-packages') and - not config('use-embedded-webserver')): - install_ceph_optimised_packages() - apt_update(fatal=True) - apt_install(PACKAGES, fatal=True) - if config('use-embedded-webserver'): - apt_purge(APACHE_PACKAGES) - else: - apt_install(APACHE_PACKAGES, fatal=True) + pkgs = filter_installed_packages(PACKAGES) + if pkgs: + status_set('maintenance', 'Installing radosgw packages') + apt_install(PACKAGES, fatal=True) + apt_purge(APACHE_PACKAGES) @hooks.hook('install.real') @@ -166,15 +145,6 @@ def config_changed(): CONFIGS.write_all() - if not config('use-embedded-webserver'): - try: - subprocess.check_call(['a2ensite', 'rgw']) - except subprocess.CalledProcessError as e: - log("Error enabling apache module 'rgw' - %s" % e, level=WARNING) - - # Ensure started but do a soft reload - subprocess.call(['service', 'apache2', 'start']) - subprocess.call(['service', 'apache2', 'reload']) update_nrpe_config() diff --git a/hooks/utils.py b/hooks/utils.py index b1b3e5b3..6d02a5d4 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -23,7 +23,6 @@ from copy import deepcopy import ceph_radosgw_context from charmhelpers.core.hookenv import ( - config, log, DEBUG, INFO, @@ -96,9 +95,6 @@ TEMPLATES_DIR = 'templates' TEMPLATES = 'templates/' HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' CEPH_CONF = '/etc/ceph/ceph.conf' -APACHE_CONF = '/etc/apache2/sites-available/rgw' -APACHE_24_CONF = '/etc/apache2/sites-available/rgw.conf' -APACHE_PORTS_CONF = '/etc/apache2/ports.conf' VERSION_PACKAGE = 'radosgw' @@ -108,18 +104,6 @@ BASE_RESOURCE_MAP = OrderedDict([ ceph_radosgw_context.HAProxyContext()], 'services': ['haproxy'], }), - (APACHE_CONF, { - 'contexts': [ceph_radosgw_context.ApacheContext()], - 'services': ['apache2'], - }), - (APACHE_24_CONF, { - 'contexts': [ceph_radosgw_context.ApacheContext()], - 'services': ['apache2'], - }), - (APACHE_PORTS_CONF, { - 'contexts': [ceph_radosgw_context.ApacheContext()], - 'services': ['apache2'], - }), (CEPH_CONF, { 'contexts': [ceph_radosgw_context.MonContext()], 'services': ['radosgw'], @@ -140,16 +124,6 @@ def resource_map(): These will be managed for a single hook execution. """ - if not config('use-embedded-webserver'): - if os.path.exists('/etc/apache2/conf-available'): - BASE_RESOURCE_MAP.pop(APACHE_CONF) - else: - BASE_RESOURCE_MAP.pop(APACHE_24_CONF) - else: - BASE_RESOURCE_MAP.pop(APACHE_CONF) - BASE_RESOURCE_MAP.pop(APACHE_24_CONF) - BASE_RESOURCE_MAP.pop(APACHE_PORTS_CONF) - resource_map = deepcopy(BASE_RESOURCE_MAP) return resource_map diff --git a/templates/ceph.conf b/templates/ceph.conf index e35fb2d0..d2674e6c 100644 --- a/templates/ceph.conf +++ b/templates/ceph.conf @@ -28,13 +28,7 @@ rgw init timeout = 1200 keyring = /etc/ceph/keyring.rados.gateway rgw socket path = /tmp/radosgw.sock log file = /var/log/ceph/radosgw.log -{% if embedded_webserver %} rgw frontends = civetweb port={{ port }} -{% elif disable_100_continue %} -# Turn off 100-continue optimization as stock mod_fastcgi -# does not support it -rgw print continue = false -{% endif %} {% if auth_type == 'keystone' %} rgw keystone url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/ rgw keystone admin token = {{ admin_token }} diff --git a/templates/ports.conf b/templates/ports.conf deleted file mode 100644 index 9b011ec1..00000000 --- a/templates/ports.conf +++ /dev/null @@ -1,11 +0,0 @@ -Listen {{ port }} - - - Listen 443 - - - - Listen 443 - - -# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/templates/rgw b/templates/rgw deleted file mode 100644 index 7a3e4724..00000000 --- a/templates/rgw +++ /dev/null @@ -1,25 +0,0 @@ - - FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock - - - - ServerName {{ hostname }} - ServerAdmin ceph@ubuntu.com - DocumentRoot /var/www - RewriteEngine On - RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] - - - Options +ExecCGI - AllowOverride All - SetHandler fastcgi-script - Order allow,deny - Allow from all - AuthBasicAuthoritative Off - - - AllowEncodedSlashes On - ErrorLog /var/log/apache2/error.log - CustomLog /var/log/apache2/access.log combined - ServerSignature Off - diff --git a/templates/rgw.conf b/templates/rgw.conf deleted file mode 100644 index 7a3e4724..00000000 --- a/templates/rgw.conf +++ /dev/null @@ -1,25 +0,0 @@ - - FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock - - - - ServerName {{ hostname }} - ServerAdmin ceph@ubuntu.com - DocumentRoot /var/www - RewriteEngine On - RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1¶ms=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] - - - Options +ExecCGI - AllowOverride All - SetHandler fastcgi-script - Order allow,deny - Allow from all - AuthBasicAuthoritative Off - - - AllowEncodedSlashes On - ErrorLog /var/log/apache2/error.log - CustomLog /var/log/apache2/access.log combined - ServerSignature Off - diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index e3ff55d7..155d5aba 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -115,13 +115,11 @@ class CephRadosGwBasicDeployment(OpenStackAmuletDeployment): 'ephemeral-unmount': '/mnt', 'osd-devices': '/dev/vdb /srv/ceph' } - radosgw_config = {"use-embedded-webserver": True} configs = {'keystone': keystone_config, 'percona-cluster': pxc_config, 'cinder': cinder_config, - 'ceph': ceph_config, - 'ceph-radosgw': radosgw_config} + 'ceph': ceph_config} super(CephRadosGwBasicDeployment, self)._configure_services(configs) def _run_action(self, unit_id, action, *args): diff --git a/unit_tests/test_ceph_radosgw_context.py b/unit_tests/test_ceph_radosgw_context.py index bdbdc965..79f5310a 100644 --- a/unit_tests/test_ceph_radosgw_context.py +++ b/unit_tests/test_ceph_radosgw_context.py @@ -28,7 +28,6 @@ TO_PATCH = [ 'related_units', 'cmp_pkgrevno', 'socket', - 'is_apache_24', ] @@ -182,8 +181,6 @@ class MonContextTest(CharmTestCase): self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2'] expect = { 'auth_supported': 'cephx', - 'embedded_webserver': False, - 'disable_100_continue': True, 'hostname': 'testhost', 'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789', 'old_auth': False, @@ -231,8 +228,6 @@ class MonContextTest(CharmTestCase): self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2'] expect = { 'auth_supported': 'none', - 'embedded_webserver': False, - 'disable_100_continue': True, 'hostname': 'testhost', 'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789', 'old_auth': False, @@ -262,8 +257,6 @@ class MonContextTest(CharmTestCase): self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2'] expect = { 'auth_supported': 'cephx', - 'embedded_webserver': False, - 'disable_100_continue': True, 'hostname': 'testhost', 'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789', 'old_auth': False, diff --git a/unit_tests/test_hooks.py b/unit_tests/test_hooks.py index 905a408d..4687b040 100644 --- a/unit_tests/test_hooks.py +++ b/unit_tests/test_hooks.py @@ -13,7 +13,6 @@ # limitations under the License. from mock import ( - call, patch, ) @@ -42,7 +41,6 @@ TO_PATCH = [ 'get_iface_for_address', 'get_netmask_for_address', 'log', - 'lsb_release', 'open_port', 'os', 'relation_ids', @@ -67,50 +65,10 @@ class CephRadosGWTests(CharmTestCase): self.test_config.set('key', 'secretkey') self.test_config.set('use-syslog', False) - def test_install_ceph_optimised_packages(self): - self.lsb_release.return_value = {'DISTRIB_CODENAME': 'vivid'} - fastcgi_source = ( - 'http://gitbuilder.ceph.com/' - 'libapache-mod-fastcgi-deb-vivid-x86_64-basic/ref/master') - apache_source = ( - 'http://gitbuilder.ceph.com/' - 'apache2-deb-vivid-x86_64-basic/ref/master') - calls = [ - call(fastcgi_source, key='6EAEAE2203C3951A'), - call(apache_source, key='6EAEAE2203C3951A'), - ] - ceph_hooks.install_ceph_optimised_packages() - self.add_source.assert_has_calls(calls) - def test_install_packages(self): - self.test_config.set('use-ceph-optimised-packages', '') ceph_hooks.install_packages() self.add_source.assert_called_with('distro', 'secretkey') self.assertTrue(self.apt_update.called) - self.apt_install.assert_called_with(['libapache2-mod-fastcgi', - 'apache2'], fatal=True) - - def test_install_optimised_packages_no_embedded(self): - self.test_config.set('use-ceph-optimised-packages', True) - self.test_config.set('use-embedded-webserver', False) - _install_packages = self.patch('install_ceph_optimised_packages') - ceph_hooks.install_packages() - self.add_source.assert_called_with('distro', 'secretkey') - self.assertTrue(self.apt_update.called) - self.assertTrue(_install_packages.called) - self.apt_install.assert_called_with(['libapache2-mod-fastcgi', - 'apache2'], fatal=True) - - def test_install_optimised_packages_embedded(self): - self.test_config.set('use-ceph-optimised-packages', True) - self.test_config.set('use-embedded-webserver', True) - _install_packages = self.patch('install_ceph_optimised_packages') - ceph_hooks.install_packages() - self.add_source.assert_called_with('distro', 'secretkey') - self.assertTrue(self.apt_update.called) - self.assertFalse(_install_packages.called) - self.apt_install.assert_called_with(ceph_hooks.PACKAGES, - fatal=True) self.apt_purge.assert_called_with(['libapache2-mod-fastcgi', 'apache2'])