From 4e9aa259b0afd71111395616d853009b9cb80e75 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 11 Dec 2017 11:37:33 -0800 Subject: [PATCH] Update HAProxy default timeout values The default HAProxy timeout values are fairly strict. On a busy cloud it is common to exceed one or more of these timeouts. The only indication that HAProxy has exceeded a timeout and dropped the connection is errors such as "BadStatusLine" or "EOF." These can be very difficult to diagnose when intermittent. This charm-helpers sync pulls in the change to update the default timeout values to more real world settings. These values have been extensively tested in ServerStack. Configured values will not be overridden. Partial Bug: #1736171 Change-Id: I0aefed05d7b06162d21bb5ded216fc32935ef52e --- .../contrib/openstack/amulet/utils.py | 9 ++-- charmhelpers/contrib/openstack/context.py | 14 +++++- .../contrib/openstack/templates/haproxy.cfg | 8 +-- charmhelpers/contrib/storage/linux/lvm.py | 50 +++++++++++++++++++ charmhelpers/core/host_factory/ubuntu.py | 1 + config.yaml | 8 +-- .../contrib/openstack/amulet/utils.py | 9 ++-- tests/charmhelpers/contrib/openstack/utils.py | 2 + .../charmhelpers/contrib/storage/linux/lvm.py | 50 +++++++++++++++++++ .../charmhelpers/core/host_factory/ubuntu.py | 1 + 10 files changed, 137 insertions(+), 15 deletions(-) diff --git a/charmhelpers/contrib/openstack/amulet/utils.py b/charmhelpers/contrib/openstack/amulet/utils.py index b71b2b1..87f364d 100644 --- a/charmhelpers/contrib/openstack/amulet/utils.py +++ b/charmhelpers/contrib/openstack/amulet/utils.py @@ -858,9 +858,12 @@ class OpenStackAmuletUtils(AmuletUtils): :returns: List of pool name, object count, kb disk space used """ df = self.get_ceph_df(sentry_unit) - pool_name = df['pools'][pool_id]['name'] - obj_count = df['pools'][pool_id]['stats']['objects'] - kb_used = df['pools'][pool_id]['stats']['kb_used'] + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + self.log.debug('Ceph {} pool (ID {}): {} objects, ' '{} kb used'.format(pool_name, pool_id, obj_count, kb_used)) diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index e6c0e9f..70850c1 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -101,6 +101,8 @@ from charmhelpers.contrib.openstack.utils import ( git_determine_python_path, enable_memcache, snap_install_requested, + CompareOpenStackReleases, + os_release, ) from charmhelpers.core.unitdata import kv @@ -1566,8 +1568,18 @@ class InternalEndpointContext(OSContextGenerator): endpoints by default so this allows admins to optionally use internal endpoints. """ + def __init__(self, ost_rel_check_pkg_name): + self.ost_rel_check_pkg_name = ost_rel_check_pkg_name + def __call__(self): - return {'use_internal_endpoints': config('use-internal-endpoints')} + ctxt = {'use_internal_endpoints': config('use-internal-endpoints')} + rel = os_release(self.ost_rel_check_pkg_name, base='icehouse') + if CompareOpenStackReleases(rel) >= 'pike': + ctxt['volume_api_version'] = '3' + else: + ctxt['volume_api_version'] = '2' + + return ctxt class AppArmorContext(OSContextGenerator): diff --git a/charmhelpers/contrib/openstack/templates/haproxy.cfg b/charmhelpers/contrib/openstack/templates/haproxy.cfg index ebc8a68..d36af2a 100644 --- a/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -17,22 +17,22 @@ defaults {%- if haproxy_queue_timeout %} timeout queue {{ haproxy_queue_timeout }} {%- else %} - timeout queue 5000 + timeout queue 9000 {%- endif %} {%- if haproxy_connect_timeout %} timeout connect {{ haproxy_connect_timeout }} {%- else %} - timeout connect 5000 + timeout connect 9000 {%- endif %} {%- if haproxy_client_timeout %} timeout client {{ haproxy_client_timeout }} {%- else %} - timeout client 30000 + timeout client 90000 {%- endif %} {%- if haproxy_server_timeout %} timeout server {{ haproxy_server_timeout }} {%- else %} - timeout server 30000 + timeout server 90000 {%- endif %} listen stats diff --git a/charmhelpers/contrib/storage/linux/lvm.py b/charmhelpers/contrib/storage/linux/lvm.py index 7f2a060..79a7a24 100644 --- a/charmhelpers/contrib/storage/linux/lvm.py +++ b/charmhelpers/contrib/storage/linux/lvm.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools from subprocess import ( CalledProcessError, check_call, @@ -101,3 +102,52 @@ def create_lvm_volume_group(volume_group, block_device): :block_device: str: Full path of PV-initialized block device. ''' check_call(['vgcreate', volume_group, block_device]) + + +def list_logical_volumes(select_criteria=None, path_mode=False): + ''' + List logical volumes + + :param select_criteria: str: Limit list to those volumes matching this + criteria (see 'lvs -S help' for more details) + :param path_mode: bool: return logical volume name in 'vg/lv' format, this + format is required for some commands like lvextend + :returns: [str]: List of logical volumes + ''' + lv_diplay_attr = 'lv_name' + if path_mode: + # Parsing output logic relies on the column order + lv_diplay_attr = 'vg_name,' + lv_diplay_attr + cmd = ['lvs', '--options', lv_diplay_attr, '--noheadings'] + if select_criteria: + cmd.extend(['--select', select_criteria]) + lvs = [] + for lv in check_output(cmd).decode('UTF-8').splitlines(): + if not lv: + continue + if path_mode: + lvs.append('/'.join(lv.strip().split())) + else: + lvs.append(lv.strip()) + return lvs + + +list_thin_logical_volume_pools = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^t') + +list_thin_logical_volumes = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^V') + + +def extend_logical_volume_by_device(lv_name, block_device): + ''' + Extends the size of logical volume lv_name by the amount of free space on + physical volume block_device. + + :param lv_name: str: name of logical volume to be extended (vg/lv format) + :param block_device: str: name of block_device to be allocated to lv_name + ''' + cmd = ['lvextend', lv_name, block_device] + check_call(cmd) diff --git a/charmhelpers/core/host_factory/ubuntu.py b/charmhelpers/core/host_factory/ubuntu.py index d8dc378..99451b5 100644 --- a/charmhelpers/core/host_factory/ubuntu.py +++ b/charmhelpers/core/host_factory/ubuntu.py @@ -20,6 +20,7 @@ UBUNTU_RELEASES = ( 'yakkety', 'zesty', 'artful', + 'bionic', ) diff --git a/config.yaml b/config.yaml index 0df9bd9..e64b1e5 100644 --- a/config.yaml +++ b/config.yaml @@ -195,25 +195,25 @@ options: default: description: | Server timeout configuration in ms for haproxy, used in HA - configurations. If not provided, default value of 30000ms is used. + configurations. If not provided, default value of 90000ms is used. haproxy-client-timeout: type: int default: description: | Client timeout configuration in ms for haproxy, used in HA - configurations. If not provided, default value of 30000ms is used. + configurations. If not provided, default value of 90000ms is used. haproxy-queue-timeout: type: int default: description: | Queue timeout configuration in ms for haproxy, used in HA - configurations. If not provided, default value of 5000ms is used. + configurations. If not provided, default value of 9000ms is used. haproxy-connect-timeout: type: int default: description: | Connect timeout configuration in ms for haproxy, used in HA - configurations. If not provided, default value of 5000ms is used. + configurations. If not provided, default value of 9000ms is used. # Network config (by default all access is over 'private-address') os-admin-network: type: string diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index b71b2b1..87f364d 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -858,9 +858,12 @@ class OpenStackAmuletUtils(AmuletUtils): :returns: List of pool name, object count, kb disk space used """ df = self.get_ceph_df(sentry_unit) - pool_name = df['pools'][pool_id]['name'] - obj_count = df['pools'][pool_id]['stats']['objects'] - kb_used = df['pools'][pool_id]['stats']['kb_used'] + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + self.log.debug('Ceph {} pool (ID {}): {} objects, ' '{} kb used'.format(pool_name, pool_id, obj_count, kb_used)) diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py index 8a541d4..9e5af34 100644 --- a/tests/charmhelpers/contrib/openstack/utils.py +++ b/tests/charmhelpers/contrib/openstack/utils.py @@ -392,6 +392,8 @@ def get_swift_codename(version): releases = UBUNTU_OPENSTACK_RELEASE release = [k for k, v in six.iteritems(releases) if codename in v] ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) + if six.PY3: + ret = ret.decode('UTF-8') if codename in ret or release[0] in ret: return codename elif len(codenames) == 1: diff --git a/tests/charmhelpers/contrib/storage/linux/lvm.py b/tests/charmhelpers/contrib/storage/linux/lvm.py index 7f2a060..79a7a24 100644 --- a/tests/charmhelpers/contrib/storage/linux/lvm.py +++ b/tests/charmhelpers/contrib/storage/linux/lvm.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools from subprocess import ( CalledProcessError, check_call, @@ -101,3 +102,52 @@ def create_lvm_volume_group(volume_group, block_device): :block_device: str: Full path of PV-initialized block device. ''' check_call(['vgcreate', volume_group, block_device]) + + +def list_logical_volumes(select_criteria=None, path_mode=False): + ''' + List logical volumes + + :param select_criteria: str: Limit list to those volumes matching this + criteria (see 'lvs -S help' for more details) + :param path_mode: bool: return logical volume name in 'vg/lv' format, this + format is required for some commands like lvextend + :returns: [str]: List of logical volumes + ''' + lv_diplay_attr = 'lv_name' + if path_mode: + # Parsing output logic relies on the column order + lv_diplay_attr = 'vg_name,' + lv_diplay_attr + cmd = ['lvs', '--options', lv_diplay_attr, '--noheadings'] + if select_criteria: + cmd.extend(['--select', select_criteria]) + lvs = [] + for lv in check_output(cmd).decode('UTF-8').splitlines(): + if not lv: + continue + if path_mode: + lvs.append('/'.join(lv.strip().split())) + else: + lvs.append(lv.strip()) + return lvs + + +list_thin_logical_volume_pools = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^t') + +list_thin_logical_volumes = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^V') + + +def extend_logical_volume_by_device(lv_name, block_device): + ''' + Extends the size of logical volume lv_name by the amount of free space on + physical volume block_device. + + :param lv_name: str: name of logical volume to be extended (vg/lv format) + :param block_device: str: name of block_device to be allocated to lv_name + ''' + cmd = ['lvextend', lv_name, block_device] + check_call(cmd) diff --git a/tests/charmhelpers/core/host_factory/ubuntu.py b/tests/charmhelpers/core/host_factory/ubuntu.py index d8dc378..99451b5 100644 --- a/tests/charmhelpers/core/host_factory/ubuntu.py +++ b/tests/charmhelpers/core/host_factory/ubuntu.py @@ -20,6 +20,7 @@ UBUNTU_RELEASES = ( 'yakkety', 'zesty', 'artful', + 'bionic', )