From 7b7188b610c0ff19c6164fcecda0d6fdaf37f199 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Wed, 24 Oct 2018 16:04:32 +1300 Subject: [PATCH] Add wsrep-slave-threads and gcs-fc-limit These options allow tuning the Galera replication to avoid flow control from slowing down the primary. Change-Id: Ib275cae0db02e4c8c0a85fcc8cb138b26eb26982 Closes-Bug: 1799622 --- config.yaml | 17 ++++ hooks/percona_hooks.py | 3 + hooks/percona_utils.py | 3 + templates/mysqld.cnf | 4 + tests/basic_deployment.py | 4 +- unit_tests/test_percona_hooks.py | 165 +++++++++++++++++++++++++++++++ unit_tests/test_percona_utils.py | 7 +- 7 files changed, 201 insertions(+), 2 deletions(-) diff --git a/config.yaml b/config.yaml index 12aea5a..1eebbd9 100644 --- a/config.yaml +++ b/config.yaml @@ -341,3 +341,20 @@ options: Cluster ID to be used when using MySQL asynchronous replication. . NOTE: This value must be different for each cluster. + wsrep-slave-threads: + type: int + default: + description: | + Specifies the number of threads that can apply replication transactions in + parallel. Galera supports true parallel replication that applies + transactions in parallel only when it is safe to do so. Unset leaves the + default value of 1. + gcs-fc-limit: + type: int + default: + description: | + This setting controls when flow control engages. Simply speaking, if the + wsrep_local_recv_queue exceeds this size on a given node, a pausing flow + control message will be sent. The fc_limit defaults to 16 transactions. + This effectively means that this is as far as a given node can be behind + committing transactions from the cluster. diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 10d4fca..54b1b6a 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -231,6 +231,9 @@ def render_config(hosts=None): if wsrep_provider_options: context['wsrep_provider_options'] = wsrep_provider_options + if config('wsrep-slave-threads') is not None: + context['wsrep_slave_threads'] = config('wsrep-slave-threads') + if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) < 'bionic': # myisam_recover is not valid for PXC 5.7 (introduced in Bionic) so we # only set it for PXC 5.6. diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 3f2da58..0abbd18 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -1068,6 +1068,9 @@ def get_wsrep_provider_options(): if config('prefer-ipv6'): wsrep_provider_options.append('gmcast.listen_addr=tcp://:::4567') + if config('gcs-fc-limit') is not None: + wsrep_provider_options.append( + 'gcs.fc_limit={}'.format(config('gcs-fc-limit'))) peer_timeout = config('peer-timeout') if peer_timeout and(not peer_timeout.startswith('PT') or diff --git a/templates/mysqld.cnf b/templates/mysqld.cnf index 656cedf..d81f97c 100644 --- a/templates/mysqld.cnf +++ b/templates/mysqld.cnf @@ -173,6 +173,10 @@ wsrep_log_conflicts wsrep_provider_options = {{ wsrep_provider_options }} {% endif %} +{% if wsrep_slave_threads -%} +wsrep_slave_threads = {{ wsrep_slave_threads }} +{% endif %} + # # * Performance Schema # diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 1f17920..afc82fb 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -73,7 +73,9 @@ class BasicDeployment(OpenStackAmuletDeployment): """Configure all of the services.""" cfg_percona = {'min-cluster-size': self.units, 'vip': self.vip, - 'root-password': PXC_ROOT_PASSWD} + 'root-password': PXC_ROOT_PASSWD, + 'wsrep-slave-threads': 2, + 'gcs-fc-limit': 32} cfg_ha = {'debug': True, 'corosync_key': ('xZP7GDWV0e8Qs0GxWThXirNNYlScgi3sRTdZk/IXKD' diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index e4d2c59..5e9adf8 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -1,10 +1,14 @@ import json +import logging import mock +import os import shutil import sys import tempfile +import yaml import charmhelpers.contrib.openstack.ha.utils as ch_ha_utils +from charmhelpers.contrib.database.mysql import PerconaClusterHelper from test_utils import CharmTestCase @@ -670,3 +674,164 @@ class TestUpgradeCharm(CharmTestCase): self.leader_set.assert_has_calls( [mock.call(**{'leader-ip': '10.10.10.10'}), mock.call(**{'root-password': 'mypasswd'})]) + + +class TestConfigs(CharmTestCase): + + TO_PATCH = [ + 'config', + 'is_leader', + ] + + def setUp(self): + CharmTestCase.setUp(self, hooks, TO_PATCH) + self.config.side_effect = self.test_config.get + self.default_config = self._get_default_config() + for key, value in self.default_config.items(): + self.test_config.set(key, value) + self.is_leader.return_value = False + + def _load_config(self): + '''Walk backwords from __file__ looking for config.yaml, + load and return the 'options' section' + ''' + config = None + f = __file__ + while config is None: + d = os.path.dirname(f) + if os.path.isfile(os.path.join(d, 'config.yaml')): + config = os.path.join(d, 'config.yaml') + break + f = d + + if not config: + logging.error('Could not find config.yaml in any parent directory ' + 'of %s. ' % f) + raise Exception + + return yaml.safe_load(open(config).read())['options'] + + def _get_default_config(self): + '''Load default charm config from config.yaml return as a dict. + If no default is set in config.yaml, its value is None. + ''' + default_config = {} + config = self._load_config() + for k, v in config.iteritems(): + if 'default' in v: + default_config[k] = v['default'] + else: + default_config[k] = None + return default_config + + @mock.patch.object(os, 'makedirs') + @mock.patch.object(hooks, 'get_cluster_host_ip') + @mock.patch.object(hooks, 'get_wsrep_provider_options') + @mock.patch.object(PerconaClusterHelper, 'parse_config') + @mock.patch.object(hooks, 'render') + @mock.patch.object(hooks, 'sst_password') + @mock.patch.object(hooks, 'lsb_release') + def test_render_config_defaults(self, + lsb_release, + sst_password, + render, + parse_config, + get_wsrep_provider_options, + get_cluster_host_ip, + makedirs): + parse_config.return_value = {'key_buffer': '32M'} + get_cluster_host_ip.return_value = '10.1.1.1' + get_wsrep_provider_options.return_value = None + sst_password.return_value = 'sstpassword' + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + context = { + 'server_id': hooks.get_server_id(), + 'server-id': hooks.get_server_id(), + 'is_leader': hooks.is_leader(), + 'series_upgrade': hooks.is_unit_upgrading_set(), + 'private_address': '10.1.1.1', + 'innodb_autoinc_lock_mode': '2', + 'cluster_hosts': '', + 'enable_binlogs': self.default_config['enable-binlogs'], + 'sst_password': 'sstpassword', + 'sst_method': self.default_config['sst-method'], + 'pxc_strict_mode': 'enforcing', + 'binlogs_max_size': self.default_config['binlogs-max-size'], + 'cluster_name': 'juju_cluster', + 'innodb_file_per_table': + self.default_config['innodb-file-per-table'], + 'table_open_cache': self.default_config['table-open-cache'], + 'binlogs_path': self.default_config['binlogs-path'], + 'binlogs_expire_days': self.default_config['binlogs-expire-days'], + 'performance_schema': self.default_config['performance-schema'], + 'key_buffer': '32M', + 'default_storage_engine': 'InnoDB', + 'wsrep_log_conflicts': True, + 'ipv6': False, + 'wsrep_provider': '/usr/lib/galera3/libgalera_smm.so', + } + + hooks.render_config() + hooks.render.assert_called_once_with( + 'mysqld.cnf', + '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf', + context, + perms=0o444) + + @mock.patch.object(os, 'makedirs') + @mock.patch.object(hooks, 'get_cluster_host_ip') + @mock.patch.object(hooks, 'get_wsrep_provider_options') + @mock.patch.object(PerconaClusterHelper, 'parse_config') + @mock.patch.object(hooks, 'render') + @mock.patch.object(hooks, 'sst_password') + @mock.patch.object(hooks, 'lsb_release') + def test_render_config_wsrep_slave_threads( + self, + lsb_release, + sst_password, + render, + parse_config, + get_wsrep_provider_options, + get_cluster_host_ip, + makedirs): + parse_config.return_value = {'key_buffer': '32M'} + get_cluster_host_ip.return_value = '10.1.1.1' + get_wsrep_provider_options.return_value = None + sst_password.return_value = 'sstpassword' + self.test_config.set('wsrep-slave-threads', 2) + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + + context = { + 'server_id': hooks.get_server_id(), + 'server-id': hooks.get_server_id(), + 'is_leader': hooks.is_leader(), + 'series_upgrade': hooks.is_unit_upgrading_set(), + 'private_address': '10.1.1.1', + 'innodb_autoinc_lock_mode': '2', + 'cluster_hosts': '', + 'enable_binlogs': self.default_config['enable-binlogs'], + 'sst_password': 'sstpassword', + 'sst_method': self.default_config['sst-method'], + 'pxc_strict_mode': 'enforcing', + 'binlogs_max_size': self.default_config['binlogs-max-size'], + 'cluster_name': 'juju_cluster', + 'innodb_file_per_table': + self.default_config['innodb-file-per-table'], + 'table_open_cache': self.default_config['table-open-cache'], + 'binlogs_path': self.default_config['binlogs-path'], + 'binlogs_expire_days': self.default_config['binlogs-expire-days'], + 'performance_schema': self.default_config['performance-schema'], + 'key_buffer': '32M', + 'default_storage_engine': 'InnoDB', + 'wsrep_log_conflicts': True, + 'ipv6': False, + 'wsrep_provider': '/usr/lib/galera3/libgalera_smm.so', + 'wsrep_slave_threads': 2, + } + + hooks.render_config() + hooks.render.assert_called_once_with( + 'mysqld.cnf', + '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf', + context, + perms=0o444) diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index 435dd92..fa597f7 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -347,7 +347,12 @@ class UtilsTests(CharmTestCase): "gmcast.peer_timeout=PT15S") self.assertEqual(percona_utils.get_wsrep_provider_options(), expected) - + # set gcs.fs_limit=10000 + _config = {"gcs-fc-limit": 10000} + mock_config.side_effect = lambda key: _config.get(key) + expected = "gcs.fc_limit=10000" + self.assertEqual(percona_utils.get_wsrep_provider_options(), + expected) # peer_timeout bad setting _config = {"peer-timeout": "10"} mock_config.side_effect = lambda key: _config.get(key)