From 60e58482f6a5887e178e3e4712f7c04d70ff7d00 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 24 Feb 2017 14:34:42 -0500 Subject: [PATCH] Improve password management for clustered deploys In the past, its mandatory to provide the sst and root password configuration options for clustered deployments to ensure consistent use of passwords across the cluster from install onwards. Rework password management and install process to seed passwords from the lead unit if not supplied via configuration options. Following units will defer installation until the leader has stored this information in leader storage for retrieval by followers. Closes-Bug: 1454317 Change-Id: I5ab70cae78ed35322bf60048af841de071a69704 --- README.md | 44 +++++++------------ charmhelpers/contrib/database/mysql.py | 30 +++++++------ charmhelpers/contrib/network/ip.py | 45 ++++++++++++++++++- config.yaml | 10 +++-- hooks/percona_hooks.py | 38 ++++++++++++---- hooks/percona_utils.py | 38 +++++++++++++++- tests/basic_deployment.py | 10 ++--- unit_tests/test_percona_hooks.py | 61 ++++++++++++++++++++++++-- unit_tests/test_percona_utils.py | 5 ++- 9 files changed, 214 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 6cb98cb..dc84acc 100644 --- a/README.md +++ b/README.md @@ -11,43 +11,29 @@ This charm deploys Percona XtraDB Cluster onto Ubuntu. Usage ===== -WARNING: Its critical that you follow the bootstrap process detailed in this -document in order to end up with a running Active/Active Percona Cluster. - -Proxy Configuration -------------------- - -If you are deploying this charm on MAAS or in an environment without direct -access to the internet, you will need to allow access to repo.percona.com -as the charm installs packages direct from the Percona respositories. If you -are using squid-deb-proxy, follow the steps below: - - echo "repo.percona.com" | sudo tee /etc/squid-deb-proxy/mirror-dstdomain.acl.d/40-percona - sudo service squid-deb-proxy restart - Deployment ---------- -The first service unit deployed acts as the seed node for the rest of the -cluster; in order for the cluster to function correctly, the same MySQL passwords -must be used across all nodes: +To deploy this charm: - cat > percona.yaml << EOF - percona-cluster: - root-password: my-root-password - sst-password: my-sst-password - EOF + juju deploy percona-cluster -Once you have created this file, you can deploy the first seed unit: +Passwords required for the correct operation of the deployment are automatically +generated and stored by the lead unit (typically the first unit). - juju deploy --config percona.yaml percona-cluster +To expand the deployment: -Once this node is full operational, you can add extra units one at a time to the -deployment: + juju add-unit -n 2 percona-cluster - juju add-unit percona-cluster +See notes in the 'HA/Clustering' section on safely deploying a PXC cluster +in a single action. -A minimum cluster size of three units is recommended. +The root password for mysql can be retrieved using the following command: + + juju run --unit percona-cluster/0 leader-get root-password + +This is only usable from within one of the units within the deployment +(access to root is restricted to localhost only). Memory Configuration ------------------- @@ -86,7 +72,7 @@ the host due to performance schema being turned on. Even with the default now turned off this value should be carefully considered against the production requirements and resources available. -[1] http://dev.mysql.com/doc/relnotes/mysql/5.6/en/news-5-6-6.html#mysqld-5-6-6-performance-schema +[1] http://dev.mysql.com/doc/relnotes/mysql/5.6/en/news-5-6-6.html#mysqld-5-6-6-performance-schema [2] http://www.mysqlcalculator.com/ diff --git a/charmhelpers/contrib/database/mysql.py b/charmhelpers/contrib/database/mysql.py index 36e783a..8aac183 100644 --- a/charmhelpers/contrib/database/mysql.py +++ b/charmhelpers/contrib/database/mysql.py @@ -35,16 +35,15 @@ from charmhelpers.core.hookenv import ( DEBUG, INFO, WARNING, + leader_get, + leader_set, + is_leader, ) from charmhelpers.fetch import ( apt_install, apt_update, filter_installed_packages, ) -from charmhelpers.contrib.peerstorage import ( - peer_store, - peer_retrieve, -) from charmhelpers.contrib.network.ip import get_host_ip try: @@ -61,14 +60,14 @@ except ImportError: class MySQLHelper(object): def __init__(self, rpasswdf_template, upasswdf_template, host='localhost', - migrate_passwd_to_peer_relation=True, + migrate_passwd_to_leader_storage=True, delete_ondisk_passwd_file=True): self.host = host # Password file path templates self.root_passwd_file_template = rpasswdf_template self.user_passwd_file_template = upasswdf_template - self.migrate_passwd_to_peer_relation = migrate_passwd_to_peer_relation + self.migrate_passwd_to_leader_storage = migrate_passwd_to_leader_storage # If we migrate we have the option to delete local copy of root passwd self.delete_ondisk_passwd_file = delete_ondisk_passwd_file @@ -157,13 +156,18 @@ class MySQLHelper(object): finally: cursor.close() - def migrate_passwords_to_peer_relation(self, excludes=None): - """Migrate any passwords storage on disk to cluster peer relation.""" + def migrate_passwords_to_leader_storage(self, excludes=None): + """Migrate any passwords storage on disk to leader storage.""" + if not is_leader(): + log("Skipping password migration as not the lead unit", + level=DEBUG) + return dirname = os.path.dirname(self.root_passwd_file_template) path = os.path.join(dirname, '*.passwd') for f in glob.glob(path): if excludes and f in excludes: - log("Excluding %s from peer migration" % (f), level=DEBUG) + log("Excluding %s from leader storage migration" % (f), + level=DEBUG) continue key = os.path.basename(f) @@ -171,7 +175,7 @@ class MySQLHelper(object): _value = passwd.read().strip() try: - peer_store(key, _value) + leader_set(settings={key: _value}) if self.delete_ondisk_passwd_file: os.unlink(f) @@ -238,7 +242,7 @@ class MySQLHelper(object): # First check peer relation. try: for key in self.passwd_keys(username): - _password = peer_retrieve(key) + _password = leader_get(key) if _password: break @@ -255,8 +259,8 @@ class MySQLHelper(object): _password = self.get_mysql_password_on_disk(username, password) # Put on wire if required - if self.migrate_passwd_to_peer_relation: - self.migrate_passwords_to_peer_relation(excludes=excludes) + if self.migrate_passwd_to_leader_storage: + self.migrate_passwords_to_leader_storage(excludes=excludes) return _password diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index dfdc021..54c76a7 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -20,13 +20,19 @@ import socket from functools import partial -from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import ( + config, log, + network_get_primary_address, + unit_get, WARNING, ) +from charmhelpers.core.host import ( + lsb_release, +) + try: import netifaces except ImportError: @@ -511,3 +517,40 @@ def port_has_listener(address, port): cmd = ['nc', '-z', address, str(port)] result = subprocess.call(cmd) return not(bool(result)) + + +def assert_charm_supports_ipv6(): + """Check whether we are able to support charms ipv6.""" + if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty": + raise Exception("IPv6 is not supported in the charms for Ubuntu " + "versions less than Trusty 14.04") + + +def get_relation_ip(interface, config_override=None): + """Return this unit's IP for the given relation. + + Allow for an arbitrary interface to use with network-get to select an IP. + Handle all address selection options including configuration parameter + override and IPv6. + + Usage: get_relation_ip('amqp', config_override='access-network') + + @param interface: string name of the relation. + @param config_override: string name of the config option for network + override. Supports legacy network override configuration parameters. + @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. + @returns IPv6 or IPv4 address + """ + + fallback = get_host_ip(unit_get('private-address')) + if config('prefer-ipv6'): + assert_charm_supports_ipv6() + return get_ipv6_addr()[0] + elif config_override and config(config_override): + return get_address_in_network(config(config_override), + fallback) + else: + try: + return network_get_primary_address(interface) + except NotImplementedError: + return fallback diff --git a/config.yaml b/config.yaml index de2d1b3..886c81b 100644 --- a/config.yaml +++ b/config.yaml @@ -104,14 +104,16 @@ options: type: string default: description: | - Root password for MySQL access; must be configured pre-deployment for - Active-Active clusters. + Root account password for new cluster nodes. Overrides the automatic + generation of a password for the root user, but must be set prior to + deployment time to have any effect. sst-password: type: string default: description: | - Re-sync account password for new cluster nodes; must be configured - pre-deployment for Active-Active clusters. + SST account password for new cluster nodes. Overrides the automatic + generation of a password for the sst user, but must be set prior to + deployment time to have any effect. sst-method: type: string default: xtrabackup-v2 diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 25fc81c..bf66de5 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -98,6 +98,9 @@ from percona_utils import ( client_node_is_ready, leader_node_is_ready, DEFAULT_MYSQL_PORT, + sst_password, + root_password, + pxc_installed, ) @@ -112,6 +115,27 @@ RES_MONITOR_PARAMS = ('params user="sstuser" password="%(sstpass)s" ' 'OCF_CHECK_LEVEL="1"') +def install_percona_xtradb_cluster(): + '''Attempt PXC install based on seeding of passwords for users''' + if pxc_installed(): + log('MySQL already installed, skipping') + return + + _root_password = root_password() + _sst_password = sst_password() + if not _root_password or not _sst_password: + log('Passwords not seeded, unable to install MySQL at this' + ' point so deferring installation') + return + configure_mysql_root_password(_root_password) + + apt_install(determine_packages(), fatal=True) + + configure_sstuser(_sst_password) + if config('harden') and 'mysql' in config('harden'): + run_mysql_checks() + + @hooks.hook('install.real') @harden() def install(): @@ -121,13 +145,9 @@ def install(): setup_percona_repo() elif config('source') is not None: add_source(config('source'), config('key')) - - configure_mysql_root_password(config('root-password')) apt_update(fatal=True) - apt_install(determine_packages(), fatal=True) - configure_sstuser(config('sst-password')) - if config('harden') and 'mysql' in config('harden'): - run_mysql_checks() + + install_percona_xtradb_cluster() def render_config(clustered=False, hosts=None): @@ -144,7 +164,7 @@ def render_config(clustered=False, hosts=None): 'clustered': clustered, 'cluster_hosts': ",".join(hosts), 'sst_method': config('sst-method'), - 'sst_password': config('sst-password'), + 'sst_password': sst_password(), 'innodb_file_per_table': config('innodb-file-per-table'), 'table_open_cache': config('table-open-cache'), 'lp1366997_workaround': config('lp1366997-workaround'), @@ -601,7 +621,7 @@ def shared_db_changed(relation_id=None, unit=None): @hooks.hook('ha-relation-joined') def ha_relation_joined(relation_id=None): cluster_config = get_hacluster_config() - sstpsswd = config('sst-password') + sstpsswd = sst_password() resources = {'res_mysql_monitor': 'ocf:percona:mysql_monitor'} resource_params = {'res_mysql_monitor': RES_MONITOR_PARAMS % {'sstpass': sstpsswd}} @@ -673,6 +693,8 @@ def ha_relation_changed(): @hooks.hook('leader-settings-changed') def leader_settings_changed(): + '''Re-trigger install once leader has seeded passwords into install''' + install_percona_xtradb_cluster() # Notify any changes to data in leader storage update_shared_db_rels() diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index 07e197b..43a9ebd 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -6,12 +6,14 @@ import tempfile import os import shutil import uuid +from functools import partial from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import ( lsb_release, mkdir, service, + pwgen, ) from charmhelpers.core.hookenv import ( charm_dir, @@ -542,7 +544,8 @@ def assess_status(configs): @returns None - this function is executed for its side-effect """ assess_status_func(configs)() - application_version_set(get_upstream_version(determine_packages()[0])) + if pxc_installed(): + application_version_set(get_upstream_version(determine_packages()[0])) def assess_status_func(configs): @@ -720,3 +723,36 @@ def leader_node_is_ready(): if is_unit_paused_set(): return False return (is_leader() and cluster_ready()) + + +def _get_password(key): + '''Retrieve named password + + This function will ensure that a consistent named password + is used across all units in the pxc cluster; the lead unit + will generate or use the root-password configuration option + to seed this value into the deployment. + + Once set, it cannot be changed. + + @requires: str: named password or None if unable to retrieve + at this point in time + ''' + _password = leader_get(key) + if not _password and is_leader(): + _password = config(key) or pwgen() + leader_set({key: _password}) + return _password + + +root_password = partial(_get_password, 'root-password') + +sst_password = partial(_get_password, 'sst-password') + + +def pxc_installed(): + '''Determine whether percona-xtradb-cluster is installed + + @returns: boolean: indicating installation + ''' + return os.path.exists('/usr/sbin/mysqld') diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index f59c4fd..75f62a1 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -65,9 +65,7 @@ class BasicDeployment(OpenStackAmuletDeployment): def _get_configs(self): """Configure all of the services.""" - cfg_percona = {'sst-password': 'ubuntu', - 'root-password': 't00r', - 'min-cluster-size': self.units, + cfg_percona = {'min-cluster-size': self.units, 'vip': self.vip} cfg_ha = {'debug': True, @@ -259,9 +257,9 @@ class BasicDeployment(OpenStackAmuletDeployment): u = unit else: u = self.master_unit - - cmd = ("mysql -uroot -pt00r -e\"show status like '%s';\"| " - "grep %s" % (attr, attr)) + root_password, _ = u.run('leader-get root-password') + cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| " + "grep {}".format(root_password, attr, attr)) output, code = u.run(cmd) if code != 0: self.log.debug("command returned non-zero '%s'" % (code)) diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py index ed8dd02..8c9fdfa 100644 --- a/unit_tests/test_percona_hooks.py +++ b/unit_tests/test_percona_hooks.py @@ -29,13 +29,15 @@ TO_PATCH = ['log', 'config', 'is_clustered', 'get_ipv6_addr', 'get_hacluster_config', - 'update_dns_ha_resource_params'] + 'update_dns_ha_resource_params', + 'sst_password'] class TestHARelation(CharmTestCase): def setUp(self): CharmTestCase.setUp(self, hooks, TO_PATCH) self.network_get_primary_address.side_effect = NotImplementedError + self.sst_password.return_value = 'ubuntu' def test_resources(self): self.relation_ids.return_value = ['ha:1'] @@ -47,7 +49,6 @@ class TestHARelation(CharmTestCase): self.get_netmask_for_address.return_value = None self.get_iface_for_address.return_value = None self.test_config.set('vip', '10.0.3.3') - self.test_config.set('sst-password', password) self.get_hacluster_config.return_value = { 'vip': '10.0.3.3', 'ha-bindiface': 'eth0', @@ -111,7 +112,7 @@ class TestHARelation(CharmTestCase): 'cidr_netmask="20" ' 'nic="eth1"'), 'res_mysql_monitor': - hooks.RES_MONITOR_PARAMS % {'sstpass': 'None'}} + hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}} call_args, call_kwargs = self.relation_set.call_args self.assertEqual(resource_params, call_kwargs['resource_params']) @@ -145,7 +146,7 @@ class TestHARelation(CharmTestCase): 'cidr_netmask="16" ' 'nic="eth1"'), 'res_mysql_monitor': - hooks.RES_MONITOR_PARAMS % {'sstpass': 'None'}} + hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}} call_args, call_kwargs = self.relation_set.call_args self.assertEqual(resource_params, call_kwargs['resource_params']) @@ -241,3 +242,55 @@ class TestConfigChanged(CharmTestCase): self.is_relation_made.return_value = False hooks.config_changed() self.open_port.assert_called_with(3306) + + +class TestInstallPerconaXtraDB(CharmTestCase): + + TO_PATCH = [ + 'log', + 'pxc_installed', + 'root_password', + 'sst_password', + 'configure_mysql_root_password', + 'apt_install', + 'determine_packages', + 'configure_sstuser', + 'config', + 'run_mysql_checks', + ] + + def setUp(self): + CharmTestCase.setUp(self, hooks, self.TO_PATCH) + self.config.side_effect = self.test_config.get + self.pxc_installed.return_value = False + + def test_installed(self): + self.pxc_installed.return_value = True + hooks.install_percona_xtradb_cluster() + self.configure_mysql_root_password.assert_not_called() + self.apt_install.assert_not_called() + + def test_passwords_not_initialized(self): + self.root_password.return_value = None + self.sst_password.return_value = None + hooks.install_percona_xtradb_cluster() + self.configure_mysql_root_password.assert_not_called() + self.configure_sstuser.assert_not_called() + self.apt_install.assert_not_called() + + self.root_password.return_value = None + self.sst_password.return_value = 'testpassword' + hooks.install_percona_xtradb_cluster() + self.configure_sstuser.assert_not_called() + self.configure_mysql_root_password.assert_not_called() + self.apt_install.assert_not_called() + + def test_passwords_initialized(self): + self.root_password.return_value = 'rootpassword' + self.sst_password.return_value = 'testpassword' + self.determine_packages.return_value = ['pxc-5.6'] + hooks.install_percona_xtradb_cluster() + self.configure_mysql_root_password.assert_called_with('rootpassword') + self.configure_sstuser.assert_called_with('testpassword') + self.apt_install.assert_called_with(['pxc-5.6'], fatal=True) + self.run_mysql_checks.assert_not_called() diff --git a/unit_tests/test_percona_utils.py b/unit_tests/test_percona_utils.py index fcd91ed..6ff141e 100644 --- a/unit_tests/test_percona_utils.py +++ b/unit_tests/test_percona_utils.py @@ -305,14 +305,17 @@ class UtilsTestsCTC(CharmTestCase): stat, _ = percona_utils.charm_check_func() assert stat == 'active' + @mock.patch.object(percona_utils, 'pxc_installed') @mock.patch.object(percona_utils, 'determine_packages') @mock.patch.object(percona_utils, 'application_version_set') @mock.patch.object(percona_utils, 'get_upstream_version') def test_assess_status(self, get_upstream_version, application_version_set, - determine_packages): + determine_packages, + pxc_installed): get_upstream_version.return_value = '5.6.17' determine_packages.return_value = ['percona-xtradb-cluster-server-5.6'] + pxc_installed.return_value = True with mock.patch.object(percona_utils, 'assess_status_func') as asf: callee = mock.Mock() asf.return_value = callee