openstack
/
charm-percona-cluster
Code Issues Proposed changes

Improve password management for clustered deploys 

In the past, its mandatory to provide the sst and root password
configuration options for clustered deployments to ensure consistent
use of passwords across the cluster from install onwards.

Rework password management and install process to seed passwords
from the lead unit if not supplied via configuration options.

Following units will defer installation until the leader has
stored this information in leader storage for retrieval by
followers.

Closes-Bug: 1454317

Change-Id: I5ab70cae78ed35322bf60048af841de071a69704
This commit is contained in:
James Page 2017-02-24 14:34:42 -05:00
parent fd6097fcb2
commit 60e58482f6
9 changed files with 214 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
README.md
View File

@ -11,43 +11,29 @@ This charm deploys Percona XtraDB Cluster onto Ubuntu.
Usage
=====
WARNING: Its critical that you follow the bootstrap process detailed in this
document in order to end up with a running Active/Active Percona Cluster.
Proxy Configuration
-------------------
If you are deploying this charm on MAAS or in an environment without direct
access to the internet, you will need to allow access to repo.percona.com
as the charm installs packages direct from the Percona respositories. If you
are using squid-deb-proxy, follow the steps below:
echo "repo.percona.com" | sudo tee /etc/squid-deb-proxy/mirror-dstdomain.acl.d/40-percona
sudo service squid-deb-proxy restart
Deployment
----------
The first service unit deployed acts as the seed node for the rest of the
cluster; in order for the cluster to function correctly, the same MySQL passwords
must be used across all nodes:
To deploy this charm:
cat > percona.yaml << EOF
percona-cluster:
root-password: my-root-password
sst-password: my-sst-password
EOF
juju deploy percona-cluster
Once you have created this file, you can deploy the first seed unit:
Passwords required for the correct operation of the deployment are automatically
generated and stored by the lead unit (typically the first unit).
juju deploy --config percona.yaml percona-cluster
To expand the deployment:
Once this node is full operational, you can add extra units one at a time to the
deployment:
juju add-unit -n 2 percona-cluster
juju add-unit percona-cluster
See notes in the 'HA/Clustering' section on safely deploying a PXC cluster
in a single action.
A minimum cluster size of three units is recommended.
The root password for mysql can be retrieved using the following command:
juju run --unit percona-cluster/0 leader-get root-password
This is only usable from within one of the units within the deployment
(access to root is restricted to localhost only).
Memory Configuration
-------------------
@ -86,7 +72,7 @@ the host due to performance schema being turned on. Even with the default now
turned off this value should be carefully considered against the production
requirements and resources available.
[1] http://dev.mysql.com/doc/relnotes/mysql/5.6/en/news-5-6-6.html#mysqld-5-6-6-performance-schema
[1] http://dev.mysql.com/doc/relnotes/mysql/5.6/en/news-5-6-6.html#mysqld-5-6-6-performance-schema
[2] http://www.mysqlcalculator.com/

30
charmhelpers/contrib/database/mysql.py
View File

@ -35,16 +35,15 @@ from charmhelpers.core.hookenv import (
DEBUG,
INFO,
WARNING,
leader_get,
leader_set,
is_leader,
)
from charmhelpers.fetch import (
apt_install,
apt_update,
filter_installed_packages,
)
from charmhelpers.contrib.peerstorage import (
peer_store,
peer_retrieve,
)
from charmhelpers.contrib.network.ip import get_host_ip
try:
@ -61,14 +60,14 @@ except ImportError:
class MySQLHelper(object):
def __init__(self, rpasswdf_template, upasswdf_template, host='localhost',
migrate_passwd_to_peer_relation=True,
migrate_passwd_to_leader_storage=True,
delete_ondisk_passwd_file=True):
self.host = host
# Password file path templates
self.root_passwd_file_template = rpasswdf_template
self.user_passwd_file_template = upasswdf_template
self.migrate_passwd_to_peer_relation = migrate_passwd_to_peer_relation
self.migrate_passwd_to_leader_storage = migrate_passwd_to_leader_storage
# If we migrate we have the option to delete local copy of root passwd
self.delete_ondisk_passwd_file = delete_ondisk_passwd_file
@ -157,13 +156,18 @@ class MySQLHelper(object):
finally:
cursor.close()
def migrate_passwords_to_peer_relation(self, excludes=None):
"""Migrate any passwords storage on disk to cluster peer relation."""
def migrate_passwords_to_leader_storage(self, excludes=None):
"""Migrate any passwords storage on disk to leader storage."""
if not is_leader():
log("Skipping password migration as not the lead unit",
level=DEBUG)
return
dirname = os.path.dirname(self.root_passwd_file_template)
path = os.path.join(dirname, '*.passwd')
for f in glob.glob(path):
if excludes and f in excludes:
log("Excluding %s from peer migration" % (f), level=DEBUG)
log("Excluding %s from leader storage migration" % (f),
level=DEBUG)
continue
key = os.path.basename(f)
@ -171,7 +175,7 @@ class MySQLHelper(object):
_value = passwd.read().strip()
try:
peer_store(key, _value)
leader_set(settings={key: _value})
if self.delete_ondisk_passwd_file:
os.unlink(f)
@ -238,7 +242,7 @@ class MySQLHelper(object):
# First check peer relation.
try:
for key in self.passwd_keys(username):
_password = peer_retrieve(key)
_password = leader_get(key)
if _password:
break
@ -255,8 +259,8 @@ class MySQLHelper(object):
_password = self.get_mysql_password_on_disk(username, password)
# Put on wire if required
if self.migrate_passwd_to_peer_relation:
self.migrate_passwords_to_peer_relation(excludes=excludes)
if self.migrate_passwd_to_leader_storage:
self.migrate_passwords_to_leader_storage(excludes=excludes)
return _password

45
charmhelpers/contrib/network/ip.py
View File

@ -20,13 +20,19 @@ import socket
from functools import partial
from charmhelpers.core.hookenv import unit_get
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import (
config,
log,
network_get_primary_address,
unit_get,
WARNING,
)
from charmhelpers.core.host import (
lsb_release,
)
try:
import netifaces
except ImportError:
@ -511,3 +517,40 @@ def port_has_listener(address, port):
cmd = ['nc', '-z', address, str(port)]
result = subprocess.call(cmd)
return not(bool(result))
def assert_charm_supports_ipv6():
"""Check whether we are able to support charms ipv6."""
if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":
raise Exception("IPv6 is not supported in the charms for Ubuntu "
"versions less than Trusty 14.04")
def get_relation_ip(interface, config_override=None):
"""Return this unit's IP for the given relation.
Allow for an arbitrary interface to use with network-get to select an IP.
Handle all address selection options including configuration parameter
override and IPv6.
Usage: get_relation_ip('amqp', config_override='access-network')
@param interface: string name of the relation.
@param config_override: string name of the config option for network
override. Supports legacy network override configuration parameters.
@raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
@returns IPv6 or IPv4 address
"""
fallback = get_host_ip(unit_get('private-address'))
if config('prefer-ipv6'):
assert_charm_supports_ipv6()
return get_ipv6_addr()[0]
elif config_override and config(config_override):
return get_address_in_network(config(config_override),
fallback)
else:
try:
return network_get_primary_address(interface)
except NotImplementedError:
return fallback

10
config.yaml
View File

@ -104,14 +104,16 @@ options:
type: string
default:
description: |
Root password for MySQL access; must be configured pre-deployment for
Active-Active clusters.
Root account password for new cluster nodes. Overrides the automatic
generation of a password for the root user, but must be set prior to
deployment time to have any effect.
sst-password:
type: string
default:
description: |
Re-sync account password for new cluster nodes; must be configured
pre-deployment for Active-Active clusters.
SST account password for new cluster nodes. Overrides the automatic
generation of a password for the sst user, but must be set prior to
deployment time to have any effect.
sst-method:
type: string
default: xtrabackup-v2

38
hooks/percona_hooks.py
View File

@ -98,6 +98,9 @@ from percona_utils import (
client_node_is_ready,
leader_node_is_ready,
DEFAULT_MYSQL_PORT,
sst_password,
root_password,
pxc_installed,
)
@ -112,6 +115,27 @@ RES_MONITOR_PARAMS = ('params user="sstuser" password="%(sstpass)s" '
'OCF_CHECK_LEVEL="1"')
def install_percona_xtradb_cluster():
'''Attempt PXC install based on seeding of passwords for users'''
if pxc_installed():
log('MySQL already installed, skipping')
return
_root_password = root_password()
_sst_password = sst_password()
if not _root_password or not _sst_password:
log('Passwords not seeded, unable to install MySQL at this'
' point so deferring installation')
return
configure_mysql_root_password(_root_password)
apt_install(determine_packages(), fatal=True)
configure_sstuser(_sst_password)
if config('harden') and 'mysql' in config('harden'):
run_mysql_checks()
@hooks.hook('install.real')
@harden()
def install():
@ -121,13 +145,9 @@ def install():
setup_percona_repo()
elif config('source') is not None:
add_source(config('source'), config('key'))
configure_mysql_root_password(config('root-password'))
apt_update(fatal=True)
apt_install(determine_packages(), fatal=True)
configure_sstuser(config('sst-password'))
if config('harden') and 'mysql' in config('harden'):
run_mysql_checks()
install_percona_xtradb_cluster()
def render_config(clustered=False, hosts=None):
@ -144,7 +164,7 @@ def render_config(clustered=False, hosts=None):
'clustered': clustered,
'cluster_hosts': ",".join(hosts),
'sst_method': config('sst-method'),
'sst_password': config('sst-password'),
'sst_password': sst_password(),
'innodb_file_per_table': config('innodb-file-per-table'),
'table_open_cache': config('table-open-cache'),
'lp1366997_workaround': config('lp1366997-workaround'),
@ -601,7 +621,7 @@ def shared_db_changed(relation_id=None, unit=None):
@hooks.hook('ha-relation-joined')
def ha_relation_joined(relation_id=None):
cluster_config = get_hacluster_config()
sstpsswd = config('sst-password')
sstpsswd = sst_password()
resources = {'res_mysql_monitor': 'ocf:percona:mysql_monitor'}
resource_params = {'res_mysql_monitor':
RES_MONITOR_PARAMS % {'sstpass': sstpsswd}}
@ -673,6 +693,8 @@ def ha_relation_changed():
@hooks.hook('leader-settings-changed')
def leader_settings_changed():
'''Re-trigger install once leader has seeded passwords into install'''
install_percona_xtradb_cluster()
# Notify any changes to data in leader storage
update_shared_db_rels()

38
hooks/percona_utils.py
View File

@ -6,12 +6,14 @@ import tempfile
import os
import shutil
import uuid
from functools import partial
from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.core.host import (
lsb_release,
mkdir,
service,
pwgen,
)
from charmhelpers.core.hookenv import (
charm_dir,
@ -542,7 +544,8 @@ def assess_status(configs):
@returns None - this function is executed for its side-effect
"""
assess_status_func(configs)()
application_version_set(get_upstream_version(determine_packages()[0]))
if pxc_installed():
application_version_set(get_upstream_version(determine_packages()[0]))
def assess_status_func(configs):
@ -720,3 +723,36 @@ def leader_node_is_ready():
if is_unit_paused_set():
return False
return (is_leader() and cluster_ready())
def _get_password(key):
'''Retrieve named password
This function will ensure that a consistent named password
is used across all units in the pxc cluster; the lead unit
will generate or use the root-password configuration option
to seed this value into the deployment.
Once set, it cannot be changed.
@requires: str: named password or None if unable to retrieve
at this point in time
'''
_password = leader_get(key)
if not _password and is_leader():
_password = config(key) or pwgen()
leader_set({key: _password})
return _password
root_password = partial(_get_password, 'root-password')
sst_password = partial(_get_password, 'sst-password')
def pxc_installed():
'''Determine whether percona-xtradb-cluster is installed
@returns: boolean: indicating installation
'''
return os.path.exists('/usr/sbin/mysqld')

10
tests/basic_deployment.py
View File

@ -65,9 +65,7 @@ class BasicDeployment(OpenStackAmuletDeployment):
def _get_configs(self):
"""Configure all of the services."""
cfg_percona = {'sst-password': 'ubuntu',
'root-password': 't00r',
'min-cluster-size': self.units,
cfg_percona = {'min-cluster-size': self.units,
'vip': self.vip}
cfg_ha = {'debug': True,
@ -259,9 +257,9 @@ class BasicDeployment(OpenStackAmuletDeployment):
u = unit
else:
u = self.master_unit
cmd = ("mysql -uroot -pt00r -e\"show status like '%s';\"| "
"grep %s" % (attr, attr))
root_password, _ = u.run('leader-get root-password')
cmd = ("mysql -uroot -p{} -e\"show status like '{}';\"| "
"grep {}".format(root_password, attr, attr))
output, code = u.run(cmd)
if code != 0:
self.log.debug("command returned non-zero '%s'" % (code))

61
unit_tests/test_percona_hooks.py
View File

@ -29,13 +29,15 @@ TO_PATCH = ['log', 'config',
'is_clustered',
'get_ipv6_addr',
'get_hacluster_config',
'update_dns_ha_resource_params']
'update_dns_ha_resource_params',
'sst_password']
class TestHARelation(CharmTestCase):
def setUp(self):
CharmTestCase.setUp(self, hooks, TO_PATCH)
self.network_get_primary_address.side_effect = NotImplementedError
self.sst_password.return_value = 'ubuntu'
def test_resources(self):
self.relation_ids.return_value = ['ha:1']
@ -47,7 +49,6 @@ class TestHARelation(CharmTestCase):
self.get_netmask_for_address.return_value = None
self.get_iface_for_address.return_value = None
self.test_config.set('vip', '10.0.3.3')
self.test_config.set('sst-password', password)
self.get_hacluster_config.return_value = {
'vip': '10.0.3.3',
'ha-bindiface': 'eth0',
@ -111,7 +112,7 @@ class TestHARelation(CharmTestCase):
'cidr_netmask="20" '
'nic="eth1"'),
'res_mysql_monitor':
hooks.RES_MONITOR_PARAMS % {'sstpass': 'None'}}
hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}}
call_args, call_kwargs = self.relation_set.call_args
self.assertEqual(resource_params, call_kwargs['resource_params'])
@ -145,7 +146,7 @@ class TestHARelation(CharmTestCase):
'cidr_netmask="16" '
'nic="eth1"'),
'res_mysql_monitor':
hooks.RES_MONITOR_PARAMS % {'sstpass': 'None'}}
hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}}
call_args, call_kwargs = self.relation_set.call_args
self.assertEqual(resource_params, call_kwargs['resource_params'])
@ -241,3 +242,55 @@ class TestConfigChanged(CharmTestCase):
self.is_relation_made.return_value = False
hooks.config_changed()
self.open_port.assert_called_with(3306)
class TestInstallPerconaXtraDB(CharmTestCase):
TO_PATCH = [
'log',
'pxc_installed',
'root_password',
'sst_password',
'configure_mysql_root_password',
'apt_install',
'determine_packages',
'configure_sstuser',
'config',
'run_mysql_checks',
]
def setUp(self):
CharmTestCase.setUp(self, hooks, self.TO_PATCH)
self.config.side_effect = self.test_config.get
self.pxc_installed.return_value = False
def test_installed(self):
self.pxc_installed.return_value = True
hooks.install_percona_xtradb_cluster()
self.configure_mysql_root_password.assert_not_called()
self.apt_install.assert_not_called()
def test_passwords_not_initialized(self):
self.root_password.return_value = None
self.sst_password.return_value = None
hooks.install_percona_xtradb_cluster()
self.configure_mysql_root_password.assert_not_called()
self.configure_sstuser.assert_not_called()
self.apt_install.assert_not_called()
self.root_password.return_value = None
self.sst_password.return_value = 'testpassword'
hooks.install_percona_xtradb_cluster()
self.configure_sstuser.assert_not_called()
self.configure_mysql_root_password.assert_not_called()
self.apt_install.assert_not_called()
def test_passwords_initialized(self):
self.root_password.return_value = 'rootpassword'
self.sst_password.return_value = 'testpassword'
self.determine_packages.return_value = ['pxc-5.6']
hooks.install_percona_xtradb_cluster()
self.configure_mysql_root_password.assert_called_with('rootpassword')
self.configure_sstuser.assert_called_with('testpassword')
self.apt_install.assert_called_with(['pxc-5.6'], fatal=True)
self.run_mysql_checks.assert_not_called()

5
unit_tests/test_percona_utils.py
View File

@ -305,14 +305,17 @@ class UtilsTestsCTC(CharmTestCase):
stat, _ = percona_utils.charm_check_func()
assert stat == 'active'
@mock.patch.object(percona_utils, 'pxc_installed')
@mock.patch.object(percona_utils, 'determine_packages')
@mock.patch.object(percona_utils, 'application_version_set')
@mock.patch.object(percona_utils, 'get_upstream_version')
def test_assess_status(self, get_upstream_version,
application_version_set,
determine_packages):
determine_packages,
pxc_installed):
get_upstream_version.return_value = '5.6.17'
determine_packages.return_value = ['percona-xtradb-cluster-server-5.6']
pxc_installed.return_value = True
with mock.patch.object(percona_utils, 'assess_status_func') as asf:
callee = mock.Mock()
asf.return_value = callee