From 78501848024df6fc4f27951585fc3eccacede8bd Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 13 Apr 2018 06:41:48 +0000 Subject: [PATCH] Add support for HA deployments using etcd Add support for relating vault to etcd to support HA deployments of vault. Depends-On: Iebb2415077b682dfdf590b4b5f35a3c593ed3d69 Depends-On: I05a04bdc237b2a698b2f2d29e18c5a33510a2513 Change-Id: I161db8296031776652425f563a11de3ee8f7f86e --- src/layer.yaml | 1 + src/metadata.yaml | 2 + src/reactive/vault.py | 81 ++++++++++++- src/templates/vault.hcl.j2 | 13 +++ src/tests/bundles/xenial-ha-mysql.yaml | 24 ++++ src/tests/tests.yaml | 1 + src/wheelhouse.txt | 1 + test-requirements.txt | 1 + unit_tests/test_vault.py | 152 ++++++++++++++++++++----- 9 files changed, 247 insertions(+), 29 deletions(-) create mode 100644 src/tests/bundles/xenial-ha-mysql.yaml create mode 100644 src/wheelhouse.txt diff --git a/src/layer.yaml b/src/layer.yaml index cbfbced..9e96a0a 100644 --- a/src/layer.yaml +++ b/src/layer.yaml @@ -4,6 +4,7 @@ includes: - interface:nrpe-external-master - interface:pgsql - interface:mysql-shared + - interface:etcd options: basic: packages: diff --git a/src/metadata.yaml b/src/metadata.yaml index 2cd1bd5..cb6be16 100644 --- a/src/metadata.yaml +++ b/src/metadata.yaml @@ -21,6 +21,8 @@ requires: interface: pgsql shared-db: interface: mysql-shared + etcd: + interface: etcd provides: nrpe-external-master: interface: nrpe-external-master diff --git a/src/reactive/vault.py b/src/reactive/vault.py index 0bce605..01ea2e7 100644 --- a/src/reactive/vault.py +++ b/src/reactive/vault.py @@ -1,7 +1,9 @@ import base64 +import hvac import psycopg2 import subprocess + from charmhelpers.contrib.charmsupport.nrpe import ( NRPE, add_init_service_checks, @@ -10,12 +12,17 @@ from charmhelpers.contrib.charmsupport.nrpe import ( ) from charmhelpers.core.hookenv import ( + DEBUG, config, + log, open_port, status_set, + unit_private_ip, ) from charmhelpers.core.host import ( + service_restart, + service_running, service_start, write_file, ) @@ -33,6 +40,10 @@ from charms.reactive import ( when_not, ) +from charms.reactive.relations import ( + endpoint_from_flag, +) + # See https://www.vaultproject.io/docs/configuration/storage/postgresql.html VAULT_TABLE_DDL = """ @@ -50,6 +61,24 @@ CREATE INDEX IF NOT EXISTS parent_path_idx ON vault_kv_store (parent_path); """ +def get_client(): + return hvac.Client(url=get_api_url()) + + +def can_restart(): + safe_restart = False + if not service_running('vault'): + safe_restart = True + else: + client = get_client() + if not client.is_initialized(): + safe_restart = True + elif client.is_sealed(): + safe_restart = True + log("Safe to restart: {}".format(safe_restart), level=DEBUG) + return safe_restart + + def ssl_available(config): if '' in (config['ssl-cert'], config['ssl-key']): return False @@ -59,7 +88,27 @@ def ssl_available(config): def configure_vault(context): context['disable_mlock'] = config()['disable-mlock'] context['ssl_available'] = is_state('vault.ssl.available') + log("Running configure_vault", level=DEBUG) + context['disable_mlock'] = config()['disable-mlock'] + context['ssl_available'] = is_state('vault.ssl.available') + etcd = endpoint_from_flag('etcd.available') + if etcd: + log("Etcd detected, adding to context", level=DEBUG) + context['etcd_conn'] = etcd.connection_string() + context['etcd_tls_ca_file'] = '/var/snap/vault/common/etcd-ca.pem' + context['etcd_tls_cert_file'] = '/var/snap/vault/common/etcd-cert.pem' + context['etcd_tls_key_file'] = '/var/snap/vault/common/etcd.key' + etcd.save_client_credentials( + context['etcd_tls_key_file'], + context['etcd_tls_cert_file'], + context['etcd_tls_ca_file']) + context['vault_api_url'] = get_api_url() + log("Etcd detected, setting vault_api_url to {}".format( + context['vault_api_url'])) + else: + log("Etcd not detected", level=DEBUG) status_set('maintenance', 'creating vault config') + log("Rendering vault.hcl.j2", level=DEBUG) render( 'vault.hcl.j2', '/var/snap/vault/common/vault.hcl', @@ -72,7 +121,10 @@ def configure_vault(context): {}, perms=0o644) status_set('maintenance', 'starting vault') - service_start('vault') # restart seals the vault + if can_restart(): + service_restart('vault') + else: + service_start('vault') # restart seals the vault status_set('maintenance', 'opening vault port') open_port(8200) set_state('configured') @@ -84,6 +136,15 @@ def configure_vault(context): status_set('active', '=^_^=') +def get_api_url(): + protocol = 'http' + port = '8200' + ip = unit_private_ip() + if is_state('vault.ssl.available'): + protocol = 'https' + return '{}://{}:{}'.format(protocol, ip, port) + + @when('snap.installed.vault') @when_not('configured') @when('db.master.available') @@ -207,6 +268,24 @@ def ssl_ca_changed(): remove_state('vault.ssl.configured') +@when_not('etcd.local.configured') +@when('etcd.available') +def etcd_setup(etcd): + log("Detected etcd.available, removing configured", level=DEBUG) + remove_state('configured') + remove_state('etcd.local.unconfigured') + set_state('etcd.local.configured') + + +@when_not('etcd.local.unconfigured') +@when_not('etcd.available') +def etcd_not_ready(): + log("Detected etcd_not_ready, removing configured", level=DEBUG) + set_state('etcd.local.unconfigured') + remove_state('etcd.local.configured') + remove_state('configured') + + @when('configured') @when('nrpe-external-master.available') @when_not('vault.nrpe.configured') diff --git a/src/templates/vault.hcl.j2 b/src/templates/vault.hcl.j2 index 1102415..2a9836a 100644 --- a/src/templates/vault.hcl.j2 +++ b/src/templates/vault.hcl.j2 @@ -1,3 +1,6 @@ +{%- if vault_api_url %} +api_addr = "{{ vault_api_url }}" +{%- endif %} {%- if disable_mlock %} disable_mlock = true {%- endif %} @@ -13,6 +16,16 @@ storage "mysql" { address = "{{ mysql_db_relation.db_host() }}:3306" } {%- endif %} +{%- if etcd_conn %} +ha_storage "etcd" { + ha_enabled = "true" + address = "{{ etcd_conn }}" + tls_ca_file = "{{ etcd_tls_ca_file }}" + tls_cert_file = "{{ etcd_tls_cert_file }}" + tls_key_file = "{{ etcd_tls_key_file }}" + etcd_api = "v3" +} +{%- endif %} listener "tcp" { address = "0.0.0.0:8200" {%- if ssl_available %} diff --git a/src/tests/bundles/xenial-ha-mysql.yaml b/src/tests/bundles/xenial-ha-mysql.yaml new file mode 100644 index 0000000..a85a2ea --- /dev/null +++ b/src/tests/bundles/xenial-ha-mysql.yaml @@ -0,0 +1,24 @@ +series: xenial +services: + vault: + num_units: 3 + series: xenial + charm: ../../../vault + mysql: + charm: cs:mysql + num_units: 1 + easyrsa: + charm: cs:~containers/easyrsa + num_units: 1 + etcd: + charm: cs:etcd + num_units: 2 + options: + channel: 3.1/stable +relations: +- - vault:shared-db + - mysql:shared-db +- - etcd:certificates + - easyrsa:client +- - etcd:db + - vault:etcd diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml index 4f2ea5b..b5b20b1 100644 --- a/src/tests/tests.yaml +++ b/src/tests/tests.yaml @@ -4,6 +4,7 @@ tests: configure: - zaza.charm_tests.vault.setup.basic_setup gate_bundles: + - xenial-ha-mysql - xenial-postgres - xenial-mysql smoke_bundles: diff --git a/src/wheelhouse.txt b/src/wheelhouse.txt new file mode 100644 index 0000000..3e6a595 --- /dev/null +++ b/src/wheelhouse.txt @@ -0,0 +1 @@ +hvac diff --git a/test-requirements.txt b/test-requirements.txt index f5617b8..8e2dfb4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ # Unit test requirements +hvac flake8>=2.2.4,<=2.4.1 os-testr>=0.4.1 charms.reactive diff --git a/unit_tests/test_vault.py b/unit_tests/test_vault.py index ba2a162..0faec6a 100644 --- a/unit_tests/test_vault.py +++ b/unit_tests/test_vault.py @@ -16,6 +16,35 @@ import reactive.vault as handlers # noqa: E402 class TestHandlers(unittest.TestCase): + def setUp(self): + super(TestHandlers, self).setUp() + self.patches = [ + 'config', + 'endpoint_from_flag', + 'is_state', + 'log', + 'open_port', + 'service_restart', + 'service_running', + 'service_start', + 'set_state', + 'status_set', + 'remove_state', + 'render', + 'unit_private_ip', + ] + self.patch_all() + + def _patch(self, method): + _m = patch.object(handlers, method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def patch_all(self): + for method in self.patches: + setattr(self, method, self._patch(method)) + def test_ssl_available(self): self.assertFalse(handlers.ssl_available({ 'ssl-cert': '', @@ -30,20 +59,15 @@ class TestHandlers(unittest.TestCase): 'ssl-cert': 'acert', 'ssl-key': 'akey'})) - @patch.object(handlers, 'is_state') - @patch.object(handlers, 'config') - @patch.object(handlers, 'open_port') - @patch.object(handlers, 'service_start') - @patch.object(handlers, 'render') - @patch.object(handlers, 'status_set') - @patch.object(handlers, 'remove_state') - def test_configure_vault(self, remove_state, status_set, render, - service_start, open_port, config, is_state): - config.return_value = {'disable-mlock': False} - is_state.return_value = True + @patch.object(handlers, 'can_restart') + def test_configure_vault(self, can_restart): + can_restart.return_value = True + self.config.return_value = {'disable-mlock': False} + self.is_state.return_value = True db_context = { 'storage_name': 'psql', 'psql_db_conn': 'myuri'} + self.endpoint_from_flag.return_value = None handlers.configure_vault(db_context) expected_context = { 'storage_name': 'psql', @@ -70,12 +94,12 @@ class TestHandlers(unittest.TestCase): {}, perms=0o644) ] - open_port.assert_called_once_with(8200) - status_set.assert_has_calls(status_set_calls) - render.assert_has_calls(render_calls) + self.open_port.assert_called_once_with(8200) + self.status_set.assert_has_calls(status_set_calls) + self.render.assert_has_calls(render_calls) # Check flipping disable-mlock makes it to the context - config.return_value = {'disable-mlock': True} + self.config.return_value = {'disable-mlock': True} expected_context['disable_mlock'] = True handlers.configure_vault(db_context) render_calls = [ @@ -90,7 +114,7 @@ class TestHandlers(unittest.TestCase): {}, perms=0o644) ] - render.assert_has_calls(render_calls) + self.render.assert_has_calls(render_calls) @patch.object(handlers, 'configure_vault') def test_configure_vault_psql(self, configure_vault): @@ -109,28 +133,24 @@ class TestHandlers(unittest.TestCase): 'storage_name': 'mysql', 'mysql_db_relation': mysql}) - @patch.object(handlers, 'remove_state') - def test_disable_mlock_changed(self, remove_state): + def test_disable_mlock_changed(self): handlers.disable_mlock_changed() - remove_state.assert_called_once_with('configured') + self.remove_state.assert_called_once_with('configured') - @patch.object(handlers, 'remove_state') - def test_upgrade_charm(self, remove_state): + def test_upgrade_charm(self): calls = [mock.call('configured'), mock.call('vault.nrpe.configured'), mock.call('vault.ssl.configured')] handlers.upgrade_charm() - remove_state.assert_has_calls(calls) + self.remove_state.assert_has_calls(calls) def test_request_db(self): psql = mock.MagicMock() handlers.request_db(psql) psql.set_database.assert_called_once_with('vault') - @patch.object(handlers, 'set_state') @patch.object(handlers, 'psycopg2') - @patch.object(handlers, 'status_set') - def test_create_vault_table(self, status_set, psycopg2, set_state): + def test_create_vault_table(self, psycopg2): psql = mock.MagicMock() psql.master = 'myuri' handlers.create_vault_table(psql) @@ -140,7 +160,83 @@ class TestHandlers(unittest.TestCase): ] psycopg2.connect().cursor().execute.assert_has_calls(db_calls) - @patch.object(handlers, 'remove_state') - def test_database_not_ready(self, remove_state): + def test_database_not_ready(self): handlers.database_not_ready() - remove_state.assert_called_once_with('vault.schema.created') + self.remove_state.assert_called_once_with('vault.schema.created') + + @patch.object(handlers, 'can_restart') + @patch.object(handlers, 'get_api_url') + def test_configure_vault_etcd(self, get_api_url, can_restart): + can_restart.return_value = True + get_api_url.return_value = 'http://this-unit' + self.config.return_value = {'disable-mlock': False} + etcd_mock = mock.MagicMock() + etcd_mock.connection_string.return_value = 'http://etcd' + self.endpoint_from_flag.return_value = etcd_mock + self.is_state.return_value = True + handlers.configure_vault({}) + expected_context = { + 'disable_mlock': False, + 'ssl_available': True, + 'etcd_conn': 'http://etcd', + 'etcd_tls_ca_file': '/var/snap/vault/common/etcd-ca.pem', + 'etcd_tls_cert_file': '/var/snap/vault/common/etcd-cert.pem', + 'etcd_tls_key_file': '/var/snap/vault/common/etcd.key', + 'vault_api_url': 'http://this-unit'} + render_calls = [ + mock.call( + 'vault.hcl.j2', + '/var/snap/vault/common/vault.hcl', + expected_context, + perms=0o600), + mock.call( + 'vault.service.j2', + '/etc/systemd/system/vault.service', + {}, + perms=0o644) + ] + self.render.assert_has_calls(render_calls) + + @patch.object(handlers.hvac, 'Client') + @patch.object(handlers, 'get_api_url') + def test_get_client(self, get_api_url, hvac_Client): + get_api_url.return_value = 'http://this-unit' + handlers.get_client() + hvac_Client.assert_called_once_with(url='http://this-unit') + + def test_can_restart_vault_down(self): + self.service_running.return_value = False + self.assertTrue(handlers.can_restart()) + + @patch.object(handlers, 'get_client') + def test_can_restart_not_initialized(self, get_client): + hvac_mock = mock.MagicMock() + hvac_mock.is_initialized.return_value = False + get_client.return_value = hvac_mock + self.assertTrue(handlers.can_restart()) + + @patch.object(handlers, 'get_client') + def test_can_restart_sealed(self, get_client): + hvac_mock = mock.MagicMock() + hvac_mock.is_initialized.return_value = True + hvac_mock.is_sealed.return_value = True + get_client.return_value = hvac_mock + self.assertTrue(handlers.can_restart()) + + @patch.object(handlers, 'get_client') + def test_can_restart_unsealed(self, get_client): + hvac_mock = mock.MagicMock() + hvac_mock.is_initialized.return_value = True + hvac_mock.is_sealed.return_value = False + get_client.return_value = hvac_mock + self.assertFalse(handlers.can_restart()) + + def test_get_api_url_ssl(self): + self.is_state.return_value = True + self.unit_private_ip.return_value = '1.2.3.4' + self.assertEqual(handlers.get_api_url(), 'https://1.2.3.4:8200') + + def test_get_api_url_nossl(self): + self.is_state.return_value = False + self.unit_private_ip.return_value = '1.2.3.4' + self.assertEqual(handlers.get_api_url(), 'http://1.2.3.4:8200')