diff --git a/src/actions/actions.py b/src/actions/actions.py index 4b97f68..1e465ad 100755 --- a/src/actions/actions.py +++ b/src/actions/actions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/local/sbin/charm-env python3 # Copyright 2018 Canonical Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,8 @@ import charmhelpers.core.hookenv as hookenv import charm.vault as vault +import charms.reactive + def authorize_charm_action(*args): """Create a role allowing the charm to perform certain vault actions. @@ -55,6 +57,8 @@ def main(args): action(args) except Exception as e: hookenv.action_fail(str(e)) + else: + charms.reactive.main() if __name__ == "__main__": diff --git a/src/layer.yaml b/src/layer.yaml index daf6a37..1ac370a 100644 --- a/src/layer.yaml +++ b/src/layer.yaml @@ -1,11 +1,13 @@ includes: - layer:basic - layer:snap + - layer:leadership - interface:nrpe-external-master - interface:pgsql - interface:mysql-shared - interface:etcd - interface:hacluster + - interface:vault-kv options: basic: packages: diff --git a/src/lib/charm/vault.py b/src/lib/charm/vault.py index 9b0274a..9c484ea 100644 --- a/src/lib/charm/vault.py +++ b/src/lib/charm/vault.py @@ -1,3 +1,17 @@ +# Copyright 2018 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import functools import json import requests @@ -50,6 +64,19 @@ path "sys/mounts/" { }""" VAULT_HEALTH_URL = '{vault_addr}/v1/sys/health' +VAULT_LOCALHOST_URL = "http://127.0.0.1:8220" + +SECRET_BACKEND_HCL = """ +path "{backend}/{hostname}/*" {{ + capabilities = ["create", "read", "update", "delete", "list"] +}} +""" + +SECRET_BACKEND_SHARED_HCL = """ +path "{backend}/*" {{ + capabilities = ["create", "read", "update", "delete", "list"] +}} +""" def binding_address(binding): @@ -59,9 +86,9 @@ def binding_address(binding): return hookenv.unit_private_ip() -def get_vault_url(binding, port): +def get_vault_url(binding, port, address=None): protocol = 'http' - ip = binding_address(binding) + ip = address or binding_address(binding) if charms.reactive.is_state('vault.ssl.available'): protocol = 'https' return '{}://{}:{}'.format(protocol, ip, port) @@ -110,9 +137,8 @@ def setup_charm_vault_access(token=None): :rtype: str""" if not token: token = hookenv.leader_get('token') - vault_url = get_api_url() client = hvac.Client( - url=vault_url, + url=VAULT_LOCALHOST_URL, token=token) enable_approle_auth(client) policies = [CHARM_POLICY_NAME] @@ -129,13 +155,13 @@ def get_local_charm_access_role_id(): return hookenv.leader_get(CHARM_ACCESS_ROLE_ID) -def get_client(): +def get_client(url=None): """Provide a client for talking to the vault api :returns: vault client :rtype: hvac.Client """ - return hvac.Client(url=get_api_url()) + return hvac.Client(url=url or get_api_url()) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=10), @@ -148,7 +174,7 @@ def get_vault_health(): :rtype: dict """ response = requests.get( - VAULT_HEALTH_URL.format(vault_addr=get_api_url())) + VAULT_HEALTH_URL.format(vault_addr=VAULT_LOCALHOST_URL)) return response.json() @@ -189,7 +215,7 @@ def initialize_vault(shares=1, threshold=1): :param threshold: Minimum number of shares needed to unlock :type threshold: int """ - client = get_client() + client = get_client(url=VAULT_LOCALHOST_URL) result = client.initialize(shares, threshold) client.token = result['root_token'] hookenv.leader_set( @@ -200,7 +226,7 @@ def initialize_vault(shares=1, threshold=1): def unseal_vault(keys=None): """Unseal vault with provided keys. If no keys are provided retrieve from leader db""" - client = get_client() + client = get_client(url=VAULT_LOCALHOST_URL) if not keys: keys = json.loads(hookenv.leader_get()['keys']) for key in keys: @@ -219,7 +245,7 @@ def can_restart(): elif hookenv.config('auto-unlock'): safe_restart = True else: - client = get_client() + client = get_client(url=VAULT_LOCALHOST_URL) if not client.is_initialized(): safe_restart = True elif client.is_sealed(): @@ -228,3 +254,51 @@ def can_restart(): "Safe to restart: {}".format(safe_restart), level=hookenv.DEBUG) return safe_restart + + +def configure_secret_backend(client, name): + """Ensure a KV backend is enabled + + :param client: Vault client + :ptype client: hvac.Client + :param name: Name of backend to enable + :ptype name: str""" + if '{}/'.format(name) not in client.list_secret_backends(): + client.enable_secret_backend(backend_type='kv', + description='Charm created KV backend', + mount_point=name) + + +def configure_policy(client, name, hcl): + """Create/update a role within vault associating the supplied policies + + :param client: Vault client + :ptype client: hvac.Client + :param name: Name of policy to create + :ptype name: str + :param hcl: Vault policy HCL + :ptype hcl: str""" + client.set_policy(name, hcl) + + +def configure_approle(client, name, cidr, policies): + """Create/update a role within vault associating the supplied policies + + :param client: Vault client + :ptype client: hvac.Client + :param name: Name of role + :ptype name: str + :param cidr: Network address of remote unit + :ptype cidr: str + :param policies: List of policy names + :ptype policies: [str, str, ...] + :returns: Id of created role + :rtype: str""" + client.create_role( + name, + token_ttl='60s', + token_max_ttl='60s', + policies=policies, + bind_secret_id='false', + bound_cidr_list=cidr) + return client.get_role_id(name) diff --git a/src/metadata.yaml b/src/metadata.yaml index 92f35e9..9b8bfd0 100644 --- a/src/metadata.yaml +++ b/src/metadata.yaml @@ -32,6 +32,8 @@ provides: nrpe-external-master: interface: nrpe-external-master scope: container + secrets: + interface: vault-kv peers: cluster: interface: vault-ha diff --git a/src/reactive/vault_handlers.py b/src/reactive/vault_handlers.py index e78ba9d..1b74cb2 100644 --- a/src/reactive/vault_handlers.py +++ b/src/reactive/vault_handlers.py @@ -1,6 +1,7 @@ import base64 import psycopg2 import subprocess +import tenacity from charmhelpers.contrib.charmsupport.nrpe import ( @@ -56,6 +57,7 @@ from charms.reactive.flags import ( ) from charms.layer import snap + import lib.charm.vault as vault # See https://www.vaultproject.io/docs/configuration/storage/postgresql.html @@ -367,6 +369,99 @@ def file_change_auto_unlock_mode(): vault.prepare_vault() +@when('leadership.is_leader') +@when('endpoint.secrets.new-request') +def configure_secrets_backend(): + """ Process requests for setup and access to simple kv secret backends """ + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=10), + stop=tenacity.stop_after_attempt(10), + reraise=True) + def _check_vault_status(client): + if (not service_running('vault') or + not client.is_initialized() or + client.is_sealed()): + return False + return True + + # NOTE: use localhost listener as policy only allows 127.0.0.1 to + # administer the local vault instances via the charm + client = vault.get_client(url=vault.VAULT_LOCALHOST_URL) + + status_ok = _check_vault_status(client) + if not status_ok: + log('Unable to process new secret backend requests,' + ' deferring until vault is fully configured', level=DEBUG) + return + + charm_role_id = vault.get_local_charm_access_role_id() + if charm_role_id is None: + log('Charm access to vault not configured, deferring' + ' secrets backend setup', level=DEBUG) + return + client.auth_approle(charm_role_id) + + secrets = endpoint_from_flag('endpoint.secrets.new-request') + requests = secrets.requests() + + # Configure KV secret backends + backends = set([request['secret_backend'] + for request in requests]) + for backend in backends: + if not backend.startswith('charm-'): + continue + vault.configure_secret_backend(client, name=backend) + + # Configure AppRoles for application unit access + for request in requests: + # NOTE: backends must start with charm- + backend_name = request['secret_backend'] + if not backend_name.startswith('charm-'): + continue + + unit = request['unit'] + hostname = request['hostname'] + access_address = request['access_address'] + isolated = request['isolated'] + unit_name = unit.unit_name.replace('/', '-') + policy_name = approle_name = 'charm-{}'.format(unit_name) + + if isolated: + policy_template = vault.SECRET_BACKEND_HCL + else: + policy_template = vault.SECRET_BACKEND_SHARED_HCL + + vault.configure_policy( + client, + name=policy_name, + hcl=policy_template.format(backend=backend_name, + hostname=hostname) + ) + + approle_id = vault.configure_approle( + client, + name=approle_name, + cidr='{}/32'.format(access_address), + policies=[policy_name]) + + secrets.set_role_id(unit=unit, + role_id=approle_id) + + clear_flag('endpoint.secrets.new-request') + + +@when('secrets.connected') +def send_vault_url_and_ca(): + secrets = endpoint_from_flag('secrets.connected') + if is_flag_set('ha.available'): + vault_url = vault.get_api_url(address=config('vip')) + else: + vault_url = vault.get_api_url() + secrets.publish_url(vault_url=vault_url) + + if config('ssl-ca'): + secrets.publish_ca(vault_ca=config('ssl-ca')) + + @when('snap.installed.vault') def prime_assess_status(): atexit(_assess_status) diff --git a/src/templates/vault.hcl.j2 b/src/templates/vault.hcl.j2 index f4719c8..7c70f2d 100644 --- a/src/templates/vault.hcl.j2 +++ b/src/templates/vault.hcl.j2 @@ -37,4 +37,10 @@ listener "tcp" { {%- else %} tls_disable = 1 {%- endif %} -} \ No newline at end of file +} + +# Localhost only listener for charm access to vault. +listener "tcp" { + address = "127.0.0.1:8220" + tls_disable = 1 +} diff --git a/unit_tests/test_lib_charm_vault.py b/unit_tests/test_lib_charm_vault.py index 525d458..825c60f 100644 --- a/unit_tests/test_lib_charm_vault.py +++ b/unit_tests/test_lib_charm_vault.py @@ -176,7 +176,7 @@ class TestLibCharmVault(unit_tests.test_utils.CharmTestCase): self.assertEqual(vault.get_vault_health(), self._health_response) requests.get.assert_called_with( - "https://vault.demo.com:8200/v1/sys/health") + "http://127.0.0.1:8220/v1/sys/health") mock_response.json.assert_called_once() @patch.object(vault, 'setup_charm_vault_access') @@ -315,3 +315,46 @@ class TestLibCharmVault(unit_tests.test_utils.CharmTestCase): can_restart.return_value = False vault.opportunistic_restart() service_start.assert_called_once_with('vault') + + def test_configure_secret_backend(self): + hvac_client = mock.MagicMock() + hvac_client.list_secret_backends.return_value = ['secrets/'] + vault.configure_secret_backend(hvac_client, 'test') + hvac_client.enable_secret_backend.assert_called_once_with( + backend_type='kv', + description=mock.ANY, + mount_point='test') + + def test_configure_secret_backend_noop(self): + hvac_client = mock.MagicMock() + hvac_client.list_secret_backends.return_value = ['secrets/'] + vault.configure_secret_backend(hvac_client, 'secrets') + hvac_client.enable_secret_backend.assert_not_called() + + def test_configure_policy(self): + hvac_client = mock.MagicMock() + vault.configure_policy(hvac_client, 'test-policy', 'test-hcl') + hvac_client.set_policy.assert_called_once_with( + 'test-policy', + 'test-hcl', + ) + + def test_configure_approle(self): + hvac_client = mock.MagicMock() + hvac_client.get_role_id.return_value = 'some-UUID' + self.assertEqual( + vault.configure_approle(hvac_client, + 'test-role', + '10.5.0.20/32', + ['test-policy']), + 'some-UUID' + ) + hvac_client.create_role.assert_called_once_with( + 'test-role', + token_ttl='60s', + token_max_ttl='60s', + policies=['test-policy'], + bind_secret_id='false', + bound_cidr_list='10.5.0.20/32' + ) + hvac_client.get_role_id.assert_called_with('test-role') diff --git a/unit_tests/test_reactive_vault_handlers.py b/unit_tests/test_reactive_vault_handlers.py index 3d25371..ada1a03 100644 --- a/unit_tests/test_reactive_vault_handlers.py +++ b/unit_tests/test_reactive_vault_handlers.py @@ -135,6 +135,7 @@ class TestHandlers(unit_tests.test_utils.CharmTestCase): perms=0o644) ] self.render.assert_has_calls(render_calls) + self.service.assert_called_with('enable', 'vault') @patch.object(handlers, 'configure_vault') def test_configure_vault_psql(self, configure_vault): @@ -467,3 +468,118 @@ class TestHandlers(unit_tests.test_utils.CharmTestCase): self.assertFalse(hacluster_mock.add_dnsha.called) self.assertFalse(hacluster_mock.bind_resources.called) self.set_flag.assert_called_once_with('config.dns_vip.invalid') + + def fixture_test_requests(self): + test_requests = [] + test_requests.append({ + 'secret_backend': 'charm-vaultlocker', + 'hostname': 'juju-123456-0', + 'isolated': True, + 'access_address': '10.20.4.5', + 'unit': mock.MagicMock() + }) + test_requests[-1]['unit'].unit_name = 'ceph-osd/0' + + test_requests.append({ + 'secret_backend': 'charm-supersecrets', + 'hostname': 'juju-789012-0', + 'isolated': True, + 'access_address': '10.20.4.20', + 'unit': mock.MagicMock() + }) + test_requests[-1]['unit'].unit_name = 'omg/0' + + return test_requests + + @mock.patch.object(handlers, 'vault') + def test_configure_secrets_backend(self, _vault): + hvac_client = mock.MagicMock() + _vault.get_client.return_value = hvac_client + # Vault is up and running, init'ed and unsealed + hvac_client.is_initialized.return_value = True + hvac_client.is_sealed.return_value = False + self.service_running.return_value = True + + _vault.get_local_charm_access_role_id.return_value = 'local-approle' + + secrets_interface = mock.MagicMock() + self.endpoint_from_flag.return_value = secrets_interface + secrets_interface.requests.return_value = self.fixture_test_requests() + _vault.configure_approle.side_effect = ['role_a', 'role_b'] + self.is_flag_set.return_value = False + _vault.get_api_url.return_value = "http://vault:8200" + + handlers.configure_secrets_backend() + + hvac_client.auth_approle.assert_called_once_with('local-approle') + _vault.configure_secret_backend.assert_has_calls([ + mock.call(hvac_client, name='charm-vaultlocker'), + mock.call(hvac_client, name='charm-supersecrets') + ]) + + _vault.configure_policy.assert_has_calls([ + mock.call(hvac_client, name='charm-ceph-osd-0', hcl=mock.ANY), + mock.call(hvac_client, name='charm-omg-0', hcl=mock.ANY) + ]) + + _vault.configure_approle.assert_has_calls([ + mock.call(hvac_client, name='charm-ceph-osd-0', + cidr="10.20.4.5/32", + policies=mock.ANY), + mock.call(hvac_client, name='charm-omg-0', + cidr="10.20.4.20/32", + policies=mock.ANY) + ]) + + secrets_interface.set_role_id.assert_has_calls([ + mock.call(unit=mock.ANY, + role_id='role_a'), + mock.call(unit=mock.ANY, + role_id='role_b'), + ]) + + self.clear_flag.assert_called_once_with('endpoint.secrets.new-request') + + @mock.patch.object(handlers, 'vault') + def send_vault_url_and_ca(self, _vault): + _test_config = { + 'vip': '10.5.100.1', + 'ssl-ca': 'test-ca', + } + self.config.side_effect = lambda key: _test_config.get(key) + mock_secrets = mock.MagicMock() + self.endpoint_from_flag.return_value = mock_secrets + self.is_flag_set.return_value = False + _vault.get_api_url.return_value = 'http://10.5.0.23:8200' + handlers.send_vault_url_and_ca() + self.endpoint_from_flag.assert_called_with('secrets.connected') + self.is_flag_set.assert_called_with('ha.available') + _vault.get_api_url.assert_called_once_with() + mock_secrets.publish_url.assert_called_once_with( + vault_url='http://10.5.0.23:8200' + ) + mock_secrets.publish_ca.assert_called_once_with( + vault_ca='test-ca' + ) + + @mock.patch.object(handlers, 'vault') + def send_vault_url_and_ca_ha(self, _vault): + _test_config = { + 'vip': '10.5.100.1', + 'ssl-ca': 'test-ca', + } + self.config.side_effect = lambda key: _test_config.get(key) + mock_secrets = mock.MagicMock() + self.endpoint_from_flag.return_value = mock_secrets + self.is_flag_set.return_value = True + _vault.get_api_url.return_value = 'http://10.5.100.1:8200' + handlers.send_vault_url_and_ca() + self.endpoint_from_flag.assert_called_with('secrets.connected') + self.is_flag_set.assert_called_with('ha.available') + _vault.get_api_url.assert_called_once_with(address='10.5.100.1') + mock_secrets.publish_url.assert_called_once_with( + vault_url='http://10.5.100.1:8200' + ) + mock_secrets.publish_ca.assert_called_once_with( + vault_ca='test-ca' + )