openstack
/
charm-vault
Code Issues Proposed changes

Add support for new secrets:vault-kv interface 

Provide implementation for the provides endpoint of the vault-kv
typed interface.

This interface type will setup a KV secret backend and create an
approle and policy to allow remote services to access the backend
using the Vault API.

Backends may be shared between remote units, allowing any unit to
access any value in the backend, or may be isolated between units
based on path suffixing using the units hostname so that stored
secrets are not visible between units of a deployment.

Change-Id: Id8fa1cbe33feccc9c2f06a61db22453d7830730d
This commit is contained in:
James Page 2018-04-20 11:22:28 +01:00
parent 34714a92ae
commit 28fb89a44b
8 changed files with 355 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/actions/actions.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/local/sbin/charm-env python3
# Copyright 2018 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -27,6 +27,8 @@ import charmhelpers.core.hookenv as hookenv
import charm.vault as vault
import charms.reactive
def authorize_charm_action(*args):
"""Create a role allowing the charm to perform certain vault actions.
@ -55,6 +57,8 @@ def main(args):
action(args)
except Exception as e:
hookenv.action_fail(str(e))
else:
charms.reactive.main()
if __name__ == "__main__":

2
src/layer.yaml
View File

@ -1,11 +1,13 @@
includes:
- layer:basic
- layer:snap
- layer:leadership
- interface:nrpe-external-master
- interface:pgsql
- interface:mysql-shared
- interface:etcd
- interface:hacluster
- interface:vault-kv
options:
basic:
packages:

94
src/lib/charm/vault.py
View File

@ -1,3 +1,17 @@
# Copyright 2018 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import json
import requests
@ -50,6 +64,19 @@ path "sys/mounts/" {
}"""
VAULT_HEALTH_URL = '{vault_addr}/v1/sys/health'
VAULT_LOCALHOST_URL = "http://127.0.0.1:8220"
SECRET_BACKEND_HCL = """
path "{backend}/{hostname}/*" {{
capabilities = ["create", "read", "update", "delete", "list"]
}}
"""
SECRET_BACKEND_SHARED_HCL = """
path "{backend}/*" {{
capabilities = ["create", "read", "update", "delete", "list"]
}}
"""
def binding_address(binding):
@ -59,9 +86,9 @@ def binding_address(binding):
return hookenv.unit_private_ip()
def get_vault_url(binding, port):
def get_vault_url(binding, port, address=None):
protocol = 'http'
ip = binding_address(binding)
ip = address or binding_address(binding)
if charms.reactive.is_state('vault.ssl.available'):
protocol = 'https'
return '{}://{}:{}'.format(protocol, ip, port)
@ -110,9 +137,8 @@ def setup_charm_vault_access(token=None):
:rtype: str"""
if not token:
token = hookenv.leader_get('token')
vault_url = get_api_url()
client = hvac.Client(
url=vault_url,
url=VAULT_LOCALHOST_URL,
token=token)
enable_approle_auth(client)
policies = [CHARM_POLICY_NAME]
@ -129,13 +155,13 @@ def get_local_charm_access_role_id():
return hookenv.leader_get(CHARM_ACCESS_ROLE_ID)
def get_client():
def get_client(url=None):
"""Provide a client for talking to the vault api
:returns: vault client
:rtype: hvac.Client
"""
return hvac.Client(url=get_api_url())
return hvac.Client(url=url or get_api_url())
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=10),
@ -148,7 +174,7 @@ def get_vault_health():
:rtype: dict
"""
response = requests.get(
VAULT_HEALTH_URL.format(vault_addr=get_api_url()))
VAULT_HEALTH_URL.format(vault_addr=VAULT_LOCALHOST_URL))
return response.json()
@ -189,7 +215,7 @@ def initialize_vault(shares=1, threshold=1):
:param threshold: Minimum number of shares needed to unlock
:type threshold: int
"""
client = get_client()
client = get_client(url=VAULT_LOCALHOST_URL)
result = client.initialize(shares, threshold)
client.token = result['root_token']
hookenv.leader_set(
@ -200,7 +226,7 @@ def initialize_vault(shares=1, threshold=1):
def unseal_vault(keys=None):
"""Unseal vault with provided keys. If no keys are provided retrieve from
leader db"""
client = get_client()
client = get_client(url=VAULT_LOCALHOST_URL)
if not keys:
keys = json.loads(hookenv.leader_get()['keys'])
for key in keys:
@ -219,7 +245,7 @@ def can_restart():
elif hookenv.config('auto-unlock'):
safe_restart = True
else:
client = get_client()
client = get_client(url=VAULT_LOCALHOST_URL)
if not client.is_initialized():
safe_restart = True
elif client.is_sealed():
@ -228,3 +254,51 @@ def can_restart():
"Safe to restart: {}".format(safe_restart),
level=hookenv.DEBUG)
return safe_restart
def configure_secret_backend(client, name):
"""Ensure a KV backend is enabled
:param client: Vault client
:ptype client: hvac.Client
:param name: Name of backend to enable
:ptype name: str"""
if '{}/'.format(name) not in client.list_secret_backends():
client.enable_secret_backend(backend_type='kv',
description='Charm created KV backend',
mount_point=name)
def configure_policy(client, name, hcl):
"""Create/update a role within vault associating the supplied policies
:param client: Vault client
:ptype client: hvac.Client
:param name: Name of policy to create
:ptype name: str
:param hcl: Vault policy HCL
:ptype hcl: str"""
client.set_policy(name, hcl)
def configure_approle(client, name, cidr, policies):
"""Create/update a role within vault associating the supplied policies
:param client: Vault client
:ptype client: hvac.Client
:param name: Name of role
:ptype name: str
:param cidr: Network address of remote unit
:ptype cidr: str
:param policies: List of policy names
:ptype policies: [str, str, ...]
:returns: Id of created role
:rtype: str"""
client.create_role(
name,
token_ttl='60s',
token_max_ttl='60s',
policies=policies,
bind_secret_id='false',
bound_cidr_list=cidr)
return client.get_role_id(name)

2
src/metadata.yaml
View File

@ -32,6 +32,8 @@ provides:
nrpe-external-master:
interface: nrpe-external-master
scope: container
secrets:
interface: vault-kv
peers:
cluster:
interface: vault-ha

95
src/reactive/vault_handlers.py
View File

@ -1,6 +1,7 @@
import base64
import psycopg2
import subprocess
import tenacity
from charmhelpers.contrib.charmsupport.nrpe import (
@ -56,6 +57,7 @@ from charms.reactive.flags import (
)
from charms.layer import snap
import lib.charm.vault as vault
# See https://www.vaultproject.io/docs/configuration/storage/postgresql.html
@ -367,6 +369,99 @@ def file_change_auto_unlock_mode():
vault.prepare_vault()
@when('leadership.is_leader')
@when('endpoint.secrets.new-request')
def configure_secrets_backend():
""" Process requests for setup and access to simple kv secret backends """
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(10),
reraise=True)
def _check_vault_status(client):
if (not service_running('vault') or
not client.is_initialized() or
client.is_sealed()):
return False
return True
# NOTE: use localhost listener as policy only allows 127.0.0.1 to
# administer the local vault instances via the charm
client = vault.get_client(url=vault.VAULT_LOCALHOST_URL)
status_ok = _check_vault_status(client)
if not status_ok:
log('Unable to process new secret backend requests,'
' deferring until vault is fully configured', level=DEBUG)
return
charm_role_id = vault.get_local_charm_access_role_id()
if charm_role_id is None:
log('Charm access to vault not configured, deferring'
' secrets backend setup', level=DEBUG)
return
client.auth_approle(charm_role_id)
secrets = endpoint_from_flag('endpoint.secrets.new-request')
requests = secrets.requests()
# Configure KV secret backends
backends = set([request['secret_backend']
for request in requests])
for backend in backends:
if not backend.startswith('charm-'):
continue
vault.configure_secret_backend(client, name=backend)
# Configure AppRoles for application unit access
for request in requests:
# NOTE: backends must start with charm-
backend_name = request['secret_backend']
if not backend_name.startswith('charm-'):
continue
unit = request['unit']
hostname = request['hostname']
access_address = request['access_address']
isolated = request['isolated']
unit_name = unit.unit_name.replace('/', '-')
policy_name = approle_name = 'charm-{}'.format(unit_name)
if isolated:
policy_template = vault.SECRET_BACKEND_HCL
else:
policy_template = vault.SECRET_BACKEND_SHARED_HCL
vault.configure_policy(
client,
name=policy_name,
hcl=policy_template.format(backend=backend_name,
hostname=hostname)
)
approle_id = vault.configure_approle(
client,
name=approle_name,
cidr='{}/32'.format(access_address),
policies=[policy_name])
secrets.set_role_id(unit=unit,
role_id=approle_id)
clear_flag('endpoint.secrets.new-request')
@when('secrets.connected')
def send_vault_url_and_ca():
secrets = endpoint_from_flag('secrets.connected')
if is_flag_set('ha.available'):
vault_url = vault.get_api_url(address=config('vip'))
else:
vault_url = vault.get_api_url()
secrets.publish_url(vault_url=vault_url)
if config('ssl-ca'):
secrets.publish_ca(vault_ca=config('ssl-ca'))
@when('snap.installed.vault')
def prime_assess_status():
atexit(_assess_status)

8
src/templates/vault.hcl.j2
View File

@ -37,4 +37,10 @@ listener "tcp" {
{%- else %}
tls_disable = 1
{%- endif %}
}
}
# Localhost only listener for charm access to vault.
listener "tcp" {
address = "127.0.0.1:8220"
tls_disable = 1
}

45
unit_tests/test_lib_charm_vault.py
View File

@ -176,7 +176,7 @@ class TestLibCharmVault(unit_tests.test_utils.CharmTestCase):
self.assertEqual(vault.get_vault_health(),
self._health_response)
requests.get.assert_called_with(
"https://vault.demo.com:8200/v1/sys/health")
"http://127.0.0.1:8220/v1/sys/health")
mock_response.json.assert_called_once()
@patch.object(vault, 'setup_charm_vault_access')
@ -315,3 +315,46 @@ class TestLibCharmVault(unit_tests.test_utils.CharmTestCase):
can_restart.return_value = False
vault.opportunistic_restart()
service_start.assert_called_once_with('vault')
def test_configure_secret_backend(self):
hvac_client = mock.MagicMock()
hvac_client.list_secret_backends.return_value = ['secrets/']
vault.configure_secret_backend(hvac_client, 'test')
hvac_client.enable_secret_backend.assert_called_once_with(
backend_type='kv',
description=mock.ANY,
mount_point='test')
def test_configure_secret_backend_noop(self):
hvac_client = mock.MagicMock()
hvac_client.list_secret_backends.return_value = ['secrets/']
vault.configure_secret_backend(hvac_client, 'secrets')
hvac_client.enable_secret_backend.assert_not_called()
def test_configure_policy(self):
hvac_client = mock.MagicMock()
vault.configure_policy(hvac_client, 'test-policy', 'test-hcl')
hvac_client.set_policy.assert_called_once_with(
'test-policy',
'test-hcl',
)
def test_configure_approle(self):
hvac_client = mock.MagicMock()
hvac_client.get_role_id.return_value = 'some-UUID'
self.assertEqual(
vault.configure_approle(hvac_client,
'test-role',
'10.5.0.20/32',
['test-policy']),
'some-UUID'
)
hvac_client.create_role.assert_called_once_with(
'test-role',
token_ttl='60s',
token_max_ttl='60s',
policies=['test-policy'],
bind_secret_id='false',
bound_cidr_list='10.5.0.20/32'
)
hvac_client.get_role_id.assert_called_with('test-role')

116
unit_tests/test_reactive_vault_handlers.py
View File

@ -135,6 +135,7 @@ class TestHandlers(unit_tests.test_utils.CharmTestCase):
perms=0o644)
]
self.render.assert_has_calls(render_calls)
self.service.assert_called_with('enable', 'vault')
@patch.object(handlers, 'configure_vault')
def test_configure_vault_psql(self, configure_vault):
@ -467,3 +468,118 @@ class TestHandlers(unit_tests.test_utils.CharmTestCase):
self.assertFalse(hacluster_mock.add_dnsha.called)
self.assertFalse(hacluster_mock.bind_resources.called)
self.set_flag.assert_called_once_with('config.dns_vip.invalid')
def fixture_test_requests(self):
test_requests = []
test_requests.append({
'secret_backend': 'charm-vaultlocker',
'hostname': 'juju-123456-0',
'isolated': True,
'access_address': '10.20.4.5',
'unit': mock.MagicMock()
})
test_requests[-1]['unit'].unit_name = 'ceph-osd/0'
test_requests.append({
'secret_backend': 'charm-supersecrets',
'hostname': 'juju-789012-0',
'isolated': True,
'access_address': '10.20.4.20',
'unit': mock.MagicMock()
})
test_requests[-1]['unit'].unit_name = 'omg/0'
return test_requests
@mock.patch.object(handlers, 'vault')
def test_configure_secrets_backend(self, _vault):
hvac_client = mock.MagicMock()
_vault.get_client.return_value = hvac_client
# Vault is up and running, init'ed and unsealed
hvac_client.is_initialized.return_value = True
hvac_client.is_sealed.return_value = False
self.service_running.return_value = True
_vault.get_local_charm_access_role_id.return_value = 'local-approle'
secrets_interface = mock.MagicMock()
self.endpoint_from_flag.return_value = secrets_interface
secrets_interface.requests.return_value = self.fixture_test_requests()
_vault.configure_approle.side_effect = ['role_a', 'role_b']
self.is_flag_set.return_value = False
_vault.get_api_url.return_value = "http://vault:8200"
handlers.configure_secrets_backend()
hvac_client.auth_approle.assert_called_once_with('local-approle')
_vault.configure_secret_backend.assert_has_calls([
mock.call(hvac_client, name='charm-vaultlocker'),
mock.call(hvac_client, name='charm-supersecrets')
])
_vault.configure_policy.assert_has_calls([
mock.call(hvac_client, name='charm-ceph-osd-0', hcl=mock.ANY),
mock.call(hvac_client, name='charm-omg-0', hcl=mock.ANY)
])
_vault.configure_approle.assert_has_calls([
mock.call(hvac_client, name='charm-ceph-osd-0',
cidr="10.20.4.5/32",
policies=mock.ANY),
mock.call(hvac_client, name='charm-omg-0',
cidr="10.20.4.20/32",
policies=mock.ANY)
])
secrets_interface.set_role_id.assert_has_calls([
mock.call(unit=mock.ANY,
role_id='role_a'),
mock.call(unit=mock.ANY,
role_id='role_b'),
])
self.clear_flag.assert_called_once_with('endpoint.secrets.new-request')
@mock.patch.object(handlers, 'vault')
def send_vault_url_and_ca(self, _vault):
_test_config = {
'vip': '10.5.100.1',
'ssl-ca': 'test-ca',
}
self.config.side_effect = lambda key: _test_config.get(key)
mock_secrets = mock.MagicMock()
self.endpoint_from_flag.return_value = mock_secrets
self.is_flag_set.return_value = False
_vault.get_api_url.return_value = 'http://10.5.0.23:8200'
handlers.send_vault_url_and_ca()
self.endpoint_from_flag.assert_called_with('secrets.connected')
self.is_flag_set.assert_called_with('ha.available')
_vault.get_api_url.assert_called_once_with()
mock_secrets.publish_url.assert_called_once_with(
vault_url='http://10.5.0.23:8200'
)
mock_secrets.publish_ca.assert_called_once_with(
vault_ca='test-ca'
)
@mock.patch.object(handlers, 'vault')
def send_vault_url_and_ca_ha(self, _vault):
_test_config = {
'vip': '10.5.100.1',
'ssl-ca': 'test-ca',
}
self.config.side_effect = lambda key: _test_config.get(key)
mock_secrets = mock.MagicMock()
self.endpoint_from_flag.return_value = mock_secrets
self.is_flag_set.return_value = True
_vault.get_api_url.return_value = 'http://10.5.100.1:8200'
handlers.send_vault_url_and_ca()
self.endpoint_from_flag.assert_called_with('secrets.connected')
self.is_flag_set.assert_called_with('ha.available')
_vault.get_api_url.assert_called_once_with(address='10.5.100.1')
mock_secrets.publish_url.assert_called_once_with(
vault_url='http://10.5.100.1:8200'
)
mock_secrets.publish_ca.assert_called_once_with(
vault_ca='test-ca'
)