x
/
omni
Code Issues Proposed changes

Added credentials manager and updated omni drivers. 

This change:
1. Adds credmanager service which handles credentials for AWS drivers.
2. Adds support for managing multiple AWS accounts through use of credmanager. Each account is mapped to a single project in keystone.
3. Adds support for multiple AZs by running one nova-compute and cinder-volume process per AZ.
4. Improves support for AWS networking in neutron.
5. Also, made few stability fixes in GCP and Azure drivers.

Change-Id: I0f87005a924423397db659ab754caaa6cde90274
This commit is contained in:
Harsha Dhake 2019-02-21 02:30:24 -05:00
parent 14f465f99d
commit 3fc8b3f97d
126 changed files with 8679 additions and 1274 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -1,8 +1,11 @@
*.pyc
*.py[cod]
*~
*venv
.idea
*.egg*
.tox
*.log
build/*
creds_manager/build/*
.testrepository
openstack
*.log

111
cinder/tests/unit/volume/drivers/test_ec2.py
View File

@ -17,52 +17,48 @@ from cinder.exception import ImageNotFound
from cinder.exception import NotFound
from cinder.exception import VolumeNotFound
from cinder import test
from cinder.tests.unit.fake_snapshot import fake_snapshot_obj
from cinder.tests.unit.fake_volume import fake_volume_obj
from cinder.volume.drivers.aws import ebs
from cinder.volume.drivers.aws.exception import AvailabilityZoneNotFound
import mock
from moto import mock_ec2_deprecated
from moto import mock_ec2
from oslo_service import loopingcall
def fake_get_credentials(*args, **kwargs):
return {
'aws_access_key_id': 'fake_access_key_id',
'aws_secret_access_key': 'fake_access_key'
}
class EBSVolumeTestCase(test.TestCase):
@mock_ec2_deprecated
@mock_ec2
def setUp(self):
super(EBSVolumeTestCase, self).setUp()
self.mock_get_credentials = mock.patch(
'cinder.volume.drivers.aws.ebs.get_credentials'
).start()
self.mock_get_credentials.side_effect = fake_get_credentials
ebs.CONF.AWS.region_name = 'us-east-1'
ebs.CONF.AWS.access_key = 'fake-key'
ebs.CONF.AWS.secret_key = 'fake-secret'
ebs.CONF.AWS.az = 'us-east-1a'
self._driver = ebs.EBSDriver()
self.ctxt = context.get_admin_context()
self._driver.do_setup(self.ctxt)
def _stub_volume(self, **kwargs):
uuid = u'c20aba21-6ef6-446b-b374-45733b4883ba'
name = u'volume-00000001'
size = 1
created_at = '2016-10-19 23:22:33'
volume = dict()
volume['id'] = kwargs.get('id', uuid)
volume['display_name'] = kwargs.get('display_name', name)
volume['size'] = kwargs.get('size', size)
volume['provider_location'] = kwargs.get('provider_location', None)
volume['volume_type_id'] = kwargs.get('volume_type_id', None)
volume['project_id'] = kwargs.get('project_id', 'aws_proj_700')
volume['created_at'] = kwargs.get('create_at', created_at)
return volume
kwargs.setdefault('display_name', 'fake_name')
kwargs.setdefault('project_id', 'fake_project_id')
kwargs.setdefault('created_at', '2016-10-19 23:22:33')
return fake_volume_obj(self.ctxt, **kwargs)
def _stub_snapshot(self, **kwargs):
uuid = u'0196f961-c294-4a2a-923e-01ef5e30c2c9'
created_at = '2016-10-19 23:22:33'
ss = dict()
ss['id'] = kwargs.get('id', uuid)
ss['project_id'] = kwargs.get('project_id', 'aws_proj_700')
ss['created_at'] = kwargs.get('create_at', created_at)
ss['volume'] = kwargs.get('volume', self._stub_volume())
ss['display_name'] = kwargs.get('display_name', 'snapshot_007')
return ss
volume = self._stub_volume()
kwargs.setdefault('volume_id', volume.id)
kwargs.setdefault('display_name', 'fake_name')
kwargs.setdefault('project_id', 'fake_project_id')
kwargs.setdefault('created_at', '2016-10-19 23:22:33')
return fake_snapshot_obj(self.ctxt, **kwargs)
def _fake_image_meta(self):
image_meta = dict()
@ -76,18 +72,11 @@ class EBSVolumeTestCase(test.TestCase):
image_meta['properties']['aws_image_id'] = 'ami-00001'
return image_meta
@mock_ec2_deprecated
def test_availability_zone_config(self):
ebs.CONF.AWS.az = 'hgkjhgkd'
driver = ebs.EBSDriver()
self.assertRaises(AvailabilityZoneNotFound, driver.do_setup, self.ctxt)
ebs.CONF.AWS.az = 'us-east-1a'
@mock_ec2_deprecated
@mock_ec2
def test_volume_create_success(self):
self.assertIsNone(self._driver.create_volume(self._stub_volume()))
@mock_ec2_deprecated
@mock_ec2
@mock.patch('cinder.volume.drivers.aws.ebs.EBSDriver._wait_for_create')
def test_volume_create_fails(self, mock_wait):
def wait(*args):
@ -101,49 +90,34 @@ class EBSVolumeTestCase(test.TestCase):
self.assertRaises(APITimeout, self._driver.create_volume,
self._stub_volume())
@mock_ec2_deprecated
@mock_ec2
def test_volume_deletion(self):
vol = self._stub_volume()
self._driver.create_volume(vol)
self.assertIsNone(self._driver.delete_volume(vol))
@mock_ec2_deprecated
@mock_ec2
@mock.patch('cinder.volume.drivers.aws.ebs.EBSDriver._find')
def test_volume_deletion_not_found(self, mock_find):
vol = self._stub_volume()
mock_find.side_effect = NotFound
self.assertIsNone(self._driver.delete_volume(vol))
@mock_ec2_deprecated
@mock_ec2
def test_snapshot(self):
vol = self._stub_volume()
snapshot = self._stub_snapshot()
self._driver.create_volume(vol)
self.assertIsNone(self._driver.create_snapshot(snapshot))
@mock_ec2_deprecated
@mock_ec2
@mock.patch('cinder.volume.drivers.aws.ebs.EBSDriver._find')
def test_snapshot_volume_not_found(self, mock_find):
mock_find.side_effect = NotFound
ss = self._stub_snapshot()
self.assertRaises(VolumeNotFound, self._driver.create_snapshot, ss)
@mock_ec2_deprecated
@mock.patch('cinder.volume.drivers.aws.ebs.EBSDriver._wait_for_snapshot')
def test_snapshot_create_fails(self, mock_wait):
def wait(*args):
def _wait():
raise loopingcall.LoopingCallDone(False)
timer = loopingcall.FixedIntervalLoopingCall(_wait)
return timer.start(interval=1).wait()
mock_wait.side_effect = wait
ss = self._stub_snapshot()
self._driver.create_volume(ss['volume'])
self.assertRaises(APITimeout, self._driver.create_snapshot, ss)
@mock_ec2_deprecated
@mock_ec2
def test_volume_from_snapshot(self):
snapshot = self._stub_snapshot()
volume = self._stub_volume()
@ -152,7 +126,7 @@ class EBSVolumeTestCase(test.TestCase):
self.assertIsNone(
self._driver.create_volume_from_snapshot(volume, snapshot))
@mock_ec2_deprecated
@mock_ec2
def test_volume_from_non_existing_snapshot(self):
self.assertRaises(NotFound, self._driver.create_volume_from_snapshot,
self._stub_volume(), self._stub_snapshot())
@ -163,27 +137,24 @@ class EBSVolumeTestCase(test.TestCase):
self.assertRaises(ImageNotFound, self._driver.clone_image,
self.ctxt, volume, '', image_meta, '')
@mock_ec2_deprecated
@mock_ec2
@mock.patch('cinder.volume.drivers.aws.ebs.EBSDriver._get_snapshot_id')
def test_clone_image(self, mock_get):
snapshot = self._stub_snapshot()
image_meta = self._fake_image_meta()
volume = fake_volume_obj(self.ctxt)
volume.id = 'd30aba21-6ef6-446b-b374-45733b4883ba'
volume.display_name = 'volume-00000001'
volume.project_id = 'fake_project_id'
volume.created_at = '2016-10-19 23:22:33'
self._driver.create_volume(snapshot['volume'])
volume = self._stub_volume()
snapshot = self._stub_snapshot()
self._driver.create_volume(volume)
self._driver.create_snapshot(snapshot)
ebs_snap = self._driver._find(snapshot['id'],
self._driver._conn.get_all_snapshots)
mock_get.return_value = ebs_snap.id
ec2_conn = self._driver._ec2_client(snapshot.obj_context)
ebs_snap = self._driver._find(
snapshot['id'], ec2_conn.describe_snapshots, is_snapshot=True)
mock_get.return_value = ebs_snap['SnapshotId']
metadata, cloned = self._driver.clone_image(self.ctxt, volume, '',
image_meta, '')
self.assertEqual(True, cloned)
self.assertTrue(isinstance(metadata, dict))
@mock_ec2_deprecated
@mock_ec2
def test_create_cloned_volume(self):
src_volume = fake_volume_obj(self.ctxt)
src_volume.display_name = 'volume-00000001'

46
cinder/volume/drivers/aws/config.py Normal file
View File

@ -0,0 +1,46 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from oslo_config import cfg
aws_group = cfg.OptGroup(name='AWS',
title='Options to connect to an AWS environment')
aws_opts = [
cfg.StrOpt('secret_key', help='Secret key of AWS account', secret=True),
cfg.StrOpt('access_key', help='Access key of AWS account', secret=True),
cfg.StrOpt('region_name', help='AWS region'),
cfg.StrOpt('az', help='AWS availability zone'),
cfg.IntOpt('wait_time_min', help='Maximum wait time for AWS operations',
default=5),
cfg.BoolOpt('use_credsmgr', help='Should credentials manager be used',
default=True)
]
ebs_opts = [
cfg.StrOpt('ebs_pool_name', help='Storage pool name'),
cfg.IntOpt('ebs_free_capacity_gb',
help='Free space available on EBS storage pool', default=1024),
cfg.IntOpt('ebs_total_capacity_gb',
help='Total space available on EBS storage pool', default=1024)
]
cinder_opts = [
cfg.StrOpt('os_region_name',
help='Region name of this node'),
]
CONF = cfg.CONF
CONF.register_group(aws_group)
CONF.register_opts(aws_opts, group=aws_group)
CONF.register_opts(ebs_opts)
CONF.register_opts(cinder_opts)

59
cinder/volume/drivers/aws/credshelper.py Normal file
View File

@ -0,0 +1,59 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from keystoneauth1.access import service_catalog
from keystoneauth1.exceptions import EndpointNotFound
from credsmgrclient.client import Client
from credsmgrclient.common import exceptions
from cinder.volume.drivers.aws.config import CONF
from cinder.volume.drivers.aws.exception import AwsCredentialsNotFound
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
def get_credentials_from_conf(CONF):
secret_key = CONF.AWS.secret_key
access_key = CONF.AWS.access_key
if not access_key or not secret_key:
raise AwsCredentialsNotFound()
return dict(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key
)
def get_credentials(context, project_id=None):
# TODO(ssudake21): Add caching support
# 1. Cache keystone endpoint
# 2. Cache recently used AWS credentials
try:
sc = service_catalog.ServiceCatalogV2(context.service_catalog)
credsmgr_endpoint = sc.url_for(
service_type='credsmgr', region_name=CONF.os_region_name)
token = context.auth_token
credsmgr_client = Client(credsmgr_endpoint, token=token)
if not project_id:
project_id = context.project_id
resp, body = credsmgr_client.credentials.credentials_get(
'aws', project_id)
except (EndpointNotFound, exceptions.HTTPBadGateway):
return get_credentials_from_conf(CONF)
except exceptions.HTTPNotFound:
if not CONF.AWS.use_credsmgr:
return get_credentials_from_conf(CONF)
raise
return body

286
cinder/volume/drivers/aws/ebs.py
View File

@ -13,9 +13,9 @@ limitations under the License.
import time
from boto import ec2
from boto.exception import EC2ResponseError
from boto.regioninfo import RegionInfo
import boto3
from botocore.exceptions import ClientError
from cinder.exception import APITimeout
from cinder.exception import ImageNotFound
from cinder.exception import InvalidConfigurationValue
@ -23,34 +23,13 @@ from cinder.exception import NotFound
from cinder.exception import VolumeBackendAPIException
from cinder.exception import VolumeNotFound
from cinder.volume.driver import BaseVD
from cinder.volume.drivers.aws.exception import AvailabilityZoneNotFound
from oslo_config import cfg
from cinder.volume.drivers.aws.config import CONF
from cinder.volume.drivers.aws.credshelper import get_credentials
from oslo_log import log as logging
from oslo_service import loopingcall
aws_group = cfg.OptGroup(name='AWS',
title='Options to connect to an AWS environment')
aws_opts = [
cfg.StrOpt('secret_key', help='Secret key of AWS account', secret=True),
cfg.StrOpt('access_key', help='Access key of AWS account', secret=True),
cfg.StrOpt('region_name', help='AWS region'),
cfg.StrOpt('az', help='AWS availability zone'),
cfg.IntOpt('wait_time_min', help='Maximum wait time for AWS operations',
default=5)
]
ebs_opts = [
cfg.StrOpt('ebs_pool_name', help='Storage pool name'),
cfg.IntOpt('ebs_free_capacity_gb',
help='Free space available on EBS storage pool', default=1024),
cfg.IntOpt('ebs_total_capacity_gb',
help='Total space available on EBS storage pool', default=1024)
]
CONF = cfg.CONF
CONF.register_group(aws_group)
CONF.register_opts(aws_opts, group=aws_group)
CONF.register_opts(ebs_opts)
LOG = logging.getLogger(__name__)
@ -61,107 +40,107 @@ class EBSDriver(BaseVD):
self.VERSION = '1.0.0'
self._wait_time_sec = 60 * (CONF.AWS.wait_time_min)
def do_setup(self, context):
self._check_config()
self.az = CONF.AWS.az
self.set_initialized()
def _check_config(self):
tbl = dict([(n, eval(n)) for n in ['CONF.AWS.access_key',
'CONF.AWS.secret_key',
'CONF.AWS.region_name',
tbl = dict([(n, eval(n)) for n in ['CONF.AWS.region_name',
'CONF.AWS.az']])
for k, v in tbl.iteritems():
if v is None:
raise InvalidConfigurationValue(value=None, option=k)
def do_setup(self, context):
self._check_config()
region_name = CONF.AWS.region_name
endpoint = '.'.join(['ec2', region_name, 'amazonaws.com'])
region = RegionInfo(name=region_name, endpoint=endpoint)
self._conn = ec2.EC2Connection(
aws_access_key_id=CONF.AWS.access_key,
aws_secret_access_key=CONF.AWS.secret_key,
region=region)
# resort to first AZ for now. TODO(do_setup): expose this through API
az = CONF.AWS.az
def _ec2_client(self, context, project_id=None):
creds = get_credentials(context, project_id=project_id)
return boto3.client(
"ec2", region_name=CONF.AWS.region_name,
aws_access_key_id=creds['aws_access_key_id'],
aws_secret_access_key=creds['aws_secret_access_key'],)
try:
self._zone = filter(lambda z: z.name == az,
self._conn.get_all_zones())[0]
except IndexError:
raise AvailabilityZoneNotFound(az=az)
self.set_initialized()
def _wait_for_create(self, id, final_state):
def _wait_for_create(self, ec2_conn, ec2_id, final_state,
is_snapshot=False):
def _wait_for_status(start_time):
current_time = time.time()
if current_time - start_time > self._wait_time_sec:
raise loopingcall.LoopingCallDone(False)
obj = self._conn.get_all_volumes([id])[0]
if obj.status == final_state:
raise loopingcall.LoopingCallDone(True)
try:
if is_snapshot:
resp = ec2_conn.describe_snapshots(SnapshotIds=[ec2_id])
obj = resp['Snapshots'][0]
else:
resp = ec2_conn.describe_volumes(VolumeIds=[ec2_id])
obj = resp['Volumes'][0]
if obj['State'] == final_state:
raise loopingcall.LoopingCallDone(True)
except ClientError as e:
LOG.warn(e.message)
timer = loopingcall.FixedIntervalLoopingCall(_wait_for_status,
time.time())
return timer.start(interval=5).wait()
return timer.start(interval=10).wait()
def _wait_for_snapshot(self, id, final_state):
def _wait_for_status(start_time):
if time.time() - start_time > self._wait_time_sec:
raise loopingcall.LoopingCallDone(False)
obj = self._conn.get_all_snapshots([id])[0]
if obj.status == final_state:
raise loopingcall.LoopingCallDone(True)
timer = loopingcall.FixedIntervalLoopingCall(_wait_for_status,
time.time())
return timer.start(interval=5).wait()
def _wait_for_tags_creation(self, id, volume):
def _wait_for_tags_creation(self, ec2_conn, ec2_id, ostack_obj,
is_clone=False, is_snapshot=False):
def _wait_for_completion(start_time):
if time.time() - start_time > self._wait_time_sec:
raise loopingcall.LoopingCallDone(False)
self._conn.create_tags([id],
{'project_id': volume['project_id'],
'uuid': volume['id'],
'is_clone': True,
'created_at': volume['created_at'],
'Name': volume['display_name']})
obj = self._conn.get_all_volumes([id])[0]
if obj.tags:
tags = [
{'Key': 'project_id', 'Value': ostack_obj['project_id']},
{'Key': 'uuid', 'Value': ostack_obj['id']},
{'Key': 'is_clone', 'Value': str(is_clone)},
{'Key': 'created_at', 'Value': str(ostack_obj['created_at'])},
{'Key': 'Name', 'Value': ostack_obj['display_name']},
]
ec2_conn.create_tags(Resources=[ec2_id], Tags=tags)
if is_snapshot:
resp = ec2_conn.describe_snapshots(SnapshotIds=[ec2_id])
obj = resp['Snapshots'][0]
else:
resp = ec2_conn.describe_volumes(VolumeIds=[ec2_id])
obj = resp['Volumes'][0]
if 'Tags' in obj and obj['Tags']:
raise loopingcall.LoopingCallDone(True)
timer = loopingcall.FixedIntervalLoopingCall(_wait_for_completion,
time.time())
return timer.start(interval=5).wait()
return timer.start(interval=10).wait()
def create_volume(self, volume):
size = volume['size']
ebs_vol = self._conn.create_volume(size, self._zone)
if self._wait_for_create(ebs_vol.id, 'available') is False:
ec2_conn = self._ec2_client(
volume.obj_context, project_id=volume.project_id)
ebs_vol = ec2_conn.create_volume(Size=size, AvailabilityZone=self.az)
vol_id = ebs_vol['VolumeId']
if not self._wait_for_create(ec2_conn, vol_id, 'available'):
raise APITimeout(service='EC2')
if not self._wait_for_tags_creation(ec2_conn, vol_id, volume):
raise APITimeout(service='EC2')
self._conn.create_tags([ebs_vol.id],
{'project_id': volume['project_id'],
'uuid': volume['id'],
'is_clone': False,
'created_at': volume['created_at'],
'Name': volume['display_name']})
def _find(self, obj_id, find_func):
ebs_objs = find_func(filters={'tag:uuid': obj_id})
if len(ebs_objs) == 0:
raise NotFound()
ebs_obj = ebs_objs[0]
return ebs_obj
def delete_volume(self, volume):
ec2_conn = self._ec2_client(
volume.obj_context, project_id=volume.project_id)
try:
ebs_vol = self._find(volume['id'], self._conn.get_all_volumes)
ebs_vol = self._find(volume['id'], ec2_conn.describe_volumes)
except NotFound:
LOG.error('Volume %s was not found' % volume['id'])
return
self._conn.delete_volume(ebs_vol.id)
ec2_conn.delete_volume(VolumeId=ebs_vol['VolumeId'])
def _find(self, obj_id, find_func, is_snapshot=False):
ebs_objs = find_func(Filters=[{'Name': 'tag:uuid',
'Values': [obj_id]}])
if is_snapshot:
if len(ebs_objs['Snapshots']) == 0:
raise NotFound()
ebs_obj = ebs_objs['Snapshots'][0]
else:
if len(ebs_objs['Volumes']) == 0:
raise NotFound()
ebs_obj = ebs_objs['Volumes'][0]
return ebs_obj
def check_for_setup_error(self):
# TODO(check_setup_error) throw errors if AWS config is broken
@ -177,11 +156,13 @@ class EBSDriver(BaseVD):
pass
def initialize_connection(self, volume, connector, initiator_data=None):
ec2_conn = self._ec2_client(
volume.obj_context, project_id=volume.project_id)
try:
ebs_vol = self._find(volume.id, self._conn.get_all_volumes)
ebs_vol = self._find(volume.id, ec2_conn.describe_volumes)
except NotFound:
raise VolumeNotFound(volume_id=volume.id)
conn_info = dict(data=dict(volume_id=ebs_vol.id))
conn_info = dict(data=dict(volume_id=ebs_vol['VolumeId']))
return conn_info
def terminate_connection(self, volume, connector, **kwargs):
@ -213,63 +194,73 @@ class EBSDriver(BaseVD):
return self._stats
def create_snapshot(self, snapshot):
os_vol = snapshot['volume']
vol_id = snapshot['volume_id']
ec2_conn = self._ec2_client(
snapshot.obj_context, project_id=snapshot.project_id)
try:
ebs_vol = self._find(os_vol['id'], self._conn.get_all_volumes)
ebs_vol = self._find(vol_id, ec2_conn.describe_volumes)
except NotFound:
raise VolumeNotFound(os_vol['id'])
raise VolumeNotFound(volume_id=vol_id)
ebs_snap = self._conn.create_snapshot(ebs_vol.id)
if self._wait_for_snapshot(ebs_snap.id, 'completed') is False:
ebs_snap = ec2_conn.create_snapshot(VolumeId=ebs_vol['VolumeId'])
if not self._wait_for_create(ec2_conn, ebs_snap['SnapshotId'],
'completed', is_snapshot=True):
raise APITimeout(service='EC2')
if not self._wait_for_tags_creation(ec2_conn, ebs_snap['SnapshotId'],
snapshot, True, True):
raise APITimeout(service='EC2')
self._conn.create_tags([ebs_snap.id],
{'project_id': snapshot['project_id'],
'uuid': snapshot['id'],
'is_clone': True,
'created_at': snapshot['created_at'],
'Name': snapshot['display_name']})
def delete_snapshot(self, snapshot):
ec2_conn = self._ec2_client(
snapshot.obj_context, project_id=snapshot.project_id)
try:
ebs_ss = self._find(snapshot['id'], self._conn.get_all_snapshots)
ebs_ss = self._find(snapshot['id'], ec2_conn.describe_snapshots,
is_snapshot=True)
except NotFound:
LOG.error('Snapshot %s was not found' % snapshot['id'])
return
self._conn.delete_snapshot(ebs_ss.id)
ec2_conn.delete_snapshot(SnapshotId=ebs_ss['SnapshotId'])
def create_volume_from_snapshot(self, volume, snapshot):
ec2_conn = self._ec2_client(
volume.obj_context, project_id=volume.project_id)
try:
ebs_ss = self._find(snapshot['id'], self._conn.get_all_snapshots)
ebs_ss = self._find(snapshot['id'], ec2_conn.describe_snapshots,
is_snapshot=True)
except NotFound:
LOG.error('Snapshot %s was not found' % snapshot['id'])
raise
ebs_vol = ebs_ss.create_volume(self._zone)
ebs_vol = ec2_conn.create_volume(AvailabilityZone=self.az,
SnapshotId=ebs_ss['SnapshotId'])
vol_id = ebs_vol['VolumeId']
if self._wait_for_create(ebs_vol.id, 'available') is False:
if not self._wait_for_create(ec2_conn, vol_id, 'available'):
raise APITimeout(service='EC2')
if not self._wait_for_tags_creation(ec2_conn, vol_id, volume):
raise APITimeout(service='EC2')
self._conn.create_tags([ebs_vol.id],
{'project_id': volume['project_id'],
'uuid': volume['id'],
'is_clone': False,
'created_at': volume['created_at'],
'Name': volume['display_name']})
def create_cloned_volume(self, volume, srcvol_ref):
ebs_snap = None
ebs_vol = None
ec2_conn = self._ec2_client(
volume.obj_context, project_id=volume.project_id)
try:
src_vol = self._find(srcvol_ref['id'], self._conn.get_all_volumes)
ebs_snap = self._conn.create_snapshot(src_vol.id)
src_vol = self._find(srcvol_ref['id'], ec2_conn.describe_volumes)
ebs_snap = ec2_conn.create_snapshot(VolumeId=src_vol['VolumeId'])
if self._wait_for_snapshot(ebs_snap.id, 'completed') is False:
if not self._wait_for_create(ec2_conn, ebs_snap['SnapshotId'],
'completed', is_snapshot=True):
raise APITimeout(service='EC2')
ebs_vol = self._conn.create_volume(
size=volume.size, zone=self._zone, snapshot=ebs_snap.id)
if self._wait_for_create(ebs_vol.id, 'available') is False:
ebs_vol = ec2_conn.create_volume(
Size=volume.size, AvailabilityZone=self.az,
SnapshotId=ebs_snap['SnapshotId'])
vol_id = ebs_vol['VolumeId']
if not self._wait_for_create(ec2_conn, vol_id, 'available'):
raise APITimeout(service='EC2')
if self._wait_for_tags_creation(ebs_vol.id, volume) is False:
if not self._wait_for_tags_creation(ec2_conn, vol_id, volume,
True):
raise APITimeout(service='EC2')
except NotFound:
raise VolumeNotFound(srcvol_ref['id'])
@ -277,36 +268,43 @@ class EBSDriver(BaseVD):
message = "create_cloned_volume failed! volume: {0}, reason: {1}"
LOG.error(message.format(volume.id, ex))
if ebs_vol:
self._conn.delete_volume(ebs_vol.id)
ec2_conn.delete_volume(VolumeId=ebs_vol['VolumeId'])
raise VolumeBackendAPIException(data=message.format(volume.id, ex))
finally:
if ebs_snap:
self._conn.delete_snapshot(ebs_snap.id)
ec2_conn.delete_snapshot(SnapshotId=ebs_snap['SnapshotId'])
def clone_image(self, context, volume, image_location, image_meta,
image_service):
ec2_conn = self._ec2_client(context, project_id=volume.project_id)
image_id = image_meta['properties']['aws_image_id']
snapshot_id = self._get_snapshot_id(image_id)
ebs_vol = self._conn.create_volume(size=volume.size, zone=self._zone,
snapshot=snapshot_id)
if self._wait_for_create(ebs_vol.id, 'available') is False:
snapshot_id = self._get_snapshot_id(ec2_conn, image_id)
ebs_vol = ec2_conn.create_volume(
Size=volume.size, AvailabilityZone=self.az,
SnapshotId=snapshot_id)
vol_id = ebs_vol['VolumeId']
if not self._wait_for_create(ec2_conn, vol_id, 'available'):
raise APITimeout(service='EC2')
if self._wait_for_tags_creation(ebs_vol.id, volume) is False:
if not self._wait_for_tags_creation(ec2_conn, vol_id, volume, True):
raise APITimeout(service='EC2')
metadata = volume['metadata']
metadata['new_volume_id'] = ebs_vol.id
metadata['new_volume_id'] = vol_id
return dict(metadata=metadata), True
def _get_snapshot_id(self, image_id):
def _get_snapshot_id(self, ec2_conn, image_id):
try:
response = self._conn.get_all_images(image_ids=[image_id])[0]
snapshot_id = response.block_device_mapping[
'/dev/sda1'].snapshot_id
resp = ec2_conn.describe_images(ImageIds=[image_id])
ec2_image = resp['Images'][0]
snapshot_id = None
for bdm in ec2_image['BlockDeviceMappings']:
if bdm['DeviceName'] == '/dev/sda1':
snapshot_id = bdm['Ebs']['SnapshotId']
break
return snapshot_id
except EC2ResponseError:
message = "Getting image {0} failed.".format(image_id)
LOG.error(message)
raise ImageNotFound(message)
except ClientError as e:
message = "Getting image {0} failed. Error: {1}"
LOG.error(message.format(image_id, e.message))
raise ImageNotFound(message.format(image_id, e.message))
def copy_image_to_volume(self, context, volume, image_service, image_id):
"""Nothing need to do here since we create volume from image in

4
cinder/volume/drivers/aws/exception.py
View File

@ -18,3 +18,7 @@ from cinder.i18n import _
class AvailabilityZoneNotFound(CinderException):
message = _("Availability Zone %(az)s was not found")
class AwsCredentialsNotFound(CinderException):
message = _("Aws credentials could not be found")

8
creds_manager/.testr.conf Normal file
View File

@ -0,0 +1,8 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-0} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-0} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./credsmgr/tests/unit} $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

7
creds_manager/README.md Normal file
View File

@ -0,0 +1,7 @@
Credsmgr is a credential manager for Openstack Omni.
## Setup
## Status
In development. Can be used for individual testing
## Contributions
Contributions are welcome.

16
creds_manager/credsmgr/__init__.py Normal file
View File

@ -0,0 +1,16 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Make sure eventlet is loaded
import eventlet # noqa

0
creds_manager/credsmgr/api/__init__.py Normal file
View File

25
creds_manager/credsmgr/api/app.py Normal file
View File

@ -0,0 +1,25 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
import paste.urlmap
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def root_app_factory(loader, global_conf, **local_conf):
return paste.urlmap.urlmap_factory(loader, global_conf, **local_conf)

0
creds_manager/credsmgr/api/controllers/__init__.py Normal file
View File

153
creds_manager/credsmgr/api/controllers/api_version_request.py Normal file
View File

@ -0,0 +1,153 @@
# Copyright 2014 IBM Corp.
# Copyright 2015 Clinton Knight
# Copyright 2017 Platform9 Systems
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
from credsmgr.api.controllers import versioned_method
from credsmgr import exception
from credsmgr import utils
# Define the minimum and maximum version of the API across all of the
# REST API. The format of the version is:
# X.Y where:
#
# - X will only be changed if a significant backwards incompatible API
# change is made which affects the API as whole. That is, something
# that is only very very rarely incremented.
#
# - Y when you make any change to the API. Note that this includes
# semantic changes which may not affect the input or output formats or
# even originate in the API code layer. We are not distinguishing
# between backwards compatible and backwards incompatible changes in
# the versioning system. It must be made clear in the documentation as
# to what is a backwards compatible change and what is a backwards
# incompatible one.
# The minimum and maximum versions of the API supported
# The default api version request is defined to be the
# minimum version of the API supported.
# Explicitly using /v1 or /v2 enpoints will still work
_MIN_API_VERSION = "1.0"
_MAX_API_VERSION = "1.0"
# NOTE(cyeoh): min and max versions declared as functions so we can
# mock them for unittests. Do not use the constants directly anywhere
# else.
def min_api_version():
return APIVersionRequest(_MIN_API_VERSION)
def max_api_version():
return APIVersionRequest(_MAX_API_VERSION)
class APIVersionRequest(utils.ComparableMixin):
"""This class represents an API Version Request.
This class includes convenience methods for manipulation
and comparison of version numbers as needed to implement
API microversions.
"""
def __init__(self, version_string=None, experimental=False):
"""Create an API version request object."""
self._ver_major = None
self._ver_minor = None
if version_string is not None:
match = re.match(r"^([1-9]\d*)\.([1-9]\d*|0)$",
version_string)
if match:
self._ver_major = int(match.group(1))
self._ver_minor = int(match.group(2))
else:
raise exception.InvalidAPIVersionString(version=version_string)
def __str__(self):
"""Debug/Logging representation of object."""
return ("API Version Request Major: %(major)s, Minor: %(minor)s"
% {'major': self._ver_major, 'minor': self._ver_minor})
def is_null(self):
return self._ver_major is None and self._ver_minor is None
def _cmpkey(self):
"""Return the value used by ComparableMixin for rich comparisons."""
return self._ver_major, self._ver_minor
def matches_versioned_method(self, method):
"""Compares this version to that of a versioned method."""
if type(method) != versioned_method.VersionedMethod:
msg = ('An API version request must be compared '
'to a VersionedMethod object.')
raise exception.InvalidAPIVersionString(err=msg)
return self.matches(method.start_version,
method.end_version,
method.experimental)
def matches(self, min_version, max_version=None, experimental=False):
"""Compares this version to the specified min/max range.
Returns whether the version object represents a version
greater than or equal to the minimum version and less than
or equal to the maximum version.
If min_version is null then there is no minimum limit.
If max_version is null then there is no maximum limit.
If self is null then raise ValueError.
:param min_version: Minimum acceptable version.
:param max_version: Maximum acceptable version.
:param experimental: Whether to match experimental APIs.
:returns: boolean
"""
if self.is_null():
raise ValueError
if isinstance(min_version, str):
min_version = APIVersionRequest(version_string=min_version)
if isinstance(max_version, str):
max_version = APIVersionRequest(version_string=max_version)
if not min_version and not max_version:
return True
elif ((min_version and max_version) and
max_version.is_null() and min_version.is_null()):
return True
elif not max_version or max_version.is_null():
return min_version <= self
elif not min_version or min_version.is_null():
return self <= max_version
else:
return min_version <= self <= max_version
def get_string(self):
"""Returns a string representation of this object.
If this method is used to create an APIVersionRequest,
the resulting object will be an equivalent request.
"""
if self.is_null():
raise ValueError
return ("%(major)s.%(minor)s" %
{'major': self._ver_major, 'minor': self._ver_minor})

0
creds_manager/credsmgr/api/controllers/v1/__init__.py Normal file
View File

197
creds_manager/credsmgr/api/controllers/v1/credentials.py Normal file
View File

@ -0,0 +1,197 @@
# Copyright 2018 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import webob
from oslo_log import log as logging
from oslo_utils import uuidutils
from credsmgr.api.controllers.v1 import microversion
from credsmgr.api.controllers import wsgi
from credsmgr.db import api as db_api
from credsmgr import exception
from credsmgrclient.encrypt import ENCRYPTOR
LOG = logging.getLogger(__name__)
def _check_body(body):
if not body:
raise webob.exc.HTTPBadRequest(explanation="No data found in request")
def _check_admin(context):
if not context.is_admin:
msg = "User does not have admin privileges"
raise webob.exc.HTTPBadRequest(explanation=msg)
def _check_uuid(uuid):
if not uuidutils.is_uuid_like(uuid):
msg = "Id {} is invalid".format(uuid)
raise webob.exc.HTTPBadRequest(explanation=msg)
def _check_values(body, values):
for value in values:
if value not in body:
msg = "Invalid request {} value not present".format(value)
raise webob.exc.HTTPBadRequest(explanation=msg)
def _check_credential_exists(context, cred_id):
credentials = db_api.credentials_get_by_id(context, cred_id).count()
if not credentials:
e = exception.CredentialNotFound(cred_id=cred_id)
raise webob.exc.HTTPNotFound(explanation=e.format_message())
class CredentialController(wsgi.Controller):
def __init__(self, provider, supported_values, encrypted_values):
self.provider = provider
self.supported_values = supported_values
self.encrypted_values = encrypted_values
super(CredentialController, self).__init__()
@wsgi.response(201)
def create(self, req, body):
LOG.debug('Create %s credentials body %s', self.provider, body)
context = req.environ['credsmgr.context']
_check_body(body)
_check_values(body, self.supported_values)
properties = dict()
for value in self.supported_values:
if value in self.encrypted_values:
properties[value] = ENCRYPTOR.encrypt(body[value])
else:
properties[value] = body[value]
try:
self._check_for_duplicate_entries(context, body)
except exception.CredentialExists as e:
raise webob.exc.HTTPConflict(explanation=e.format_message())
cred_id = db_api.credentials_create(context, **properties)
return dict(cred_id=cred_id)
def _check_for_duplicate_entries(self, context, body):
all_credentials = db_api.credential_get_all(context)
creds_info = {}
for credentials in all_credentials:
if credentials.id not in creds_info:
creds_info[credentials.id] = {}
if credentials.name in self.encrypted_values:
value = ENCRYPTOR.decrypt(credentials.value)
else:
value = credentials.value
creds_info[credentials.id][credentials.name] = value
for creds in creds_info.values():
if body == creds:
raise exception.CredentialExists()
def update(self, req, cred_id, body):
context = req.environ['credsmgr.context']
_check_body(body)
_check_uuid(cred_id)
_check_credential_exists(context, cred_id)
credentials = db_api.credentials_get_by_id(context, cred_id)
_body = copy.deepcopy(body)
for credential in credentials:
name = credential.name
_value = str(credential.value)
if name in _body and _body[name] != _value:
value = _body.pop(name)
if name in self.encrypted_values:
value = ENCRYPTOR.encrypt(value)
db_api.credential_update(context, cred_id, name, value)
@wsgi.response(204)
def delete(self, req, cred_id):
context = req.environ['credsmgr.context']
_check_uuid(cred_id)
_check_credential_exists(context, cred_id)
try:
db_api.credentials_delete_by_id(context, cred_id)
except Exception as e:
LOG.exception("Error occurred while deleting credentials: %s" % e)
msg = "Delete failed for credential {}".format(cred_id)
raise webob.exc.HTTPBadRequest(explanation=msg)
def show(self, req, body=None):
context = req.environ['credsmgr.context']
mversion = microversion.get_and_validate_microversion(req)
tenant_id = req.params.get('tenant_id')
if not tenant_id:
_check_body(body)
_check_values(body, ('tenant_id', ))
tenant_id = body['tenant_id']
_check_uuid(tenant_id)
try:
rows = db_api.credential_association_get_credentials(context,
tenant_id)
except exception.CredentialAssociationNotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.format_message())
credential_info = {}
for row in rows:
credential_info[row.name] = row.value
if mversion >= microversion.add_cred_id:
credential_info['id'] = row.id
if not credential_info:
e = exception.CredentialAssociationNotFound(tenant_id=tenant_id)
raise webob.exc.HTTPNotFound(explanation=e.format_message())
return credential_info
def list(self, req):
context = req.environ['credsmgr.context']
_check_admin(context)
mversion = microversion.get_and_validate_microversion(req)
populate_id = mversion >= microversion.add_cred_id
return db_api.credential_association_get_all_credentials(
context, populate_id=populate_id)
@wsgi.response(201)
def association_create(self, req, cred_id, body):
context = req.environ['credsmgr.context']
_check_body(body)
_check_uuid(cred_id)
_check_values(body, ('tenant_id', ))
tenant_id = body['tenant_id']
_check_uuid(tenant_id)
# TODO(ssudake21): Verify tenant_id exists in keystone
try:
db_api.credential_association_create(context, cred_id, tenant_id)
except exception.CredentialAssociationExists as e:
raise webob.exc.HTTPConflict(explanation=e.format_message())
@wsgi.response(204)
def association_delete(self, req, cred_id, tenant_id):
context = req.environ['credsmgr.context']
_check_uuid(cred_id)
_check_uuid(tenant_id)
try:
db_api.credential_association_delete(context, cred_id, tenant_id)
except exception.CredentialAssociationNotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.format_message())
def association_list(self, req):
context = req.environ['credsmgr.context']
_check_admin(context)
credential_info = db_api.credential_association_list(context)
return credential_info
def create_resource(provider, supported_properties, encrypted_properties):
return wsgi.Resource(
CredentialController(provider, supported_properties,
encrypted_properties))

42
creds_manager/credsmgr/api/controllers/v1/microversion.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2018 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
microversion_header = 'OpenStack-API-Version'
default_microversion = 1.0
# 1.1: adds credential ID to GET /aws?tenant_id=<> and /aws/list
# APIs.
add_cred_id = 1.1
valid_microversions = [default_microversion, add_cred_id]
def get_and_validate_microversion(request):
"""
:param request: API request object to parse
"""
microversion_str = request.headers.get(microversion_header,
str(default_microversion))
try:
microversion = float(microversion_str)
except ValueError:
LOG.error('Incorrect microversion specified - %s', microversion_str)
microversion = default_microversion
if microversion not in valid_microversions:
LOG.error('Invalid microversion specified - %s, using default'
' microversion', microversion_str)
microversion = default_microversion
return microversion

93
creds_manager/credsmgr/api/controllers/v1/router.py Normal file
View File

@ -0,0 +1,93 @@
# Copyright 2011 OpenStack Foundation
# Copyright 2011 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2017 Platform9 Systems.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from credsmgr.api.controllers.v1 import credentials
from credsmgr.api import router
from credsmgrclient.common import constants
LOG = logging.getLogger(__name__)
class APIRouter(router.APIRouter):
"""Routes requests on the API to the appropriate controller and method."""
def _setup_routes(self, mapper):
LOG.info("Setup routes in for credentials API")
for provider, values_info in constants.provider_values.items():
self.resources[provider] = credentials.create_resource(
provider, values_info['supported_values'],
values_info['encrypted_values'])
self._set_resource_apis(provider, mapper)
def _set_resource_apis(self, provider, mapper):
controller = self.resources[provider]
url_info = [
{
'action': 'create',
'r_type': 'POST',
'suffix': ''
},
{
'action': 'show',
'r_type': 'GET',
'suffix': ''
},
{
'action': 'list',
'r_type': 'GET',
'suffix': '/list'
},
{
'action': 'update',
'r_type': 'PUT',
'suffix': '/{cred_id}'
},
{
'action': 'update',
'r_type': 'PATCH',
'suffix': '/{cred_id}'
},
{
'action': 'delete',
'r_type': 'DELETE',
'suffix': '/{cred_id}'
},
{
'action': 'association_create',
'r_type': 'POST',
'suffix': '/{cred_id}/association'
},
{
'action': 'association_delete',
'r_type': 'DELETE',
'suffix': '/{cred_id}/association/{tenant_id}'
},
{
'action': 'association_list',
'r_type': 'GET',
'suffix': '/associations'
}
]
for info in url_info:
uri = '/{0}{1}'.format(provider, info['suffix'])
LOG.debug("Setup URI {0} Info {1}".format(uri, info))
mapper.connect(uri, controller=controller, action=info['action'],
conditions={'method': [info['r_type']]})

50
creds_manager/credsmgr/api/controllers/versioned_method.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2014 IBM Corp.
# Copyright 2015 Clinton Knight
# Copyright 2017 Platform9 Systems
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from credsmgr import utils
class VersionedMethod(utils.ComparableMixin):
def __init__(self, name, start_version, end_version, experimental, func):
"""Versioning information for a single method.
Minimum and maximums are inclusive.
:param name: Name of the method
:param start_version: Minimum acceptable version
:param end_version: Maximum acceptable_version
:param func: Method to call
"""
self.name = name
self.start_version = start_version
self.end_version = end_version
self.experimental = experimental
self.func = func
def __str__(self):
args = {
'name': self.name,
'start': self.start_version,
'end': self.end_version
}
return "Version Method %(name)s: min: %(start)s, max: %(end)s" % args
def _cmpkey(self):
"""Return the value used by ComparableMixin for rich comparisons."""
return self.start_version

1118
creds_manager/credsmgr/api/controllers/wsgi.py Normal file
View File

File diff suppressed because it is too large Load Diff

0
creds_manager/credsmgr/api/middleware/__init__.py Normal file
View File

67
creds_manager/credsmgr/api/middleware/context.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright 2017 Platform9 Systems.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_middleware import request_id as oslo_request_id
from oslo_serialization import jsonutils
import credsmgr.context
from credsmgr.wsgi import common
context_opts = [
cfg.StrOpt('admin_role', default='admin',
help='Role used to identify an authenticated user as '
'administrator.')]
CONF = cfg.CONF
CONF.register_opts(context_opts)
CONF = cfg.CONF
class ContextMiddleware(common.Middleware):
def process_request(self, req):
"""Convert authentication information into a request context
Generate a murano.context.RequestContext object from the available
authentication headers and store on the 'context' attribute
of the req object.
:param req: wsgi request object that will be given the context object
"""
# FIXME: To be uncommented after keystone auth is enabled
roles = [r.strip() for r in req.headers.get('X-Roles').split(',')]
kwargs = {
'user': req.headers.get('X-User-Id'),
'tenant': req.headers.get('X-Project-Id'),
'project_name': req.headers.get('X-Project-Name'),
'auth_token': req.headers.get('X-Auth-Token'),
# 'session': req.headers.get('X-Configuration-Session'),
'is_admin': CONF.admin_role in roles,
'request_id': req.environ.get(oslo_request_id.ENV_REQUEST_ID),
'roles': roles
}
sc_header = req.headers.get('X-Service-Catalog')
sc_header = None
if sc_header:
kwargs['service_catalog'] = jsonutils.loads(sc_header)
req.environ['credsmgr.context'] = \
credsmgr.context.RequestContext(**kwargs)
@classmethod
def factory(cls, global_conf, **local_conf):
def filter(app):
return cls(app)
return filter

58
creds_manager/credsmgr/api/router.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright (c) 2013 OpenStack Foundation
# Copyright 2017 Platform9 Systems.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
WSGI middleware for OpenStack API controllers.
"""
from oslo_log import log as logging
from oslo_service import wsgi as base_wsgi
import routes
LOG = logging.getLogger(__name__)
class APIMapper(routes.Mapper):
def routematch(self, url=None, environ=None):
if url is "":
result = self._match("", environ)
return result[0], result[1]
return routes.Mapper.routematch(self, url, environ)
def connect(self, *args, **kwargs):
# NOTE(inhye): Default the format part of a route to only accept json
# so it doesn't eat all characters after a '.'
# in the url.
kwargs.setdefault('requirements', {})
if not kwargs['requirements'].get('format'):
kwargs['requirements']['format'] = 'json'
return routes.Mapper.connect(self, *args, **kwargs)
class APIRouter(base_wsgi.Router):
"""Routes requests on the API to the appropriate controller and method."""
@classmethod
def factory(cls, global_config, **local_config):
"""Simple paste factory, :class:`cinder.wsgi.Router` doesn't have."""
return cls()
def __init__(self):
LOG.info("Initializing APIRouter .....")
mapper = APIMapper()
self.resources = {}
self._setup_routes(mapper)
super(APIRouter, self).__init__(mapper)

0
creds_manager/credsmgr/cmd/__init__.py Normal file
View File

48
creds_manager/credsmgr/cmd/api.py Normal file
View File

@ -0,0 +1,48 @@
#!/usr/bin/env python
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Starter script for Credsmgr API."""
import eventlet # noqa
eventlet.monkey_patch() # noqa
import socket
import sys
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import service as oslo_service
# Need to register global_opts
from credsmgr import conf as credsmgr_conf # noqa
from credsmgr import service
CONF = cfg.CONF
host_opt = cfg.StrOpt('host', default=socket.gethostname(),
help='Credsmgr host')
CONF.register_opts([host_opt])
def main():
logging.register_options(CONF)
CONF(sys.argv[1:], project='credsmgr', version=".1")
logging.setup(CONF, "credsmgr")
service_instance = service.WSGIService('credsmgr_api')
service_launcher = oslo_service.ProcessLauncher(CONF)
service_launcher.launch_service(service_instance,
workers=service_instance.workers)
service_launcher.wait()

26
creds_manager/credsmgr/cmd/manage.py Normal file
View File

@ -0,0 +1,26 @@
# Copyright 2017 Platform9 Systems.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from oslo_config import cfg
from credsmgr.db import migration
def main():
CONF = cfg.CONF
CONF(sys.argv[1:], project='credsmgr')
migration.db_sync()

34
creds_manager/credsmgr/cmd/service.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2017 Platform9 Systems.
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import service as oslo_service
from credsmgr import conf # noqa
from credsmgr import service
CONF = cfg.CONF
logging.register_options(CONF)
CONF(sys.argv[1:], project='credsmgr', version=".1")
logging.setup(CONF, "credsmgr")
service_instance = service.WSGIService('credsmgr_api')
service_launcher = oslo_service.ProcessLauncher(CONF)
service_launcher.launch_service(service_instance,
workers=service_instance.workers)
service_launcher.wait()

22
creds_manager/credsmgr/conf/__init__.py Normal file
View File

@ -0,0 +1,22 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import default
import paths
from oslo_config import cfg
CONF = cfg.CONF
default.register_opts(CONF)
paths.register_opts(CONF)

34
creds_manager/credsmgr/conf/default.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2017 Platform9 Systems
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
CONF = cfg.CONF
default_opts = [
cfg.StrOpt('credsmgr_api_listen_port',
help='Credential Manager API listen Port'),
cfg.BoolOpt('credsmgr_api_use_ssl',
default=False,
help='SSL for Credential Manager API'),
cfg.IntOpt('credsmgr_api_workers',
default=1,
help='Number of workers for Credential Manager API service')
]
def register_opts(conf):
conf.register_opts(default_opts)

92
creds_manager/credsmgr/conf/paths.py Normal file
View File

@ -0,0 +1,92 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
# Copyright 2017 Platform9, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from oslo_config import cfg
ALL_OPTS = [
cfg.StrOpt('pybasedir', default=os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../')), help="""
The directory where the Nova python modules are installed.
This directory is used to store template files for networking and remote
console access. It is also the default path for other config options which
need to persist Nova internal data. It is very unlikely that you need to
change this option from its default value.
Possible values:
* The full path to a directory.
Related options:
* ``state_path``
"""),
cfg.StrOpt('bindir', default=os.path.join(sys.prefix, 'local', 'bin'),
help="""
The directory where the Nova binaries are installed.
This option is only relevant if the networking capabilities from Nova are
used (see services below). Nova's networking capabilities are targeted to
be fully replaced by Neutron in the future. It is very unlikely that you need
to change this option from its default value.
Possible values:
* The full path to a directory.
"""),
cfg.StrOpt('state_path', default='$pybasedir', help="""
The top-level directory for maintaining Nova's state.
This directory is used to store Nova's internal state. It is used by a
variety of other config options which derive from this. In some scenarios
(for example migrations) it makes sense to use a storage location which is
shared between multiple compute hosts (for example via NFS). Unless the
option ``instances_path`` gets overwritten, this directory can grow very
large.
Possible values:
* The full path to a directory. Defaults to value provided in ``pybasedir``.
"""),
]
def basedir_def(*args):
"""Return an uninterpolated path relative to $pybasedir."""
return os.path.join('$pybasedir', *args)
def bindir_def(*args):
"""Return an uninterpolated path relative to $bindir."""
return os.path.join('$bindir', *args)
def state_path_def(*args):
"""Return an uninterpolated path relative to $state_path."""
return os.path.join('$state_path', *args)
def register_opts(conf):
conf.register_opts(ALL_OPTS)
def list_opts():
return {"DEFAULT": ALL_OPTS}

194
creds_manager/credsmgr/context.py Normal file
View File

@ -0,0 +1,194 @@
# Copyright 2011 OpenStack Foundation
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""RequestContext: context for requests that persist through credsmgr."""
import copy
import policy
import six
from oslo_config import cfg
from oslo_context import context
from oslo_log import log as logging
from oslo_utils import timeutils
from credsmgr.db import api as db_api
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class RequestContext(context.RequestContext):
"""Security context and request information.
Represents the user taking a given action within the system.
"""
def __init__(self, user_id=None, project_id=None, is_admin=None,
read_deleted="no", project_name=None, remote_address=None,
timestamp=None, quota_class=None, service_catalog=None,
**kwargs):
"""Initialize RequestContext.
:param read_deleted: 'no' indicates deleted records are hidden, 'yes'
indicates deleted records are visible, 'only' indicates that
*only* deleted records are visible.
:param overwrite: Set to False to ensure that the greenthread local
copy of the index is not overwritten.
"""
# NOTE(jamielennox): oslo.context still uses some old variables names.
# These arguments are maintained instead of passed as kwargs to
# maintain the interface for tests.
kwargs.setdefault('user', user_id)
kwargs.setdefault('tenant', project_id)
super(RequestContext, self).__init__(is_admin=is_admin, **kwargs)
self.project_name = project_name
self.read_deleted = read_deleted
self.remote_address = remote_address
if not timestamp:
timestamp = timeutils.utcnow()
elif isinstance(timestamp, six.string_types):
timestamp = timeutils.parse_isotime(timestamp)
self.timestamp = timestamp
self.quota_class = quota_class
self._session = None
if service_catalog:
# Only include required parts of service_catalog
self.service_catalog = [s for s in service_catalog
if s.get('type') in
('identity', 'compute', 'credsmgr')]
else:
# if list is empty or none
self.service_catalog = []
# We need to have RequestContext attributes defined
# when policy.check_is_admin invokes request logging
# to make it loggable.
if self.is_admin is None:
self.is_admin = policy.check_is_admin(self.roles, self)
elif self.is_admin and 'admin' not in self.roles:
self.roles.append('admin')
def _get_read_deleted(self):
return self._read_deleted
def _set_read_deleted(self, read_deleted):
if read_deleted not in ('no', 'yes', 'only'):
raise ValueError("read_deleted can only be one of 'no',"
"'yes' or 'only', not %r" % read_deleted)
self._read_deleted = read_deleted
def _del_read_deleted(self):
del self._read_deleted
read_deleted = property(_get_read_deleted, _set_read_deleted,
_del_read_deleted)
def to_dict(self):
result = super(RequestContext, self).to_dict()
result['user_id'] = self.user_id
result['project_id'] = self.project_id
result['project_name'] = self.project_name
result['domain'] = self.domain
result['read_deleted'] = self.read_deleted
result['remote_address'] = self.remote_address
result['timestamp'] = self.timestamp.isoformat()
result['quota_class'] = self.quota_class
result['service_catalog'] = self.service_catalog
result['request_id'] = self.request_id
return result
@classmethod
def from_dict(cls, values):
return cls(user_id=values.get('user_id'),
project_id=values.get('project_id'),
project_name=values.get('project_name'),
domain=values.get('domain'),
read_deleted=values.get('read_deleted'),
remote_address=values.get('remote_address'),
timestamp=values.get('timestamp'),
quota_class=values.get('quota_class'),
service_catalog=values.get('service_catalog'),
request_id=values.get('request_id'),
is_admin=values.get('is_admin'),
roles=values.get('roles'),
auth_token=values.get('auth_token'),
user_domain=values.get('user_domain'),
project_domain=values.get('project_domain'))
def to_policy_values(self):
policy = super(RequestContext, self).to_policy_values()
policy['is_admin'] = self.is_admin
return policy
def elevated(self, read_deleted=None, overwrite=False):
"""Return a version of this context with admin flag set."""
context = self.deepcopy()
context.is_admin = True
if 'admin' not in context.roles:
context.roles.append('admin')
if read_deleted is not None:
context.read_deleted = read_deleted
return context
def deepcopy(self):
return copy.deepcopy(self)
# NOTE(sirp): the openstack/common version of RequestContext uses
# tenant/user whereas the Credsmgr version uses project_id/user_id.
# NOTE(adrienverge): The Credsmgr version of RequestContext now uses
# tenant/user internally, so it is compatible with context-aware code from
# openstack/common. We still need this shim for the rest of Credsmgr's
# code.
@property
def project_id(self):
return self.tenant
@project_id.setter
def project_id(self, value):
self.tenant = value
@property
def user_id(self):
return self.user
@user_id.setter
def user_id(self, value):
self.user = value
@property
def session(self):
if self._session is None:
self._session = db_api.get_session()
return self._session
def get_admin_context(read_deleted="no"):
return RequestContext(user_id=None,
project_id=None,
is_admin=True,
read_deleted=read_deleted,
overwrite=False)

19
creds_manager/credsmgr/credsmgr.conf Normal file
View File

@ -0,0 +1,19 @@
[DEFAULT]
credsmgr_api_listen_port = 8091
credsmgr_api_use_ssl = False
credsmgr_api_workers = 1
[keystone_authtoken]
auth_uri = http://localhost:8080/keystone/v3
auth_url = http://localhost:8080/keystone_admin
auth_version = v3
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = services
username = credsmgr
password = password
region_name = RegionOne
[database]
connection = mysql+pymysql://credsmgr:password@localhost/credsmgr

0
creds_manager/credsmgr/db/__init__.py Normal file
View File

87
creds_manager/credsmgr/db/api.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright 2018 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_db import api
from oslo_log import log as logging
CONF = cfg.CONF
log = logging.getLogger(__name__)
_BACKEND_MAPPING = {'sqlalchemy': 'credsmgr.db.sqlalchemy.api'}
IMPL = api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING)
def get_engine():
return IMPL.get_engine()
def get_session():
return IMPL.get_session()
def credential_create(context, name, value):
return IMPL.credential_create(context, name, value)
def credential_get(context, name):
return IMPL.credential_get(context, name)
def credential_get_all(context):
return IMPL.credential_get_all(context)
def credential_update(context, cred_id, name, value):
return IMPL.credential_update(context, cred_id, name, value)
def credential_delete(context, cred_id, name):
return IMPL.credential_delete(context, cred_id, name)
def credentials_create(context, **kwargs):
return IMPL.credentials_create(context, **kwargs)
def credentials_get_by_id(context, cred_id):
return IMPL.credentials_get_by_id(context, cred_id)
def credentials_delete_by_id(context, cred_id):
return IMPL.credentials_delete_by_id(context, cred_id)
def credential_association_get_all_credentials(context, populate_id=False):
return IMPL.credential_association_get_all_credentials(
context, populate_id=populate_id)
def credential_association_list(context):
return IMPL.credential_association_list(context)
def credential_association_get_credentials(context, project_id):
return IMPL.credential_association_get_credentials(context, project_id)
def credential_association_create(context, cred_id, project_id):
return IMPL.credential_association_create(context, cred_id, project_id)
def credential_association_delete(context, cred_id, project_id):
return IMPL.credential_association_delete(context, cred_id, project_id)

70
creds_manager/credsmgr/db/migration.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database setup and migration commands."""
import os
import threading
from oslo_config import cfg
from oslo_db import options
from stevedore import driver
from credsmgr.db.sqlalchemy import api as db_api
INIT_VERSION = 00
_IMPL = None
_LOCK = threading.Lock()
print("Set defaults")
options.set_defaults(cfg.CONF)
MIGRATE_REPO_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'sqlalchemy',
'migrate_repo',
)
def get_backend():
global _IMPL
if _IMPL is None:
with _LOCK:
if _IMPL is None:
_IMPL = driver.DriverManager(
"credsmgr.database.migration_backend",
cfg.CONF.database.backend).driver
return _IMPL
def db_sync(version=None, init_version=INIT_VERSION, engine=None):
"""Migrate the database to `version` or the most recent version."""
if engine is None:
engine = db_api.get_engine()
print("DB sync")
current_db_version = get_backend().db_version(engine,
MIGRATE_REPO_PATH,
init_version)
# TODO(e0ne): drop version validation when new oslo.db will be released
if version and int(version) < current_db_version:
msg = 'Database schema downgrade is not allowed.'
raise Exception(msg)
return get_backend().db_sync(engine=engine,
abs_path=MIGRATE_REPO_PATH,
version=version,
init_version=init_version)

0
creds_manager/credsmgr/db/sqlalchemy/__init__.py Normal file
View File

184
creds_manager/credsmgr/db/sqlalchemy/api.py Normal file
View File

@ -0,0 +1,184 @@
# Copyright 2018 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from oslo_config import cfg
from oslo_db import exception as db_exception
from oslo_db import options
from oslo_db.sqlalchemy import session as db_session
from oslo_db.sqlalchemy import utils as db_utils
from oslo_log import log as logging
from oslo_utils import uuidutils
from sqlalchemy.orm import exc as orm_exception
from credsmgr.db.sqlalchemy import models
from credsmgr import exception
CONF = cfg.CONF
options.set_defaults(CONF)
LOG = logging.getLogger(__name__)
_facade = None
def get_facade():
global _facade
if not _facade:
_facade = db_session.EngineFacade.from_config(CONF)
return _facade
def get_engine():
return get_facade().get_engine()
def get_session():
return get_facade().get_session()
def get_backend():
"""The backend is this module itself."""
return sys.modules[__name__]
def _get_context_values(context):
return {
'owner_user_id': context.user_id,
'owner_project_id': context.project_id
}
def credential_create(context, cred_id, name, value):
pass
def credential_get(context, cred_id, name):
try:
return db_utils.model_query(
models.Credential, context.session, deleted=False).\
filter_by(id=cred_id, name=name).one()
except orm_exception.NoResultFound:
raise exception.CredentialNotFound(cred_id=cred_id)
def credential_get_all(context):
return db_utils.model_query(models.Credential, context.session,
deleted=False)
def credential_update(context, cred_id, name, value):
_credential = credential_get(context, cred_id, name)
_credential.value = value
_credential.save(context.session)
def credential_delete(context, cred_id, name):
pass
def credentials_create(context, **kwargs):
session = context.session
cred_id = uuidutils.generate_uuid()
context_values = _get_context_values(context)
with session.begin():
for k, v in kwargs.items():
cp = models.Credential(id=cred_id, name=k, value=v)
cp.update(context_values)
session.add(cp)
return cred_id
def credentials_get_by_id(context, cred_id):
try:
return db_utils.model_query(
models.Credential, context.session, deleted=False).\
filter_by(id=cred_id)
except orm_exception.NoResultFound:
raise exception.CredentialNotFound(cred_id=cred_id)
def credentials_delete_by_id(context, cred_id):
query = credentials_get_by_id(context, cred_id)
for credential in query:
credential.soft_delete(context.session)
def credential_association_list(context):
all_credentials = {}
all_associations = db_utils.model_query(
models.CredentialsAssociation, context.session, deleted=False)
for association in all_associations:
all_credentials[association.project_id] = association.credential_id
return all_credentials
def credential_association_get_all_credentials(context, populate_id=False):
def _extract_creds(credentials):
credential_info = {}
for credential in credentials:
credential_info[credential.name] = credential.value
if populate_id:
credential_info['id'] = credential.id
return credential_info
all_credentials = {}
all_associations = db_utils.model_query(
models.CredentialsAssociation, context.session, deleted=False)
for association in all_associations:
creds = _extract_creds(association.credentials)
all_credentials[association.project_id] = creds
return all_credentials
def credential_association_get_credentials(context, project_id):
try:
creds_association = db_utils.model_query(
models.CredentialsAssociation, context.session, deleted=False).\
filter_by(project_id=project_id).one()
except orm_exception.NoResultFound:
raise exception.CredentialAssociationNotFound(tenant_id=project_id)
credentials = creds_association.credentials
return credentials
def credential_association_create(context, cred_id, project_id):
session = context.session
context_values = _get_context_values(context)
try:
with session.begin():
creds_association = models.CredentialsAssociation(
project_id=project_id,
credential_id=cred_id)
creds_association.update(context_values)
session.add(creds_association)
except db_exception.DBDuplicateEntry:
raise exception.CredentialAssociationExists(tenant_id=project_id)
def credential_association_delete(context, cred_id, project_id):
try:
creds_association = db_utils.model_query(
models.CredentialsAssociation, context.session, deleted=False).\
filter_by(credential_id=cred_id).\
filter_by(project_id=project_id).one()
except orm_exception.NoResultFound:
raise exception.CredentialAssociationNotFound(tenant_id=project_id)
creds_association.soft_delete(context.session)

4
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/README.md Normal file
View File

@ -0,0 +1,4 @@
This is database migration repository
More information at:
http://code.google.com/p/sqlalchemy-migrate/

0
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/__init__.py Normal file
View File

22
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/manage.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from credsmgr.db.sqlalchemy import migrate_repo
from migrate.versioning.shell import main
import os
if __name__ == "__main__":
main(debug=False,
repository=os.path.abspath(os.path.dirname(migrate_repo.__file__)))

20
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/migrate.cfg Normal file
View File

@ -0,0 +1,20 @@
[db_settings]
# Used to identify which repository this database is versioned under.
# You can use the name of your project.
repository_id=credsmgr
# The name of the database table used to track the schema version.
# This name shouldn't already be used by your project.
# If this is changed once a database is under version control, you'll need to
# change the table name in each database too.
version_table=migrate_version
# When committing a change script, Migrate will attempt to generate the
# sql for all supported databases; normally, if one of them fails - probably
# because you don't have that database installed - it is ignored and the
# commit continues, perhaps ending successfully.
# Databases in this list MUST compile successfully during a commit, or the
# entire commit will fail. List the databases your application will actually
# be using to ensure your updates to that database work properly.
# This must be a list; example: ['postgres','sqlite']
required_dbs=[]

72
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/versions/001_credsmgr_init.py Normal file
View File

@ -0,0 +1,72 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sqlalchemy import Column, MetaData, Table
from sqlalchemy import Integer, DateTime, String, ForeignKey, Text
from migrate.changeset import UniqueConstraint
def define_tables(meta):
credentials = Table(
'credentials', meta,
Column('created_at', DateTime(timezone=False), nullable=False),
Column('updated_at', DateTime(timezone=False)),
Column('deleted_at', DateTime(timezone=False)),
Column('deleted', Integer),
Column('owner_user_id', String(36)),
Column('owner_project_id', String(36)),
Column('id', String(36), nullable=False, primary_key=True),
Column('name', String(255), nullable=False, primary_key=True),
Column('value', Text(), nullable=False),
mysql_engine='InnoDB', mysql_charset='utf8')
credentials_association = Table(
'credentials_association', meta,
Column('created_at', DateTime(timezone=False), nullable=False),
Column('updated_at', DateTime(timezone=False)),
Column('deleted_at', DateTime(timezone=False)),
Column('deleted', Integer),
Column('owner_user_id', String(36)),
Column('owner_project_id', String(36)),
Column('id', Integer, primary_key=True, autoincrement=True),
Column('project_id', String(36), nullable=False),
Column('credential_id',
String(36),
ForeignKey('credentials.id'), nullable=False),
UniqueConstraint(
'project_id', 'deleted',
name='uniq_credentials_association0'
'project_id0deleted'),
mysql_engine='InnoDB', mysql_charset='utf8')
return [credentials, credentials_association]
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = define_tables(meta)
for table in tables:
table.create()
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
tables = define_tables(meta)
for table in tables:
table.drop()

0
creds_manager/credsmgr/db/sqlalchemy/migrate_repo/versions/__init__.py Normal file
View File

69
creds_manager/credsmgr/db/sqlalchemy/migration.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database setup and migration commands."""
import os
import threading
from oslo_config import cfg
from oslo_db import options
from stevedore import driver
from credsmgr.db import api as db_api
from credsmgr import exception
INIT_VERSION = 1
_IMPL = None
_LOCK = threading.Lock()
options.set_defaults(cfg.CONF)
MIGRATE_REPO_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'sqlalchemy',
'migrate_repo',
)
def get_backend():
global _IMPL
if _IMPL is None:
with _LOCK:
if _IMPL is None:
_IMPL = driver.DriverManager(
"credsmgr.database.migration_backend",
cfg.CONF.database.backend).driver
return _IMPL
def db_sync(version=None, init_version=INIT_VERSION, engine=None):
"""Migrate the database to `version` or the most recent version."""
if engine is None:
engine = db_api.get_engine()
current_db_version = get_backend().db_version(engine,
MIGRATE_REPO_PATH,
init_version)
# TODO(e0ne): drop version validation when new oslo.db will be released
if version and int(version) < current_db_version:
msg = 'Database schema downgrade is not allowed.'
raise exception.InvalidInput(reason=msg)
return get_backend().db_sync(engine=engine,
abs_path=MIGRATE_REPO_PATH,
version=version,
init_version=init_version)

87
creds_manager/credsmgr/db/sqlalchemy/models.py Normal file
View File

@ -0,0 +1,87 @@
# Copyright 2017 Platform9 Systems, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
SQLAlchemy models for credsmgr data.
"""
from oslo_config import cfg
from oslo_db.sqlalchemy import models
from oslo_utils import timeutils
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, Text
from sqlalchemy import ForeignKey, UniqueConstraint
from sqlalchemy.orm import relationship
CONF = cfg.CONF
BASE = declarative_base()
class CredsMgrBase(models.TimestampMixin, models.SoftDeleteMixin,
models.ModelBase):
"""Base class for Credsmgr Models."""
__table_args__ = {'mysql_engine': 'InnoDB'}
owner_user_id = Column(String(36), nullable=False)
owner_project_id = Column(String(36), nullable=False)
metadata = None
class Credential(BASE, CredsMgrBase):
__tablename__ = 'credentials'
id = Column(String(36), nullable=False, primary_key=True)
name = Column(String(255), nullable=False, primary_key=True)
value = Column(Text(), nullable=False)
def soft_delete(self, session):
# NOTE(ssudake21): oslo_db directly assigns object id to deleted field.
# Here we have string, so need to override soft_delete method.
self.deleted = 1
self.deleted_at = timeutils.utcnow()
self.save(session=session)
class CredentialsAssociation(BASE, CredsMgrBase):
"""Represents credentials association with tenant"""
__tablename__ = 'credentials_association'
__table_args__ = (
UniqueConstraint(
'project_id', 'deleted',
name='uniq_credentials_association0'
'project_id0deleted'
), {})
id = Column(Integer, primary_key=True, autoincrement=True)
project_id = Column(String(36), nullable=False)
credential_id = Column(
String(36), ForeignKey('credentials.id'), nullable=False)
primaryjoin = 'and_({0}.{1} == {2}.id, {2}.deleted == 0)'.format(
'CredentialsAssociation', 'credential_id', 'Credential')
credentials = relationship('Credential', uselist=True,
primaryjoin=primaryjoin)
def register_models(engine):
"""Creates database tables for all models with the given engine."""
models = (Credential, CredentialsAssociation)
for model in models:
model.metadata.create_all(engine)
def unregister_models(engine):
"""Drops database tables for all models with the given engine."""
models = (Credential, CredentialsAssociation)
for model in models:
model.metadata.drop_all(engine)

198
creds_manager/credsmgr/exception.py Normal file
View File

@ -0,0 +1,198 @@
# Copyright 2017 Platform9 Systems
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Credsmgr base exception handling.
Includes decorator for re-raising Credsmgr-type exceptions.
SHOULD include dedicated exception logging.
"""
import sys
from oslo_config import cfg
from oslo_log import log as logging
import six
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class CredsMgrException(Exception):
"""Base Credsmgr Exception
To correctly use this class, inherit from it and define
a 'msg_fmt' property. That msg_fmt will get printf'd
with the keyword arguments provided to the constructor.
"""
msg_fmt = "An unknown exception occurred."
code = 500
headers = {}
safe = False
def __init__(self, message=None, **kwargs):
self.kwargs = kwargs
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
if not message:
try:
message = self.msg_fmt % kwargs
except Exception:
exc_info = sys.exc_info()
# kwargs doesn't match a variable in the message
# log the issue and the kwargs
LOG.exception('Exception in string format operation')
for name, value in six.iteritems(kwargs):
LOG.error("%s: %s" % (name, value)) # noqa
if CONF.fatal_exception_format_errors:
six.reraise(*exc_info)
else:
# at least get the core message out if something happened
message = self.msg_fmt
self.message = message
super(CredsMgrException, self).__init__(message)
def format_message(self):
# NOTE: use the first argument to the python Exception object
# which should be our full CredsMgrException message, (see __init__)
return self.args[0]
class APIException(CredsMgrException):
msg_fmt = "Error while requesting %(service)s API."
def __init__(self, message=None, **kwargs):
if 'service' not in kwargs:
kwargs['service'] = 'unknown'
super(APIException, self).__init__(message, **kwargs)
class APITimeout(APIException):
msg_fmt = "Timeout while requesting %(service)s API."
class Conflict(CredsMgrException):
msg_fmt = "Conflict"
code = 409
class Invalid(CredsMgrException):
msg_fmt = "Bad Request - Invalid Parameters"
code = 400
class InvalidName(Invalid):
msg_fmt = "An invalid 'name' value was provided. "\
"The name must be: %(reason)s"
class InvalidInput(Invalid):
msg_fmt = "Invalid input received: %(reason)s"
class InvalidAPIVersionString(Invalid):
msg_fmt = "API Version String %(version)s is of invalid format. Must "\
"be of format MajorNum.MinorNum."
class MalformedRequestBody(CredsMgrException):
msg_fmt = "Malformed message body: %(reason)s"
# NOTE: NotFound should only be used when a 404 error is
# appropriate to be returned
class NotFound(CredsMgrException):
msg_fmt = "Resource could not be found."
code = 404
class ConfigNotFound(NotFound):
msg_fmt = "Could not find config at %(path)s"
class Forbidden(CredsMgrException):
msg_fmt = "Forbidden"
code = 403
class AdminRequired(Forbidden):
msg_fmt = "User does not have admin privileges"
class PolicyNotAuthorized(Forbidden):
msg_fmt = "Policy doesn't allow %(action)s to be performed."
class PasteAppNotFound(CredsMgrException):
msg_fmt = "Could not load paste app '%(name)s' from %(path)s"
class InvalidContentType(Invalid):
msg_fmt = "Invalid content type %(content_type)s."
class VersionNotFoundForAPIMethod(Invalid):
msg_fmt = "API version %(version)s is not supported on this method."
class InvalidGlobalAPIVersion(Invalid):
msg_fmt = "Version %(req_ver)s is not supported by the API. Minimum " \
"is %(min_ver)s and maximum is %(max_ver)s."
class ApiVersionsIntersect(Invalid):
msg_fmt = "Version of %(name) %(min_ver) %(max_ver) intersects " \
"with another versions."
class ValidationError(Invalid):
msg_fmt = "%(detail)s"
class Unauthorized(CredsMgrException):
msg_fmt = "Not authorized."
code = 401
class NoResources(CredsMgrException):
msg_fmt = "No resources available"
class CredentialNotFound(NotFound):
msg_fmt = "Credential with id %(cred_id)s could not be found."
class CredentialAssociationNotFound(NotFound):
msg_fmt = "Credential associated with tenant %(tenant_id)s "\
"could not be found."
class CredentialAssociationExists(Conflict):
msg_fmt = "Credential associated with tenant %(tenant_id)s exists"
class CredentialExists(Conflict):
msg_fmt = "credentials with provided parameters already exists"

81
creds_manager/credsmgr/policy.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2011 OpenStack Foundation
# All Rights Reserved.
#
# Copyright (c) 2017 Platform9 Systems
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Policy Engine For Credsmgr"""
from oslo_config import cfg
from oslo_policy import opts as policy_opts
from oslo_policy import policy
from credsmgr import exception
CONF = cfg.CONF
policy_opts.set_defaults(cfg.CONF, 'policy.json')
_ENFORCER = None
def init():
global _ENFORCER
if not _ENFORCER:
_ENFORCER = policy.Enforcer(CONF)
def enforce(context, action, target):
"""Verifies that the action is valid on the target in this context.
:param context: credsmgr context
:param action: string representing the action to be checked
this should be colon separated for clarity.
i.e. ``compute:create_instance``,
``compute:attach_volume``,
``volume:attach_volume``
:param object: dictionary representing the object of the action
for object creation this should be a dictionary representing the
location of the object e.g. ``{'project_id': context.project_id}``
:raises PolicyNotAuthorized: if verification fails.
"""
init()
return _ENFORCER.enforce(action,
target,
context.to_policy_values(),
do_raise=True,
exc=exception.PolicyNotAuthorized,
action=action)
def check_is_admin(roles, context=None):
"""Whether or not user is admin according to policy setting.
"""
init()
# include project_id on target to avoid KeyError if context_is_admin
# policy definition is missing, and default admin_or_owner rule
# attempts to apply.
target = {'project_id': ''}
if context is None:
credentials = {'roles': roles}
else:
credentials = context.to_dict()
return _ENFORCER.enforce('context_is_admin', target, credentials)

115
creds_manager/credsmgr/service.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_service import service
from oslo_service import wsgi
from oslo_utils import importutils
CONF = cfg.CONF
class WSGIService(service.ServiceBase):
"""Provides ability to launch API from a 'paste' configuration."""
def __init__(self, name, loader=None):
"""Initialize, but do not start the WSGI server.
:param name: The name of the WSGI server given to the loader.
:param loader: Loads the WSGI application using the given name.
:returns: None
"""
self.name = name
self.manager = self._get_manager()
self.loader = loader or wsgi.Loader(CONF)
self.app = self.loader.load_app(name)
self.host = getattr(CONF, '%s_listen' % name, "0.0.0.0")
self.port = getattr(CONF, '%s_listen_port' % name, 0)
self.use_ssl = getattr(CONF, '%s_use_ssl' % name, False)
self.workers = getattr(CONF, '%s_workers' % name, 1)
if self.workers and self.workers < 1:
worker_name = '%s_workers' % name
msg = ("%(worker_name)s value of %(workers)d is invalid, "
"must be greater than 0." %
{'worker_name': worker_name,
'workers': self.workers})
raise Exception(msg)
# setup_profiler(name, self.host)
self.server = wsgi.Server(CONF,
name,
self.app,
host=self.host,
port=self.port,
use_ssl=self.use_ssl)
def _get_manager(self):
"""Initialize a Manager object appropriate for this service.
Use the service name to look up a Manager subclass from the
configuration and initialize an instance. If no class name
is configured, just return None.
:returns: a Manager instance, or None.
"""
fl = '%s_manager' % self.name
if fl not in CONF:
return None
manager_class_name = CONF.get(fl, None)
if not manager_class_name:
return None
manager_class = importutils.import_class(manager_class_name)
return manager_class()
def start(self):
"""Start serving this service using loaded configuration.
Also, retrieve updated port number in case '0' was passed in, which
indicates a random port should be used.
:returns: None
"""
if self.manager:
self.manager.init_host()
self.server.start()
self.port = self.server.port
def stop(self):
"""Stop serving this API.
:returns: None
"""
self.server.stop()
def wait(self):
"""Wait for the service to stop serving this API.
:returns: None
"""
self.server.wait()
def reset(self):
"""Reset server greenpool size to default.
:returns: None
"""
self.server.reset()

68
creds_manager/credsmgr/test.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import fixtures
from oslo_config import cfg
from oslo_log import log as logging
from oslotest import moxstubout
import testtools
from credsmgr.tests import utils
CONF = cfg.CONF
logging.register_options(CONF)
logging.setup(CONF, 'credsmgr')
_DB_CACHE = None
class Database(fixtures.Fixture):
def __init__(self, db_api, db_migrate, sql_connection):
self.sql_connection = sql_connection
self.engine = db_api.get_engine()
self.engine.dispose()
def setUp(self):
super(Database, self).setUp()
conn = self.engine.connect()
conn.connection.executescript(self._DB)
self.addCleanup(self.engine.dispose)
class TestCase(testtools.TestCase):
"""
Base class for all credsmgr unit tests
"""
def setUp(self):
super(TestCase, self).setUp()
self.useFixture(fixtures.FakeLogger('credsmgr'))
CONF.set_default('connection', 'sqlite://', 'database')
CONF.set_default('sqlite_synchronous', True, 'database')
utils.setup_dummy_db()
self.addCleanup(utils.reset_dummy_db)
mox_fixture = self.useFixture(moxstubout.MoxStubout())
self.mox = mox_fixture.mox
self.stubs = mox_fixture.stubs
class DBObject(dict):
def __init__(self, **kwargs):
super(DBObject, self).__init__(kwargs)
def __getattr__(self, item):
return self[item]

0
creds_manager/credsmgr/tests/__init__.py Normal file
View File

0
creds_manager/credsmgr/tests/unit/__init__.py Normal file
View File

0
creds_manager/credsmgr/tests/unit/api/__init__.py Normal file
View File

19
creds_manager/credsmgr/tests/unit/api/api_base.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from credsmgr import test
class ApiBaseTest(test.TestCase):
def setUp(self):
super(ApiBaseTest, self).setUp()

0
creds_manager/credsmgr/tests/unit/api/controllers/__init__.py Normal file
View File

0
creds_manager/credsmgr/tests/unit/api/controllers/v1/__init__.py Normal file
View File

214
creds_manager/credsmgr/tests/unit/api/controllers/v1/test_credentials.py Normal file
View File

@ -0,0 +1,214 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from credsmgr.api.controllers.v1 import credentials
from credsmgr.db import api as db_api
from credsmgrclient.common import constants
from credsmgr.tests.unit.api import api_base
from credsmgr.tests.unit.api import fakes
import webob
LOG = logging.getLogger(__name__)
def fake_creds():
return dict(aws_access_key_id='fake_access_key',
aws_secret_access_key='fake_secret_key')
class CredentialControllerTest(api_base.ApiBaseTest):
def setUp(self):
super(CredentialControllerTest, self).setUp()
provider_values = constants.provider_values[constants.AWS]
self.controller = credentials.CredentialController(
constants.AWS, provider_values['supported_values'],
provider_values['encrypted_values'])
def get_credentials(self, cred_id):
context = fakes.HTTPRequest.blank('v1/credentials').environ[
'credsmgr.context']
credentials = db_api.credentials_get_by_id(context, cred_id)
creds_info = {}
for credential in credentials:
creds_info[credential.name] = credential.value
return creds_info
def _call_request(self, action, *args, **kwargs):
use_admin_context = kwargs.pop('use_admin_context', False)
project_name = kwargs.pop('project_name', 'service')
microversion = kwargs.pop('microversion', None)
req = fakes.HTTPRequest.blank('v1/credentials',
use_admin_context=use_admin_context,
project_name=project_name)
if microversion:
req.headers['OpenStack-API-Version'] = microversion
action = getattr(self.controller, action)
return action(req, *args, **kwargs)
def test_credentials_create(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
creds_info = self.get_credentials(resp['cred_id'])
self.assertEqual(creds, creds_info)
def test_credentials_create_duplicate(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
self.assertRaises(webob.exc.HTTPConflict, self._call_request,
'create', creds)
def test_credentials_create_with_duplicate_values_after_deleting(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
self._call_request('delete', resp['cred_id'])
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
def test_credentials_update(self):
creds = fake_creds()
resp = self._call_request('create', creds)
creds['aws_access_key_id'] = 'fake_access_key2'
creds['aws_secret_access_key'] = 'fake_secret_key2'
self._call_request('update', resp['cred_id'], creds)
creds_info = self.get_credentials(resp['cred_id'])
self.assertEqual(creds, creds_info)
def test_credentials_delete(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
self._call_request('delete', resp['cred_id'])
creds_info = self.get_credentials(resp['cred_id'])
self.assertFalse(creds_info)
def test_credentials_association_create(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
body = {'tenant_id': 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'}
self._call_request('association_create', resp['cred_id'], body)
creds_info = self._call_request('show', body)
self.assertEqual(creds, creds_info)
def test_credential_get_with_microversion(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
body = {'tenant_id': 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'}
self._call_request('association_create', resp['cred_id'], body)
creds_info = self._call_request('show', body, microversion="1.1")
creds_id = creds_info.pop('id')
self.assertEqual(creds_id, resp['cred_id'])
self.assertEqual(creds, creds_info)
def test_credential_get_with_wrong_microversion(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
body = {'tenant_id': 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'}
self._call_request('association_create', resp['cred_id'], body)
creds_info = self._call_request('show', body, microversion="a.b")
creds_id = creds_info.pop('id', None)
self.assertIsNone(creds_id)
self.assertEqual(creds, creds_info)
def test_credentials_association_delete(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
body = {'tenant_id': 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'}
self._call_request('association_create', resp['cred_id'], body)
creds_info = self._call_request('show', body)
self.assertEqual(creds, creds_info)
self._call_request('association_delete', resp['cred_id'],
body['tenant_id'])
self.assertRaises(webob.exc.HTTPNotFound, self._call_request, 'show',
body)
def test_credentials_list_without_admin(self):
self.assertRaises(webob.exc.HTTPBadRequest, self._call_request, 'list')
def test_credentials_list(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
body = {'tenant_id': project_id1}
self._call_request('association_create', resp['cred_id'], body)
project_id2 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5d'
body = {'tenant_id': project_id2}
self._call_request('association_create', resp['cred_id'], body)
all_creds = self._call_request('list', use_admin_context=True,
project_name='services')
self.assertEqual(len(all_creds), 2)
self.assertIn(project_id1, all_creds)
self.assertIn(project_id2, all_creds)
def test_credentials_list_with_microversion(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
body = {'tenant_id': project_id1}
self._call_request('association_create', resp['cred_id'], body)
project_id2 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5d'
body = {'tenant_id': project_id2}
self._call_request('association_create', resp['cred_id'], body)
all_creds = self._call_request('list', use_admin_context=True,
project_name='services',
microversion="1.1")
self.assertEqual(len(all_creds), 2)
for _, creds in all_creds.items():
self.assertIn('id', creds)
def test_credentials_list_with_incorrect_microversion(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
body = {'tenant_id': project_id1}
self._call_request('association_create', resp['cred_id'], body)
project_id2 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5d'
body = {'tenant_id': project_id2}
self._call_request('association_create', resp['cred_id'], body)
all_creds = self._call_request('list', use_admin_context=True,
project_name='services',
microversion="a.b")
self.assertEqual(len(all_creds), 2)
for _, creds in all_creds.items():
self.assertNotIn('id', creds)
def test_credential_association_list(self):
creds = fake_creds()
resp = self._call_request('create', creds)
self.assertTrue('cred_id' in resp)
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
body = {'tenant_id': project_id1}
self._call_request('association_create', resp['cred_id'], body)
all_creds = self._call_request('association_list',
use_admin_context=True)
self.assertEqual(len(all_creds), 1)
self.assertIn(project_id1, all_creds)
def test_credential_association_list_with_no_associations(self):
all_creds = self._call_request('association_list',
use_admin_context=True)
self.assertEqual(len(all_creds), 0)
self.assertEqual(all_creds, {})

56
creds_manager/credsmgr/tests/unit/api/fakes.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2010 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_service import wsgi
import webob
import webob.dec
import webob.request
from credsmgr.api.controllers import api_version_request as api_version
from credsmgr import context
FAKE_PROJECT_ID = '9a06b8ce-4803-4b4c-89a5-27b75c1cba4b'
FAKE_USER_ID = '9a2d073f-5fd4-41ec-98b8-ee775a8f6a04'
class FakeRequestContext(context.RequestContext):
def __init__(self, *args, **kwargs):
kwargs['auth_token'] = kwargs.get(FAKE_USER_ID, FAKE_PROJECT_ID)
super(FakeRequestContext, self).__init__(*args, **kwargs)
class HTTPRequest(webob.Request):
@classmethod
def blank(cls, *args, **kwargs):
if args is not None:
if 'v1' in args[0]:
kwargs['base_url'] = 'http://localhost/v1'
if 'v2' in args[0]:
kwargs['base_url'] = 'http://localhost/v2'
if 'v3' in args[0]:
kwargs['base_url'] = 'http://localhost/v3'
use_admin_context = kwargs.pop('use_admin_context', False)
project_name = kwargs.pop('project_name', 'service')
version = kwargs.pop('version', api_version._MIN_API_VERSION)
out = wsgi.Request.blank(*args, **kwargs)
out.environ['credsmgr.context'] = FakeRequestContext(
FAKE_USER_ID,
FAKE_PROJECT_ID,
is_admin=use_admin_context,
project_name=project_name)
out.api_version_request = api_version.APIVersionRequest(version)
return out

0
creds_manager/credsmgr/tests/unit/db/__init__.py Normal file
View File

0
creds_manager/credsmgr/tests/unit/db/sqlalchemy/__init__.py Normal file
View File

143
creds_manager/credsmgr/tests/unit/db/sqlalchemy/test_db_api.py Normal file
View File

@ -0,0 +1,143 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from credsmgr import context
from credsmgr.db import api as db_api
from credsmgr import exception
import credsmgr.tests.unit.db.test_db as test_db
LOG = logging.getLogger(__name__)
class TestDBApi(test_db.BaseTest):
def setUp(self):
super(TestDBApi, self).setUp()
self.ctxt = context.get_admin_context()
self.ctxt.user_id = 'fake-user'
self.ctxt.project_id = 'fake-project'
@staticmethod
def default_aws_credential_values():
return dict(
aws_access_key_id='fake_access_key',
aws_secret_access_key='fake_secret_key', )
def get_credentials(self, cred_id):
credentials = db_api.credentials_get_by_id(self.ctxt, cred_id)
creds_info = {}
for credential in credentials:
creds_info[credential.name] = credential.value
return creds_info
def _setup_credentials(self):
values = self.default_aws_credential_values()
cred_id = db_api.credentials_create(self.ctxt, **values)
creds_info = self.get_credentials(cred_id)
self.assertEqual(len(creds_info), 2)
self.assertEqual(values, creds_info)
return cred_id
def test_credentials_create(self):
self._setup_credentials()
def test_credentials_update(self):
cred_id = self._setup_credentials()
values = self.default_aws_credential_values()
values['aws_access_key_id'] = 'fake_access_key2'
values['aws_secret_access_key'] = 'fake_secret_key2'
for k, v in values.items():
db_api.credential_update(self.ctxt, cred_id, k, v)
creds_info = self.get_credentials(cred_id)
self.assertEqual(len(creds_info), 2)
self.assertEqual(values, creds_info)
def test_credential_update_with_different_keys(self):
cred_id = self._setup_credentials()
values = {
'x-aws_access_key_id': 'fake_access_key2',
'x-aws_secret_access_key': 'fake_secret_key2'
}
for k, v in values.items():
self.assertRaises(exception.CredentialNotFound,
db_api.credential_update, self.ctxt, cred_id, k,
v)
def test_credentials_delete(self):
cred_id = self._setup_credentials()
db_api.credentials_delete_by_id(self.ctxt, cred_id)
creds_info = self.get_credentials(cred_id)
self.assertEqual(len(creds_info), 0)
def test_credentials_association(self):
cred_id = self._setup_credentials()
project_id = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
db_api.credential_association_create(self.ctxt, cred_id, project_id)
credentials = db_api.credential_association_get_credentials(
self.ctxt, project_id)
creds_info = {}
for credential in credentials:
creds_info[credential.name] = credential.value
values = self.default_aws_credential_values()
self.assertEqual(len(creds_info), 2)
self.assertEqual(values, creds_info)
db_api.credential_association_delete(self.ctxt, cred_id, project_id)
def test_credentials_association_exists(self):
cred_id = self._setup_credentials()
project_id = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
db_api.credential_association_create(self.ctxt, cred_id, project_id)
self.assertRaises(exception.CredentialAssociationExists,
db_api.credential_association_create, self.ctxt,
cred_id, project_id)
db_api.credential_association_delete(self.ctxt, cred_id, project_id)
db_api.credential_association_create(self.ctxt, cred_id, project_id)
def test_credential_association_does_not_exist(self):
project_id = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
self.assertRaises(exception.CredentialAssociationNotFound,
db_api.credential_association_get_credentials,
self.ctxt, project_id)
def test_credential_association_does_not_exist_after_delete(self):
cred_id = self._setup_credentials()
project_id = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
db_api.credential_association_create(self.ctxt, cred_id, project_id)
self.assertRaises(exception.CredentialAssociationExists,
db_api.credential_association_create, self.ctxt,
cred_id, project_id)
db_api.credential_association_delete(self.ctxt, cred_id, project_id)
self.assertRaises(exception.CredentialAssociationNotFound,
db_api.credential_association_get_credentials,
self.ctxt, project_id)
def test_credential_association_get_all_credentials(self):
cred_id = self._setup_credentials()
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
project_id2 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5d'
db_api.credential_association_create(self.ctxt, cred_id, project_id1)
db_api.credential_association_create(self.ctxt, cred_id, project_id2)
all_creds = db_api.credential_association_get_all_credentials(
self.ctxt)
self.assertIn(project_id1, all_creds)
self.assertIn(project_id2, all_creds)
self.assertEqual(len(all_creds), 2)
def test_credential_association_list(self):
cred_id = self._setup_credentials()
project_id1 = 'd37da4ea-8249-4bb7-94a2-d6a12f1b1a5c'
db_api.credential_association_create(self.ctxt, cred_id, project_id1)
all_creds = db_api.credential_association_list(self.ctxt)
self.assertIn(project_id1, all_creds)
self.assertEqual(len(all_creds), 1)

19
creds_manager/credsmgr/tests/unit/db/test_db.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from credsmgr.test import TestCase
class BaseTest(TestCase):
pass

41
creds_manager/credsmgr/tests/utils.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sqlalchemy
from oslo_config import cfg
from oslo_db import options
from credsmgr.db import api as db_api
from credsmgr.db.sqlalchemy import models
get_engine = db_api.get_engine
def setup_dummy_db():
options.cfg.set_defaults(options.database_opts, sqlite_synchronous=False)
options.set_defaults(cfg.CONF, connection="sqlite://")
engine = get_engine()
models.BASE.metadata.create_all(engine)
engine.connect()
def reset_dummy_db():
engine = get_engine()
meta = sqlalchemy.MetaData()
meta.reflect(bind=engine)
for table in reversed(meta.sorted_tables):
if table.name == 'migrate_version':
continue
engine.execute(table.delete())

89
creds_manager/credsmgr/utils.py Normal file
View File

@ -0,0 +1,89 @@
# Copyright 2017 Platform9 Systems
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_concurrency import lockutils
from oslo_utils import encodeutils
from oslo_utils import strutils
import six
from credsmgr import exception
synchronized = lockutils.synchronized_with_prefix('credsmgr-')
class ComparableMixin(object):
def _compare(self, other, method):
try:
return method(self._cmpkey(), other._cmpkey())
except (AttributeError, TypeError):
# _cmpkey not implemented, or return different type,
# so I can't compare with "other".
return NotImplemented
def __lt__(self, other):
return self._compare(other, lambda s, o: s < o)
def __le__(self, other):
return self._compare(other, lambda s, o: s <= o)
def __eq__(self, other):
return self._compare(other, lambda s, o: s == o)
def __ge__(self, other):
return self._compare(other, lambda s, o: s >= o)
def __gt__(self, other):
return self._compare(other, lambda s, o: s > o)
def __ne__(self, other):
return self._compare(other, lambda s, o: s != o)
def check_string_length(value, name, min_length=0, max_length=None,
allow_all_spaces=True):
"""Check the length of specified string.
:param value: the value of the string
:param name: the name of the string
:param min_length: the min_length of the string
:param max_length: the max_length of the string
"""
try:
strutils.check_string_length(value, name=name, min_length=min_length,
max_length=max_length)
except (ValueError, TypeError) as exc:
raise exception.InvalidInput(reason=exc)
if not allow_all_spaces and value.isspace():
msg = '%(name)s cannot be all spaces.'
raise exception.InvalidInput(reason=msg)
def convert_str(text):
"""Convert to native string.
Convert bytes and Unicode strings to native strings:
* convert to bytes on Python 2:
encode Unicode using encodeutils.safe_encode()
* convert to Unicode on Python 3: decode bytes from UTF-8
"""
if six.PY2:
return encodeutils.to_utf8(text)
else:
if isinstance(text, bytes):
return text.decode('utf-8')
else:
return text

0
creds_manager/credsmgr/wsgi/__init__.py Normal file
View File

219
creds_manager/credsmgr/wsgi/common.py Normal file
View File

@ -0,0 +1,219 @@
# Copyright 2017 Platform9 Systems.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Utility methods for working with WSGI servers."""
import webob.dec
import webob.exc
from credsmgr.api.controllers import api_version_request
from credsmgr import exception
SUPPORTED_CONTENT_TYPES = ('application/json')
SUPPORTED_ACCEPT_TYPES = ('application/json')
class Request(webob.Request):
def __init__(self, *args, **kwargs):
super(Request, self).__init__(*args, **kwargs)
self.support_api_request_version = False
if not hasattr(self, 'api_version_request'):
self.api_version_request = api_version_request.APIVersionRequest()
def set_api_version_request(self, url):
"""Set API version request based on the request header information.
"""
if 'v1' in url:
self.api_version_request = 'v1'
def get_content_type(self):
"""Determine content type of the request body.
Does not do any body introspection, only checks header
"""
if "Content-Type" not in self.headers:
return None
allowed_types = SUPPORTED_CONTENT_TYPES
content_type = self.content_type
if content_type not in allowed_types:
raise exception.InvalidContentType(content_type=content_type)
return content_type
def best_match_content_type(self):
"""Determine the requested response content-type."""
if 'credsmgr.best_content_type' not in self.environ:
# Calculate the best MIME type
content_type = None
# Check URL path suffix
parts = self.path.rsplit('.', 1)
if len(parts) > 1:
possible_type = 'application/' + parts[1]
if possible_type in SUPPORTED_CONTENT_TYPES:
content_type = possible_type
if not content_type:
# FIXME: Implement Accept best match algorithm when needed
# content_type = self.accept.best_match(
# SUPPORTED_CONTENT_TYPES)
content_type = 'application/json'
self.environ['credsmgr.best_content_type'] = content_type
return self.environ['credsmgr.best_content_type']
def best_match_language(self):
"""Determines best available locale from the Accept-Language header.
:returns: the best language match or None if the 'Accept-Language'
header was not available in the request.
"""
# if not self.accept_language:
# return None
# FIXME: TO be fixed when language support is added
return None
class Application(object):
"""Base WSGI application wrapper. Subclasses need to implement __call__."""
@classmethod
def factory(cls, global_config, **local_config):
"""Used for paste app factories in paste.deploy config files.
Any local configuration (that is, values under the [app:APPNAME]
section of the paste config) will be passed into the `__init__` method
as kwargs.
A hypothetical configuration would look like:
[app:wadl]
latest_version = 1.3
paste.app_factory = credsmgr.api.fancy_api:Wadl.factory
which would result in a call to the `Wadl` class as
import credsmgr.api.fancy_api
fancy_api.Wadl(latest_version='1.3')
You could of course re-implement the `factory` method in subclasses,
but using the kwarg passing it shouldn't be necessary.
"""
return cls(**local_config)
def __call__(self, environ, start_response):
r"""Subclasses will probably want to implement __call__ like this:
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, req):
# Any of the following objects work as responses:
# Option 1: simple string
res = 'message\n'
# Option 2: a nicely formatted HTTP exception page
res = exc.HTTPForbidden(explanation='Nice try')
# Option 3: a webob Response object (in case you need to play with
# headers, or you want to be treated like an iterable)
res = Response();
res.app_iter = open('somefile')
# Option 4: any wsgi app to be run next
res = self.application
# Option 5: you can get a Response object for a wsgi app, too, to
# play with headers etc
res = req.get_response(self.application)
# You can then just return your response...
return res
# ... or set req.response and return None.
req.response = res
See the end of http://pythonpaste.org/webob/modules/dec.html
for more info.
"""
raise NotImplementedError('You must implement __call__')
class Middleware(Application):
"""Base WSGI middleware.
These classes require an application to be
initialized that will be called next. By default the middleware will
simply call its wrapped app, or you can override __call__ to customize its
behavior.
"""
@classmethod
def factory(cls, global_config, **local_config):
"""Used for paste app factories in paste.deploy config files.
Any local configuration (that is, values under the [filter:APPNAME]
section of the paste config) will be passed into the `__init__` method
as kwargs.
A hypothetical configuration would look like:
[filter:analytics]
redis_host = 127.0.0.1
paste.filter_factory = credsmgr.api.analytics:Analytics.factory
which would result in a call to the `Analytics` class as
import credsmgr.api.analytics
analytics.Analytics(app_from_paste, redis_host='127.0.0.1')
You could of course re-implement the `factory` method in subclasses,
but using the kwarg passing it shouldn't be necessary.
"""
def _factory(app):
return cls(app, **local_config)
return _factory
def __init__(self, application):
self.application = application
def process_request(self, req):
"""Called on each request.
If this returns None, the next application down the stack will be
executed. If it returns a response then that response will be returned
and execution will stop here.
"""
return None
def process_response(self, response):
"""Do whatever you'd like to the response."""
return response
@webob.dec.wsgify(RequestClass=Request)
def __call__(self, req):
response = self.process_request(req)
if response:
return response
response = req.get_response(self.application)
return self.process_response(response)

26
creds_manager/etc/credsmgr/api-paste.ini Normal file
View File

@ -0,0 +1,26 @@
[pipeline:credsmgr_api]
pipeline = request_id authtoken context rootapp
;pipeline = cors request_id http_proxy_to_wsgi versionnegotiation faultwrap authtoken context rootapp
[composite:rootapp]
use = call:credsmgr.api.app:root_app_factory
/v1/credentials: credsmgr_api_v1
[app:credsmgr_api_v1]
paste.app_factory = credsmgr.api.controllers.v1.router:APIRouter.factory
[filter:cors]
paste.filter_factory = oslo_middleware.cors:filter_factory
oslo_config_project = credsmgr
[filter:request_id]
paste.filter_factory = oslo_middleware.request_id:RequestId.factory
[filter:http_proxy_to_wsgi]
paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory
[filter:authtoken]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
[filter:context]
paste.filter_factory = credsmgr.api.middleware.context:ContextMiddleware.factory

19
creds_manager/etc/credsmgr/credsmgr.conf Normal file
View File

@ -0,0 +1,19 @@
[DEFAULT]
credsmgr_api_listen_port = 8091
credsmgr_api_use_ssl = False
credsmgr_api_workers = 1
[keystone_authtoken]
auth_uri = http://localhost:8080/keystone/v3
auth_url = http://localhost:8080/keystone_admin
auth_version = v3
auth_type = password
project_domain_name = default
user_domain_name = default
project_name = services
username = credsmgr
password = credsmgr
region_name = RegionOne
[database]
connection = mysql+pymysql://credsmgr:credsmgr@localhost/credsmgr

3
creds_manager/etc/credsmgr/policy.json Normal file
View File

@ -0,0 +1,3 @@
{
}

10
creds_manager/etc/logrotate.d/credsmanager Normal file
View File

@ -0,0 +1,10 @@
/var/log/credsmgr/*.log {
daily
rotate 10
missingok
compress
delaycompress
notifempty
minsize 100k
copytruncate
}

4
creds_manager/etc/rsyslog.d/credsmgr.conf Normal file
View File

@ -0,0 +1,4 @@
if $programname == 'credsmgr_api' then {
/var/log/credsmgr/credsmgr-api.log
~
}

20
creds_manager/requirements.txt Normal file
View File

@ -0,0 +1,20 @@
pbr
enum34
eventlet
keystoneauth1
keystonemiddleware
greenlet
MySQL-python
pymysql
oslo.config
oslo.concurrency
oslo.db
oslo.log
oslo.messaging
oslo.middleware
oslo.policy
oslo.service
SQLAlchemy
sqlalchemy-migrate
webob
cryptography

59
creds_manager/setup.cfg Normal file
View File

@ -0,0 +1,59 @@
[metadata]
name = credsmgr
summary = OpenStack Credentials Manager for Omni
description-file =
README.md
author = Platform9
author-email = info@platform9.com
home-page = http://www.platform9.com
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
[global]
setup-hooks =
pbr.hooks.setup_hook
[files]
packages =
credsmgr
[entry_points]
oslo.config.opts =
credsmgr = credsmgr.opts:list_opts
console_scripts =
credsmgr-api = credsmgr.cmd.api:main
credsmgr-manage = credsmgr.cmd.manage:main
credsmgr.database.migration_backend =
sqlalchemy = oslo_db.sqlalchemy.migration
[build_sphinx]
all_files = 1
build-dir = doc/build
source-dir = doc/source
[egg_info]
tag_build =
tag_date = 0
tag_svn_revision = 0
[compile_catalog]
directory = credsmgr/locale
domain = credsmgr credsmgr-log-error credsmgr-log-info credsmgr-log-warning
[update_catalog]
domain = credsmgr
output_dir = credsmgr/locale
input_file = credsmgr/locale/credsmgr.pot
[extract_messages]
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = credsmgr/locale/credsmgr.pot

29
creds_manager/setup.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright (c) 2017 Platform9 Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
# In python < 2.7.4, a lazy loading of package `pbr` will break
# setuptools if some other modules registered functions in `atexit`.
# solution from: http://bugs.python.org/issue15881#msg170215
try:
import multiprocessing # noqa
except ImportError:
pass
setuptools.setup(
setup_requires=['pbr>=1.8'],
pbr=True)

27
creds_manager/test-requirements.txt Normal file
View File

@ -0,0 +1,27 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
# Install bounded pep8/pyflakes first, then let flake8 install
hacking<0.11,>=0.10.0
anyjson>=0.3.3 # BSD
coverage>=3.6 # Apache-2.0
ddt>=1.0.1 # MIT
fixtures>=3.0.0 # Apache-2.0/BSD
mock>=2.0 # BSD
mox3>=0.7.0 # Apache-2.0
os-api-ref>=1.0.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
python-subunit>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
testrepository>=0.0.18 # Apache-2.0/BSD
testresources>=0.2.4 # Apache-2.0/BSD
testscenarios>=0.4 # Apache-2.0/BSD
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
os-testr>=0.7.0 # Apache-2.0
tempest-lib>=0.14.0 # Apache-2.0
bandit>=1.1.0 # Apache-2.0
reno>=1.8.0 # Apache2

6
creds_manager/tools/pretty_tox.sh Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -o pipefail
TESTRARGS=$1
python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | subunit-trace -f

28
creds_manager/tox.ini Normal file
View File

@ -0,0 +1,28 @@
[tox]
minversion = 2.0
envlist = py27
skipsdist = True
[testenv]
setenv = VIRTUAL_ENV={envdir}
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
PYTHONHASHSEED=0
usedevelop = True
install_command =
pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/newton} {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
cp -r {toxinidir}/../credsmgrclient {envdir}/lib/python2.7/site-packages/
find . -type f -name "*.pyc" -delete
bash tools/pretty_tox.sh '{posargs}'
whitelist_externals =
bash
find
cp
passenv = *_proxy *_PROXY
[testenv:venv]
commands = {posargs}

0
credsmgrclient/__init__.py Normal file
View File

23
credsmgrclient/client.py Normal file
View File

@ -0,0 +1,23 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from oslo_utils import importutils
def Client(endpoint, version=1, **kwargs):
"""Client for the OpenStack Credential manager API."""
module_string = '.'.join(('credsmgrclient', 'v%s' % int(version),
'client'))
module = importutils.import_module(module_string)
client_class = getattr(module, 'Client')
return client_class(endpoint, **kwargs)

0
credsmgrclient/common/__init__.py Normal file
View File

36
credsmgrclient/common/constants.py Normal file
View File

@ -0,0 +1,36 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
AWS = 'aws'
GCE = 'gce'
AZURE = 'azure'
VMWARE = 'vmware'
provider_values = {
AWS: {
'supported_values': ['aws_access_key_id', 'aws_secret_access_key'],
'encrypted_values': ['aws_secret_access_key']
},
AZURE: {
'supported_values': ['tenant_id', 'client_id', 'client_secret',
'subscription_id'],
'encrypted_values': ['client_secret']
},
GCE: {
'supported_values': ['b64_key'],
'encrypted_values': ['b64_key']
},
VMWARE: {
'supported_values': ['host_username', 'host_password'],
'encrypted_values': ['host_password']
}
}

120
credsmgrclient/common/exceptions.py Normal file
View File

@ -0,0 +1,120 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import sys
import six
class _BaseException(Exception):
"""An error occurred."""
def __init__(self, message=None):
super(_BaseException, self).__init__()
self.message = message
class InvalidEndpoint(_BaseException):
"""The provided endpoint is invalid."""
class InvalidToken(_BaseException):
"""Provided token is invalid or token not provided"""
class CommunicationError(_BaseException):
"""Unable to communicate with server."""
class InvalidJson(_BaseException):
"Provided JSON is invalid"
class HTTPException(Exception):
"""Base exception for all HTTP-derived exceptions."""
code = 'N/A'
def __init__(self, details=None):
super(HTTPException, self).__init__()
self.details = details or self.__class__.__name__
def __str__(self):
return "%s (HTTP %s)" % (self.details, self.code)
class HTTPBadRequest(HTTPException):
code = 400
class HTTPUnauthorized(HTTPException):
code = 401
class HTTPForbidden(HTTPException):
code = 403
class HTTPNotFound(HTTPException):
code = 404
class HTTPMethodNotAllowed(HTTPException):
code = 405
class HTTPConflict(HTTPException):
code = 409
class HTTPOverLimit(HTTPException):
code = 413
class HTTPInternalServerError(HTTPException):
code = 500
class HTTPNotImplemented(HTTPException):
code = 501
class HTTPBadGateway(HTTPException):
code = 502
class HTTPServiceUnavailable(HTTPException):
code = 503
_code_map = {}
for obj_name in dir(sys.modules[__name__]):
if obj_name.startswith('HTTP'):
obj = getattr(sys.modules[__name__], obj_name)
_code_map[obj.code] = obj
def from_response(response, body=None):
"""Return an instance of an HTTPException based on httplib response."""
cls = _code_map.get(response.status_code, HTTPException)
if body and 'json' in response.headers['content-type']:
# Iterate over the nested objects and retrieve the "message" attribute.
messages = [obj.get('message') for obj in response.json().values()]
# Join all of the messages together nicely and filter out any objects
# that don't have a "message" attr.
details = '\n'.join(i for i in messages if i is not None)
return cls(details=details)
elif body:
if six.PY3:
body = body.decode('utf-8')
details = body.replace('\n\n', '\n')
return cls(details=details)
return cls()

228
credsmgrclient/common/http.py Normal file
View File

@ -0,0 +1,228 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import copy
import json
import logging
from keystoneauth1 import adapter
from keystoneauth1 import exceptions as ksa_exc
from oslo_utils import encodeutils
from oslo_utils import netutils
import requests
import six
from credsmgrclient.common import exceptions
from credsmgrclient.common import utils
LOG = logging.getLogger(__name__)
USER_AGENT = 'python-credentialclient'
def encode_headers(headers):
"""Encodes headers.
:param headers: Headers to encode
:returns: Dictionary with encoded headers names and values
"""
return dict((encodeutils.safe_encode(h), encodeutils.safe_encode(v))
for h, v in headers.items() if v is not None)
class _BaseHTTPClient(object):
def _set_common_request_kwargs(self, headers, kwargs):
"""Handle the common parameters used to send the request."""
# Default Content-Type is json
content_type = headers.get('Content-Type', 'application/json')
if 'data' in kwargs:
data = json.dumps(kwargs.pop("data"))
else:
data = {}
headers['Content-Type'] = content_type
kwargs['stream'] = False
return data
def _handle_response(self, resp):
if not resp.ok:
LOG.debug("Request returned failure status %s.", resp.status_code)
raise exceptions.from_response(resp, resp.content)
content_type = resp.headers.get('Content-Type')
content = resp.text
if content_type and content_type.startswith('application/json'):
body_iter = resp.json()
else:
body_iter = six.StringIO(content)
try:
body_iter = json.loads(''.join([c for c in body_iter]))
except ValueError:
body_iter = None
return resp, body_iter
class HTTPClient(_BaseHTTPClient):
def __init__(self, base_url, **kwargs):
self.base_url = base_url
self.identity_headers = kwargs.get('identity_headers')
self.auth_token = kwargs.get('token')
if self.identity_headers:
self.auth_token = self.identity_headers.pop('X-Auth-Token',
self.auth_token)
self.session = requests.Session()
self.session.headers["User-Agent"] = USER_AGENT
self.timeout = float(kwargs.get('timeout', 600))
if self.base_url.startswith("https"):
if kwargs.get('insecure', False) is True:
self.session.verify = False
else:
if kwargs.get('cacert', None) is not None:
self.session.verify = kwargs.get('cacert', True)
self.session.cert = (kwargs.get('cert_file'),
kwargs.get('key_file'))
@staticmethod
def parse_endpoint(endpoint):
return netutils.urlsplit(endpoint)
def log_curl_request(self, method, url, headers, data):
curl = ['curl -i -X %s' % method]
headers = copy.deepcopy(headers)
headers.update(self.session.headers)
for (key, value) in headers.items():
header = "-H '%s: %s'" % (key, value)
curl.append(header)
if not self.session.verify:
curl.append('-k')
else:
if isinstance(self.session.verify, six.string_types):
curl.append('--cacert %s' % self.session.verify)
if self.session.cert:
curl.append('--cert %s --key %s' % self.session.cert)
if data and isinstance(data, six.string_types):
curl.append("-d '%s'" % data)
curl.append(url)
msg = ' '.join([encodeutils.safe_decode(item, errors='ignore')
for item in curl])
LOG.debug(msg)
@staticmethod
def log_http_response(resp):
status = (resp.raw.version / 10.0, resp.status_code, resp.reason)
dump = ['\nHTTP/%.1f %s %s' % status]
headers = resp.headers.items()
dump.extend(['%s: %s' % utils.safe_header(k, v) for k, v in headers])
dump.append('')
dump.extend([resp.text, ''])
LOG.debug('\n'.join([encodeutils.safe_decode(x, errors='ignore')
for x in dump]))
def _request(self, method, url, **kwargs):
"""Send an http request with the specified characteristics.
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
as setting headers and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
headers = copy.deepcopy(kwargs.pop('headers', {}))
if self.identity_headers:
for k, v in self.identity_headers.items():
headers.setdefault(k, v)
data = self._set_common_request_kwargs(headers, kwargs)
# add identity header to the request
if not headers.get('X-Auth-Token'):
headers['X-Auth-Token'] = self.auth_token
headers = encode_headers(headers)
conn_url = "%s%s" % (self.base_url, url)
self.log_curl_request(method, conn_url, headers, data)
try:
resp = self.session.request(method, conn_url, data=data,
headers=headers, **kwargs)
except requests.exceptions.Timeout as e:
message = ("Error communicating with %(url)s: %(e)s" %
dict(url=conn_url, e=e))
raise exceptions.InvalidEndpoint(message=message)
except requests.exceptions.ConnectionError as e:
message = ("Error finding address for %(url)s: %(e)s" %
dict(url=conn_url, e=e))
raise exceptions.CommunicationError(message=message)
request_id = resp.headers.get('x-openstack-request-id')
if request_id:
LOG.debug('%(method)s call to image for %(url)s used request id '
'%(response_request_id)s',
{'method': resp.request.method, 'url': resp.url,
'response_request_id': request_id})
resp, body_iter = self._handle_response(resp)
self.log_http_response(resp)
return resp, body_iter
def get(self, url, **kwargs):
return self._request('GET', url, **kwargs)
def post(self, url, **kwargs):
return self._request('POST', url, **kwargs)
def put(self, url, **kwargs):
return self._request('PUT', url, **kwargs)
def delete(self, url, **kwargs):
return self._request('DELETE', url, **kwargs)
class SessionClient(adapter.Adapter, _BaseHTTPClient):
def __init__(self, session, **kwargs):
kwargs.setdefault('user_agent', USER_AGENT)
kwargs.setdefault('service_type', 'credsmgr')
super(SessionClient, self).__init__(session, **kwargs)
def request(self, url, method, **kwargs):
headers = kwargs.pop('headers', {})
kwargs['raise_exc'] = False
data = self._set_common_request_kwargs(headers, kwargs)
try:
resp = super(SessionClient, self).request(
url, method, headers=encode_headers(headers), data=data,
**kwargs)
except ksa_exc.ConnectTimeout as e:
conn_url = self.get_endpoint(auth=kwargs.get('auth'))
conn_url = "%s/%s" % (conn_url.rstrip('/'), url.lstrip('/'))
message = ("Error communicating with %(url)s %(e)s" %
dict(url=conn_url, e=e))
raise exceptions.InvalidEndpoint(message=message)
except ksa_exc.ConnectFailure as e:
conn_url = self.get_endpoint(auth=kwargs.get('auth'))
conn_url = "%s/%s" % (conn_url.rstrip('/'), url.lstrip('/'))
message = ("Error finding address for %(url)s: %(e)s" %
dict(url=conn_url, e=e))
raise exceptions.CommunicationError(message=message)
return self._handle_response(resp)
def get_http_client(endpoint=None, session=None, **kwargs):
if session:
return SessionClient(session, **kwargs)
elif endpoint:
return HTTPClient(endpoint, **kwargs)
else:
raise AttributeError('Constructing a client must contain either an '
'endpoint or a session')

24
credsmgrclient/common/utils.py Normal file
View File

@ -0,0 +1,24 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import hashlib
SENSITIVE_HEADERS = ('X-Auth-Token', )
def safe_header(name, value):
if value is not None and name in SENSITIVE_HEADERS:
h = hashlib.sha1(value)
d = h.hexdigest()
return name, "{SHA1}%s" % d
return name, value

31
credsmgrclient/encrypt.py Normal file
View File

@ -0,0 +1,31 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import importutils
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
no_encryption = 'credsmgrclient.encryption.noop.NoEncryption'
encryptor_opts = [
cfg.StrOpt('encryptor', help='Encryption driver',
default=no_encryption),
]
CONF.register_opts(encryptor_opts, group='credsmgr')
try:
ENCRYPTOR = importutils.import_object(CONF.credsmgr.encryptor)
except ImportError:
LOG.error('Could not load encryption class: %s' % CONF.credsmgr.encryptor)
raise

0
credsmgrclient/encryption/__init__.py Normal file
View File

27
credsmgrclient/encryption/base.py Normal file
View File

@ -0,0 +1,27 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from abc import ABCMeta
from abc import abstractmethod
from six import add_metaclass
@add_metaclass(ABCMeta)
class Encryptor(object):
@abstractmethod
def encrypt(self, data):
pass
@abstractmethod
def decrypt(self, data):
pass

63
credsmgrclient/encryption/fernet.py Normal file
View File

@ -0,0 +1,63 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import base64
import six
from credsmgrclient.encryption import base
from cryptography.fernet import Fernet
from cryptography.fernet import InvalidToken
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
from oslo_config import cfg
from oslo_log import log as logging
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
encrypt_opts = [
cfg.StrOpt('fernet_salt', help='Salt to be used for generating fernet key.'
'Should be 16 bytes', required=True),
cfg.StrOpt('fernet_password', help='Password to be used for generating'
'fernet key', required=True),
cfg.IntOpt('iterations', help='Number of iterations for generating key'
'from password and salt',
default=100000)
]
CONF.register_opts(encrypt_opts, group='credsmgr')
class FernetKeyEncryption(base.Encryptor):
def __init__(self):
fernet_password = CONF.credsmgr.fernet_password
fernet_salt = CONF.credsmgr.fernet_salt
iterations = CONF.credsmgr.iterations
kdf = PBKDF2HMAC(algorithm=hashes.SHA512(), length=32,
salt=fernet_salt, iterations=iterations,
backend=default_backend())
key = base64.urlsafe_b64encode(kdf.derive(fernet_password))
self.fernet_key = Fernet(key)
def encrypt(self, data):
if isinstance(data, six.types.UnicodeType):
data = data.encode('utf-8')
return self.fernet_key.encrypt(data)
def decrypt(self, data):
if isinstance(data, six.types.UnicodeType):
data = data.encode('utf-8')
try:
return self.fernet_key.decrypt(data)
except InvalidToken:
return data

26
credsmgrclient/encryption/noop.py Normal file
View File

@ -0,0 +1,26 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from credsmgrclient.encryption import base
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class NoEncryption(base.Encryptor):
def encrypt(self, data):
LOG.warn('Data will be stored without encryption')
return data
def decrypt(self, data):
return data

14
credsmgrclient/v1/__init__.py Normal file
View File

@ -0,0 +1,14 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from credsmgrclient.v1.client import Client # noqa

36
credsmgrclient/v1/client.py Normal file
View File

@ -0,0 +1,36 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import six
from credsmgrclient.common import exceptions
from credsmgrclient.common import http
from credsmgrclient.v1 import credentials
class Client(object):
"""Client for the OpenStack Credential Manager v1 API.
:param string endpoint: A user-supplied endpoint URL for the glance
service.
:param string token: Token for authentication.
:param integer timeout: Allows customization of the timeout for client
http requests. (optional)
"""
def __init__(self, endpoint, **kwargs):
"""Initialize a new client for the Images v1 API."""
if not isinstance(endpoint, six.string_types):
raise exceptions.InvalidEndpoint("Endpoint must be a string")
base_url = endpoint + "/v1/credentials"
self.http_client = http.get_http_client(base_url, **kwargs)
self.credentials = credentials.CredentialManager(self.http_client)

129
credsmgrclient/v1/credentials.py Normal file
View File

@ -0,0 +1,129 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import logging
from credsmgrclient.common.constants import provider_values
from credsmgrclient.encrypt import ENCRYPTOR
LOG = logging.getLogger(__name__)
def _get_encrypted_values(provider):
try:
return provider_values[provider]['encrypted_values']
except KeyError:
raise Exception("Provider %s is not valid" % provider)
def _decrypt_creds(creds, encrypted_values):
for k, v in creds.items():
if k in encrypted_values:
creds[k] = ENCRYPTOR.decrypt(v)
class CredentialManager(object):
def __init__(self, http_client):
self.client = http_client
def credentials_get(self, provider, tenant_id):
"""Get the information about Credentials.
:param provider: Name of Omni provider
:type: str
:param tenant_id: tenant id to look up
:type: str
:rtype: dict
"""
resp, body = self.client.get("/%s" % provider,
data={"tenant_id": tenant_id})
LOG.debug("Get Credentials response: {0}, body: {1}".format(
resp, body))
if body:
encrypted_values = _get_encrypted_values(provider)
_decrypt_creds(body, encrypted_values)
return resp, body
def credentials_list(self, provider):
"""Get the information about Credentials for all tenants.
:param provider: Name of Omni provider
:type: str
:rtype: dict
"""
resp, body = self.client.get("/%s/list" % provider)
LOG.debug("Get Credentials list response: {0}, body: {1}".format(
resp, body))
if body:
encrypted_values = _get_encrypted_values(provider)
for creds in body.values():
_decrypt_creds(creds, encrypted_values)
return resp, body
def credentials_create(self, provider, **kwargs):
"""Create a credential.
:param provider: Name of Omni provider
:type: str
:param body: Credentials for Omni provider
:type: dict
:rtype: dict
"""
resp, body = self.client.post("/%s" % provider,
data=kwargs.get('body'))
LOG.debug("Post Credentials response: {0}, body: {1}".format(resp,
body))
return resp, body
def credentials_delete(self, provider, credential_id):
"""Delete a credential.
:param provider: Name of Omni provider
:type: str
:param credential_id: ID for credential
:type: str
"""
resp, body = self.client.delete("/%s/%s" % (provider, credential_id))
LOG.debug("Delete Credentials response: {0}, body: {1}".format(
resp, body))
def credentials_update(self, provider, credential_id, **kwargs):
"""Update credential.
:param provider: Name of Omni provider
:type: str
:param credential_id: ID for credential
:type: str
"""
resp, body = self.client.put("/%s/%s" % (provider, credential_id),
data=kwargs.get('body'))
LOG.debug("Update Credentials response: {0}, body: {1}".format(
resp, body))
return resp, body
def credentials_association_create(self, provider, credential_id,
**kwargs):
resp, body = self.client.post(
"/%s/%s/association" % (provider, credential_id),
data=kwargs.get('body'))
LOG.debug("Create Association response: {0}, body: {1}".format(
resp, body))
def credentials_association_delete(self, provider, credential_id,
tenant_id):
resp, body = self.client.delete(
"/%s/%s/association/%s" % (provider, credential_id, tenant_id))
LOG.debug("Delete Association response: {0}, body: {1}".format(
resp, body))
def credentials_association_list(self, provider):
resp, body = self.client.get("/%s/associations" % provider)
LOG.debug("List associations response: {0}, body: {1}".format(
resp, body))
return resp, body

100
glance/glance_store/_drivers/aws.py
View File

@ -10,20 +10,25 @@ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
import hashlib
import logging
import uuid
import glance.registry.client.v1.api as registry
from glance_store import capabilities
import glance_store.driver
from glance_store import exceptions
from glance_store.i18n import _
import glance_store.location
from oslo_config import cfg
from oslo_utils import units
from six.moves import urllib
import boto3
import botocore.exceptions
from glance_store._drivers import awsutils
LOG = logging.getLogger(__name__)
MAX_REDIRECTS = 5
@ -34,6 +39,17 @@ aws_opts = [cfg.StrOpt('access_key', help='AWS access key ID'),
cfg.StrOpt('secret_key', help='AWS secret access key'),
cfg.StrOpt('region_name', help='AWS region name')]
keystone_opts_group = cfg.OptGroup(
name='keystone_credentials', title='Keystone credentials')
keystone_opts = [cfg.StrOpt('region_name', help='Keystone region name'), ]
def _get_image_uuid(ami_id):
md = hashlib.md5()
md.update(ami_id)
return str(uuid.UUID(bytes=md.digest()))
class StoreLocation(glance_store.location.StoreLocation):
@ -67,6 +83,7 @@ class StoreLocation(glance_store.location.StoreLocation):
LOG.info(_("No image ami_id specified in URL"))
raise exceptions.BadStoreUri(uri=uri)
self.ami_id = ami_id
self.image_id = pieces.path.strip('/')
class Store(glance_store.driver.Store):
@ -80,22 +97,20 @@ class Store(glance_store.driver.Store):
super(Store, self).__init__(conf)
conf.register_group(aws_opts_group)
conf.register_opts(aws_opts, group=aws_opts_group)
self.credentials = {}
self.credentials['aws_access_key_id'] = conf.aws.access_key
self.credentials['aws_secret_access_key'] = conf.aws.secret_key
self.credentials['region_name'] = conf.aws.region_name
self.__ec2_client = None
self.__ec2_resource = None
conf.register_group(keystone_opts_group)
conf.register_opts(keystone_opts, group=keystone_opts_group)
self.conf = conf
self.region_name = conf.aws.region_name
def _get_ec2_client(self):
if self.__ec2_client is None:
self.__ec2_client = boto3.client('ec2', **self.credentials)
return self.__ec2_client
def _get_ec2_client(self, context, tenant):
creds = awsutils.get_credentials(context, tenant, conf=self.conf)
creds['region_name'] = self.region_name
return boto3.client('ec2', **creds)
def _get_ec2_resource(self):
if self.__ec2_resource is None:
self.__ec2_resource = boto3.resource('ec2', **self.credentials)
return self.__ec2_resource
def _get_ec2_resource(self, context, tenant):
creds = awsutils.get_credentials(context, tenant, conf=self.conf)
creds['region_name'] = self.region_name
return boto3.resource('ec2', **creds)
@capabilities.check
def get(self, location, offset=0, chunk_size=None, context=None):
@ -118,8 +133,11 @@ class Store(glance_store.driver.Store):
from glance_store.location.get_location_from_uri()
:raises NotFound if image does not exist
"""
ami_id = location.get_store_uri().split('/')[2]
aws_client = self._get_ec2_client()
ami_id = location.store_location.ami_id
image_id = location.store_location.image_id
image_info = registry.get_image_metadata(context, image_id)
project_id = image_info['owner']
aws_client = self._get_ec2_client(context, project_id)
aws_imgs = aws_client.describe_images(Owners=['self'])['Images']
for img in aws_imgs:
if ami_id == img.get('ImageId'):
@ -133,6 +151,33 @@ class Store(glance_store.driver.Store):
"""
return ('aws',)
def _get_size_from_properties(self, image_info):
"""
:param image_info dict object, supplied from
registry.get_image_metadata
:retval int: size of image in bytes or -1 if size could not be fetched
from image properties alone
"""
img_size = -1
if 'properties' in image_info:
img_props = image_info['properties']
if img_props.get('aws_root_device_type') == 'ebs' and \
'aws_ebs_vol_sizes' in img_props:
ebs_vol_size_str = img_props['aws_ebs_vol_sizes']
img_size = 0
# sizes are stored as string - "[8, 16]"
# Convert it to array of int
ebs_vol_sizes = [int(vol.strip()) for vol in
ebs_vol_size_str.replace('[', '').
replace(']', '').split(',')]
for vol_size in ebs_vol_sizes:
img_size += vol_size
elif img_props.get('aws_root_device_type') != 'ebs':
istore_vols = int(img_props.get('aws_num_istore_vols', '0'))
if istore_vols >= 1:
img_size = 0
return img_size
def get_size(self, location, context=None):
"""
Takes a `glance_store.location.Location` object that indicates
@ -142,20 +187,31 @@ class Store(glance_store.driver.Store):
from glance_store.location.get_location_from_uri()
:retval int: size of image file in bytes
"""
ami_id = location.get_store_uri().split('/')[2]
ec2_resource = self._get_ec2_resource()
ami_id = location.store_location.ami_id
image_id = location.store_location.image_id
image_info = registry.get_image_metadata(context, image_id)
project_id = image_info['owner']
ec2_resource = self._get_ec2_resource(context, project_id)
image = ec2_resource.Image(ami_id)
size = 0
size = self._get_size_from_properties(image_info)
if size >= 0:
LOG.debug('Got image size from properties as %d' % size)
# Convert size in gb to bytes
size *= units.Gi
return size
try:
image.load()
# no size info for instance-store volumes, so return 0 in that case
# no size info for instance-store volumes, so return 1 in that case
# Setting size as 0 fails multiple checks in glance required for
# successful creation of image record.
size = 1
if image.root_device_type == 'ebs':
for bdm in image.block_device_mappings:
if 'Ebs' in bdm and 'VolumeSize' in bdm['Ebs']:
LOG.debug('ebs info: %s' % bdm['Ebs'])
size += bdm['Ebs']['VolumeSize']
# convert size in gb to bytes
size *= 1073741824
size *= units.Gi
except botocore.exceptions.ClientError as ce:
if ce.response['Error']['Code'] == 'InvalidAMIID.NotFound':
raise exceptions.ImageDataNotFound()

62
glance/glance_store/_drivers/awsutils.py Normal file
View File

@ -0,0 +1,62 @@
"""
Copyright 2017 Platform9 Systems Inc.(http://www.platform9.com)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from keystoneauth1.access import service_catalog
from keystoneauth1.exceptions import EndpointNotFound
from credsmgrclient.client import Client
from credsmgrclient.common import exceptions as credsmgr_ex
from glance_store import exceptions as glance_ex
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class AwsCredentialsNotFound(glance_ex.GlanceStoreException):
message = "Aws credentials could not be found"
def get_credentials_from_conf(conf):
secret_key = conf.aws.secret_key
access_key = conf.aws.access_key
if not access_key or not secret_key:
raise AwsCredentialsNotFound()
return dict(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key
)
def get_credentials(context, tenant, conf=None):
# TODO(ssudake21): Add caching support
# 1. Cache keystone endpoint
# 2. Cache recently used AWS credentials
try:
if context is None or tenant is None:
raise glance_ex.AuthorizationFailure()
sc = service_catalog.ServiceCatalogV2(context.service_catalog)
region_name = conf.keystone_credentials.region_name
credsmgr_endpoint = sc.url_for(
service_type='credsmgr', region_name=region_name)
token = context.auth_token
credsmgr_client = Client(credsmgr_endpoint, token=token)
resp, body = credsmgr_client.credentials.credentials_get(
'aws', tenant)
except (EndpointNotFound, credsmgr_ex.HTTPBadGateway,
credsmgr_ex.HTTPNotFound):
if conf is not None:
return get_credentials_from_conf(conf)
raise AwsCredentialsNotFound()
return body

911
neutron/neutron/common/aws_utils.py
View File

File diff suppressed because it is too large Load Diff

1
neutron/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD Normal file
View File

@ -0,0 +1 @@
f14ac1703ee2

50
neutron/neutron/db/migration/alembic_migrations/versions/pike/expand/f14ac1703ee2_add_omni_resource_mapping.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2018 OpenStack Foundation
# Copyright 2018 Platform9 Systems Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Add Omni resource mapping
Revision ID: f14ac1703ee2
Revises: 7d32f979895f
Create Date: 2018-09-04 21:04:41.357943
"""
# revision identifiers, used by Alembic.
revision = 'f14ac1703ee2'
down_revision = '7d32f979895f'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
omni_resources_table = 'omni_resource_map'
def MediumText():
return sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql')
def upgrade():
op.create_table(
omni_resources_table,
sa.Column('openstack_id',
sa.String(length=36),
nullable=False,
primary_key=True),
sa.Column('omni_resource',
MediumText(),
nullable=False),
)

41
neutron/neutron/db/models/omni_resources.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2018 OpenStack Foundation
# Copyright 2018 Platform9 Systems Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from neutron_lib.db import model_base
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def MediumText():
return sa.Text().with_variant(mysql.MEDIUMTEXT(), 'mysql')
class OmniResources(model_base.BASEV2):
__tablename__ = 'omni_resource_map'
openstack_id = sa.Column(sa.String(36), nullable=False, primary_key=True)
# Omni resource field is set to MEDIUMTEXT because so that it can be used
# to store larger information e.g. security groups along with group rules
# for different VPCs. For networks and subnets the table will simply be a
# mapping from OpenStack ID to public cloud ID.
omni_resource = sa.Column(MediumText(), nullable=False)
def __repr__(self):
return "<%s(%s, %s)>" % (self.__class__.__name__, self.openstack_id,
self.omni_resource)
def __init__(self, openstack_id, omni_resource):
self.openstack_id = openstack_id
self.omni_resource = omni_resource

57
neutron/neutron/db/omni_resources.py Normal file
View File

@ -0,0 +1,57 @@
# Copyright 2018 OpenStack Foundation
# Copyright 2018 Platform9 Systems Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from neutron.db import api as db_api
from neutron.db.models import omni_resources
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
def add_mapping(openstack_id, omni_resource):
LOG.debug('Adding mapping as - %s --> %s', openstack_id, omni_resource)
session = db_api.get_session()
with db_api.autonested_transaction(session) as tx:
check_existing = tx.session.query(omni_resources.OmniResources).\
filter_by(openstack_id=openstack_id).first()
if check_existing:
LOG.info('Updating to add %s-%s since already present',
openstack_id, omni_resource)
check_existing.omni_resource = omni_resource
tx.session.flush()
else:
mapping = omni_resources.OmniResources(openstack_id, omni_resource)
tx.session.add(mapping)
def get_omni_resource(openstack_id):
session = db_api.get_reader_session()
result = session.query(omni_resources.OmniResources).filter_by(
openstack_id=openstack_id).first()
if not result:
return None
return result.omni_resource
def delete_mapping(openstack_id):
LOG.debug('Deleting mapping for - %s', openstack_id)
session = db_api.get_session()
with db_api.autonested_transaction(session) as tx:
mapping = tx.session.query(omni_resources.OmniResources).filter_by(
openstack_id=openstack_id).first()
if mapping:
tx.session.delete(mapping)

59
neutron/neutron/extensions/subnet_availability_zone.py Normal file
View File

@ -0,0 +1,59 @@
"""
Copyright 2018 Platform9 Systems Inc.(http://www.platform9.com).
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
"""
from neutron_lib.api import extensions
from neutron.extensions import availability_zone as az_ext
EXTENDED_ATTRIBUTES_2_0 = {
'subnets': {
az_ext.RESOURCE_NAME: {
'allow_post': True, 'allow_put': False, 'is_visible': True,
'default': None}},
}
class Subnet_availability_zone(extensions.ExtensionDescriptor):
"""Subnet availability zone extension."""
@classmethod
def get_name(cls):
"""Get name of extension."""
return "Subnet Availability Zone"
@classmethod
def get_alias(cls):
"""Get alias of extension."""
return "subnet_availability_zone"
@classmethod
def get_description(cls):
"""Get description of extension."""
return "Availability zone support for subnet."
@classmethod
def get_updated(cls):
"""Get updated date of extension."""
return "2018-08-10T10:00:00-00:00"
def get_required_extensions(self):
"""Get list of required extensions."""
return ["availability_zone"]
def get_extended_resources(self, version):
"""Get extended resources for extension."""
if version == "2.0":
return EXTENDED_ATTRIBUTES_2_0
else:
return {}

2
neutron/neutron/plugins/ml2/drivers/aws/callbacks.py
View File

@ -24,6 +24,8 @@ def subscribe(mech_driver):
events.BEFORE_DELETE)
registry.subscribe(mech_driver.secgroup_callback, resources.SECURITY_GROUP,
events.BEFORE_UPDATE)
registry.subscribe(mech_driver.secgroup_callback, resources.SECURITY_GROUP,
events.AFTER_CREATE)
registry.subscribe(mech_driver.secgroup_callback,
resources.SECURITY_GROUP_RULE, events.BEFORE_DELETE)
registry.subscribe(mech_driver.secgroup_callback,

310
neutron/neutron/plugins/ml2/drivers/aws/mechanism_aws.py
View File

@ -14,22 +14,50 @@ under the License.
import json
import random
import requests
import six
from neutron.callbacks import events
from neutron.callbacks import resources
from neutron.common.aws_utils import AwsException
from neutron.common.aws_utils import AwsUtils
from neutron import manager
from neutron.db import omni_resources
from neutron.plugins.ml2 import driver_api as api
from neutron.plugins.ml2.drivers.aws import callbacks
from neutron_lib import exceptions
from neutron_lib.plugins import directory
from oslo_config import cfg
from oslo_log import log
LOG = log.getLogger(__name__)
AZ = 'availability_zone'
AZ_HINT = 'availability_zone_hints'
class NetworkWithMultipleAZs(exceptions.NeutronException):
message = "Network shouldn't have more than one availability zone"
class AzNotProvided(exceptions.NeutronException):
"""Raise exception if AZ is not provided in subnet or network."""
message = "No AZ provided either in Subnet or in Network context"
class InvalidAzValue(exceptions.NeutronException):
"""Raise exception if AZ value is incorrect."""
message = ("Invalid AZ value. It should be a single string value and from"
" provided AWS region")
class AwsMechanismDriver(api.MechanismDriver):
"""Ml2 Mechanism driver for AWS"""
def __init__(self):
self.aws_utils = None
self._default_sgr_to_remove = []
super(AwsMechanismDriver, self).__init__()
def initialize(self):
@ -46,9 +74,18 @@ class AwsMechanismDriver(api.MechanismDriver):
def update_network_precommit(self, context):
try:
network_name = context.current['name']
original_network_name = context.original['name']
LOG.debug("Update network original: %s current: %s",
original_network_name, network_name)
if network_name == original_network_name:
return
neutron_network_id = context.current['id']
project_id = context.current['project_id']
tags_list = [{'Key': 'Name', 'Value': network_name}]
self.aws_utils.create_tags_for_vpc(neutron_network_id, tags_list)
self.aws_utils.create_tags_for_vpc(neutron_network_id, tags_list,
context=context._plugin_context,
project_id=project_id)
except Exception as e:
LOG.error("Error in update subnet precommit: %s" % e)
raise e
@ -58,33 +95,53 @@ class AwsMechanismDriver(api.MechanismDriver):
def delete_network_precommit(self, context):
neutron_network_id = context.current['id']
project_id = context.current['project_id']
# If user is deleting an empty neutron network then nothing to be done
# on AWS side
if len(context.current['subnets']) > 0:
vpc_id = self.aws_utils.get_vpc_from_neutron_network_id(
neutron_network_id)
neutron_network_id, context=context._plugin_context,
project_id=project_id)
if vpc_id is not None:
LOG.info("Deleting network %s (VPC_ID: %s)" %
(neutron_network_id, vpc_id))
self.aws_utils.delete_vpc(vpc_id=vpc_id)
try:
self.aws_utils.delete_vpc(vpc_id=vpc_id,
context=context._plugin_context,
project_id=project_id)
except AwsException as e:
if 'InvalidVpcID.NotFound' in e.msg:
LOG.warn(e.msg)
else:
raise e
omni_resources.delete_mapping(context.current['id'])
def delete_network_postcommit(self, context):
pass
# SUBNET
def create_subnet_precommit(self, context):
LOG.info("Create subnet for network %s" %
context.network.current['id'])
network_id = context.network.current['id']
LOG.info("Create subnet for network %s" % network_id)
# External Network doesn't exist on AWS, so no operations permitted
if 'provider:physical_network' in context.network.current:
if context.network.current[
'provider:physical_network'] == "external":
# Do not create subnets for external & provider networks. Only
# allow tenant network subnet creation at the moment.
LOG.info('Creating external network {0}'.format(
context.network.current['id']))
return
physical_network = context.network.current.get(
'provider:physical_network')
if physical_network == "external":
# Do not create subnets for external & provider networks. Only
# allow tenant network subnet creation at the moment.
LOG.info('Creating external network {0}'.format(
network_id))
return
elif physical_network and physical_network.startswith('vpc'):
LOG.info('Registering AWS network with vpc %s',
physical_network)
subnet_cidr = context.current['cidr']
subnet_id = self.aws_utils.get_subnet_from_vpc_and_cidr(
context._plugin_context, physical_network, subnet_cidr,
context.current['project_id'])
omni_resources.add_mapping(network_id, physical_network)
omni_resources.add_mapping(context.current['id'], subnet_id)
return
if context.current['ip_version'] == 6:
raise AwsException(error_code="IPv6Error",
message="Cannot create subnets with IPv6")
@ -96,7 +153,7 @@ class AwsMechanismDriver(api.MechanismDriver):
# Check if this is the first subnet to be added to a network
neutron_network = context.network.current
associated_vpc_id = self.aws_utils.get_vpc_from_neutron_network_id(
neutron_network['id'])
neutron_network['id'], context=context._plugin_context)
if associated_vpc_id is None:
# Need to create EC2 VPC
vpc_cidr = context.current['cidr'][:-2] + '16'
@ -108,7 +165,10 @@ class AwsMechanismDriver(api.MechanismDriver):
'Value': context.current['tenant_id']}
]
associated_vpc_id = self.aws_utils.create_vpc_and_tags(
cidr=vpc_cidr, tags_list=tags)
cidr=vpc_cidr, tags_list=tags,
context=context._plugin_context)
omni_resources.add_mapping(neutron_network['id'],
associated_vpc_id)
# Create Subnet in AWS
tags = [
{'Key': 'Name', 'Value': context.current['name']},
@ -116,13 +176,46 @@ class AwsMechanismDriver(api.MechanismDriver):
{'Key': 'openstack_tenant_id',
'Value': context.current['tenant_id']}
]
self.aws_utils.create_subnet_and_tags(vpc_id=associated_vpc_id,
cidr=context.current['cidr'],
tags_list=tags)
if AZ in context.current and context.current[AZ]:
aws_az = context.current[AZ]
elif context.network.current[AZ_HINT]:
network_az_hints = context.network.current[AZ_HINT]
if len(network_az_hints) > 1:
# We use only one AZ hint even if multiple AZ values
# are passed while creating network.
raise NetworkWithMultipleAZs()
aws_az = network_az_hints[0]
else:
raise AzNotProvided()
self._validate_az(aws_az)
ec2_subnet_id = self.aws_utils.create_subnet_and_tags(
vpc_id=associated_vpc_id, cidr=context.current['cidr'],
tags_list=tags, aws_az=aws_az, context=context._plugin_context)
omni_resources.add_mapping(context.current['id'], ec2_subnet_id)
except Exception as e:
LOG.error("Error in create subnet precommit: %s" % e)
raise e
def _send_request(self, session, url):
headers = {'Content-Type': 'application/json',
'X-Auth-Token': session.get_token()}
response = requests.get(url + "/v1/zones", headers=headers)
response.raise_for_status()
return response.json()
def _validate_az(self, aws_az):
if not isinstance(aws_az, six.string_types):
raise InvalidAzValue()
if ',' in aws_az:
raise NetworkWithMultipleAZs()
session = self.aws_utils.get_keystone_session()
azmgr_url = session.get_endpoint(service_type='azmanager',
region_name=cfg.CONF.nova_region_name)
zones = self._send_request(session, azmgr_url)
if aws_az not in zones:
LOG.error("Provided az %s not found in zones %s", aws_az, zones)
raise InvalidAzValue()
def create_subnet_postcommit(self, context):
pass
@ -131,7 +224,8 @@ class AwsMechanismDriver(api.MechanismDriver):
subnet_name = context.current['name']
neutron_subnet_id = context.current['id']
tags_list = [{'Key': 'Name', 'Value': subnet_name}]
self.aws_utils.create_subnet_tags(neutron_subnet_id, tags_list)
self.aws_utils.create_subnet_tags(neutron_subnet_id, tags_list,
context=context._plugin_context)
except Exception as e:
LOG.error("Error in update subnet precommit: %s" % e)
raise e
@ -140,30 +234,32 @@ class AwsMechanismDriver(api.MechanismDriver):
pass
def delete_subnet_precommit(self, context):
if 'provider:physical_network' in context.network.current:
if context.network.current[
'provider:physical_network'] == "external":
LOG.error("Deleting provider and external networks not "
"supported")
return
try:
LOG.info("Deleting subnet %s" % context.current['id'])
project_id = context.current['project_id']
subnet_id = self.aws_utils.get_subnet_from_neutron_subnet_id(
context.current['id'])
if subnet_id is not None:
self.aws_utils.delete_subnet(subnet_id=subnet_id)
context.current['id'], context=context._plugin_context,
project_id=project_id)
if not subnet_id:
raise Exception("Subnet mapping %s not found" % (
context.current['id']))
try:
self.aws_utils.delete_subnet(
subnet_id=subnet_id, context=context._plugin_context,
project_id=project_id)
omni_resources.delete_mapping(context.current['id'])
except AwsException as e:
if 'InvalidSubnetID.NotFound' in e.msg:
LOG.warn(e.msg)
omni_resources.delete_mapping(context.current['id'])
else:
raise e
except Exception as e:
LOG.error("Error in delete subnet precommit: %s" % e)
raise e
def delete_subnet_postcommit(self, context):
neutron_network = context.network.current
if 'provider:physical_network' in context.network.current and \
context.network.current['provider:physical_network'] == \
"external":
LOG.info('Deleting %s external network' %
context.network.current['id'])
return
try:
subnets = neutron_network['subnets']
if (len(subnets) == 1 and subnets[0] == context.current['id'] or
@ -171,11 +267,19 @@ class AwsMechanismDriver(api.MechanismDriver):
# Last subnet for this network was deleted, so delete VPC
# because VPC gets created during first subnet creation under
# an OpenStack network
project_id = context.current['project_id']
vpc_id = self.aws_utils.get_vpc_from_neutron_network_id(
neutron_network['id'])
neutron_network['id'], context=context._plugin_context,
project_id=project_id)
if not vpc_id:
raise Exception("Network mapping %s not found",
neutron_network['id'])
LOG.info("Deleting VPC %s since this was the last subnet in "
"the vpc" % vpc_id)
self.aws_utils.delete_vpc(vpc_id=vpc_id)
self.aws_utils.delete_vpc(
vpc_id=vpc_id, context=context._plugin_context,
project_id=project_id)
omni_resources.delete_mapping(context.network.current['id'])
except Exception as e:
LOG.error("Error in delete subnet postcommit: %s" % e)
raise e
@ -187,7 +291,20 @@ class AwsMechanismDriver(api.MechanismDriver):
pass
def update_port_precommit(self, context):
pass
original_port = context._original_port
updated_port = context._port
sorted_original_sgs = sorted(original_port['security_groups'])
sorted_updated_sgs = sorted(updated_port['security_groups'])
aws_sgs = []
project_id = context.current['project_id']
if sorted_updated_sgs != sorted_original_sgs:
for sg in updated_port['security_groups']:
aws_secgrps = self.aws_utils.get_sec_group_by_id(
sg, context._plugin_context, project_id=project_id)
aws_sgs.append(aws_secgrps[0]['GroupId'])
if aws_sgs:
self.aws_utils.modify_ports(aws_sgs, updated_port['name'],
context._plugin_context, project_id)
def update_port_postcommit(self, context):
pass
@ -203,20 +320,28 @@ class AwsMechanismDriver(api.MechanismDriver):
if 'fixed_ips' in context.current:
if len(context.current['fixed_ips']) > 0:
fixed_ip_dict = context.current['fixed_ips'][0]
fixed_ip_dict['subnet_id'] = \
openstack_subnet_id = fixed_ip_dict['subnet_id']
aws_subnet_id = \
self.aws_utils.get_subnet_from_neutron_subnet_id(
fixed_ip_dict['subnet_id'])
openstack_subnet_id, context._plugin_context,
project_id=context.current['project_id'])
fixed_ip_dict['subnet_id'] = aws_subnet_id
secgroup_ids = context.current['security_groups']
self.create_security_groups_if_needed(context, secgroup_ids)
ec2_secgroup_ids = self.create_security_groups_if_needed(
context, secgroup_ids)
fixed_ip_dict['ec2_security_groups'] = ec2_secgroup_ids
segment_id = random.choice(context.network.network_segments)[api.ID]
context.set_binding(segment_id, "vip_type_a",
json.dumps(fixed_ip_dict), status='ACTIVE')
return True
def create_security_groups_if_needed(self, context, secgrp_ids):
core_plugin = manager.NeutronManager.get_plugin()
project_id = context.current.get('project_id')
core_plugin = directory.get_plugin()
vpc_id = self.aws_utils.get_vpc_from_neutron_network_id(
context.current['network_id'])
context.current['network_id'], context=context._plugin_context,
project_id=project_id)
ec2_secgroup_ids = []
for secgrp_id in secgrp_ids:
tags = [
{'Key': 'openstack_id', 'Value': secgrp_id},
@ -225,40 +350,61 @@ class AwsMechanismDriver(api.MechanismDriver):
]
secgrp = core_plugin.get_security_group(context._plugin_context,
secgrp_id)
aws_secgrp = self.aws_utils.get_sec_group_by_id(secgrp_id,
vpc_id=vpc_id)
if not aws_secgrp and secgrp['name'] != 'default':
aws_secgrps = self.aws_utils.get_sec_group_by_id(
secgrp_id, group_name=secgrp['name'], vpc_id=vpc_id,
context=context._plugin_context, project_id=project_id)
if not aws_secgrps and secgrp['name'] != 'default':
grp_name = secgrp['name']
tags.append({"Key": "Name", "Value": grp_name})
desc = secgrp['description']
rules = secgrp['security_group_rules']
ec2_secgrp = self.aws_utils.create_security_group(
grp_name, desc, vpc_id, secgrp_id, tags)
grp_name, desc, vpc_id, secgrp_id, tags,
context=context._plugin_context,
project_id=project_id
)
self.aws_utils.create_security_group_rules(ec2_secgrp, rules)
# Make sure that omni_resources table is populated with newly
# created security group
aws_secgrps = self.aws_utils.get_sec_group_by_id(
secgrp_id, group_name=secgrp['name'], vpc_id=vpc_id,
context=context._plugin_context, project_id=project_id)
for aws_secgrp in aws_secgrps:
ec2_secgroup_ids.append(aws_secgrp['GroupId'])
return ec2_secgroup_ids
def delete_security_group(self, security_group_id):
self.aws_utils.delete_security_group(security_group_id)
def delete_security_group(self, security_group_id, context, project_id):
core_plugin = directory.get_plugin()
secgrp = core_plugin.get_security_group(context, security_group_id)
self.aws_utils.delete_security_group(security_group_id, context,
project_id,
group_name=secgrp['name'])
def remove_security_group_rule(self, context, rule_id):
core_plugin = manager.NeutronManager.get_plugin()
core_plugin = directory.get_plugin()
rule = core_plugin.get_security_group_rule(context, rule_id)
secgrp_id = rule['security_group_id']
secgrp = core_plugin.get_security_group(context, secgrp_id)
old_rules = secgrp['security_group_rules']
for idx in range(len(old_rules) - 1, -1, -1):
if old_rules[idx]['id'] == rule_id:
old_rules.pop(idx)
self.aws_utils.update_sec_group(secgrp_id, old_rules)
if "project_id" in rule:
project_id = rule['project_id']
else:
project_id = context.tenant
self.aws_utils.delete_security_group_rule_if_needed(
context, secgrp_id, secgrp['name'], project_id, rule)
def add_security_group_rule(self, context, rule):
core_plugin = manager.NeutronManager.get_plugin()
core_plugin = directory.get_plugin()
secgrp_id = rule['security_group_id']
secgrp = core_plugin.get_security_group(context, secgrp_id)
old_rules = secgrp['security_group_rules']
old_rules.append(rule)
self.aws_utils.update_sec_group(secgrp_id, old_rules)
if "project_id" in rule:
project_id = rule['project_id']
else:
project_id = context.tenant
self.aws_utils.create_security_group_rule_if_needed(
context, secgrp_id, secgrp['name'], project_id, rule)
def update_security_group_rules(self, context, rule_id):
core_plugin = manager.NeutronManager.get_plugin()
core_plugin = directory.get_plugin()
rule = core_plugin.get_security_group_rule(context, rule_id)
secgrp_id = rule['security_group_id']
secgrp = core_plugin.get_security_group(context, secgrp_id)
@ -268,24 +414,58 @@ class AwsMechanismDriver(api.MechanismDriver):
old_rules.pop(idx)
break
old_rules.append(rule)
self.aws_utils.update_sec_group(secgrp_id, old_rules)
if "project_id" in rule:
project_id = rule['project_id']
else:
project_id = context.tenant
self.aws_utils.update_sec_group(secgrp_id, old_rules, context=context,
project_id=project_id,
group_name=secgrp['name'])
def secgroup_callback(self, resource, event, trigger, **kwargs):
context = kwargs['context']
if resource == resources.SECURITY_GROUP:
if event == events.AFTER_CREATE:
project_id = kwargs.get('security_group')['project_id']
secgrp = kwargs.get('security_group')
security_group_id = secgrp.get('id')
core_plugin = directory.get_plugin()
aws_secgrps = self.aws_utils.get_sec_group_by_id(
security_group_id, group_name=secgrp.get('name'),
context=context, project_id=project_id)
if len(aws_secgrps) == 0:
return
for sgr in secgrp.get('security_group_rules', []):
# This is invoked for discovered security groups only. For
# discovered security groups we do not need default egress
# rules. Those should be reported by discovery service.
# When removing these default security group rules we do
# not need to check against AWS. Store the security group
# rule IDs so that we can ignore them when delete security
# group rule is called here.
self._default_sgr_to_remove.append(sgr.get('id'))
core_plugin.delete_security_group_rule(context,
sgr.get('id'))
if event == events.BEFORE_DELETE:
project_id = kwargs.get('security_group')['project_id']
security_group_id = kwargs.get('security_group_id')
if security_group_id:
self.delete_security_group(security_group_id)
self.delete_security_group(security_group_id, context,
project_id)
else:
LOG.warn('Security group ID not found in delete request')
elif resource == resources.SECURITY_GROUP_RULE:
context = kwargs['context']
if event == events.BEFORE_CREATE:
rule = kwargs['security_group_rule']
self.add_security_group_rule(context, rule)
elif event == events.BEFORE_DELETE:
rule_id = kwargs['security_group_rule_id']
self.remove_security_group_rule(context, rule_id)
if rule_id in self._default_sgr_to_remove:
# Check the comment above in security group rule
# AFTER_CREATE event handling
self._default_sgr_to_remove.remove(rule_id)
else:
self.remove_security_group_rule(context, rule_id)
elif event == events.BEFORE_UPDATE:
rule_id = kwargs['security_group_rule_id']
self.update_security_group_rules(context, rule_id)

Some files were not shown because too many files have changed in this diff Show More