Datasource driver for Swift
Description: Data source driver for Swift added. Also configuration changes and unit test code has been added. This driver provides swift(object storage) containers list and objects list in each container Change-Id: I956ca6bf46ad7f8fba445d271e48d47b7da570fd
This commit is contained in:
Srinivasa Rao Ragolu
parent
92e56fc615
commit
f7d5ee8c1d
|
|
@ -0,0 +1,158 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright (c) 2014 Montavista Software, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
import swiftclient.service
|
||||
|
||||
from congress.datasources.datasource_driver import DataSourceDriver
|
||||
from congress.openstack.common import log as logging
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def d6service(name, keys, inbox, datapath, args):
|
||||
"""This method is called by d6cage to create a dataservice instance."""
|
||||
return SwiftDriver(name, keys, inbox, datapath, args)
|
||||
|
||||
|
||||
class SwiftDriver(DataSourceDriver):
|
||||
CONTAINERS = "containers"
|
||||
OBJECTS = "objects"
|
||||
|
||||
value_trans = {'translation-type': 'VALUE'}
|
||||
|
||||
containers_translator = {
|
||||
'translation-type': 'HDICT',
|
||||
'table-name': CONTAINERS,
|
||||
'selector-type': 'DICT_SELECTOR',
|
||||
'field-translators':
|
||||
({'fieldname': 'count', 'translator': value_trans},
|
||||
{'fieldname': 'bytes', 'translator': value_trans},
|
||||
{'fieldname': 'name', 'translator': value_trans})}
|
||||
|
||||
objects_translator = {
|
||||
'translation-type': 'HDICT',
|
||||
'table-name': OBJECTS,
|
||||
'selector-type': 'DICT_SELECTOR',
|
||||
'field-translators':
|
||||
({'fieldname': 'bytes', 'translator': value_trans},
|
||||
{'fieldname': 'last_modified', 'translator': value_trans},
|
||||
{'fieldname': 'hash', 'translator': value_trans},
|
||||
{'fieldname': 'name', 'translator': value_trans},
|
||||
{'fieldname': 'content_type', 'translator': value_trans},
|
||||
{'fieldname': 'container_name', 'translator': value_trans})}
|
||||
|
||||
def __init__(self, name='', keys='', inbox=None, datapath=None, args=None):
|
||||
if args is None:
|
||||
args = self.empty_credentials()
|
||||
super(SwiftDriver, self).__init__(name, keys, inbox, datapath, args)
|
||||
if 'client' in args:
|
||||
self.swift_client = args['client']
|
||||
else:
|
||||
self.swift_service = swiftclient.service.SwiftService()
|
||||
|
||||
self.raw_state = {}
|
||||
|
||||
def update_from_datasource(self):
|
||||
'''Read data from swift and populate the policy engine
|
||||
tables with current state as specified by translators
|
||||
'''
|
||||
container_list = self.swift_service.list()
|
||||
object_list = []
|
||||
self.obj_list = []
|
||||
cont_list = []
|
||||
objects = []
|
||||
|
||||
LOG.debug("Swift obtaining containers List")
|
||||
for stats in container_list:
|
||||
containers = stats['listing']
|
||||
for item in containers:
|
||||
cont_list.append(item['name'])
|
||||
|
||||
LOG.debug("Swift obtaining objects List")
|
||||
for container in cont_list:
|
||||
object_list = self.swift_service.list(container)
|
||||
for items in object_list:
|
||||
item_list = items['listing']
|
||||
for obj in item_list:
|
||||
obj['container_name'] = container
|
||||
for obj in item_list:
|
||||
objects.append(obj)
|
||||
|
||||
LOG.debug("Containers Lists--->: %s" % containers)
|
||||
LOG.debug("Object Lists--->: %s " % objects)
|
||||
|
||||
if ('containers' not in self.raw_state or containers !=
|
||||
self.raw_state['containers']):
|
||||
self.raw_state['containers'] = containers
|
||||
self._translate_containers(containers)
|
||||
|
||||
if ('objects' not in self.raw_state or objects !=
|
||||
self.raw_state['objects']):
|
||||
self.raw_state['objects'] = objects
|
||||
self._translate_objects(objects)
|
||||
|
||||
@classmethod
|
||||
def get_translators(cls):
|
||||
return (cls.containers_translator, cls.objects_translator)
|
||||
|
||||
def _translate_containers(self, obj):
|
||||
"""Translate the containers represented by OBJ into tables.
|
||||
Assign self.state[tablename] for the table names
|
||||
generated from OBJ: CONTAINERS.
|
||||
"""
|
||||
row_data = SwiftDriver.convert_objs(obj,
|
||||
self.containers_translator)
|
||||
|
||||
container_tables = (self.CONTAINERS)
|
||||
self.state[container_tables] = set()
|
||||
for table, row in row_data:
|
||||
assert table in container_tables
|
||||
self.state[table].add(row)
|
||||
|
||||
LOG.debug("CONTAINERS: %s" % str(self.state[self.CONTAINERS]))
|
||||
return tuple(self.state[self.CONTAINERS])
|
||||
|
||||
def _translate_objects(self, obj):
|
||||
"""Translate the objects represented by OBJ into tables.
|
||||
Assign self.state[tablename] for the table names
|
||||
generated from OBJ: OBJECTS.
|
||||
"""
|
||||
row_data = SwiftDriver.convert_objs(obj,
|
||||
self.objects_translator)
|
||||
|
||||
object_tables = (self.OBJECTS)
|
||||
self.state[object_tables] = set()
|
||||
for table, row in row_data:
|
||||
assert table in object_tables
|
||||
self.state[table].add(row)
|
||||
|
||||
LOG.debug("OBJECTS: %s" % str(self.state[self.OBJECTS]))
|
||||
return tuple(self.state[self.OBJECTS])
|
||||
|
||||
|
||||
def main():
|
||||
driver = SwiftDriver()
|
||||
driver.update_from_datasource()
|
||||
|
||||
print "Starting Swift Sync Service"
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except SystemExit:
|
||||
# Let system.exit() calls complete normally
|
||||
raise
|
||||
except Exception:
|
||||
raise
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright (c) 2014 Montavista Software, LLC.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
from mock import MagicMock
|
||||
|
||||
from congress.datasources.swift_driver import SwiftDriver
|
||||
from congress.tests import base
|
||||
from congress.tests import helper
|
||||
|
||||
|
||||
class TestSwiftDriver(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSwiftDriver, self).setUp()
|
||||
self.swift_client = MagicMock()
|
||||
|
||||
args = helper.datasource_openstack_args()
|
||||
args['poll_time'] = 0
|
||||
args['client'] = self.swift_client
|
||||
self.driver = SwiftDriver(name='testswift', args=args)
|
||||
|
||||
def test_list_containers(self):
|
||||
containers_data = [{'count': '1',
|
||||
'bytes': '1048',
|
||||
'name': 'container1'},
|
||||
{'count': '2',
|
||||
'bytes': '2086',
|
||||
'name': 'container2'}]
|
||||
|
||||
container_list = self.driver._translate_containers(containers_data)
|
||||
self.assertIsNotNone(container_list)
|
||||
self.assertEqual(2, len(container_list))
|
||||
|
||||
if container_list[0][2] == 'container1':
|
||||
self.assertEqual(('1', '1048', 'container1'), container_list[0])
|
||||
self.assertEqual(('2', '2086', 'container2'), container_list[1])
|
||||
if container_list[1][2] == 'container1':
|
||||
self.assertEqual(('1', '1048', 'container1'), container_list[1])
|
||||
self.assertEqual(('2', '2086', 'container2'), container_list[0])
|
||||
|
||||
def test_list_objects(self):
|
||||
objects_data = [{'bytes': '2200',
|
||||
'last_modified': '2014-11-06T05:40:34.052100',
|
||||
'hash': '9204776814ca62c92c7996de725ecc6b',
|
||||
'name': 'file-1',
|
||||
'content_type': 'application/octet-stream',
|
||||
'container_name': 'container1'},
|
||||
{'bytes': '2350',
|
||||
'last_modified': '2014-11-06T05:39:57.424800',
|
||||
'hash': 'c2b86044dd50a29d60c0e92e23e3ceea',
|
||||
'name': 'file-2',
|
||||
'content_type': 'application/octet-stream',
|
||||
'container_name': 'container2'}]
|
||||
|
||||
object_list = self.driver._translate_objects(objects_data)
|
||||
self.assertIsNotNone(object_list)
|
||||
self.assertEqual(2, len(object_list))
|
||||
|
||||
if object_list[0][5] == 'container1':
|
||||
self.assertEqual(('2200', '2014-11-06T05:40:34.052100',
|
||||
'9204776814ca62c92c7996de725ecc6b', 'file-1',
|
||||
'application/octet-stream',
|
||||
'container1'), object_list[0])
|
||||
|
||||
self.assertEqual(('2350', '2014-11-06T05:39:57.424800',
|
||||
'c2b86044dd50a29d60c0e92e23e3ceea', 'file-2',
|
||||
'application/octet-stream',
|
||||
'container2'), object_list[1])
|
||||
|
||||
if object_list[1][5] == 'container1':
|
||||
self.assertEqual(('2200', '2014-11-06T05:40:34.052100',
|
||||
'9204776814ca62c92c7996de725ecc6b', 'file-1',
|
||||
'application/octet-stream',
|
||||
'container1'), object_list[1])
|
||||
|
||||
self.assertEqual(('2350', '2014-11-06T05:39:57.424800',
|
||||
'c2b86044dd50a29d60c0e92e23e3ceea', 'file-2',
|
||||
'application/octet-stream',
|
||||
'container2'), object_list[0])
|
||||
|
|
@ -32,3 +32,10 @@ username: admin
|
|||
password: password
|
||||
auth_url: http://127.0.0.1:5000/v2.0
|
||||
tenant_name: admin
|
||||
|
||||
[swift]
|
||||
module: datasources/swift_driver.py
|
||||
username: admin
|
||||
password: password
|
||||
auth_url: http://127.0.0.1:5000/v2.0
|
||||
tenant_name: admin
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ python-novaclient>=2.18.0
|
|||
python-neutronclient>=2.3.6,<3
|
||||
python-ceilometerclient>=1.0.6
|
||||
python-cinderclient>=1.1.0
|
||||
python-swiftclient>=2.2.0
|
||||
alembic>=0.6.4
|
||||
Routes>=1.12.3,!=2.0
|
||||
six>=1.7.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue