This commit is contained in:
Kashyap Kopparam 2014-11-12 15:32:14 +05:30
commit a813df1687
26 changed files with 473 additions and 58 deletions

4
ec2_group_transformer.py Normal file
View File

@ -0,0 +1,4 @@
class EC2GroupTransformer:
def to_group(self, ec2_group):
pass

9
ec2_rule_service.py Normal file
View File

@ -0,0 +1,9 @@
class EC2RuleService:
def __init__(self, ec2_connection, ec2_rule_transformer):
self.ec2_connection = ec2_connection
self.ec2_rule_transformer = ec2_rule_transformer
def get_rules_for_group(self, group_name):
group = self.ec2_connection.get_all_security_groups(groupnames=group_name)[0]
return set([self.ec2_rule_transformer.to_rule(rule) for rule in group.rules])

22
ec2_rule_transformer.py Normal file
View File

@ -0,0 +1,22 @@
from copy import deepcopy
from rule import Rule
class EC2RuleTransformer:
def __init__(self, ec2_connection):
self.ec2_connection = ec2_connection
def to_rule(self, ec2_rule):
rule_args = {}
rule_args['ip_protocol'] = ec2_rule.ip_protocol
rule_args['from_port'] = ec2_rule.from_port
rule_args['to_port'] = ec2_rule.to_port
if ec2_rule.grants[0].cidr_ip:
rule_args['ip_range'] = ec2_rule.grants[0].cidr_ip
else:
group_id = ec2_rule.grants[0].group_id
rule_args['group_name'] = self.ec2_connection.get_all_security_groups(group_ids=group_id)[0].name
return Rule(**rule_args)

View File

@ -25,6 +25,8 @@ from boto.exception import EC2ResponseError
from boto.regioninfo import RegionInfo
from oslo.config import cfg
from novaclient.v1_1 import client
from ec2_rule_service import EC2RuleService
from ec2_rule_transformer import EC2RuleTransformer
from ec2driver_config import *
from nova import block_device
@ -39,8 +41,13 @@ from nova.openstack.common import loopingcall
from nova.virt import driver
from nova.virt import virtapi
from credentials import get_nova_creds
from instance_rule_refresher import InstanceRuleRefresher
from openstack_group_service import OpenstackGroupService
from openstack_rule_service import OpenstackRuleService
from openstack_rule_transformer import OpenstackRuleTransformer
import rule_comparator
from group_rule_refresher import GroupRuleRefresher
LOG = logging.getLogger(__name__)
@ -156,7 +163,20 @@ class EC2Driver(driver.ComputeDriver):
aws_region, aws_access_key_id=CONF.ec2driver.ec2_access_key_id, aws_secret_access_key=CONF.ec2driver.ec2_secret_access_key)
self.security_group_lock = Lock()
self.rule_comparator = rule_comparator.RuleComparator(self.ec2_conn)
self.instance_rule_refresher = InstanceRuleRefresher(
GroupRuleRefresher(
ec2_connection=self.ec2_conn,
openstack_rule_service=OpenstackRuleService(
group_service=OpenstackGroupService(self.nova.security_groups),
openstack_rule_transformer=OpenstackRuleTransformer()
),
ec2_rule_service=EC2RuleService(
ec2_connection=self.ec2_conn,
ec2_rule_transformer=EC2RuleTransformer(self.ec2_conn)
)
)
)
if not '_EC2_NODES' in globals():
set_nodes([CONF.host])
@ -720,48 +740,7 @@ class EC2Driver(driver.ComputeDriver):
# TODO: lock for case when group is associated with multiple instances [Cameron & Ed]
openstack_instance = self.nova.servers.get(instance['id'])
for group_dict in openstack_instance.security_groups:
openstack_group =\
[group for group in self.nova.security_groups.list() if group.name == group_dict['name']][0]
ec2_group = self.ec2_conn.get_all_security_groups(groupnames=group_dict['name'])[0]
for openstack_rule in openstack_group.rules:
equivalent_rule_found_in_ec2 = False
for ec2_rule in ec2_group.rules:
if self.rule_comparator.rules_are_equal(openstack_rule, ec2_rule):
equivalent_rule_found_in_ec2 = True
break
if not equivalent_rule_found_in_ec2:
self.ec2_conn.authorize_security_group(
group_name=ec2_group.name,
ip_protocol=openstack_rule['ip_protocol'],
from_port=openstack_rule['from_port'],
to_port=openstack_rule['to_port'],
src_security_group_name=self._get_allowed_group_name_from_openstack_rule_if_present(openstack_rule),
cidr_ip=self._get_allowed_ip_range_from_openstack_rule_if_present(openstack_rule)
)
for ec2_rule in ec2_group.rules:
equivalent_rule_found_in_openstack = False
for openstack_rule in openstack_group.rules:
if self.rule_comparator.rules_are_equal(openstack_rule, ec2_rule):
equivalent_rule_found_in_openstack = True
break
if not equivalent_rule_found_in_openstack:
self.ec2_conn.revoke_security_group(
group_name=ec2_group.name,
ip_protocol=ec2_rule.ip_protocol,
from_port=ec2_rule.from_port,
to_port=ec2_rule.to_port,
cidr_ip=ec2_rule.grants[0].cidr_ip,
src_security_group_group_id=ec2_rule.grants[0].group_id
)
self.instance_rule_refresher.refresh(self.nova.servers.get(instance['id']))
return

4
group.py Normal file
View File

@ -0,0 +1,4 @@
class Group:
def rule_diff(self, other_group):
pass

39
group_rule_refresher.py Normal file
View File

@ -0,0 +1,39 @@
class GroupRuleRefresher:
def __init__(self, ec2_connection, openstack_rule_service, ec2_rule_service):
self.ec2_conn = ec2_connection
self.openstack_rule_service = openstack_rule_service
self.ec2_rule_service = ec2_rule_service
def refresh(self, group_name):
openstack_rules = self.openstack_rule_service.get_rules_for_group(group_name)
ec2_rules = self.ec2_rule_service.get_rules_for_group(group_name)
self._add_rules_to_ec2(ec2_rules, group_name, openstack_rules)
self._remove_rules_from_ec2(ec2_rules, group_name, openstack_rules)
def _add_rules_to_ec2(self, ec2_rules, group_name, openstack_rules):
for rule in openstack_rules - ec2_rules:
self._add_rule_on_ec2(group_name, rule)
def _remove_rules_from_ec2(self, ec2_rules, group_name, openstack_rules):
for rule in ec2_rules - openstack_rules:
self._remove_rule_from_ec2(group_name, rule)
def _remove_rule_from_ec2(self, group_name, rule):
self.ec2_conn.revoke_security_group(
group_name=group_name,
ip_protocol=rule.ip_protocol,
from_port=rule.from_port,
to_port=rule.to_port,
cidr_ip=rule.ip_range
)
def _add_rule_on_ec2(self, group_name, rule):
self.ec2_conn.authorize_security_group(
group_name=group_name,
ip_protocol=rule.ip_protocol,
from_port=rule.from_port,
to_port=rule.to_port,
cidr_ip=rule.ip_range
)

View File

@ -0,0 +1,11 @@
class InstanceRuleRefresher:
def __init__(self, group_rule_refresher):
self.group_rule_refresher = group_rule_refresher
def refresh(self, instance):
for group_name in self._get_group_names(instance):
self.group_rule_refresher.refresh(group_name)
def _get_group_names(self, instance):
return [group['name'] for group in instance.security_groups]

View File

@ -0,0 +1,7 @@
class OpenstackGroupService():
def __init__(self, security_group_manager):
self.security_group_manager = security_group_manager
def get_group(self, group_name):
return [group for group in self.security_group_manager.list() if group.name == group_name][0]

View File

@ -0,0 +1,3 @@
class OpenstackGroupTransformer:
def to_group(self, openstack_group):
pass

View File

@ -0,0 +1,9 @@
class OpenstackRuleService:
def __init__(self, group_service, openstack_rule_transformer):
self.group_service = group_service
self.openstack_rule_transformer = openstack_rule_transformer
def get_rules_for_group(self, group_name):
openstack_group = self.group_service.get_group(group_name)
return set([self.openstack_rule_transformer.to_rule(rule) for rule in openstack_group.rules])
# return self.group_service.get_group(group_name).rules

View File

@ -0,0 +1,17 @@
from copy import deepcopy
from rule import Rule
class OpenstackRuleTransformer:
def to_rule(self, openstack_rule):
rule_args = {}
rule_args['ip_protocol'] = openstack_rule['ip_protocol']
rule_args['from_port'] = str(openstack_rule['from_port'])
rule_args['to_port'] = str(openstack_rule['to_port'])
if 'cidr' in openstack_rule['ip_range']:
rule_args['ip_range'] = openstack_rule['ip_range']['cidr']
else:
rule_args['group_name'] = openstack_rule['group']['name']
return Rule(**rule_args)

16
rule.py Normal file
View File

@ -0,0 +1,16 @@
class Rule:
def __init__(self, ip_protocol, from_port, to_port, ip_range=None, group_name=None):
self.ip_protocol = ip_protocol
self.from_port = from_port
self.to_port = to_port
self.ip_range = ip_range
self.group_name = group_name
def __key(self):
return self.ip_protocol, self.from_port, self.to_port, self.ip_range, self.group_name
def __eq__(self, other):
return self.__key() == other.__key()
def __hash__(self):
return hash(self.__key())

View File

View File

@ -7,8 +7,8 @@ import datetime
from novaclient.v1_1 import client
from boto import ec2
from ..credentials import get_nova_creds
from ..ec2driver_config import *
from ...credentials import get_nova_creds
from ...ec2driver_config import *
class EC2TestBase(unittest.TestCase):

View File

@ -1,10 +1,11 @@
import unittest
import urllib2
from boto.exception import EC2ResponseError
from boto.exception import EC2ResponseError
from novaclient.v1_1 import client
from ..ec2driver_config import *
from ec2_test_base import EC2TestBase
from ...ec2driver_config import *
from tests.integration.ec2_test_base import EC2TestBase
class TestEC2Driver(EC2TestBase):

View File

@ -2,9 +2,10 @@ from time import sleep
import unittest
from random import randint
from ..ec2driver_config import *
from ...ec2driver_config import *
from ec2_test_base import EC2TestBase
class TestSecurityGroups(EC2TestBase):
def setUp(self):
@ -35,9 +36,9 @@ class TestSecurityGroups(EC2TestBase):
updated_matching_ec2_security_group = self._wait_for_ec2_group_to_have_no_instances(self.security_group)
self.assertEqual(updated_matching_ec2_security_group.instances(), [])
def test_should_add_rule_to_ec2_security_group_when_group_is_added_to_an_instance(self):
@unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto')
def test_should_add_rule_to_ec2_security_group_when_rule_is_added_to_openstack_group_associated_with_instance(self):
security_group_rule = self.nova.security_group_rules.create(
parent_group_id=self.security_group.id,
ip_protocol='tcp',
@ -46,13 +47,13 @@ class TestSecurityGroups(EC2TestBase):
cidr='0.0.0.0/0'
)
updated_security_group = self.nova.security_groups.get(self.security_group.id)
ec2_security_group = self.ec2_conn.get_all_security_groups(groupnames=self.security_group.name)[0]
ec2_rule = ec2_security_group.rules[0]
self.assertEqual(ec2_rule.ip_protocol, security_group_rule.ip_protocol)
#etc
self.assertEqual(ec2_rule.ip_protocol, security_group_rule['ip_protocol'])
self.assertEqual(ec2_rule.from_port, security_group_rule['from_port'])
self.assertEqual(ec2_rule.to_port, security_group_rule['to_port'])
self.assertEqual(ec2_rule.grants[0].cidr_ip, security_group_rule['ip_range']['cidr'])
def _destroy_security_group(self):
print "Cleanup: Destroying security group"

0
tests/unit/__init__.py Normal file
View File

View File

@ -2,7 +2,7 @@ from collections import namedtuple
class FakeEC2RuleBuilder():
EC2Rule = namedtuple('EC2Rule', 'ip_protocol from_port to_port grants')
EC2Rule = namedtuple('EC2Rule', 'ip_protocol from_port to_port grants parent item ipRanges groups')
GroupOrCIDR = namedtuple('GroupOrCIDR', 'cidr_ip group_id')
def __init__(self):
@ -11,6 +11,10 @@ class FakeEC2RuleBuilder():
self.to_port = '3333'
self.ip_range = '0.0.0.0/0'
self.allowed_security_group_id = None
self.parent = None
self.item = '\n'
self.ip_ranges = '\n'
self.groups = ''
@staticmethod
def an_ec2_rule():
@ -40,4 +44,5 @@ class FakeEC2RuleBuilder():
def build(self):
grants = [self.GroupOrCIDR(self.ip_range, self.allowed_security_group_id)]
return self.EC2Rule(self.ip_protocol, self.from_port, self.to_port, grants)
return self.EC2Rule(self.ip_protocol, self.from_port, self.to_port, grants,
self.parent, self.item, self.ip_ranges, self.groups)

View File

@ -0,0 +1,50 @@
import unittest
from boto.ec2 import EC2Connection
from boto.ec2.securitygroup import SecurityGroup
from mock import Mock, call
from nova.virt.ec2.ec2_rule_service import EC2RuleService
from nova.virt.ec2.ec2_rule_transformer import EC2RuleTransformer
class TestEC2RuleService(unittest.TestCase):
def setUp(self):
self.security_group = Mock(spec=SecurityGroup)
self.security_group.name = "secGroup"
self.ec2_connection = Mock(spec=EC2Connection)
self.ec2_connection.get_all_security_groups.return_value = [self.security_group]
self.ec2_rule_transformer = Mock(spec=EC2RuleTransformer)
self.ec2_rule_service = EC2RuleService(self.ec2_connection, self.ec2_rule_transformer)
def test_should_get_security_group_from_ec2_connection(self):
self.security_group.rules = []
self.ec2_rule_service.get_rules_for_group(self.security_group.name)
self.ec2_connection.get_all_security_groups.assert_called_once_with(groupnames=self.security_group.name)
def test_should_transform_rules_from_security_group(self):
first_rule = Mock()
second_rule = Mock()
self.security_group.rules = [first_rule, second_rule]
self.ec2_rule_service.get_rules_for_group(self.security_group.name)
self.ec2_rule_transformer.to_rule.assert_has_calls([call(first_rule), call(second_rule)])
def test_should_return_transformed_security_group_rules(self):
first_rule = Mock()
second_rule = Mock()
self.security_group.rules = [first_rule, second_rule]
first_transformed_rule = Mock()
second_transformed_rule = Mock()
self.ec2_rule_transformer.to_rule.side_effect = [first_transformed_rule, second_transformed_rule]
actual_rules = self.ec2_rule_service.get_rules_for_group(self.security_group.name)
self.assertEqual(actual_rules, set([first_transformed_rule, second_transformed_rule]))

View File

@ -0,0 +1,46 @@
import unittest
from mock import Mock
from nova.virt.ec2.ec2_rule_transformer import EC2RuleTransformer
from fake_ec2_rule_builder import FakeEC2RuleBuilder
class TestEC2RuleTransformer(unittest.TestCase):
def setUp(self):
self.ec2_connection = Mock()
self.ec2_rule_transformer = EC2RuleTransformer(self.ec2_connection)
def test_should_copy_ip_protocol(self):
ec2_rule = FakeEC2RuleBuilder.an_ec2_rule().build()
rule = self.ec2_rule_transformer.to_rule(ec2_rule)
self.assertEqual(rule.ip_protocol, ec2_rule.ip_protocol)
def test_should_copy_from_port(self):
ec2_rule = FakeEC2RuleBuilder.an_ec2_rule().build()
rule = self.ec2_rule_transformer.to_rule(ec2_rule)
self.assertEqual(rule.from_port, ec2_rule.from_port)
def test_should_copy_to_port(self):
ec2_rule = FakeEC2RuleBuilder.an_ec2_rule().build()
rule = self.ec2_rule_transformer.to_rule(ec2_rule)
self.assertEqual(rule.to_port, ec2_rule.to_port)
def test_should_copy_ip_range_attribute_from_grant(self):
ec2_rule = FakeEC2RuleBuilder.an_ec2_rule().with_ip_range('5.6.7.8/90').build()
rule = self.ec2_rule_transformer.to_rule(ec2_rule)
self.assertEqual(rule.ip_range, ec2_rule.grants[0].cidr_ip)
def test_should_set_group_name_from_grant(self):
ec2_rule = FakeEC2RuleBuilder.an_ec2_rule().with_allowed_security_group_id(123).build()
ec2_group = Mock()
ec2_group.name = 'secGroup'
self.ec2_connection.get_all_security_groups.return_value = [ec2_group]
rule = self.ec2_rule_transformer.to_rule(ec2_rule)
self.assertEqual(rule.group_name, 'secGroup')

View File

@ -0,0 +1,55 @@
import unittest
from boto.ec2 import EC2Connection
from mock import Mock
from nova.virt.ec2.group_rule_refresher import GroupRuleRefresher
from nova.virt.ec2.rule import Rule
from nova.virt.ec2.openstack_rule_service import OpenstackRuleService
from nova.virt.ec2.ec2_rule_service import EC2RuleService
GROUP_NAME = 'secGroup'
OTHER_GROUP_NAME = "otherSecGroup"
class TestGroupRuleRefresher(unittest.TestCase):
def setUp(self):
self.rule = Rule('hjkl', 7, 8, '9.9.9.9/99')
self.openstack_instance = Mock()
self.ec2_connection = Mock(EC2Connection)
self.openstack_rule_service = Mock(OpenstackRuleService)
self.ec2_rule_service = Mock(EC2RuleService)
self.group_rule_refresher = GroupRuleRefresher(
self.ec2_connection,
self.openstack_rule_service,
self.ec2_rule_service
)
def test_should_add_rule_to_ec2_security_group_when_rule_associated_with_group_on_openstack(self):
self.openstack_rule_service.get_rules_for_group.return_value = set([self.rule])
self.ec2_rule_service.get_rules_for_group.return_value = set()
self.group_rule_refresher.refresh(GROUP_NAME)
self.ec2_connection.authorize_security_group.assert_called_once_with(
group_name=GROUP_NAME,
ip_protocol=self.rule.ip_protocol,
from_port=self.rule.from_port,
to_port=self.rule.to_port,
cidr_ip=self.rule.ip_range
)
def test_should_remove_rule_from_ec2_security_group_when_rule_not_associated_with_group_on_openstack(self):
self.openstack_rule_service.get_rules_for_group.return_value = set()
self.ec2_rule_service.get_rules_for_group.return_value = set([self.rule])
self.group_rule_refresher.refresh(GROUP_NAME)
self.ec2_connection.revoke_security_group.assert_called_once_with(
group_name=GROUP_NAME,
ip_protocol=self.rule.ip_protocol,
from_port=self.rule.from_port,
to_port=self.rule.to_port,
cidr_ip=self.rule.ip_range
)

View File

@ -0,0 +1,21 @@
import unittest
from mock import Mock, call
from nova.virt.ec2.group_rule_refresher import GroupRuleRefresher
from nova.virt.ec2.instance_rule_refresher import InstanceRuleRefresher
class TestInstanceRuleRefresher(unittest.TestCase):
def test_should_call_group_rule_refresher_on_every_group_for_instance(self):
group_rule_refresher = Mock(spec=GroupRuleRefresher)
instance = Mock()
first_group = {'name': 'firstGroup'}
second_group = {'name': 'secondGroup'}
instance.security_groups = [first_group, second_group]
instance_rule_refresher = InstanceRuleRefresher(group_rule_refresher)
instance_rule_refresher.refresh(instance)
group_rule_refresher.refresh.assert_has_calls([call(first_group['name']), call(second_group['name'])])

View File

@ -0,0 +1,29 @@
import unittest
from mock import Mock
from novaclient.v1_1.security_groups import SecurityGroupManager
from nova.virt.ec2.openstack_group_service import OpenstackGroupService
class TestOpenstackGroupService(unittest.TestCase):
def setUp(self):
self.security_group_manager = Mock(spec=SecurityGroupManager)
self.openstack_group_service = OpenstackGroupService(self.security_group_manager)
def test_should_get_group_from_nova_security_group_manager(self):
security_group = Mock()
security_group.name = 'secGroup'
self.security_group_manager.list.return_value = [security_group]
self.assertEqual(self.openstack_group_service.get_group(security_group.name), security_group)
def test_should_get_group_from_nova_security_group_manager_when_multiple_groups_present(self):
security_group1 = Mock()
security_group1.name = 'secGroup'
security_group2 = Mock()
security_group2.name = 'otherGroup'
self.security_group_manager.list.return_value = [security_group1, security_group2]
self.assertEqual(self.openstack_group_service.get_group(security_group2.name), security_group2)

View File

@ -0,0 +1,52 @@
import unittest
from mock import Mock, call
from novaclient.v1_1.security_groups import SecurityGroup
from nova.virt.ec2.openstack_rule_service import OpenstackRuleService
from nova.virt.ec2.openstack_group_service import OpenstackGroupService
from nova.virt.ec2.openstack_rule_transformer import OpenstackRuleTransformer
class TestOpenstackRuleService(unittest.TestCase):
def setUp(self):
self.security_group = Mock(spec=SecurityGroup)
self.security_group.name = "secGroup"
self.openstack_group_service = Mock(OpenstackGroupService)
self.openstack_group_service.get_group.return_value = self.security_group
self.openstack_rule_transformer = Mock(OpenstackRuleTransformer)
self.openstack_rule_service = OpenstackRuleService(
self.openstack_group_service,
self.openstack_rule_transformer
)
def test_should_get_security_group_from_group_service(self):
self.security_group.rules = []
self.openstack_rule_service.get_rules_for_group(self.security_group.name)
self.openstack_group_service.get_group.assert_called_once_with(self.security_group.name)
def test_should_transform_rules_from_security_group(self):
first_rule = Mock()
second_rule = Mock()
self.security_group.rules = [first_rule, second_rule]
self.openstack_rule_service.get_rules_for_group(self.security_group.name)
self.openstack_rule_transformer.to_rule.assert_has_calls([call(first_rule), call(second_rule)])
def test_should_return_transformed_security_group_rules(self):
first_rule = Mock()
second_rule = Mock()
self.security_group.rules = [first_rule, second_rule]
first_transformed_rule = Mock()
second_transformed_rule = Mock()
self.openstack_rule_transformer.to_rule.side_effect = [first_transformed_rule, second_transformed_rule]
actual_rules = self.openstack_rule_service.get_rules_for_group(self.security_group.name)
self.assertEqual(actual_rules, set([first_transformed_rule, second_transformed_rule]))

View File

@ -0,0 +1,35 @@
import unittest
from nova.virt.ec2.openstack_rule_transformer import OpenstackRuleTransformer
class TestOpenstackRuleTransformer(unittest.TestCase):
def setUp(self):
self.openstack_rule_transformer = OpenstackRuleTransformer()
def test_should_copy_to_port_as_string(self):
openstack_rule = {
'ip_protocol': 'abc',
'from_port': 123,
'to_port': 456,
'group': {},
'parent_group_id': 55,
'ip_range': {'cidr': '9.8.7.6/55'},
'id': 18
}
rule = self.openstack_rule_transformer.to_rule(openstack_rule)
self.assertEqual(rule.to_port, str(openstack_rule['to_port']))
def test_should_copy_from_port_as_string(self):
openstack_rule = {
'ip_protocol': 'abc',
'from_port': 123,
'to_port': 456,
'group': {},
'parent_group_id': 55,
'ip_range': {'cidr': '9.8.7.6/55'},
'id': 18
}
rule = self.openstack_rule_transformer.to_rule(openstack_rule)
self.assertEqual(rule.from_port, str(openstack_rule['from_port']))