summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-06-16 00:15:54 +0000
committerGerrit Code Review <review@openstack.org>2017-06-16 00:15:54 +0000
commit934641e59ed35563d0290d068038c1dfeffc4788 (patch)
tree69ea5d386ba01673701cad23c700b32519689ca1
parent6509f92e4f2d18a8d66d1e64de6eaa08879154bb (diff)
parenta919ab2959310b6677036132304a9e89e1a82e05 (diff)
Merge "Lazy load of resources in resource cache"
-rw-r--r--neutron/agent/resource_cache.py38
-rw-r--r--neutron/agent/rpc.py3
-rw-r--r--neutron/tests/unit/agent/test_resource_cache.py26
3 files changed, 40 insertions, 27 deletions
diff --git a/neutron/agent/resource_cache.py b/neutron/agent/resource_cache.py
index c83f0b7..e031c01 100644
--- a/neutron/agent/resource_cache.py
+++ b/neutron/agent/resource_cache.py
@@ -36,6 +36,9 @@ class RemoteResourceCache(object):
36 self.resource_types = resource_types 36 self.resource_types = resource_types
37 self._cache_by_type_and_id = {rt: {} for rt in self.resource_types} 37 self._cache_by_type_and_id = {rt: {} for rt in self.resource_types}
38 self._deleted_ids_by_type = {rt: set() for rt in self.resource_types} 38 self._deleted_ids_by_type = {rt: set() for rt in self.resource_types}
39 # track everything we've asked the server so we don't ask again
40 self._satisfied_server_queries = set()
41 self._puller = resources_rpc.ResourcesPullRpcApi()
39 42
40 def _type_cache(self, rtype): 43 def _type_cache(self, rtype):
41 if rtype not in self.resource_types: 44 if rtype not in self.resource_types:
@@ -45,22 +48,37 @@ class RemoteResourceCache(object):
45 def start_watcher(self): 48 def start_watcher(self):
46 self._watcher = RemoteResourceWatcher(self) 49 self._watcher = RemoteResourceWatcher(self)
47 50
48 def bulk_flood_cache(self):
49 # TODO(kevinbenton): this fills the cache with *every* server record.
50 # make this more intelligent to only pull ports on host, then all
51 # networks and security groups for those ports, etc.
52 context = n_ctx.get_admin_context()
53 puller = resources_rpc.ResourcesPullRpcApi()
54 for rtype in self.resource_types:
55 for resource in puller.bulk_pull(context, rtype):
56 self._type_cache(rtype)[resource.id] = resource
57
58 def get_resource_by_id(self, rtype, obj_id): 51 def get_resource_by_id(self, rtype, obj_id):
59 """Returns None if it doesn't exist.""" 52 """Returns None if it doesn't exist."""
53 if obj_id in self._deleted_ids_by_type[rtype]:
54 return None
55 cached_item = self._type_cache(rtype).get(obj_id)
56 if cached_item:
57 return cached_item
58 # try server in case object existed before agent start
59 self._flood_cache_for_query(rtype, id=obj_id)
60 return self._type_cache(rtype).get(obj_id) 60 return self._type_cache(rtype).get(obj_id)
61 61
62 def _flood_cache_for_query(self, rtype, **filter_kwargs):
63 """Load info from server for first query.
64
65 Queries the server if this is the first time a given query for
66 rtype has been issued.
67 """
68 query_id = (rtype, ) + tuple(sorted(filter_kwargs.items()))
69 if query_id in self._satisfied_server_queries:
70 # we've already asked the server this question so we don't
71 # ask again because any updates will have been pushed to us
72 return
73 context = n_ctx.get_admin_context()
74 for resource in self._puller.bulk_pull(context, rtype,
75 filter_kwargs=filter_kwargs):
76 self._type_cache(rtype)[resource.id] = resource
77 self._satisfied_server_queries.add(query_id)
78
62 def get_resources(self, rtype, filters): 79 def get_resources(self, rtype, filters):
63 """Find resources that match key:value in filters dict.""" 80 """Find resources that match key:value in filters dict."""
81 self._flood_cache_for_query(rtype, **filters)
64 match = lambda o: all(v == getattr(o, k) for k, v in filters.items()) 82 match = lambda o: all(v == getattr(o, k) for k, v in filters.items())
65 return self.match_resources_with_func(rtype, match) 83 return self.match_resources_with_func(rtype, match)
66 84
diff --git a/neutron/agent/rpc.py b/neutron/agent/rpc.py
index d081088..a9e056e 100644
--- a/neutron/agent/rpc.py
+++ b/neutron/agent/rpc.py
@@ -169,9 +169,6 @@ def create_cache_for_l2_agent():
169 ] 169 ]
170 rcache = resource_cache.RemoteResourceCache(resource_types) 170 rcache = resource_cache.RemoteResourceCache(resource_types)
171 rcache.start_watcher() 171 rcache.start_watcher()
172 # TODO(kevinbenton): ensure flood uses filters or that this has a long
173 # timeout before Pike release.
174 rcache.bulk_flood_cache()
175 return rcache 172 return rcache
176 173
177 174
diff --git a/neutron/tests/unit/agent/test_resource_cache.py b/neutron/tests/unit/agent/test_resource_cache.py
index 5fa8679..63b4d3a 100644
--- a/neutron/tests/unit/agent/test_resource_cache.py
+++ b/neutron/tests/unit/agent/test_resource_cache.py
@@ -46,20 +46,7 @@ class RemoteResourceCacheTestCase(base.BaseTestCase):
46 self.duck = OVOLikeThing(2) 46 self.duck = OVOLikeThing(2)
47 self.ctx = context.get_admin_context() 47 self.ctx = context.get_admin_context()
48 self.rcache = resource_cache.RemoteResourceCache(rtypes) 48 self.rcache = resource_cache.RemoteResourceCache(rtypes)
49 49 self._pullmock = mock.patch.object(self.rcache, '_puller').start()
50 def test_bulk_flood_cache(self):
51 with mock.patch('neutron.api.rpc.handlers.resources_rpc.'
52 'ResourcesPullRpcApi') as rpc_pull_mock:
53 pull = rpc_pull_mock.return_value.bulk_pull
54 pull.side_effect = [[self.duck], [self.goose]]
55 self.rcache.bulk_flood_cache()
56 pull.assert_any_call(mock.ANY, 'duck')
57 pull.assert_any_call(mock.ANY, 'goose')
58 self.assertEqual(self.goose,
59 self.rcache.get_resource_by_id('goose', 1))
60 self.assertEqual(self.duck,
61 self.rcache.get_resource_by_id('duck', 2))
62 self.assertIsNone(self.rcache.get_resource_by_id('duck', 1))
63 50
64 def test_get_resource_by_id(self): 51 def test_get_resource_by_id(self):
65 self.rcache.record_resource_update(self.ctx, 'goose', self.goose) 52 self.rcache.record_resource_update(self.ctx, 'goose', self.goose)
@@ -67,6 +54,17 @@ class RemoteResourceCacheTestCase(base.BaseTestCase):
67 self.rcache.get_resource_by_id('goose', 1)) 54 self.rcache.get_resource_by_id('goose', 1))
68 self.assertIsNone(self.rcache.get_resource_by_id('goose', 2)) 55 self.assertIsNone(self.rcache.get_resource_by_id('goose', 2))
69 56
57 def test__flood_cache_for_query_pulls_once(self):
58 self.rcache._flood_cache_for_query('goose', id=66)
59 self._pullmock.bulk_pull.assert_called_once_with(
60 mock.ANY, 'goose', filter_kwargs={'id': 66})
61 self._pullmock.bulk_pull.reset_mock()
62 self.rcache._flood_cache_for_query('goose', id=66)
63 self.assertFalse(self._pullmock.called)
64 self.rcache._flood_cache_for_query('goose', id=67)
65 self._pullmock.bulk_pull.assert_called_once_with(
66 mock.ANY, 'goose', filter_kwargs={'id': 67})
67
70 def test_get_resources(self): 68 def test_get_resources(self):
71 geese = [OVOLikeThing(3, size='large'), OVOLikeThing(5, size='medium'), 69 geese = [OVOLikeThing(3, size='large'), OVOLikeThing(5, size='medium'),
72 OVOLikeThing(4, size='large'), OVOLikeThing(6, size='small')] 70 OVOLikeThing(4, size='large'), OVOLikeThing(6, size='small')]