Cleanup deleted resource from the tenant resource cache

When a resource is deleted, its not currently removed from the tenant
resource cache.  This causes a cache hit if the tenant attempts to re-create
the same type of resource, but the resource is then later ignored because
it has been deleted.  This adds a callback used by the TRM to remove it
from the resource cache when its state machine is deleted.

Change-Id: I5dcbeda7de240a693fc7a4944dd34a37b10d174b
Closes-bug: #1531597
This commit is contained in:
Adam Gandelman 2016-01-06 12:24:50 -08:00
parent 3219e39c4d
commit 5fed2a7041
6 changed files with 73 additions and 11 deletions

View File

@ -87,23 +87,25 @@ class TenantResourceManager(object):
tenant.
"""
def __init__(self, tenant_id, notify_callback,
def __init__(self, tenant_id, delete_callback, notify_callback,
queue_warning_threshold,
reboot_error_threshold):
self.tenant_id = tenant_id
self.delete = delete_callback
self.notify = notify_callback
self._queue_warning_threshold = queue_warning_threshold
self._reboot_error_threshold = reboot_error_threshold
self.state_machines = ResourceContainer()
self._default_resource_id = None
def _delete_resource(self, resource_id):
def _delete_resource(self, resource):
"Called when the Automaton decides the resource can be deleted"
if resource_id in self.state_machines:
LOG.debug('deleting state machine for %s', resource_id)
del self.state_machines[resource_id]
if self._default_resource_id == resource_id:
if resource.id in self.state_machines:
LOG.debug('deleting state machine for %s', resource.id)
del self.state_machines[resource.id]
if self._default_resource_id == resource.id:
self._default_resource_id = None
self.delete(resource)
def shutdown(self):
LOG.info('shutting down')
@ -179,7 +181,7 @@ class TenantResourceManager(object):
return []
def deleter():
self._delete_resource(message.resource.id)
self._delete_resource(message.resource)
new_state_machine = state.Automaton(
driver=driver_obj,

View File

@ -39,8 +39,10 @@ class TestTenantResourceManager(unittest.TestCase):
mock.patch('astara.instance_manager.InstanceManager').start()
self.addCleanup(mock.patch.stopall)
self.notifier = mock.Mock()
self.deleter = mock.Mock()
self.trm = tenant.TenantResourceManager(
'1234',
delete_callback=self.deleter,
notify_callback=self.notifier,
queue_warning_threshold=10,
reboot_error_threshold=5,
@ -193,27 +195,46 @@ class TestTenantResourceManager(unittest.TestCase):
self.assertIs(sm2, sms['5678'])
def test_delete_resource(self):
r = event.Resource(
id='1234',
tenant_id=self.tenant_id,
driver=router.Router.RESOURCE_NAME,
)
self.trm.state_machines['1234'] = mock.Mock()
self.trm._delete_resource('1234')
self.trm._delete_resource(r)
self.assertNotIn('1234', self.trm.state_machines)
self.assertTrue(self.deleter.called)
def test_delete_default_resource(self):
r = event.Resource(
id='1234',
tenant_id=self.tenant_id,
driver=router.Router.RESOURCE_NAME)
self.trm._default_resource_id = '1234'
self.trm.state_machines['1234'] = mock.Mock()
self.trm._delete_resource('1234')
self.trm._delete_resource(r)
self.assertNotIn('1234', self.trm.state_machines)
self.assertIs(None, self.trm._default_resource_id)
def test_delete_not_default_resource(self):
r = event.Resource(
id='1234',
tenant_id=self.tenant_id,
driver=router.Router.RESOURCE_NAME)
self.trm._default_resource_id = 'abcd'
self.trm.state_machines['1234'] = mock.Mock()
self.trm._delete_resource('1234')
self.trm._delete_resource(r)
self.assertEqual('abcd', self.trm._default_resource_id)
def test_no_update_deleted_resource(self):
r = event.Resource(
tenant_id='1234',
id='5678',
driver=router.Router.RESOURCE_NAME,
)
self.trm._default_resource_id = 'abcd'
self.trm.state_machines['5678'] = mock.Mock()
self.trm._delete_resource('5678')
self.trm._delete_resource(r)
self.assertEqual(self.trm.state_machines.values(), [])
r = event.Resource(
tenant_id='1234',

View File

@ -398,6 +398,29 @@ class TestResourceCache(WorkerTestBase):
self.w._context.neutron.get_router_for_tenant.assert_called_with(
'fake_tenant_id')
def test_resource_cache_delete(self):
r = event.Resource(
tenant_id='fake_tenant_id',
id='fake_fetched_resource_id',
driver=router.Router.RESOURCE_NAME,
)
msg = event.Event(
resource=r,
crud=event.UPDATE,
body={},
)
self.resource_cache.get_by_tenant(
resource=r,
worker_context=self.worker_context,
message=msg)
self.assertEqual(
self.resource_cache._tenant_resources[r.driver][r.tenant_id],
r.id)
self.resource_cache.delete(r)
self.assertNotIn(
r.tenant_id,
self.resource_cache._tenant_resources[r.driver])
class TestCreatingResource(WorkerTestBase):
def setUp(self):

View File

@ -27,6 +27,7 @@ import six
from logging import INFO
from oslo_config import cfg
from oslo_concurrency import lockutils
from oslo_log import log as logging
from astara import commands
@ -93,6 +94,7 @@ class TenantResourceCache(object):
# across mulitple rugs.
_tenant_resources = {}
@lockutils.synchronized('astara-trm')
def get_by_tenant(self, resource, worker_context, message):
tenant_id = resource.tenant_id
driver = resource.driver
@ -111,6 +113,14 @@ class TenantResourceCache(object):
return self._tenant_resources[driver][tenant_id]
@lockutils.synchronized('astara-trm')
def delete(self, resource):
"""Callback used to remove a resource from the cache upon deletion"""
try:
del self._tenant_resources[resource.driver][resource.tenant_id]
except KeyError:
pass
class WorkerContext(object):
"""Holds resources owned by the worker and used by the Automaton.
@ -298,10 +308,12 @@ class Worker(object):
LOG.debug('creating tenant manager for %s', tenant_id)
self.tenant_managers[tenant_id] = tenant.TenantResourceManager(
tenant_id=tenant_id,
delete_callback=self.resource_cache.delete,
notify_callback=self.notifier.publish,
queue_warning_threshold=self._queue_warning_threshold,
reboot_error_threshold=self._reboot_error_threshold,
)
return [self.tenant_managers[tenant_id]]
def _populate_resource_id(self, message):

View File

@ -0,0 +1,3 @@
---
fixes:
- Bug `1531597 <https://bugs.launchpad.net/astara/+bug/1531597/>`_ \- Deleted resources are properly invalidated from the local tenant resource cache

View File

@ -6,6 +6,7 @@ eventlet!=0.18.0,>=0.17.4 # MIT
netaddr!=0.7.16,>=0.7.12 # BSD
httplib2>=0.7.5 # MIT
python-neutronclient>=2.6.0 # Apache-2.0
oslo.concurrency>=2.3.0 # Apache-2.0
oslo.config>=3.2.0 # Apache-2.0
oslo.context>=0.2.0 # Apache-2.0
oslo.db>=4.1.0 # Apache-2.0