openstack
/
heat
Code Issues Proposed changes

Merge "Fix database purge query" into stable/kilo

This commit is contained in:
Jenkins 2016-03-10 17:06:14 +00:00 committed by Gerrit Code Review
parent 5cd0d123bc cb98419d62
commit f403f320e1
5 changed files with 124 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
heat/db/sqlalchemy/api.py
View File

@ -946,13 +946,14 @@ def purge_deleted(age, granularity='days'):
elif granularity == 'minutes':
age = age * 60
time_line = datetime.datetime.now() - datetime.timedelta(seconds=age)
time_line = datetime.datetime.utcnow() - datetime.timedelta(seconds=age)
engine = get_engine()
meta = sqlalchemy.MetaData()
meta.bind = engine
stack = sqlalchemy.Table('stack', meta, autoload=True)
stack_lock = sqlalchemy.Table('stack_lock', meta, autoload=True)
stack_tag = sqlalchemy.Table('stack_tag', meta, autoload=True)
resource = sqlalchemy.Table('resource', meta, autoload=True)
resource_data = sqlalchemy.Table('resource_data', meta, autoload=True)
event = sqlalchemy.Table('event', meta, autoload=True)
@ -962,40 +963,68 @@ def purge_deleted(age, granularity='days'):
syncpoint = sqlalchemy.Table('sync_point', meta, autoload=True)
# find the soft-deleted stacks that are past their expiry
stack_where = sqlalchemy.select([stack.c.id]).where(
stack.c.deleted_at < time_line)
# delete stack locks (just in case some got stuck)
stack_lock_del = stack_lock.delete().where(
stack_lock.c.stack_id.in_(stack_where))
engine.execute(stack_lock_del)
# delete resource_data
res_where = sqlalchemy.select([resource.c.id]).where(
resource.c.stack_id.in_(stack_where))
res_data_del = resource_data.delete().where(
resource_data.c.resource_id.in_(res_where))
engine.execute(res_data_del)
# delete resources
res_del = resource.delete().where(resource.c.stack_id.in_(stack_where))
engine.execute(res_del)
# delete events
event_del = event.delete().where(event.c.stack_id.in_(stack_where))
engine.execute(event_del)
# clean up any sync_points that may have lingered
sync_del = syncpoint.delete().where(syncpoint.c.stack_id.in_(stack_where))
engine.execute(sync_del)
# delete the stacks
stack_del = stack.delete().where(stack.c.deleted_at < time_line)
engine.execute(stack_del)
# delete orphaned raw templates
stack_templ_sel = sqlalchemy.select([stack.c.raw_template_id])
raw_templ_sel = sqlalchemy.not_(raw_template.c.id.in_(stack_templ_sel))
raw_templ_del = raw_template.delete().where(raw_templ_sel)
engine.execute(raw_templ_del)
# purge any user creds that are no longer referenced
stack_creds_sel = sqlalchemy.select([stack.c.user_creds_id])
user_creds_sel = sqlalchemy.not_(user_creds.c.id.in_(stack_creds_sel))
usr_creds_del = user_creds.delete().where(user_creds_sel)
engine.execute(usr_creds_del)
stack_where = sqlalchemy.select([stack.c.id, stack.c.raw_template_id,
stack.c.prev_raw_template_id,
stack.c.user_creds_id]).where(
stack.c.deleted_at < time_line)
stacks = list(engine.execute(stack_where))
if stacks:
stack_ids = [i[0] for i in stacks]
# delete stack locks (just in case some got stuck)
stack_lock_del = stack_lock.delete().where(
stack_lock.c.stack_id.in_(stack_ids))
engine.execute(stack_lock_del)
# delete stack tags
stack_tag_del = stack_tag.delete().where(
stack_tag.c.stack_id.in_(stack_ids))
engine.execute(stack_tag_del)
# delete resource_data
res_where = sqlalchemy.select([resource.c.id]).where(
resource.c.stack_id.in_(stack_ids))
res_data_del = resource_data.delete().where(
resource_data.c.resource_id.in_(res_where))
engine.execute(res_data_del)
# delete resources
res_del = resource.delete().where(resource.c.stack_id.in_(stack_ids))
engine.execute(res_del)
# delete events
event_del = event.delete().where(event.c.stack_id.in_(stack_ids))
engine.execute(event_del)
# clean up any sync_points that may have lingered
sync_del = syncpoint.delete().where(
syncpoint.c.stack_id.in_(stack_ids))
engine.execute(sync_del)
# delete the stacks
stack_del = stack.delete().where(stack.c.id.in_(stack_ids))
engine.execute(stack_del)
# delete orphaned raw templates
raw_template_ids = [i[1] for i in stacks if i[1] is not None]
raw_template_ids.extend(i[2] for i in stacks if i[2] is not None)
if raw_template_ids:
# keep those still referenced
raw_tmpl_sel = sqlalchemy.select([stack.c.raw_template_id]).where(
stack.c.raw_template_id.in_(raw_template_ids))
raw_tmpl = [i[0] for i in engine.execute(raw_tmpl_sel)]
raw_template_ids = set(raw_template_ids) - set(raw_tmpl)
raw_tmpl_sel = sqlalchemy.select(
[stack.c.prev_raw_template_id]).where(
stack.c.prev_raw_template_id.in_(raw_template_ids))
raw_tmpl = [i[0] for i in engine.execute(raw_tmpl_sel)]
raw_template_ids = raw_template_ids - set(raw_tmpl)
raw_templ_del = raw_template.delete().where(
raw_template.c.id.in_(raw_template_ids))
engine.execute(raw_templ_del)
# purge any user creds that are no longer referenced
user_creds_ids = [i[3] for i in stacks if i[3] is not None]
if user_creds_ids:
# keep those still referenced
user_sel = sqlalchemy.select([stack.c.user_creds_id]).where(
stack.c.user_creds_id.in_(user_creds_ids))
users = [i[0] for i in engine.execute(user_sel)]
user_creds_ids = set(user_creds_ids) - set(users)
usr_creds_del = user_creds.delete().where(
user_creds.c.id.in_(user_creds_ids))
engine.execute(usr_creds_del)
# Purge deleted services
srvc_del = service.delete().where(service.c.deleted_at < time_line)
engine.execute(srvc_del)

5
heat/objects/stack.py
View File

@ -21,6 +21,8 @@ from oslo_versionedobjects import base
from oslo_versionedobjects import fields
from heat.common import exception
from heat.common.i18n import _
from heat.db import api as db_api
from heat.objects import fields as heat_fields
from heat.objects import raw_template
@ -167,6 +169,9 @@ class Stack(
def refresh(self):
db_stack = db_api.stack_get(
self._context, self.id, show_deleted=True)
if db_stack is None:
message = _('No stack exists with id "%s"') % str(self.id)
raise exception.NotFound(message)
db_stack.refresh()
return self.__class__._from_db_object(
self._context,

2
heat/tests/db/test_sqlalchemy_api.py
View File

@ -1547,7 +1547,7 @@ class DBAPIStackTest(common.HeatTestCase):
db_api.stack_count_all(self.ctx, tenant_safe=False))
def test_purge_deleted(self):
now = datetime.datetime.now()
now = datetime.datetime.utcnow()
delta = datetime.timedelta(seconds=3600 * 7)
deleted = [now - delta * i for i in range(1, 6)]
templates = [create_raw_template(self.ctx) for i in range(5)]

7
heat_integrationtests/common/test.py
View File

@ -342,7 +342,7 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
return dict((r.resource_name, r.resource_type) for r in resources)
def stack_create(self, stack_name=None, template=None, files=None,
parameters=None, environment=None,
parameters=None, environment=None, tags=None,
expected_status='CREATE_COMPLETE', disable_rollback=True):
name = stack_name or self._stack_rand_name()
templ = template or self.template
@ -355,10 +355,11 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
files=templ_files,
disable_rollback=disable_rollback,
parameters=params,
environment=env
environment=env,
tags=tags
)
if expected_status not in ['ROLLBACK_COMPLETE']:
self.addCleanup(self.client.stacks.delete, name)
self.addCleanup(self._stack_delete, name)
stack = self.client.stacks.get(name)
stack_identifier = '%s/%s' % (name, stack.id)

50
heat_integrationtests/functional/test_purge.py Normal file
View File

@ -0,0 +1,50 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import subprocess
from heat_integrationtests.common import test
class PurgeTest(test.HeatIntegrationTest):
template = '''
heat_template_version: 2014-10-16
parameters:
resources:
test_resource:
type: OS::Heat::TestResource
'''
def setUp(self):
super(PurgeTest, self).setUp()
self.client = self.orchestration_client
def test_purge(self):
stack_identifier = self.stack_create(template=self.template)
self._stack_delete(stack_identifier)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertIn(stack_identifier.split('/')[1], stacks)
cmd = "heat-manage purge_deleted 0"
subprocess.check_call(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)
# Test with tags
stack_identifier = self.stack_create(template=self.template,
tags="foo,bar")
self._stack_delete(stack_identifier)
subprocess.check_call(cmd, shell=True)
stacks = dict((stack.id, stack) for stack in
self.client.stacks.list(show_deleted=True))
self.assertNotIn(stack_identifier.split('/')[1], stacks)