openstack
/
ceilometer
Code Issues Proposed changes

add newly added constraints to expire clear_expired_metering_data 

- cascade deletes to metadata tables on meter deletion
- delete associations on object deletion
- add check to not delete user/project entries associated with Alarm

Change-Id: I24cbaea9ede6d142dc22eaf070b3f7d1f14b484a
Closes-Bug: #1258071
This commit is contained in:
Gordon Chung 2013-12-09 17:53:04 -05:00
parent 5472a1dba0
commit 13764537a4
4 changed files with 139 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
ceilometer/storage/impl_sqlalchemy.py
View File

@ -279,28 +279,49 @@ class Connection(base.Connection):
:param ttl: Number of seconds to keep records for.
"""
session = sqlalchemy_session.get_session()
query = session.query(models.Meter.id)
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
query = query.filter(models.Meter.timestamp < end)
query.delete()
with session.begin():
end = timeutils.utcnow() - datetime.timedelta(seconds=ttl)
meter_query = session.query(models.Meter)\
.filter(models.Meter.timestamp < end)
for meter_obj in meter_query.all():
session.delete(meter_obj)
query = session.query(models.User.id).filter(~models.User.id.in_(
session.query(models.Meter.user_id).group_by(models.Meter.user_id)
))
query.delete(synchronize_session='fetch')
query = session.query(models.User).filter(
~models.User.id.in_(session.query(models.Meter.user_id)
.group_by(models.Meter.user_id)),
~models.User.id.in_(session.query(models.Alarm.user_id)
.group_by(models.Alarm.user_id)),
~models.User.id.in_(session.query(models.AlarmChange.user_id)
.group_by(models.AlarmChange.user_id))
)
for user_obj in query.all():
session.delete(user_obj)
query = session.query(models.Project.id)\
.filter(~models.Project.id.in_(
session.query(models.Meter.project_id).group_by(
models.Meter.project_id)))
query.delete(synchronize_session='fetch')
query = session.query(models.Project)\
.filter(~models.Project.id.in_(
session.query(models.Meter.project_id)
.group_by(models.Meter.project_id)),
~models.Project.id.in_(
session.query(models.Alarm.project_id)
.group_by(models.Alarm.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.project_id)
.group_by(models.AlarmChange.project_id)),
~models.Project.id.in_(
session.query(models.AlarmChange.on_behalf_of)
.group_by(models.AlarmChange.on_behalf_of))
)
for project_obj in query.all():
session.delete(project_obj)
query = session.query(models.Resource.id)\
.filter(~models.Resource.id.in_(
session.query(models.Meter.resource_id).group_by(
models.Meter.resource_id)))
query.delete(synchronize_session='fetch')
query = session.query(models.Resource)\
.filter(~models.Resource.id.in_(
session.query(models.Meter.resource_id).group_by(
models.Meter.resource_id)))
for res_obj in query.all():
session.delete(res_obj)
@staticmethod
def get_users(source=None):

10
ceilometer/storage/sqlalchemy/models.py
View File

@ -201,7 +201,6 @@ class Meter(Base):
)
id = Column(Integer, primary_key=True)
counter_name = Column(String(255))
sources = relationship("Source", secondary=lambda: sourceassoc)
user_id = Column(String(255), ForeignKey('user.id'))
project_id = Column(String(255), ForeignKey('project.id'))
resource_id = Column(String(255), ForeignKey('resource.id'))
@ -212,6 +211,15 @@ class Meter(Base):
timestamp = Column(PreciseTimestamp(), default=timeutils.utcnow)
message_signature = Column(String(1000))
message_id = Column(String(1000))
sources = relationship("Source", secondary=lambda: sourceassoc)
meta_text = relationship("MetaText", backref="meter",
cascade="all, delete-orphan")
meta_float = relationship("MetaFloat", backref="meter",
cascade="all, delete-orphan")
meta_int = relationship("MetaBigInt", backref="meter",
cascade="all, delete-orphan")
meta_bool = relationship("MetaBool", backref="meter",
cascade="all, delete-orphan")
class User(Base):

42
ceilometer/tests/storage/test_impl_sqlalchemy.py
View File

@ -28,9 +28,12 @@ import repr
from mock import patch
import ceilometer.openstack.common.db.sqlalchemy.session as sqlalchemy_session
from ceilometer.openstack.common import timeutils
from ceilometer.storage import models
from ceilometer.storage.sqlalchemy import models as sql_models
from ceilometer.tests import db as tests_db
from ceilometer.tests.storage import test_storage_scenarios as scenarios
from ceilometer import utils
@ -175,3 +178,42 @@ class ModelTest(tests_db.TestBase):
def test_model_table_args(self):
self.assertIsNotNone(sql_models.table_args())
class RelationshipTest(scenarios.DBTestBase):
database_connection = 'mysql://localhost'
def test_clear_metering_data_meta_tables(self):
timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45)
self.conn.clear_expired_metering_data(3 * 60)
session = sqlalchemy_session.get_session()
meta_tables = [sql_models.MetaText, sql_models.MetaFloat,
sql_models.MetaBigInt, sql_models.MetaBool]
for table in meta_tables:
self.assertEqual(session.query(table)
.filter(~table.id.in_(
session.query(sql_models.Meter.id)
.group_by(sql_models.Meter.id)
)).count(), 0)
def test_clear_metering_data_associations(self):
timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45)
self.conn.clear_expired_metering_data(3 * 60)
session = sqlalchemy_session.get_session()
self.assertEqual(session.query(sql_models.sourceassoc)
.filter(~sql_models.sourceassoc.c.meter_id.in_(
session.query(sql_models.Meter.id)
.group_by(sql_models.Meter.id)
)).count(), 0)
self.assertEqual(session.query(sql_models.sourceassoc)
.filter(~sql_models.sourceassoc.c.project_id.in_(
session.query(sql_models.Project.id)
.group_by(sql_models.Project.id)
)).count(), 0)
self.assertEqual(session.query(sql_models.sourceassoc)
.filter(~sql_models.sourceassoc.c.user_id.in_(
session.query(sql_models.User.id)
.group_by(sql_models.User.id)
)).count(), 0)

50
ceilometer/tests/storage/test_storage_scenarios.py
View File

@ -16,7 +16,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
""" Base classes for DB backend implemtation test
""" Base classes for DB backend implementation test
"""
import datetime
@ -662,6 +662,54 @@ class RawSampleTest(DBTestBase,
results = list(self.conn.get_resources())
self.assertEqual(len(results), 9)
def test_clear_metering_data_with_alarms(self):
# NOTE(jd) Override this test in MongoDB because our code doesn't clear
# the collections, this is handled by MongoDB TTL feature.
if self.CONF.database.connection.startswith('mongodb://'):
return
alarm = models.Alarm(alarm_id='r3d',
enabled=True,
type='threshold',
name='red-alert',
description='my red-alert',
timestamp=None,
user_id='user-id',
project_id='project-id',
state="insufficient data",
state_timestamp=None,
ok_actions=[],
alarm_actions=['http://nowhere/alarms'],
insufficient_data_actions=[],
repeat_actions=False,
rule=dict(comparison_operator='eq',
threshold=36,
statistic='count',
evaluation_periods=1,
period=60,
meter_name='test.one',
query=[{'field': 'key',
'op': 'eq',
'value': 'value',
'type': 'string'}]),
)
self.conn.create_alarm(alarm)
timeutils.utcnow.override_time = datetime.datetime(2012, 7, 2, 10, 45)
self.conn.clear_expired_metering_data(5)
# user and project with Alarms associated with it aren't deleted.
f = storage.SampleFilter(meter='instance')
results = list(self.conn.get_samples(f))
self.assertEqual(len(results), 2)
results = list(self.conn.get_users())
self.assertEqual(len(results), 3)
self.assertIn('user-id', results)
results = list(self.conn.get_projects())
self.assertEqual(len(results), 3)
self.assertIn('project-id', results)
results = list(self.conn.get_resources())
self.assertEqual(len(results), 2)
class StatisticsTest(DBTestBase,
tests_db.MixinTestsWithBackendScenarios):