From a6ac8ce1db302a599d933e37e0879c5a22cae0fa Mon Sep 17 00:00:00 2001 From: Robert Myers Date: Thu, 1 Aug 2013 10:55:02 -0500 Subject: [PATCH] Adding volume size to the backup views/models. * Save the size of the guest before running a backup * Display the size in the backup GET/LIST views * Simplify the dbaas module removing redundant class. * Switch get_filesystem_volume_stats to use os.statvfs() * Change output of volume stats to return used in GB to simplify various view that display it. Implements: Blueprint backup-volume-size Change-Id: I860427d3491acb34e43ee23f91e4ffbb39e0042c --- trove/backup/views.py | 1 + trove/extensions/mgmt/instances/views.py | 6 +- trove/guestagent/api.py | 2 +- trove/guestagent/backup/backupagent.py | 4 + trove/guestagent/dbaas.py | 56 +++++++------ trove/guestagent/manager/mysql.py | 2 +- trove/instance/views.py | 5 +- trove/tests/api/backups.py | 3 + trove/tests/api/instances.py | 6 +- trove/tests/fakes/guestagent.py | 4 +- .../unittests/backup/test_backup_models.py | 6 ++ .../unittests/backup/test_backupagent.py | 9 +++ .../tests/unittests/guestagent/test_dbaas.py | 80 ++++++------------- .../unittests/taskmanager/test_models.py | 1 + 14 files changed, 91 insertions(+), 94 deletions(-) diff --git a/trove/backup/views.py b/trove/backup/views.py index 3685e4b99b..bd17efd1d1 100644 --- a/trove/backup/views.py +++ b/trove/backup/views.py @@ -30,6 +30,7 @@ class BackupView(object): "instance_id": self.backup.instance_id, "created": self.backup.created, "updated": self.backup.updated, + "size": self.backup.size, "status": self.backup.state } } diff --git a/trove/extensions/mgmt/instances/views.py b/trove/extensions/mgmt/instances/views.py index faaaf69db7..f5cd4e4ad1 100644 --- a/trove/extensions/mgmt/instances/views.py +++ b/trove/extensions/mgmt/instances/views.py @@ -72,10 +72,6 @@ class MgmtInstanceDetailView(MgmtInstanceView): result['instance']['root_enabled_by'] = self.root_history.user if self.instance.volume: volume = self.instance.volume - if self.instance.volume_used: - used = self._to_gb(self.instance.volume_used) - else: - used = None result['instance']['volume'] = { "attachments": volume.attachments, "availability_zone": volume.availability_zone, @@ -83,7 +79,7 @@ class MgmtInstanceDetailView(MgmtInstanceView): "id": volume.id, "size": volume.size, "status": volume.status, - "used": used, + "used": self.instance.volume_used or None, } else: result['instance']['volume'] = None diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py index b768e54e25..0700e373b4 100644 --- a/trove/guestagent/api.py +++ b/trove/guestagent/api.py @@ -253,7 +253,7 @@ class API(proxy.RpcProxy): LOG.debug(_("Check Volume Info on Instance %s"), self.id) # self._check_for_hearbeat() return self._call("get_filesystem_stats", AGENT_LOW_TIMEOUT, - fs_path="/var/lib/mysql") + fs_path=CONF.mount_point) def update_guest(self): """Make a synchronous call to update the guest agent.""" diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py index 884b55cf3d..e467dfc447 100644 --- a/trove/guestagent/backup/backupagent.py +++ b/trove/guestagent/backup/backupagent.py @@ -18,6 +18,7 @@ import logging from trove.backup.models import DBBackup from trove.backup.models import BackupState from trove.common import cfg, utils +from trove.guestagent.dbaas import get_filesystem_volume_stats from trove.guestagent.manager.mysql_service import ADMIN_USER_NAME from trove.guestagent.manager.mysql_service import get_auth_password from trove.guestagent.strategies.backup.base import BackupError @@ -59,6 +60,9 @@ class BackupAgent(object): CONF.storage_strategy, CONF.storage_namespace)(context) + # Store the size of the filesystem before the backup. + stats = get_filesystem_volume_stats(CONF.mount_point) + backup.size = stats.get('used', 0.0) backup.state = BackupState.BUILDING backup.save() diff --git a/trove/guestagent/dbaas.py b/trove/guestagent/dbaas.py index 5e698bb150..5fc7442a4f 100644 --- a/trove/guestagent/dbaas.py +++ b/trove/guestagent/dbaas.py @@ -25,33 +25,43 @@ handles RPC calls relating to Platform specific operations. """ +import os -from trove.common import utils -from trove.openstack.common import log as logging +from trove.openstack.common import log -LOG = logging.getLogger(__name__) +LOG = log.getLogger(__name__) SERVICE_REGISTRY = { 'mysql': 'trove.guestagent.manager.mysql.Manager', - 'percona': 'trove.guestagent.manager.mysql.Manager', } + 'percona': 'trove.guestagent.manager.mysql.Manager', +} -class Interrogator(object): - def get_filesystem_volume_stats(self, fs_path): - out, err = utils.execute_with_timeout( - "stat", - "-f", - "-t", - fs_path) - if err: - LOG.error(err) - raise RuntimeError("Filesystem not found (%s) : %s" - % (fs_path, err)) - stats = out.split() - output = {'block_size': int(stats[4]), - 'total_blocks': int(stats[6]), - 'free_blocks': int(stats[7]), - 'total': int(stats[6]) * int(stats[4]), - 'free': int(stats[7]) * int(stats[4])} - output['used'] = int(output['total']) - int(output['free']) - return output +def to_gb(bytes): + if bytes == 0: + return 0.0 + size = bytes / 1024.0 ** 3 + return round(size, 2) + + +def get_filesystem_volume_stats(fs_path): + try: + stats = os.statvfs(fs_path) + except OSError: + LOG.exception("Error getting volume stats.") + raise RuntimeError("Filesystem not found (%s)" % fs_path) + + total = stats.f_blocks * stats.f_bsize + free = stats.f_bfree * stats.f_bsize + # return the size in GB + used = to_gb(total - free) + + output = { + 'block_size': stats.f_bsize, + 'total_blocks': stats.f_blocks, + 'free_blocks': stats.f_bfree, + 'total': total, + 'free': free, + 'used': used + } + return output diff --git a/trove/guestagent/manager/mysql.py b/trove/guestagent/manager/mysql.py index bc277274fb..cde360715a 100644 --- a/trove/guestagent/manager/mysql.py +++ b/trove/guestagent/manager/mysql.py @@ -140,7 +140,7 @@ class Manager(periodic_task.PeriodicTasks): def get_filesystem_stats(self, context, fs_path): """ Gets the filesystem stats for the path given """ - return dbaas.Interrogator().get_filesystem_volume_stats(fs_path) + return dbaas.get_filesystem_volume_stats(fs_path) def create_backup(self, context, backup_id): """ diff --git a/trove/instance/views.py b/trove/instance/views.py index e136525296..55347fbf8a 100644 --- a/trove/instance/views.py +++ b/trove/instance/views.py @@ -79,9 +79,6 @@ class InstanceDetailView(InstanceView): super(InstanceDetailView, self).__init__(instance, req=req) - def _to_gb(self, bytes): - return bytes / 1024.0 ** 3 - def data(self): result = super(InstanceDetailView, self).data() result['instance']['created'] = self.instance.created @@ -98,7 +95,7 @@ class InstanceDetailView(InstanceView): if isinstance(self.instance, models.DetailInstance) and \ self.instance.volume_used: - used = self._to_gb(self.instance.volume_used) + used = self.instance.volume_used if CONF.trove_volume_support: result['instance']['volume']['used'] = used else: diff --git a/trove/tests/api/backups.py b/trove/tests/api/backups.py index 982f427011..1f13f9ce4e 100644 --- a/trove/tests/api/backups.py +++ b/trove/tests/api/backups.py @@ -145,6 +145,7 @@ class ListBackups(object): backup = result[0] assert_equal(BACKUP_NAME, backup.name) assert_equal(BACKUP_DESC, backup.description) + assert_not_equal(0.0, backup.size) assert_equal(instance_info.id, backup.instance_id) assert_equal('COMPLETED', backup.status) @@ -156,6 +157,7 @@ class ListBackups(object): backup = result[0] assert_equal(BACKUP_NAME, backup.name) assert_equal(BACKUP_DESC, backup.description) + assert_not_equal(0.0, backup.size) assert_equal(instance_info.id, backup.instance_id) assert_equal('COMPLETED', backup.status) @@ -167,6 +169,7 @@ class ListBackups(object): assert_equal(backup_info.name, backup.name) assert_equal(backup_info.description, backup.description) assert_equal(instance_info.id, backup.instance_id) + assert_not_equal(0.0, backup.size) assert_equal('COMPLETED', backup.status) # Test to make sure that user in other tenant is not able diff --git a/trove/tests/api/instances.py b/trove/tests/api/instances.py index 2aef81ae1e..2a95406afe 100644 --- a/trove/tests/api/instances.py +++ b/trove/tests/api/instances.py @@ -892,19 +892,19 @@ class TestInstanceListing(object): if create_new_instance(): assert_equal(instance_info.volume['size'], instance.volume['size']) else: - assert_true(isinstance(instance_info.volume['size'], int)) + assert_true(isinstance(instance_info.volume['size'], float)) if create_new_instance(): assert_true(0.12 < instance.volume['used'] < 0.25) @test(enabled=EPHEMERAL_SUPPORT) def test_ephemeral_mount(self): instance = dbaas.instances.get(instance_info.id) - assert_true(isinstance(instance_info.local_storage['used'], int)) + assert_true(isinstance(instance.local_storage['used'], float)) @test(enabled=ROOT_PARTITION) def test_root_partition(self): instance = dbaas.instances.get(instance_info.id) - assert_true(isinstance(instance_info.local_storage['used'], int)) + assert_true(isinstance(instance.local_storage['used'], float)) @test(enabled=do_not_delete_instance()) def test_instance_not_shown_to_other_user(self): diff --git a/trove/tests/fakes/guestagent.py b/trove/tests/fakes/guestagent.py index 3703b74a27..aada60905f 100644 --- a/trove/tests/fakes/guestagent.py +++ b/trove/tests/fakes/guestagent.py @@ -234,8 +234,8 @@ class FakeGuest(object): self._set_status('SHUTDOWN') def get_volume_info(self): - """Return used volume information in bytes.""" - return {'used': 175756487} + """Return used volume information in GB.""" + return {'used': 0.16} def grant_access(self, username, hostname, databases): """Add a database to a users's grant list.""" diff --git a/trove/tests/unittests/backup/test_backup_models.py b/trove/tests/unittests/backup/test_backup_models.py index 492b95f6ea..64da90b6c6 100644 --- a/trove/tests/unittests/backup/test_backup_models.py +++ b/trove/tests/unittests/backup/test_backup_models.py @@ -140,6 +140,7 @@ class BackupORMTest(testtools.TestCase): state=BACKUP_STATE, instance_id=self.instance_id, deleted=False, + size=2.0, location=BACKUP_LOCATION) self.deleted = False @@ -158,6 +159,7 @@ class BackupORMTest(testtools.TestCase): name=BACKUP_NAME_2, state=BACKUP_STATE, instance_id=self.instance_id, + size=2.0, deleted=False) db_record = models.Backup.list_for_instance(self.instance_id) self.assertEqual(2, db_record.count()) @@ -191,6 +193,10 @@ class BackupORMTest(testtools.TestCase): def test_not_is_done(self): self.assertFalse(self.backup.is_done) + def test_backup_size(self): + db_record = models.DBBackup.find_by(id=self.backup.id) + self.assertEqual(db_record.size, self.backup.size) + def test_backup_delete(self): backup = models.DBBackup.find_by(id=self.backup.id) backup.delete() diff --git a/trove/tests/unittests/backup/test_backupagent.py b/trove/tests/unittests/backup/test_backupagent.py index 5025199de5..718b9c845a 100644 --- a/trove/tests/unittests/backup/test_backupagent.py +++ b/trove/tests/unittests/backup/test_backupagent.py @@ -13,6 +13,7 @@ #limitations under the License. import hashlib +import os from trove.common import utils from trove.common.context import TroveContext from trove.guestagent.strategies.restore.base import RestoreRunner @@ -129,6 +130,13 @@ class MockRestoreRunner(RestoreRunner): def is_zipped(self): return False + +class MockStats: + f_blocks = 1024 ** 2 + f_bsize = 4096 + f_bfree = 512 * 1024 + + BACKUP_NS = 'trove.guestagent.strategies.backup' @@ -139,6 +147,7 @@ class BackupAgentTest(testtools.TestCase): when(backupagent).get_auth_password().thenReturn('secret') when(backupagent).get_storage_strategy(any(), any()).thenReturn( MockSwift) + when(os).statvfs(any()).thenReturn(MockStats) def tearDown(self): super(BackupAgentTest, self).tearDown() diff --git a/trove/tests/unittests/guestagent/test_dbaas.py b/trove/tests/unittests/guestagent/test_dbaas.py index a4a06d12ac..5e49f79e64 100644 --- a/trove/tests/unittests/guestagent/test_dbaas.py +++ b/trove/tests/unittests/guestagent/test_dbaas.py @@ -13,7 +13,6 @@ # under the License. import os -import __builtin__ from random import randint import time @@ -39,12 +38,13 @@ from trove.common.context import TroveContext from trove.guestagent import pkg from trove.common import utils import trove.guestagent.manager.mysql_service as dbaas +from trove.guestagent.dbaas import to_gb +from trove.guestagent.dbaas import get_filesystem_volume_stats from trove.guestagent.manager.mysql_service import MySqlAdmin from trove.guestagent.manager.mysql_service import MySqlRootAccess from trove.guestagent.manager.mysql_service import MySqlApp from trove.guestagent.manager.mysql_service import MySqlAppStatus from trove.guestagent.manager.mysql_service import KeepAliveConnection -from trove.guestagent.dbaas import Interrogator from trove.guestagent.db import models from trove.instance.models import ServiceStatuses from trove.instance.models import InstanceServiceStatus @@ -834,67 +834,37 @@ class MySqlRootStatusTest(testtools.TestCase): verify(mock_db_api).save(any(RootHistory)) +class MockStats: + f_blocks = 1024 ** 2 + f_bsize = 4096 + f_bfree = 512 * 1024 + + class InterrogatorTest(testtools.TestCase): - def setUp(self): - super(InterrogatorTest, self).setUp() - self.orig_utils_execute_with_timeout = dbaas.utils.execute_with_timeout - self.orig_LOG_err = dbaas.LOG + def test_to_gb(self): + result = to_gb(123456789) + self.assertEqual(result, 0.11) - def tearDown(self): - super(InterrogatorTest, self).tearDown() - dbaas.utils.execute_with_timeout = self.orig_utils_execute_with_timeout - dbaas.LOG = self.orig_LOG_err + def test_to_gb_zero(self): + result = to_gb(0) + self.assertEqual(result, 0.0) def test_get_filesystem_volume_stats(self): + when(os).statvfs(any()).thenReturn(MockStats) + result = get_filesystem_volume_stats('/some/path/') - path = 'aPath' - block_size = 4096 - total_block = 2582828 - free_block = 767118 - total = total_block * block_size - free = free_block * block_size - used = total - free - out = " ".join(str(x) for x in (path, 'fb518d79428291bb', 255, 'ef53', - block_size, '4096', total_block, - free_block, 636216, 655360, 583768)) - err = None - return_exp = out, err - dbaas.utils.execute_with_timeout = Mock(return_value=return_exp) - - self.interrogator = Interrogator() - result = self.interrogator.get_filesystem_volume_stats(path) - - self.assertTrue(dbaas.utils.execute_with_timeout.called) - self.assertTrue('stat' in - dbaas.utils.execute_with_timeout.call_args[0]) - self.assertTrue(path in dbaas.utils.execute_with_timeout.call_args[0]) - - self.assertEqual(result['block_size'], block_size) - self.assertEqual(result['total_blocks'], total_block) - self.assertEqual(result['free_blocks'], free_block) - self.assertEqual(result['total'], total) - self.assertEqual(result['free'], free) - self.assertEqual(result['used'], used) + self.assertEqual(result['block_size'], 4096) + self.assertEqual(result['total_blocks'], 1048576) + self.assertEqual(result['free_blocks'], 524288) + self.assertEqual(result['total'], 4294967296) + self.assertEqual(result['free'], 2147483648) + self.assertEqual(result['used'], 2.0) def test_get_filesystem_volume_stats_error(self): - - path = 'aPath' - block_size = 4096 - total_block = 2582828 - free_block = 767118 - - out = " ".join(str(x) for x in (path, 'fb518d79428291bb', 255, 'ef53', - block_size, '4096', total_block, - free_block, 636216, 655360, 583768)) - err = "Error found" - return_exp = out, err - dbaas.utils.execute_with_timeout = Mock(return_value=return_exp) - dbaas.LOG.err = Mock() - - self.interrogator = Interrogator() - self.assertRaises(RuntimeError, - self.interrogator.get_filesystem_volume_stats, path) + self.assertRaises( + RuntimeError, + get_filesystem_volume_stats, '/nonexistent/path') class KeepAliveConnectionTest(testtools.TestCase): diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py index 6c3a4a2830..c73919de82 100644 --- a/trove/tests/unittests/taskmanager/test_models.py +++ b/trove/tests/unittests/taskmanager/test_models.py @@ -32,6 +32,7 @@ class BackupTasksTest(testtools.TestCase): self.backup.instance_id = 'instance id' self.backup.created = 'yesterday' self.backup.updated = 'today' + self.backup.size = 2.0 self.backup.state = backup_models.BackupState.NEW self.container_content = (None, [{'name': 'first'},