Merge "Adding volume size to the backup views/models."

trove/backup/views.py

@@ -30,6 +30,7 @@ class BackupView(object):
             "instance_id": self.backup.instance_id,
             "created": self.backup.created,
             "updated": self.backup.updated,
+            "size": self.backup.size,
             "status": self.backup.state
         }
         }

trove/extensions/mgmt/instances/views.py

@@ -72,10 +72,6 @@ class MgmtInstanceDetailView(MgmtInstanceView):
             result['instance']['root_enabled_by'] = self.root_history.user
         if self.instance.volume:
             volume = self.instance.volume
-            if self.instance.volume_used:
-                used = self._to_gb(self.instance.volume_used)
-            else:
-                used = None
             result['instance']['volume'] = {
                 "attachments": volume.attachments,
                 "availability_zone": volume.availability_zone,
@@ -83,7 +79,7 @@ class MgmtInstanceDetailView(MgmtInstanceView):
                 "id": volume.id,
                 "size": volume.size,
                 "status": volume.status,
-                "used": used,
+                "used": self.instance.volume_used or None,
             }
         else:
             result['instance']['volume'] = None

trove/guestagent/api.py

@@ -259,7 +259,7 @@ class API(proxy.RpcProxy):
         LOG.debug(_("Check Volume Info on Instance %s"), self.id)
         # self._check_for_hearbeat()
         return self._call("get_filesystem_stats", AGENT_LOW_TIMEOUT,
-                          fs_path="/var/lib/mysql")
+                          fs_path=CONF.mount_point)
 
     def update_guest(self):
         """Make a synchronous call to update the guest agent."""

trove/guestagent/backup/backupagent.py

@@ -18,6 +18,7 @@ import logging
 from trove.backup.models import DBBackup
 from trove.backup.models import BackupState
 from trove.common import cfg, utils
+from trove.guestagent.dbaas import get_filesystem_volume_stats
 from trove.guestagent.manager.mysql_service import ADMIN_USER_NAME
 from trove.guestagent.manager.mysql_service import get_auth_password
 from trove.guestagent.strategies.backup.base import BackupError
@@ -59,6 +60,9 @@ class BackupAgent(object):
             CONF.storage_strategy,
             CONF.storage_namespace)(context)
 
+        # Store the size of the filesystem before the backup.
+        stats = get_filesystem_volume_stats(CONF.mount_point)
+        backup.size = stats.get('used', 0.0)
         backup.state = BackupState.BUILDING
         backup.save()
 

trove/guestagent/dbaas.py

@@ -25,33 +25,43 @@ handles RPC calls relating to Platform specific operations.
 
 """
 
+import os
 
-from trove.common import utils
+from trove.openstack.common import log
-from trove.openstack.common import log as logging
 
 
-LOG = logging.getLogger(__name__)
+LOG = log.getLogger(__name__)
 SERVICE_REGISTRY = {
     'mysql': 'trove.guestagent.manager.mysql.Manager',
-    'percona': 'trove.guestagent.manager.mysql.Manager', }
+    'percona': 'trove.guestagent.manager.mysql.Manager',
+}
 
 
-class Interrogator(object):
+def to_gb(bytes):
-    def get_filesystem_volume_stats(self, fs_path):
+    if bytes == 0:
-        out, err = utils.execute_with_timeout(
+        return 0.0
-            "stat",
+    size = bytes / 1024.0 ** 3
-            "-f",
+    return round(size, 2)
-            "-t",
+
-            fs_path)
+
-        if err:
+def get_filesystem_volume_stats(fs_path):
-            LOG.error(err)
+    try:
-            raise RuntimeError("Filesystem not found (%s) : %s"
+        stats = os.statvfs(fs_path)
-                               % (fs_path, err))
+    except OSError:
-        stats = out.split()
+        LOG.exception("Error getting volume stats.")
-        output = {'block_size': int(stats[4]),
+        raise RuntimeError("Filesystem not found (%s)" % fs_path)
-                  'total_blocks': int(stats[6]),
+
-                  'free_blocks': int(stats[7]),
+    total = stats.f_blocks * stats.f_bsize
-                  'total': int(stats[6]) * int(stats[4]),
+    free = stats.f_bfree * stats.f_bsize
-                  'free': int(stats[7]) * int(stats[4])}
+    # return the size in GB
-        output['used'] = int(output['total']) - int(output['free'])
+    used = to_gb(total - free)
-        return output
+
+    output = {
+        'block_size': stats.f_bsize,
+        'total_blocks': stats.f_blocks,
+        'free_blocks': stats.f_bfree,
+        'total': total,
+        'free': free,
+        'used': used
+    }
+    return output

trove/guestagent/manager/mysql.py

@@ -143,7 +143,7 @@ class Manager(periodic_task.PeriodicTasks):
 
     def get_filesystem_stats(self, context, fs_path):
         """ Gets the filesystem stats for the path given """
-        return dbaas.Interrogator().get_filesystem_volume_stats(fs_path)
+        return dbaas.get_filesystem_volume_stats(fs_path)
 
     def create_backup(self, context, backup_id):
         """

trove/instance/views.py

@@ -79,9 +79,6 @@ class InstanceDetailView(InstanceView):
         super(InstanceDetailView, self).__init__(instance,
                                                  req=req)
 
-    def _to_gb(self, bytes):
-        return bytes / 1024.0 ** 3
-
     def data(self):
         result = super(InstanceDetailView, self).data()
         result['instance']['created'] = self.instance.created
@@ -98,7 +95,7 @@ class InstanceDetailView(InstanceView):
 
         if isinstance(self.instance, models.DetailInstance) and \
             self.instance.volume_used:
-            used = self._to_gb(self.instance.volume_used)
+            used = self.instance.volume_used
             if CONF.trove_volume_support:
                 result['instance']['volume']['used'] = used
             else:

trove/tests/api/backups.py

@@ -145,6 +145,7 @@ class ListBackups(object):
         backup = result[0]
         assert_equal(BACKUP_NAME, backup.name)
         assert_equal(BACKUP_DESC, backup.description)
+        assert_not_equal(0.0, backup.size)
         assert_equal(instance_info.id, backup.instance_id)
         assert_equal('COMPLETED', backup.status)
 
@@ -156,6 +157,7 @@ class ListBackups(object):
         backup = result[0]
         assert_equal(BACKUP_NAME, backup.name)
         assert_equal(BACKUP_DESC, backup.description)
+        assert_not_equal(0.0, backup.size)
         assert_equal(instance_info.id, backup.instance_id)
         assert_equal('COMPLETED', backup.status)
 
@@ -167,6 +169,7 @@ class ListBackups(object):
         assert_equal(backup_info.name, backup.name)
         assert_equal(backup_info.description, backup.description)
         assert_equal(instance_info.id, backup.instance_id)
+        assert_not_equal(0.0, backup.size)
         assert_equal('COMPLETED', backup.status)
 
         # Test to make sure that user in other tenant is not able

trove/tests/api/instances.py

@@ -890,19 +890,19 @@ class TestInstanceListing(object):
         if create_new_instance():
             assert_equal(instance_info.volume['size'], instance.volume['size'])
         else:
-            assert_true(isinstance(instance_info.volume['size'], int))
+            assert_true(isinstance(instance_info.volume['size'], float))
         if create_new_instance():
             assert_true(0.12 < instance.volume['used'] < 0.25)
 
     @test(enabled=EPHEMERAL_SUPPORT)
     def test_ephemeral_mount(self):
         instance = dbaas.instances.get(instance_info.id)
-        assert_true(isinstance(instance_info.local_storage['used'], int))
+        assert_true(isinstance(instance.local_storage['used'], float))
 
     @test(enabled=ROOT_PARTITION)
     def test_root_partition(self):
         instance = dbaas.instances.get(instance_info.id)
-        assert_true(isinstance(instance_info.local_storage['used'], int))
+        assert_true(isinstance(instance.local_storage['used'], float))
 
     @test(enabled=do_not_delete_instance())
     def test_instance_not_shown_to_other_user(self):

trove/tests/fakes/guestagent.py

@@ -261,8 +261,8 @@ class FakeGuest(object):
         self._set_status('SHUTDOWN')
 
     def get_volume_info(self):
-        """Return used volume information in bytes."""
+        """Return used volume information in GB."""
-        return {'used': 175756487}
+        return {'used': 0.16}
 
     def grant_access(self, username, hostname, databases):
         """Add a database to a users's grant list."""

trove/tests/unittests/backup/test_backup_models.py

@@ -140,6 +140,7 @@ class BackupORMTest(testtools.TestCase):
                                              state=BACKUP_STATE,
                                              instance_id=self.instance_id,
                                              deleted=False,
+                                             size=2.0,
                                              location=BACKUP_LOCATION)
         self.deleted = False
 
@@ -158,6 +159,7 @@ class BackupORMTest(testtools.TestCase):
                                name=BACKUP_NAME_2,
                                state=BACKUP_STATE,
                                instance_id=self.instance_id,
+                               size=2.0,
                                deleted=False)
         db_record = models.Backup.list_for_instance(self.instance_id)
         self.assertEqual(2, db_record.count())
@@ -191,6 +193,10 @@ class BackupORMTest(testtools.TestCase):
     def test_not_is_done(self):
         self.assertFalse(self.backup.is_done)
 
+    def test_backup_size(self):
+        db_record = models.DBBackup.find_by(id=self.backup.id)
+        self.assertEqual(db_record.size, self.backup.size)
+
     def test_backup_delete(self):
         backup = models.DBBackup.find_by(id=self.backup.id)
         backup.delete()

trove/tests/unittests/backup/test_backupagent.py

@@ -13,6 +13,7 @@
 #limitations under the License.
 
 import hashlib
+import os
 from trove.common import utils
 from trove.common.context import TroveContext
 from trove.guestagent.strategies.restore.base import RestoreRunner
@@ -129,6 +130,13 @@ class MockRestoreRunner(RestoreRunner):
     def is_zipped(self):
         return False
 
+
+class MockStats:
+    f_blocks = 1024 ** 2
+    f_bsize = 4096
+    f_bfree = 512 * 1024
+
+
 BACKUP_NS = 'trove.guestagent.strategies.backup'
 
 
@@ -139,6 +147,7 @@ class BackupAgentTest(testtools.TestCase):
         when(backupagent).get_auth_password().thenReturn('secret')
         when(backupagent).get_storage_strategy(any(), any()).thenReturn(
             MockSwift)
+        when(os).statvfs(any()).thenReturn(MockStats)
 
     def tearDown(self):
         super(BackupAgentTest, self).tearDown()

trove/tests/unittests/guestagent/test_dbaas.py

@@ -13,7 +13,6 @@
 #    under the License.
 
 import os
-import __builtin__
 from random import randint
 import time
 
@@ -39,12 +38,13 @@ from trove.common.context import TroveContext
 from trove.guestagent import pkg
 from trove.common import utils
 import trove.guestagent.manager.mysql_service as dbaas
+from trove.guestagent.dbaas import to_gb
+from trove.guestagent.dbaas import get_filesystem_volume_stats
 from trove.guestagent.manager.mysql_service import MySqlAdmin
 from trove.guestagent.manager.mysql_service import MySqlRootAccess
 from trove.guestagent.manager.mysql_service import MySqlApp
 from trove.guestagent.manager.mysql_service import MySqlAppStatus
 from trove.guestagent.manager.mysql_service import KeepAliveConnection
-from trove.guestagent.dbaas import Interrogator
 from trove.guestagent.db import models
 from trove.instance.models import ServiceStatuses
 from trove.instance.models import InstanceServiceStatus
@@ -834,67 +834,37 @@ class MySqlRootStatusTest(testtools.TestCase):
         verify(mock_db_api).save(any(RootHistory))
 
 
+class MockStats:
+    f_blocks = 1024 ** 2
+    f_bsize = 4096
+    f_bfree = 512 * 1024
+
+
 class InterrogatorTest(testtools.TestCase):
 
-    def setUp(self):
+    def test_to_gb(self):
-        super(InterrogatorTest, self).setUp()
+        result = to_gb(123456789)
-        self.orig_utils_execute_with_timeout = dbaas.utils.execute_with_timeout
+        self.assertEqual(result, 0.11)
-        self.orig_LOG_err = dbaas.LOG
 
-    def tearDown(self):
+    def test_to_gb_zero(self):
-        super(InterrogatorTest, self).tearDown()
+        result = to_gb(0)
-        dbaas.utils.execute_with_timeout = self.orig_utils_execute_with_timeout
+        self.assertEqual(result, 0.0)
-        dbaas.LOG = self.orig_LOG_err
 
     def test_get_filesystem_volume_stats(self):
+        when(os).statvfs(any()).thenReturn(MockStats)
+        result = get_filesystem_volume_stats('/some/path/')
 
-        path = 'aPath'
+        self.assertEqual(result['block_size'], 4096)
-        block_size = 4096
+        self.assertEqual(result['total_blocks'], 1048576)
-        total_block = 2582828
+        self.assertEqual(result['free_blocks'], 524288)
-        free_block = 767118
+        self.assertEqual(result['total'], 4294967296)
-        total = total_block * block_size
+        self.assertEqual(result['free'], 2147483648)
-        free = free_block * block_size
+        self.assertEqual(result['used'], 2.0)
-        used = total - free
-        out = " ".join(str(x) for x in (path, 'fb518d79428291bb', 255, 'ef53',
-                                        block_size, '4096', total_block,
-                                        free_block, 636216, 655360, 583768))
-        err = None
-        return_exp = out, err
-        dbaas.utils.execute_with_timeout = Mock(return_value=return_exp)
-
-        self.interrogator = Interrogator()
-        result = self.interrogator.get_filesystem_volume_stats(path)
-
-        self.assertTrue(dbaas.utils.execute_with_timeout.called)
-        self.assertTrue('stat' in
-                        dbaas.utils.execute_with_timeout.call_args[0])
-        self.assertTrue(path in dbaas.utils.execute_with_timeout.call_args[0])
-
-        self.assertEqual(result['block_size'], block_size)
-        self.assertEqual(result['total_blocks'], total_block)
-        self.assertEqual(result['free_blocks'], free_block)
-        self.assertEqual(result['total'], total)
-        self.assertEqual(result['free'], free)
-        self.assertEqual(result['used'], used)
 
     def test_get_filesystem_volume_stats_error(self):
-
+        self.assertRaises(
-        path = 'aPath'
+            RuntimeError,
-        block_size = 4096
+            get_filesystem_volume_stats, '/nonexistent/path')
-        total_block = 2582828
-        free_block = 767118
-
-        out = " ".join(str(x) for x in (path, 'fb518d79428291bb', 255, 'ef53',
-                                        block_size, '4096', total_block,
-                                        free_block, 636216, 655360, 583768))
-        err = "Error found"
-        return_exp = out, err
-        dbaas.utils.execute_with_timeout = Mock(return_value=return_exp)
-        dbaas.LOG.err = Mock()
-
-        self.interrogator = Interrogator()
-        self.assertRaises(RuntimeError,
-                          self.interrogator.get_filesystem_volume_stats, path)
 
 
 class KeepAliveConnectionTest(testtools.TestCase):

trove/tests/unittests/taskmanager/test_models.py

@@ -32,6 +32,7 @@ class BackupTasksTest(testtools.TestCase):
         self.backup.instance_id = 'instance id'
         self.backup.created = 'yesterday'
         self.backup.updated = 'today'
+        self.backup.size = 2.0
         self.backup.state = backup_models.BackupState.NEW
         self.container_content = (None,
                                   [{'name': 'first'},