Merge "Clean up swift-{account, container}-info"

This commit is contained in:
Jenkins 2014-04-03 19:54:52 +00:00 committed by Gerrit Code Review
commit bd65514ed0
4 changed files with 463 additions and 168 deletions

View File

@ -1,90 +1,34 @@
#!/usr/bin/python
# Copyright (c) 2010-2012 OpenStack Foundation
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from datetime import datetime
from optparse import OptionParser
from swift.common.ring import Ring
from swift.common.utils import hash_path, storage_directory
from swift.account.backend import AccountBroker
from swift.cli.info import print_info, InfoSystemExit
def print_account_info(db_file, swift_dir='/etc/swift'):
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print "DB file doesn't exist"
sys.exit(1)
try:
ring = Ring(swift_dir, ring_name='account')
except Exception:
ring = None
metadata = {}
broker = AccountBroker(db_file)
for key, (value, timestamp) in broker.metadata.iteritems():
if value and key.lower().startswith('x-account-meta-'):
metadata[key] = value
info = broker.get_info()
account = info['account']
account_hash = hash_path(account)
print ' Account: %s' % info['account']
print ' Account Hash: %s' % account_hash
print (' Created at: %s (%s)' %
(datetime.fromtimestamp(float(info['created_at'])),
info['created_at']))
print (' Put Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['put_timestamp'])),
info['put_timestamp']))
print (' Delete Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['delete_timestamp'])),
info['delete_timestamp']))
print ' Container Count: %s' % info['container_count']
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
print ' Chexor: %s' % info['hash']
print ' ID: %s' % info['id']
if metadata:
print ' User Metadata: %s' % metadata
else:
print 'No user metadata found in db file'
print
if ring is not None:
print 'Ring locations:'
part, nodes = ring.get_nodes(account)
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.db' %
(node['ip'], node['port'], node['device'],
storage_directory('accounts', part, account_hash),
account_hash))
print
print 'note: /srv/node is used as default value of `devices`, '\
'the real value is set in account-server.conf '\
'on each storage node.'
if __name__ == '__main__':
parser = OptionParser()
parser.set_defaults(swift_dir='/etc/swift')
parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
parser.add_option(
'-d', '--swift-dir',
'-d', '--swift-dir', default='/etc/swift',
help="Pass location of swift directory")
options, args = parser.parse_args()
if len(args) < 1:
print "Usage: %s [--swift-dir] ACCOUNT_DB_FILE" % sys.argv[0]
if len(args) != 1:
sys.exit(parser.print_help())
try:
print_info('account', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)
print_account_info(args[0], swift_dir=options.swift_dir)

View File

@ -1,107 +1,22 @@
#!/usr/bin/python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from datetime import datetime
from optparse import OptionParser
from swift.common.ring import Ring
from swift.common.utils import hash_path, storage_directory
from swift.container.backend import ContainerBroker
from swift.common.request_helpers import (
is_user_meta, strip_user_meta_prefix, is_sys_meta, strip_sys_meta_prefix)
from swift.cli.info import print_info, InfoSystemExit
def print_container_info(db_file, swift_dir='/etc/swift'):
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print "DB file doesn't exist"
sys.exit(1)
broker = ContainerBroker(db_file)
info = broker.get_info()
account = info['account']
container = info['container']
print 'Path: /%s/%s' % (account, container)
print ' Account: %s' % account
print ' Container: %s' % container
container_hash = hash_path(account, container)
print ' Container Hash: %s' % container_hash
print 'Metadata:'
print (' Created at: %s (%s)' %
(datetime.fromtimestamp(float(info['created_at'])),
info['created_at']))
print (' Put Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['put_timestamp'])),
info['put_timestamp']))
print (' Delete Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['delete_timestamp'])),
info['delete_timestamp']))
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
print (' Reported Put Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['reported_put_timestamp'])),
info['reported_put_timestamp']))
print (' Reported Delete Timestamp: %s (%s)' %
(datetime.fromtimestamp(float(info['reported_delete_timestamp'])),
info['reported_delete_timestamp']))
print ' Reported Object Count: %s' % info['reported_object_count']
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
print ' Chexor: %s' % info['hash']
print ' UUID: %s' % info['id']
for key, value in info.iteritems():
if key.lower().startswith('x_container_'):
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
user_metadata = {}
sys_metadata = {}
for key, (value, timestamp) in broker.metadata.iteritems():
if is_user_meta('container', key):
user_metadata[strip_user_meta_prefix('container', key)] = value
elif is_sys_meta('container', key):
sys_metadata[strip_sys_meta_prefix('container', key)] = value
else:
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
if sys_metadata:
print ' System Metadata: %s' % sys_metadata
else:
print 'No system metadata found in db file'
if user_metadata:
print ' User Metadata: %s' % user_metadata
else:
print 'No user metadata found in db file'
print
try:
ring = Ring(swift_dir, ring_name='container')
except Exception:
ring = None
else:
print 'Ring locations:'
part, nodes = ring.get_nodes(account, container)
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.db' %
(node['ip'], node['port'], node['device'],
storage_directory('containers', part, container_hash),
container_hash))
print
print 'note: /srv/node is used as default value of `devices`, the ' \
'real value is set in container-server.conf on each storage node.'
if __name__ == '__main__':
parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
parser.add_option(
@ -112,4 +27,8 @@ if __name__ == '__main__':
if len(args) != 1:
sys.exit(parser.print_help())
print_container_info(*args, **vars(options))
try:
print_info('container', *args, **vars(options))
except InfoSystemExit:
sys.exit(1)

176
swift/cli/info.py Normal file
View File

@ -0,0 +1,176 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from datetime import datetime
from swift.common.utils import hash_path, storage_directory
from swift.common.ring import Ring
from swift.common.request_helpers import is_sys_meta, is_user_meta, \
strip_sys_meta_prefix, strip_user_meta_prefix
from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
class InfoSystemExit(Exception):
"""
Indicates to the caller that a sys.exit(1) should be performed.
"""
pass
def print_ring_locations(ring, datadir, account, container=None):
"""
print out ring locations of specified type
:param ring: ring instance
:param datadir: high level directory to store account/container/objects
:param acount: account name
:param container: container name
"""
if ring is None or datadir is None or account is None:
raise ValueError('None type')
storage_type = 'account'
if container:
storage_type = 'container'
try:
part, nodes = ring.get_nodes(account, container, None)
except (ValueError, AttributeError):
raise ValueError('Ring error')
else:
path_hash = hash_path(account, container, None)
print '\nRing locations:'
for node in nodes:
print (' %s:%s - /srv/node/%s/%s/%s.db' %
(node['ip'], node['port'], node['device'],
storage_directory(datadir, part, path_hash),
path_hash))
print '\nnote: /srv/node is used as default value of `devices`, the ' \
'real value is set in the %s config file on each storage node.' % \
storage_type
def print_db_info_metadata(db_type, info, metadata):
"""
print out data base info/metadata based on its type
:param db_type: database type, account or container
:param info: dict of data base info
:param metadata: dict of data base metadata
"""
if info is None:
raise ValueError('DB info is None')
if db_type not in ['container', 'account']:
raise ValueError('Wrong DB type')
try:
account = info['account']
container = None
if db_type == 'container':
container = info['container']
path = '/%s/%s' % (account, container)
else:
path = '/%s' % account
print 'Path: %s' % path
print ' Account: %s' % account
if db_type == 'container':
print ' Container: %s' % container
path_hash = hash_path(account, container)
if db_type == 'container':
print ' Container Hash: %s' % path_hash
else:
print ' Account Hash: %s' % path_hash
print 'Metadata:'
print (' Created at: %s (%s)' %
(datetime.utcfromtimestamp(float(info['created_at'])),
info['created_at']))
print (' Put Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['put_timestamp'])),
info['put_timestamp']))
print (' Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(float(info['delete_timestamp'])),
info['delete_timestamp']))
print ' Object Count: %s' % info['object_count']
print ' Bytes Used: %s' % info['bytes_used']
if db_type == 'container':
print (' Reported Put Timestamp: %s (%s)' %
(datetime.utcfromtimestamp(
float(info['reported_put_timestamp'])),
info['reported_put_timestamp']))
print (' Reported Delete Timestamp: %s (%s)' %
(datetime.utcfromtimestamp
(float(info['reported_delete_timestamp'])),
info['reported_delete_timestamp']))
print ' Reported Object Count: %s' % info['reported_object_count']
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
print ' Chexor: %s' % info['hash']
print ' UUID: %s' % info['id']
except KeyError:
raise ValueError('Info is incomplete')
meta_prefix = 'x_' + db_type + '_'
for key, value in info.iteritems():
if key.lower().startswith(meta_prefix):
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
user_metadata = {}
sys_metadata = {}
for key, (value, timestamp) in metadata.iteritems():
if is_user_meta(db_type, key):
user_metadata[strip_user_meta_prefix(db_type, key)] = value
elif is_sys_meta(db_type, key):
sys_metadata[strip_sys_meta_prefix(db_type, key)] = value
else:
title = key.replace('_', '-').title()
print ' %s: %s' % (title, value)
if sys_metadata:
print ' System Metadata: %s' % sys_metadata
else:
print 'No system metadata found in db file'
if user_metadata:
print ' User Metadata: %s' % user_metadata
else:
print 'No user metadata found in db file'
def print_info(db_type, db_file, swift_dir='/etc/swift'):
if db_type not in ('account', 'container'):
print "Unrecognized DB type: internal error"
raise InfoSystemExit()
if not os.path.exists(db_file) or not db_file.endswith('.db'):
print "DB file doesn't exist"
raise InfoSystemExit()
if not db_file.startswith(('/', './')):
db_file = './' + db_file # don't break if the bare db file is given
if db_type == 'account':
broker = AccountBroker(db_file)
datadir = ABDATADIR
else:
broker = ContainerBroker(db_file)
datadir = CBDATADIR
info = broker.get_info()
account = info['account']
container = info['container'] if db_type == 'container' else None
print_db_info_metadata(db_type, info, broker.metadata)
try:
ring = Ring(swift_dir, ring_name=db_type)
except Exception:
ring = None
else:
print_ring_locations(ring, datadir, account, container)

256
test/unit/cli/test_info.py Normal file
View File

@ -0,0 +1,256 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Tests for swift.cli.info"""
import os
import unittest
import cPickle as pickle
import mock
from cStringIO import StringIO
from contextlib import closing
from gzip import GzipFile
from shutil import rmtree
from tempfile import mkdtemp
from swift.common import ring, utils
from swift.common.swob import Request
from swift.cli.info import print_db_info_metadata, print_ring_locations, \
print_info, InfoSystemExit
from swift.account.server import AccountController
from swift.container.server import ContainerController
class TestCliInfo(unittest.TestCase):
def setUp(self):
self.orig_hp = utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX
utils.HASH_PATH_PREFIX = 'info'
utils.HASH_PATH_SUFFIX = 'info'
self.testdir = os.path.join(mkdtemp(), 'tmp_test_cli_info')
utils.mkdirs(self.testdir)
rmtree(self.testdir)
utils.mkdirs(os.path.join(self.testdir, 'sda1'))
utils.mkdirs(os.path.join(self.testdir, 'sda1', 'tmp'))
utils.mkdirs(os.path.join(self.testdir, 'sdb1'))
utils.mkdirs(os.path.join(self.testdir, 'sdb1', 'tmp'))
self.account_ring_path = os.path.join(self.testdir, 'account.ring.gz')
with closing(GzipFile(self.account_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1',
'ip': '127.0.0.1', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.2', 'port': 43}], 30),
f)
self.container_ring_path = os.path.join(self.testdir,
'container.ring.gz')
with closing(GzipFile(self.container_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1',
'ip': '127.0.0.3', 'port': 42},
{'id': 1, 'zone': 1, 'device': 'sdb1',
'ip': '127.0.0.4', 'port': 43}], 30),
f)
def tearDown(self):
utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = self.orig_hp
rmtree(os.path.dirname(self.testdir))
def assertRaisesMessage(self, exc, msg, func, *args, **kwargs):
try:
func(*args, **kwargs)
except Exception, e:
self.assertEqual(msg, str(e))
self.assertTrue(isinstance(e, exc),
"Expected %s, got %s" % (exc, type(e)))
def test_print_db_info_metadata(self):
self.assertRaisesMessage(ValueError, 'Wrong DB type',
print_db_info_metadata, 't', {}, {})
self.assertRaisesMessage(ValueError, 'DB info is None',
print_db_info_metadata, 'container', None, {})
self.assertRaisesMessage(ValueError, 'Info is incomplete',
print_db_info_metadata, 'container', {}, {})
info = dict(
account='acct',
created_at=100.1,
put_timestamp=106.3,
delete_timestamp=107.9,
object_count='20',
bytes_used='42')
info['hash'] = 'abaddeadbeefcafe'
info['id'] = 'abadf100d0ddba11'
md = {'x-account-meta-mydata': ('swift', '0000000000.00000'),
'x-other-something': ('boo', '0000000000.00000')}
out = StringIO()
with mock.patch('sys.stdout', out):
print_db_info_metadata('account', info, md)
exp_out = '''Path: /acct
Account: acct
Account Hash: dc5be2aa4347a22a0fee6bc7de505b47
Metadata:
Created at: 1970-01-01 00:01:40.100000 (100.1)
Put Timestamp: 1970-01-01 00:01:46.300000 (106.3)
Delete Timestamp: 1970-01-01 00:01:47.900000 (107.9)
Object Count: 20
Bytes Used: 42
Chexor: abaddeadbeefcafe
UUID: abadf100d0ddba11
X-Other-Something: boo
No system metadata found in db file
User Metadata: {'mydata': 'swift'}'''
self.assertEquals(out.getvalue().strip(), exp_out)
info = dict(
account='acct',
container='cont',
created_at='0000000100.10000',
put_timestamp='0000000106.30000',
delete_timestamp='0000000107.90000',
object_count='20',
bytes_used='42',
reported_put_timestamp='0000010106.30000',
reported_delete_timestamp='0000010107.90000',
reported_object_count='20',
reported_bytes_used='42',
x_container_foo='bar',
x_container_bar='goo')
info['hash'] = 'abaddeadbeefcafe'
info['id'] = 'abadf100d0ddba11'
md = {'x-container-sysmeta-mydata': ('swift', '0000000000.00000')}
out = StringIO()
with mock.patch('sys.stdout', out):
print_db_info_metadata('container', info, md)
exp_out = '''Path: /acct/cont
Account: acct
Container: cont
Container Hash: d49d0ecbb53be1fcc49624f2f7c7ccae
Metadata:
Created at: 1970-01-01 00:01:40.100000 (0000000100.10000)
Put Timestamp: 1970-01-01 00:01:46.300000 (0000000106.30000)
Delete Timestamp: 1970-01-01 00:01:47.900000 (0000000107.90000)
Object Count: 20
Bytes Used: 42
Reported Put Timestamp: 1970-01-01 02:48:26.300000 (0000010106.30000)
Reported Delete Timestamp: 1970-01-01 02:48:27.900000 (0000010107.90000)
Reported Object Count: 20
Reported Bytes Used: 42
Chexor: abaddeadbeefcafe
UUID: abadf100d0ddba11
X-Container-Bar: goo
X-Container-Foo: bar
System Metadata: {'mydata': 'swift'}
No user metadata found in db file'''
self.assertEquals(out.getvalue().strip(), exp_out)
def test_print_ring_locations(self):
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
None, 'dir', 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
[], None, 'acct')
self.assertRaisesMessage(ValueError, 'None type', print_ring_locations,
[], 'dir', None)
self.assertRaisesMessage(ValueError, 'Ring error',
print_ring_locations,
[], 'dir', 'acct', 'con')
out = StringIO()
with mock.patch('sys.stdout', out):
acctring = ring.Ring(self.testdir, ring_name='account')
print_ring_locations(acctring, 'dir', 'acct')
exp_db2 = os.path.join('/srv', 'node', 'sdb1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_db1 = os.path.join('/srv', 'node', 'sda1', 'dir', '3', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
exp_out = ('Ring locations:\n 127.0.0.2:43 - %s\n'
' 127.0.0.1:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the account config file on'
' each storage node.' % (exp_db2, exp_db1))
self.assertEquals(out.getvalue().strip(), exp_out)
out = StringIO()
with mock.patch('sys.stdout', out):
contring = ring.Ring(self.testdir, ring_name='container')
print_ring_locations(contring, 'dir', 'acct', 'con')
exp_db4 = os.path.join('/srv', 'node', 'sdb1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_db3 = os.path.join('/srv', 'node', 'sda1', 'dir', '1', 'fe6',
'63e70955d78dfc62821edc07d6ec1fe6',
'63e70955d78dfc62821edc07d6ec1fe6.db')
exp_out = ('Ring locations:\n 127.0.0.4:43 - %s\n'
' 127.0.0.3:42 - %s\n'
'\nnote: /srv/node is used as default value of `devices`,'
' the real value is set in the container config file on'
' each storage node.' % (exp_db4, exp_db3))
self.assertEquals(out.getvalue().strip(), exp_out)
def test_print_info(self):
db_file = 'foo'
self.assertRaises(InfoSystemExit, print_info, 'object', db_file)
db_file = os.path.join(self.testdir, './acct.db')
self.assertRaises(InfoSystemExit, print_info, 'account', db_file)
controller = AccountController(
{'devices': self.testdir, 'mount_check': 'false'})
req = Request.blank('/sda1/1/acct', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = req.get_response(controller)
self.assertEqual(resp.status_int, 201)
out = StringIO()
exp_raised = False
with mock.patch('sys.stdout', out):
db_file = os.path.join(self.testdir, 'sda1', 'accounts',
'1', 'b47',
'dc5be2aa4347a22a0fee6bc7de505b47',
'dc5be2aa4347a22a0fee6bc7de505b47.db')
try:
print_info('account', db_file, swift_dir=self.testdir)
except Exception:
exp_raised = True
if exp_raised:
self.fail("Unexpected exception raised")
else:
self.assertTrue(len(out.getvalue().strip()) > 800)
controller = ContainerController(
{'devices': self.testdir, 'mount_check': 'false'})
req = Request.blank('/sda1/1/acct/cont',
environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = req.get_response(controller)
self.assertEqual(resp.status_int, 201)
out = StringIO()
exp_raised = False
with mock.patch('sys.stdout', out):
db_file = os.path.join(self.testdir, 'sda1', 'containers',
'1', 'cae',
'd49d0ecbb53be1fcc49624f2f7c7ccae',
'd49d0ecbb53be1fcc49624f2f7c7ccae.db')
orig_cwd = os.getcwd()
try:
os.chdir(os.path.dirname(db_file))
print_info('container', os.path.basename(db_file),
swift_dir='/dev/null')
except Exception:
exp_raised = True
finally:
os.chdir(orig_cwd)
if exp_raised:
self.fail("Unexpected exception raised")
else:
self.assertTrue(len(out.getvalue().strip()) > 600)