openstack
/
swift
Code Issues Proposed changes

Cleanup for review 

reverting unnecessary changes frm master

relocating some methods within modules

Change-Id: I33a46f4daa99e57d946793323d9396a2ad62cd1a
This commit is contained in:
Alistair Coles 2018-05-02 09:29:48 +01:00
parent b72d040884
commit 28514903e0
10 changed files with 65 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/saio/swift/container-server/1.conf
View File

@ -36,4 +36,4 @@ rsync_module = {replication_ip}::container{replication_port}
shard_container_size = 100
# The probe tests make explicit assumptions about the batch sizes
shard_scanner_batch_size = 10
shard_batch_size = 2
cleave_batch_size = 2

2
doc/saio/swift/container-server/2.conf
View File

@ -36,4 +36,4 @@ rsync_module = {replication_ip}::container{replication_port}
shard_container_size = 100
# The probe tests make explicit assumptions about the batch sizes
shard_scanner_batch_size = 10
shard_batch_size = 2
cleave_batch_size = 2

2
doc/saio/swift/container-server/3.conf
View File

@ -36,4 +36,4 @@ rsync_module = {replication_ip}::container{replication_port}
shard_container_size = 100
# The probe tests make explicit assumptions about the batch sizes
shard_scanner_batch_size = 10
shard_batch_size = 2
cleave_batch_size = 2

2
doc/saio/swift/container-server/4.conf
View File

@ -36,4 +36,4 @@ rsync_module = {replication_ip}::container{replication_port}
shard_container_size = 100
# The probe tests make explicit assumptions about the batch sizes
shard_scanner_batch_size = 10
shard_batch_size = 2
cleave_batch_size = 2

3
swift/common/db_replicator.py
View File

@ -800,9 +800,6 @@ class Replicator(Daemon):
if elapsed < self.interval:
sleep(self.interval - elapsed)
def _is_locked(self, broker):
return False
class ReplicatorRpc(object):
"""Handle Replication RPC calls. TODO(redbo): document please :)"""

4
swift/common/exceptions.py
View File

@ -280,7 +280,3 @@ class ClientException(Exception):
class InvalidPidFileException(Exception):
pass
class RangeAnalyserException(SwiftException):
pass

107
swift/container/backend.py
View File

@ -49,12 +49,6 @@ SHARDED = 3
COLLAPSED = 4
DB_STATES = ['not_found', 'unsharded', 'sharding', 'sharded', 'collapsed']
SHARD_STATS_STATES = [ShardRange.ACTIVE, ShardRange.SHARDING,
ShardRange.SHRINKING]
SHARD_LISTING_STATES = SHARD_STATS_STATES + [ShardRange.CLEAVED]
SHARD_UPDATE_STATES = [ShardRange.CREATED, ShardRange.CLEAVED,
ShardRange.ACTIVE, ShardRange.SHARDING]
def db_state_text(state):
try:
@ -63,6 +57,13 @@ def db_state_text(state):
return 'unknown (%d)' % state
SHARD_STATS_STATES = [ShardRange.ACTIVE, ShardRange.SHARDING,
ShardRange.SHRINKING]
SHARD_LISTING_STATES = SHARD_STATS_STATES + [ShardRange.CLEAVED]
SHARD_UPDATE_STATES = [ShardRange.CREATED, ShardRange.CLEAVED,
ShardRange.ACTIVE, ShardRange.SHARDING]
# attribute names in order used when transforming shard ranges from dicts to
# tuples and vice-versa
SHARD_RANGE_KEYS = ('name', 'timestamp', 'lower', 'upper', 'object_count',
@ -420,45 +421,6 @@ class ContainerBroker(DatabaseBroker):
hash_, epoch, ext = parse_db_filename(self.db_file)
return epoch
# TODO: needs unit test
def update_sharding_info(self, info):
"""
Updates the broker's metadata with the given ``info``. Each key in
``info`` is prefixed with a sharding specific namespace.
:param info: a dict of info to be persisted
"""
prefix = 'X-Container-Sysmeta-Shard-'
timestamp = Timestamp.now()
metadata = dict(
('%s%s' % (prefix, key),
(value, timestamp.internal))
for key, value in info.items()
)
self.update_metadata(metadata)
# TODO: needs unit test
def get_sharding_info(self, key=None, default=None):
"""
Returns sharding specific info from the broker's metadata.
:param key: if given the value stored under ``key`` in the sharding
info will be returned. If ``key`` is not found in the info then the
value of ``default`` will be returned or None if ``default`` is not
given.
:param default: a default value to return if ``key`` is given but not
found in the sharding info.
:return: either a dict of sharding info or the value stored under
``key`` in that dict.
"""
prefix = 'X-Container-Sysmeta-Shard-'
metadata = self.metadata
info = dict((k[len(prefix):], v[0]) for
k, v in metadata.items() if k.startswith(prefix))
if key:
return info.get(key, default)
return info
@property
def storage_policy_index(self):
if not hasattr(self, '_storage_policy_index'):
@ -466,6 +428,13 @@ class ContainerBroker(DatabaseBroker):
self.get_info()['storage_policy_index']
return self._storage_policy_index
@property
def path(self):
if self.container is None:
# Ensure account/container get populated
self.get_info()
return '%s/%s' % (self.account, self.container)
def _initialize(self, conn, put_timestamp, storage_policy_index):
"""
Create a brand new container database (tables, indices, triggers, etc.)
@ -837,7 +806,7 @@ class ContainerBroker(DatabaseBroker):
except sqlite3.OperationalError as err:
err_msg = str(err)
if err_msg in errors:
# only attempt each migration once
# only attempt migration once
raise
errors.add(err_msg)
if 'no such column: storage_policy_index' in err_msg:
@ -1954,6 +1923,45 @@ class ContainerBroker(DatabaseBroker):
brokers.append(self)
return brokers
# TODO: needs unit test
def update_sharding_info(self, info):
"""
Updates the broker's metadata with the given ``info``. Each key in
``info`` is prefixed with a sharding specific namespace.
:param info: a dict of info to be persisted
"""
prefix = 'X-Container-Sysmeta-Shard-'
timestamp = Timestamp.now()
metadata = dict(
('%s%s' % (prefix, key),
(value, timestamp.internal))
for key, value in info.items()
)
self.update_metadata(metadata)
# TODO: needs unit test
def get_sharding_info(self, key=None, default=None):
"""
Returns sharding specific info from the broker's metadata.
:param key: if given the value stored under ``key`` in the sharding
info will be returned. If ``key`` is not found in the info then the
value of ``default`` will be returned or None if ``default`` is not
given.
:param default: a default value to return if ``key`` is given but not
found in the sharding info.
:return: either a dict of sharding info or the value stored under
``key`` in that dict.
"""
prefix = 'X-Container-Sysmeta-Shard-'
metadata = self.metadata
info = dict((k[len(prefix):], v[0]) for
k, v in metadata.items() if k.startswith(prefix))
if key:
return info.get(key, default)
return info
def get_items_since(self, start, count, include_sharding=False):
"""
Get a list of objects in the database between start and end.
@ -2003,13 +2011,6 @@ class ContainerBroker(DatabaseBroker):
"of the form 'account/container', got %r" % path)
self._root_account, self._root_container = tuple(path.split('/'))
@property
def path(self):
if self.container is None:
# Ensure account/container get populated
self.get_info()
return '%s/%s' % (self.account, self.container)
@property
def root_account(self):
if not self._root_account:

13
swift/container/server.py
View File

@ -35,8 +35,7 @@ from swift.common.request_helpers import get_param, \
from swift.common.utils import get_logger, hash_path, public, \
Timestamp, storage_directory, validate_sync_to, \
config_true_value, timing_stats, replication, \
override_bytes_from_content_type, get_log_line, whataremyips, ShardRange, \
list_from_csv
override_bytes_from_content_type, get_log_line, ShardRange, list_from_csv
from swift.common.constraints import valid_timestamp, check_utf8, check_drive
from swift.common import constraints
@ -98,13 +97,12 @@ class ContainerController(BaseStorageServer):
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.node_timeout = float(conf.get('node_timeout', 3))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
swift_dir = conf.get('swift_dir', '/etc/swift')
#: ContainerSyncCluster instance for validating sync-to values.
self.realms_conf = ContainerSyncRealms(
os.path.join(swift_dir, 'container-sync-realms.conf'),
os.path.join(
conf.get('swift_dir', '/etc/swift'),
'container-sync-realms.conf'),
self.logger)
self.ips = whataremyips()
self.port = int(conf.get('bind_port', 6201))
#: The list of hosts we're allowed to send syncs to. This can be
#: overridden by data in self.realms_conf
self.allowed_sync_hosts = [
@ -527,8 +525,10 @@ class ContainerController(BaseStorageServer):
"""
Perform any mutations to container listing records that are common to
all serialization formats, and returns it as a dict.
Converts created time to iso timestamp.
Replaces size with 'swift_bytes' content type parameter.
:params record: object entry record
:returns: modified record
"""
@ -618,7 +618,6 @@ class ContainerController(BaseStorageServer):
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'],
reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)

1
swift/proxy/controllers/base.py
View File

@ -64,6 +64,7 @@ from swift.common.request_helpers import strip_sys_meta_prefix, \
strip_object_transient_sysmeta_prefix
from swift.common.storage_policy import POLICIES
DEFAULT_RECHECK_ACCOUNT_EXISTENCE = 60 # seconds
DEFAULT_RECHECK_CONTAINER_EXISTENCE = 60 # seconds

1
test/unit/proxy/test_server.py
View File

@ -35,7 +35,6 @@ from tempfile import mkdtemp, NamedTemporaryFile
import weakref
import operator
import functools
from swift.obj import diskfile
import re
import random