openstack
/
swift
Code Issues Proposed changes

Allow hostnames for nodes in Rings 

This change modifies the swift-ring-builder and introduces new format
of sub-commands (search, list_parts, set_weight, set_info and remove)
in addition to add sub-command so that hostnames can be used in place
of an ip-address for the sub-commands.
The account reaper, container synchronizer, and replicators were also
updated so that they still have a way to identify a particular device
as being "local".

Previously this was Change-Id:
Ie471902413002872fc6755bacd36af3b9c613b74

Change-Id: Ieff583ffb932133e3820744a3f8f9f491686b08d
Co-Authored-By: Alex Pecoraro <alex.pecoraro@emc.com>
Implements: blueprint allow-hostnames-for-nodes-in-rings
This commit is contained in:
Hisashi Osanai 2014-12-09 04:44:52 +09:00
parent df529a225f
commit efb39a5665
13 changed files with 2612 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/source/overview_ring.rst
View File

@ -75,7 +75,7 @@ weight float The relative weight of the device in comparison to other
back into balance a device that has ended up with more or less
data than desired over time. A good average weight of 100.0
allows flexibility in lowering the weight later if necessary.
ip string The IP address of the server containing the device.
ip string The IP address or hostname of the server containing the device.
port int The TCP port the listening server process uses that serves
requests for the device.
device string The on disk name of the device on the server.

5
swift/account/reaper.py
View File

@ -28,6 +28,7 @@ from swift.common.direct_client import direct_delete_container, \
direct_delete_object, direct_get_container
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.ring.utils import is_local_device
from swift.common.utils import get_logger, whataremyips, ismount, \
config_true_value, Timestamp
from swift.common.daemon import Daemon
@ -58,6 +59,7 @@ class AccountReaper(Daemon):
def __init__(self, conf, logger=None):
self.conf = conf
self.logger = logger or get_logger(conf, log_route='account-reaper')
self.bind_port = conf.get('bind_port', 6002)
self.devices = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.interval = int(conf.get('interval', 3600))
@ -159,7 +161,8 @@ class AccountReaper(Daemon):
if not partition.isdigit():
continue
nodes = self.get_account_ring().get_part_nodes(int(partition))
if nodes[0]['ip'] not in self.myips or \
if not is_local_device(self.myips, self.bind_port,
nodes[0]['ip'], nodes[0]['port']) or \
not os.path.isdir(partition_path):
continue
for suffix in os.listdir(partition_path):

718
swift/cli/ringbuilder.py
View File

@ -16,6 +16,7 @@
from errno import EEXIST
from itertools import islice, izip
from operator import itemgetter
from os import mkdir
from os.path import basename, abspath, dirname, exists, join as pathjoin
from sys import argv as sys_argv, exit, stderr
@ -27,10 +28,12 @@ import math
from swift.common import exceptions
from swift.common.ring import RingBuilder, Ring
from swift.common.ring.builder import MAX_BALANCE
from swift.common.utils import lock_parent_directory
from swift.common.ring.utils import parse_search_value, parse_args, \
build_dev_from_opts, parse_builder_ring_filename_args, find_parts, \
from swift.common.ring.utils import validate_args, \
validate_and_normalize_ip, build_dev_from_opts, \
parse_builder_ring_filename_args, parse_search_value, \
parse_search_values_from_opts, parse_change_values_from_opts, \
dispersion_report
from swift.common.utils import lock_parent_directory
MAJOR_VERSION = 1
MINOR_VERSION = 3
@ -55,6 +58,106 @@ def format_device(dev):
'"%(meta)s"' % copy_dev)
def _parse_search_values(argvish):
new_cmd_format, opts, args = validate_args(argvish)
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
try:
search_values = {}
if len(args) > 0:
if new_cmd_format or len(args) != 1:
print Commands.search.__doc__.strip()
exit(EXIT_ERROR)
search_values = parse_search_value(args[0])
else:
search_values = parse_search_values_from_opts(opts)
return search_values
except ValueError as e:
print e
exit(EXIT_ERROR)
def _find_parts(devs):
devs = [d['id'] for d in devs]
if not devs or not builder._replica2part2dev:
return None
partition_count = {}
for replica in builder._replica2part2dev:
for partition, device in enumerate(replica):
if device in devs:
if partition not in partition_count:
partition_count[partition] = 0
partition_count[partition] += 1
# Sort by number of found replicas to keep the output format
sorted_partition_count = sorted(
partition_count.iteritems(), key=itemgetter(1), reverse=True)
return sorted_partition_count
def _parse_list_parts_values(argvish):
new_cmd_format, opts, args = validate_args(argvish)
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
try:
devs = []
if len(args) > 0:
if new_cmd_format:
print Commands.list_parts.__doc__.strip()
exit(EXIT_ERROR)
for arg in args:
devs.extend(
builder.search_devs(parse_search_value(arg)) or [])
else:
devs.extend(builder.search_devs(
parse_search_values_from_opts(opts)) or [])
return devs
except ValueError as e:
print e
exit(EXIT_ERROR)
def _parse_address(rest):
if rest.startswith('['):
# remove first [] for ip
rest = rest.replace('[', '', 1).replace(']', '', 1)
pos = 0
while (pos < len(rest) and
not (rest[pos] == 'R' or rest[pos] == '/')):
pos += 1
address = rest[:pos]
rest = rest[pos:]
port_start = address.rfind(':')
if port_start == -1:
raise ValueError('Invalid port in add value')
ip = address[:port_start]
try:
port = int(address[(port_start + 1):])
except (TypeError, ValueError):
raise ValueError(
'Invalid port %s in add value' % address[port_start:])
# if this is an ipv6 address then we want to convert it
# to all lowercase and use its fully expanded representation
# to make searches easier
ip = validate_and_normalize_ip(ip)
return (ip, port, rest)
def _parse_add_values(argvish):
"""
Parse devices to add as specified on the command line.
@ -63,23 +166,17 @@ def _parse_add_values(argvish):
:returns: array of device dicts
"""
new_cmd_format, opts, args = validate_args(argvish)
opts, args = parse_args(argvish)
# We'll either parse the all-in-one-string format or the --options format,
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
opts_used = opts.region or opts.zone or opts.ip or opts.port or \
opts.device or opts.weight or opts.meta
if len(args) > 0 and opts_used:
print Commands.add.__doc__.strip()
exit(EXIT_ERROR)
elif len(args) > 0:
if len(args) % 2 != 0:
parsed_devs = []
if len(args) > 0:
if new_cmd_format or len(args) % 2 != 0:
print Commands.add.__doc__.strip()
exit(EXIT_ERROR)
parsed_devs = []
devs_and_weights = izip(islice(args, 0, len(args), 2),
islice(args, 1, len(args), 2))
@ -93,12 +190,11 @@ def _parse_add_values(argvish):
region = int(devstr[1:i])
rest = devstr[i:]
else:
stderr.write("WARNING: No region specified for %s. "
"Defaulting to region 1.\n" % devstr)
stderr.write('WARNING: No region specified for %s. '
'Defaulting to region 1.\n' % devstr)
if not rest.startswith('z'):
print 'Invalid add value: %s' % devstr
exit(EXIT_ERROR)
raise ValueError('Invalid add value: %s' % devstr)
i = 1
while i < len(rest) and rest[i].isdigit():
i += 1
@ -106,64 +202,18 @@ def _parse_add_values(argvish):
rest = rest[i:]
if not rest.startswith('-'):
print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
i = 1
if rest[i] == '[':
i += 1
while i < len(rest) and rest[i] != ']':
i += 1
i += 1
ip = rest[1:i].lstrip('[').rstrip(']')
rest = rest[i:]
else:
while i < len(rest) and rest[i] in '0123456789.':
i += 1
ip = rest[1:i]
rest = rest[i:]
raise ValueError('Invalid add value: %s' % devstr)
if not rest.startswith(':'):
print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
i = 1
while i < len(rest) and rest[i].isdigit():
i += 1
port = int(rest[1:i])
rest = rest[i:]
ip, port, rest = _parse_address(rest[1:])
replication_ip = ip
replication_port = port
if rest.startswith('R'):
i = 1
if rest[i] == '[':
i += 1
while i < len(rest) and rest[i] != ']':
i += 1
i += 1
replication_ip = rest[1:i].lstrip('[').rstrip(']')
rest = rest[i:]
else:
while i < len(rest) and rest[i] in '0123456789.':
i += 1
replication_ip = rest[1:i]
rest = rest[i:]
if not rest.startswith(':'):
print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
i = 1
while i < len(rest) and rest[i].isdigit():
i += 1
replication_port = int(rest[1:i])
rest = rest[i:]
replication_ip, replication_port, rest = \
_parse_address(rest[1:])
if not rest.startswith('/'):
print 'Invalid add value: %s' % devstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
raise ValueError(
'Invalid add value: %s' % devstr)
i = 1
while i < len(rest) and rest[i] != '_':
i += 1
@ -174,32 +224,227 @@ def _parse_add_values(argvish):
if rest.startswith('_'):
meta = rest[1:]
try:
weight = float(weightstr)
except ValueError:
print 'Invalid weight value: %s' % weightstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
weight = float(weightstr)
if weight < 0:
print 'Invalid weight value (must be positive): %s' % weightstr
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
raise ValueError('Invalid weight value: %s' % devstr)
parsed_devs.append({'region': region, 'zone': zone, 'ip': ip,
'port': port, 'device': device_name,
'replication_ip': replication_ip,
'replication_port': replication_port,
'weight': weight, 'meta': meta})
return parsed_devs
else:
try:
dev = build_dev_from_opts(opts)
except ValueError as e:
print e
print "The on-disk ring builder is unchanged.\n"
parsed_devs.append(build_dev_from_opts(opts))
return parsed_devs
def _set_weight_values(devs, weight):
if not devs:
print('Search value matched 0 devices.\n'
'The on-disk ring builder is unchanged.')
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to update the weight for '
'these %s devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device modifications'
exit(EXIT_ERROR)
return [dev]
for dev in devs:
builder.set_dev_weight(dev['id'], weight)
print '%s weight set to %s' % (format_device(dev),
dev['weight'])
def _parse_set_weight_values(argvish):
new_cmd_format, opts, args = validate_args(argvish)
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
try:
devs = []
if not new_cmd_format:
if len(args) % 2 != 0:
print Commands.set_weight.__doc__.strip()
exit(EXIT_ERROR)
devs_and_weights = izip(islice(argvish, 0, len(argvish), 2),
islice(argvish, 1, len(argvish), 2))
for devstr, weightstr in devs_and_weights:
devs.extend(builder.search_devs(
parse_search_value(devstr)) or [])
weight = float(weightstr)
_set_weight_values(devs, weight)
else:
if len(args) != 1:
print Commands.set_weight.__doc__.strip()
exit(EXIT_ERROR)
devs.extend(builder.search_devs(
parse_search_values_from_opts(opts)) or [])
weight = float(args[0])
_set_weight_values(devs, weight)
except ValueError as e:
print e
exit(EXIT_ERROR)
def _set_info_values(devs, change):
if not devs:
print("Search value matched 0 devices.\n"
"The on-disk ring builder is unchanged.")
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to update the info for '
'these %s devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device modifications'
exit(EXIT_ERROR)
for dev in devs:
orig_dev_string = format_device(dev)
test_dev = dict(dev)
for key in change:
test_dev[key] = change[key]
for check_dev in builder.devs:
if not check_dev or check_dev['id'] == test_dev['id']:
continue
if check_dev['ip'] == test_dev['ip'] and \
check_dev['port'] == test_dev['port'] and \
check_dev['device'] == test_dev['device']:
print 'Device %d already uses %s:%d/%s.' % \
(check_dev['id'], check_dev['ip'],
check_dev['port'], check_dev['device'])
exit(EXIT_ERROR)
for key in change:
dev[key] = change[key]
print 'Device %s is now %s' % (orig_dev_string,
format_device(dev))
def _parse_set_info_values(argvish):
new_cmd_format, opts, args = validate_args(argvish)
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
if not new_cmd_format:
if len(args) % 2 != 0:
print Commands.search.__doc__.strip()
exit(EXIT_ERROR)
searches_and_changes = izip(islice(argvish, 0, len(argvish), 2),
islice(argvish, 1, len(argvish), 2))
for search_value, change_value in searches_and_changes:
devs = builder.search_devs(parse_search_value(search_value))
change = {}
ip = ''
if len(change_value) and change_value[0].isdigit():
i = 1
while (i < len(change_value) and
change_value[i] in '0123456789.'):
i += 1
ip = change_value[:i]
change_value = change_value[i:]
elif len(change_value) and change_value[0] == '[':
i = 1
while i < len(change_value) and change_value[i] != ']':
i += 1
i += 1
ip = change_value[:i].lstrip('[').rstrip(']')
change_value = change_value[i:]
if ip:
change['ip'] = validate_and_normalize_ip(ip)
if change_value.startswith(':'):
i = 1
while i < len(change_value) and change_value[i].isdigit():
i += 1
change['port'] = int(change_value[1:i])
change_value = change_value[i:]
if change_value.startswith('R'):
change_value = change_value[1:]
replication_ip = ''
if len(change_value) and change_value[0].isdigit():
i = 1
while (i < len(change_value) and
change_value[i] in '0123456789.'):
i += 1
replication_ip = change_value[:i]
change_value = change_value[i:]
elif len(change_value) and change_value[0] == '[':
i = 1
while i < len(change_value) and change_value[i] != ']':
i += 1
i += 1
replication_ip = \
change_value[:i].lstrip('[').rstrip(']')
change_value = change_value[i:]
if replication_ip:
change['replication_ip'] = \
validate_and_normalize_ip(replication_ip)
if change_value.startswith(':'):
i = 1
while i < len(change_value) and change_value[i].isdigit():
i += 1
change['replication_port'] = int(change_value[1:i])
change_value = change_value[i:]
if change_value.startswith('/'):
i = 1
while i < len(change_value) and change_value[i] != '_':
i += 1
change['device'] = change_value[1:i]
change_value = change_value[i:]
if change_value.startswith('_'):
change['meta'] = change_value[1:]
change_value = ''
if change_value or not change:
raise ValueError('Invalid set info change value: %s' %
repr(argvish[1]))
_set_info_values(devs, change)
else:
devs = builder.search_devs(parse_search_values_from_opts(opts))
change = parse_change_values_from_opts(opts)
_set_info_values(devs, change)
def _parse_remove_values(argvish):
new_cmd_format, opts, args = validate_args(argvish)
# We'll either parse the all-in-one-string format or the
# --options format,
# but not both. If both are specified, raise an error.
try:
devs = []
if len(args) > 0:
if new_cmd_format:
print Commands.remove.__doc__.strip()
exit(EXIT_ERROR)
for arg in args:
devs.extend(builder.search_devs(
parse_search_value(arg)) or [])
else:
devs.extend(builder.search_devs(
parse_search_values_from_opts(opts)))
return devs
except ValueError as e:
print e
exit(EXIT_ERROR)
class Commands(object):
@ -286,6 +531,18 @@ swift-ring-builder <builder_file>
def search():
"""
swift-ring-builder <builder_file> search <search-value>
or
swift-ring-builder <builder_file> search
--region <region> --zone <zone> --ip <ip or hostname> --port <port>
--replication-ip <r_ip or r_hostname> --replication-port <r_port>
--device <device_name> --meta <meta> --weight <weight>
Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
and port.
Any of the options are optional in both cases.
Shows information about matching devices.
"""
if len(argv) < 4:
@ -293,7 +550,9 @@ swift-ring-builder <builder_file> search <search-value>
print
print parse_search_value.__doc__.strip()
exit(EXIT_ERROR)
devs = builder.search_devs(parse_search_value(argv[3]))
devs = builder.search_devs(_parse_search_values(argv[3:]))
if not devs:
print 'No matching devices found'
exit(EXIT_ERROR)
@ -322,6 +581,18 @@ swift-ring-builder <builder_file> search <search-value>
def list_parts():
"""
swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
or
swift-ring-builder <builder_file> list_parts
--region <region> --zone <zone> --ip <ip or hostname> --port <port>
--replication-ip <r_ip or r_hostname> --replication-port <r_port>
--device <device_name> --meta <meta> --weight <weight>
Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
and port.
Any of the options are optional in both cases.
Returns a 2 column list of all the partitions that are assigned to any of
the devices matching the search values given. The first column is the
assigned partition number and the second column is the number of device
@ -340,7 +611,12 @@ swift-ring-builder <builder_file> list_parts <search-value> [<search-value>] ..
'Please rebalance first.' % argv[1])
exit(EXIT_ERROR)
sorted_partition_count = find_parts(builder, argv)
devs = _parse_list_parts_values(argv[3:])
if not devs:
print 'No matching devices found'
exit(EXIT_ERROR)
sorted_partition_count = _find_parts(devs)
if not sorted_partition_count:
print 'No matching devices found'
@ -364,8 +640,8 @@ swift-ring-builder <builder_file> add
or
swift-ring-builder <builder_file> add
--region <region> --zone <zone> --ip <ip> --port <port>
[--replication-ip <r_ip> --replication-port <r_port>]
--region <region> --zone <zone> --ip <ip or hostname> --port <port>
[--replication-ip <r_ip or r_hostname>] [--replication-port <r_port>]
--device <device_name> --weight <weight>
[--meta <meta>]
@ -373,24 +649,30 @@ swift-ring-builder <builder_file> add
assigned to the new device until after running 'rebalance'. This is so you
can make multiple device changes and rebalance them all just once.
"""
if len(argv) < 5 or len(argv) % 2 != 1:
if len(argv) < 5:
print Commands.add.__doc__.strip()
exit(EXIT_ERROR)
for new_dev in _parse_add_values(argv[3:]):
for dev in builder.devs:
if dev is None:
continue
if dev['ip'] == new_dev['ip'] and \
dev['port'] == new_dev['port'] and \
dev['device'] == new_dev['device']:
print 'Device %d already uses %s:%d/%s.' % \
(dev['id'], dev['ip'], dev['port'], dev['device'])
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
builder.add_dev(new_dev)
print('Device %s weight %s' %
(format_device(new_dev), new_dev['weight']))
try:
for new_dev in _parse_add_values(argv[3:]):
for dev in builder.devs:
if dev is None:
continue
if dev['ip'] == new_dev['ip'] and \
dev['port'] == new_dev['port'] and \
dev['device'] == new_dev['device']:
print 'Device %d already uses %s:%d/%s.' % \
(dev['id'], dev['ip'],
dev['port'], dev['device'])
print "The on-disk ring builder is unchanged.\n"
exit(EXIT_ERROR)
dev_id = builder.add_dev(new_dev)
print('Device %s with %s weight got id %s' %
(format_device(new_dev), new_dev['weight'], dev_id))
except ValueError as err:
print err
print 'The on-disk ring builder is unchanged.'
exit(EXIT_ERROR)
builder.save(argv[1])
exit(EXIT_SUCCESS)
@ -400,38 +682,30 @@ swift-ring-builder <builder_file> add
swift-ring-builder <builder_file> set_weight <search-value> <weight>
[<search-value> <weight] ...
or
swift-ring-builder <builder_file> set_weight
--region <region> --zone <zone> --ip <ip or hostname> --port <port>
--replication-ip <r_ip or r_hostname> --replication-port <r_port>
--device <device_name> --meta <meta> --weight <weight>
Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
and port.
Any of the options are optional in both cases.
Resets the devices' weights. No partitions will be reassigned to or from
the device until after running 'rebalance'. This is so you can make
multiple device changes and rebalance them all just once.
"""
if len(argv) < 5 or len(argv) % 2 != 1:
# if len(argv) < 5 or len(argv) % 2 != 1:
if len(argv) < 5:
print Commands.set_weight.__doc__.strip()
print
print parse_search_value.__doc__.strip()
exit(EXIT_ERROR)
devs_and_weights = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
for devstr, weightstr in devs_and_weights:
devs = builder.search_devs(parse_search_value(devstr))
weight = float(weightstr)
if not devs:
print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n"
% devstr)
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to update the weight for '
'these %s devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device modifications'
exit(EXIT_ERROR)
for dev in devs:
builder.set_dev_weight(dev['id'], weight)
print '%s weight set to %s' % (format_device(dev),
dev['weight'])
_parse_set_weight_values(argv[3:])
builder.save(argv[1])
exit(EXIT_SUCCESS)
@ -441,7 +715,21 @@ swift-ring-builder <builder_file> set_info
<search-value> <ip>:<port>[R<r_ip>:<r_port>]/<device_name>_<meta>
[<search-value> <ip>:<port>[R<r_ip>:<r_port>]/<device_name>_<meta>] ...
Where <r_ip> and <r_port> are replication ip and port.
or
swift-ring-builder <builder_file> set_info
--ip <ip or hostname> --port <port>
--replication-ip <r_ip or r_hostname> --replication-port <r_port>
--device <device_name> --meta <meta>
--change-ip <ip or hostname> --change-port <port>
--change-replication-ip <r_ip or r_hostname>
--change-replication-port <r_port>
--change-device <device_name>
--change-meta <meta>
Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
and port.
Any of the options are optional in both cases.
For each search-value, resets the matched device's information.
This information isn't used to assign partitions, so you can use
@ -451,111 +739,36 @@ swift-ring-builder <builder_file> set_info
want to change. For instance set_info d74 _"snet: 5.6.7.8" would
just update the meta data for device id 74.
"""
if len(argv) < 5 or len(argv) % 2 != 1:
if len(argv) < 5:
print Commands.set_info.__doc__.strip()
print
print parse_search_value.__doc__.strip()
exit(EXIT_ERROR)
searches_and_changes = izip(islice(argv, 3, len(argv), 2),
islice(argv, 4, len(argv), 2))
try:
_parse_set_info_values(argv[3:])
except ValueError as err:
print err
exit(EXIT_ERROR)
for search_value, change_value in searches_and_changes:
devs = builder.search_devs(parse_search_value(search_value))
change = []
if len(change_value) and change_value[0].isdigit():
i = 1
while (i < len(change_value) and
change_value[i] in '0123456789.'):
i += 1
change.append(('ip', change_value[:i]))
change_value = change_value[i:]
elif len(change_value) and change_value[0] == '[':
i = 1
while i < len(change_value) and change_value[i] != ']':
i += 1
i += 1
change.append(('ip', change_value[:i].lstrip('[').rstrip(']')))
change_value = change_value[i:]
if change_value.startswith(':'):
i = 1
while i < len(change_value) and change_value[i].isdigit():
i += 1
change.append(('port', int(change_value[1:i])))
change_value = change_value[i:]
if change_value.startswith('R'):
change_value = change_value[1:]
if len(change_value) and change_value[0].isdigit():
i = 1
while (i < len(change_value) and
change_value[i] in '0123456789.'):
i += 1
change.append(('replication_ip', change_value[:i]))
change_value = change_value[i:]
elif len(change_value) and change_value[0] == '[':
i = 1
while i < len(change_value) and change_value[i] != ']':
i += 1
i += 1
change.append(('replication_ip',
change_value[:i].lstrip('[').rstrip(']')))
change_value = change_value[i:]
if change_value.startswith(':'):
i = 1
while i < len(change_value) and change_value[i].isdigit():
i += 1
change.append(('replication_port', int(change_value[1:i])))
change_value = change_value[i:]
if change_value.startswith('/'):
i = 1
while i < len(change_value) and change_value[i] != '_':
i += 1
change.append(('device', change_value[1:i]))
change_value = change_value[i:]
if change_value.startswith('_'):
change.append(('meta', change_value[1:]))
change_value = ''
if change_value or not change:
raise ValueError('Invalid set info change value: %s' %
repr(argv[4]))
if not devs:
print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged.\n"
% search_value)
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to update the info for '
'these %s devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device modifications'
exit(EXIT_ERROR)
for dev in devs:
orig_dev_string = format_device(dev)
test_dev = dict(dev)
for key, value in change:
test_dev[key] = value
for check_dev in builder.devs:
if not check_dev or check_dev['id'] == test_dev['id']:
continue
if check_dev['ip'] == test_dev['ip'] and \
check_dev['port'] == test_dev['port'] and \
check_dev['device'] == test_dev['device']:
print 'Device %d already uses %s:%d/%s.' % \
(check_dev['id'], check_dev['ip'],
check_dev['port'], check_dev['device'])
exit(EXIT_ERROR)
for key, value in change:
dev[key] = value
print 'Device %s is now %s' % (orig_dev_string,
format_device(dev))
builder.save(argv[1])
exit(EXIT_SUCCESS)
def remove():
"""
swift-ring-builder <builder_file> remove <search-value> [search-value ...]
or
swift-ring-builder <builder_file> search
--region <region> --zone <zone> --ip <ip or hostname> --port <port>
--replication-ip <r_ip or r_hostname> --replication-port <r_port>
--device <device_name> --meta <meta> --weight <weight>
Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
and port.
Any of the options are optional in both cases.
Removes the device(s) from the ring. This should normally just be used for
a device that has failed. For a device you wish to decommission, it's best
to set its weight to 0, wait for it to drain all its data, then use this
@ -569,36 +782,37 @@ swift-ring-builder <builder_file> remove <search-value> [search-value ...]
print parse_search_value.__doc__.strip()
exit(EXIT_ERROR)
for search_value in argv[3:]:
devs = builder.search_devs(parse_search_value(search_value))
if not devs:
print("Search value \"%s\" matched 0 devices.\n"
"The on-disk ring builder is unchanged." % search_value)
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to remove these %s '
'devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device removals'
exit(EXIT_ERROR)
devs = _parse_remove_values(argv[3:])
if not devs:
print('Search value matched 0 devices.\n'
'The on-disk ring builder is unchanged.')
exit(EXIT_ERROR)
if len(devs) > 1:
print 'Matched more than one device:'
for dev in devs:
try:
builder.remove_dev(dev['id'])
except exceptions.RingBuilderError as e:
print '-' * 79
print(
"An error occurred while removing device with id %d\n"
"This usually means that you attempted to remove\n"
"the last device in a ring. If this is the case,\n"
"consider creating a new ring instead.\n"
"The on-disk ring builder is unchanged.\n"
"Original exception message: %s" %
(dev['id'], e)
)
print '-' * 79
exit(EXIT_ERROR)
print ' %s' % format_device(dev)
if raw_input('Are you sure you want to remove these %s '
'devices? (y/N) ' % len(devs)) != 'y':
print 'Aborting device removals'
exit(EXIT_ERROR)
for dev in devs:
try:
builder.remove_dev(dev['id'])
except exceptions.RingBuilderError as e:
print '-' * 79
print(
'An error occurred while removing device with id %d\n'
'This usually means that you attempted to remove\n'
'the last device in a ring. If this is the case,\n'
'consider creating a new ring instead.\n'
'The on-disk ring builder is unchanged.\n'
'Original exception message: %s' %
(dev['id'], e))
print '-' * 79
exit(EXIT_ERROR)
print '%s marked for removal and will ' \
'be removed next rebalance.' % format_device(dev)

6
swift/common/db_replicator.py
View File

@ -33,6 +33,7 @@ from swift.common.utils import get_logger, whataremyips, storage_directory, \
renamer, mkdirs, lock_parent_directory, config_true_value, \
unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp
from swift.common import ring
from swift.common.ring.utils import is_local_device
from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE
from swift.common.bufferedhttp import BufferedHTTPConnection
from swift.common.exceptions import DriveNotMounted
@ -543,8 +544,9 @@ class Replicator(Daemon):
return
self._local_device_ids = set()
for node in self.ring.devs:
if (node and node['replication_ip'] in ips and
node['replication_port'] == self.port):
if node and is_local_device(ips, self.port,
node['replication_ip'],
node['replication_port']):
if self.mount_check and not ismount(
os.path.join(self.root, node['device'])):
self.logger.warn(

12
swift/common/ring/builder.py
View File

@ -27,7 +27,8 @@ from time import time
from swift.common import exceptions
from swift.common.ring import RingData
from swift.common.ring.utils import tiers_for_dev, build_tier_tree
from swift.common.ring.utils import tiers_for_dev, build_tier_tree, \
validate_and_normalize_address
MAX_BALANCE = 999.99
@ -1305,6 +1306,15 @@ class RingBuilder(object):
if key == 'meta':
if value not in dev.get(key):
matched = False
elif key == 'ip' or key == 'replication_ip':
cdev = ''
try:
cdev = validate_and_normalize_address(
dev.get(key, ''))
except ValueError:
pass
if cdev != value:
matched = False
elif dev.get(key) != value:
matched = False
if matched:

245
swift/common/ring/utils.py
View File

@ -13,9 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import defaultdict
from operator import itemgetter
import optparse
import re
import socket
from swift.common.utils import expand_ipv6
def tiers_for_dev(dev):
@ -128,10 +130,136 @@ def build_tier_tree(devices):
return tier2children
def validate_and_normalize_ip(ip):
"""
Return normalized ip if the ip is a valid ip.
Otherwise raise ValueError Exception. The hostname is
normalized to all lower case. IPv6-addresses are converted to
lowercase and fully expanded.
"""
# first convert to lower case
new_ip = ip.lower()
if is_valid_ipv4(new_ip):
return new_ip
elif is_valid_ipv6(new_ip):
return expand_ipv6(new_ip)
else:
raise ValueError('Invalid ip %s' % ip)
def validate_and_normalize_address(address):
"""
Return normalized address if the address is a valid ip or hostname.
Otherwise raise ValueError Exception. The hostname is
normalized to all lower case. IPv6-addresses are converted to
lowercase and fully expanded.
RFC1123 2.1 Host Names and Nubmers
DISCUSSION
This last requirement is not intended to specify the complete
syntactic form for entering a dotted-decimal host number;
that is considered to be a user-interface issue. For
example, a dotted-decimal number must be enclosed within
"[ ]" brackets for SMTP mail (see Section 5.2.17). This
notation could be made universal within a host system,
simplifying the syntactic checking for a dotted-decimal
number.
If a dotted-decimal number can be entered without such
identifying delimiters, then a full syntactic check must be
made, because a segment of a host domain name is now allowed
to begin with a digit and could legally be entirely numeric
(see Section 6.1.2.4). However, a valid host name can never
have the dotted-decimal form #.#.#.#, since at least the
highest-level component label will be alphabetic.
"""
new_address = address.lstrip('[').rstrip(']')
if address.startswith('[') and address.endswith(']'):
return validate_and_normalize_ip(new_address)
new_address = new_address.lower()
if is_valid_ipv4(new_address):
return new_address
elif is_valid_ipv6(new_address):
return expand_ipv6(new_address)
elif is_valid_hostname(new_address):
return new_address
else:
raise ValueError('Invalid address %s' % address)
def is_valid_ip(ip):
"""
Return True if the provided ip is a valid IP-address
"""
return is_valid_ipv4(ip) or is_valid_ipv6(ip)
def is_valid_ipv4(ip):
"""
Return True if the provided ip is a valid IPv4-address
"""
try:
socket.inet_pton(socket.AF_INET, ip)
except socket.error:
return False
return True
def is_valid_ipv6(ip):
"""
Return True if the provided ip is a valid IPv6-address
"""
try:
socket.inet_pton(socket.AF_INET6, ip)
except socket.error: # not a valid address
return False
return True
def is_valid_hostname(hostname):
"""
Return True if the provided hostname is a valid hostname
"""
if len(hostname) < 1 or len(hostname) > 255:
return False
if hostname[-1] == ".":
# strip exactly one dot from the right, if present
hostname = hostname[:-1]
allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
return all(allowed.match(x) for x in hostname.split("."))
def is_local_device(my_ips, my_port, dev_ip, dev_port):
"""
Return True if the provided dev_ip and dev_port are among the IP
addresses specified in my_ips and my_port respectively.
If dev_ip is a hostname then it is first translated to an IP
address before checking it against my_ips.
"""
if not is_valid_ip(dev_ip) and is_valid_hostname(dev_ip):
try:
# get the ip for this host; use getaddrinfo so that
# it works for both ipv4 and ipv6 addresses
addrinfo = socket.getaddrinfo(dev_ip, dev_port)
for addr in addrinfo:
family = addr[0]
dev_ip = addr[4][0] # get the ip-address
if family == socket.AF_INET6:
dev_ip = expand_ipv6(dev_ip)
if dev_ip in my_ips and dev_port == my_port:
return True
return False
except socket.gaierror:
return False
return dev_ip in my_ips and dev_port == my_port
def parse_search_value(search_value):
"""The <search-value> can be of the form::
d<device_id>r<region>z<zone>-<ip>:<port>[R<r_ip>:<r_port>]/
d<device_id>r<region>z<zone>-<ip>:<port>R<r_ip>:<r_port>/
<device_name>_<meta>
Where <r_ip> and <r_port> are replication ip and port.
@ -201,6 +329,12 @@ def parse_search_value(search_value):
i += 1
match['ip'] = search_value[:i].lstrip('[').rstrip(']')
search_value = search_value[i:]
if 'ip' in match:
# ipv6 addresses are converted to all lowercase
# and use the fully expanded representation
match['ip'] = validate_and_normalize_ip(match['ip'])
if search_value.startswith(':'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
@ -224,6 +358,13 @@ def parse_search_value(search_value):
i += 1
match['replication_ip'] = search_value[:i].lstrip('[').rstrip(']')
search_value = search_value[i:]
if 'replication_ip' in match:
# ipv6 addresses are converted to all lowercase
# and use the fully expanded representation
match['replication_ip'] = \
validate_and_normalize_ip(match['replication_ip'])
if search_value.startswith(':'):
i = 1
while i < len(search_value) and search_value[i].isdigit():
@ -245,11 +386,68 @@ def parse_search_value(search_value):
return match
def parse_search_values_from_opts(opts):
"""
Convert optparse style options into a dictionary for searching.
:param opts: optparse style options
:returns: a dictonary with search values to filter devices,
supported parameters are id, region, zone, ip, port,
replication_ip, replication_port, device, weight, meta
"""
search_values = {}
for key in ('id', 'region', 'zone', 'ip', 'port', 'replication_ip',
'replication_port', 'device', 'weight', 'meta'):
value = getattr(opts, key, None)
if value:
if key == 'ip' or key == 'replication_ip':
value = validate_and_normalize_address(value)
search_values[key] = value
return search_values
def parse_change_values_from_opts(opts):
"""
Convert optparse style options into a dictionary for changing.
:param opts: optparse style options
:returns: a dictonary with change values to filter devices,
supported parameters are ip, port, replication_ip,
replication_port
"""
change_values = {}
for key in ('change_ip', 'change_port', 'change_replication_ip',
'change_replication_port', 'change_device', 'change_meta'):
value = getattr(opts, key, None)
if value:
if key == 'change_ip' or key == 'change_replication_ip':
value = validate_and_normalize_address(value)
change_values[key.replace('change_', '')] = value
return change_values
def validate_args(argvish):
"""
Build OptionParse and validate it whether the format is new command-line
format or not.
"""
opts, args = parse_args(argvish)
new_cmd_format = opts.id or opts.region or opts.zone or \
opts.ip or opts.port or \
opts.replication_ip or opts.replication_port or \
opts.device or opts.weight or opts.meta
return (new_cmd_format, opts, args)
def parse_args(argvish):
"""
Build OptionParser and evaluate command line arguments.
"""
parser = optparse.OptionParser()
parser.add_option('-u', '--id', type="int",
help="Device ID")
parser.add_option('-r', '--region', type="int",
help="Region")
parser.add_option('-z', '--zone', type="int",
@ -268,6 +466,18 @@ def parse_args(argvish):
help="Device weight")
parser.add_option('-m', '--meta', type="string", default="",
help="Extra device info (just a string)")
parser.add_option('-I', '--change-ip', type="string",
help="IP address for change")
parser.add_option('-P', '--change-port', type="int",
help="Port number for change")
parser.add_option('-J', '--change-replication-ip', type="string",
help="Replication IP address for change")
parser.add_option('-Q', '--change-replication-port', type="int",
help="Replication port number for change")
parser.add_option('-D', '--change-device', type="string",
help="Device name (e.g. md0, sdb1) for change")
parser.add_option('-M', '--change-meta', type="string", default="",
help="Extra device info (just a string) for change")
return parser.parse_args(argvish)
@ -300,40 +510,17 @@ def build_dev_from_opts(opts):
raise ValueError('Required argument %s/%s not specified.' %
(shortopt, longopt))
replication_ip = opts.replication_ip or opts.ip
ip = validate_and_normalize_address(opts.ip)
replication_ip = validate_and_normalize_address(
(opts.replication_ip or opts.ip))
replication_port = opts.replication_port or opts.port
return {'region': opts.region, 'zone': opts.zone, 'ip': opts.ip,
return {'region': opts.region, 'zone': opts.zone, 'ip': ip,
'port': opts.port, 'device': opts.device, 'meta': opts.meta,
'replication_ip': replication_ip,
'replication_port': replication_port, 'weight': opts.weight}
def find_parts(builder, argv):
devs = []
for arg in argv[3:]:
devs.extend(builder.search_devs(parse_search_value(arg)) or [])
devs = [d['id'] for d in devs]
if not devs:
return None
partition_count = {}
for replica in builder._replica2part2dev:
for partition, device in enumerate(replica):
if device in devs:
if partition not in partition_count:
partition_count[partition] = 0
partition_count[partition] += 1
# Sort by number of found replicas to keep the output format
sorted_partition_count = sorted(
partition_count.iteritems(), key=itemgetter(1), reverse=True)
return sorted_partition_count
def dispersion_report(builder, search_filter=None, verbose=False):
if not builder._dispersion_graph:
builder._build_dispersion_graph()

23
swift/common/utils.py
View File

@ -907,7 +907,7 @@ class NullLogger(object):
"""A no-op logger for eventlet wsgi."""
def write(self, *args):
#"Logs" the args to nowhere
# "Logs" the args to nowhere
pass
@ -1069,6 +1069,7 @@ class LoggingHandlerWeakRef(weakref.ref):
Like a weak reference, but passes through a couple methods that logging
handlers need.
"""
def close(self):
referent = self()
try:
@ -1542,6 +1543,17 @@ def parse_options(parser=None, once=False, test_args=None):
return config, options
def expand_ipv6(address):
"""
Expand ipv6 address.
:param address: a string indicating valid ipv6 address
:returns: a string indicating fully expanded ipv6 address
"""
packed_ip = socket.inet_pton(socket.AF_INET6, address)
return socket.inet_ntop(socket.AF_INET6, packed_ip)
def whataremyips():
"""
Get the machine's ip addresses
@ -1561,7 +1573,7 @@ def whataremyips():
# If we have an ipv6 address remove the
# %ether_interface at the end
if family == netifaces.AF_INET6:
addr = addr.split('%')[0]
addr = expand_ipv6(addr.split('%')[0])
addresses.append(addr)
except ValueError:
pass
@ -2388,7 +2400,7 @@ def dump_recon_cache(cache_dict, cache_file, logger, lock_timeout=2):
if existing_entry:
cache_entry = json.loads(existing_entry)
except ValueError:
#file doesn't have a valid entry, we'll recreate it
# file doesn't have a valid entry, we'll recreate it
pass
for cache_key, cache_value in cache_dict.items():
put_recon_cache_entry(cache_entry, cache_key, cache_value)
@ -2724,14 +2736,15 @@ def tpool_reraise(func, *args, **kwargs):
class ThreadPool(object):
BYTE = 'a'.encode('utf-8')
"""
Perform blocking operations in background threads.
Call its methods from within greenlets to green-wait for results without
blocking the eventlet reactor (hopefully).
"""
BYTE = 'a'.encode('utf-8')
def __init__(self, nthreads=2):
self.nthreads = nthreads
self._run_queue = Queue()

4
swift/container/sync.py
View File

@ -29,6 +29,7 @@ from swift.common.direct_client import direct_get_object
from swift.common.internal_client import delete_object, put_object
from swift.common.exceptions import ClientException
from swift.common.ring import Ring
from swift.common.ring.utils import is_local_device
from swift.common.utils import (
audit_location_generator, clean_content_type, config_true_value,
FileLikeIter, get_logger, hash_path, quote, urlparse, validate_sync_to,
@ -239,7 +240,8 @@ class ContainerSync(Daemon):
x, nodes = self.container_ring.get_nodes(info['account'],
info['container'])
for ordinal, node in enumerate(nodes):
if node['ip'] in self._myips and node['port'] == self._myport:
if is_local_device(self._myips, self._myport,
node['ip'], node['port']):
break
else:
return

7
swift/obj/replicator.py
View File

@ -27,6 +27,7 @@ from eventlet import GreenPool, tpool, Timeout, sleep, hubs
from eventlet.green import subprocess
from eventlet.support.greenlets import GreenletExit
from swift.common.ring.utils import is_local_device
from swift.common.utils import whataremyips, unlink_older_than, \
compute_eta, get_logger, dump_recon_cache, ismount, \
rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \
@ -417,8 +418,10 @@ class ObjectReplicator(Daemon):
data_dir = get_data_dir(policy.idx)
for local_dev in [dev for dev in obj_ring.devs
if (dev
and dev['replication_ip'] in ips
and dev['replication_port'] == self.port
and is_local_device(ips,
self.port,
dev['replication_ip'],
dev['replication_port'])
and (override_devices is None
or dev['device'] in override_devices))]:
dev_path = join(self.devices_dir, local_dev['device'])

6
test/unit/account/test_reaper.py
View File

@ -95,15 +95,15 @@ class FakeRing(object):
def __init__(self):
self.nodes = [{'id': '1',
'ip': '10.10.10.1',
'port': None,
'port': 6002,
'device': None},
{'id': '2',
'ip': '10.10.10.1',
'port': None,
'port': 6002,
'device': None},
{'id': '3',
'ip': '10.10.10.1',
'port': None,
'port': 6002,
'device': None},
]

1442
test/unit/cli/test_ringbuilder.py
View File

File diff suppressed because it is too large Load Diff

504
test/unit/common/ring/test_utils.py
View File

@ -16,11 +16,17 @@
import unittest
from swift.common import ring
from swift.common.ring.utils import (build_tier_tree, tiers_for_dev,
parse_search_value, parse_args,
build_dev_from_opts, find_parts,
from swift.common.ring.utils import (tiers_for_dev, build_tier_tree,
validate_and_normalize_ip,
validate_and_normalize_address,
is_valid_ip, is_valid_ipv4,
is_valid_ipv6, is_valid_hostname,
is_local_device, parse_search_value,
parse_search_values_from_opts,
parse_change_values_from_opts,
validate_args, parse_args,
parse_builder_ring_filename_args,
dispersion_report)
build_dev_from_opts, dispersion_report)
class TestUtils(unittest.TestCase):
@ -95,6 +101,121 @@ class TestUtils(unittest.TestCase):
(1, 2, '192.168.2.2:6000', 10),
(1, 2, '192.168.2.2:6000', 11)]))
def test_is_valid_ip(self):
self.assertTrue(is_valid_ip("127.0.0.1"))
self.assertTrue(is_valid_ip("10.0.0.1"))
ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80::204:61ff:fe9d:f156"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80::204:61ff:254.157.241.86"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "fe80::"
self.assertTrue(is_valid_ip(ipv6))
ipv6 = "::1"
self.assertTrue(is_valid_ip(ipv6))
not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a"
self.assertFalse(is_valid_ip(not_ipv6))
not_ipv6 = "1:2:3:4:5:6::7:8"
self.assertFalse(is_valid_ip(not_ipv6))
def test_is_valid_ipv4(self):
self.assertTrue(is_valid_ipv4("127.0.0.1"))
self.assertTrue(is_valid_ipv4("10.0.0.1"))
ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80::204:61ff:fe9d:f156"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80::204:61ff:254.157.241.86"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "fe80::"
self.assertFalse(is_valid_ipv4(ipv6))
ipv6 = "::1"
self.assertFalse(is_valid_ipv4(ipv6))
not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a"
self.assertFalse(is_valid_ipv4(not_ipv6))
not_ipv6 = "1:2:3:4:5:6::7:8"
self.assertFalse(is_valid_ipv4(not_ipv6))
def test_is_valid_ipv6(self):
self.assertFalse(is_valid_ipv6("127.0.0.1"))
self.assertFalse(is_valid_ipv6("10.0.0.1"))
ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80:0:0:0:204:61ff:fe9d:f156"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80::204:61ff:fe9d:f156"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80:0000:0000:0000:0204:61ff:254.157.241.86"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80:0:0:0:0204:61ff:254.157.241.86"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80::204:61ff:254.157.241.86"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "fe80::"
self.assertTrue(is_valid_ipv6(ipv6))
ipv6 = "::1"
self.assertTrue(is_valid_ipv6(ipv6))
not_ipv6 = "3ffe:0b00:0000:0001:0000:0000:000a"
self.assertFalse(is_valid_ipv6(not_ipv6))
not_ipv6 = "1:2:3:4:5:6::7:8"
self.assertFalse(is_valid_ipv6(not_ipv6))
def test_is_valid_hostname(self):
self.assertTrue(is_valid_hostname("local"))
self.assertTrue(is_valid_hostname("test.test.com"))
hostname = "test." * 51
self.assertTrue(is_valid_hostname(hostname))
hostname = hostname.rstrip('.')
self.assertTrue(is_valid_hostname(hostname))
hostname = hostname + "00"
self.assertFalse(is_valid_hostname(hostname))
self.assertFalse(is_valid_hostname("$blah#"))
def test_is_local_device(self):
my_ips = ["127.0.0.1",
"0000:0000:0000:0000:0000:0000:0000:0001"]
my_port = 6000
self.assertTrue(is_local_device(my_ips, my_port,
"localhost",
my_port))
def test_validate_and_normalize_ip(self):
ipv4 = "10.0.0.1"
self.assertEqual(ipv4, validate_and_normalize_ip(ipv4))
ipv6 = "fe80::204:61ff:fe9d:f156"
self.assertEqual(ipv6, validate_and_normalize_ip(ipv6.upper()))
hostname = "test.test.com"
self.assertRaises(ValueError,
validate_and_normalize_ip, hostname)
hostname = "$blah#"
self.assertRaises(ValueError,
validate_and_normalize_ip, hostname)
def test_validate_and_normalize_address(self):
ipv4 = "10.0.0.1"
self.assertEqual(ipv4, validate_and_normalize_address(ipv4))
ipv6 = "fe80::204:61ff:fe9d:f156"
self.assertEqual(ipv6, validate_and_normalize_address(ipv6.upper()))
hostname = "test.test.com"
self.assertEqual(hostname,
validate_and_normalize_address(hostname.upper()))
hostname = "$blah#"
self.assertRaises(ValueError,
validate_and_normalize_address, hostname)
def test_parse_search_value(self):
res = parse_search_value('r0')
self.assertEqual(res, {'region': 0})
@ -108,6 +229,8 @@ class TestUtils(unittest.TestCase):
self.assertEqual(res, {'zone': 1})
res = parse_search_value('-127.0.0.1')
self.assertEqual(res, {'ip': '127.0.0.1'})
res = parse_search_value('127.0.0.1')
self.assertEqual(res, {'ip': '127.0.0.1'})
res = parse_search_value('-[127.0.0.1]:10001')
self.assertEqual(res, {'ip': '127.0.0.1', 'port': 10001})
res = parse_search_value(':10001')
@ -125,22 +248,268 @@ class TestUtils(unittest.TestCase):
self.assertEqual(res, {'meta': 'meta1'})
self.assertRaises(ValueError, parse_search_value, 'OMGPONIES')
def test_replication_defaults(self):
args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split()
opts, _ = parse_args(args)
device = build_dev_from_opts(opts)
def test_parse_search_values_from_opts(self):
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "test.test.com",
"--port", "6000",
"--replication-ip", "r.test.com",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "change.test.test.com",
"--change-port", "6001",
"--change-replication-ip", "change.r.test.com",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
expected = {
'device': 'd1',
'ip': '127.0.0.1',
'meta': '',
'port': 6010,
'region': 1,
'replication_ip': '127.0.0.1',
'replication_port': 6010,
'weight': 100.0,
'zone': 1,
'id': 1,
'region': 2,
'zone': 3,
'ip': "test.test.com",
'port': 6000,
'replication_ip': "r.test.com",
'replication_port': 7000,
'device': "sda3",
'meta': "some meta data",
'weight': 3.14159265359,
}
self.assertEquals(device, expected)
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_search_values_from_opts(opts)
self.assertEquals(search_values, expected)
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "127.0.0.1",
"--port", "6000",
"--replication-ip", "127.0.0.10",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "127.0.0.2",
"--change-port", "6001",
"--change-replication-ip", "127.0.0.20",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
expected = {
'id': 1,
'region': 2,
'zone': 3,
'ip': "127.0.0.1",
'port': 6000,
'replication_ip': "127.0.0.10",
'replication_port': 7000,
'device': "sda3",
'meta': "some meta data",
'weight': 3.14159265359,
}
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_search_values_from_opts(opts)
self.assertEquals(search_values, expected)
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "[127.0.0.1]",
"--port", "6000",
"--replication-ip", "[127.0.0.10]",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "[127.0.0.2]",
"--change-port", "6001",
"--change-replication-ip", "[127.0.0.20]",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_search_values_from_opts(opts)
self.assertEquals(search_values, expected)
def test_parse_change_values_from_opts(self):
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "test.test.com",
"--port", "6000",
"--replication-ip", "r.test.com",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "change.test.test.com",
"--change-port", "6001",
"--change-replication-ip", "change.r.test.com",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
expected = {
'ip': "change.test.test.com",
'port': 6001,
'replication_ip': "change.r.test.com",
'replication_port': 7001,
'device': "sdb3",
'meta': "some meta data for change",
}
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_change_values_from_opts(opts)
self.assertEquals(search_values, expected)
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "127.0.0.1",
"--port", "6000",
"--replication-ip", "127.0.0.10",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "127.0.0.2",
"--change-port", "6001",
"--change-replication-ip", "127.0.0.20",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
expected = {
'ip': "127.0.0.2",
'port': 6001,
'replication_ip': "127.0.0.20",
'replication_port': 7001,
'device': "sdb3",
'meta': "some meta data for change",
}
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_change_values_from_opts(opts)
self.assertEquals(search_values, expected)
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "[127.0.0.1]",
"--port", "6000",
"--replication-ip", "[127.0.0.10]",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "[127.0.0.2]",
"--change-port", "6001",
"--change-replication-ip", "[127.0.0.20]",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
new_cmd_format, opts, args = validate_args(argv)
search_values = parse_change_values_from_opts(opts)
self.assertEquals(search_values, expected)
def test_validate_args(self):
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "test.test.com",
"--port", "6000",
"--replication-ip", "r.test.com",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "change.test.test.com",
"--change-port", "6001",
"--change-replication-ip", "change.r.test.com",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
new_cmd_format, opts, args = validate_args(argv)
self.assertTrue(new_cmd_format)
self.assertEqual(opts.id, 1)
self.assertEqual(opts.region, 2)
self.assertEqual(opts.zone, 3)
self.assertEqual(opts.ip, "test.test.com")
self.assertEqual(opts.port, 6000)
self.assertEqual(opts.replication_ip, "r.test.com")
self.assertEqual(opts.replication_port, 7000)
self.assertEqual(opts.device, "sda3")
self.assertEqual(opts.meta, "some meta data")
self.assertEqual(opts.weight, 3.14159265359)
self.assertEqual(opts.change_ip, "change.test.test.com")
self.assertEqual(opts.change_port, 6001)
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
self.assertEqual(opts.change_replication_port, 7001)
self.assertEqual(opts.change_device, "sdb3")
self.assertEqual(opts.change_meta, "some meta data for change")
argv = \
["--id", "0", "--region", "0", "--zone", "0",
"--ip", "",
"--port", "0",
"--replication-ip", "",
"--replication-port", "0",
"--device", "",
"--meta", "",
"--weight", "0",
"--change-ip", "",
"--change-port", "0",
"--change-replication-ip", "",
"--change-replication-port", "0",
"--change-device", "",
"--change-meta", ""]
new_cmd_format, opts, args = validate_args(argv)
self.assertFalse(new_cmd_format)
argv = \
["--id", "0", "--region", "0", "--zone", "0",
"--ip", "",
"--port", "0",
"--replication-ip", "",
"--replication-port", "0",
"--device", "",
"--meta", "",
"--weight", "0",
"--change-ip", "change.test.test.com",
"--change-port", "6001",
"--change-replication-ip", "change.r.test.com",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
new_cmd_format, opts, args = validate_args(argv)
self.assertFalse(new_cmd_format)
def test_parse_args(self):
argv = \
["--id", "1", "--region", "2", "--zone", "3",
"--ip", "test.test.com",
"--port", "6000",
"--replication-ip", "r.test.com",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359",
"--change-ip", "change.test.test.com",
"--change-port", "6001",
"--change-replication-ip", "change.r.test.com",
"--change-replication-port", "7001",
"--change-device", "sdb3",
"--change-meta", "some meta data for change"]
opts, args = parse_args(argv)
self.assertEqual(opts.id, 1)
self.assertEqual(opts.region, 2)
self.assertEqual(opts.zone, 3)
self.assertEqual(opts.ip, "test.test.com")
self.assertEqual(opts.port, 6000)
self.assertEqual(opts.replication_ip, "r.test.com")
self.assertEqual(opts.replication_port, 7000)
self.assertEqual(opts.device, "sda3")
self.assertEqual(opts.meta, "some meta data")
self.assertEqual(opts.weight, 3.14159265359)
self.assertEqual(opts.change_ip, "change.test.test.com")
self.assertEqual(opts.change_port, 6001)
self.assertEqual(opts.change_replication_ip, "change.r.test.com")
self.assertEqual(opts.change_replication_port, 7001)
self.assertEqual(opts.change_device, "sdb3")
self.assertEqual(opts.change_meta, "some meta data for change")
self.assertEqual(len(args), 0)
def test_parse_builder_ring_filename_args(self):
args = 'swift-ring-builder object.builder write_ring'
@ -161,33 +530,86 @@ class TestUtils(unittest.TestCase):
'my.file.name', 'my.file.name.ring.gz'
), parse_builder_ring_filename_args(args.split()))
def test_find_parts(self):
rb = ring.RingBuilder(8, 3, 0)
rb.add_dev({'id': 0, 'region': 1, 'zone': 0, 'weight': 100,
'ip': '127.0.0.1', 'port': 10000, 'device': 'sda1'})
rb.add_dev({'id': 1, 'region': 1, 'zone': 1, 'weight': 100,
'ip': '127.0.0.1', 'port': 10001, 'device': 'sda1'})
rb.add_dev({'id': 2, 'region': 1, 'zone': 2, 'weight': 100,
'ip': '127.0.0.1', 'port': 10002, 'device': 'sda1'})
rb.rebalance()
def test_build_dev_from_opts(self):
argv = \
["--region", "2", "--zone", "3",
"--ip", "test.test.com",
"--port", "6000",
"--replication-ip", "r.test.com",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359"]
expected = {
'region': 2,
'zone': 3,
'ip': "test.test.com",
'port': 6000,
'replication_ip': "r.test.com",
'replication_port': 7000,
'device': "sda3",
'meta': "some meta data",
'weight': 3.14159265359,
}
opts, args = parse_args(argv)
device = build_dev_from_opts(opts)
self.assertEquals(device, expected)
rb.add_dev({'id': 3, 'region': 2, 'zone': 1, 'weight': 10,
'ip': '127.0.0.1', 'port': 10004, 'device': 'sda1'})
rb.pretend_min_part_hours_passed()
rb.rebalance()
argv = \
["--region", "2", "--zone", "3",
"--ip", "[test.test.com]",
"--port", "6000",
"--replication-ip", "[r.test.com]",
"--replication-port", "7000",
"--device", "sda3",
"--meta", "some meta data",
"--weight", "3.14159265359"]
opts, args = parse_args(argv)
self.assertRaises(ValueError, build_dev_from_opts, opts)
argv = ['swift-ring-builder', 'object.builder',
'list_parts', '127.0.0.1']
sorted_partition_count = find_parts(rb, argv)
argv = \
["--region", "2", "--zone", "3",
"--ip", "[test.test.com]",
"--port", "6000",
"--replication-ip", "[r.test.com]",
"--replication-port", "7000",
"--meta", "some meta data",
"--weight", "3.14159265359"]
opts, args = parse_args(argv)
self.assertRaises(ValueError, build_dev_from_opts, opts)
# Expect 256 partitions in the output
self.assertEqual(256, len(sorted_partition_count))
def test_replication_defaults(self):
args = '-r 1 -z 1 -i 127.0.0.1 -p 6010 -d d1 -w 100'.split()
opts, _ = parse_args(args)
device = build_dev_from_opts(opts)
expected = {
'device': 'd1',
'ip': '127.0.0.1',
'meta': '',
'port': 6010,
'region': 1,
'replication_ip': '127.0.0.1',
'replication_port': 6010,
'weight': 100.0,
'zone': 1,
}
self.assertEquals(device, expected)
# Each partitions should have 3 replicas
for partition, count in sorted_partition_count:
self.assertEqual(
3, count, "Partition %d has only %d replicas" %
(partition, count))
args = '-r 1 -z 1 -i test.com -p 6010 -d d1 -w 100'.split()
opts, _ = parse_args(args)
device = build_dev_from_opts(opts)
expected = {
'device': 'd1',
'ip': 'test.com',
'meta': '',
'port': 6010,
'region': 1,
'replication_ip': 'test.com',
'replication_port': 6010,
'weight': 100.0,
'zone': 1,
}
self.assertEquals(device, expected)
def test_dispersion_report(self):
rb = ring.RingBuilder(8, 3, 0)

15
test/unit/common/test_utils.py
View File

@ -1455,6 +1455,15 @@ class TestUtils(unittest.TestCase):
self.assertEquals(utils.storage_directory('objects', '1', 'ABCDEF'),
'objects/1/DEF/ABCDEF')
def test_expand_ipv6(self):
expanded_ipv6 = "fe80::204:61ff:fe9d:f156"
upper_ipv6 = "fe80:0000:0000:0000:0204:61ff:fe9d:f156"
self.assertEqual(expanded_ipv6, utils.expand_ipv6(upper_ipv6))
omit_ipv6 = "fe80:0000:0000::0204:61ff:fe9d:f156"
self.assertEqual(expanded_ipv6, utils.expand_ipv6(omit_ipv6))
less_num_ipv6 = "fe80:0:00:000:0204:61ff:fe9d:f156"
self.assertEqual(expanded_ipv6, utils.expand_ipv6(less_num_ipv6))
def test_whataremyips(self):
myips = utils.whataremyips()
self.assert_(len(myips) > 1)
@ -2935,9 +2944,9 @@ class TestSwiftInfo(unittest.TestCase):
utils._swift_info = {'swift': {'foo': 'bar'},
'cap1': cap1}
# expect no exceptions
info = utils.get_swift_info(disallowed_sections=
['cap2.cap1_foo', 'cap1.no_match',
'cap1.cap1_foo.no_match.no_match'])
info = utils.get_swift_info(
disallowed_sections=['cap2.cap1_foo', 'cap1.no_match',
'cap1.cap1_foo.no_match.no_match'])
self.assertEquals(info['cap1'], cap1)