1770 lines
78 KiB
Python
1770 lines
78 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
import numbers
|
|
|
|
import mock
|
|
import operator
|
|
import time
|
|
import unittest
|
|
import socket
|
|
import os
|
|
import errno
|
|
import itertools
|
|
import random
|
|
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from six.moves import urllib
|
|
from swift.container import reconciler
|
|
from swift.container.server import gen_resp_headers
|
|
from swift.common.direct_client import ClientException
|
|
from swift.common import swob
|
|
from swift.common.header_key_dict import HeaderKeyDict
|
|
from swift.common.utils import split_path, Timestamp, encode_timestamps
|
|
|
|
from test.unit import debug_logger, FakeRing, fake_http_connect
|
|
from test.unit.common.middleware.helpers import FakeSwift
|
|
|
|
|
|
def timestamp_to_last_modified(timestamp):
|
|
return datetime.utcfromtimestamp(
|
|
float(Timestamp(timestamp))).strftime('%Y-%m-%dT%H:%M:%S.%f')
|
|
|
|
|
|
def container_resp_headers(**kwargs):
|
|
return HeaderKeyDict(gen_resp_headers(kwargs))
|
|
|
|
|
|
class FakeStoragePolicySwift(object):
|
|
|
|
def __init__(self):
|
|
self.storage_policy = defaultdict(FakeSwift)
|
|
self._mock_oldest_spi_map = {}
|
|
|
|
def __getattribute__(self, name):
|
|
try:
|
|
return object.__getattribute__(self, name)
|
|
except AttributeError:
|
|
return getattr(self.storage_policy[None], name)
|
|
|
|
def __call__(self, env, start_response):
|
|
method = env['REQUEST_METHOD']
|
|
path = env['PATH_INFO']
|
|
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
|
|
rest_with_last=True)
|
|
if not obj:
|
|
policy_index = None
|
|
else:
|
|
policy_index = self._mock_oldest_spi_map.get(cont, 0)
|
|
# allow backend policy override
|
|
if 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX' in env:
|
|
policy_index = int(env['HTTP_X_BACKEND_STORAGE_POLICY_INDEX'])
|
|
|
|
try:
|
|
return self.storage_policy[policy_index].__call__(
|
|
env, start_response)
|
|
except KeyError:
|
|
pass
|
|
|
|
if method == 'PUT':
|
|
resp_class = swob.HTTPCreated
|
|
else:
|
|
resp_class = swob.HTTPNotFound
|
|
self.storage_policy[policy_index].register(
|
|
method, path, resp_class, {}, '')
|
|
|
|
return self.storage_policy[policy_index].__call__(
|
|
env, start_response)
|
|
|
|
|
|
class FakeInternalClient(reconciler.InternalClient):
|
|
def __init__(self, listings):
|
|
self.app = FakeStoragePolicySwift()
|
|
self.user_agent = 'fake-internal-client'
|
|
self.request_tries = 1
|
|
self.parse(listings)
|
|
|
|
def parse(self, listings):
|
|
self.accounts = defaultdict(lambda: defaultdict(list))
|
|
for item, timestamp in listings.items():
|
|
# XXX this interface is stupid
|
|
if isinstance(timestamp, tuple):
|
|
timestamp, content_type = timestamp
|
|
else:
|
|
timestamp, content_type = timestamp, 'application/x-put'
|
|
storage_policy_index, path = item
|
|
account, container_name, obj_name = split_path(
|
|
path.encode('utf-8'), 0, 3, rest_with_last=True)
|
|
self.accounts[account][container_name].append(
|
|
(obj_name, storage_policy_index, timestamp, content_type))
|
|
for account_name, containers in self.accounts.items():
|
|
for con in containers:
|
|
self.accounts[account_name][con].sort(key=lambda t: t[0])
|
|
for account, containers in self.accounts.items():
|
|
account_listing_data = []
|
|
account_path = '/v1/%s' % account
|
|
for container, objects in containers.items():
|
|
container_path = account_path + '/' + container
|
|
container_listing_data = []
|
|
for entry in objects:
|
|
(obj_name, storage_policy_index,
|
|
timestamp, content_type) = entry
|
|
if storage_policy_index is None and not obj_name:
|
|
# empty container
|
|
continue
|
|
obj_path = container_path + '/' + obj_name
|
|
ts = Timestamp(timestamp)
|
|
headers = {'X-Timestamp': ts.normal,
|
|
'X-Backend-Timestamp': ts.internal}
|
|
# register object response
|
|
self.app.storage_policy[storage_policy_index].register(
|
|
'GET', obj_path, swob.HTTPOk, headers)
|
|
self.app.storage_policy[storage_policy_index].register(
|
|
'DELETE', obj_path, swob.HTTPNoContent, {})
|
|
# container listing entry
|
|
last_modified = timestamp_to_last_modified(timestamp)
|
|
# some tests setup mock listings using floats, some use
|
|
# strings, so normalize here
|
|
if isinstance(timestamp, numbers.Number):
|
|
timestamp = '%f' % timestamp
|
|
obj_data = {
|
|
'bytes': 0,
|
|
# listing data is unicode
|
|
'name': obj_name.decode('utf-8'),
|
|
'last_modified': last_modified,
|
|
'hash': timestamp.decode('utf-8'),
|
|
'content_type': content_type,
|
|
}
|
|
container_listing_data.append(obj_data)
|
|
container_listing_data.sort(key=operator.itemgetter('name'))
|
|
# register container listing response
|
|
container_headers = {}
|
|
container_qry_string = \
|
|
'?format=json&marker=&end_marker=&prefix='
|
|
self.app.register('GET', container_path + container_qry_string,
|
|
swob.HTTPOk, container_headers,
|
|
json.dumps(container_listing_data))
|
|
if container_listing_data:
|
|
obj_name = container_listing_data[-1]['name']
|
|
# client should quote and encode marker
|
|
end_qry_string = \
|
|
'?format=json&marker=%s&end_marker=&prefix=' % (
|
|
urllib.parse.quote(obj_name.encode('utf-8')))
|
|
self.app.register('GET', container_path + end_qry_string,
|
|
swob.HTTPOk, container_headers,
|
|
json.dumps([]))
|
|
self.app.register('DELETE', container_path,
|
|
swob.HTTPConflict, {}, '')
|
|
# simple account listing entry
|
|
container_data = {'name': container}
|
|
account_listing_data.append(container_data)
|
|
# register account response
|
|
account_listing_data.sort(key=operator.itemgetter('name'))
|
|
account_headers = {}
|
|
account_qry_string = '?format=json&marker=&end_marker=&prefix='
|
|
self.app.register('GET', account_path + account_qry_string,
|
|
swob.HTTPOk, account_headers,
|
|
json.dumps(account_listing_data))
|
|
end_qry_string = '?format=json&marker=%s&end_marker=&prefix=' % (
|
|
urllib.parse.quote(account_listing_data[-1]['name']))
|
|
self.app.register('GET', account_path + end_qry_string,
|
|
swob.HTTPOk, account_headers,
|
|
json.dumps([]))
|
|
|
|
|
|
class TestReconcilerUtils(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.fake_ring = FakeRing()
|
|
reconciler.direct_get_container_policy_index.reset()
|
|
|
|
def test_parse_raw_obj(self):
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "2:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(2017551.49350).internal,
|
|
'last_modified': timestamp_to_last_modified(2017551.49352),
|
|
'content_type': 'application/x-delete',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 2)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 2017551.49350)
|
|
self.assertEqual(got['q_record'], 2017551.49352)
|
|
self.assertEqual(got['q_op'], 'DELETE')
|
|
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(1234.20190).internal,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
'content_type': 'application/x-put',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 1)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 1234.20190)
|
|
self.assertEqual(got['q_record'], 1234.20192)
|
|
self.assertEqual(got['q_op'], 'PUT')
|
|
|
|
# the 'hash' field in object listing has the raw 'created_at' value
|
|
# which could be a composite of timestamps
|
|
timestamp_str = encode_timestamps(Timestamp(1234.20190),
|
|
Timestamp(1245.20190),
|
|
Timestamp(1256.20190),
|
|
explicit=True)
|
|
got = reconciler.parse_raw_obj({
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': timestamp_str,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
'content_type': 'application/x-put',
|
|
})
|
|
self.assertEqual(got['q_policy_index'], 1)
|
|
self.assertEqual(got['account'], 'AUTH_bob')
|
|
self.assertEqual(got['container'], 'con')
|
|
self.assertEqual(got['obj'], 'obj')
|
|
self.assertEqual(got['q_ts'], 1234.20190)
|
|
self.assertEqual(got['q_record'], 1234.20192)
|
|
self.assertEqual(got['q_op'], 'PUT')
|
|
|
|
# negative test
|
|
obj_info = {
|
|
'name': "1:/AUTH_bob/con/obj",
|
|
'hash': Timestamp(1234.20190).internal,
|
|
'last_modified': timestamp_to_last_modified(1234.20192),
|
|
}
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
obj_info['content_type'] = 'foo'
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
obj_info['content_type'] = 'appliation/x-post'
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj, obj_info)
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': 'bogus'})
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': '-1:/AUTH_test/container'})
|
|
self.assertRaises(ValueError, reconciler.parse_raw_obj,
|
|
{'name': 'asdf:/AUTH_test/c/obj'})
|
|
self.assertRaises(KeyError, reconciler.parse_raw_obj,
|
|
{'name': '0:/AUTH_test/c/obj',
|
|
'content_type': 'application/x-put'})
|
|
|
|
def test_get_container_policy_index(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
]
|
|
for permutation in itertools.permutations((0, 1, 2)):
|
|
reconciler.direct_get_container_policy_index.reset()
|
|
resp_headers = [stub_resp_headers[i] for i in permutation]
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
test_values = [(info['x-storage-policy-index'],
|
|
info['x-backend-status-changed-at']) for
|
|
info in resp_headers]
|
|
self.assertEqual(oldest_spi, 0,
|
|
"oldest policy index wrong "
|
|
"for permutation %r" % test_values)
|
|
|
|
def test_get_container_policy_index_with_error(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_change_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
# old timestamp, but 500 should be ignored...
|
|
ClientException(
|
|
'Container Server blew up',
|
|
http_status=500, http_reason='Server Error',
|
|
http_headers=container_resp_headers(
|
|
status_changed_at=Timestamp(0).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_with_socket_error(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_with_too_many_errors(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
ClientException(
|
|
'Container Server blew up',
|
|
http_status=500, http_reason='Server Error',
|
|
http_headers=container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertIsNone(oldest_spi)
|
|
|
|
def test_get_container_policy_index_for_deleted(self):
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
headers = container_resp_headers(
|
|
status_changed_at=Timestamp.now().internal,
|
|
storage_policy_index=1,
|
|
)
|
|
stub_resp_headers = [
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=headers,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_for_recently_deleted(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_for_recently_recreated(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
# old put, no recreate
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
# recently deleted
|
|
ClientException(
|
|
'Container Not Found',
|
|
http_status=404, http_reason='Not Found',
|
|
http_headers=container_resp_headers(
|
|
put_timestamp=next(ts),
|
|
delete_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
),
|
|
# recently recreated
|
|
container_resp_headers(
|
|
delete_timestamp=next(ts),
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 2)
|
|
|
|
def test_get_container_policy_index_for_recently_split_brain(self):
|
|
ts = itertools.count(int(time.time()))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
# oldest put
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=0,
|
|
),
|
|
# old recreate
|
|
container_resp_headers(
|
|
delete_timestamp=next(ts),
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=1,
|
|
),
|
|
# recently put
|
|
container_resp_headers(
|
|
delete_timestamp=0,
|
|
put_timestamp=next(ts),
|
|
status_changed_at=next(ts),
|
|
storage_policy_index=2,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 1)
|
|
|
|
def test_get_container_policy_index_cache(self):
|
|
now = time.time()
|
|
ts = itertools.count(int(now))
|
|
mock_path = 'swift.container.reconciler.direct_head_container'
|
|
stub_resp_headers = [
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=1,
|
|
),
|
|
container_resp_headers(
|
|
status_changed_at=Timestamp(next(ts)).internal,
|
|
storage_policy_index=0,
|
|
),
|
|
]
|
|
random.shuffle(stub_resp_headers)
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
self.assertEqual(oldest_spi, 0)
|
|
# re-mock with errors
|
|
stub_resp_headers = [
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
]
|
|
with mock.patch('time.time', new=lambda: now):
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
# still cached
|
|
self.assertEqual(oldest_spi, 0)
|
|
# propel time forward
|
|
the_future = now + 31
|
|
with mock.patch('time.time', new=lambda: the_future):
|
|
with mock.patch(mock_path) as direct_head:
|
|
direct_head.side_effect = stub_resp_headers
|
|
oldest_spi = reconciler.direct_get_container_policy_index(
|
|
self.fake_ring, 'a', 'con')
|
|
# expired
|
|
self.assertIsNone(oldest_spi)
|
|
|
|
def test_direct_delete_container_entry(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
x_timestamp = Timestamp.now()
|
|
headers = {'x-timestamp': x_timestamp.internal}
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
with mock.patch(mock_path, fake_hc):
|
|
reconciler.direct_delete_container_entry(
|
|
self.fake_ring, 'a', 'c', 'o', headers=headers)
|
|
|
|
self.assertEqual(len(connect_args), 3)
|
|
for args in connect_args:
|
|
self.assertEqual(args['method'], 'DELETE')
|
|
self.assertEqual(args['path'], '/a/c/o')
|
|
self.assertEqual(args['headers'].get('x-timestamp'),
|
|
headers['x-timestamp'])
|
|
|
|
def test_direct_delete_container_entry_with_errors(self):
|
|
# setup mock direct_delete
|
|
mock_path = \
|
|
'swift.container.reconciler.direct_delete_container_object'
|
|
stub_resp = [
|
|
None,
|
|
socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED)),
|
|
ClientException(
|
|
'Container Server blew up',
|
|
'10.0.0.12', 6201, 'sdj', 404, 'Not Found'
|
|
),
|
|
]
|
|
mock_direct_delete = mock.MagicMock()
|
|
mock_direct_delete.side_effect = stub_resp
|
|
|
|
with mock.patch(mock_path, mock_direct_delete), \
|
|
mock.patch('eventlet.greenpool.DEBUG', False):
|
|
rv = reconciler.direct_delete_container_entry(
|
|
self.fake_ring, 'a', 'c', 'o')
|
|
self.assertIsNone(rv)
|
|
self.assertEqual(len(mock_direct_delete.mock_calls), 3)
|
|
|
|
def test_add_to_reconciler_queue(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'DELETE')
|
|
|
|
self.assertTrue(ret)
|
|
self.assertEqual(ret, str(int(5948918.63946 // 3600 * 3600)))
|
|
self.assertEqual(len(connect_args), 3)
|
|
|
|
required_headers = ('x-content-type', 'x-etag')
|
|
|
|
for args in connect_args:
|
|
self.assertEqual(args['headers']['X-Timestamp'], '5948918.63946')
|
|
self.assertEqual(args['path'],
|
|
'/.misplaced_objects/5947200/17:/a/c/o')
|
|
self.assertEqual(args['headers']['X-Content-Type'],
|
|
'application/x-delete')
|
|
for header in required_headers:
|
|
self.assertTrue(header in args['headers'],
|
|
'%r was missing request headers %r' % (
|
|
header, args['headers']))
|
|
|
|
def test_add_to_reconciler_queue_force(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
connect_args = []
|
|
|
|
def test_connect(ipaddr, port, device, partition, method, path,
|
|
headers=None, query_string=None):
|
|
connect_args.append({
|
|
'ipaddr': ipaddr, 'port': port, 'device': device,
|
|
'partition': partition, 'method': method, 'path': path,
|
|
'headers': headers, 'query_string': query_string})
|
|
|
|
fake_hc = fake_http_connect(200, 200, 200, give_connect=test_connect)
|
|
now = time.time()
|
|
with mock.patch(mock_path, fake_hc), \
|
|
mock.patch('swift.container.reconciler.time.time',
|
|
lambda: now):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'PUT',
|
|
force=True)
|
|
|
|
self.assertTrue(ret)
|
|
self.assertEqual(ret, str(int(5948918.63946 // 3600 * 3600)))
|
|
self.assertEqual(len(connect_args), 3)
|
|
|
|
required_headers = ('x-size', 'x-content-type')
|
|
|
|
for args in connect_args:
|
|
self.assertEqual(args['headers']['X-Timestamp'],
|
|
Timestamp(now).internal)
|
|
self.assertEqual(args['headers']['X-Etag'], '5948918.63946')
|
|
self.assertEqual(args['path'],
|
|
'/.misplaced_objects/5947200/17:/a/c/o')
|
|
for header in required_headers:
|
|
self.assertTrue(header in args['headers'],
|
|
'%r was missing request headers %r' % (
|
|
header, args['headers']))
|
|
|
|
def test_add_to_reconciler_queue_fails(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
|
|
fake_connects = [fake_http_connect(200),
|
|
fake_http_connect(200, raise_timeout_exc=True),
|
|
fake_http_connect(507)]
|
|
|
|
def fake_hc(*a, **kw):
|
|
return fake_connects.pop()(*a, **kw)
|
|
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'PUT')
|
|
self.assertFalse(ret)
|
|
|
|
def test_add_to_reconciler_queue_socket_error(self):
|
|
mock_path = 'swift.common.direct_client.http_connect'
|
|
|
|
exc = socket.error(errno.ECONNREFUSED,
|
|
os.strerror(errno.ECONNREFUSED))
|
|
fake_connects = [fake_http_connect(200),
|
|
fake_http_connect(200, raise_timeout_exc=True),
|
|
fake_http_connect(500, raise_exc=exc)]
|
|
|
|
def fake_hc(*a, **kw):
|
|
return fake_connects.pop()(*a, **kw)
|
|
|
|
with mock.patch(mock_path, fake_hc):
|
|
ret = reconciler.add_to_reconciler_queue(
|
|
self.fake_ring, 'a', 'c', 'o', 17, 5948918.63946, 'DELETE')
|
|
self.assertFalse(ret)
|
|
|
|
|
|
def listing_qs(marker):
|
|
return "?format=json&marker=%s&end_marker=&prefix=" % \
|
|
urllib.parse.quote(marker.encode('utf-8'))
|
|
|
|
|
|
class TestReconciler(unittest.TestCase):
|
|
|
|
maxDiff = None
|
|
|
|
def setUp(self):
|
|
self.logger = debug_logger()
|
|
conf = {}
|
|
with mock.patch('swift.container.reconciler.InternalClient'):
|
|
self.reconciler = reconciler.ContainerReconciler(conf)
|
|
self.reconciler.logger = self.logger
|
|
self.start_interval = int(time.time() // 3600 * 3600)
|
|
self.current_container_path = '/v1/.misplaced_objects/%d' % (
|
|
self.start_interval) + listing_qs('')
|
|
|
|
def _mock_listing(self, objects):
|
|
self.reconciler.swift = FakeInternalClient(objects)
|
|
self.fake_swift = self.reconciler.swift.app
|
|
|
|
def _mock_oldest_spi(self, container_oldest_spi_map):
|
|
self.fake_swift._mock_oldest_spi_map = container_oldest_spi_map
|
|
|
|
def _run_once(self):
|
|
"""
|
|
Helper method to run the reconciler once with appropriate direct-client
|
|
mocks in place.
|
|
|
|
Returns the list of direct-deleted container entries in the format
|
|
[(acc1, con1, obj1), ...]
|
|
"""
|
|
|
|
def mock_oldest_spi(ring, account, container_name):
|
|
return self.fake_swift._mock_oldest_spi_map.get(container_name, 0)
|
|
|
|
items = {
|
|
'direct_get_container_policy_index': mock_oldest_spi,
|
|
'direct_delete_container_entry': mock.DEFAULT,
|
|
}
|
|
|
|
mock_time_iter = itertools.count(self.start_interval)
|
|
with mock.patch.multiple(reconciler, **items) as mocks:
|
|
self.mock_delete_container_entry = \
|
|
mocks['direct_delete_container_entry']
|
|
with mock.patch('time.time', mock_time_iter.next):
|
|
self.reconciler.run_once()
|
|
|
|
return [c[1][1:4] for c in
|
|
mocks['direct_delete_container_entry'].mock_calls]
|
|
|
|
def test_invalid_queue_name(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/bogus"): 3618.84187,
|
|
})
|
|
deleted_container_entries = self._run_once()
|
|
# we try to find something useful
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('bogus'))])
|
|
# but only get the bogus record
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
# and just leave it on the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertFalse(deleted_container_entries)
|
|
|
|
def test_invalid_queue_name_marches_onward(self):
|
|
# there's something useful there on the queue
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/00000bogus"): 3600.0000,
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1}) # already in the right spot!
|
|
deleted_container_entries = self._run_once()
|
|
# we get all the queue entries we can
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and one is garbage
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
# but the other is workable
|
|
self.assertEqual(self.reconciler.stats['noop_object'], 1)
|
|
# so pop the queue for that one
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_queue_name_with_policy_index_delimiter_in_name(self):
|
|
q_path = '.misplaced_objects/3600'
|
|
obj_path = "AUTH_bob/c:sneaky/o1:sneaky"
|
|
# there's something useful there on the queue
|
|
self._mock_listing({
|
|
(None, "/%s/1:/%s" % (q_path, obj_path)): 3618.84187,
|
|
(1, '/%s' % obj_path): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
# we find the misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/%s' % obj_path))])
|
|
# move it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/%s' % obj_path),
|
|
('DELETE', '/v1/%s' % obj_path)])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/%s' % obj_path),
|
|
('PUT', '/v1/%s' % obj_path)])
|
|
# clean up the source
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
# and pop the queue for that one
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries, [(
|
|
'.misplaced_objects', '3600', '1:/%s' % obj_path)])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_unable_to_direct_get_oldest_storage_policy(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
# the reconciler gets "None" if we can't quorum the container
|
|
self._mock_oldest_spi({'c': None})
|
|
deleted_container_entries = self._run_once()
|
|
# we look for misplaced objects
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but can't really say where to go looking
|
|
self.assertEqual(self.reconciler.stats['unavailable_container'], 1)
|
|
# we don't clean up anything
|
|
self.assertEqual(self.reconciler.stats['cleanup_object'], 0)
|
|
# and we definitely should not pop_queue
|
|
self.assertFalse(deleted_container_entries)
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'),
|
|
('PUT', '/v1/AUTH_bob/c/o1')])
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2))
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1))
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_the_other_direction(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/0:/AUTH_bob/c/o1"): 3618.84187,
|
|
(0, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('0:/AUTH_bob/c/o1'))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '0:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_with_unicode_and_spaces(self):
|
|
# the "name" in listings and the unicode string passed to all
|
|
# functions where we call them with (account, container, obj)
|
|
obj_name = u"AUTH_bob/c \u062a/o1 \u062a"
|
|
# anytime we talk about a call made to swift for a path
|
|
obj_path = obj_name.encode('utf-8')
|
|
# this mock expects unquoted unicode because it handles container
|
|
# listings as well as paths
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/%s" % obj_name): 3618.84187,
|
|
(1, "/%s" % obj_name): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
# listing_qs encodes and quotes - so give it name
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/%s' % obj_name))])
|
|
# moves it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
# these calls are to the real path
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/%s' % obj_path), # 2
|
|
('DELETE', '/v1/%s' % obj_path)]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/%s' % obj_path), # 1
|
|
('PUT', '/v1/%s' % obj_path)]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we PUT the object in the right place with q_ts + offset 2
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3618.84187, offset=1).internal)
|
|
self.assertEqual(
|
|
delete_headers.get('X-Backend-Storage-Policy-Index'), '1')
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
# this mock received the name, it's encoded down in buffered_http
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/%s' % obj_name)])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_delete(self):
|
|
q_ts = time.time()
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): (
|
|
Timestamp(q_ts).internal, 'application/x-delete'),
|
|
# object exists in "correct" storage policy - slightly older
|
|
(0, "/AUTH_bob/c/o1"): Timestamp(q_ts - 1).internal,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
# the tombstone exists in the enqueued storage policy
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound,
|
|
{'X-Backend-Timestamp': Timestamp(q_ts).internal})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# delete it
|
|
self.assertEqual(self.reconciler.stats['delete_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['delete_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'),
|
|
('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
reconcile_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# we DELETE the object in the right place with q_ts + offset 2
|
|
self.assertEqual(reconcile_headers.get('X-Timestamp'),
|
|
Timestamp(q_ts, offset=2).internal)
|
|
# cleans up the old
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# we DELETE the object from the wrong place with source_ts + offset 1
|
|
# timestamp to make sure the change takes effect
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(q_ts, offset=1))
|
|
# and when we're done, we pop the entry from the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_enqueued_for_the_correct_dest_noop(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3618.84187,
|
|
(1, "/AUTH_bob/c/o1"): 3618.84187,
|
|
})
|
|
self._mock_oldest_spi({'c': 1}) # already in the right spot!
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# nothing to see here
|
|
self.assertEqual(self.reconciler.stats['noop_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# so we just pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_src_object_newer_than_queue_entry(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
|
|
(1, '/AUTH_bob/c/o1'): 3600.234567, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0}) # destination
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# proceed with the move
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# .. with source timestamp + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3600.234567, offset=2))
|
|
# src object is cleaned up
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# ... with q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3600.123456, offset=1))
|
|
# and queue is popped
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_src_object_older_than_queue_entry(self):
|
|
# should be some sort of retry case
|
|
q_ts = time.time()
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
(1, '/AUTH_bob/c/o1'): q_ts - 0.00001, # slightly older
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_src_object_unavailable_with_slightly_newer_tombstone(self):
|
|
# should be some sort of retry case
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPNotFound,
|
|
{'X-Backend-Timestamp': Timestamp(q_ts, offset=2).internal})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_src_object_unavailable_server_error(self):
|
|
# should be some sort of retry case
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_bob/c/o1" % q_path): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
self.fake_swift.storage_policy[1].register(
|
|
'GET', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/%s' % q_path + listing_qs('')),
|
|
('GET', '/v1/%s' % q_path +
|
|
listing_qs('1:/AUTH_bob/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but no object copy is attempted
|
|
self.assertEqual(self.reconciler.stats['unavailable_source'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# src object is un-modified
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# queue is un-changed, we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_fails_cleanup(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3600.123456,
|
|
(1, '/AUTH_bob/c/o1'): 3600.123457, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0}) # destination
|
|
|
|
# make the DELETE blow up
|
|
self.fake_swift.storage_policy[1].register(
|
|
'DELETE', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# proceed with the move
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# .. with source timestamp + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(3600.123457, offset=2))
|
|
# we try to cleanup
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
# ... with q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3600.12346, offset=1))
|
|
# but cleanup fails!
|
|
self.assertEqual(self.reconciler.stats['cleanup_failed'], 1)
|
|
# so the queue is not popped
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
# and we'll have to retry
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_src_object_is_forever_gone(self):
|
|
# oh boy, hate to be here - this is an oldy
|
|
q_ts = self.start_interval - self.reconciler.reclaim_age - 1
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): q_ts,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# but it's gone :\
|
|
self.assertEqual(self.reconciler.stats['lost_source'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')])
|
|
# gah, look, even if it was out there somewhere - we've been at this
|
|
# two weeks and haven't found it. We can't just keep looking forever,
|
|
# so... we're done
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
# dunno if this is helpful, but FWIW we don't throw tombstones?
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
self.assertEqual(self.reconciler.stats['success'], 1) # lol
|
|
|
|
def test_object_move_dest_already_moved(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
|
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
|
(0, "/AUTH_bob/c/o1"): 3679.2019,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we look for misplaced objects
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but we found it already in the right place!
|
|
self.assertEqual(self.reconciler.stats['found_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# so no attempt to read the source is made, but we do cleanup
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[0]
|
|
# rather we just clean up the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3679.2019, offset=1))
|
|
# and wipe our hands of it
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_dest_object_newer_than_queue_entry(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/3600/1:/AUTH_bob/c/o1"): 3679.2019,
|
|
(1, "/AUTH_bob/c/o1"): 3679.2019,
|
|
(0, "/AUTH_bob/c/o1"): 3679.2019 + 0.00001, # slightly newer
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we look for misplaced objects...
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('3600')),
|
|
('GET', '/v1/.misplaced_objects/3600' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/3600' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# but we found it already in the right place!
|
|
self.assertEqual(self.reconciler.stats['found_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1')])
|
|
# so not attempt to read is made, but we do cleanup
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 0)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('DELETE', '/v1/AUTH_bob/c/o1')])
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[0]
|
|
# rather we just clean up the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(3679.2019, offset=1))
|
|
# and since we cleaned up the old object, so this counts as done
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '3600', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_dest_object_older_than_queue_entry(self):
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.38393,
|
|
(1, "/AUTH_bob/c/o1"): 36123.38393,
|
|
(0, "/AUTH_bob/c/o1"): 36123.38393 - 0.00001, # slightly older
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we found a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and since our version is *newer*, we overwrite
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1'), # 2
|
|
('DELETE', '/v1/AUTH_bob/c/o1')]) # 4
|
|
delete_headers = self.fake_swift.storage_policy[1].headers[1]
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
# ... with a q_ts + offset 2
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.38393, offset=2))
|
|
# then clean the dark matter
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 1)
|
|
self.assertEqual(self.reconciler.stats['cleanup_success'], 1)
|
|
# ... with a q_ts + offset 1
|
|
self.assertEqual(delete_headers.get('X-Timestamp'),
|
|
Timestamp(36123.38393, offset=1))
|
|
|
|
# and pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 1)
|
|
self.assertEqual(deleted_container_entries,
|
|
[('.misplaced_objects', '36000', '1:/AUTH_bob/c/o1')])
|
|
self.assertEqual(self.reconciler.stats['success'], 1)
|
|
|
|
def test_object_move_put_fails(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.383925,
|
|
(1, "/AUTH_bob/c/o1"): 36123.383925,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
# make the put to dest fail!
|
|
self.fake_swift.storage_policy[0].register(
|
|
'PUT', '/v1/AUTH_bob/c/o1', swob.HTTPServiceUnavailable, {})
|
|
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we find a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and try to move it, but it fails
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')]) # 2
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# ...with q_ts + offset 2 (20-microseconds)
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.383925, offset=2))
|
|
# but it failed
|
|
self.assertEqual(self.reconciler.stats['copy_success'], 0)
|
|
self.assertEqual(self.reconciler.stats['copy_failed'], 1)
|
|
# ... so we don't clean up the source
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# and we don't pop the queue
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['unhandled_errors'], 0)
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_put_blows_up_crazy_town(self):
|
|
# setup the cluster
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/36000/1:/AUTH_bob/c/o1"): 36123.383925,
|
|
(1, "/AUTH_bob/c/o1"): 36123.383925,
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
# make the put to dest blow up crazy town
|
|
def blow_up(*args, **kwargs):
|
|
raise Exception('kaboom!')
|
|
|
|
self.fake_swift.storage_policy[0].register(
|
|
'PUT', '/v1/AUTH_bob/c/o1', blow_up, {})
|
|
|
|
# turn the crank
|
|
deleted_container_entries = self._run_once()
|
|
|
|
# we find a misplaced object
|
|
self.assertEqual(self.reconciler.stats['misplaced_object'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('36000')),
|
|
('GET', '/v1/.misplaced_objects/36000' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/36000' +
|
|
listing_qs('1:/AUTH_bob/c/o1'))])
|
|
# and attempt to move it
|
|
self.assertEqual(self.reconciler.stats['copy_attempt'], 1)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_bob/c/o1')]) # 2
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_bob/c/o1'), # 1
|
|
('PUT', '/v1/AUTH_bob/c/o1')]) # 3
|
|
put_headers = self.fake_swift.storage_policy[0].headers[1]
|
|
# ...with q_ts + offset 2 (20-microseconds)
|
|
self.assertEqual(put_headers.get('X-Timestamp'),
|
|
Timestamp(36123.383925, offset=2))
|
|
# but it blows up hard
|
|
self.assertEqual(self.reconciler.stats['unhandled_error'], 1)
|
|
# so we don't cleanup
|
|
self.assertEqual(self.reconciler.stats['cleanup_attempt'], 0)
|
|
# and we don't pop the queue
|
|
self.assertEqual(self.reconciler.stats['pop_queue'], 0)
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(self.reconciler.stats['retry'], 1)
|
|
|
|
def test_object_move_no_such_object_no_tombstone_recent(self):
|
|
q_ts = float(Timestamp.now())
|
|
container = str(int(q_ts // 3600 * 3600))
|
|
q_path = '.misplaced_objects/%s' % container
|
|
|
|
self._mock_listing({
|
|
(None, "/%s/1:/AUTH_jeb/c/o1" % q_path): q_ts
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
deleted_container_entries = self._run_once()
|
|
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('1:/AUTH_jeb/c/o1')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
# the queue entry is recent enough that there could easily be
|
|
# tombstones on offline nodes or something, so we'll just leave it
|
|
# here and try again later
|
|
self.assertEqual(deleted_container_entries, [])
|
|
|
|
def test_object_move_no_such_object_no_tombstone_ancient(self):
|
|
queue_ts = float(Timestamp.now()) - \
|
|
self.reconciler.reclaim_age * 1.1
|
|
container = str(int(queue_ts // 3600 * 3600))
|
|
|
|
self._mock_listing({
|
|
(
|
|
None, "/.misplaced_objects/%s/1:/AUTH_jeb/c/o1" % container
|
|
): queue_ts
|
|
})
|
|
self._mock_oldest_spi({'c': 0})
|
|
|
|
deleted_container_entries = self._run_once()
|
|
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container)),
|
|
('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('1:/AUTH_jeb/c/o1'))])
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[0].calls,
|
|
[('HEAD', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
self.assertEqual(
|
|
self.fake_swift.storage_policy[1].calls,
|
|
[('GET', '/v1/AUTH_jeb/c/o1')],
|
|
)
|
|
|
|
# the queue entry is old enough that the tombstones, if any, have
|
|
# probably been reaped, so we'll just give up
|
|
self.assertEqual(
|
|
deleted_container_entries,
|
|
[('.misplaced_objects', container, '1:/AUTH_jeb/c/o1')])
|
|
|
|
def test_delete_old_empty_queue_containers(self):
|
|
ts = time.time() - self.reconciler.reclaim_age * 1.1
|
|
container = str(int(ts // 3600 * 3600))
|
|
older_ts = ts - 3600
|
|
older_container = str(int(older_ts // 3600 * 3600))
|
|
self._mock_listing({
|
|
(None, "/.misplaced_objects/%s/" % container): 0,
|
|
(None, "/.misplaced_objects/%s/something" % older_container): 0,
|
|
})
|
|
deleted_container_entries = self._run_once()
|
|
self.assertEqual(deleted_container_entries, [])
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(container)),
|
|
('GET', '/v1/.misplaced_objects/%s' % container + listing_qs('')),
|
|
('DELETE', '/v1/.misplaced_objects/%s' % container),
|
|
('GET', '/v1/.misplaced_objects/%s' % older_container +
|
|
listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects/%s' % older_container +
|
|
listing_qs('something'))])
|
|
self.assertEqual(self.reconciler.stats['invalid_record'], 1)
|
|
|
|
def test_iter_over_old_containers_in_reverse(self):
|
|
step = reconciler.MISPLACED_OBJECTS_CONTAINER_DIVISOR
|
|
now = self.start_interval
|
|
containers = []
|
|
for i in range(10):
|
|
container_ts = int(now - step * i)
|
|
container_name = str(container_ts // 3600 * 3600)
|
|
containers.append(container_name)
|
|
# add some old containers too
|
|
now -= self.reconciler.reclaim_age
|
|
old_containers = []
|
|
for i in range(10):
|
|
container_ts = int(now - step * i)
|
|
container_name = str(container_ts // 3600 * 3600)
|
|
old_containers.append(container_name)
|
|
containers.sort()
|
|
old_containers.sort()
|
|
all_containers = old_containers + containers
|
|
self._mock_listing(dict((
|
|
(None, "/.misplaced_objects/%s/" % container), 0
|
|
) for container in all_containers))
|
|
deleted_container_entries = self._run_once()
|
|
self.assertEqual(deleted_container_entries, [])
|
|
last_container = all_containers[-1]
|
|
account_listing_calls = [
|
|
('GET', '/v1/.misplaced_objects' + listing_qs('')),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(last_container)),
|
|
]
|
|
new_container_calls = [
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('')) for container in reversed(containers)
|
|
][1:] # current_container get's skipped the second time around...
|
|
old_container_listings = [
|
|
('GET', '/v1/.misplaced_objects/%s' % container +
|
|
listing_qs('')) for container in reversed(old_containers)
|
|
]
|
|
old_container_deletes = [
|
|
('DELETE', '/v1/.misplaced_objects/%s' % container)
|
|
for container in reversed(old_containers)
|
|
]
|
|
old_container_calls = list(itertools.chain(*zip(
|
|
old_container_listings, old_container_deletes)))
|
|
self.assertEqual(self.fake_swift.calls,
|
|
[('GET', self.current_container_path)] +
|
|
account_listing_calls + new_container_calls +
|
|
old_container_calls)
|
|
|
|
def test_error_in_iter_containers(self):
|
|
self._mock_listing({})
|
|
|
|
# make the listing return an error
|
|
self.fake_swift.storage_policy[None].register(
|
|
'GET', '/v1/.misplaced_objects' + listing_qs(''),
|
|
swob.HTTPServiceUnavailable, {})
|
|
|
|
self._run_once()
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(''))])
|
|
self.assertEqual(self.reconciler.stats, {})
|
|
errors = self.reconciler.logger.get_lines_for_level('error')
|
|
self.assertEqual(errors, [
|
|
'Error listing containers in account '
|
|
'.misplaced_objects (Unexpected response: '
|
|
'503 Service Unavailable)'])
|
|
|
|
def test_unhandled_exception_in_reconcile(self):
|
|
self._mock_listing({})
|
|
|
|
# make the listing blow up
|
|
def blow_up(*args, **kwargs):
|
|
raise Exception('kaboom!')
|
|
|
|
self.fake_swift.storage_policy[None].register(
|
|
'GET', '/v1/.misplaced_objects' + listing_qs(''),
|
|
blow_up, {})
|
|
self._run_once()
|
|
self.assertEqual(
|
|
self.fake_swift.calls,
|
|
[('GET', self.current_container_path),
|
|
('GET', '/v1/.misplaced_objects' + listing_qs(''))])
|
|
self.assertEqual(self.reconciler.stats, {})
|
|
errors = self.reconciler.logger.get_lines_for_level('error')
|
|
self.assertEqual(errors,
|
|
['Unhandled Exception trying to reconcile: '])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|