303 lines
12 KiB
Python
303 lines
12 KiB
Python
# Copyright (c) 2010-2014 OpenStack Foundation.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from base64 import standard_b64encode as b64encode
|
|
from base64 import standard_b64decode as b64decode
|
|
|
|
from six.moves.urllib.parse import quote
|
|
|
|
from swift.common import swob
|
|
from swift.common.http import HTTP_OK
|
|
from swift.common.utils import json, public, config_true_value
|
|
|
|
from swift.common.middleware.s3api.controllers.base import Controller
|
|
from swift.common.middleware.s3api.etree import Element, SubElement, tostring, \
|
|
fromstring, XMLSyntaxError, DocumentInvalid
|
|
from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
|
|
InvalidArgument, \
|
|
MalformedXML, InvalidLocationConstraint, NoSuchBucket, \
|
|
BucketNotEmpty, InternalError, ServiceUnavailable, NoSuchKey
|
|
from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX
|
|
|
|
MAX_PUT_BUCKET_BODY_SIZE = 10240
|
|
|
|
|
|
class BucketController(Controller):
|
|
"""
|
|
Handles bucket request.
|
|
"""
|
|
def _delete_segments_bucket(self, req):
|
|
"""
|
|
Before delete bucket, delete segments bucket if existing.
|
|
"""
|
|
container = req.container_name + MULTIUPLOAD_SUFFIX
|
|
marker = ''
|
|
seg = ''
|
|
|
|
try:
|
|
resp = req.get_response(self.app, 'HEAD')
|
|
if int(resp.sw_headers['X-Container-Object-Count']) > 0:
|
|
raise BucketNotEmpty()
|
|
# FIXME: This extra HEAD saves unexpected segment deletion
|
|
# but if a complete multipart upload happen while cleanup
|
|
# segment container below, completed object may be missing its
|
|
# segments unfortunately. To be safer, it might be good
|
|
# to handle if the segments can be deleted for each object.
|
|
except NoSuchBucket:
|
|
pass
|
|
|
|
try:
|
|
while True:
|
|
# delete all segments
|
|
resp = req.get_response(self.app, 'GET', container,
|
|
query={'format': 'json',
|
|
'marker': marker})
|
|
segments = json.loads(resp.body)
|
|
for seg in segments:
|
|
try:
|
|
req.get_response(
|
|
self.app, 'DELETE', container,
|
|
swob.bytes_to_wsgi(seg['name'].encode('utf8')))
|
|
except NoSuchKey:
|
|
pass
|
|
except InternalError:
|
|
raise ServiceUnavailable()
|
|
if segments:
|
|
marker = seg['name']
|
|
else:
|
|
break
|
|
req.get_response(self.app, 'DELETE', container)
|
|
except NoSuchBucket:
|
|
return
|
|
except (BucketNotEmpty, InternalError):
|
|
raise ServiceUnavailable()
|
|
|
|
@public
|
|
def HEAD(self, req):
|
|
"""
|
|
Handle HEAD Bucket (Get Metadata) request
|
|
"""
|
|
resp = req.get_response(self.app)
|
|
|
|
return HTTPOk(headers=resp.headers)
|
|
|
|
@public
|
|
def GET(self, req):
|
|
"""
|
|
Handle GET Bucket (List Objects) request
|
|
"""
|
|
|
|
max_keys = req.get_validated_param(
|
|
'max-keys', self.conf.max_bucket_listing)
|
|
# TODO: Separate max_bucket_listing and default_bucket_listing
|
|
tag_max_keys = max_keys
|
|
max_keys = min(max_keys, self.conf.max_bucket_listing)
|
|
|
|
encoding_type = req.params.get('encoding-type')
|
|
if encoding_type is not None and encoding_type != 'url':
|
|
err_msg = 'Invalid Encoding Method specified in Request'
|
|
raise InvalidArgument('encoding-type', encoding_type, err_msg)
|
|
|
|
query = {
|
|
'format': 'json',
|
|
'limit': max_keys + 1,
|
|
}
|
|
if 'prefix' in req.params:
|
|
query.update({'prefix': req.params['prefix']})
|
|
if 'delimiter' in req.params:
|
|
query.update({'delimiter': req.params['delimiter']})
|
|
fetch_owner = False
|
|
if 'versions' in req.params:
|
|
listing_type = 'object-versions'
|
|
if 'key-marker' in req.params:
|
|
query.update({'marker': req.params['key-marker']})
|
|
elif 'version-id-marker' in req.params:
|
|
err_msg = ('A version-id marker cannot be specified without '
|
|
'a key marker.')
|
|
raise InvalidArgument('version-id-marker',
|
|
req.params['version-id-marker'], err_msg)
|
|
elif int(req.params.get('list-type', '1')) == 2:
|
|
listing_type = 'version-2'
|
|
if 'start-after' in req.params:
|
|
query.update({'marker': req.params['start-after']})
|
|
# continuation-token overrides start-after
|
|
if 'continuation-token' in req.params:
|
|
decoded = b64decode(req.params['continuation-token'])
|
|
query.update({'marker': decoded})
|
|
if 'fetch-owner' in req.params:
|
|
fetch_owner = config_true_value(req.params['fetch-owner'])
|
|
else:
|
|
listing_type = 'version-1'
|
|
if 'marker' in req.params:
|
|
query.update({'marker': req.params['marker']})
|
|
|
|
resp = req.get_response(self.app, query=query)
|
|
|
|
objects = json.loads(resp.body)
|
|
|
|
# in order to judge that truncated is valid, check whether
|
|
# max_keys + 1 th element exists in swift.
|
|
is_truncated = max_keys > 0 and len(objects) > max_keys
|
|
objects = objects[:max_keys]
|
|
|
|
if listing_type == 'object-versions':
|
|
elem = Element('ListVersionsResult')
|
|
SubElement(elem, 'Name').text = req.container_name
|
|
SubElement(elem, 'Prefix').text = req.params.get('prefix')
|
|
SubElement(elem, 'KeyMarker').text = req.params.get('key-marker')
|
|
SubElement(elem, 'VersionIdMarker').text = req.params.get(
|
|
'version-id-marker')
|
|
if is_truncated:
|
|
if 'name' in objects[-1]:
|
|
SubElement(elem, 'NextKeyMarker').text = \
|
|
objects[-1]['name']
|
|
if 'subdir' in objects[-1]:
|
|
SubElement(elem, 'NextKeyMarker').text = \
|
|
objects[-1]['subdir']
|
|
SubElement(elem, 'NextVersionIdMarker').text = 'null'
|
|
else:
|
|
elem = Element('ListBucketResult')
|
|
SubElement(elem, 'Name').text = req.container_name
|
|
SubElement(elem, 'Prefix').text = req.params.get('prefix')
|
|
if listing_type == 'version-1':
|
|
SubElement(elem, 'Marker').text = req.params.get('marker')
|
|
if is_truncated and 'delimiter' in req.params:
|
|
if 'name' in objects[-1]:
|
|
name = objects[-1]['name']
|
|
else:
|
|
name = objects[-1]['subdir']
|
|
if encoding_type == 'url':
|
|
name = quote(name.encode('utf-8'))
|
|
SubElement(elem, 'NextMarker').text = name
|
|
elif listing_type == 'version-2':
|
|
if is_truncated:
|
|
if 'name' in objects[-1]:
|
|
SubElement(elem, 'NextContinuationToken').text = \
|
|
b64encode(objects[-1]['name'].encode('utf-8'))
|
|
if 'subdir' in objects[-1]:
|
|
SubElement(elem, 'NextContinuationToken').text = \
|
|
b64encode(objects[-1]['subdir'].encode('utf-8'))
|
|
if 'continuation-token' in req.params:
|
|
SubElement(elem, 'ContinuationToken').text = \
|
|
req.params['continuation-token']
|
|
if 'start-after' in req.params:
|
|
SubElement(elem, 'StartAfter').text = \
|
|
req.params['start-after']
|
|
SubElement(elem, 'KeyCount').text = str(len(objects))
|
|
|
|
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
|
|
|
|
if 'delimiter' in req.params:
|
|
SubElement(elem, 'Delimiter').text = req.params['delimiter']
|
|
|
|
if encoding_type == 'url':
|
|
SubElement(elem, 'EncodingType').text = encoding_type
|
|
|
|
SubElement(elem, 'IsTruncated').text = \
|
|
'true' if is_truncated else 'false'
|
|
|
|
for o in objects:
|
|
if 'subdir' not in o:
|
|
name = o['name']
|
|
if encoding_type == 'url':
|
|
name = quote(name.encode('utf-8'))
|
|
|
|
if listing_type == 'object-versions':
|
|
contents = SubElement(elem, 'Version')
|
|
SubElement(contents, 'Key').text = name
|
|
SubElement(contents, 'VersionId').text = 'null'
|
|
SubElement(contents, 'IsLatest').text = 'true'
|
|
else:
|
|
contents = SubElement(elem, 'Contents')
|
|
SubElement(contents, 'Key').text = name
|
|
SubElement(contents, 'LastModified').text = \
|
|
o['last_modified'][:-3] + 'Z'
|
|
if 's3_etag' in o:
|
|
# New-enough MUs are already in the right format
|
|
etag = o['s3_etag']
|
|
elif 'slo_etag' in o:
|
|
# SLOs may be in something *close* to the MU format
|
|
etag = '"%s-N"' % o['slo_etag'].strip('"')
|
|
else:
|
|
# Normal objects just use the MD5
|
|
etag = '"%s"' % o['hash']
|
|
# This also catches sufficiently-old SLOs, but we have
|
|
# no way to identify those from container listings
|
|
SubElement(contents, 'ETag').text = etag
|
|
SubElement(contents, 'Size').text = str(o['bytes'])
|
|
if fetch_owner or listing_type != 'version-2':
|
|
owner = SubElement(contents, 'Owner')
|
|
SubElement(owner, 'ID').text = req.user_id
|
|
SubElement(owner, 'DisplayName').text = req.user_id
|
|
SubElement(contents, 'StorageClass').text = 'STANDARD'
|
|
|
|
for o in objects:
|
|
if 'subdir' in o:
|
|
common_prefixes = SubElement(elem, 'CommonPrefixes')
|
|
name = o['subdir']
|
|
if encoding_type == 'url':
|
|
name = quote(name.encode('utf-8'))
|
|
SubElement(common_prefixes, 'Prefix').text = name
|
|
|
|
body = tostring(elem)
|
|
|
|
return HTTPOk(body=body, content_type='application/xml')
|
|
|
|
@public
|
|
def PUT(self, req):
|
|
"""
|
|
Handle PUT Bucket request
|
|
"""
|
|
xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
|
|
if xml:
|
|
# check location
|
|
try:
|
|
elem = fromstring(
|
|
xml, 'CreateBucketConfiguration', self.logger)
|
|
location = elem.find('./LocationConstraint').text
|
|
except (XMLSyntaxError, DocumentInvalid):
|
|
raise MalformedXML()
|
|
except Exception as e:
|
|
self.logger.error(e)
|
|
raise
|
|
|
|
if location != self.conf.location:
|
|
# s3api cannot support multiple regions currently.
|
|
raise InvalidLocationConstraint()
|
|
|
|
resp = req.get_response(self.app)
|
|
|
|
resp.status = HTTP_OK
|
|
resp.location = '/' + req.container_name
|
|
|
|
return resp
|
|
|
|
@public
|
|
def DELETE(self, req):
|
|
"""
|
|
Handle DELETE Bucket request
|
|
"""
|
|
if self.conf.allow_multipart_uploads:
|
|
self._delete_segments_bucket(req)
|
|
resp = req.get_response(self.app)
|
|
return resp
|
|
|
|
@public
|
|
def POST(self, req):
|
|
"""
|
|
Handle POST Bucket request
|
|
"""
|
|
raise S3NotImplemented()
|