openstack
/
karbor
Code Issues Proposed changes

Merge "Fix glance image restore error"

This commit is contained in:
Jenkins 2017-07-26 16:19:03 +00:00 committed by Gerrit Code Review
parent a7b2eb6432 0e56552b24
commit c8a03e5a34
8 changed files with 59 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/providers.d/openstack-s3-bank.conf
View File

@ -8,6 +8,8 @@ plugin=karbor-image-protection-plugin
plugin=karbor-server-protection-plugin
bank=karbor-s3-bank-plugin
enabled=False
[s3_client]
s3_endpoint=http://127.0.0.1:7480
s3_access_key=demo

4
karbor/services/protection/bank_plugins/file_system_bank_plugin.py
View File

@ -182,8 +182,8 @@ class FileSystemBankPlugin(BankPlugin):
LOG.debug("FsBank: list_objects. key: %s", prefix)
try:
file_lists = self._list_object(prefix)
return (
file_lists[-limit:] if limit is not None else file_lists)
except OSError as err:
LOG.error("List objects failed. err: %s", err)
raise exception.BankListObjectsFailed(reason=err)
else:
return file_lists[-limit:] if limit is not None else file_lists

42
karbor/services/protection/bank_plugins/s3_bank_plugin.py
View File

@ -155,10 +155,7 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
limit=limit,
marker=marker
)
if 'Contents' not in response:
return []
else:
return [obj['Key'] for obj in response['Contents']]
return [obj['Key'] for obj in response]
except S3ConnectionFailed as err:
LOG.error("list objects failed, err: %s.", err)
raise exception.BankListObjectsFailed(reason=err)
@ -222,15 +219,38 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
except ClientError as err:
raise S3ConnectionFailed(reason=err)
def _get_bucket(self, bucket, prefix=None, limit=None, marker=None):
def _get_bucket(self, bucket, prefix=None, limit=None,
marker=None):
try:
prefix = '' if prefix is None else prefix
marker = '' if marker is None else marker
response = self.connection.list_objects(
Bucket=bucket,
Prefix=prefix,
MaxKeys=limit,
Marker=marker)
return response
objects_to_return = []
if limit is None:
is_truncated = True
while is_truncated:
response = self.connection.list_objects(
Bucket=bucket,
Prefix=prefix,
Marker=marker
)
if 'Contents' not in response:
break
is_truncated = response['IsTruncated']
objects_to_return.extend(response['Contents'])
marker = response['Contents'][-1]['Key']
else:
response = self.connection.list_objects(
Bucket=bucket,
Prefix=prefix,
MaxKeys=limit,
Marker=marker
)
if 'Contents' in response:
objects_to_return.extend(response['Contents'])
except ClientError as err:
raise S3ConnectionFailed(reason=err)
else:
return objects_to_return

10
karbor/services/protection/bank_plugins/swift_bank_plugin.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import logging as log
import math
import time
@ -33,6 +34,7 @@ swift_bank_plugin_opts = [
]
LOG = logging.getLogger(__name__)
log.getLogger('swiftclient').setLevel(log.WARNING)
lease_opt = [cfg.IntOpt('lease_expire_window',
default=600,
@ -148,8 +150,7 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
body = self._get_container(
container=self.bank_object_container,
prefix=prefix, end_marker=marker)
limit_objects = body[-limit:] if limit is not None else body
return [obj.get("name") for obj in limit_objects]
return [obj.get("name") for obj in body]
else:
body = self._get_container(
container=self.bank_object_container,
@ -237,13 +238,16 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
def _get_container(self, container, prefix=None, limit=None, marker=None,
end_marker=None):
full_listing = True if limit is None else False
try:
(_resp, body) = self.connection.get_container(
container=container,
prefix=prefix,
limit=limit,
marker=marker,
end_marker=end_marker)
end_marker=end_marker,
full_listing=full_listing
)
return body
except ClientException as err:
raise SwiftConnectionFailed(reason=err)

2
karbor/services/protection/clients/s3.py
View File

@ -48,6 +48,6 @@ def create(context, conf, **kwargs):
return botocore.session.get_session().create_client(
's3',
aws_access_key_id=client_config.s3_access_key,
aws_secret_access_key=client_config.s3_ecret_key,
aws_secret_access_key=client_config.s3_secret_key,
endpoint_url=client_config.s3_endpoint
)

4
karbor/services/protection/protection_plugins/image/image_protection_plugin.py
View File

@ -130,6 +130,8 @@ class ProtectOperation(protection_plugin.Operation):
data = image_response_data.read()
if data != '':
bank_section.update_object("data_" + str(chunks_num), data)
else:
chunks_num -= 1
# Save the chunks_num to metadata
resource_definition = bank_section.get_object("metadata")
@ -204,6 +206,8 @@ class RestoreOperation(protection_plugin.Operation):
# check the chunks_num
chunks_num = resource_definition.get("chunks_num", 0)
if len(objects) != int(chunks_num):
LOG.debug('object num: {0}, chunk num: {1}'.
format(len(objects), chunks_num))
raise exception.RestoreBackupFailed(
reason=" The chunks_num of restored image is invalid.",
resource_id=original_image_id,

12
karbor/tests/unit/protection/fake_s3_client.py
View File

@ -33,7 +33,7 @@ class FakeS3Connection(object):
'Keys': {}
}
def list_objects(self, Bucket, Prefix, MaxKeys, Marker):
def list_objects(self, Bucket, Prefix, Marker):
body = []
prefix = '' if not Prefix else Prefix
for obj in self.s3_dir[Bucket]['Keys'].keys():
@ -41,7 +41,15 @@ class FakeS3Connection(object):
body.append({
'Key': obj
})
return {'Contents': body}
if len(body) == 0:
return {
'IsTruncated': False
}
else:
return {
'Contents': body,
'IsTruncated': False
}
def put_object(self, Bucket, Key, Body, Metadata=None):
if Bucket in self.s3_dir.keys():

3
karbor/tests/unit/protection/fake_swift_client.py
View File

@ -38,7 +38,8 @@ class FakeSwiftConnection(object):
else:
os.makedirs(container_dir)
def get_container(self, container, prefix, limit, marker, end_marker):
def get_container(self, container, prefix, limit, marker,
end_marker, full_listing):
container_dir = self.swiftdir + "/" + container
body = []
if prefix: