Implements the S3 store to the level of the swift store.
This branch is Chris' work with a merge of trunk, fix of merge conflicts from trunk, and moving the import of boto into a conditional block so tests can run with the fakes when boto is not installed on the local machine.
This commit is contained in:
jaypipes@gmail.com
Tarmac
commit
0afe4cc554
|
|
@ -53,13 +53,17 @@ def get_backend_class(backend):
|
|||
"""
|
||||
# NOTE(sirp): avoiding circular import
|
||||
from glance.store.http import HTTPBackend
|
||||
from glance.store.s3 import S3Backend
|
||||
from glance.store.swift import SwiftBackend
|
||||
from glance.store.filesystem import FilesystemBackend
|
||||
|
||||
BACKENDS = {"file": FilesystemBackend,
|
||||
"http": HTTPBackend,
|
||||
"https": HTTPBackend,
|
||||
"swift": SwiftBackend}
|
||||
BACKENDS = {
|
||||
"file": FilesystemBackend,
|
||||
"http": HTTPBackend,
|
||||
"https": HTTPBackend,
|
||||
"swift": SwiftBackend,
|
||||
"s3": S3Backend
|
||||
}
|
||||
|
||||
try:
|
||||
return BACKENDS[backend]
|
||||
|
|
@ -99,3 +103,41 @@ def get_store_from_location(location):
|
|||
"""
|
||||
loc_pieces = urlparse.urlparse(location)
|
||||
return loc_pieces.scheme
|
||||
|
||||
|
||||
def parse_uri_tokens(parsed_uri, example_url):
|
||||
"""
|
||||
Given a URI and an example_url, attempt to parse the uri to assemble an
|
||||
authurl. This method returns the user, key, authurl, referenced container,
|
||||
and the object we're looking for in that container.
|
||||
|
||||
Parsing the uri is three phases:
|
||||
1) urlparse to split the tokens
|
||||
2) use RE to split on @ and /
|
||||
3) reassemble authurl
|
||||
|
||||
"""
|
||||
path = parsed_uri.path.lstrip('//')
|
||||
netloc = parsed_uri.netloc
|
||||
|
||||
try:
|
||||
try:
|
||||
creds, netloc = netloc.split('@')
|
||||
except ValueError:
|
||||
# Python 2.6.1 compat
|
||||
# see lp659445 and Python issue7904
|
||||
creds, path = path.split('@')
|
||||
user, key = creds.split(':')
|
||||
path_parts = path.split('/')
|
||||
obj = path_parts.pop()
|
||||
container = path_parts.pop()
|
||||
except (ValueError, IndexError):
|
||||
raise BackendException(
|
||||
"Expected four values to unpack in: %s:%s. "
|
||||
"Should have received something like: %s."
|
||||
% (parsed_uri.scheme, parsed_uri.path, example_url))
|
||||
|
||||
authurl = "https://%s" % '/'.join(path_parts)
|
||||
|
||||
return user, key, authurl, container, obj
|
||||
|
||||
|
|
|
|||
|
|
@ -14,107 +14,3 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import urlparse
|
||||
|
||||
from glance.common import exception
|
||||
|
||||
|
||||
# TODO(sirp): should this be moved out to common/utils.py ?
|
||||
def _file_iter(f, size):
|
||||
"""
|
||||
Return an iterator for a file-like object
|
||||
"""
|
||||
chunk = f.read(size)
|
||||
while chunk:
|
||||
yield chunk
|
||||
chunk = f.read(size)
|
||||
|
||||
|
||||
class BackendException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedBackend(BackendException):
|
||||
pass
|
||||
|
||||
|
||||
class Backend(object):
|
||||
CHUNKSIZE = 4096
|
||||
|
||||
|
||||
class FilesystemBackend(Backend):
|
||||
@classmethod
|
||||
def get(cls, parsed_uri, expected_size, opener=lambda p: open(p, "rb")):
|
||||
""" Filesystem-based backend
|
||||
|
||||
file:///path/to/file.tar.gz.0
|
||||
"""
|
||||
#FIXME: must prevent attacks using ".." and "." paths
|
||||
with opener(parsed_uri.path) as f:
|
||||
return _file_iter(f, cls.CHUNKSIZE)
|
||||
|
||||
@classmethod
|
||||
def delete(cls, parsed_uri):
|
||||
"""
|
||||
Removes a file from the filesystem backend.
|
||||
|
||||
:param parsed_uri: Parsed pieces of URI in form of::
|
||||
file:///path/to/filename.ext
|
||||
|
||||
:raises NotFound if file does not exist
|
||||
:raises NotAuthorized if cannot delete because of permissions
|
||||
"""
|
||||
fn = parsed_uri.path
|
||||
if os.path.exists(fn):
|
||||
try:
|
||||
os.unlink(fn)
|
||||
except OSError:
|
||||
raise exception.NotAuthorized("You cannot delete file %s" % fn)
|
||||
else:
|
||||
raise exception.NotFound("File %s does not exist" % fn)
|
||||
|
||||
|
||||
def get_backend_class(backend):
|
||||
"""
|
||||
Returns the backend class as designated in the
|
||||
backend name
|
||||
|
||||
:param backend: Name of backend to create
|
||||
"""
|
||||
# NOTE(sirp): avoiding circular import
|
||||
from glance.store.backends.http import HTTPBackend
|
||||
from glance.store.backends.swift import SwiftBackend
|
||||
|
||||
BACKENDS = {"file": FilesystemBackend,
|
||||
"http": HTTPBackend,
|
||||
"https": HTTPBackend,
|
||||
"swift": SwiftBackend}
|
||||
|
||||
try:
|
||||
return BACKENDS[backend]
|
||||
except KeyError:
|
||||
raise UnsupportedBackend("No backend found for '%s'" % scheme)
|
||||
|
||||
|
||||
def get_from_backend(uri, **kwargs):
|
||||
"""Yields chunks of data from backend specified by uri"""
|
||||
|
||||
parsed_uri = urlparse.urlparse(uri)
|
||||
scheme = parsed_uri.scheme
|
||||
|
||||
backend_class = get_backend_class(scheme)
|
||||
|
||||
return backend_class.get(parsed_uri, **kwargs)
|
||||
|
||||
|
||||
def delete_from_backend(uri, **kwargs):
|
||||
"""Removes chunks of data from backend specified by uri"""
|
||||
|
||||
parsed_uri = urlparse.urlparse(uri)
|
||||
scheme = parsed_uri.scheme
|
||||
|
||||
backend_class = get_backend_class(scheme)
|
||||
|
||||
return backend_class.delete(parsed_uri, **kwargs)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2010 OpenStack, LLC
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""The s3 backend adapter"""
|
||||
|
||||
import glance.store
|
||||
|
||||
|
||||
class S3Backend(glance.store.Backend):
|
||||
"""An implementation of the s3 adapter."""
|
||||
|
||||
EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
|
||||
|
||||
@classmethod
|
||||
def get(cls, parsed_uri, expected_size, conn_class=None):
|
||||
"""
|
||||
Takes a parsed_uri in the format of:
|
||||
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
|
||||
to s3 and downloads the file. Returns the generator resp_body provided
|
||||
by get_object.
|
||||
"""
|
||||
|
||||
if conn_class:
|
||||
pass
|
||||
else:
|
||||
import boto.s3.connection
|
||||
conn_class = boto.s3.connection.S3Connection
|
||||
|
||||
(access_key, secret_key, host, bucket, obj) = \
|
||||
cls._parse_s3_tokens(parsed_uri)
|
||||
|
||||
# Close the connection when we're through.
|
||||
with conn_class(access_key, secret_key, host=host) as s3_conn:
|
||||
bucket = cls._get_bucket(s3_conn, bucket)
|
||||
|
||||
# Close the key when we're through.
|
||||
with cls._get_key(bucket, obj) as key:
|
||||
if not key.size == expected_size:
|
||||
raise glance.store.BackendException(
|
||||
"Expected %s bytes, got %s" %
|
||||
(expected_size, key.size))
|
||||
|
||||
key.BufferSize = cls.CHUNKSIZE
|
||||
for chunk in key:
|
||||
yield chunk
|
||||
|
||||
@classmethod
|
||||
def delete(cls, parsed_uri, conn_class=None):
|
||||
"""
|
||||
Takes a parsed_uri in the format of:
|
||||
s3://access_key:secret_key@s3.amazonaws.com/bucket/file.gz.0, connects
|
||||
to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete()
|
||||
returns.
|
||||
"""
|
||||
|
||||
if conn_class:
|
||||
pass
|
||||
else:
|
||||
conn_class = boto.s3.connection.S3Connection
|
||||
|
||||
(access_key, secret_key, host, bucket, obj) = \
|
||||
cls._parse_s3_tokens(parsed_uri)
|
||||
|
||||
# Close the connection when we're through.
|
||||
with conn_class(access_key, secret_key, host=host) as s3_conn:
|
||||
bucket = cls._get_bucket(s3_conn, bucket)
|
||||
|
||||
# Close the key when we're through.
|
||||
with cls._get_key(bucket, obj) as key:
|
||||
return key.delete()
|
||||
|
||||
@classmethod
|
||||
def _get_bucket(cls, conn, bucket_id):
|
||||
"""Get a bucket from an s3 connection"""
|
||||
|
||||
bucket = conn.get_bucket(bucket_id)
|
||||
if not bucket:
|
||||
raise glance.store.BackendException("Could not find bucket: %s" %
|
||||
bucket_id)
|
||||
|
||||
return bucket
|
||||
|
||||
@classmethod
|
||||
def _get_key(cls, bucket, obj):
|
||||
"""Get a key from a bucket"""
|
||||
|
||||
key = bucket.get_key(obj)
|
||||
if not key:
|
||||
raise glance.store.BackendException("Could not get key: %s" % key)
|
||||
return key
|
||||
|
||||
@classmethod
|
||||
def _parse_s3_tokens(cls, parsed_uri):
|
||||
"""Parse tokens from the parsed_uri"""
|
||||
return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)
|
||||
|
|
@ -15,6 +15,7 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
import glance.store
|
||||
|
||||
|
||||
|
|
@ -114,21 +115,7 @@ class SwiftBackend(glance.store.Backend):
|
|||
|
||||
|
||||
def get_connection_class(conn_class):
|
||||
if conn_class:
|
||||
pass # Use the provided conn_class
|
||||
else:
|
||||
# NOTE(sirp): A standard import statement won't work here because
|
||||
# this file ('swift.py') is shadowing the swift module, and since
|
||||
# the import statement searches locally before globally, we'd end
|
||||
# up importing ourselves.
|
||||
#
|
||||
# NOTE(jaypipes): This can be resolved by putting this code in
|
||||
# /glance/store/swift/__init__.py
|
||||
#
|
||||
# see http://docs.python.org/library/functions.html#__import__
|
||||
PERFORM_ABSOLUTE_IMPORTS = 0
|
||||
swift = __import__('swift.common.client', globals(), locals(), [],
|
||||
PERFORM_ABSOLUTE_IMPORTS)
|
||||
|
||||
if not conn_class:
|
||||
import swift.common.client
|
||||
conn_class = swift.common.client.Connection
|
||||
return conn_class
|
||||
|
|
|
|||
|
|
@ -105,6 +105,44 @@ def stub_out_filesystem_backend():
|
|||
f.close()
|
||||
|
||||
|
||||
def stub_out_s3_backend(stubs):
|
||||
""" Stubs out the S3 Backend with fake data and calls.
|
||||
|
||||
The stubbed swift backend provides back an iterator over
|
||||
the data ""
|
||||
|
||||
:param stubs: Set of stubout stubs
|
||||
|
||||
"""
|
||||
|
||||
class FakeSwiftAuth(object):
|
||||
pass
|
||||
class FakeS3Connection(object):
|
||||
pass
|
||||
|
||||
class FakeS3Backend(object):
|
||||
CHUNK_SIZE = 2
|
||||
DATA = 'I am a teapot, short and stout\n'
|
||||
|
||||
@classmethod
|
||||
def get(cls, parsed_uri, expected_size, conn_class=None):
|
||||
S3Backend = glance.store.s3.S3Backend
|
||||
|
||||
# raise BackendException if URI is bad.
|
||||
(user, key, authurl, container, obj) = \
|
||||
S3Backend._parse_s3_tokens(parsed_uri)
|
||||
|
||||
def chunk_it():
|
||||
for i in xrange(0, len(cls.DATA), cls.CHUNK_SIZE):
|
||||
yield cls.DATA[i:i+cls.CHUNK_SIZE]
|
||||
|
||||
return chunk_it()
|
||||
|
||||
fake_swift_backend = FakeS3Backend()
|
||||
stubs.Set(glance.store.s3.S3Backend, 'get',
|
||||
fake_swift_backend.get)
|
||||
|
||||
|
||||
def stub_out_swift_backend(stubs):
|
||||
"""Stubs out the Swift Glance backend with fake data
|
||||
and calls.
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import stubout
|
|||
import unittest
|
||||
import urlparse
|
||||
|
||||
from glance.store.s3 import S3Backend
|
||||
from glance.store.swift import SwiftBackend
|
||||
from glance.store import Backend, BackendException, get_from_backend
|
||||
from tests import stubs
|
||||
|
|
@ -87,6 +88,25 @@ class TestHTTPBackend(TestBackend):
|
|||
self.assertEqual(chunks, expected_returns)
|
||||
|
||||
|
||||
class TestS3Backend(TestBackend):
|
||||
def setUp(self):
|
||||
super(TestS3Backend, self).setUp()
|
||||
stubs.stub_out_s3_backend(self.stubs)
|
||||
|
||||
def test_get(self):
|
||||
s3_uri = "s3://user:password@localhost/bucket1/file.tar.gz"
|
||||
|
||||
expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s',
|
||||
'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n']
|
||||
fetcher = get_from_backend(s3_uri,
|
||||
expected_size=8,
|
||||
conn_class=S3Backend)
|
||||
|
||||
chunks = [c for c in fetcher]
|
||||
self.assertEqual(chunks, expected_returns)
|
||||
|
||||
|
||||
|
||||
class TestSwiftBackend(TestBackend):
|
||||
|
||||
def setUp(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue