Merge "Split out general implementation for containers"
This commit is contained in:
Zuul
Gerrit Code Review
commit
f1ee4f16a3
|
|
@ -36,8 +36,8 @@ paste.filter_factory =
|
|||
storlet_handler = storlets.swift_middleware.storlet_handler:filter_factory
|
||||
|
||||
storlets.gateways =
|
||||
stub = storlets.gateway.gateways.stub:StorletGatewayStub
|
||||
docker = storlets.gateway.gateways.docker:StorletGatewayDocker
|
||||
stub = storlets.gateway.gateways.stub:StubStorletGateway
|
||||
docker = storlets.gateway.gateways.docker:DockerStorletGateway
|
||||
|
||||
console_scripts =
|
||||
sbus = storlets.sbus.cli:main
|
||||
|
|
|
|||
|
|
@ -0,0 +1,372 @@
|
|||
# Copyright IBM Corp. 2015, 2015 All Rights Reserved
|
||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from storlets.gateway.common.stob import StorletRequest
|
||||
from storlets.gateway.gateways.base import StorletGatewayBase
|
||||
from storlets.gateway.gateways.container.runtime import RunTimePaths, \
|
||||
StorletInvocationProtocol
|
||||
|
||||
|
||||
class ContainerStorletRequest(StorletRequest):
|
||||
"""
|
||||
The ContainerStorletRequest class represents a request to be processed by
|
||||
the storlet the request is derived from the Swift request and
|
||||
essentially consists of:
|
||||
1. A data stream to be processed
|
||||
2. Metadata identifying the stream
|
||||
"""
|
||||
|
||||
# TODO(takashi): Some of following parameters should be defined common
|
||||
# parameters for StorletRequest
|
||||
required_options = ['storlet_main', 'storlet_language', 'file_manager']
|
||||
|
||||
def __init__(self, storlet_id, params, user_metadata, data_iter=None,
|
||||
data_fd=None, options=None):
|
||||
"""
|
||||
:param storlet_id: storlet id
|
||||
:param params: execution parameters
|
||||
:param user_metadata: user metadata
|
||||
:param data_iter: an iterator to read data
|
||||
:param data_fd: a file descriptor to read data
|
||||
:param options: a dictionaly which stores gateway specific options.
|
||||
:raises ValueError: when some of the required options (storlet_main
|
||||
and file_manager) are missing
|
||||
"""
|
||||
super(ContainerStorletRequest, self).__init__(
|
||||
storlet_id, params, user_metadata, data_iter, data_fd,
|
||||
options=options)
|
||||
|
||||
self.generate_log = self.options.get('generate_log', False)
|
||||
|
||||
self.storlet_main = self.options['storlet_main']
|
||||
self.storlet_language = self.options['storlet_language']
|
||||
self.storlet_language_version = \
|
||||
self.options.get('storlet_language_version')
|
||||
|
||||
if self.options.get('storlet_dependency'):
|
||||
self.dependencies = [
|
||||
x.strip() for x
|
||||
in self.options['storlet_dependency'].split(',')
|
||||
if x.strip()]
|
||||
else:
|
||||
self.dependencies = []
|
||||
|
||||
self.file_manager = self.options['file_manager']
|
||||
|
||||
self.start = self.options.get('range_start')
|
||||
self.end = self.options.get('range_end')
|
||||
|
||||
@property
|
||||
def has_range(self):
|
||||
"""
|
||||
Whether the input range is given
|
||||
"""
|
||||
return self.start is not None and self.end is not None
|
||||
|
||||
|
||||
class StorletGatewayContainer(StorletGatewayBase):
|
||||
|
||||
request_class = ContainerStorletRequest
|
||||
sandbox = None
|
||||
|
||||
def __init__(self, conf, logger, scope):
|
||||
"""
|
||||
:param conf: a dict for gateway conf
|
||||
:param logger: a logger instance
|
||||
:param scope: scope name to identify the container
|
||||
"""
|
||||
super(StorletGatewayContainer, self).__init__(conf, logger, scope)
|
||||
self.storlet_timeout = float(self.conf.get('storlet_timeout', 40))
|
||||
self.paths = RunTimePaths(scope, conf)
|
||||
|
||||
@classmethod
|
||||
def validate_storlet_registration(cls, params, name):
|
||||
"""
|
||||
Validate required parameters for storlet file
|
||||
|
||||
:param params: parameters related to the storlet file
|
||||
:param name: name of the storlet file
|
||||
:raises ValueError: if some of the required parameters are missing,
|
||||
or some of the parameters are invalid
|
||||
"""
|
||||
mandatory = ['Language', 'Interface-Version', 'Object-Metadata',
|
||||
'Main']
|
||||
cls._check_mandatory_params(params, mandatory)
|
||||
|
||||
if params['Language'].lower() == 'java':
|
||||
if '-' not in name or '.' not in name:
|
||||
raise ValueError('Storlet name is incorrect')
|
||||
elif params['Language'].lower() == 'python':
|
||||
try:
|
||||
version = float(params.get('Language-Version', 3))
|
||||
except ValueError:
|
||||
raise ValueError('Language-Version is invalid')
|
||||
|
||||
if int(version) != 3:
|
||||
# TODO(kota_): more strict version check should be nice.
|
||||
raise ValueError('Not supported version specified')
|
||||
|
||||
if name.endswith('.py'):
|
||||
cls_name = params['Main']
|
||||
if not cls_name.startswith(name[:-3] + '.'):
|
||||
raise ValueError('Main class should be included in '
|
||||
'storlet file')
|
||||
|
||||
if len(cls_name.split('.')) != 2:
|
||||
raise ValueError('Submodule is currently not supported')
|
||||
# TODO(takashi): Add support for sdist tar.gz
|
||||
else:
|
||||
raise ValueError('Storlet name is incorrect')
|
||||
else:
|
||||
raise ValueError('Unsupported Language')
|
||||
|
||||
dep = params.get('Dependency')
|
||||
if dep:
|
||||
deps = dep.split(',')
|
||||
if name in deps:
|
||||
raise ValueError('Using the same name for storlet and '
|
||||
'dependency is not allowed')
|
||||
if len(deps) != len(set(deps)):
|
||||
raise ValueError('Duplicated name in dependencies')
|
||||
|
||||
@classmethod
|
||||
def validate_dependency_registration(cls, params, name):
|
||||
"""
|
||||
Validate required parameters for dependency file
|
||||
|
||||
:param params: parameters related to the dependency file
|
||||
:param name: name of the dependency file
|
||||
:raises ValueError: if some of the required parameters are missing,
|
||||
or some of the parameters are invalid
|
||||
"""
|
||||
mandatory = ['Dependency-Version']
|
||||
cls._check_mandatory_params(params, mandatory)
|
||||
|
||||
perm = params.get('Dependency-Permissions')
|
||||
if perm is not None:
|
||||
try:
|
||||
perm_int = int(perm, 8)
|
||||
except ValueError:
|
||||
raise ValueError('Dependency permission is incorrect')
|
||||
if (perm_int & int('600', 8)) != int('600', 8):
|
||||
raise ValueError('The owner should have rw permission')
|
||||
|
||||
@classmethod
|
||||
def _check_mandatory_params(cls, params, mandatory):
|
||||
"""
|
||||
Ensure that we have all mandatory parameters in the given parameters
|
||||
|
||||
:param params: file parameters
|
||||
:param mandatory: required parameters
|
||||
:raises ValueError: if some of the required parameters are missing
|
||||
"""
|
||||
for md in mandatory:
|
||||
if md not in params:
|
||||
raise ValueError('Mandatory parameter is missing'
|
||||
': {0}'.format(md))
|
||||
|
||||
def invocation_flow(self, sreq, extra_sources=None):
|
||||
"""
|
||||
Invoke the backend protocol with gateway
|
||||
|
||||
:param sreq: StorletRequest instance
|
||||
:param extra_sources (WIP): A list of StorletRequest instance to gather
|
||||
as extra resoureces to feed to storlet
|
||||
container as data source
|
||||
:return: StorletResponse instance
|
||||
"""
|
||||
run_time_sbox = self.sandbox(self.scope, self.conf, self.logger)
|
||||
container_updated = self.update_container_from_cache(sreq)
|
||||
run_time_sbox.activate_storlet_daemon(sreq, container_updated)
|
||||
self._add_system_params(sreq)
|
||||
|
||||
slog_path = self.paths.get_host_slog_path(sreq.storlet_main)
|
||||
storlet_pipe_path = \
|
||||
self.paths.get_host_storlet_pipe(sreq.storlet_main)
|
||||
|
||||
sprotocol = StorletInvocationProtocol(sreq,
|
||||
storlet_pipe_path,
|
||||
slog_path,
|
||||
self.storlet_timeout,
|
||||
self.logger,
|
||||
extra_sources=extra_sources)
|
||||
|
||||
sresp = sprotocol.communicate()
|
||||
|
||||
self._upload_storlet_logs(slog_path, sreq)
|
||||
|
||||
return sresp
|
||||
|
||||
def _add_system_params(self, sreq):
|
||||
"""
|
||||
Adds Storlet engine specific parameters to the invocation
|
||||
|
||||
currently, this consists only of the execution path of the
|
||||
Storlet within the container.
|
||||
|
||||
:params params: Request parameters
|
||||
"""
|
||||
sreq.params['storlet_execution_path'] = self. \
|
||||
paths.get_sbox_storlet_dir(sreq.storlet_main)
|
||||
|
||||
def _upload_storlet_logs(self, slog_path, sreq):
|
||||
"""
|
||||
Upload storlet execution log as a swift object
|
||||
|
||||
:param slog_path: target path
|
||||
:params sreq: ContainerStorletRequest instance
|
||||
"""
|
||||
if sreq.generate_log:
|
||||
with open(slog_path, 'rb') as logfile:
|
||||
storlet_name = sreq.storlet_id.split('-')[0]
|
||||
log_obj_name = '%s.log' % storlet_name
|
||||
sreq.file_manager.put_log(log_obj_name, logfile)
|
||||
|
||||
def bring_from_cache(self, obj_name, sreq, is_storlet):
|
||||
"""
|
||||
Auxiliary function that:
|
||||
|
||||
(1) Brings from Swift obj_name, either this is in a
|
||||
storlet or a storlet dependency.
|
||||
(2) Copies from local cache into the conrainer
|
||||
If this is a Storlet then also validates that the cache is updated
|
||||
with most recent copy of the Storlet compared to the copy residing in
|
||||
Swift.
|
||||
|
||||
:params obj_name: name of the object
|
||||
:params sreq: ContainerStorletRequest instance
|
||||
:params is_storlet: True if the object is a storlet object
|
||||
False if the object is a dependency object
|
||||
:returns: Whether the container was updated with obj_name
|
||||
"""
|
||||
# Determine the cache we are to work with
|
||||
# e.g. dependency or storlet
|
||||
if is_storlet:
|
||||
cache_dir = self.paths.host_storlet_cache_dir
|
||||
get_func = sreq.file_manager.get_storlet
|
||||
else:
|
||||
cache_dir = self.paths.host_dependency_cache_dir
|
||||
get_func = sreq.file_manager.get_dependency
|
||||
|
||||
if not os.path.exists(cache_dir):
|
||||
os.makedirs(cache_dir, 0o700)
|
||||
|
||||
# cache_target_path is the actual object we need to deal with
|
||||
# e.g. a concrete storlet or dependency we need to bring/update
|
||||
cache_target_path = os.path.join(cache_dir, obj_name)
|
||||
|
||||
# Determine if we need to update the cache for cache_target_path
|
||||
# We default for no
|
||||
update_cache = False
|
||||
|
||||
# If it does not exist in cache, we obviously need to bring
|
||||
if not os.path.isfile(cache_target_path):
|
||||
update_cache = True
|
||||
elif is_storlet:
|
||||
# The cache_target_path exists, we test if it is up-to-date
|
||||
# with the metadata we got.
|
||||
# We mention that this is currently applicable for storlets
|
||||
# only, and not for dependencies.
|
||||
# This will change when we will head dependencies as well
|
||||
fstat = os.stat(cache_target_path)
|
||||
storlet_or_size = int(
|
||||
sreq.options['storlet_content_length'].rstrip("L"))
|
||||
storlet_or_time = float(sreq.options['storlet_x_timestamp'])
|
||||
b_storlet_size_changed = fstat.st_size != storlet_or_size
|
||||
b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time
|
||||
if b_storlet_size_changed or b_storlet_file_updated:
|
||||
update_cache = True
|
||||
|
||||
if update_cache:
|
||||
# If the cache needs to be updated, then get on with it
|
||||
# bring the object from storage
|
||||
data_iter, perm = get_func(obj_name)
|
||||
|
||||
if perm:
|
||||
perm = int(perm, 8) & 0o700
|
||||
else:
|
||||
perm = 0o600
|
||||
|
||||
# TODO(takashi): Do not directly write to target path
|
||||
with open(cache_target_path, 'wb') as fn:
|
||||
os.chmod(cache_target_path, perm)
|
||||
for data in data_iter:
|
||||
fn.write(data)
|
||||
|
||||
# The node's local cache is now updated.
|
||||
# We now verify if we need to update the
|
||||
# container itself.
|
||||
# The container needs to be updated if:
|
||||
# 1. The container does not hold a copy of the object
|
||||
# 2. The container holds an older version of the object
|
||||
update_container = False
|
||||
container_storlet_path = \
|
||||
self.paths.get_host_storlet_dir(sreq.storlet_main)
|
||||
container_target_path = os.path.join(container_storlet_path, obj_name)
|
||||
|
||||
if not os.path.exists(container_storlet_path):
|
||||
os.makedirs(container_storlet_path, 0o700)
|
||||
update_container = True
|
||||
elif not os.path.isfile(container_target_path):
|
||||
update_container = True
|
||||
else:
|
||||
fstat_cached = os.stat(cache_target_path)
|
||||
fstat_container = os.stat(container_target_path)
|
||||
if fstat_cached.st_size != fstat_container.st_size:
|
||||
update_container = True
|
||||
if fstat_cached.st_mtime < fstat_container.st_mtime:
|
||||
update_container = True
|
||||
|
||||
if update_container:
|
||||
# need to copy from cache to container
|
||||
# copy2 also copies the permissions
|
||||
shutil.copy2(cache_target_path, container_target_path)
|
||||
|
||||
return update_container
|
||||
|
||||
def update_container_from_cache(self, sreq):
|
||||
"""
|
||||
Iterates over the storlet name and its dependencies appearing
|
||||
|
||||
in the invocation data and make sure they are brought to the
|
||||
local cache, and from there to the container.
|
||||
Uses the bring_from_cache auxiliary function.
|
||||
|
||||
:params sreq: ContainerStorletRequest instance
|
||||
:returns: True if the container was updated
|
||||
"""
|
||||
# where at the host side, reside the storlet containers
|
||||
storlet_path = self.paths.host_storlet_base_dir
|
||||
if not os.path.exists(storlet_path):
|
||||
os.makedirs(storlet_path, 0o755)
|
||||
|
||||
# Iterate over storlet and dependencies, and make sure
|
||||
# they are updated within the container.
|
||||
# return True if any of them wea actually
|
||||
# updated within the container
|
||||
container_updated = False
|
||||
|
||||
updated = self.bring_from_cache(sreq.storlet_id, sreq, True)
|
||||
container_updated = container_updated or updated
|
||||
|
||||
for dep in sreq.dependencies:
|
||||
updated = self.bring_from_cache(dep, sreq, False)
|
||||
container_updated = container_updated or updated
|
||||
|
||||
return container_updated
|
||||
|
|
@ -0,0 +1,744 @@
|
|||
# Copyright (c) 2015, 2016 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import errno
|
||||
import os
|
||||
import select
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
import eventlet
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
from storlets.sbus.client import SBusClient
|
||||
from storlets.sbus.client.exceptions import SBusClientException
|
||||
from storlets.sbus.datagram import SBusFileDescriptor
|
||||
from storlets.sbus import file_description as sbus_fd
|
||||
from storlets.gateway.common.exceptions import StorletRuntimeException, \
|
||||
StorletTimeout
|
||||
from storlets.gateway.common.logger import StorletLogger
|
||||
from storlets.gateway.common.stob import StorletResponse
|
||||
|
||||
MAX_METADATA_SIZE = 4096
|
||||
|
||||
|
||||
eventlet.monkey_patch()
|
||||
|
||||
|
||||
class RunTimePaths(object):
|
||||
"""
|
||||
The Storlet Engine need to be access stuff located in many paths:
|
||||
|
||||
1. The various communication channels represented as pipes in the
|
||||
filesystem
|
||||
2. Directories where to place Storlets
|
||||
3. Directories where to place logs
|
||||
|
||||
Communication channels
|
||||
----------------------
|
||||
The RunTimeSandbox communicates with the Sandbox via two types of pipes
|
||||
1. factory pipe - defined per scope, used for communication with the
|
||||
sandbox
|
||||
for e.g. start/stop a storlet daemon
|
||||
2. Storlet pipe - defined per scope and Storlet, used for communication
|
||||
with a storlet daemon, e.g. to call the invoke API
|
||||
|
||||
Each pipe type has two paths:
|
||||
1. A path that is inside the sandbox
|
||||
2. A path that is outside of the sandbox or at the host side. As such
|
||||
this path is prefixed by 'host_'
|
||||
|
||||
Thus, we have the following 4 paths of interest:
|
||||
1. sandbox_factory_pipe_path
|
||||
2. host_factory_pipe_path
|
||||
3. sandbox_storlet_pipe_path
|
||||
4. host_storlet_pipe_path
|
||||
|
||||
Our implementation uses the following path structure for the various pipes:
|
||||
In the host, all pipes belonging to a given scope are prefixed by
|
||||
<pipes_dir>/<scope>, where <pipes_dir> comes from the configuration
|
||||
Thus:
|
||||
host_factory_pipe_path is of the form <pipes_dir>/<scope>/factory_pipe
|
||||
host_storlet_pipe_path is of the form <pipes_dir>/<scope>/<storlet_id>
|
||||
|
||||
In The sandbox side
|
||||
sandbox_factory_pipe_path is of the form /mnt/channels/factory_pipe
|
||||
sandbox_storlet_pipe_path is of the form /mnt/channels/<storlet_id>
|
||||
|
||||
Storlets Locations
|
||||
------------------
|
||||
The Storlet binaries are accessible from the sandbox using a mounted
|
||||
directory.
|
||||
This directory is called the storlet directories.
|
||||
On the host side it is of the form <storlet_dir>/<scope>/<storlet_name>
|
||||
On the sandbox side it is of the form /home/swift/<storlet_name>
|
||||
<storlet_dir> comes from the configuration
|
||||
<storlet_name> is the prefix of the jar.
|
||||
|
||||
Logs
|
||||
----
|
||||
Logs are located in paths of the form:
|
||||
<log_dir>/<scope>/<storlet_name>.log
|
||||
"""
|
||||
|
||||
def __init__(self, scope, conf):
|
||||
"""
|
||||
Construct RunTimePaths instance
|
||||
|
||||
:param scope: scope name to be used as container name
|
||||
:param conf: gateway conf
|
||||
"""
|
||||
self.scope = scope
|
||||
self.factory_pipe_name = 'factory_pipe'
|
||||
self.sandbox_pipe_dir = '/mnt/channels'
|
||||
|
||||
self.sandbox_storlet_base_dir = '/home/swift'
|
||||
self.host_root_dir = conf.get('host_root', '/var/lib/storlets')
|
||||
self.host_pipe_root_dir = \
|
||||
conf.get('pipes_dir',
|
||||
os.path.join(self.host_root_dir, 'pipes', 'scopes'))
|
||||
self.host_storlet_root_dir = \
|
||||
conf.get('storlets_dir',
|
||||
os.path.join(self.host_root_dir, 'storlets', 'scopes'))
|
||||
self.host_log_root_dir = \
|
||||
conf.get('log_dir',
|
||||
os.path.join(self.host_root_dir, 'logs', 'scopes'))
|
||||
self.host_cache_root_dir = \
|
||||
conf.get('cache_dir',
|
||||
os.path.join(self.host_root_dir, 'cache', 'scopes'))
|
||||
self.host_restart_script_dir = \
|
||||
conf.get('script_dir',
|
||||
os.path.join(self.host_root_dir, 'scripts'))
|
||||
|
||||
self.host_storlet_native_lib_dir = '/usr/local/lib/storlets'
|
||||
self.sandbox_storlet_native_lib_dir = '/usr/local/lib/storlets'
|
||||
self.host_storlet_native_bin_dir = '/usr/local/libexec/storlets'
|
||||
self.sandbox_storlet_native_bin_dir = '/usr/local/libexec/storlets'
|
||||
|
||||
@property
|
||||
def host_pipe_dir(self):
|
||||
return os.path.join(self.host_pipe_root_dir, self.scope)
|
||||
|
||||
def create_host_pipe_dir(self):
|
||||
path = self.host_pipe_dir
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
# 0777 should be 0700 when we get user namespaces in container
|
||||
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
||||
return path
|
||||
|
||||
@property
|
||||
def host_factory_pipe(self):
|
||||
return os.path.join(self.host_pipe_dir, self.factory_pipe_name)
|
||||
|
||||
@property
|
||||
def sandbox_factory_pipe(self):
|
||||
return os.path.join(self.sandbox_pipe_dir, self.factory_pipe_name)
|
||||
|
||||
def get_host_storlet_pipe(self, storlet_id):
|
||||
return os.path.join(self.host_pipe_dir, storlet_id)
|
||||
|
||||
def get_sbox_storlet_pipe(self, storlet_id):
|
||||
return os.path.join(self.sandbox_pipe_dir, storlet_id)
|
||||
|
||||
def get_sbox_storlet_dir(self, storlet_id):
|
||||
return os.path.join(self.sandbox_storlet_base_dir, storlet_id)
|
||||
|
||||
@property
|
||||
def host_storlet_base_dir(self):
|
||||
return os.path.join(self.host_storlet_root_dir, self.scope)
|
||||
|
||||
def get_host_storlet_dir(self, storlet_id):
|
||||
return os.path.join(self.host_storlet_base_dir, storlet_id)
|
||||
|
||||
def get_host_slog_path(self, storlet_id):
|
||||
return os.path.join(
|
||||
self.host_log_root_dir, self.scope, storlet_id,
|
||||
'storlet_invoke.log')
|
||||
|
||||
@property
|
||||
def host_storlet_cache_dir(self):
|
||||
return os.path.join(self.host_cache_root_dir, self.scope, 'storlet')
|
||||
|
||||
@property
|
||||
def host_dependency_cache_dir(self):
|
||||
return os.path.join(self.host_cache_root_dir, self.scope, 'dependency')
|
||||
|
||||
|
||||
class RunTimeSandbox(object, metaclass=abc.ABCMeta):
|
||||
"""
|
||||
The RunTimeSandbox represents a reusable per scope sandbox.
|
||||
|
||||
The sandbox is reusable in the sense that it can run several storlet
|
||||
daemons.
|
||||
|
||||
The following methods are supported:
|
||||
ping - pings the sandbox for liveness
|
||||
wait - wait for the sandbox to be ready for processing commands
|
||||
restart - restart the sandbox
|
||||
start_storlet_daemon - start a daemon for a given storlet
|
||||
stop_storlet_daemon - stop a daemon of a given storlet
|
||||
get_storlet_daemon_status - test if a given storlet daemon is running
|
||||
"""
|
||||
|
||||
def __init__(self, scope, conf, logger):
|
||||
"""
|
||||
:param scope: scope name to be used as container name
|
||||
:param conf: gateway conf
|
||||
:param logger: logger instance
|
||||
"""
|
||||
self.paths = RunTimePaths(scope, conf)
|
||||
self.scope = scope
|
||||
|
||||
self.sandbox_ping_interval = \
|
||||
float(conf.get('sandbox_ping_interval', 0.5))
|
||||
self.sandbox_stop_timeout = \
|
||||
float(conf.get('stop_linux_container_timeout', 1))
|
||||
self.sandbox_wait_timeout = \
|
||||
float(conf.get('restart_linux_container_timeout', 10))
|
||||
|
||||
self.container_image_namespace = \
|
||||
conf.get('docker_repo', conf.get('container_image_namespace'))
|
||||
self.container_image_name_prefix = 'tenant'
|
||||
|
||||
# TODO(add line in conf)
|
||||
self.storlet_daemon_thread_pool_size = \
|
||||
int(conf.get('storlet_daemon_thread_pool_size', 5))
|
||||
self.storlet_daemon_factory_debug_level = \
|
||||
conf.get('storlet_daemon_factory_debug_level', 'DEBUG')
|
||||
self.storlet_daemon_debug_level = \
|
||||
conf.get('storlet_daemon_debug_level', 'DEBUG')
|
||||
|
||||
# TODO(change logger's route if possible)
|
||||
self.logger = logger
|
||||
|
||||
self.default_container_image_name = conf.get(
|
||||
'default_docker_image_name',
|
||||
conf.get('default_container_image_name', 'storlet_engine_image')
|
||||
)
|
||||
|
||||
self.max_containers_per_node = \
|
||||
int(conf.get('max_containers_per_node', 0))
|
||||
|
||||
self.container_cpu_period = int(conf.get('container_cpu_period', 0))
|
||||
self.container_cpu_quota = int(conf.get('container_cpu_quota', 0))
|
||||
self.container_mem_limit = conf.get('container_mem_limit', 0)
|
||||
# NOTE(tkajinam): memory limit can be a string with unit like 1024m
|
||||
try:
|
||||
self.container_mem_limit = int(self.container_mem_limit)
|
||||
except TypeError:
|
||||
pass
|
||||
self.container_cpuset_cpus = conf.get('container_cpuset_cpus')
|
||||
self.container_cpuset_mems = conf.get('container_cpuset_mems')
|
||||
self.container_pids_limit = int(conf.get('container_pids_limit', 0))
|
||||
|
||||
def ping(self):
|
||||
"""
|
||||
Ping to daemon factory process inside container
|
||||
|
||||
:returns: True when the daemon factory is responsive
|
||||
False when the daemon factory is not responsive or it fails
|
||||
to send command to the process
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.ping()
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to ping to daemon factory: %s' %
|
||||
resp.message)
|
||||
return resp.status
|
||||
except SBusClientException:
|
||||
return False
|
||||
|
||||
def wait(self):
|
||||
"""
|
||||
Wait while scope's sandbox is starting
|
||||
|
||||
:raises StorletTimeout: the sandbox has not started in
|
||||
sandbox_wait_timeout
|
||||
"""
|
||||
with StorletTimeout(self.sandbox_wait_timeout):
|
||||
while not self.ping():
|
||||
time.sleep(self.sandbox_ping_interval)
|
||||
|
||||
@abc.abstractmethod
|
||||
def _restart(self, container_image_name):
|
||||
"""
|
||||
Restarts the scope's sandbox using the specified container image
|
||||
|
||||
:param container_image_name: name of the container image to start
|
||||
:raises StorletRuntimeException: when failed to restart the container
|
||||
"""
|
||||
pass
|
||||
|
||||
def restart(self):
|
||||
"""
|
||||
Restarts the scope's sandbox
|
||||
|
||||
"""
|
||||
self.paths.create_host_pipe_dir()
|
||||
|
||||
container_image_name = self.scope
|
||||
try:
|
||||
self._restart(container_image_name)
|
||||
self.wait()
|
||||
except StorletTimeout:
|
||||
raise
|
||||
except StorletRuntimeException:
|
||||
# We were unable to start a container from the tenant image.
|
||||
# Let us try to start a container from default image.
|
||||
self.logger.exception("Failed to start a container from "
|
||||
"tenant image %s" % container_image_name)
|
||||
|
||||
self.logger.info("Trying to start a container from default "
|
||||
"image: %s" % self.default_container_image_name)
|
||||
self._restart(self.default_container_image_name)
|
||||
self.wait()
|
||||
|
||||
def start_storlet_daemon(
|
||||
self, spath, storlet_id, language, language_version=None):
|
||||
"""
|
||||
Start SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.start_daemon(
|
||||
language.lower(), spath, storlet_id,
|
||||
self.paths.get_sbox_storlet_pipe(storlet_id),
|
||||
self.storlet_daemon_debug_level,
|
||||
self.storlet_daemon_thread_pool_size,
|
||||
language_version)
|
||||
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to start storlet daemon: %s' %
|
||||
resp.message)
|
||||
raise StorletRuntimeException('Daemon start failed')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Daemon start failed')
|
||||
|
||||
def stop_storlet_daemon(self, storlet_id):
|
||||
"""
|
||||
Stop SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.stop_daemon(storlet_id)
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to stop storlet daemon: %s' %
|
||||
resp.message)
|
||||
raise StorletRuntimeException('Daemon stop failed')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Daemon stop failed')
|
||||
|
||||
def get_storlet_daemon_status(self, storlet_id):
|
||||
"""
|
||||
Get the status of SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.daemon_status(storlet_id)
|
||||
if resp.status:
|
||||
return 1
|
||||
else:
|
||||
self.logger.error('Failed to get status about storlet '
|
||||
'daemon: %s' % resp.message)
|
||||
return 0
|
||||
except SBusClientException:
|
||||
return -1
|
||||
|
||||
def _get_storlet_classpath(self, storlet_main, storlet_id, dependencies):
|
||||
"""
|
||||
Get classpath required to run storlet application
|
||||
|
||||
:param storlet_main: Main class name of the storlet
|
||||
:param storlet_id: Name of the storlet file
|
||||
:param dependencies: A list of dependency file
|
||||
:returns: classpath string
|
||||
"""
|
||||
class_path = os.path.join(
|
||||
self.paths.get_sbox_storlet_dir(storlet_main), storlet_id)
|
||||
|
||||
dep_path_list = \
|
||||
[os.path.join(self.paths.get_sbox_storlet_dir(storlet_main), dep)
|
||||
for dep in dependencies]
|
||||
|
||||
return class_path + ':' + ':'.join(dep_path_list)
|
||||
|
||||
def activate_storlet_daemon(self, sreq, cache_updated=True):
|
||||
storlet_daemon_status = \
|
||||
self.get_storlet_daemon_status(sreq.storlet_main)
|
||||
if (storlet_daemon_status == -1):
|
||||
# We failed to send a command to the factory.
|
||||
# Best we can do is execute the container.
|
||||
self.logger.debug('Failed to check the storlet daemon status. '
|
||||
'Restart its container')
|
||||
self.restart()
|
||||
storlet_daemon_status = 0
|
||||
|
||||
if (cache_updated is True and storlet_daemon_status == 1):
|
||||
# The cache was updated while the daemon is running we need to
|
||||
# stop it.
|
||||
self.logger.debug('The cache was updated, and the storlet daemon '
|
||||
'is running. Stopping daemon')
|
||||
|
||||
try:
|
||||
self.stop_storlet_daemon(sreq.storlet_main)
|
||||
except StorletRuntimeException:
|
||||
self.logger.warning('Failed to stop the storlet daemon. '
|
||||
'Restart its container')
|
||||
self.restart()
|
||||
else:
|
||||
self.logger.debug('Deamon stopped')
|
||||
storlet_daemon_status = 0
|
||||
|
||||
if (storlet_daemon_status == 0):
|
||||
self.logger.debug('Going to start the storlet daemon!')
|
||||
|
||||
# TODO(takashi): This is not needed for python application
|
||||
classpath = self._get_storlet_classpath(
|
||||
sreq.storlet_main, sreq.storlet_id, sreq.dependencies)
|
||||
|
||||
self.start_storlet_daemon(
|
||||
classpath, sreq.storlet_main, sreq.storlet_language,
|
||||
sreq.storlet_language_version)
|
||||
self.logger.debug('Daemon started')
|
||||
|
||||
|
||||
class StorletInvocationProtocol(object):
|
||||
"""
|
||||
StorletInvocationProtocol class
|
||||
|
||||
This class serves communictaion with a container to run an
|
||||
application
|
||||
|
||||
:param srequest: StorletRequest instance
|
||||
:param storlet_pipe_path: path string to pipe
|
||||
:param storlet_logger_path: path string to log file
|
||||
:param timeout: integer of timeout for waiting the resp from container
|
||||
:param logger: logger instance
|
||||
:param extra_sources (WIP): a list of StorletRequest instances
|
||||
which keep data_iter for adding extra source
|
||||
as data stream
|
||||
"""
|
||||
def __init__(self, srequest, storlet_pipe_path, storlet_logger_path,
|
||||
timeout, logger, extra_sources=None):
|
||||
self.srequest = srequest
|
||||
self.storlet_pipe_path = storlet_pipe_path
|
||||
self.storlet_logger = StorletLogger(storlet_logger_path)
|
||||
self.logger = logger
|
||||
self.timeout = timeout
|
||||
|
||||
# local side file descriptors
|
||||
self.data_read_fd = None
|
||||
self.data_write_fd = None
|
||||
self.metadata_read_fd = None
|
||||
self.metadata_write_fd = None
|
||||
self.task_id = None
|
||||
self._input_data_read_fd = None
|
||||
self._input_data_write_fd = None
|
||||
|
||||
self.extra_data_sources = []
|
||||
extra_sources = extra_sources or []
|
||||
for source in extra_sources:
|
||||
if source.has_fd:
|
||||
# TODO(kota_): it may be data_fd in the future.
|
||||
raise Exception(
|
||||
'extra_source no requires data_fd just data_iter')
|
||||
self.extra_data_sources.append(
|
||||
{'read_fd': None, 'write_fd': None,
|
||||
'user_metadata': source.user_metadata,
|
||||
'data_iter': source.data_iter})
|
||||
|
||||
@property
|
||||
def input_data_read_fd(self):
|
||||
"""
|
||||
File descriptor to read the input body content
|
||||
"""
|
||||
if self.srequest.has_fd:
|
||||
return self.srequest.data_fd
|
||||
else:
|
||||
return self._input_data_read_fd
|
||||
|
||||
@property
|
||||
def remote_fds(self):
|
||||
"""
|
||||
A list of sbus file descriptors passed to remote side
|
||||
"""
|
||||
storlets_metadata = {}
|
||||
if self.srequest.has_range:
|
||||
storlets_metadata.update(
|
||||
{'start': str(self.srequest.start),
|
||||
'end': str(self.srequest.end)})
|
||||
|
||||
fds = [SBusFileDescriptor(sbus_fd.SBUS_FD_INPUT_OBJECT,
|
||||
self.input_data_read_fd,
|
||||
storage_metadata=self.srequest.user_metadata,
|
||||
storlets_metadata=storlets_metadata),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT,
|
||||
self.data_write_fd),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
|
||||
self.metadata_write_fd),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_LOGGER,
|
||||
self.storlet_logger.getfd())]
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
fd = SBusFileDescriptor(
|
||||
sbus_fd.SBUS_FD_INPUT_OBJECT,
|
||||
source['read_fd'],
|
||||
storage_metadata=source['user_metadata'])
|
||||
fds.append(fd)
|
||||
|
||||
return fds
|
||||
|
||||
@contextmanager
|
||||
def _activate_invocation_descriptors(self):
|
||||
"""
|
||||
Contextmanager about file descriptors used in storlet invocation
|
||||
|
||||
NOTE: This context manager now only closes remote side fds,
|
||||
so you should close local side fds
|
||||
"""
|
||||
self._prepare_invocation_descriptors()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._close_remote_side_descriptors()
|
||||
|
||||
def _prepare_invocation_descriptors(self):
|
||||
"""
|
||||
Create all pipse used for Storlet execution
|
||||
"""
|
||||
if not self.srequest.has_fd:
|
||||
self._input_data_read_fd, self._input_data_write_fd = os.pipe()
|
||||
self.data_read_fd, self.data_write_fd = os.pipe()
|
||||
self.metadata_read_fd, self.metadata_write_fd = os.pipe()
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
source['read_fd'], source['write_fd'] = os.pipe()
|
||||
|
||||
def _safe_close(self, fds):
|
||||
"""
|
||||
Make sure that all of the file descriptors get closed
|
||||
|
||||
:param fds: a list of file descriptors
|
||||
"""
|
||||
for fd in fds:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError as err:
|
||||
if err.errno != errno.EBADF:
|
||||
raise
|
||||
# TODO(kota_): fd might be closed already, so if already
|
||||
# closed, OSError will be raised. we need more refactor to
|
||||
# keep clean the file descriptors.
|
||||
pass
|
||||
|
||||
def _close_remote_side_descriptors(self):
|
||||
"""
|
||||
Close all of the container side descriptors
|
||||
"""
|
||||
fds = [self.data_write_fd, self.metadata_write_fd]
|
||||
if not self.srequest.has_fd:
|
||||
fds.append(self.input_data_read_fd)
|
||||
fds.extend([source['read_fd'] for source in self.extra_data_sources])
|
||||
for fd in fds:
|
||||
os.close(fd)
|
||||
|
||||
def _close_local_side_descriptors(self):
|
||||
"""
|
||||
Close all of the host side descriptors
|
||||
"""
|
||||
fds = [self.data_read_fd, self.metadata_read_fd]
|
||||
fds.extend([source['write_fd'] for source in self.extra_data_sources])
|
||||
self._safe_close(fds)
|
||||
|
||||
def _cancel(self):
|
||||
"""
|
||||
Cancel on-going storlet execution
|
||||
"""
|
||||
client = SBusClient(self.storlet_pipe_path)
|
||||
try:
|
||||
resp = client.cancel(self.task_id)
|
||||
if not resp.status:
|
||||
raise StorletRuntimeException('Failed to cancel task')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Failed to cancel task')
|
||||
|
||||
def _invoke(self):
|
||||
"""
|
||||
Send an execution command to the remote daemon factory
|
||||
"""
|
||||
with self.storlet_logger.activate(),\
|
||||
self._activate_invocation_descriptors():
|
||||
self._send_execute_command()
|
||||
|
||||
def _send_execute_command(self):
|
||||
"""
|
||||
Send execute command to the remote daemon factory to invoke storlet
|
||||
execution
|
||||
"""
|
||||
client = SBusClient(self.storlet_pipe_path)
|
||||
try:
|
||||
resp = client.execute(self.srequest.params, self.remote_fds)
|
||||
if not resp.status:
|
||||
raise StorletRuntimeException("Failed to send execute command")
|
||||
|
||||
if not resp.task_id:
|
||||
raise StorletRuntimeException("Missing task id")
|
||||
else:
|
||||
self.task_id = resp.task_id
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException("Failed to send execute command")
|
||||
|
||||
def _wait_for_read_with_timeout(self, fd):
|
||||
"""
|
||||
Wait while the read file descriptor gets ready
|
||||
|
||||
:param fd: File descriptor to read
|
||||
:raises StorletTimeout: Exception raised when it times out to cancel
|
||||
the existing task
|
||||
:raises StorletRuntimeException: Exception raised when it fails to
|
||||
cancel the existing task
|
||||
"""
|
||||
try:
|
||||
with StorletTimeout(self.timeout):
|
||||
r, w, e = select.select([fd], [], [])
|
||||
except StorletTimeout:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
|
||||
# When there is a task already running, we should cancel it.
|
||||
if self.task_id:
|
||||
try:
|
||||
self._cancel()
|
||||
except StorletRuntimeException:
|
||||
self.logger.warning(
|
||||
'Task %s timed out, but failed to get canceled'
|
||||
% self.task_id)
|
||||
pass
|
||||
|
||||
if exc_value is None:
|
||||
exc_value = exc_traceback
|
||||
if exc_value.__traceback__ is not exc_traceback:
|
||||
raise exc_value.with_traceback(exc_traceback)
|
||||
raise exc_value
|
||||
|
||||
if fd not in r:
|
||||
raise StorletRuntimeException('Read fd is not ready')
|
||||
|
||||
def _read_metadata(self):
|
||||
"""
|
||||
Read metadata in the storlet execution result from fd
|
||||
|
||||
:returns: a dict of metadata
|
||||
"""
|
||||
self._wait_for_read_with_timeout(self.metadata_read_fd)
|
||||
flat_json = os.read(self.metadata_read_fd, MAX_METADATA_SIZE)
|
||||
os.close(self.metadata_read_fd)
|
||||
try:
|
||||
return json.loads(flat_json)
|
||||
except ValueError:
|
||||
self.logger.exception('Failed to load metadata from json')
|
||||
raise StorletRuntimeException('Got invalid format about metadata')
|
||||
|
||||
def _wait_for_write_with_timeout(self, fd):
|
||||
"""
|
||||
Wait while the write file descriptor gets ready
|
||||
|
||||
:param fd: File descriptor to write
|
||||
:raises StorletTimeout: Exception raised when it times out to cancel
|
||||
the existing task
|
||||
:raises StorletRuntimeException: Exception raised when it fails to
|
||||
cancel the existing task
|
||||
"""
|
||||
with StorletTimeout(self.timeout):
|
||||
r, w, e = select.select([], [fd], [])
|
||||
if fd not in w:
|
||||
raise StorletRuntimeException('Write fd is not ready')
|
||||
|
||||
def _close_input_data_descriptors(self):
|
||||
fds = [self._input_data_read_fd, self._input_data_write_fd]
|
||||
self._safe_close(fds)
|
||||
|
||||
def communicate(self):
|
||||
try:
|
||||
self._invoke()
|
||||
|
||||
if not self.srequest.has_fd:
|
||||
self._wait_for_write_with_timeout(self._input_data_write_fd)
|
||||
|
||||
# We do the writing in a different thread.
|
||||
# Otherwise, we can run into the following deadlock
|
||||
# 1. middleware writes to Storlet
|
||||
# 2. Storlet reads and starts to write metadata and then data
|
||||
# 3. middleware continues writing
|
||||
# 4. Storlet continues writing and gets stuck as middleware
|
||||
# is busy writing, but still not consuming the reader end
|
||||
# of the Storlet writer.
|
||||
eventlet.spawn_n(self._write_input_data,
|
||||
self._input_data_write_fd,
|
||||
self.srequest.data_iter)
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
# NOTE(kota_): not sure right now if using eventlet.spawn_n is
|
||||
# right way. GreenPool is better? I don't get
|
||||
# whole for the dead lock described in above.
|
||||
self._wait_for_write_with_timeout(source['write_fd'])
|
||||
eventlet.spawn_n(self._write_input_data,
|
||||
source['write_fd'],
|
||||
source['data_iter'])
|
||||
|
||||
out_md = self._read_metadata()
|
||||
self._wait_for_read_with_timeout(self.data_read_fd)
|
||||
|
||||
return StorletResponse(out_md, data_fd=self.data_read_fd,
|
||||
cancel=self._cancel)
|
||||
except Exception:
|
||||
self._close_local_side_descriptors()
|
||||
if not self.srequest.has_fd:
|
||||
self._close_input_data_descriptors()
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _open_writer(self, fd):
|
||||
with os.fdopen(fd, 'wb') as writer:
|
||||
yield writer
|
||||
|
||||
def _write_input_data(self, fd, data_iter):
|
||||
try:
|
||||
# double try/except block saving from unexpected errors
|
||||
try:
|
||||
with self._open_writer(fd) as writer:
|
||||
for chunk in data_iter:
|
||||
with StorletTimeout(self.timeout):
|
||||
writer.write(chunk)
|
||||
except (OSError, TypeError, ValueError):
|
||||
self.logger.exception('fdopen failed')
|
||||
except IOError:
|
||||
# this will happen at sort of broken pipe while writer.write
|
||||
self.logger.exception('IOError with writing fd %s' % fd)
|
||||
except StorletTimeout:
|
||||
self.logger.exception(
|
||||
'Timeout (%s)s with writing fd %s' % (self.timeout, fd))
|
||||
except Exception:
|
||||
# _write_input_data is designed to run eventlet thread
|
||||
# so that we should catch and suppress it here
|
||||
self.logger.exception('Unexpected error at writing input data')
|
||||
|
|
@ -13,8 +13,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from storlets.gateway.gateways.docker.gateway import StorletGatewayDocker
|
||||
from storlets.gateway.gateways.docker.gateway import DockerStorletGateway
|
||||
|
||||
__all__ = [
|
||||
'StorletGatewayDocker',
|
||||
'DockerStorletGateway',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -14,370 +14,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from storlets.gateway.common.stob import StorletRequest
|
||||
from storlets.gateway.gateways.base import StorletGatewayBase
|
||||
from storlets.gateway.gateways.docker.runtime import RunTimePaths, \
|
||||
RunTimeSandbox, StorletInvocationProtocol
|
||||
from storlets.gateway.gateways.container.gateway import StorletGatewayContainer
|
||||
from storlets.gateway.gateways.docker.runtime import DockerRunTimeSandbox
|
||||
|
||||
|
||||
"""---------------------------------------------------------------------------
|
||||
The Storlet Gateway API
|
||||
The API is made of:
|
||||
(1) The class DockerStorletRequest. This encapsulates what goes in and comes
|
||||
out of the gateway
|
||||
(2) The StorletGateway is the Docker flavor of the StorletGateway API:
|
||||
validate_storlet_registration
|
||||
validate_dependency_registration
|
||||
invocation_flow
|
||||
---------------------------------------------------------------------------"""
|
||||
|
||||
|
||||
class DockerStorletRequest(StorletRequest):
|
||||
"""
|
||||
The DockerStorletRequest class represents a request to be processed by the
|
||||
storlet the request is derived from the Swift request and
|
||||
essentially consists of:
|
||||
1. A data stream to be processed
|
||||
2. Metadata identifying the stream
|
||||
"""
|
||||
|
||||
# TODO(takashi): Some of following parameters should be defined common
|
||||
# parameters for StorletRequest
|
||||
required_options = ['storlet_main', 'storlet_language', 'file_manager']
|
||||
|
||||
def __init__(self, storlet_id, params, user_metadata, data_iter=None,
|
||||
data_fd=None, options=None):
|
||||
"""
|
||||
:param storlet_id: storlet id
|
||||
:param params: execution parameters
|
||||
:param user_metadata: user metadata
|
||||
:param data_iter: an iterator to read data
|
||||
:param data_fd: a file descriptor to read data
|
||||
:param options: a dictionaly which stores gateway specific options.
|
||||
:raises ValueError: when some of the required options (storlet_main
|
||||
and file_manager) are missing
|
||||
"""
|
||||
super(DockerStorletRequest, self).__init__(
|
||||
storlet_id, params, user_metadata, data_iter, data_fd,
|
||||
options=options)
|
||||
|
||||
self.generate_log = self.options.get('generate_log', False)
|
||||
|
||||
self.storlet_main = self.options['storlet_main']
|
||||
self.storlet_language = self.options['storlet_language']
|
||||
self.storlet_language_version = \
|
||||
self.options.get('storlet_language_version')
|
||||
|
||||
if self.options.get('storlet_dependency'):
|
||||
self.dependencies = [
|
||||
x.strip() for x
|
||||
in self.options['storlet_dependency'].split(',')
|
||||
if x.strip()]
|
||||
else:
|
||||
self.dependencies = []
|
||||
|
||||
self.file_manager = self.options['file_manager']
|
||||
|
||||
self.start = self.options.get('range_start')
|
||||
self.end = self.options.get('range_end')
|
||||
|
||||
@property
|
||||
def has_range(self):
|
||||
"""
|
||||
Whether the input range is given
|
||||
"""
|
||||
return self.start is not None and self.end is not None
|
||||
|
||||
|
||||
class StorletGatewayDocker(StorletGatewayBase):
|
||||
|
||||
request_class = DockerStorletRequest
|
||||
|
||||
def __init__(self, conf, logger, scope):
|
||||
"""
|
||||
:param conf: a dict for gateway conf
|
||||
:param logger: a logger instance
|
||||
:param scope: scope name to identify the container
|
||||
"""
|
||||
super(StorletGatewayDocker, self).__init__(conf, logger, scope)
|
||||
self.storlet_timeout = float(self.conf.get('storlet_timeout', 40))
|
||||
self.paths = RunTimePaths(scope, conf)
|
||||
|
||||
@classmethod
|
||||
def validate_storlet_registration(cls, params, name):
|
||||
"""
|
||||
Validate required parameters for storlet file
|
||||
|
||||
:param params: parameters related to the storlet file
|
||||
:param name: name of the storlet file
|
||||
:raises ValueError: if some of the required parameters are missing,
|
||||
or some of the parameters are invalid
|
||||
"""
|
||||
mandatory = ['Language', 'Interface-Version', 'Object-Metadata',
|
||||
'Main']
|
||||
cls._check_mandatory_params(params, mandatory)
|
||||
|
||||
if params['Language'].lower() == 'java':
|
||||
if '-' not in name or '.' not in name:
|
||||
raise ValueError('Storlet name is incorrect')
|
||||
elif params['Language'].lower() == 'python':
|
||||
try:
|
||||
version = float(params.get('Language-Version', 3))
|
||||
except ValueError:
|
||||
raise ValueError('Language-Version is invalid')
|
||||
|
||||
if int(version) != 3:
|
||||
# TODO(kota_): more strict version check should be nice.
|
||||
raise ValueError('Not supported version specified')
|
||||
|
||||
if name.endswith('.py'):
|
||||
cls_name = params['Main']
|
||||
if not cls_name.startswith(name[:-3] + '.'):
|
||||
raise ValueError('Main class should be included in '
|
||||
'storlet file')
|
||||
|
||||
if len(cls_name.split('.')) != 2:
|
||||
raise ValueError('Submodule is currently not supported')
|
||||
# TODO(takashi): Add support for sdist tar.gz
|
||||
else:
|
||||
raise ValueError('Storlet name is incorrect')
|
||||
else:
|
||||
raise ValueError('Unsupported Language')
|
||||
|
||||
dep = params.get('Dependency')
|
||||
if dep:
|
||||
deps = dep.split(',')
|
||||
if name in deps:
|
||||
raise ValueError('Using the same name for storlet and '
|
||||
'dependency is not allowed')
|
||||
if len(deps) != len(set(deps)):
|
||||
raise ValueError('Duplicated name in dependencies')
|
||||
|
||||
@classmethod
|
||||
def validate_dependency_registration(cls, params, name):
|
||||
"""
|
||||
Validate required parameters for dependency file
|
||||
|
||||
:param params: parameters related to the dependency file
|
||||
:param name: name of the dependency file
|
||||
:raises ValueError: if some of the required parameters are missing,
|
||||
or some of the parameters are invalid
|
||||
"""
|
||||
mandatory = ['Dependency-Version']
|
||||
cls._check_mandatory_params(params, mandatory)
|
||||
|
||||
perm = params.get('Dependency-Permissions')
|
||||
if perm is not None:
|
||||
try:
|
||||
perm_int = int(perm, 8)
|
||||
except ValueError:
|
||||
raise ValueError('Dependency permission is incorrect')
|
||||
if (perm_int & int('600', 8)) != int('600', 8):
|
||||
raise ValueError('The owner should have rw permission')
|
||||
|
||||
@classmethod
|
||||
def _check_mandatory_params(cls, params, mandatory):
|
||||
"""
|
||||
Ensure that we have all mandatory parameters in the given parameters
|
||||
|
||||
:param params: file parameters
|
||||
:param mandatory: required parameters
|
||||
:raises ValueError: if some of the required parameters are missing
|
||||
"""
|
||||
for md in mandatory:
|
||||
if md not in params:
|
||||
raise ValueError('Mandatory parameter is missing'
|
||||
': {0}'.format(md))
|
||||
|
||||
def invocation_flow(self, sreq, extra_sources=None):
|
||||
"""
|
||||
Invoke the backend protocol with gateway
|
||||
|
||||
:param sreq: StorletRequest instance
|
||||
:param extra_sources (WIP): A list of StorletRequest instance to gather
|
||||
as extra resoureces to feed to storlet
|
||||
container as data source
|
||||
:return: StorletResponse instance
|
||||
"""
|
||||
run_time_sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
|
||||
docker_updated = self.update_docker_container_from_cache(sreq)
|
||||
run_time_sbox.activate_storlet_daemon(sreq, docker_updated)
|
||||
self._add_system_params(sreq)
|
||||
|
||||
slog_path = self.paths.get_host_slog_path(sreq.storlet_main)
|
||||
storlet_pipe_path = \
|
||||
self.paths.get_host_storlet_pipe(sreq.storlet_main)
|
||||
|
||||
sprotocol = StorletInvocationProtocol(sreq,
|
||||
storlet_pipe_path,
|
||||
slog_path,
|
||||
self.storlet_timeout,
|
||||
self.logger,
|
||||
extra_sources=extra_sources)
|
||||
|
||||
sresp = sprotocol.communicate()
|
||||
|
||||
self._upload_storlet_logs(slog_path, sreq)
|
||||
|
||||
return sresp
|
||||
|
||||
def _add_system_params(self, sreq):
|
||||
"""
|
||||
Adds Storlet engine specific parameters to the invocation
|
||||
|
||||
currently, this consists only of the execution path of the
|
||||
Storlet within the Docker container.
|
||||
|
||||
:params params: Request parameters
|
||||
"""
|
||||
sreq.params['storlet_execution_path'] = self. \
|
||||
paths.get_sbox_storlet_dir(sreq.storlet_main)
|
||||
|
||||
def _upload_storlet_logs(self, slog_path, sreq):
|
||||
"""
|
||||
Upload storlet execution log as a swift object
|
||||
|
||||
:param slog_path: target path
|
||||
:params sreq: DockerStorletRequest instance
|
||||
"""
|
||||
if sreq.generate_log:
|
||||
with open(slog_path, 'rb') as logfile:
|
||||
storlet_name = sreq.storlet_id.split('-')[0]
|
||||
log_obj_name = '%s.log' % storlet_name
|
||||
sreq.file_manager.put_log(log_obj_name, logfile)
|
||||
|
||||
def bring_from_cache(self, obj_name, sreq, is_storlet):
|
||||
"""
|
||||
Auxiliary function that:
|
||||
|
||||
(1) Brings from Swift obj_name, either this is in a
|
||||
storlet or a storlet dependency.
|
||||
(2) Copies from local cache into the Docker conrainer
|
||||
If this is a Storlet then also validates that the cache is updated
|
||||
with most recent copy of the Storlet compared to the copy residing in
|
||||
Swift.
|
||||
|
||||
:params obj_name: name of the object
|
||||
:params sreq: DockerStorletRequest instance
|
||||
:params is_storlet: True if the object is a storlet object
|
||||
False if the object is a dependency object
|
||||
:returns: Whether the Docker container was updated with obj_name
|
||||
"""
|
||||
# Determine the cache we are to work with
|
||||
# e.g. dependency or storlet
|
||||
if is_storlet:
|
||||
cache_dir = self.paths.host_storlet_cache_dir
|
||||
get_func = sreq.file_manager.get_storlet
|
||||
else:
|
||||
cache_dir = self.paths.host_dependency_cache_dir
|
||||
get_func = sreq.file_manager.get_dependency
|
||||
|
||||
if not os.path.exists(cache_dir):
|
||||
os.makedirs(cache_dir, 0o700)
|
||||
|
||||
# cache_target_path is the actual object we need to deal with
|
||||
# e.g. a concrete storlet or dependency we need to bring/update
|
||||
cache_target_path = os.path.join(cache_dir, obj_name)
|
||||
|
||||
# Determine if we need to update the cache for cache_target_path
|
||||
# We default for no
|
||||
update_cache = False
|
||||
|
||||
# If it does not exist in cache, we obviously need to bring
|
||||
if not os.path.isfile(cache_target_path):
|
||||
update_cache = True
|
||||
elif is_storlet:
|
||||
# The cache_target_path exists, we test if it is up-to-date
|
||||
# with the metadata we got.
|
||||
# We mention that this is currently applicable for storlets
|
||||
# only, and not for dependencies.
|
||||
# This will change when we will head dependencies as well
|
||||
fstat = os.stat(cache_target_path)
|
||||
storlet_or_size = int(
|
||||
sreq.options['storlet_content_length'].rstrip("L"))
|
||||
storlet_or_time = float(sreq.options['storlet_x_timestamp'])
|
||||
b_storlet_size_changed = fstat.st_size != storlet_or_size
|
||||
b_storlet_file_updated = float(fstat.st_mtime) < storlet_or_time
|
||||
if b_storlet_size_changed or b_storlet_file_updated:
|
||||
update_cache = True
|
||||
|
||||
if update_cache:
|
||||
# If the cache needs to be updated, then get on with it
|
||||
# bring the object from storage
|
||||
data_iter, perm = get_func(obj_name)
|
||||
|
||||
if perm:
|
||||
perm = int(perm, 8) & 0o700
|
||||
else:
|
||||
perm = 0o600
|
||||
|
||||
# TODO(takashi): Do not directly write to target path
|
||||
with open(cache_target_path, 'wb') as fn:
|
||||
os.chmod(cache_target_path, perm)
|
||||
for data in data_iter:
|
||||
fn.write(data)
|
||||
|
||||
# The node's local cache is now updated.
|
||||
# We now verify if we need to update the
|
||||
# Docker container itself.
|
||||
# The Docker container needs to be updated if:
|
||||
# 1. The Docker container does not hold a copy of the object
|
||||
# 2. The Docker container holds an older version of the object
|
||||
update_docker = False
|
||||
docker_storlet_path = \
|
||||
self.paths.get_host_storlet_dir(sreq.storlet_main)
|
||||
docker_target_path = os.path.join(docker_storlet_path, obj_name)
|
||||
|
||||
if not os.path.exists(docker_storlet_path):
|
||||
os.makedirs(docker_storlet_path, 0o700)
|
||||
update_docker = True
|
||||
elif not os.path.isfile(docker_target_path):
|
||||
update_docker = True
|
||||
else:
|
||||
fstat_cached = os.stat(cache_target_path)
|
||||
fstat_docker = os.stat(docker_target_path)
|
||||
if fstat_cached.st_size != fstat_docker.st_size:
|
||||
update_docker = True
|
||||
if fstat_cached.st_mtime < fstat_docker.st_mtime:
|
||||
update_docker = True
|
||||
|
||||
if update_docker:
|
||||
# need to copy from cache to docker
|
||||
# copy2 also copies the permissions
|
||||
shutil.copy2(cache_target_path, docker_target_path)
|
||||
|
||||
return update_docker
|
||||
|
||||
def update_docker_container_from_cache(self, sreq):
|
||||
"""
|
||||
Iterates over the storlet name and its dependencies appearing
|
||||
|
||||
in the invocation data and make sure they are brought to the
|
||||
local cache, and from there to the Docker container.
|
||||
Uses the bring_from_cache auxiliary function.
|
||||
|
||||
:params sreq: DockerStorletRequest instance
|
||||
:returns: True if the Docker container was updated
|
||||
"""
|
||||
# where at the host side, reside the storlet containers
|
||||
storlet_path = self.paths.host_storlet_base_dir
|
||||
if not os.path.exists(storlet_path):
|
||||
os.makedirs(storlet_path, 0o755)
|
||||
|
||||
# Iterate over storlet and dependencies, and make sure
|
||||
# they are updated within the Docker container.
|
||||
# return True if any of them wea actually
|
||||
# updated within the Docker container
|
||||
docker_updated = False
|
||||
|
||||
updated = self.bring_from_cache(sreq.storlet_id, sreq, True)
|
||||
docker_updated = docker_updated or updated
|
||||
|
||||
for dep in sreq.dependencies:
|
||||
updated = self.bring_from_cache(dep, sreq, False)
|
||||
docker_updated = docker_updated or updated
|
||||
|
||||
return docker_updated
|
||||
class DockerStorletGateway(StorletGatewayContainer):
|
||||
sandbox = DockerRunTimeSandbox
|
||||
|
|
|
|||
|
|
@ -13,283 +13,17 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import os
|
||||
import select
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
import docker
|
||||
import docker.errors
|
||||
from docker.types import Mount as DockerMount
|
||||
|
||||
import eventlet
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
|
||||
from storlets.sbus.client import SBusClient
|
||||
from storlets.sbus.client.exceptions import SBusClientException
|
||||
from storlets.sbus.datagram import SBusFileDescriptor
|
||||
from storlets.sbus import file_description as sbus_fd
|
||||
from storlets.gateway.common.exceptions import StorletRuntimeException, \
|
||||
StorletTimeout
|
||||
from storlets.gateway.common.logger import StorletLogger
|
||||
from storlets.gateway.common.stob import StorletResponse
|
||||
|
||||
MAX_METADATA_SIZE = 4096
|
||||
from storlets.gateway.common.exceptions import StorletRuntimeException
|
||||
from storlets.gateway.gateways.container.runtime import RunTimeSandbox
|
||||
|
||||
|
||||
eventlet.monkey_patch()
|
||||
|
||||
|
||||
"""---------------------------------------------------------------------------
|
||||
Sandbox API
|
||||
"""
|
||||
|
||||
|
||||
class RunTimePaths(object):
|
||||
"""
|
||||
The Storlet Engine need to be access stuff located in many paths:
|
||||
|
||||
1. The various communication channels represented as pipes in the
|
||||
filesystem
|
||||
2. Directories where to place Storlets
|
||||
3. Directories where to place logs
|
||||
|
||||
Communication channels
|
||||
----------------------
|
||||
The RunTimeSandbox communicates with the Sandbox via two types of pipes
|
||||
1. factory pipe - defined per scope, used for communication with the
|
||||
sandbox
|
||||
for e.g. start/stop a storlet daemon
|
||||
2. Storlet pipe - defined per scope and Storlet, used for communication
|
||||
with a storlet daemon, e.g. to call the invoke API
|
||||
|
||||
Each pipe type has two paths:
|
||||
1. A path that is inside the sandbox
|
||||
2. A path that is outside of the sandbox or at the host side. As such
|
||||
this path is prefixed by 'host_'
|
||||
|
||||
Thus, we have the following 4 paths of interest:
|
||||
1. sandbox_factory_pipe_path
|
||||
2. host_factory_pipe_path
|
||||
3. sandbox_storlet_pipe_path
|
||||
4. host_storlet_pipe_path
|
||||
|
||||
Our implementation uses the following path structure for the various pipes:
|
||||
In the host, all pipes belonging to a given scope are prefixed by
|
||||
<pipes_dir>/<scope>, where <pipes_dir> comes from the configuration
|
||||
Thus:
|
||||
host_factory_pipe_path is of the form <pipes_dir>/<scope>/factory_pipe
|
||||
host_storlet_pipe_path is of the form <pipes_dir>/<scope>/<storlet_id>
|
||||
|
||||
In The sandbox side
|
||||
sandbox_factory_pipe_path is of the form /mnt/channels/factory_pipe
|
||||
sandbox_storlet_pipe_path is of the form /mnt/channels/<storlet_id>
|
||||
|
||||
Storlets Locations
|
||||
------------------
|
||||
The Storlet binaries are accessible from the sandbox using a mounted
|
||||
directory.
|
||||
This directory is called the storlet directories.
|
||||
On the host side it is of the form <storlet_dir>/<scope>/<storlet_name>
|
||||
On the sandbox side it is of the form /home/swift/<storlet_name>
|
||||
<storlet_dir> comes from the configuration
|
||||
<storlet_name> is the prefix of the jar.
|
||||
|
||||
Logs
|
||||
----
|
||||
Logs are located in paths of the form:
|
||||
<log_dir>/<scope>/<storlet_name>.log
|
||||
"""
|
||||
|
||||
def __init__(self, scope, conf):
|
||||
"""
|
||||
Construct RunTimePaths instance
|
||||
|
||||
:param scope: scope name to be used as container name
|
||||
:param conf: gateway conf
|
||||
"""
|
||||
self.scope = scope
|
||||
self.factory_pipe_name = 'factory_pipe'
|
||||
self.sandbox_pipe_dir = '/mnt/channels'
|
||||
|
||||
self.sandbox_storlet_base_dir = '/home/swift'
|
||||
self.host_root_dir = conf.get('host_root', '/var/lib/storlets')
|
||||
self.host_pipe_root_dir = \
|
||||
conf.get('pipes_dir',
|
||||
os.path.join(self.host_root_dir, 'pipes', 'scopes'))
|
||||
self.host_storlet_root_dir = \
|
||||
conf.get('storlets_dir',
|
||||
os.path.join(self.host_root_dir, 'storlets', 'scopes'))
|
||||
self.host_log_root_dir = \
|
||||
conf.get('log_dir',
|
||||
os.path.join(self.host_root_dir, 'logs', 'scopes'))
|
||||
self.host_cache_root_dir = \
|
||||
conf.get('cache_dir',
|
||||
os.path.join(self.host_root_dir, 'cache', 'scopes'))
|
||||
self.host_restart_script_dir = \
|
||||
conf.get('script_dir',
|
||||
os.path.join(self.host_root_dir, 'scripts'))
|
||||
|
||||
self.host_storlet_native_lib_dir = '/usr/local/lib/storlets'
|
||||
self.sandbox_storlet_native_lib_dir = '/usr/local/lib/storlets'
|
||||
self.host_storlet_native_bin_dir = '/usr/local/libexec/storlets'
|
||||
self.sandbox_storlet_native_bin_dir = '/usr/local/libexec/storlets'
|
||||
|
||||
@property
|
||||
def host_pipe_dir(self):
|
||||
return os.path.join(self.host_pipe_root_dir, self.scope)
|
||||
|
||||
def create_host_pipe_dir(self):
|
||||
path = self.host_pipe_dir
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
# 0777 should be 0700 when we get user namespaces in Docker
|
||||
os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
|
||||
return path
|
||||
|
||||
@property
|
||||
def host_factory_pipe(self):
|
||||
return os.path.join(self.host_pipe_dir, self.factory_pipe_name)
|
||||
|
||||
@property
|
||||
def sandbox_factory_pipe(self):
|
||||
return os.path.join(self.sandbox_pipe_dir, self.factory_pipe_name)
|
||||
|
||||
def get_host_storlet_pipe(self, storlet_id):
|
||||
return os.path.join(self.host_pipe_dir, storlet_id)
|
||||
|
||||
def get_sbox_storlet_pipe(self, storlet_id):
|
||||
return os.path.join(self.sandbox_pipe_dir, storlet_id)
|
||||
|
||||
def get_sbox_storlet_dir(self, storlet_id):
|
||||
return os.path.join(self.sandbox_storlet_base_dir, storlet_id)
|
||||
|
||||
@property
|
||||
def host_storlet_base_dir(self):
|
||||
return os.path.join(self.host_storlet_root_dir, self.scope)
|
||||
|
||||
def get_host_storlet_dir(self, storlet_id):
|
||||
return os.path.join(self.host_storlet_base_dir, storlet_id)
|
||||
|
||||
def get_host_slog_path(self, storlet_id):
|
||||
return os.path.join(
|
||||
self.host_log_root_dir, self.scope, storlet_id,
|
||||
'storlet_invoke.log')
|
||||
|
||||
@property
|
||||
def host_storlet_cache_dir(self):
|
||||
return os.path.join(self.host_cache_root_dir, self.scope, 'storlet')
|
||||
|
||||
@property
|
||||
def host_dependency_cache_dir(self):
|
||||
return os.path.join(self.host_cache_root_dir, self.scope, 'dependency')
|
||||
|
||||
|
||||
"""---------------------------------------------------------------------------
|
||||
Docker Stateful Container API
|
||||
The RunTimeSandbox serve as an API between the Docker Gateway and
|
||||
a reusable per scope sandbox
|
||||
---------------------------------------------------------------------------"""
|
||||
|
||||
|
||||
class RunTimeSandbox(object):
|
||||
"""
|
||||
The RunTimeSandbox represents a reusable per scope sandbox.
|
||||
|
||||
The sandbox is reusable in the sense that it can run several storlet
|
||||
daemons.
|
||||
|
||||
The following methods are supported:
|
||||
ping - pings the sandbox for liveness
|
||||
wait - wait for the sandbox to be ready for processing commands
|
||||
restart - restart the sandbox
|
||||
start_storlet_daemon - start a daemon for a given storlet
|
||||
stop_storlet_daemon - stop a daemon of a given storlet
|
||||
get_storlet_daemon_status - test if a given storlet daemon is running
|
||||
"""
|
||||
|
||||
def __init__(self, scope, conf, logger):
|
||||
"""
|
||||
:param scope: scope name to be used as container name
|
||||
:param conf: gateway conf
|
||||
:param logger: logger instance
|
||||
"""
|
||||
self.paths = RunTimePaths(scope, conf)
|
||||
self.scope = scope
|
||||
|
||||
self.sandbox_ping_interval = \
|
||||
float(conf.get('sandbox_ping_interval', 0.5))
|
||||
self.sandbox_stop_timeout = \
|
||||
float(conf.get('stop_linux_container_timeout', 1))
|
||||
self.sandbox_wait_timeout = \
|
||||
float(conf.get('restart_linux_container_timeout', 10))
|
||||
|
||||
self.container_image_namespace = \
|
||||
conf.get('docker_repo', conf.get('container_image_namespace'))
|
||||
self.container_image_name_prefix = 'tenant'
|
||||
|
||||
# TODO(add line in conf)
|
||||
self.storlet_daemon_thread_pool_size = \
|
||||
int(conf.get('storlet_daemon_thread_pool_size', 5))
|
||||
self.storlet_daemon_factory_debug_level = \
|
||||
conf.get('storlet_daemon_factory_debug_level', 'DEBUG')
|
||||
self.storlet_daemon_debug_level = \
|
||||
conf.get('storlet_daemon_debug_level', 'DEBUG')
|
||||
|
||||
# TODO(change logger's route if possible)
|
||||
self.logger = logger
|
||||
|
||||
self.default_container_image_name = conf.get(
|
||||
'default_docker_image_name',
|
||||
conf.get('default_container_image_name', 'storlet_engine_image')
|
||||
)
|
||||
|
||||
self.max_containers_per_node = \
|
||||
int(conf.get('max_containers_per_node', 0))
|
||||
|
||||
self.container_cpu_period = int(conf.get('container_cpu_period', 0))
|
||||
self.container_cpu_quota = int(conf.get('container_cpu_quota', 0))
|
||||
self.container_mem_limit = conf.get('container_mem_limit', 0)
|
||||
# NOTE(tkajinam): memory limit can be a string with unit like 1024m
|
||||
try:
|
||||
self.container_mem_limit = int(self.container_mem_limit)
|
||||
except TypeError:
|
||||
pass
|
||||
self.container_cpuset_cpus = conf.get('container_cpuset_cpus')
|
||||
self.container_cpuset_mems = conf.get('container_cpuset_mems')
|
||||
self.container_pids_limit = int(conf.get('container_pids_limit', 0))
|
||||
|
||||
def ping(self):
|
||||
"""
|
||||
Ping to daemon factory process inside container
|
||||
|
||||
:returns: True when the daemon factory is responsive
|
||||
False when the daemon factory is not responsive or it fails
|
||||
to send command to the process
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.ping()
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to ping to daemon factory: %s' %
|
||||
resp.message)
|
||||
return resp.status
|
||||
except SBusClientException:
|
||||
return False
|
||||
|
||||
def wait(self):
|
||||
"""
|
||||
Wait while scope's sandbox is starting
|
||||
|
||||
:raises StorletTimeout: the sandbox has not started in
|
||||
sandbox_wait_timeout
|
||||
"""
|
||||
with StorletTimeout(self.sandbox_wait_timeout):
|
||||
while not self.ping():
|
||||
time.sleep(self.sandbox_ping_interval)
|
||||
class DockerRunTimeSandbox(RunTimeSandbox):
|
||||
|
||||
def _restart(self, container_image_name):
|
||||
"""
|
||||
|
|
@ -375,468 +109,3 @@ class RunTimeSandbox(object):
|
|||
except docker.errors.APIError:
|
||||
self.logger.exception("Failed to manage docker containers")
|
||||
raise StorletRuntimeException("Docker runtime error")
|
||||
|
||||
def restart(self):
|
||||
"""
|
||||
Restarts the scope's sandbox
|
||||
|
||||
"""
|
||||
self.paths.create_host_pipe_dir()
|
||||
|
||||
container_image_name = self.scope
|
||||
try:
|
||||
self._restart(container_image_name)
|
||||
self.wait()
|
||||
except StorletTimeout:
|
||||
raise
|
||||
except StorletRuntimeException:
|
||||
# We were unable to start docker container from the tenant image.
|
||||
# Let us try to start docker container from default image.
|
||||
self.logger.exception("Failed to start docker container from "
|
||||
"tenant image %s" % container_image_name)
|
||||
|
||||
self.logger.info("Trying to start docker container from default "
|
||||
"image: %s" % self.default_container_image_name)
|
||||
self._restart(self.default_container_image_name)
|
||||
self.wait()
|
||||
|
||||
def start_storlet_daemon(
|
||||
self, spath, storlet_id, language, language_version=None):
|
||||
"""
|
||||
Start SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.start_daemon(
|
||||
language.lower(), spath, storlet_id,
|
||||
self.paths.get_sbox_storlet_pipe(storlet_id),
|
||||
self.storlet_daemon_debug_level,
|
||||
self.storlet_daemon_thread_pool_size,
|
||||
language_version)
|
||||
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to start storlet daemon: %s' %
|
||||
resp.message)
|
||||
raise StorletRuntimeException('Daemon start failed')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Daemon start failed')
|
||||
|
||||
def stop_storlet_daemon(self, storlet_id):
|
||||
"""
|
||||
Stop SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.stop_daemon(storlet_id)
|
||||
if not resp.status:
|
||||
self.logger.error('Failed to stop storlet daemon: %s' %
|
||||
resp.message)
|
||||
raise StorletRuntimeException('Daemon stop failed')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Daemon stop failed')
|
||||
|
||||
def get_storlet_daemon_status(self, storlet_id):
|
||||
"""
|
||||
Get the status of SDaemon process in the scope's sandbox
|
||||
"""
|
||||
pipe_path = self.paths.host_factory_pipe
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
resp = client.daemon_status(storlet_id)
|
||||
if resp.status:
|
||||
return 1
|
||||
else:
|
||||
self.logger.error('Failed to get status about storlet '
|
||||
'daemon: %s' % resp.message)
|
||||
return 0
|
||||
except SBusClientException:
|
||||
return -1
|
||||
|
||||
def _get_storlet_classpath(self, storlet_main, storlet_id, dependencies):
|
||||
"""
|
||||
Get classpath required to run storlet application
|
||||
|
||||
:param storlet_main: Main class name of the storlet
|
||||
:param storlet_id: Name of the storlet file
|
||||
:param dependencies: A list of dependency file
|
||||
:returns: classpath string
|
||||
"""
|
||||
class_path = os.path.join(
|
||||
self.paths.get_sbox_storlet_dir(storlet_main), storlet_id)
|
||||
|
||||
dep_path_list = \
|
||||
[os.path.join(self.paths.get_sbox_storlet_dir(storlet_main), dep)
|
||||
for dep in dependencies]
|
||||
|
||||
return class_path + ':' + ':'.join(dep_path_list)
|
||||
|
||||
def activate_storlet_daemon(self, sreq, cache_updated=True):
|
||||
storlet_daemon_status = \
|
||||
self.get_storlet_daemon_status(sreq.storlet_main)
|
||||
if (storlet_daemon_status == -1):
|
||||
# We failed to send a command to the factory.
|
||||
# Best we can do is execute the container.
|
||||
self.logger.debug('Failed to check the storlet daemon status. '
|
||||
'Restart Docker container')
|
||||
self.restart()
|
||||
storlet_daemon_status = 0
|
||||
|
||||
if (cache_updated is True and storlet_daemon_status == 1):
|
||||
# The cache was updated while the daemon is running we need to
|
||||
# stop it.
|
||||
self.logger.debug('The cache was updated, and the storlet daemon '
|
||||
'is running. Stopping daemon')
|
||||
|
||||
try:
|
||||
self.stop_storlet_daemon(sreq.storlet_main)
|
||||
except StorletRuntimeException:
|
||||
self.logger.warning('Failed to stop the storlet daemon. '
|
||||
'Restart Docker container')
|
||||
self.restart()
|
||||
else:
|
||||
self.logger.debug('Deamon stopped')
|
||||
storlet_daemon_status = 0
|
||||
|
||||
if (storlet_daemon_status == 0):
|
||||
self.logger.debug('Going to start the storlet daemon!')
|
||||
|
||||
# TODO(takashi): This is not needed for python application
|
||||
classpath = self._get_storlet_classpath(
|
||||
sreq.storlet_main, sreq.storlet_id, sreq.dependencies)
|
||||
|
||||
self.start_storlet_daemon(
|
||||
classpath, sreq.storlet_main, sreq.storlet_language,
|
||||
sreq.storlet_language_version)
|
||||
self.logger.debug('Daemon started')
|
||||
|
||||
|
||||
"""---------------------------------------------------------------------------
|
||||
Storlet Daemon API
|
||||
StorletInvocationProtocol
|
||||
server as an API between the Docker Gateway and the Storlet Daemon which
|
||||
runs inside the Docker container. These classes implement the Storlet execution
|
||||
protocol
|
||||
---------------------------------------------------------------------------"""
|
||||
|
||||
|
||||
class StorletInvocationProtocol(object):
|
||||
"""
|
||||
StorletInvocationProtocol class
|
||||
|
||||
This class serves communictaion with a Docker container to run an
|
||||
application
|
||||
|
||||
:param srequest: StorletRequest instance
|
||||
:param storlet_pipe_path: path string to pipe
|
||||
:param storlet_logger_path: path string to log file
|
||||
:param timeout: integer of timeout for waiting the resp from container
|
||||
:param logger: logger instance
|
||||
:param extra_sources (WIP): a list of StorletRequest instances
|
||||
which keep data_iter for adding extra source
|
||||
as data stream
|
||||
"""
|
||||
def __init__(self, srequest, storlet_pipe_path, storlet_logger_path,
|
||||
timeout, logger, extra_sources=None):
|
||||
self.srequest = srequest
|
||||
self.storlet_pipe_path = storlet_pipe_path
|
||||
self.storlet_logger = StorletLogger(storlet_logger_path)
|
||||
self.logger = logger
|
||||
self.timeout = timeout
|
||||
|
||||
# local side file descriptors
|
||||
self.data_read_fd = None
|
||||
self.data_write_fd = None
|
||||
self.metadata_read_fd = None
|
||||
self.metadata_write_fd = None
|
||||
self.task_id = None
|
||||
self._input_data_read_fd = None
|
||||
self._input_data_write_fd = None
|
||||
|
||||
self.extra_data_sources = []
|
||||
extra_sources = extra_sources or []
|
||||
for source in extra_sources:
|
||||
if source.has_fd:
|
||||
# TODO(kota_): it may be data_fd in the future.
|
||||
raise Exception(
|
||||
'extra_source no requires data_fd just data_iter')
|
||||
self.extra_data_sources.append(
|
||||
{'read_fd': None, 'write_fd': None,
|
||||
'user_metadata': source.user_metadata,
|
||||
'data_iter': source.data_iter})
|
||||
|
||||
@property
|
||||
def input_data_read_fd(self):
|
||||
"""
|
||||
File descriptor to read the input body content
|
||||
"""
|
||||
if self.srequest.has_fd:
|
||||
return self.srequest.data_fd
|
||||
else:
|
||||
return self._input_data_read_fd
|
||||
|
||||
@property
|
||||
def remote_fds(self):
|
||||
"""
|
||||
A list of sbus file descriptors passed to remote side
|
||||
"""
|
||||
storlets_metadata = {}
|
||||
if self.srequest.has_range:
|
||||
storlets_metadata.update(
|
||||
{'start': str(self.srequest.start),
|
||||
'end': str(self.srequest.end)})
|
||||
|
||||
fds = [SBusFileDescriptor(sbus_fd.SBUS_FD_INPUT_OBJECT,
|
||||
self.input_data_read_fd,
|
||||
storage_metadata=self.srequest.user_metadata,
|
||||
storlets_metadata=storlets_metadata),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT,
|
||||
self.data_write_fd),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
|
||||
self.metadata_write_fd),
|
||||
SBusFileDescriptor(sbus_fd.SBUS_FD_LOGGER,
|
||||
self.storlet_logger.getfd())]
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
fd = SBusFileDescriptor(
|
||||
sbus_fd.SBUS_FD_INPUT_OBJECT,
|
||||
source['read_fd'],
|
||||
storage_metadata=source['user_metadata'])
|
||||
fds.append(fd)
|
||||
|
||||
return fds
|
||||
|
||||
@contextmanager
|
||||
def _activate_invocation_descriptors(self):
|
||||
"""
|
||||
Contextmanager about file descriptors used in storlet invocation
|
||||
|
||||
NOTE: This context manager now only closes remote side fds,
|
||||
so you should close local side fds
|
||||
"""
|
||||
self._prepare_invocation_descriptors()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._close_remote_side_descriptors()
|
||||
|
||||
def _prepare_invocation_descriptors(self):
|
||||
"""
|
||||
Create all pipse used for Storlet execution
|
||||
"""
|
||||
if not self.srequest.has_fd:
|
||||
self._input_data_read_fd, self._input_data_write_fd = os.pipe()
|
||||
self.data_read_fd, self.data_write_fd = os.pipe()
|
||||
self.metadata_read_fd, self.metadata_write_fd = os.pipe()
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
source['read_fd'], source['write_fd'] = os.pipe()
|
||||
|
||||
def _safe_close(self, fds):
|
||||
"""
|
||||
Make sure that all of the file descriptors get closed
|
||||
|
||||
:param fds: a list of file descriptors
|
||||
"""
|
||||
for fd in fds:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError as err:
|
||||
if err.errno != errno.EBADF:
|
||||
raise
|
||||
# TODO(kota_): fd might be closed already, so if already
|
||||
# closed, OSError will be raised. we need more refactor to
|
||||
# keep clean the file descriptors.
|
||||
pass
|
||||
|
||||
def _close_remote_side_descriptors(self):
|
||||
"""
|
||||
Close all of the container side descriptors
|
||||
"""
|
||||
fds = [self.data_write_fd, self.metadata_write_fd]
|
||||
if not self.srequest.has_fd:
|
||||
fds.append(self.input_data_read_fd)
|
||||
fds.extend([source['read_fd'] for source in self.extra_data_sources])
|
||||
for fd in fds:
|
||||
os.close(fd)
|
||||
|
||||
def _close_local_side_descriptors(self):
|
||||
"""
|
||||
Close all of the host side descriptors
|
||||
"""
|
||||
fds = [self.data_read_fd, self.metadata_read_fd]
|
||||
fds.extend([source['write_fd'] for source in self.extra_data_sources])
|
||||
self._safe_close(fds)
|
||||
|
||||
def _cancel(self):
|
||||
"""
|
||||
Cancel on-going storlet execution
|
||||
"""
|
||||
client = SBusClient(self.storlet_pipe_path)
|
||||
try:
|
||||
resp = client.cancel(self.task_id)
|
||||
if not resp.status:
|
||||
raise StorletRuntimeException('Failed to cancel task')
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException('Failed to cancel task')
|
||||
|
||||
def _invoke(self):
|
||||
"""
|
||||
Send an execution command to the remote daemon factory
|
||||
"""
|
||||
with self.storlet_logger.activate(),\
|
||||
self._activate_invocation_descriptors():
|
||||
self._send_execute_command()
|
||||
|
||||
def _send_execute_command(self):
|
||||
"""
|
||||
Send execute command to the remote daemon factory to invoke storlet
|
||||
execution
|
||||
"""
|
||||
client = SBusClient(self.storlet_pipe_path)
|
||||
try:
|
||||
resp = client.execute(self.srequest.params, self.remote_fds)
|
||||
if not resp.status:
|
||||
raise StorletRuntimeException("Failed to send execute command")
|
||||
|
||||
if not resp.task_id:
|
||||
raise StorletRuntimeException("Missing task id")
|
||||
else:
|
||||
self.task_id = resp.task_id
|
||||
except SBusClientException:
|
||||
raise StorletRuntimeException("Failed to send execute command")
|
||||
|
||||
def _wait_for_read_with_timeout(self, fd):
|
||||
"""
|
||||
Wait while the read file descriptor gets ready
|
||||
|
||||
:param fd: File descriptor to read
|
||||
:raises StorletTimeout: Exception raised when it times out to cancel
|
||||
the existing task
|
||||
:raises StorletRuntimeException: Exception raised when it fails to
|
||||
cancel the existing task
|
||||
"""
|
||||
try:
|
||||
with StorletTimeout(self.timeout):
|
||||
r, w, e = select.select([fd], [], [])
|
||||
except StorletTimeout:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
|
||||
# When there is a task already running, we should cancel it.
|
||||
if self.task_id:
|
||||
try:
|
||||
self._cancel()
|
||||
except StorletRuntimeException:
|
||||
self.logger.warning(
|
||||
'Task %s timed out, but failed to get canceled'
|
||||
% self.task_id)
|
||||
pass
|
||||
|
||||
if exc_value is None:
|
||||
exc_value = exc_traceback
|
||||
if exc_value.__traceback__ is not exc_traceback:
|
||||
raise exc_value.with_traceback(exc_traceback)
|
||||
raise exc_value
|
||||
|
||||
if fd not in r:
|
||||
raise StorletRuntimeException('Read fd is not ready')
|
||||
|
||||
def _read_metadata(self):
|
||||
"""
|
||||
Read metadata in the storlet execution result from fd
|
||||
|
||||
:returns: a dict of metadata
|
||||
"""
|
||||
self._wait_for_read_with_timeout(self.metadata_read_fd)
|
||||
flat_json = os.read(self.metadata_read_fd, MAX_METADATA_SIZE)
|
||||
os.close(self.metadata_read_fd)
|
||||
try:
|
||||
return json.loads(flat_json)
|
||||
except ValueError:
|
||||
self.logger.exception('Failed to load metadata from json')
|
||||
raise StorletRuntimeException('Got invalid format about metadata')
|
||||
|
||||
def _wait_for_write_with_timeout(self, fd):
|
||||
"""
|
||||
Wait while the write file descriptor gets ready
|
||||
|
||||
:param fd: File descriptor to write
|
||||
:raises StorletTimeout: Exception raised when it times out to cancel
|
||||
the existing task
|
||||
:raises StorletRuntimeException: Exception raised when it fails to
|
||||
cancel the existing task
|
||||
"""
|
||||
with StorletTimeout(self.timeout):
|
||||
r, w, e = select.select([], [fd], [])
|
||||
if fd not in w:
|
||||
raise StorletRuntimeException('Write fd is not ready')
|
||||
|
||||
def _close_input_data_descriptors(self):
|
||||
fds = [self._input_data_read_fd, self._input_data_write_fd]
|
||||
self._safe_close(fds)
|
||||
|
||||
def communicate(self):
|
||||
try:
|
||||
self._invoke()
|
||||
|
||||
if not self.srequest.has_fd:
|
||||
self._wait_for_write_with_timeout(self._input_data_write_fd)
|
||||
|
||||
# We do the writing in a different thread.
|
||||
# Otherwise, we can run into the following deadlock
|
||||
# 1. middleware writes to Storlet
|
||||
# 2. Storlet reads and starts to write metadata and then data
|
||||
# 3. middleware continues writing
|
||||
# 4. Storlet continues writing and gets stuck as middleware
|
||||
# is busy writing, but still not consuming the reader end
|
||||
# of the Storlet writer.
|
||||
eventlet.spawn_n(self._write_input_data,
|
||||
self._input_data_write_fd,
|
||||
self.srequest.data_iter)
|
||||
|
||||
for source in self.extra_data_sources:
|
||||
# NOTE(kota_): not sure right now if using eventlet.spawn_n is
|
||||
# right way. GreenPool is better? I don't get
|
||||
# whole for the dead lock described in above.
|
||||
self._wait_for_write_with_timeout(source['write_fd'])
|
||||
eventlet.spawn_n(self._write_input_data,
|
||||
source['write_fd'],
|
||||
source['data_iter'])
|
||||
|
||||
out_md = self._read_metadata()
|
||||
self._wait_for_read_with_timeout(self.data_read_fd)
|
||||
|
||||
return StorletResponse(out_md, data_fd=self.data_read_fd,
|
||||
cancel=self._cancel)
|
||||
except Exception:
|
||||
self._close_local_side_descriptors()
|
||||
if not self.srequest.has_fd:
|
||||
self._close_input_data_descriptors()
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _open_writer(self, fd):
|
||||
with os.fdopen(fd, 'wb') as writer:
|
||||
yield writer
|
||||
|
||||
def _write_input_data(self, fd, data_iter):
|
||||
try:
|
||||
# double try/except block saving from unexpected errors
|
||||
try:
|
||||
with self._open_writer(fd) as writer:
|
||||
for chunk in data_iter:
|
||||
with StorletTimeout(self.timeout):
|
||||
writer.write(chunk)
|
||||
except (OSError, TypeError, ValueError):
|
||||
self.logger.exception('fdopen failed')
|
||||
except IOError:
|
||||
# this will happen at sort of broken pipe while writer.write
|
||||
self.logger.exception('IOError with writing fd %s' % fd)
|
||||
except StorletTimeout:
|
||||
self.logger.exception(
|
||||
'Timeout (%s)s with writing fd %s' % (self.timeout, fd))
|
||||
except Exception:
|
||||
# _write_input_data is designed to run eventlet thread
|
||||
# so that we should catch and suppress it here
|
||||
self.logger.exception('Unexpected error at writing input data')
|
||||
|
|
|
|||
|
|
@ -18,12 +18,12 @@ from storlets.gateway.common.stob import StorletRequest, StorletResponse
|
|||
from storlets.gateway.gateways.base import StorletGatewayBase
|
||||
|
||||
|
||||
class StorletGatewayStub(StorletGatewayBase):
|
||||
class StubStorletGateway(StorletGatewayBase):
|
||||
|
||||
request_class = StorletRequest
|
||||
|
||||
def __init__(self, conf, logger, scope):
|
||||
super(StorletGatewayStub, self).__init__(conf, logger, scope)
|
||||
super(StubStorletGateway, self).__init__(conf, logger, scope)
|
||||
self.logger = logger
|
||||
self.conf = conf
|
||||
self.scope = scope
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class TestCapabilities(StorletBaseFunctionalTest):
|
|||
# TODO(eranr): take values from conf
|
||||
self.assertEqual('dependency', options['storlet_dependency'])
|
||||
self.assertEqual('storlet', options['storlet_container'])
|
||||
self.assertEqual('StorletGatewayDocker',
|
||||
self.assertEqual('DockerStorletGateway',
|
||||
options['storlet_gateway_class'])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,561 @@
|
|||
# Copyright (c) 2010-2015 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import eventlet
|
||||
from io import BytesIO, StringIO
|
||||
import json
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from swift.common.swob import Request, Response
|
||||
from swift.common.utils import FileLikeIter
|
||||
|
||||
from storlets.sbus.client import SBusResponse
|
||||
from storlets.gateway.gateways.container.gateway import ContainerStorletRequest
|
||||
from tests.unit import FakeLogger
|
||||
from tests.unit.gateway.gateways import FakeFileManager
|
||||
|
||||
|
||||
class TestContainerStorletRequest(unittest.TestCase):
|
||||
|
||||
def test_init(self):
|
||||
# Java
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {'Param1': 'Value1', 'Param2': 'Value2'}
|
||||
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
|
||||
|
||||
# with dependencies
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# without dependencies
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# storlet_language is not given
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
with self.assertRaises(ValueError):
|
||||
ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# storlet_main is not given
|
||||
options = {'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
with self.assertRaises(ValueError):
|
||||
ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# file_manager is not given
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_language': 'java'}
|
||||
with self.assertRaises(ValueError):
|
||||
ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# Python
|
||||
storlet_id = 'storlet.py'
|
||||
params = {'Param1': 'Value1', 'Param2': 'Value2'}
|
||||
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
|
||||
|
||||
# without language version
|
||||
options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_language': 'python',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('storlet.py', dsreq.storlet_id)
|
||||
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('python', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# with language version
|
||||
options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_language': 'python',
|
||||
'storlet_language_version': '3.6',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('storlet.py', dsreq.storlet_id)
|
||||
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('python', dsreq.storlet_language)
|
||||
self.assertEqual('3.6', dsreq.storlet_language_version)
|
||||
|
||||
def test_init_with_range(self):
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {}
|
||||
metadata = {}
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 1,
|
||||
'range_end': 6}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
self.assertEqual(1, dsreq.start)
|
||||
self.assertEqual(6, dsreq.end)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 0,
|
||||
'range_end': 0}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
self.assertEqual(0, dsreq.start)
|
||||
self.assertEqual(0, dsreq.end)
|
||||
|
||||
def test_has_range(self):
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {}
|
||||
metadata = {}
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertFalse(dsreq.has_range)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 1,
|
||||
'range_end': 6}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertTrue(dsreq.has_range)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 0,
|
||||
'range_end': 6}
|
||||
dsreq = ContainerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertTrue(dsreq.has_range)
|
||||
|
||||
|
||||
class ContainerGatewayTestMixin(object):
|
||||
|
||||
def setUp(self):
|
||||
# TODO(takashi): take these values from config file
|
||||
self.tempdir = mkdtemp()
|
||||
self.sconf = {
|
||||
'host_root': self.tempdir,
|
||||
'swift_dir': self.tempdir,
|
||||
'storlet_timeout': '9',
|
||||
'storlet_container': 'storlet',
|
||||
'storlet_dependency': 'dependency',
|
||||
'reseller_prefix': 'AUTH'
|
||||
}
|
||||
self.logger = FakeLogger()
|
||||
|
||||
self.storlet_container = self.sconf['storlet_container']
|
||||
self.storlet_dependency = self.sconf['storlet_dependency']
|
||||
|
||||
self.version = 'v1'
|
||||
self.account = 'AUTH_account'
|
||||
self.container = 'container'
|
||||
self.obj = 'object'
|
||||
self.sobj = 'storlet-1.0.jar'
|
||||
|
||||
self.gateway = self.gateway_class(
|
||||
self.sconf, self.logger, self.account)
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.tempdir)
|
||||
|
||||
@property
|
||||
def req_path(self):
|
||||
return self._create_proxy_path(
|
||||
self.version, self.account, self.container,
|
||||
self.obj)
|
||||
|
||||
@property
|
||||
def storlet_path(self):
|
||||
return self._create_proxy_path(
|
||||
self.version, self.account, self.storlet_container,
|
||||
self.sobj)
|
||||
|
||||
def _create_proxy_path(self, version, account, container, obj):
|
||||
return '/'.join(['', version, account, container, obj])
|
||||
|
||||
def test_check_mandatory_params(self):
|
||||
params = {'keyA': 'valueA',
|
||||
'keyB': 'valueB',
|
||||
'keyC': 'valueC'}
|
||||
|
||||
# all mandatory headers are included
|
||||
self.gateway_class._check_mandatory_params(
|
||||
params, ['keyA', 'keyB'])
|
||||
|
||||
# some of mandatory headers are missing
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class._check_mandatory_params(
|
||||
params, ['keyA', 'KeyD'])
|
||||
|
||||
def test_validate_storlet_registration_java(self):
|
||||
# correct name and headers w/ dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# correct name and headers w/o dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# some header keys are missing
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong name
|
||||
obj = 'storlet.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_storlet_registration_python(self):
|
||||
# correct name and headers w/ dependency
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '3.6',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong version
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '1.7',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# py2 is no more supported
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '2.7',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong name
|
||||
obj = 'storlet.pyfoo'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong main class
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'another_storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.foo.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_storlet_registration_not_suppoeted(self):
|
||||
# unsupported language
|
||||
obj = 'storlet.foo'
|
||||
params = {'Language': 'bar',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# same name for storlet and dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'storlet-1.0.jar',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
# duplicated name in dependencies
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file,dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_dependency_registration(self):
|
||||
# w/o dependency parameter
|
||||
obj = 'dep_file'
|
||||
params = {'Dependency-Version': '1.0'}
|
||||
self.gateway_class.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ correct dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': '755',
|
||||
'Dependency-Version': '1.0'}
|
||||
self.gateway_class.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ wrong dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': '400',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ invalid dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': 'foo',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_dependency_registration(params, obj)
|
||||
params = {
|
||||
'Dependency-Permissions': '888',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
self.gateway_class.validate_dependency_registration(params, obj)
|
||||
|
||||
def _test_invocation_flow(self, extra_sources=None):
|
||||
extra_sources = extra_sources or []
|
||||
options = {'generate_log': False,
|
||||
'scope': 'AUTH_account',
|
||||
'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
|
||||
st_req = ContainerStorletRequest(
|
||||
storlet_id=self.sobj,
|
||||
params={},
|
||||
user_metadata={},
|
||||
data_iter=iter('body'), options=options)
|
||||
|
||||
# TODO(kota_): need more efficient way for emuration of return value
|
||||
# from SDaemon
|
||||
value_generator = iter([
|
||||
# first, we get metadata json
|
||||
json.dumps({'metadata': 'return'}),
|
||||
# then we get object data
|
||||
'something', '',
|
||||
])
|
||||
|
||||
def mock_read(fd, size):
|
||||
try:
|
||||
value = next(value_generator)
|
||||
except StopIteration:
|
||||
raise Exception('called more then expected')
|
||||
# NOTE(takashi): Make sure that we return bytes in PY3
|
||||
return value.encode('utf-8')
|
||||
|
||||
def mock_close(fd):
|
||||
pass
|
||||
|
||||
called_fd_and_bodies = []
|
||||
invocation_protocol = \
|
||||
'storlets.gateway.gateways.container.runtime.' \
|
||||
'StorletInvocationProtocol._write_input_data'
|
||||
|
||||
def mock_writer(self, fd, app_iter):
|
||||
body = ''
|
||||
for chunk in app_iter:
|
||||
body += chunk
|
||||
called_fd_and_bodies.append((fd, body))
|
||||
|
||||
# prepare nested mock patch
|
||||
# SBus -> mock SBus.send() for container communication
|
||||
# os.read -> mock reading the file descriptor from container
|
||||
# select.select -> mock fd communication which can be readable
|
||||
@mock.patch('storlets.gateway.gateways.container.runtime.SBusClient')
|
||||
@mock.patch('storlets.gateway.gateways.container.runtime.os.read',
|
||||
mock_read)
|
||||
@mock.patch('storlets.gateway.gateways.container.runtime.os.close',
|
||||
mock_close)
|
||||
@mock.patch('storlets.gateway.gateways.container.runtime.select.'
|
||||
'select',
|
||||
lambda r, w, x, timeout=None: (r, w, x))
|
||||
@mock.patch('storlets.gateway.common.stob.os.read', mock_read)
|
||||
@mock.patch(invocation_protocol, mock_writer)
|
||||
def test_invocation_flow(client):
|
||||
client.ping.return_value = SBusResponse(True, 'OK')
|
||||
client.stop_daemon.return_value = SBusResponse(True, 'OK')
|
||||
client.start_daemon.return_value = SBusResponse(True, 'OK')
|
||||
client.execute.return_value = SBusResponse(True, 'OK', 'someid')
|
||||
|
||||
sresp = self.gateway.invocation_flow(st_req, extra_sources)
|
||||
eventlet.sleep(0.1)
|
||||
file_like = FileLikeIter(sresp.data_iter)
|
||||
self.assertEqual(b'something', file_like.read())
|
||||
|
||||
# I hate the decorator to return an instance but to track current
|
||||
# implementation, we have to make a mock class for this. Need to fix.
|
||||
|
||||
class MockFileManager(object):
|
||||
def get_storlet(self, req):
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
def get_dependency(self, req):
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
st_req.file_manager = MockFileManager()
|
||||
|
||||
test_invocation_flow()
|
||||
|
||||
# ensure st_req.app_iter is drawn
|
||||
self.assertRaises(StopIteration, next, st_req.data_iter)
|
||||
expected_mock_writer_calls = len(extra_sources) + 1
|
||||
self.assertEqual(expected_mock_writer_calls,
|
||||
len(called_fd_and_bodies))
|
||||
self.assertEqual('body', called_fd_and_bodies[0][1])
|
||||
return called_fd_and_bodies
|
||||
|
||||
def test_invocation_flow(self):
|
||||
self._test_invocation_flow()
|
||||
|
||||
def test_invocation_flow_with_extra_sources(self):
|
||||
options = {'generate_log': False,
|
||||
'scope': 'AUTH_account',
|
||||
'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
|
||||
data_sources = []
|
||||
|
||||
def generate_extra_st_request():
|
||||
# This works similarly with build_storlet_request
|
||||
# TODO(kota_): think of more generarl way w/o
|
||||
# build_storlet_request
|
||||
sw_req = Request.blank(
|
||||
self.req_path, environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'X-Run-Storlet': self.sobj})
|
||||
|
||||
sw_resp = Response(
|
||||
app_iter=iter(['This is a response body']), status=200)
|
||||
|
||||
st_req = ContainerStorletRequest(
|
||||
storlet_id=sw_req.headers['X-Run-Storlet'],
|
||||
params=sw_req.params,
|
||||
user_metadata={},
|
||||
data_iter=sw_resp.app_iter, options=options)
|
||||
data_sources.append(sw_resp.app_iter)
|
||||
return st_req
|
||||
|
||||
extra_request = generate_extra_st_request()
|
||||
mock_calls = self._test_invocation_flow(
|
||||
extra_sources=[extra_request])
|
||||
self.assertEqual('This is a response body', mock_calls[1][1])
|
||||
|
||||
# run all existing eventlet threads
|
||||
for app_iter in data_sources:
|
||||
# ensure all app_iters are drawn
|
||||
self.assertRaises(StopIteration, next, app_iter)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,411 @@
|
|||
# Copyright (c) 2010-2015 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from contextlib import contextmanager
|
||||
import errno
|
||||
from io import StringIO
|
||||
import os
|
||||
from stat import ST_MODE
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from storlets.sbus.client import SBusResponse
|
||||
from storlets.sbus.client.exceptions import SBusClientIOError
|
||||
from storlets.gateway.common.exceptions import StorletRuntimeException, \
|
||||
StorletTimeout
|
||||
from storlets.gateway.gateways.container.gateway import ContainerStorletRequest
|
||||
from storlets.gateway.gateways.container.runtime import \
|
||||
RunTimePaths, StorletInvocationProtocol
|
||||
from tests.unit import FakeLogger, with_tempdir
|
||||
from tests.unit.gateway.gateways import FakeFileManager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_os_pipe(bufs):
|
||||
class FakeFd(object):
|
||||
def __init__(self, rbuf=''):
|
||||
self.rbuf = rbuf.encode('utf-8')
|
||||
self.closed = False
|
||||
|
||||
def read(self, size):
|
||||
size = min(len(self.rbuf), size)
|
||||
ret = self.rbuf[:size]
|
||||
self.rbuf = self.rbuf[size:]
|
||||
return ret
|
||||
|
||||
def close(self):
|
||||
if self.closed:
|
||||
raise OSError(errno.EBADF, os.strerror(errno.EBADF))
|
||||
self.closed = True
|
||||
|
||||
def fake_os_read(fd, size):
|
||||
return fd.read(size)
|
||||
|
||||
def fake_os_close(fd):
|
||||
fd.close()
|
||||
|
||||
pipes = [(FakeFd(buf), FakeFd()) for buf in bufs]
|
||||
pipe_generator = iter(pipes)
|
||||
|
||||
def mock_os_pipe():
|
||||
try:
|
||||
return next(pipe_generator)
|
||||
except StopIteration:
|
||||
raise AssertionError('pipe called more than expected')
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.os.pipe',
|
||||
mock_os_pipe), \
|
||||
mock.patch('storlets.gateway.gateways.container.runtime.os.read',
|
||||
fake_os_read) as fake_os_read,\
|
||||
mock.patch('storlets.gateway.gateways.container.runtime.os.close',
|
||||
fake_os_close) as fake_os_close:
|
||||
yield pipes
|
||||
|
||||
|
||||
class TestRuntimePaths(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.scope = '0123456789abc'
|
||||
self._initialize()
|
||||
|
||||
def _initialize(self):
|
||||
# TODO(takashi): take these values from config file
|
||||
base_dir = '/var/lib/storlets'
|
||||
self.script_dir = os.path.join(base_dir, 'scripts')
|
||||
self.pipes_dir = os.path.join(base_dir, 'pipes', 'scopes')
|
||||
self.storlets_dir = os.path.join(base_dir, 'storlets', 'scopes')
|
||||
self.log_dir = os.path.join(base_dir, 'logs', 'scopes')
|
||||
self.cache_dir = os.path.join(base_dir, 'cache', 'scopes')
|
||||
|
||||
self.conf = {}
|
||||
self.storlet_id = 'org.openstack.storlet.mystorlet'
|
||||
self.paths = RunTimePaths(self.scope, self.conf)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_host_pipe_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.host_pipe_dir)
|
||||
|
||||
def test_create_host_pipe_dir(self):
|
||||
pipedir = self.paths.host_pipe_dir
|
||||
|
||||
# When the directory exists
|
||||
with mock.patch('os.path.exists', return_value=True), \
|
||||
mock.patch('os.makedirs') as m, \
|
||||
mock.patch('os.chmod') as c:
|
||||
self.assertEqual(os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.create_host_pipe_dir())
|
||||
self.assertEqual(0, m.call_count)
|
||||
cargs, ckwargs = c.call_args
|
||||
# Make sure about the target directory
|
||||
self.assertEqual(cargs[0], pipedir)
|
||||
|
||||
# When the directory does not exist
|
||||
with mock.patch('os.path.exists', return_value=False), \
|
||||
mock.patch('os.makedirs') as m, \
|
||||
mock.patch('os.chmod') as c:
|
||||
self.assertEqual(os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.create_host_pipe_dir())
|
||||
self.assertEqual(1, m.call_count)
|
||||
# Make sure about the target directory
|
||||
margs, mkwargs = m.call_args
|
||||
self.assertEqual(margs[0], pipedir)
|
||||
cargs, ckwargs = c.call_args
|
||||
self.assertEqual(cargs[0], pipedir)
|
||||
|
||||
def test_host_factory_pipe(self):
|
||||
self.assertEqual(
|
||||
self.paths.host_factory_pipe,
|
||||
os.path.join(self.pipes_dir, self.scope, 'factory_pipe'))
|
||||
|
||||
def test_get_host_storlet_pipe(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.pipes_dir, self.scope, self.storlet_id),
|
||||
self.paths.get_host_storlet_pipe(self.storlet_id))
|
||||
|
||||
def test_get_sbox_storlet_pipe(self):
|
||||
self.assertEqual(
|
||||
os.path.join('/mnt/channels', self.storlet_id),
|
||||
self.paths.get_sbox_storlet_pipe(self.storlet_id))
|
||||
|
||||
def test_get_sbox_storlet_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join('/home/swift', self.storlet_id),
|
||||
self.paths.get_sbox_storlet_dir(self.storlet_id))
|
||||
|
||||
def test_host_storlet_base_dir(self):
|
||||
self.assertEqual(
|
||||
self.paths.host_storlet_base_dir,
|
||||
os.path.join(self.storlets_dir, self.scope))
|
||||
|
||||
def test_get_host_storlet_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.storlets_dir, self.scope, self.storlet_id),
|
||||
self.paths.get_host_storlet_dir(self.storlet_id))
|
||||
|
||||
def test_get_host_slog_path(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.log_dir, self.scope, self.storlet_id,
|
||||
'storlet_invoke.log'),
|
||||
self.paths.get_host_slog_path(self.storlet_id))
|
||||
|
||||
def test_host_storlet_cache_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.cache_dir, self.scope, 'storlet'),
|
||||
self.paths.host_storlet_cache_dir)
|
||||
|
||||
def test_host_dependency_cache_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.cache_dir, self.scope, 'dependency'),
|
||||
self.paths.host_dependency_cache_dir)
|
||||
|
||||
def test_runtime_paths_default(self):
|
||||
# CHECK: docs says we need 4 dirs for communicate
|
||||
# ====================================================================
|
||||
# |1| host_factory_pipe_path | <pipes_dir>/<scope>/factory_pipe |
|
||||
# ====================================================================
|
||||
# |2| host_storlet_pipe_path | <pipes_dir>/<scope>/<storlet_id> |
|
||||
# ====================================================================
|
||||
# |3| sandbox_factory_pipe_path | /mnt/channels/factory_pipe |
|
||||
# ====================================================================
|
||||
# |4| sandbox_storlet_pipe_path | /mnt/channels/<storlet_id> |
|
||||
# ====================================================================
|
||||
#
|
||||
# With this test, the scope value is "account" and the storlet_id is
|
||||
# "Storlet-1.0.jar" (app name?)
|
||||
# ok, let's check for these values
|
||||
|
||||
runtime_paths = RunTimePaths('account', {})
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
|
||||
# For pipe
|
||||
self.assertEqual('/var/lib/storlets/pipes/scopes/account',
|
||||
runtime_paths.host_pipe_dir)
|
||||
|
||||
# 1. host_factory_pipe_path <pipes_dir>/<scope>/factory_pipe
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/pipes/scopes/account/factory_pipe',
|
||||
runtime_paths.host_factory_pipe)
|
||||
# 2. host_storlet_pipe_path <pipes_dir>/<scope>/<storlet_id>
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/pipes/scopes/account/Storlet-1.0.jar',
|
||||
runtime_paths.get_host_storlet_pipe(storlet_id))
|
||||
# 3. Yes, right now, we don't have the path for #3 in Python
|
||||
# 4. sandbox_storlet_pipe_path | /mnt/channels/<storlet_id>
|
||||
self.assertEqual('/mnt/channels/Storlet-1.0.jar',
|
||||
runtime_paths.get_sbox_storlet_pipe(storlet_id))
|
||||
|
||||
# This looks like for jar load?
|
||||
self.assertEqual('/var/lib/storlets/storlets/scopes/account',
|
||||
runtime_paths.host_storlet_base_dir)
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/storlets/scopes/account/Storlet-1.0.jar',
|
||||
runtime_paths.get_host_storlet_dir(storlet_id))
|
||||
# And this one is a mount point in sand box?
|
||||
self.assertEqual('/home/swift/Storlet-1.0.jar',
|
||||
runtime_paths.get_sbox_storlet_dir(storlet_id))
|
||||
|
||||
@with_tempdir
|
||||
def test_create_host_pipe_dir_with_real_dir(self, temp_dir):
|
||||
runtime_paths = RunTimePaths('account', {'host_root': temp_dir})
|
||||
runtime_paths.create_host_pipe_dir()
|
||||
path = runtime_paths.host_pipe_dir
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertTrue(os.path.isdir(path))
|
||||
permission = oct(os.stat(path)[ST_MODE])[-3:]
|
||||
# TODO(kota_): make sure if this is really acceptable
|
||||
self.assertEqual('777', permission)
|
||||
|
||||
|
||||
class TestRuntimePathsTempauth(TestRuntimePaths):
|
||||
def setUp(self):
|
||||
self.scope = 'test'
|
||||
self._initialize()
|
||||
|
||||
|
||||
class TestStorletInvocationProtocol(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.pipe_path = tempfile.mktemp()
|
||||
self.log_file = tempfile.mktemp()
|
||||
self.logger = FakeLogger()
|
||||
self.storlet_id = 'Storlet-1.0.jar'
|
||||
self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
storlet_request = ContainerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
self.protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
|
||||
def tearDown(self):
|
||||
for path in [self.pipe_path, self.log_file]:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_send_execute_command(self):
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.execute') as execute:
|
||||
execute.return_value = SBusResponse(True, 'OK', 'someid')
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
self.assertEqual('someid', self.protocol.task_id)
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.execute') as execute:
|
||||
execute.return_value = SBusResponse(True, 'OK')
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.execute') as execute:
|
||||
execute.return_value = SBusResponse(False, 'NG', 'someid')
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.execute') as execute:
|
||||
execute.side_effect = SBusClientIOError()
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
def test_invocation_protocol(self):
|
||||
# os.pipe will be called 3 times
|
||||
pipe_called = 3
|
||||
|
||||
with _mock_os_pipe([''] * pipe_called) as pipes:
|
||||
with mock.patch.object(self.protocol,
|
||||
'_wait_for_read_with_timeout'), \
|
||||
mock.patch.object(self.protocol, '_send_execute_command'):
|
||||
self.protocol._invoke()
|
||||
|
||||
self.assertEqual(pipe_called, len(pipes))
|
||||
pipes = iter(pipes)
|
||||
|
||||
# data write is not directly closed
|
||||
# data read is closed
|
||||
input_data_read_fd, input_data_write_fd = next(pipes)
|
||||
self.assertTrue(input_data_read_fd.closed)
|
||||
self.assertFalse(input_data_write_fd.closed)
|
||||
|
||||
# data write is closed but data read is still open
|
||||
data_read_fd, data_write_fd = next(pipes)
|
||||
self.assertFalse(data_read_fd.closed)
|
||||
self.assertTrue(data_write_fd.closed)
|
||||
|
||||
# metadata write fd is closed, metadata read fd is still open.
|
||||
metadata_read_fd, metadata_write_fd = next(pipes)
|
||||
self.assertFalse(metadata_read_fd.closed)
|
||||
self.assertTrue(metadata_write_fd.closed)
|
||||
|
||||
# sanity
|
||||
self.assertRaises(StopIteration, next, pipes)
|
||||
|
||||
def test_invocation_protocol_remote_fds(self):
|
||||
# In default, we have 4 fds in remote_fds
|
||||
storlet_request = ContainerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(4, len(protocol.remote_fds))
|
||||
|
||||
# extra_resources expands the remote_fds
|
||||
storlet_request = ContainerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
|
||||
extra_sources=[storlet_request])
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(5, len(protocol.remote_fds))
|
||||
|
||||
# 2 more extra_resources expands the remote_fds
|
||||
storlet_request = ContainerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
|
||||
extra_sources=[storlet_request] * 3)
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(7, len(protocol.remote_fds))
|
||||
|
||||
def test_open_writer_with_invalid_fd(self):
|
||||
invalid_fds = (
|
||||
(None, TypeError), (-1, ValueError), ('blah', TypeError))
|
||||
|
||||
for invalid_fd, expected_error in invalid_fds:
|
||||
with self.assertRaises(expected_error):
|
||||
with self.protocol._open_writer(invalid_fd):
|
||||
pass
|
||||
|
||||
def _test_writer_with_exception(self, exception_cls):
|
||||
pipes = [os.pipe()]
|
||||
|
||||
def raise_in_the_context():
|
||||
with self.protocol._open_writer(pipes[0][1]):
|
||||
raise exception_cls()
|
||||
try:
|
||||
# writer context doesn't suppress any exception
|
||||
self.assertRaises(exception_cls, raise_in_the_context)
|
||||
|
||||
# since _open_writer closes the write fd, the os.close will fail as
|
||||
# BadFileDescriptor
|
||||
with self.assertRaises(OSError) as os_error:
|
||||
os.close(pipes[0][1])
|
||||
self.assertEqual(9, os_error.exception.errno)
|
||||
|
||||
finally:
|
||||
for fd in pipes[0]:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_writer_raise_while_in_writer_context(self):
|
||||
# basic storlet timeout
|
||||
self._test_writer_with_exception(StorletTimeout)
|
||||
# unexpected IOError
|
||||
self._test_writer_with_exception(IOError)
|
||||
# else
|
||||
self._test_writer_with_exception(Exception)
|
||||
|
||||
|
||||
class TestStorletInvocationProtocolPython(TestStorletInvocationProtocol):
|
||||
def setUp(self):
|
||||
self.pipe_path = tempfile.mktemp()
|
||||
self.log_file = tempfile.mktemp()
|
||||
self.logger = FakeLogger()
|
||||
self.storlet_id = 'Storlet-1.0.py'
|
||||
self.options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'python',
|
||||
'language_version': '3.6',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
storlet_request = ContainerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
self.protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -13,554 +13,17 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import eventlet
|
||||
from io import BytesIO, StringIO
|
||||
import json
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from swift.common.swob import Request, Response
|
||||
from swift.common.utils import FileLikeIter
|
||||
from storlets.gateway.gateways.docker.gateway import \
|
||||
DockerStorletGateway
|
||||
from tests.unit.gateway.gateways.container.test_gateway import \
|
||||
ContainerGatewayTestMixin
|
||||
|
||||
from storlets.sbus.client import SBusResponse
|
||||
|
||||
from tests.unit import FakeLogger
|
||||
from tests.unit.gateway.gateways import FakeFileManager
|
||||
from storlets.gateway.gateways.docker.gateway import DockerStorletRequest, \
|
||||
StorletGatewayDocker
|
||||
class TestStorletDockerGateway(ContainerGatewayTestMixin, unittest.TestCase):
|
||||
|
||||
|
||||
class MockInternalClient(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
class TestDockerStorletRequest(unittest.TestCase):
|
||||
|
||||
def test_init(self):
|
||||
# Java
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {'Param1': 'Value1', 'Param2': 'Value2'}
|
||||
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
|
||||
|
||||
# with dependencies
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# without dependencies
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# storlet_language is not given
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
with self.assertRaises(ValueError):
|
||||
DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# storlet_main is not given
|
||||
options = {'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
with self.assertRaises(ValueError):
|
||||
DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# file_manager is not given
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_language': 'java'}
|
||||
with self.assertRaises(ValueError):
|
||||
DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
|
||||
# Python
|
||||
storlet_id = 'storlet.py'
|
||||
params = {'Param1': 'Value1', 'Param2': 'Value2'}
|
||||
metadata = {'MetaKey1': 'MetaValue1', 'MetaKey2': 'MetaValue2'}
|
||||
|
||||
# without language version
|
||||
options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_language': 'python',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('storlet.py', dsreq.storlet_id)
|
||||
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('python', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
|
||||
# with language version
|
||||
options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_language': 'python',
|
||||
'storlet_language_version': '3.6',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
iter(StringIO()), options=options)
|
||||
self.assertEqual(metadata, dsreq.user_metadata)
|
||||
self.assertEqual(params, dsreq.params)
|
||||
self.assertEqual('storlet.py', dsreq.storlet_id)
|
||||
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual([], dsreq.dependencies)
|
||||
self.assertEqual('python', dsreq.storlet_language)
|
||||
self.assertEqual('3.6', dsreq.storlet_language_version)
|
||||
|
||||
def test_init_with_range(self):
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {}
|
||||
metadata = {}
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 1,
|
||||
'range_end': 6}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
self.assertEqual(1, dsreq.start)
|
||||
self.assertEqual(6, dsreq.end)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 0,
|
||||
'range_end': 0}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
|
||||
self.assertEqual('Storlet-1.0.jar', dsreq.storlet_id)
|
||||
self.assertEqual('org.openstack.storlet.Storlet', dsreq.storlet_main)
|
||||
self.assertEqual(['dep1', 'dep2'], dsreq.dependencies)
|
||||
self.assertEqual('java', dsreq.storlet_language)
|
||||
self.assertIsNone(dsreq.storlet_language_version)
|
||||
self.assertEqual(0, dsreq.start)
|
||||
self.assertEqual(0, dsreq.end)
|
||||
|
||||
def test_has_range(self):
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
params = {}
|
||||
metadata = {}
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertFalse(dsreq.has_range)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 1,
|
||||
'range_end': 6}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertTrue(dsreq.has_range)
|
||||
|
||||
options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep'),
|
||||
'range_start': 0,
|
||||
'range_end': 6}
|
||||
dsreq = DockerStorletRequest(storlet_id, params, metadata,
|
||||
None, 0, options=options)
|
||||
self.assertTrue(dsreq.has_range)
|
||||
|
||||
|
||||
class TestStorletDockerGateway(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# TODO(takashi): take these values from config file
|
||||
self.tempdir = mkdtemp()
|
||||
self.sconf = {
|
||||
'host_root': self.tempdir,
|
||||
'swift_dir': self.tempdir,
|
||||
'storlet_timeout': '9',
|
||||
'storlet_container': 'storlet',
|
||||
'storlet_dependency': 'dependency',
|
||||
'reseller_prefix': 'AUTH'
|
||||
}
|
||||
self.logger = FakeLogger()
|
||||
|
||||
self.storlet_container = self.sconf['storlet_container']
|
||||
self.storlet_dependency = self.sconf['storlet_dependency']
|
||||
|
||||
self.version = 'v1'
|
||||
self.account = 'AUTH_account'
|
||||
self.container = 'container'
|
||||
self.obj = 'object'
|
||||
self.sobj = 'storlet-1.0.jar'
|
||||
|
||||
self.gateway = StorletGatewayDocker(
|
||||
self.sconf, self.logger, self.account)
|
||||
|
||||
def tearDown(self):
|
||||
rmtree(self.tempdir)
|
||||
|
||||
@property
|
||||
def req_path(self):
|
||||
return self._create_proxy_path(
|
||||
self.version, self.account, self.container,
|
||||
self.obj)
|
||||
|
||||
@property
|
||||
def storlet_path(self):
|
||||
return self._create_proxy_path(
|
||||
self.version, self.account, self.storlet_container,
|
||||
self.sobj)
|
||||
|
||||
def _create_proxy_path(self, version, account, container, obj):
|
||||
return '/'.join(['', version, account, container, obj])
|
||||
|
||||
def test_check_mandatory_params(self):
|
||||
params = {'keyA': 'valueA',
|
||||
'keyB': 'valueB',
|
||||
'keyC': 'valueC'}
|
||||
|
||||
# all mandatory headers are included
|
||||
StorletGatewayDocker._check_mandatory_params(
|
||||
params, ['keyA', 'keyB'])
|
||||
|
||||
# some of mandatory headers are missing
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker._check_mandatory_params(
|
||||
params, ['keyA', 'KeyD'])
|
||||
|
||||
def test_validate_storlet_registration_java(self):
|
||||
# correct name and headers w/ dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# correct name and headers w/o dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# some header keys are missing
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong name
|
||||
obj = 'storlet.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_storlet_registration_python(self):
|
||||
# correct name and headers w/ dependency
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '3.6',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong version
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '1.7',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# py2 is no more supported
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Language-Version': '2.7',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong name
|
||||
obj = 'storlet.pyfoo'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# wrong main class
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'another_storlet.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
obj = 'storlet.py'
|
||||
params = {'Language': 'python',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'storlet.foo.Storlet'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_storlet_registration_not_suppoeted(self):
|
||||
# unsupported language
|
||||
obj = 'storlet.foo'
|
||||
params = {'Language': 'bar',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# same name for storlet and dependency
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'storlet-1.0.jar',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
# duplicated name in dependencies
|
||||
obj = 'storlet-1.0.jar'
|
||||
params = {'Language': 'java',
|
||||
'Interface-Version': '1.0',
|
||||
'Dependency': 'dep_file,dep_file',
|
||||
'Object-Metadata': 'no',
|
||||
'Main': 'path.to.storlet.class'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_storlet_registration(params, obj)
|
||||
|
||||
def test_validate_dependency_registration(self):
|
||||
# w/o dependency parameter
|
||||
obj = 'dep_file'
|
||||
params = {'Dependency-Version': '1.0'}
|
||||
StorletGatewayDocker.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ correct dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': '755',
|
||||
'Dependency-Version': '1.0'}
|
||||
StorletGatewayDocker.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ wrong dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': '400',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_dependency_registration(params, obj)
|
||||
|
||||
# w/ invalid dependency parameter
|
||||
params = {
|
||||
'Dependency-Permissions': 'foo',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_dependency_registration(params, obj)
|
||||
params = {
|
||||
'Dependency-Permissions': '888',
|
||||
'Dependency-Version': '1.0'}
|
||||
with self.assertRaises(ValueError):
|
||||
StorletGatewayDocker.validate_dependency_registration(params, obj)
|
||||
|
||||
def _test_docker_gateway_communicate(self, extra_sources=None):
|
||||
extra_sources = extra_sources or []
|
||||
options = {'generate_log': False,
|
||||
'scope': 'AUTH_account',
|
||||
'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
|
||||
st_req = DockerStorletRequest(
|
||||
storlet_id=self.sobj,
|
||||
params={},
|
||||
user_metadata={},
|
||||
data_iter=iter('body'), options=options)
|
||||
|
||||
# TODO(kota_): need more efficient way for emuration of return value
|
||||
# from SDaemon
|
||||
value_generator = iter([
|
||||
# first, we get metadata json
|
||||
json.dumps({'metadata': 'return'}),
|
||||
# then we get object data
|
||||
'something', '',
|
||||
])
|
||||
|
||||
def mock_read(fd, size):
|
||||
try:
|
||||
value = next(value_generator)
|
||||
except StopIteration:
|
||||
raise Exception('called more then expected')
|
||||
# NOTE(takashi): Make sure that we return bytes in PY3
|
||||
return value.encode('utf-8')
|
||||
|
||||
def mock_close(fd):
|
||||
pass
|
||||
|
||||
called_fd_and_bodies = []
|
||||
invocation_protocol = \
|
||||
'storlets.gateway.gateways.docker.runtime.' \
|
||||
'StorletInvocationProtocol._write_input_data'
|
||||
|
||||
def mock_writer(self, fd, app_iter):
|
||||
body = ''
|
||||
for chunk in app_iter:
|
||||
body += chunk
|
||||
called_fd_and_bodies.append((fd, body))
|
||||
|
||||
# prepare nested mock patch
|
||||
# SBus -> mock SBus.send() for container communication
|
||||
# os.read -> mock reading the file descriptor from container
|
||||
# select.select -> mock fd communication which can be readable
|
||||
@mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient')
|
||||
@mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
|
||||
mock_read)
|
||||
@mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
|
||||
mock_close)
|
||||
@mock.patch('storlets.gateway.gateways.docker.runtime.select.select',
|
||||
lambda r, w, x, timeout=None: (r, w, x))
|
||||
@mock.patch('storlets.gateway.common.stob.os.read', mock_read)
|
||||
@mock.patch(invocation_protocol, mock_writer)
|
||||
def test_invocation_flow(client):
|
||||
client.ping.return_value = SBusResponse(True, 'OK')
|
||||
client.stop_daemon.return_value = SBusResponse(True, 'OK')
|
||||
client.start_daemon.return_value = SBusResponse(True, 'OK')
|
||||
client.execute.return_value = SBusResponse(True, 'OK', 'someid')
|
||||
|
||||
sresp = self.gateway.invocation_flow(st_req, extra_sources)
|
||||
eventlet.sleep(0.1)
|
||||
file_like = FileLikeIter(sresp.data_iter)
|
||||
self.assertEqual(b'something', file_like.read())
|
||||
|
||||
# I hate the decorator to return an instance but to track current
|
||||
# implementation, we have to make a mock class for this. Need to fix.
|
||||
|
||||
class MockFileManager(object):
|
||||
def get_storlet(self, req):
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
def get_dependency(self, req):
|
||||
return BytesIO(b'mock'), None
|
||||
|
||||
st_req.file_manager = MockFileManager()
|
||||
|
||||
test_invocation_flow()
|
||||
|
||||
# ensure st_req.app_iter is drawn
|
||||
self.assertRaises(StopIteration, next, st_req.data_iter)
|
||||
expected_mock_writer_calls = len(extra_sources) + 1
|
||||
self.assertEqual(expected_mock_writer_calls,
|
||||
len(called_fd_and_bodies))
|
||||
self.assertEqual('body', called_fd_and_bodies[0][1])
|
||||
return called_fd_and_bodies
|
||||
|
||||
def test_docker_gateway_communicate(self):
|
||||
self._test_docker_gateway_communicate()
|
||||
|
||||
def test_docker_gateway_communicate_with_extra_sources(self):
|
||||
options = {'generate_log': False,
|
||||
'scope': 'AUTH_account',
|
||||
'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
|
||||
data_sources = []
|
||||
|
||||
def generate_extra_st_request():
|
||||
# This works similarly with build_storlet_request
|
||||
# TODO(kota_): think of more generarl way w/o
|
||||
# build_storlet_request
|
||||
sw_req = Request.blank(
|
||||
self.req_path, environ={'REQUEST_METHOD': 'GET'},
|
||||
headers={'X-Run-Storlet': self.sobj})
|
||||
|
||||
sw_resp = Response(
|
||||
app_iter=iter(['This is a response body']), status=200)
|
||||
|
||||
st_req = DockerStorletRequest(
|
||||
storlet_id=sw_req.headers['X-Run-Storlet'],
|
||||
params=sw_req.params,
|
||||
user_metadata={},
|
||||
data_iter=sw_resp.app_iter, options=options)
|
||||
data_sources.append(sw_resp.app_iter)
|
||||
return st_req
|
||||
|
||||
extra_request = generate_extra_st_request()
|
||||
mock_calls = self._test_docker_gateway_communicate(
|
||||
extra_sources=[extra_request])
|
||||
self.assertEqual('This is a response body', mock_calls[1][1])
|
||||
|
||||
# run all existing eventlet threads
|
||||
for app_iter in data_sources:
|
||||
# ensure all app_iters are drawn
|
||||
self.assertRaises(StopIteration, next, app_iter)
|
||||
gateway_class = DockerStorletGateway
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -13,12 +13,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from contextlib import contextmanager
|
||||
import errno
|
||||
from io import StringIO
|
||||
import os
|
||||
from stat import ST_MODE
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -31,266 +25,57 @@ from storlets.sbus.client.exceptions import SBusClientIOError, \
|
|||
SBusClientMalformedResponse, SBusClientSendError
|
||||
from storlets.gateway.common.exceptions import StorletRuntimeException, \
|
||||
StorletTimeout
|
||||
from storlets.gateway.gateways.docker.gateway import DockerStorletRequest
|
||||
from storlets.gateway.gateways.docker.runtime import RunTimeSandbox, \
|
||||
RunTimePaths, StorletInvocationProtocol
|
||||
from tests.unit import FakeLogger, with_tempdir
|
||||
from tests.unit.gateway.gateways import FakeFileManager
|
||||
from storlets.gateway.gateways.docker.runtime import DockerRunTimeSandbox
|
||||
from tests.unit import FakeLogger
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_os_pipe(bufs):
|
||||
class FakeFd(object):
|
||||
def __init__(self, rbuf=''):
|
||||
self.rbuf = rbuf.encode('utf-8')
|
||||
self.closed = False
|
||||
|
||||
def read(self, size):
|
||||
size = min(len(self.rbuf), size)
|
||||
ret = self.rbuf[:size]
|
||||
self.rbuf = self.rbuf[size:]
|
||||
return ret
|
||||
|
||||
def close(self):
|
||||
if self.closed:
|
||||
raise OSError(errno.EBADF, os.strerror(errno.EBADF))
|
||||
self.closed = True
|
||||
|
||||
def fake_os_read(fd, size):
|
||||
return fd.read(size)
|
||||
|
||||
def fake_os_close(fd):
|
||||
fd.close()
|
||||
|
||||
pipes = [(FakeFd(buf), FakeFd()) for buf in bufs]
|
||||
pipe_generator = iter(pipes)
|
||||
|
||||
def mock_os_pipe():
|
||||
try:
|
||||
return next(pipe_generator)
|
||||
except StopIteration:
|
||||
raise AssertionError('pipe called more than expected')
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.os.pipe',
|
||||
mock_os_pipe), \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
|
||||
fake_os_read) as fake_os_read,\
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
|
||||
fake_os_close) as fake_os_close:
|
||||
yield pipes
|
||||
|
||||
|
||||
class TestRuntimePaths(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.scope = '0123456789abc'
|
||||
self._initialize()
|
||||
|
||||
def _initialize(self):
|
||||
# TODO(takashi): take these values from config file
|
||||
base_dir = '/var/lib/storlets'
|
||||
self.script_dir = os.path.join(base_dir, 'scripts')
|
||||
self.pipes_dir = os.path.join(base_dir, 'pipes', 'scopes')
|
||||
self.storlets_dir = os.path.join(base_dir, 'storlets', 'scopes')
|
||||
self.log_dir = os.path.join(base_dir, 'logs', 'scopes')
|
||||
self.cache_dir = os.path.join(base_dir, 'cache', 'scopes')
|
||||
|
||||
self.conf = {}
|
||||
self.storlet_id = 'org.openstack.storlet.mystorlet'
|
||||
self.paths = RunTimePaths(self.scope, self.conf)
|
||||
|
||||
def tearDown(self):
|
||||
pass
|
||||
|
||||
def test_host_pipe_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.host_pipe_dir)
|
||||
|
||||
def test_create_host_pipe_dir(self):
|
||||
pipedir = self.paths.host_pipe_dir
|
||||
|
||||
# When the directory exists
|
||||
with mock.patch('os.path.exists', return_value=True), \
|
||||
mock.patch('os.makedirs') as m, \
|
||||
mock.patch('os.chmod') as c:
|
||||
self.assertEqual(os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.create_host_pipe_dir())
|
||||
self.assertEqual(0, m.call_count)
|
||||
cargs, ckwargs = c.call_args
|
||||
# Make sure about the target directory
|
||||
self.assertEqual(cargs[0], pipedir)
|
||||
|
||||
# When the directory does not exist
|
||||
with mock.patch('os.path.exists', return_value=False), \
|
||||
mock.patch('os.makedirs') as m, \
|
||||
mock.patch('os.chmod') as c:
|
||||
self.assertEqual(os.path.join(self.pipes_dir, self.scope),
|
||||
self.paths.create_host_pipe_dir())
|
||||
self.assertEqual(1, m.call_count)
|
||||
# Make sure about the target directory
|
||||
margs, mkwargs = m.call_args
|
||||
self.assertEqual(margs[0], pipedir)
|
||||
cargs, ckwargs = c.call_args
|
||||
self.assertEqual(cargs[0], pipedir)
|
||||
|
||||
def test_host_factory_pipe(self):
|
||||
self.assertEqual(
|
||||
self.paths.host_factory_pipe,
|
||||
os.path.join(self.pipes_dir, self.scope, 'factory_pipe'))
|
||||
|
||||
def test_get_host_storlet_pipe(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.pipes_dir, self.scope, self.storlet_id),
|
||||
self.paths.get_host_storlet_pipe(self.storlet_id))
|
||||
|
||||
def test_get_sbox_storlet_pipe(self):
|
||||
self.assertEqual(
|
||||
os.path.join('/mnt/channels', self.storlet_id),
|
||||
self.paths.get_sbox_storlet_pipe(self.storlet_id))
|
||||
|
||||
def test_get_sbox_storlet_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join('/home/swift', self.storlet_id),
|
||||
self.paths.get_sbox_storlet_dir(self.storlet_id))
|
||||
|
||||
def test_host_storlet_base_dir(self):
|
||||
self.assertEqual(
|
||||
self.paths.host_storlet_base_dir,
|
||||
os.path.join(self.storlets_dir, self.scope))
|
||||
|
||||
def test_get_host_storlet_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.storlets_dir, self.scope, self.storlet_id),
|
||||
self.paths.get_host_storlet_dir(self.storlet_id))
|
||||
|
||||
def test_get_host_slog_path(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.log_dir, self.scope, self.storlet_id,
|
||||
'storlet_invoke.log'),
|
||||
self.paths.get_host_slog_path(self.storlet_id))
|
||||
|
||||
def test_host_storlet_cache_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.cache_dir, self.scope, 'storlet'),
|
||||
self.paths.host_storlet_cache_dir)
|
||||
|
||||
def test_host_dependency_cache_dir(self):
|
||||
self.assertEqual(
|
||||
os.path.join(self.cache_dir, self.scope, 'dependency'),
|
||||
self.paths.host_dependency_cache_dir)
|
||||
|
||||
def test_runtime_paths_default(self):
|
||||
# CHECK: docs says we need 4 dirs for communicate
|
||||
# ====================================================================
|
||||
# |1| host_factory_pipe_path | <pipes_dir>/<scope>/factory_pipe |
|
||||
# ====================================================================
|
||||
# |2| host_storlet_pipe_path | <pipes_dir>/<scope>/<storlet_id> |
|
||||
# ====================================================================
|
||||
# |3| sandbox_factory_pipe_path | /mnt/channels/factory_pipe |
|
||||
# ====================================================================
|
||||
# |4| sandbox_storlet_pipe_path | /mnt/channels/<storlet_id> |
|
||||
# ====================================================================
|
||||
#
|
||||
# With this test, the scope value is "account" and the storlet_id is
|
||||
# "Storlet-1.0.jar" (app name?)
|
||||
# ok, let's check for these values
|
||||
|
||||
runtime_paths = RunTimePaths('account', {})
|
||||
storlet_id = 'Storlet-1.0.jar'
|
||||
|
||||
# For pipe
|
||||
self.assertEqual('/var/lib/storlets/pipes/scopes/account',
|
||||
runtime_paths.host_pipe_dir)
|
||||
|
||||
# 1. host_factory_pipe_path <pipes_dir>/<scope>/factory_pipe
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/pipes/scopes/account/factory_pipe',
|
||||
runtime_paths.host_factory_pipe)
|
||||
# 2. host_storlet_pipe_path <pipes_dir>/<scope>/<storlet_id>
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/pipes/scopes/account/Storlet-1.0.jar',
|
||||
runtime_paths.get_host_storlet_pipe(storlet_id))
|
||||
# 3. Yes, right now, we don't have the path for #3 in Python
|
||||
# 4. sandbox_storlet_pipe_path | /mnt/channels/<storlet_id>
|
||||
self.assertEqual('/mnt/channels/Storlet-1.0.jar',
|
||||
runtime_paths.get_sbox_storlet_pipe(storlet_id))
|
||||
|
||||
# This looks like for jar load?
|
||||
self.assertEqual('/var/lib/storlets/storlets/scopes/account',
|
||||
runtime_paths.host_storlet_base_dir)
|
||||
self.assertEqual(
|
||||
'/var/lib/storlets/storlets/scopes/account/Storlet-1.0.jar',
|
||||
runtime_paths.get_host_storlet_dir(storlet_id))
|
||||
# And this one is a mount point in sand box?
|
||||
self.assertEqual('/home/swift/Storlet-1.0.jar',
|
||||
runtime_paths.get_sbox_storlet_dir(storlet_id))
|
||||
|
||||
@with_tempdir
|
||||
def test_create_host_pipe_dir_with_real_dir(self, temp_dir):
|
||||
runtime_paths = RunTimePaths('account', {'host_root': temp_dir})
|
||||
runtime_paths.create_host_pipe_dir()
|
||||
path = runtime_paths.host_pipe_dir
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertTrue(os.path.isdir(path))
|
||||
permission = oct(os.stat(path)[ST_MODE])[-3:]
|
||||
# TODO(kota_): make sure if this is really acceptable
|
||||
self.assertEqual('777', permission)
|
||||
|
||||
|
||||
class TestRuntimePathsTempauth(TestRuntimePaths):
|
||||
def setUp(self):
|
||||
self.scope = 'test'
|
||||
self._initialize()
|
||||
|
||||
|
||||
class TestRunTimeSandbox(unittest.TestCase):
|
||||
class TestDockerRunTimeSandbox(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.logger = FakeLogger()
|
||||
# TODO(takashi): take these values from config file
|
||||
self.conf = {'container_image_namespace': 'localhost:5001',
|
||||
'default_container_image_name': 'defaultimage'}
|
||||
self.scope = '0123456789abc'
|
||||
self.sbox = RunTimeSandbox(self.scope, self.conf, self.logger)
|
||||
self.sbox = DockerRunTimeSandbox(self.scope, self.conf, self.logger)
|
||||
|
||||
def test_ping(self):
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping:
|
||||
ping.return_value = SBusResponse(True, 'OK')
|
||||
self.assertTrue(self.sbox.ping())
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping:
|
||||
ping.return_value = SBusResponse(False, 'Error')
|
||||
self.assertFalse(self.sbox.ping())
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping:
|
||||
ping.side_effect = SBusClientSendError()
|
||||
self.assertFalse(self.sbox.ping())
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping:
|
||||
ping.side_effect = SBusClientMalformedResponse()
|
||||
self.assertFalse(self.sbox.ping())
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping:
|
||||
ping.side_effect = SBusClientIOError()
|
||||
self.assertFalse(self.sbox.ping())
|
||||
|
||||
def test_wait(self):
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'time.sleep') as sleep:
|
||||
ping.return_value = SBusResponse(True, 'OK')
|
||||
self.sbox.wait()
|
||||
self.assertEqual(sleep.call_count, 0)
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'SBusClient.ping') as ping, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'time.sleep') as sleep:
|
||||
ping.side_effect = [SBusResponse(False, 'Error'),
|
||||
SBusResponse(True, 'OK')]
|
||||
|
|
@ -433,24 +218,24 @@ class TestRunTimeSandbox(unittest.TestCase):
|
|||
self.assertEqual(0, mock_containers.run.call_count)
|
||||
|
||||
def test_restart(self):
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox._restart') as _restart, \
|
||||
'DockerRunTimeSandbox._restart') as _restart, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox.wait') as wait:
|
||||
'DockerRunTimeSandbox.wait') as wait:
|
||||
self.sbox.restart()
|
||||
self.assertEqual(1, pipe_dir.call_count)
|
||||
self.assertEqual(1, _restart.call_count)
|
||||
self.assertEqual((self.scope,), _restart.call_args.args)
|
||||
self.assertEqual(1, wait.call_count)
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox._restart') as _restart, \
|
||||
'DockerRunTimeSandbox._restart') as _restart, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox.wait') as wait:
|
||||
'DockerRunTimeSandbox.wait') as wait:
|
||||
_restart.side_effect = [StorletRuntimeException(), None]
|
||||
self.sbox.restart()
|
||||
self.assertEqual(1, pipe_dir.call_count)
|
||||
|
|
@ -461,12 +246,12 @@ class TestRunTimeSandbox(unittest.TestCase):
|
|||
_restart.call_args_list[1].args)
|
||||
self.assertEqual(1, wait.call_count)
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
with mock.patch('storlets.gateway.gateways.container.runtime.'
|
||||
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox._restart') as _restart, \
|
||||
'DockerRunTimeSandbox._restart') as _restart, \
|
||||
mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||
'RunTimeSandbox.wait') as wait:
|
||||
'DockerRunTimeSandbox.wait') as wait:
|
||||
_restart.side_effect = StorletTimeout()
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
self.sbox.restart()
|
||||
|
|
@ -487,173 +272,5 @@ class TestRunTimeSandbox(unittest.TestCase):
|
|||
dependencies),)
|
||||
|
||||
|
||||
class TestStorletInvocationProtocol(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.pipe_path = tempfile.mktemp()
|
||||
self.log_file = tempfile.mktemp()
|
||||
self.logger = FakeLogger()
|
||||
self.storlet_id = 'Storlet-1.0.jar'
|
||||
self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'java',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
storlet_request = DockerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
self.protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
|
||||
def tearDown(self):
|
||||
for path in [self.pipe_path, self.log_file]:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_send_execute_command(self):
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
|
||||
'execute') as execute:
|
||||
execute.return_value = SBusResponse(True, 'OK', 'someid')
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
self.assertEqual('someid', self.protocol.task_id)
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
|
||||
'execute') as execute:
|
||||
execute.return_value = SBusResponse(True, 'OK')
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
|
||||
'execute') as execute:
|
||||
execute.return_value = SBusResponse(False, 'NG', 'someid')
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
with mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient.'
|
||||
'execute') as execute:
|
||||
execute.side_effect = SBusClientIOError()
|
||||
with self.assertRaises(StorletRuntimeException):
|
||||
with self.protocol.storlet_logger.activate():
|
||||
self.protocol._send_execute_command()
|
||||
|
||||
def test_invocation_protocol(self):
|
||||
# os.pipe will be called 3 times
|
||||
pipe_called = 3
|
||||
|
||||
with _mock_os_pipe([''] * pipe_called) as pipes:
|
||||
with mock.patch.object(self.protocol,
|
||||
'_wait_for_read_with_timeout'), \
|
||||
mock.patch.object(self.protocol, '_send_execute_command'):
|
||||
self.protocol._invoke()
|
||||
|
||||
self.assertEqual(pipe_called, len(pipes))
|
||||
pipes = iter(pipes)
|
||||
|
||||
# data write is not directly closed
|
||||
# data read is closed
|
||||
input_data_read_fd, input_data_write_fd = next(pipes)
|
||||
self.assertTrue(input_data_read_fd.closed)
|
||||
self.assertFalse(input_data_write_fd.closed)
|
||||
|
||||
# data write is closed but data read is still open
|
||||
data_read_fd, data_write_fd = next(pipes)
|
||||
self.assertFalse(data_read_fd.closed)
|
||||
self.assertTrue(data_write_fd.closed)
|
||||
|
||||
# metadata write fd is closed, metadata read fd is still open.
|
||||
metadata_read_fd, metadata_write_fd = next(pipes)
|
||||
self.assertFalse(metadata_read_fd.closed)
|
||||
self.assertTrue(metadata_write_fd.closed)
|
||||
|
||||
# sanity
|
||||
self.assertRaises(StopIteration, next, pipes)
|
||||
|
||||
def test_invocation_protocol_remote_fds(self):
|
||||
# In default, we have 4 fds in remote_fds
|
||||
storlet_request = DockerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(4, len(protocol.remote_fds))
|
||||
|
||||
# extra_resources expands the remote_fds
|
||||
storlet_request = DockerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
|
||||
extra_sources=[storlet_request])
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(5, len(protocol.remote_fds))
|
||||
|
||||
# 2 more extra_resources expands the remote_fds
|
||||
storlet_request = DockerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
|
||||
extra_sources=[storlet_request] * 3)
|
||||
with protocol.storlet_logger.activate():
|
||||
self.assertEqual(7, len(protocol.remote_fds))
|
||||
|
||||
def test_open_writer_with_invalid_fd(self):
|
||||
invalid_fds = (
|
||||
(None, TypeError), (-1, ValueError), ('blah', TypeError))
|
||||
|
||||
for invalid_fd, expected_error in invalid_fds:
|
||||
with self.assertRaises(expected_error):
|
||||
with self.protocol._open_writer(invalid_fd):
|
||||
pass
|
||||
|
||||
def _test_writer_with_exception(self, exception_cls):
|
||||
pipes = [os.pipe()]
|
||||
|
||||
def raise_in_the_context():
|
||||
with self.protocol._open_writer(pipes[0][1]):
|
||||
raise exception_cls()
|
||||
try:
|
||||
# writer context doesn't suppress any exception
|
||||
self.assertRaises(exception_cls, raise_in_the_context)
|
||||
|
||||
# since _open_writer closes the write fd, the os.close will fail as
|
||||
# BadFileDescriptor
|
||||
with self.assertRaises(OSError) as os_error:
|
||||
os.close(pipes[0][1])
|
||||
self.assertEqual(9, os_error.exception.errno)
|
||||
|
||||
finally:
|
||||
for fd in pipes[0]:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_writer_raise_while_in_writer_context(self):
|
||||
# basic storlet timeout
|
||||
self._test_writer_with_exception(StorletTimeout)
|
||||
# unexpected IOError
|
||||
self._test_writer_with_exception(IOError)
|
||||
# else
|
||||
self._test_writer_with_exception(Exception)
|
||||
|
||||
|
||||
class TestStorletInvocationProtocolPython(TestStorletInvocationProtocol):
|
||||
def setUp(self):
|
||||
self.pipe_path = tempfile.mktemp()
|
||||
self.log_file = tempfile.mktemp()
|
||||
self.logger = FakeLogger()
|
||||
self.storlet_id = 'Storlet-1.0.py'
|
||||
self.options = {'storlet_main': 'storlet.Storlet',
|
||||
'storlet_dependency': 'dep1,dep2',
|
||||
'storlet_language': 'python',
|
||||
'language_version': '3.6',
|
||||
'file_manager': FakeFileManager('storlet', 'dep')}
|
||||
storlet_request = DockerStorletRequest(
|
||||
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
|
||||
self.protocol = StorletInvocationProtocol(
|
||||
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@
|
|||
import unittest
|
||||
from storlets.gateway.common.exceptions import StorletGatewayLoadError
|
||||
from storlets.gateway.loader import load_gateway
|
||||
from storlets.gateway.gateways.stub import StorletGatewayStub
|
||||
from storlets.gateway.gateways.docker import StorletGatewayDocker
|
||||
from storlets.gateway.gateways.stub import StubStorletGateway
|
||||
from storlets.gateway.gateways.docker import DockerStorletGateway
|
||||
|
||||
|
||||
class TestLoader(unittest.TestCase):
|
||||
|
|
@ -26,11 +26,11 @@ class TestLoader(unittest.TestCase):
|
|||
def test_load_gateway_entry_point(self):
|
||||
# existing entry point
|
||||
self.assertEqual(
|
||||
StorletGatewayStub,
|
||||
StubStorletGateway,
|
||||
load_gateway('stub'))
|
||||
|
||||
self.assertEqual(
|
||||
StorletGatewayDocker,
|
||||
DockerStorletGateway,
|
||||
load_gateway('docker'))
|
||||
|
||||
# If the given entry point does not exist
|
||||
|
|
@ -40,18 +40,18 @@ class TestLoader(unittest.TestCase):
|
|||
def test_load_gateway_full_class_path(self):
|
||||
# If the given class path exists
|
||||
self.assertEqual(
|
||||
StorletGatewayStub,
|
||||
load_gateway('storlets.gateway.gateways.stub.StorletGatewayStub'))
|
||||
StubStorletGateway,
|
||||
load_gateway('storlets.gateway.gateways.stub.StubStorletGateway'))
|
||||
|
||||
self.assertEqual(
|
||||
StorletGatewayDocker,
|
||||
DockerStorletGateway,
|
||||
load_gateway('storlets.gateway.gateways.docker.'
|
||||
'StorletGatewayDocker'))
|
||||
'DockerStorletGateway'))
|
||||
|
||||
# If module does not exist
|
||||
with self.assertRaises(StorletGatewayLoadError):
|
||||
load_gateway('storlets.gateway.gateways.another_stub.'
|
||||
'StorletGatewayStub')
|
||||
'StubStorletGateway')
|
||||
|
||||
# If class does not exist
|
||||
with self.assertRaises(StorletGatewayLoadError):
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from storlets.gateway.gateways.stub import StorletGatewayStub
|
||||
from storlets.gateway.gateways.stub import StubStorletGateway
|
||||
from storlets.swift_middleware import storlet_handler
|
||||
|
||||
from tests.unit import FakeLogger
|
||||
|
|
@ -25,7 +25,7 @@ from tests.unit.swift_middleware import FakeApp
|
|||
|
||||
def create_handler_config(exec_server):
|
||||
return {'execution_server': exec_server,
|
||||
'gateway_module': StorletGatewayStub}
|
||||
'gateway_module': StubStorletGateway}
|
||||
|
||||
|
||||
class BaseTestStorletMiddleware(unittest.TestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue