diff --git a/gyanclient/api_versions.py b/gyanclient/api_versions.py new file mode 100644 index 0000000..02eff40 --- /dev/null +++ b/gyanclient/api_versions.py @@ -0,0 +1,304 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import logging +import os +import pkgutil +import re +import traceback + +from oslo_utils import strutils + +from gyanclient import exceptions +from gyanclient.i18n import _ + +LOG = logging.getLogger(__name__) +if not LOG.handlers: + LOG.addHandler(logging.StreamHandler()) + + +HEADER_NAME = "OpenStack-API-Version" +SERVICE_TYPE = "ml-infra" +MIN_API_VERSION = '1.1' +MAX_API_VERSION = '1.25' +DEFAULT_API_VERSION = MAX_API_VERSION + +_SUBSTITUTIONS = {} + + +_type_error_msg = _("'%(other)s' should be an instance of '%(cls)s'") + + +class APIVersion(object): + """This class represents an API Version Request. + + This class provides convenience methods for manipulation + and comparison of version numbers that we need to do to + implement microversions. + """ + + def __init__(self, version_str=None): + """Create an API version object. + + :param version_str: String representation of APIVersionRequest. + Correct format is 'X.Y', where 'X' and 'Y' + are int values. None value should be used + to create Null APIVersionRequest, which is + equal to 0.0 + """ + self.ver_major = 0 + self.ver_minor = 0 + + if version_str is not None: + match = re.match(r"^([1-9]\d*)\.([1-9]\d*|0|latest)$", version_str) + if match: + self.ver_major = int(match.group(1)) + if match.group(2) == "latest": + self.ver_minor = float("inf") + else: + self.ver_minor = int(match.group(2)) + else: + msg = _("Invalid format of client version '%s'. " + "Expected format 'X.Y', where X is a major part and Y " + "is a minor part of version.") % version_str + raise exceptions.UnsupportedVersion(msg) + + def __str__(self): + """Debug/Logging representation of object.""" + if self.is_latest(): + return "Latest API Version Major: %s" % self.ver_major + return ("API Version Major: %s, Minor: %s" + % (self.ver_major, self.ver_minor)) + + def __repr__(self): + if self.is_null(): + return "" + else: + return "" % self.get_string() + + def is_null(self): + return self.ver_major == 0 and self.ver_minor == 0 + + def is_latest(self): + return self.ver_minor == float("inf") + + def __lt__(self, other): + if not isinstance(other, APIVersion): + raise TypeError(_type_error_msg % {"other": other, + "cls": self.__class__}) + + return ((self.ver_major, self.ver_minor) < + (other.ver_major, other.ver_minor)) + + def __eq__(self, other): + if not isinstance(other, APIVersion): + raise TypeError(_type_error_msg % {"other": other, + "cls": self.__class__}) + + return ((self.ver_major, self.ver_minor) == + (other.ver_major, other.ver_minor)) + + def __gt__(self, other): + if not isinstance(other, APIVersion): + raise TypeError(_type_error_msg % {"other": other, + "cls": self.__class__}) + + return ((self.ver_major, self.ver_minor) > + (other.ver_major, other.ver_minor)) + + def __le__(self, other): + return self < other or self == other + + def __ne__(self, other): + return not self.__eq__(other) + + def __ge__(self, other): + return self > other or self == other + + def matches(self, min_version, max_version): + """Matches the version object. + + Returns whether the version object represents a version + greater than or equal to the minimum version and less than + or equal to the maximum version. + + :param min_version: Minimum acceptable version. + :param max_version: Maximum acceptable version. + :returns: boolean + + If min_version is null then there is no minimum limit. + If max_version is null then there is no maximum limit. + If self is null then raise ValueError + """ + + if self.is_null(): + raise ValueError(_("Null APIVersion doesn't support 'matches'.")) + if max_version.is_null() and min_version.is_null(): + return True + elif max_version.is_null(): + return min_version <= self + elif min_version.is_null(): + return self <= max_version + else: + return min_version <= self <= max_version + + def get_string(self): + """Version string representation. + + Converts object to string representation which if used to create + an APIVersion object results in the same version. + """ + if self.is_null(): + raise ValueError( + _("Null APIVersion cannot be converted to string.")) + elif self.is_latest(): + return "%s.%s" % (self.ver_major, "latest") + return "%s.%s" % (self.ver_major, self.ver_minor) + + +class VersionedMethod(object): + + def __init__(self, name, start_version, end_version, func): + """Versioning information for a single method + + :param name: Name of the method + :param start_version: Minimum acceptable version + :param end_version: Maximum acceptable_version + :param func: Method to call + + Minimum and maximums are inclusive + """ + self.name = name + self.start_version = start_version + self.end_version = end_version + self.func = func + + def __str__(self): + return ("Version Method %s: min: %s, max: %s" + % (self.name, self.start_version, self.end_version)) + + def __repr__(self): + return "" % self.name + + +def get_available_major_versions(): + matcher = re.compile(r"v[0-9]*$") + submodules = pkgutil.iter_modules([os.path.dirname(__file__)]) + available_versions = [name[1:] for loader, name, ispkg in submodules + if matcher.search(name)] + + return available_versions + + +def check_major_version(api_version): + """Checks major part of ``APIVersion`` obj is supported. + + :raises exceptions.UnsupportedVersion: if major part is not supported + """ + available_versions = get_available_major_versions() + if (not api_version.is_null() and + str(api_version.ver_major) not in available_versions): + if len(available_versions) == 1: + msg = _("Invalid client version '%(version)s'. " + "Major part should be '%(major)s'") % { + "version": api_version.get_string(), + "major": available_versions[0]} + else: + msg = _("Invalid client version '%(version)s'. " + "Major part must be one of: '%(major)s'") % { + "version": api_version.get_string(), + "major": ", ".join(available_versions)} + raise exceptions.UnsupportedVersion(msg) + + +def get_api_version(version_string): + """Returns checked APIVersion object""" + version_string = str(version_string) + if strutils.is_int_like(version_string): + version_string = "%s.0" % version_string + + api_version = APIVersion(version_string) + check_major_version(api_version) + return api_version + + +def update_headers(headers, api_version): + """Set microversion headers if api_version is not null""" + + if not api_version.is_null() and api_version.ver_minor != 0: + version_string = api_version.get_string() + headers[HEADER_NAME] = '%s %s' % (SERVICE_TYPE, version_string) + + +def _add_substitution(versioned_method): + _SUBSTITUTIONS.setdefault(versioned_method.name, []) + _SUBSTITUTIONS[versioned_method.name].append(versioned_method) + + +def _get_function_name(func): + filename, _lineno, _name, line = traceback.extract_stack()[-4] + module, _file_extension = os.path.splitext(filename) + module = module.replace("/", ".") + if module.endswith(func.__module__): + return "%s.[%s].%s" % (func.__module__, line, func.__name__) + else: + return "%s.%s" % (func.__module__, func.__name__) + + +def get_substitutions(func_name, api_version=None): + if hasattr(func_name, "__id__"): + func_name = func_name.__id__ + + substitutions = _SUBSTITUTIONS.get(func_name, []) + if api_version and not api_version.is_null(): + return [m for m in substitutions + if api_version.matches(m.start_version, m.end_version)] + return sorted(substitutions, key=lambda m: m.start_version) + + +def wraps(start_version, end_version=None): + start_version = APIVersion(start_version) + if end_version: + end_version = APIVersion(end_version) + else: + end_version = APIVersion("%s.latest" % start_version.ver_major) + + def decor(func): + func.versioned = True + name = _get_function_name(func) + + versioned_method = VersionedMethod(name, start_version, + end_version, func) + _add_substitution(versioned_method) + + @functools.wraps(func) + def substitution(obj, *args, **kwargs): + methods = get_substitutions(name, obj.api_version) + + if not methods: + raise exceptions.VersionNotFoundForAPIMethod( + obj.api_version.get_string(), name) + return methods[-1].func(obj, *args, **kwargs) + + # Let's share "arguments" with original method and substitution to + # allow put cliutils.arg and wraps decorators in any order + if not hasattr(func, 'arguments'): + func.arguments = [] + substitution.arguments = func.arguments + + substitution.__id__ = name + + return substitution + + return decor diff --git a/gyanclient/client.py b/gyanclient/client.py new file mode 100644 index 0000000..a4109ca --- /dev/null +++ b/gyanclient/client.py @@ -0,0 +1,93 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + +from oslo_utils import importutils + +from gyanclient import api_versions +from gyanclient import exceptions +from gyanclient.i18n import _ + +osprofiler_profiler = importutils.try_import("osprofiler.profiler") + + +def _get_client_class_and_version(version): + if not isinstance(version, api_versions.APIVersion): + version = api_versions.get_api_version(version) + else: + api_versions.check_major_version(version) + if version.is_latest(): + raise exceptions.UnsupportedVersion( + _('The version should be explicit, not latest.')) + return version, importutils.import_class( + 'gyanclient.v%s.client.Client' % version.ver_major) + + +def _check_arguments(kwargs, release, deprecated_name, right_name=None): + """Process deprecation of arguments. + + Check presence of deprecated argument in kwargs, prints proper warning + message, renames key to right one it needed. + """ + if deprecated_name in kwargs: + if right_name: + if right_name in kwargs: + msg = ('The %(old)s argument is deprecated in %(release)s' + 'and its use may result in errors in future releases.' + 'As %(new)s is provided, the %(old)s argument will ' + 'be ignored.') % {'old': deprecated_name, + 'release': release, + 'new': right_name} + kwargs.pop(deprecated_name) + else: + msg = ('The %(old)s argument is deprecated in %(release)s ' + 'and its use may result in errors in future releases. ' + 'Use %(new)s instead.') % {'old': deprecated_name, + 'release': release, + 'new': right_name} + kwargs[right_name] = kwargs.pop(deprecated_name) + else: + msg = ('The %(old)s argument is deprecated in %(release)s ' + 'and its use may result in errors in future ' + 'releases') % {'old': deprecated_name, + 'release': release} + kwargs.pop(deprecated_name) + warnings.warn(msg) + + +def Client(version='1', username=None, auth_url=None, **kwargs): + """Initialize client objects based on given version""" + _check_arguments(kwargs, 'Queens', 'api_key', right_name='password') + _check_arguments(kwargs, 'Queens', 'endpoint_type', + right_name='interface') + _check_arguments(kwargs, 'Queens', 'gyan_url', + right_name='endpoint_override') + _check_arguments(kwargs, 'Queens', 'tenant_name', + right_name='project_name') + _check_arguments(kwargs, 'Queens', 'tenant_id', right_name='project_id') + + profile = kwargs.pop('profile', None) + if osprofiler_profiler and profile: + # Initialize the root of the future trace: the created trace ID + # will be used as the very first parent to which all related + # traces will be bound to. The given HMAC key must correspond to + # the one set in gyan-api gyan.conf, otherwise the latter + # will fail to check the request signature and will skip + # initialization of osprofiler on the server side. + osprofiler_profiler.init(profile) + + api_version, client_class = _get_client_class_and_version(version) + return client_class(api_version=api_version, + auth_url=auth_url, + username=username, + **kwargs) diff --git a/gyanclient/common/__init__.py b/gyanclient/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyanclient/common/apiclient/__init__.py b/gyanclient/common/apiclient/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyanclient/common/apiclient/auth.py b/gyanclient/common/apiclient/auth.py new file mode 100644 index 0000000..240c652 --- /dev/null +++ b/gyanclient/common/apiclient/auth.py @@ -0,0 +1,148 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import argparse +import os +import six + +from gyanclient.common.apiclient import exceptions + + +_discovered_plugins = {} + + +def load_auth_system_opts(parser): + """Load options needed by the available auth-systems into a parser. + + This function will try to populate the parser with options from the + available plugins. + """ + group = parser.add_argument_group("Common auth options") + BaseAuthPlugin.add_common_opts(group) + for name, auth_plugin in _discovered_plugins.items(): + group = parser.add_argument_group( + "Auth-system '%s' options" % name, + conflict_handler="resolve") + auth_plugin.add_opts(group) + + +def load_plugin(auth_system): + try: + plugin_class = _discovered_plugins[auth_system] + except KeyError: + raise exceptions.AuthSystemNotFound(auth_system) + return plugin_class(auth_system=auth_system) + + +@six.add_metaclass(abc.ABCMeta) +class BaseAuthPlugin(object): + """Base class for authentication plugins. + + An authentication plugin needs to override at least the authenticate + method to be a valid plugin. + """ + + auth_system = None + opt_names = [] + common_opt_names = [ + "auth_system", + "username", + "password", + "auth_url", + ] + + def __init__(self, auth_system=None, **kwargs): + self.auth_system = auth_system or self.auth_system + self.opts = dict((name, kwargs.get(name)) + for name in self.opt_names) + + @staticmethod + def _parser_add_opt(parser, opt): + """Add an option to parser in two variants. + + :param opt: option name (with underscores) + """ + dashed_opt = opt.replace("_", "-") + env_var = "OS_%s" % opt.upper() + arg_default = os.environ.get(env_var, "") + arg_help = "Defaults to env[%s]." % env_var + parser.add_argument( + "--os-%s" % dashed_opt, + metavar="<%s>" % dashed_opt, + default=arg_default, + help=arg_help) + parser.add_argument( + "--os_%s" % opt, + metavar="<%s>" % dashed_opt, + help=argparse.SUPPRESS) + + @classmethod + def add_opts(cls, parser): + """Populate the parser with the options for this plugin.""" + for opt in cls.opt_names: + # use `BaseAuthPlugin.common_opt_names` since it is never + # changed in child classes + if opt not in BaseAuthPlugin.common_opt_names: + cls._parser_add_opt(parser, opt) + + @classmethod + def add_common_opts(cls, parser): + """Add options that are common for several plugins.""" + for opt in cls.common_opt_names: + cls._parser_add_opt(parser, opt) + + @staticmethod + def get_opt(opt_name, args): + """Return option name and value. + + :param opt_name: name of the option, e.g., "username" + :param args: parsed arguments + """ + return (opt_name, getattr(args, "os_%s" % opt_name, None)) + + def parse_opts(self, args): + """Parse the actual auth-system options if any. + + This method is expected to populate the attribute `self.opts` with a + dict containing the options and values needed to make authentication. + """ + self.opts.update(dict(self.get_opt(opt_name, args) + for opt_name in self.opt_names)) + + def authenticate(self, http_client): + """Authenticate using plugin defined method. + + The method usually analyses `self.opts` and performs + a request to authentication server. + + :param http_client: client object that needs authentication + :type http_client: HTTPClient + :raises: AuthorizationFailure + """ + self.sufficient_options() + self._do_authenticate(http_client) + + @abc.abstractmethod + def _do_authenticate(self, http_client): + """Protected method for authentication.""" + + def sufficient_options(self): + """Check if all required options are present. + + :raises: AuthPluginOptionsMissing + """ + missing = [opt + for opt in self.opt_names + if not self.opts.get(opt)] + if missing: + raise exceptions.AuthPluginOptionsMissing(missing) diff --git a/gyanclient/common/apiclient/base.py b/gyanclient/common/apiclient/base.py new file mode 100644 index 0000000..ff98e2c --- /dev/null +++ b/gyanclient/common/apiclient/base.py @@ -0,0 +1,98 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base utilities to build API operation managers and objects on top of. +""" + +import copy + + +class Resource(object): + """Base class for OpenStack resources (tenant, user, etc.). + + This is pretty much just a bag for attributes. + """ + + def __init__(self, manager, info, loaded=False): + """Populate and bind to a manager. + + :param manager: BaseManager object + :param info: dictionary representing resource attributes + :param loaded: prevent lazy-loading if set to True + """ + self.manager = manager + self._info = info + self._add_details(info) + self._loaded = loaded + + def __repr__(self): + reprkeys = sorted(k + for k in self.__dict__.keys() + if k[0] != '_' and k != 'manager') + info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys) + return "<%s %s>" % (self.__class__.__name__, info) + + def _add_details(self, info): + for (k, v) in info.items(): + try: + setattr(self, k, v) + self._info[k] = v + except AttributeError: + # In this case we already defined the attribute on the class + pass + + def __getattr__(self, k): + if k not in self.__dict__: + if not self.is_loaded(): + self.get() + return self.__getattr__(k) + + raise AttributeError(k) + else: + return self.__dict__[k] + + def get(self): + """Support for lazy loading details. + + Some clients, such as novaclient have the option to lazy load the + details, details which can be loaded with this function. + """ + # set_loaded() first ... so if we have to bail, we know we tried. + self.set_loaded(True) + if not hasattr(self.manager, 'get'): + return + + new = self.manager.get(self.id) + if new: + self._add_details(new._info) + self._add_details( + {'x_request_id': self.manager.client.last_request_id}) + + def __eq__(self, other): + if not isinstance(other, Resource): + return NotImplemented + # two resources of different types are not equal + if not isinstance(other, self.__class__): + return False + if hasattr(self, 'id') and hasattr(other, 'id'): + return self.id == other.id + return self._info == other._info + + def is_loaded(self): + return self._loaded + + def set_loaded(self, val): + self._loaded = val + + def to_dict(self): + return copy.deepcopy(self._info) diff --git a/gyanclient/common/apiclient/exceptions.py b/gyanclient/common/apiclient/exceptions.py new file mode 100644 index 0000000..a22372c --- /dev/null +++ b/gyanclient/common/apiclient/exceptions.py @@ -0,0 +1,463 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exception definitions. +""" + +import inspect +import sys + +import six + +from gyanclient.i18n import _ + + +class VersionNotFoundForAPIMethod(Exception): + msg_fmt = "API version '%(vers)s' is not supported on '%(method)s' method." + + def __init__(self, version, method): + self.version = version + self.method = method + + def __str__(self): + return self.msg_fmt % {"vers": self.version, "method": self.method} + + +class ClientException(Exception): + """The base exception class for all exceptions this library raises.""" + pass + + +class UnsupportedVersion(ClientException): + """User is trying to use an unsupported version of the API.""" + pass + + +class CommandError(ClientException): + """Error in CLI tool.""" + pass + + +class AuthorizationFailure(ClientException): + """Cannot authorize API client.""" + pass + + +class ConnectionError(ClientException): + """Cannot connect to API service.""" + pass + + +class ConnectionRefused(ConnectionError): + """Connection refused while trying to connect to API service.""" + pass + + +class AuthPluginOptionsMissing(AuthorizationFailure): + """Auth plugin misses some options.""" + def __init__(self, opt_names): + super(AuthPluginOptionsMissing, self).__init__( + _("Authentication failed. Missing options: %s") % + ", ".join(opt_names)) + self.opt_names = opt_names + + +class AuthSystemNotFound(AuthorizationFailure): + """User has specified an AuthSystem that is not installed.""" + def __init__(self, auth_system): + super(AuthSystemNotFound, self).__init__( + _("AuthSystemNotFound: %r") % auth_system) + self.auth_system = auth_system + + +class NoUniqueMatch(ClientException): + """Multiple entities found instead of one.""" + pass + + +class EndpointException(ClientException): + """Something is rotten in Service Catalog.""" + pass + + +class EndpointNotFound(EndpointException): + """Could not find requested endpoint in Service Catalog.""" + pass + + +class AmbiguousEndpoints(EndpointException): + """Found more than one matching endpoint in Service Catalog.""" + def __init__(self, endpoints=None): + super(AmbiguousEndpoints, self).__init__( + _("AmbiguousEndpoints: %r") % endpoints) + self.endpoints = endpoints + + +class HttpError(ClientException): + """The base exception class for all HTTP exceptions.""" + http_status = 0 + message = _("HTTP Error") + + def __init__(self, message=None, details=None, + response=None, request_id=None, + url=None, method=None, http_status=None): + self.http_status = http_status or self.http_status + self.message = message or self.message + self.details = details + self.request_id = request_id + self.response = response + self.url = url + self.method = method + formatted_string = "%s (HTTP %s)" % (self.message, self.http_status) + if request_id: + formatted_string += " (Request-ID: %s)" % request_id + super(HttpError, self).__init__(formatted_string) + + +class HTTPRedirection(HttpError): + """HTTP Redirection.""" + message = _("HTTP Redirection") + + +class HTTPClientError(HttpError): + """Client-side HTTP error. + + Exception for cases in which the client seems to have erred. + """ + message = _("HTTP Client Error") + + +class HttpServerError(HttpError): + """Server-side HTTP error. + + Exception for cases in which the server is aware that it has + erred or is incapable of performing the request. + """ + message = _("HTTP Server Error") + + +class MultipleChoices(HTTPRedirection): + """HTTP 300 - Multiple Choices. + + Indicates multiple options for the resource that the client may follow. + """ + + http_status = 300 + message = _("Multiple Choices") + + +class BadRequest(HTTPClientError): + """HTTP 400 - Bad Request. + + The request cannot be fulfilled due to bad syntax. + """ + http_status = 400 + message = _("Bad Request") + + +class Unauthorized(HTTPClientError): + """HTTP 401 - Unauthorized. + + Similar to 403 Forbidden, but specifically for use when authentication + is required and has failed or has not yet been provided. + """ + http_status = 401 + message = _("Unauthorized") + + +class PaymentRequired(HTTPClientError): + """HTTP 402 - Payment Required. + + Reserved for future use. + """ + http_status = 402 + message = _("Payment Required") + + +class Forbidden(HTTPClientError): + """HTTP 403 - Forbidden. + + The request was a valid request, but the server is refusing to respond + to it. + """ + http_status = 403 + message = _("Forbidden") + + +class NotFound(HTTPClientError): + """HTTP 404 - Not Found. + + The requested resource could not be found but may be available again + in the future. + """ + http_status = 404 + message = _("Not Found") + + +class MethodNotAllowed(HTTPClientError): + """HTTP 405 - Method Not Allowed. + + A request was made of a resource using a request method not supported + by that resource. + """ + http_status = 405 + message = _("Method Not Allowed") + + +class NotAcceptable(HTTPClientError): + """HTTP 406 - Not Acceptable. + + The requested resource is only capable of generating content not + acceptable according to the Accept headers sent in the request. + """ + http_status = 406 + message = _("Not Acceptable") + + +class ProxyAuthenticationRequired(HTTPClientError): + """HTTP 407 - Proxy Authentication Required. + + The client must first authenticate itself with the proxy. + """ + http_status = 407 + message = _("Proxy Authentication Required") + + +class RequestTimeout(HTTPClientError): + """HTTP 408 - Request Timeout. + + The server timed out waiting for the request. + """ + http_status = 408 + message = _("Request Timeout") + + +class Conflict(HTTPClientError): + """HTTP 409 - Conflict. + + Indicates that the request could not be processed because of conflict + in the request, such as an edit conflict. + """ + http_status = 409 + message = _("Conflict") + + +class Gone(HTTPClientError): + """HTTP 410 - Gone. + + Indicates that the resource requested is no longer available and will + not be available again. + """ + http_status = 410 + message = _("Gone") + + +class LengthRequired(HTTPClientError): + """HTTP 411 - Length Required. + + The request did not specify the length of its content, which is + required by the requested resource. + """ + http_status = 411 + message = _("Length Required") + + +class PreconditionFailed(HTTPClientError): + """HTTP 412 - Precondition Failed. + + The server does not meet one of the preconditions that the requester + put on the request. + """ + http_status = 412 + message = _("Precondition Failed") + + +class RequestEntityTooLarge(HTTPClientError): + """HTTP 413 - Request Entity Too Large. + + The request is larger than the server is willing or able to process. + """ + http_status = 413 + message = _("Request Entity Too Large") + + def __init__(self, *args, **kwargs): + try: + self.retry_after = int(kwargs.pop('retry_after')) + except (KeyError, ValueError): + self.retry_after = 0 + + super(RequestEntityTooLarge, self).__init__(*args, **kwargs) + + +class RequestUriTooLong(HTTPClientError): + """HTTP 414 - Request-URI Too Long. + + The URI provided was too long for the server to process. + """ + http_status = 414 + message = _("Request-URI Too Long") + + +class UnsupportedMediaType(HTTPClientError): + """HTTP 415 - Unsupported Media Type. + + The request entity has a media type which the server or resource does + not support. + """ + http_status = 415 + message = _("Unsupported Media Type") + + +class RequestedRangeNotSatisfiable(HTTPClientError): + """HTTP 416 - Requested Range Not Satisfiable. + + The client has asked for a portion of the file, but the server cannot + supply that portion. + """ + http_status = 416 + message = _("Requested Range Not Satisfiable") + + +class ExpectationFailed(HTTPClientError): + """HTTP 417 - Expectation Failed. + + The server cannot meet the requirements of the Expect request-header field. + """ + http_status = 417 + message = _("Expectation Failed") + + +class UnprocessableEntity(HTTPClientError): + """HTTP 422 - Unprocessable Entity. + + The request was well-formed but was unable to be followed due to semantic + errors. + """ + http_status = 422 + message = _("Unprocessable Entity") + + +class InternalServerError(HttpServerError): + """HTTP 500 - Internal Server Error. + + A generic error message, given when no more specific message is suitable. + """ + http_status = 500 + message = _("Internal Server Error") + + +# NotImplemented is a python keyword. +class HttpNotImplemented(HttpServerError): + """HTTP 501 - Not Implemented. + + The server either does not recognize the request method, or it lacks + the ability to fulfill the request. + """ + http_status = 501 + message = _("Not Implemented") + + +class BadGateway(HttpServerError): + """HTTP 502 - Bad Gateway. + + The server was acting as a gateway or proxy and received an invalid + response from the upstream server. + """ + http_status = 502 + message = _("Bad Gateway") + + +class ServiceUnavailable(HttpServerError): + """HTTP 503 - Service Unavailable. + + The server is currently unavailable. + """ + http_status = 503 + message = _("Service Unavailable") + + +class GatewayTimeout(HttpServerError): + """HTTP 504 - Gateway Timeout. + + The server was acting as a gateway or proxy and did not receive a timely + response from the upstream server. + """ + http_status = 504 + message = _("Gateway Timeout") + + +class HttpVersionNotSupported(HttpServerError): + """HTTP 505 - HttpVersion Not Supported. + + The server does not support the HTTP protocol version used in the request. + """ + http_status = 505 + message = _("HTTP Version Not Supported") + + +# _code_map contains all the classes that have http_status attribute. +_code_map = dict( + (getattr(obj, 'http_status', None), obj) + for name, obj in vars(sys.modules[__name__]).items() + if inspect.isclass(obj) and getattr(obj, 'http_status', False) +) + + +def from_response(response, method, url): + """Returns an instance of :class:`HttpError` or subclass based on response. + + :param response: instance of `requests.Response` class + :param method: HTTP method used for request + :param url: URL used for request + """ + + req_id = response.headers.get("x-openstack-request-id") + if not req_id: + req_id = response.headers.get("x-compute-request-id") + kwargs = { + "http_status": response.status_code, + "response": response, + "method": method, + "url": url, + "request_id": req_id, + } + if "retry-after" in response.headers: + kwargs["retry_after"] = response.headers["retry-after"] + + content_type = response.headers.get("Content-Type", "") + if content_type.startswith("application/json"): + try: + body = response.json() + except ValueError: + pass + else: + if isinstance(body, dict): + error = body.get(list(body)[0]) + if isinstance(error, dict): + kwargs["message"] = (error.get("message") or + error.get("faultstring")) + kwargs["details"] = (error.get("details") or + six.text_type(body)) + elif content_type.startswith("text/"): + kwargs["details"] = getattr(response, 'text', '') + + try: + cls = _code_map[response.status_code] + except KeyError: + if 500 <= response.status_code < 600: + cls = HttpServerError + elif 400 <= response.status_code < 500: + cls = HTTPClientError + else: + cls = HttpError + return cls(**kwargs) diff --git a/gyanclient/common/base.py b/gyanclient/common/base.py new file mode 100644 index 0000000..0f65fdb --- /dev/null +++ b/gyanclient/common/base.py @@ -0,0 +1,160 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base utilities to build API operation managers and objects on top of. +""" + +import copy + +import six.moves.urllib.parse as urlparse + +from gyanclient.common.apiclient import base + + +def getid(obj): + """Wrapper to get object's ID. + + Abstracts the common pattern of allowing both an object or an + object's ID (UUID) as a parameter when dealing with relationships. + """ + try: + return obj.id + except AttributeError: + return obj + + +class Manager(object): + """Provides CRUD operations with a particular API.""" + resource_class = None + + def __init__(self, api): + self.api = api + + @property + def api_version(self): + return self.api.api_version + + def _create(self, url, body): + resp, body = self.api.json_request('POST', url, body=body) + if body: + return self.resource_class(self, body) + + def _format_body_data(self, body, response_key): + if response_key: + try: + data = body[response_key] + except KeyError: + return [] + else: + data = body + + if not isinstance(data, list): + data = [data] + + return data + + def _list_pagination(self, url, response_key=None, obj_class=None, + limit=None): + """Retrieve a list of items. + + The Gyan API is configured to return a maximum number of + items per request, (FIXME: see Gyan's api.max_limit option). This + iterates over the 'next' link (pagination) in the responses, + to get the number of items specified by 'limit'. If 'limit' + is None this function will continue pagination until there are + no more values to be returned. + + :param url: a partial URL, e.g. '/nodes' + :param response_key: the key to be looked up in response + dictionary, e.g. 'nodes' + :param obj_class: class for constructing the returned objects. + :param limit: maximum number of items to return. If None returns + everything. + + """ + if obj_class is None: + obj_class = self.resource_class + + if limit is not None: + limit = int(limit) + + object_list = [] + object_count = 0 + limit_reached = False + while url: + resp, body = self.api.json_request('GET', url) + data = self._format_body_data(body, response_key) + for obj in data: + object_list.append(obj_class(self, obj, loaded=True)) + object_count += 1 + if limit and object_count >= limit: + # break the for loop + limit_reached = True + break + + # break the while loop and return + if limit_reached: + break + + url = body.get('next') + if url: + url_parts = list(urlparse.urlparse(url)) + url_parts[0] = url_parts[1] = '' + url = urlparse.urlunparse(url_parts) + + return object_list + + def _list(self, url, response_key=None, obj_class=None, body=None, + qparams=None): + if qparams: + url = "%s?%s" % (url, urlparse.urlencode(qparams)) + + resp, body = self.api.json_request('GET', url) + + if obj_class is None: + obj_class = self.resource_class + + data = self._format_body_data(body, response_key) + return [obj_class(self, res, loaded=True) for res in data if res] + + def _update(self, url, body, method='PATCH', response_key=None): + resp, body = self.api.json_request(method, url, body=body) + # PATCH/PUT requests may not return a body + if body: + return self.resource_class(self, body) + + def _delete(self, url, qparams=None): + if qparams: + url = "%s?%s" % (url, urlparse.urlencode(qparams)) + self.api.raw_request('DELETE', url) + + def _search(self, url, qparams=None, response_key=None, obj_class=None, + body=None): + if qparams: + url = "%s?%s" % (url, urlparse.urlencode(qparams)) + + resp, body = self.api.json_request('GET', url, body=body) + data = self._format_body_data(body, response_key) + if obj_class is None: + obj_class = self.resource_class + return [obj_class(self, res, loaded=True) for res in data if res] + + +class Resource(base.Resource): + """Represents a particular instance of an object (tenant, user, etc). + + This is pretty much just a bag for attributes. + """ + + def to_dict(self): + return copy.deepcopy(self._info) diff --git a/gyanclient/common/cliutils.py b/gyanclient/common/cliutils.py new file mode 100644 index 0000000..5bcd1c6 --- /dev/null +++ b/gyanclient/common/cliutils.py @@ -0,0 +1,319 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from __future__ import print_function + +import collections +import getpass +import inspect +import os +import sys +import textwrap + +import decorator +from oslo_utils import encodeutils +from oslo_utils import strutils +import prettytable +import six +from six import moves + +from gyanclient.i18n import _ + + +class MissingArgs(Exception): + """Supplied arguments are not sufficient for calling a function.""" + def __init__(self, missing): + self.missing = missing + msg = _("Missing arguments: %s") % ", ".join(missing) + super(MissingArgs, self).__init__(msg) + + +def validate_args(fn, *args, **kwargs): + """Check that the supplied args are sufficient for calling a function. + + >>> validate_args(lambda a: None) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): a + >>> validate_args(lambda a, b, c, d: None, 0, c=1) + Traceback (most recent call last): + ... + MissingArgs: Missing argument(s): b, d + + :param fn: the function to check + :param arg: the positional arguments supplied + :param kwargs: the keyword arguments supplied + """ + argspec = inspect.getargspec(fn) + + num_defaults = len(argspec.defaults or []) + required_args = argspec.args[:len(argspec.args) - num_defaults] + + def isbound(method): + return getattr(method, '__self__', None) is not None + + if isbound(fn): + required_args.pop(0) + + missing = [arg for arg in required_args if arg not in kwargs] + missing = missing[len(args):] + if missing: + raise MissingArgs(missing) + + +def arg(*args, **kwargs): + """Decorator for CLI args. + + Example: + + >>> @arg("name", help="Name of the new entity") + ... def entity_create(args): + ... pass + """ + def _decorator(func): + add_arg(func, *args, **kwargs) + return func + return _decorator + + +def exclusive_arg(group_name, *args, **kwargs): + """Decorator for CLI mutually exclusive args.""" + def _decorator(func): + required = kwargs.pop('required', None) + add_exclusive_arg(func, group_name, required, *args, **kwargs) + return func + return _decorator + + +def env(*args, **kwargs): + """Returns the first environment variable set. + + If all are empty, defaults to '' or keyword arg `default`. + """ + for arg in args: + value = os.environ.get(arg) + if value: + return value + return kwargs.get('default', '') + + +def add_arg(func, *args, **kwargs): + """Bind CLI arguments to a shell.py `do_foo` function.""" + + if not hasattr(func, 'arguments'): + func.arguments = [] + + if (args, kwargs) not in func.arguments: + # Because of the semantics of decorator composition if we just append + # to the options list positional options will appear to be backwards. + func.arguments.insert(0, (args, kwargs)) + + +def add_exclusive_arg(func, group_name, required, *args, **kwargs): + """Bind CLI mutally exclusive arguments to a shell.py `do_foo` function.""" + + if not hasattr(func, 'exclusive_args'): + func.exclusive_args = collections.defaultdict(list) + # Default required to False + func.exclusive_args['__required__'] = collections.defaultdict(bool) + + if (args, kwargs) not in func.exclusive_args[group_name]: + # Because of the semantics of decorator composition if we just append + # to the options list positional options will appear to be backwards. + func.exclusive_args[group_name].insert(0, (args, kwargs)) + if required is not None: + func.exclusive_args['__required__'][group_name] = required + + +def unauthenticated(func): + """Adds 'unauthenticated' attribute to decorated function. + + Usage: + + >>> @unauthenticated + ... def mymethod(f): + ... pass + """ + func.unauthenticated = True + return func + + +def isunauthenticated(func): + """Checks if the function does not require authentication. + + Mark such functions with the `@unauthenticated` decorator. + + :returns: bool + """ + return getattr(func, 'unauthenticated', False) + + +def print_list(objs, fields, formatters=None, sortby_index=0, + mixed_case_fields=None, field_labels=None): + """Print a list or objects as a table, one row per object. + + :param objs: iterable of :class:`Resource` + :param fields: attributes that correspond to columns, in order + :param formatters: `dict` of callables for field formatting + :param sortby_index: index of the field for sorting table rows + :param mixed_case_fields: fields corresponding to object attributes that + have mixed case names (e.g., 'serverId') + :param field_labels: Labels to use in the heading of the table, default to + fields. + """ + formatters = formatters or {} + mixed_case_fields = mixed_case_fields or [] + field_labels = field_labels or fields + if len(field_labels) != len(fields): + raise ValueError(_("Field labels list %(labels)s has different number " + "of elements than fields list %(fields)s"), + {'labels': field_labels, 'fields': fields}) + + if sortby_index is None: + kwargs = {} + else: + kwargs = {'sortby': field_labels[sortby_index]} + pt = prettytable.PrettyTable(field_labels) + pt.align = 'l' + + for o in objs: + row = [] + for field in fields: + if field in formatters: + row.append(formatters[field](o)) + else: + if field in mixed_case_fields: + field_name = field.replace(' ', '_') + else: + field_name = field.lower().replace(' ', '_') + data = getattr(o, field_name, '') + row.append(data) + pt.add_row(row) + + if six.PY3: + print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode()) + else: + print(encodeutils.safe_encode(pt.get_string(**kwargs))) + + +def keys_and_vals_to_strs(dictionary): + """Recursively convert a dictionary's keys and values to strings. + + :param dictionary: dictionary whose keys/vals are to be converted to strs + """ + def to_str(k_or_v): + if isinstance(k_or_v, dict): + return keys_and_vals_to_strs(k_or_v) + elif isinstance(k_or_v, six.text_type): + return str(k_or_v) + else: + return k_or_v + return dict((to_str(k), to_str(v)) for k, v in dictionary.items()) + + +def print_dict(dct, dict_property="Property", wrap=0): + """Print a `dict` as a table of two columns. + + :param dct: `dict` to print + :param dict_property: name of the first column + :param wrap: wrapping for the second column + """ + pt = prettytable.PrettyTable([dict_property, 'Value']) + pt.align = 'l' + for k, v in dct.items(): + # convert dict to str to check length + if isinstance(v, dict): + v = six.text_type(keys_and_vals_to_strs(v)) + if wrap > 0: + v = textwrap.fill(six.text_type(v), wrap) + elif wrap < 0: + raise ValueError(_("Wrap argument should be a positive integer")) + # if value has a newline, add in multiple rows + # e.g. fault with stacktrace + if v and isinstance(v, six.string_types) and r'\n' in v: + lines = v.strip().split(r'\n') + col1 = k + for line in lines: + pt.add_row([col1, line]) + col1 = '' + elif isinstance(v, list): + val = str([str(i) for i in v]) + pt.add_row([k, val]) + else: + pt.add_row([k, v]) + + if six.PY3: + print(encodeutils.safe_encode(pt.get_string()).decode()) + else: + print(encodeutils.safe_encode(pt.get_string())) + + +def get_password(max_password_prompts=3): + """Read password from TTY.""" + verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD")) + pw = None + if hasattr(sys.stdin, "isatty") and sys.stdin.isatty(): + # Check for Ctrl-D + try: + for __ in moves.range(max_password_prompts): + pw1 = getpass.getpass("OS Password: ") + if verify: + pw2 = getpass.getpass("Please verify: ") + else: + pw2 = pw1 + if pw1 == pw2 and pw1: + pw = pw1 + break + except EOFError: + pass + return pw + + +def service_type(stype): + """Adds 'service_type' attribute to decorated function. + + Usage: + + .. code-block:: python + + @service_type('volume') + def mymethod(f): + ... + """ + def inner(f): + f.service_type = stype + return f + return inner + + +def get_service_type(f): + """Retrieves service type from function.""" + return getattr(f, 'service_type', None) + + +def pretty_choice_list(l): + return ', '.join("'%s'" % i for i in l) + + +def exit(msg=''): + if msg: + print(msg, file=sys.stderr) + sys.exit(1) + + +def deprecated(message): + @decorator.decorator + def wrapper(func, *args, **kwargs): + print(message) + return func(*args, **kwargs) + return wrapper diff --git a/gyanclient/common/httpclient.py b/gyanclient/common/httpclient.py new file mode 100644 index 0000000..46c4f39 --- /dev/null +++ b/gyanclient/common/httpclient.py @@ -0,0 +1,421 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import json +import os +from oslo_log import log as logging +import socket +import ssl + +from keystoneauth1 import adapter +from oslo_utils import importutils +import six +import six.moves.urllib.parse as urlparse + +from gyanclient import api_versions +from gyanclient import exceptions + +osprofiler_web = importutils.try_import("osprofiler.web") + +LOG = logging.getLogger(__name__) +USER_AGENT = 'python-gyanclient' +CHUNKSIZE = 1024 * 64 # 64kB + +API_VERSION = '/v1' +DEFAULT_API_VERSION = '1.latest' + + +def _extract_error_json(body): + """Return error_message from the HTTP response body.""" + error_json = {} + try: + body_json = json.loads(body) + if 'error_message' in body_json: + raw_msg = body_json['error_message'] + error_json = json.loads(raw_msg) + elif 'error' in body_json: + error_body = body_json['error'] + error_json = {'faultstring': error_body['title'], + 'debuginfo': error_body['message']} + else: + error_body = body_json['errors'][0] + error_json = {'faultstring': error_body['title']} + if 'detail' in error_body: + error_json['debuginfo'] = error_body['detail'] + elif 'description' in error_body: + error_json['debuginfo'] = error_body['description'] + + except ValueError: + return {} + + return error_json + + +class HTTPClient(object): + + def __init__(self, endpoint, api_version=DEFAULT_API_VERSION, **kwargs): + self.endpoint = endpoint + self.auth_token = kwargs.get('token') + self.auth_ref = kwargs.get('auth_ref') + self.api_version = api_version or api_versions.APIVersion() + self.connection_params = self.get_connection_params(endpoint, **kwargs) + + @staticmethod + def get_connection_params(endpoint, **kwargs): + parts = urlparse.urlparse(endpoint) + + # trim API version and trailing slash from endpoint + path = parts.path + path = path.rstrip('/').rstrip(API_VERSION) + + _args = (parts.hostname, parts.port, path) + _kwargs = {'timeout': (float(kwargs.get('timeout')) + if kwargs.get('timeout') else 600)} + + if parts.scheme == 'https': + _class = VerifiedHTTPSConnection + _kwargs['ca_file'] = kwargs.get('ca_file', None) + _kwargs['cert_file'] = kwargs.get('cert_file', None) + _kwargs['key_file'] = kwargs.get('key_file', None) + _kwargs['insecure'] = kwargs.get('insecure', False) + elif parts.scheme == 'http': + _class = six.moves.http_client.HTTPConnection + else: + msg = 'Unsupported scheme: %s' % parts.scheme + raise exceptions.EndpointException(msg) + + return (_class, _args, _kwargs) + + def get_connection(self): + _class = self.connection_params[0] + try: + return _class(*self.connection_params[1][0:2], + **self.connection_params[2]) + except six.moves.http_client.InvalidURL: + raise exceptions.EndpointException() + + def log_curl_request(self, method, url, kwargs): + curl = ['curl -i -X %s' % method] + + for (key, value) in kwargs['headers'].items(): + header = '-H \'%s: %s\'' % (key, value) + curl.append(header) + + conn_params_fmt = [ + ('key_file', '--key %s'), + ('cert_file', '--cert %s'), + ('ca_file', '--cacert %s'), + ] + for (key, fmt) in conn_params_fmt: + value = self.connection_params[2].get(key) + if value: + curl.append(fmt % value) + + if self.connection_params[2].get('insecure'): + curl.append('-k') + + if 'body' in kwargs: + curl.append('-d \'%s\'' % kwargs['body']) + + curl.append('%s/%s' % (self.endpoint, url.lstrip(API_VERSION))) + LOG.debug(' '.join(curl)) + + @staticmethod + def log_http_response(resp, body=None): + status = (resp.version / 10.0, resp.status, resp.reason) + dump = ['\nHTTP/%.1f %s %s' % status] + dump.extend(['%s: %s' % (k, v) for k, v in resp.getheaders()]) + dump.append('') + if body: + dump.extend([body, '']) + LOG.debug('\n'.join(dump)) + + def _make_connection_url(self, url): + (_class, _args, _kwargs) = self.connection_params + base_url = _args[2] + return '%s/%s' % (base_url, url.lstrip('/')) + + def _http_request(self, url, method, **kwargs): + """Send an http request with the specified characteristics. + + Wrapper around httplib.HTTP(S)Connection.request to handle tasks such + as setting headers and error handling. + """ + # Copy the kwargs so we can reuse the original in case of redirects + kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) + kwargs['headers'].setdefault('User-Agent', USER_AGENT) + api_versions.update_headers(kwargs["headers"], self.api_version) + + if self.auth_token: + kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) + + self.log_curl_request(method, url, kwargs) + conn = self.get_connection() + + try: + conn_url = self._make_connection_url(url) + conn.request(method, conn_url, **kwargs) + resp = conn.getresponse() + except socket.gaierror as e: + message = ("Error finding address for %(url)s: %(e)s" + % dict(url=url, e=e)) + raise exceptions.EndpointNotFound(message) + except (socket.error, socket.timeout) as e: + endpoint = self.endpoint + message = ("Error communicating with %(endpoint)s %(e)s" + % dict(endpoint=endpoint, e=e)) + raise exceptions.ConnectionRefused(message) + + body_iter = ResponseBodyIterator(resp) + + # Read body into string if it isn't obviously image data + body_str = None + if resp.getheader('content-type', None) != 'application/octet-stream': + # decoding byte to string is necessary for Python 3.4 compatibility + # this issues has not been found with Python 3.4 unit tests + # because the test creates a fake http response of type str + # the if statement satisfies test (str) and real (bytes) behavior + body_list = [ + chunk.decode("utf-8") if isinstance(chunk, bytes) + else chunk for chunk in body_iter + ] + body_str = ''.join(body_list) + self.log_http_response(resp, body_str) + body_iter = six.StringIO(body_str) + else: + self.log_http_response(resp) + + if 400 <= resp.status < 600: + LOG.warning("Request returned failure status.") + error_json = _extract_error_json(body_str) + raise exceptions.from_response( + resp, error_json.get('faultstring'), + error_json.get('debuginfo'), method, url) + elif resp.status in (301, 302, 305): + # Redirected. Reissue the request to the new location. + return self._http_request(resp['location'], method, **kwargs) + elif resp.status == 300: + raise exceptions.from_response(resp, method=method, url=url) + + return resp, body_iter + + def json_request(self, method, url, **kwargs): + kwargs.setdefault('headers', {}) + kwargs['headers'].setdefault('Content-Type', 'application/json') + kwargs['headers'].setdefault('Accept', 'application/json') + + if 'body' in kwargs: + kwargs['body'] = json.dumps(kwargs['body']) + + resp, body_iter = self._http_request(url, method, **kwargs) + content_type = resp.getheader('content-type', None) + + if resp.status == 204 or resp.status == 205 or content_type is None: + return resp, list() + + if 'application/json' in content_type: + body = ''.join([chunk for chunk in body_iter]) + try: + body = json.loads(body) + except ValueError: + LOG.error('Could not decode response body as JSON') + else: + body = None + + return resp, body + + def raw_request(self, method, url, **kwargs): + kwargs.setdefault('headers', {}) + kwargs['headers'].setdefault('Content-Type', + 'application/octet-stream') + return self._http_request(url, method, **kwargs) + + +class VerifiedHTTPSConnection(six.moves.http_client.HTTPSConnection): + """httplib-compatibile connection using client-side SSL authentication + + :see http://code.activestate.com/recipes/ + 577548-https-httplib-client-connection-with-certificate-v/ + """ + + def __init__(self, host, port, key_file=None, cert_file=None, + ca_file=None, timeout=None, insecure=False): + six.moves.http_client.HTTPSConnection.__init__(self, host, port, + key_file=key_file, + cert_file=cert_file) + self.key_file = key_file + self.cert_file = cert_file + if ca_file is not None: + self.ca_file = ca_file + else: + self.ca_file = self.get_system_ca_file() + self.timeout = timeout + self.insecure = insecure + + def connect(self): + """Connect to a host on a given (SSL) port. + + If ca_file is pointing somewhere, use it to check Server Certificate. + + Redefined/copied and extended from httplib.py:1105 (Python 2.6.x). + This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to + ssl.wrap_socket(), which forces SSL to check server certificate against + our client certificate. + """ + sock = socket.create_connection((self.host, self.port), self.timeout) + + if self._tunnel_host: + self.sock = sock + self._tunnel() + + if self.insecure is True: + kwargs = {'cert_reqs': ssl.CERT_NONE} + else: + kwargs = {'cert_reqs': ssl.CERT_REQUIRED, 'ca_certs': self.ca_file} + + if self.cert_file: + kwargs['certfile'] = self.cert_file + if self.key_file: + kwargs['keyfile'] = self.key_file + + self.sock = ssl.wrap_socket(sock, **kwargs) + + @staticmethod + def get_system_ca_file(): + """Return path to system default CA file.""" + # Standard CA file locations for Debian/Ubuntu, RedHat/Fedora, + # Suse, FreeBSD/OpenBSD + ca_path = ['/etc/ssl/certs/ca-certificates.crt', + '/etc/pki/tls/certs/ca-bundle.crt', + '/etc/ssl/ca-bundle.pem', + '/etc/ssl/cert.pem'] + for ca in ca_path: + if os.path.exists(ca): + return ca + return None + + +class SessionClient(adapter.LegacyJsonAdapter): + """HTTP client based on Keystone client session.""" + + def __init__(self, user_agent=USER_AGENT, logger=LOG, + api_version=DEFAULT_API_VERSION, *args, **kwargs): + self.user_agent = USER_AGENT + self.api_version = api_version or api_versions.APIVersion() + super(SessionClient, self).__init__(*args, **kwargs) + + def _http_request(self, url, method, **kwargs): + if url.startswith(API_VERSION): + url = url[len(API_VERSION):] + + kwargs.setdefault('user_agent', self.user_agent) + kwargs.setdefault('auth', self.auth) + kwargs.setdefault('endpoint_override', self.endpoint_override) + + # Copy the kwargs so we can reuse the original in case of redirects + kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) + kwargs['headers'].setdefault('User-Agent', self.user_agent) + api_versions.update_headers(kwargs["headers"], self.api_version) + + if osprofiler_web: + kwargs['headers'].update(osprofiler_web.get_trace_id_headers()) + + endpoint_filter = kwargs.setdefault('endpoint_filter', {}) + endpoint_filter.setdefault('interface', self.interface) + endpoint_filter.setdefault('service_type', self.service_type) + endpoint_filter.setdefault('region_name', self.region_name) + resp = self.session.request(url, method, + raise_exc=False, **kwargs) + + if 400 <= resp.status_code < 600: + error_json = _extract_error_json(resp.content) + raise exceptions.from_response( + resp, error_json.get('faultstring'), + error_json.get('debuginfo'), method, url) + elif resp.status_code in (301, 302, 305): + # Redirected. Reissue the request to the new location. + location = resp.headers.get('location') + resp = self._http_request(location, method, **kwargs) + elif resp.status_code == 300: + raise exceptions.from_response(resp, method=method, url=url) + return resp + + def json_request(self, method, url, **kwargs): + kwargs.setdefault('headers', {}) + kwargs['headers'].setdefault('Content-Type', 'application/json') + kwargs['headers'].setdefault('Accept', 'application/json') + + if 'body' in kwargs: + kwargs['data'] = json.dumps(kwargs.pop('body')) + + resp = self._http_request(url, method, **kwargs) + body = resp.content + content_type = resp.headers.get('content-type', None) + status = resp.status_code + if status == 204 or status == 205 or content_type is None: + return resp, list() + if 'application/json' in content_type: + try: + body = resp.json() + except ValueError: + LOG.error('Could not decode response body as JSON') + else: + body = None + + return resp, body + + def raw_request(self, method, url, **kwargs): + kwargs.setdefault('headers', {}) + kwargs['headers'].setdefault('Content-Type', + 'application/octet-stream') + return self._http_request(url, method, **kwargs) + + +class ResponseBodyIterator(object): + """A class that acts as an iterator over an HTTP response.""" + + def __init__(self, resp): + self.resp = resp + + def __iter__(self): + while True: + try: + yield self.next() + except StopIteration: + return + + def next(self): + chunk = self.resp.read(CHUNKSIZE) + if chunk: + return chunk + else: + raise StopIteration() + + +def _construct_http_client(*args, **kwargs): + session = kwargs.pop('session', None) + auth = kwargs.pop('auth', None) + + if session: + service_type = kwargs.pop('service_type', 'container') + interface = kwargs.pop('endpoint_type', None) + region_name = kwargs.pop('region_name', None) + return SessionClient(session=session, + auth=auth, + interface=interface, + service_type=service_type, + region_name=region_name, + service_name=None, + user_agent='python-gyanclient') + else: + return HTTPClient(*args, **kwargs) diff --git a/gyanclient/common/template_format.py b/gyanclient/common/template_format.py new file mode 100644 index 0000000..ab65218 --- /dev/null +++ b/gyanclient/common/template_format.py @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import yaml + + +if hasattr(yaml, 'CSafeDumper'): + yaml_dumper_base = yaml.CSafeDumper +else: + yaml_dumper_base = yaml.SafeDumper + + +# We create custom class to not overriden the default yaml behavior +class yaml_loader(yaml.SafeLoader): + pass + + +class yaml_dumper(yaml_dumper_base): + pass + + +def _construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) +yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str) +# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type +# datetime.data which causes problems in API layer when being processed by +# openstack.common.jsonutils. Therefore, make unicode string out of timestamps +# until jsonutils can handle dates. +yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp', + _construct_yaml_str) + + +def parse(tmpl_str): + """Takes a string and returns a dict containing the parsed structure. + + This includes determination of whether the string is using the + JSON or YAML format. + """ + # strip any whitespace before the check + tmpl_str = tmpl_str.strip() + if tmpl_str.startswith('{'): + tpl = json.loads(tmpl_str) + else: + try: + tpl = yaml.safe_load(tmpl_str) + except yaml.YAMLError as yea: + raise ValueError(yea) + else: + if tpl is None: + tpl = {} + + return tpl diff --git a/gyanclient/common/template_utils.py b/gyanclient/common/template_utils.py new file mode 100644 index 0000000..e46d067 --- /dev/null +++ b/gyanclient/common/template_utils.py @@ -0,0 +1,85 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils +import six +from six.moves.urllib import parse +from six.moves.urllib import request + +from gyanclient.common import template_format +from gyanclient.common import utils +from gyanclient import exceptions +from gyanclient.i18n import _ + + +def get_template_contents(template_file=None, template_url=None, + files=None): + + # Transform a bare file path to a file:// URL. + if template_file: # nosec + template_url = utils.normalise_file_path_to_url(template_file) + tpl = request.urlopen(template_url).read() + else: + raise exceptions.CommandErrorException(_('Need to specify exactly ' + 'one of %(arg1)s, %(arg2)s ' + 'or %(arg3)s') % + {'arg1': '--template-file', + 'arg2': '--template-url'}) + + if not tpl: + raise exceptions.CommandErrorException(_('Could not fetch ' + 'template from %s') % + template_url) + + try: + if isinstance(tpl, six.binary_type): + tpl = tpl.decode('utf-8') + template = template_format.parse(tpl) + except ValueError as e: + raise exceptions.CommandErrorException(_('Error parsing template ' + '%(url)s %(error)s') % + {'url': template_url, + 'error': e}) + return template + + +def is_template(file_content): + try: + if isinstance(file_content, six.binary_type): + file_content = file_content.decode('utf-8') + template_format.parse(file_content) + except (ValueError, TypeError): + return False + return True + + +def get_file_contents(from_data, files, base_url=None, + ignore_if=None): + + if isinstance(from_data, dict): + for key, value in from_data.items(): + if ignore_if and ignore_if(key, value): + continue + + if base_url and not base_url.endswith('/'): + base_url = base_url + '/' + + str_url = parse.urljoin(base_url, value) + if str_url not in files: + file_content = utils.read_url_content(str_url) + if is_template(file_content): + template = get_template_contents( + template_url=str_url, files=files)[1] + file_content = jsonutils.dumps(template) + files[str_url] = file_content + # replace the data value with the normalised absolute URL + from_data[key] = str_url diff --git a/gyanclient/common/utils.py b/gyanclient/common/utils.py new file mode 100644 index 0000000..c8f9eda --- /dev/null +++ b/gyanclient/common/utils.py @@ -0,0 +1,173 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import binascii +import json +import os +import re + +from oslo_utils import netutils +import six +from six.moves.urllib import parse +from six.moves.urllib import request +from gyanclient.common.apiclient import exceptions as apiexec +from gyanclient.common import cliutils as utils +from gyanclient import exceptions as exc +from gyanclient.i18n import _ + +VALID_UNITS = ( + K, + M, + G, +) = ( + 1024, + 1024 * 1024, + 1024 * 1024 * 1024, +) + + +def common_filters(marker=None, limit=None, sort_key=None, + sort_dir=None, all_projects=False): + """Generate common filters for any list request. + + :param all_projects: list containers in all projects or not + :param marker: entity ID from which to start returning entities. + :param limit: maximum number of entities to return. + :param sort_key: field to use for sorting. + :param sort_dir: direction of sorting: 'asc' or 'desc'. + :returns: list of string filters. + """ + filters = [] + if all_projects is True: + filters.append('all_projects=1') + if isinstance(limit, int): + filters.append('limit=%s' % limit) + if marker is not None: + filters.append('marker=%s' % marker) + if sort_key is not None: + filters.append('sort_key=%s' % sort_key) + if sort_dir is not None: + filters.append('sort_dir=%s' % sort_dir) + return filters + + +def split_and_deserialize(string): + """Split and try to JSON deserialize a string. + + Gets a string with the KEY=VALUE format, split it (using '=' as the + separator) and try to JSON deserialize the VALUE. + :returns: A tuple of (key, value). + """ + try: + key, value = string.split("=", 1) + except ValueError: + raise exc.CommandError(_('Attributes must be a list of ' + 'PATH=VALUE not "%s"') % string) + try: + value = json.loads(value) + except ValueError: + pass + + return (key, value) + + +def args_array_to_patch(attributes): + patch = [] + for attr in attributes: + path, value = split_and_deserialize(attr) + patch.append({path: value}) + return patch + + +def format_args(args, parse_comma=True): + '''Reformat a list of key-value arguments into a dict. + + Convert arguments into format expected by the API. + ''' + if not args: + return {} + + if parse_comma: + # expect multiple invocations of --label (or other arguments) but fall + # back to either , or ; delimited if only one --label is specified + if len(args) == 1: + args = args[0].replace(';', ',').split(',') + + fmt_args = {} + for arg in args: + try: + (k, v) = arg.split(('='), 1) + except ValueError: + raise exc.CommandError(_('arguments must be a list of KEY=VALUE ' + 'not %s') % arg) + if k not in fmt_args: + fmt_args[k] = v + else: + if not isinstance(fmt_args[k], list): + fmt_args[k] = [fmt_args[k]] + fmt_args[k].append(v) + + return fmt_args + + +def print_list_field(field): + return lambda obj: ', '.join(getattr(obj, field)) + + +def remove_null_parms(**kwargs): + new = {} + for (key, value) in kwargs.items(): + if value is not None: + new[key] = value + return new + + +def list_nodes(nodes): + columns = ('uuid', 'name', 'type', 'status') + utils.print_list(nodes, columns, + {'versions': print_list_field('versions')}, + sortby_index=None) + + +def list_models(models): + columns = ('uuid', 'name', 'type', 'status', 'state', 'deployed_url', + 'deployed_on') + utils.print_list(models, columns, + {'versions': print_list_field('versions')}, + sortby_index=None) + +def normalise_file_path_to_url(path): + if parse.urlparse(path).scheme: + return path + path = os.path.abspath(path) + return parse.urljoin('file:', request.pathname2url(path)) + + +def base_url_for_url(url): + parsed = parse.urlparse(url) + parsed_dir = os.path.dirname(parsed.path) + return parse.urljoin(url, parsed_dir) + + +def encode_file_data(data): + if six.PY3 and isinstance(data, str): + data = data.encode('utf-8') + return base64.b64encode(data).decode('utf-8') + + +def decode_file_data(data): + # Py3 raises binascii.Error instead of TypeError as in Py27 + try: + return base64.b64decode(data) + except (TypeError, binascii.Error): + raise exc.CommandError(_('Invalid Base 64 file data.')) diff --git a/gyanclient/exceptions.py b/gyanclient/exceptions.py new file mode 100644 index 0000000..27411c4 --- /dev/null +++ b/gyanclient/exceptions.py @@ -0,0 +1,60 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyanclient.common.apiclient import exceptions +from gyanclient.common.apiclient.exceptions import * # noqa + + +InvalidEndpoint = EndpointException +CommunicationError = ConnectionRefused +HTTPBadRequest = BadRequest +HTTPInternalServerError = InternalServerError +HTTPNotFound = NotFound +HTTPServiceUnavailable = ServiceUnavailable +CommandErrorException = CommandError + + +class AmbiguousAuthSystem(ClientException): + """Could not obtain token and endpoint using provided credentials.""" + pass + +# Alias for backwards compatibility +AmbigiousAuthSystem = AmbiguousAuthSystem + + +class InvalidAttribute(ClientException): + pass + + +def from_response(response, message=None, traceback=None, method=None, + url=None): + """Return an HttpError instance based on response from httplib/requests.""" + + error_body = {} + if message: + error_body['message'] = message + if traceback: + error_body['details'] = traceback + + if hasattr(response, 'status') and not hasattr(response, 'status_code'): + response.status_code = response.status + response.headers = { + 'Content-Type': response.getheader('content-type', "")} + + if hasattr(response, 'status_code'): + response.json = lambda: {'error': error_body} + + if (response.headers.get('Content-Type', '').startswith('text/') and + not hasattr(response, 'text')): + response.text = '' + + return exceptions.from_response(response, method, url) diff --git a/gyanclient/i18n.py b/gyanclient/i18n.py new file mode 100644 index 0000000..64e54bd --- /dev/null +++ b/gyanclient/i18n.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""oslo_i18n integration module for gyanclient. + +See https://docs.openstack.org/oslo.i18n/latest/user/usage.html. + +""" + +import oslo_i18n + + +_translators = oslo_i18n.TranslatorFactory(domain='gyanclient') + +# The primary translation function using the well-known name "_" +_ = _translators.primary diff --git a/gyanclient/shell.py b/gyanclient/shell.py new file mode 100644 index 0000000..ef5e945 --- /dev/null +++ b/gyanclient/shell.py @@ -0,0 +1,701 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +### +# This code is taken from python-zunclient. Goal is minimal modification. +### + +""" +Command-line interface to the OpenStack Gyan API. +""" + +from __future__ import print_function +import argparse +import getpass +import logging +import os +import sys + +from oslo_utils import encodeutils +from oslo_utils import importutils +from oslo_utils import strutils +import six + +profiler = importutils.try_import("osprofiler.profiler") + +HAS_KEYRING = False +all_errors = ValueError +try: + import keyring + HAS_KEYRING = True + try: + if isinstance(keyring.get_keyring(), keyring.backend.GnomeKeyring): + import gnomekeyring + all_errors = (ValueError, + gnomekeyring.IOError, + gnomekeyring.NoKeyringDaemonError) + except Exception: + pass +except ImportError: + pass + +from gyanclient import api_versions +from gyanclient import client as base_client +from gyanclient.common.apiclient import auth +from gyanclient.common import cliutils +from gyanclient import exceptions as exc +from gyanclient.i18n import _ +from gyanclient.v1 import shell as shell_v1 +from gyanclient import version + +DEFAULT_API_VERSION = api_versions.DEFAULT_API_VERSION +DEFAULT_ENDPOINT_TYPE = 'publicURL' +DEFAULT_SERVICE_TYPE = 'ml-infra' + +logger = logging.getLogger(__name__) + + +def positive_non_zero_float(text): + if text is None: + return None + try: + value = float(text) + except ValueError: + msg = "%s must be a float" % text + raise argparse.ArgumentTypeError(msg) + if value <= 0: + msg = "%s must be greater than 0" % text + raise argparse.ArgumentTypeError(msg) + return value + + +class SecretsHelper(object): + def __init__(self, args, client): + self.args = args + self.client = client + self.key = None + + def _validate_string(self, text): + if text is None or len(text) == 0: + return False + return True + + def _make_key(self): + if self.key is not None: + return self.key + keys = [ + self.client.auth_url, + self.client.projectid, + self.client.user, + self.client.region_name, + self.client.endpoint_type, + self.client.service_type, + self.client.service_name, + self.client.volume_service_name, + ] + for (index, key) in enumerate(keys): + if key is None: + keys[index] = '?' + else: + keys[index] = str(keys[index]) + self.key = "/".join(keys) + return self.key + + def _prompt_password(self, verify=True): + pw = None + if hasattr(sys.stdin, 'isatty') and sys.stdin.isatty(): + # Check for Ctl-D + try: + while True: + pw1 = getpass.getpass('OS Password: ') + if verify: + pw2 = getpass.getpass('Please verify: ') + else: + pw2 = pw1 + if pw1 == pw2 and self._validate_string(pw1): + pw = pw1 + break + except EOFError: + pass + return pw + + def save(self, auth_token, management_url, tenant_id): + if not HAS_KEYRING or not self.args.os_cache: + return + if (auth_token == self.auth_token and + management_url == self.management_url): + # Nothing changed.... + return + if not all([management_url, auth_token, tenant_id]): + raise ValueError("Unable to save empty management url/auth token") + value = "|".join([str(auth_token), + str(management_url), + str(tenant_id)]) + keyring.set_password("gyanclient_auth", self._make_key(), value) + + @property + def password(self): + if self._validate_string(self.args.os_password): + return self.args.os_password + verify_pass = ( + strutils.bool_from_string(cliutils.env("OS_VERIFY_PASSWORD")) + ) + return self._prompt_password(verify_pass) + + @property + def management_url(self): + if not HAS_KEYRING or not self.args.os_cache: + return None + management_url = None + try: + block = keyring.get_password('gyanclient_auth', + self._make_key()) + if block: + _token, management_url, _tenant_id = block.split('|', 2) + except all_errors: + pass + return management_url + + @property + def auth_token(self): + # Now is where it gets complicated since we + # want to look into the keyring module, if it + # exists and see if anything was provided in that + # file that we can use. + if not HAS_KEYRING or not self.args.os_cache: + return None + token = None + try: + block = keyring.get_password('gyanclient_auth', + self._make_key()) + if block: + token, _management_url, _tenant_id = block.split('|', 2) + except all_errors: + pass + return token + + @property + def tenant_id(self): + if not HAS_KEYRING or not self.args.os_cache: + return None + tenant_id = None + try: + block = keyring.get_password('gyanclient_auth', + self._make_key()) + if block: + _token, _management_url, tenant_id = block.split('|', 2) + except all_errors: + pass + return tenant_id + + +class GyanClientArgumentParser(argparse.ArgumentParser): + + def __init__(self, *args, **kwargs): + super(GyanClientArgumentParser, self).__init__(*args, **kwargs) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + """ + self.print_usage(sys.stderr) + # FIXME(lzyeval): if changes occur in argparse.ArgParser._check_value + choose_from = ' (choose from' + progparts = self.prog.partition(' ') + self.exit(2, "error: %(errmsg)s\nTry '%(mainp)s help %(subp)s'" + " for more information.\n" % + {'errmsg': message.split(choose_from)[0], + 'mainp': progparts[0], + 'subp': progparts[2]}) + + +class OpenStackGyanShell(object): + + def get_base_parser(self): + parser = GyanClientArgumentParser( + prog='gyan', + description=__doc__.strip(), + epilog='See "gyan help COMMAND" ' + 'for help on a specific command.', + add_help=False, + formatter_class=OpenStackHelpFormatter, + ) + + # Global arguments + parser.add_argument('-h', '--help', + action='store_true', + help=argparse.SUPPRESS) + + parser.add_argument('--version', + action='version', + version=version.version_info.version_string()) + + parser.add_argument('--debug', + default=False, + action='store_true', + help="Print debugging output.") + + parser.add_argument('--os-cache', + default=strutils.bool_from_string( + cliutils.env('OS_CACHE', default=False)), + action='store_true', + help="Use the auth token cache. Defaults to False " + "if env[OS_CACHE] is not set.") + + parser.add_argument('--os-region-name', + metavar='', + default=os.environ.get('OS_REGION_NAME'), + help='Region name. Default=env[OS_REGION_NAME].') + + +# TODO(mattf) - add get_timings support to Client +# parser.add_argument('--timings', +# default=False, +# action='store_true', +# help="Print call timing info") + +# TODO(mattf) - use timeout +# parser.add_argument('--timeout', +# default=600, +# metavar='', +# type=positive_non_zero_float, +# help="Set HTTP call timeout (in seconds)") + + parser.add_argument('--os-project-id', + metavar='', + default=cliutils.env('OS_PROJECT_ID', + default=None), + help='Defaults to env[OS_PROJECT_ID].') + + parser.add_argument('--os-project-name', + metavar='', + default=cliutils.env('OS_PROJECT_NAME', + default=None), + help='Defaults to env[OS_PROJECT_NAME].') + + parser.add_argument('--os-user-domain-id', + metavar='', + default=cliutils.env('OS_USER_DOMAIN_ID'), + help='Defaults to env[OS_USER_DOMAIN_ID].') + + parser.add_argument('--os-user-domain-name', + metavar='', + default=cliutils.env('OS_USER_DOMAIN_NAME'), + help='Defaults to env[OS_USER_DOMAIN_NAME].') + + parser.add_argument('--os-project-domain-id', + metavar='', + default=cliutils.env('OS_PROJECT_DOMAIN_ID'), + help='Defaults to env[OS_PROJECT_DOMAIN_ID].') + + parser.add_argument('--os-project-domain-name', + metavar='', + default=cliutils.env('OS_PROJECT_DOMAIN_NAME'), + help='Defaults to env[OS_PROJECT_DOMAIN_NAME].') + + parser.add_argument('--service-type', + metavar='', + help='Defaults to container for all ' + 'actions.') + parser.add_argument('--service_type', + help=argparse.SUPPRESS) + + parser.add_argument('--endpoint-type', + metavar='', + default=cliutils.env( + 'OS_ENDPOINT_TYPE', + default=DEFAULT_ENDPOINT_TYPE), + help='Defaults to env[OS_ENDPOINT_TYPE] or ' + + DEFAULT_ENDPOINT_TYPE + '.') + + parser.add_argument('--gyan-api-version', + metavar='', + default=cliutils.env( + 'GYAN_API_VERSION', + default=DEFAULT_API_VERSION), + help='Accepts X, X.Y (where X is major, Y is minor' + ' part), defaults to env[GYAN_API_VERSION].') + parser.add_argument('--gyan_api_version', + help=argparse.SUPPRESS) + + parser.add_argument('--os-cacert', + metavar='', + default=cliutils.env('OS_CACERT', default=None), + help='Specify a CA bundle file to use in ' + 'verifying a TLS (https) server certificate. ' + 'Defaults to env[OS_CACERT].') + + parser.add_argument('--bypass-url', + metavar='', + default=cliutils.env('BYPASS_URL', default=None), + dest='bypass_url', + help="Use this API endpoint instead of the " + "Service Catalog.") + parser.add_argument('--bypass_url', + help=argparse.SUPPRESS) + + parser.add_argument('--insecure', + default=cliutils.env('GYANCLIENT_INSECURE', + default=False), + action='store_true', + help="Do not verify https connections") + + if profiler: + parser.add_argument('--profile', + metavar='HMAC_KEY', + default=cliutils.env('OS_PROFILE', + default=None), + help='HMAC key to use for encrypting context ' + 'data for performance profiling of ' + 'operation. This key should be the ' + 'value of the HMAC key configured for ' + 'the OSprofiler middleware in gyan; it ' + 'is specified in the Gyan configuration ' + 'file at "/etc/gyan/gyan.conf". Without ' + 'the key, profiling functions will not ' + 'be triggered even if OSprofiler is ' + 'enabled on the server side.') + + # The auth-system-plugins might require some extra options + auth.load_auth_system_opts(parser) + + return parser + + def get_subcommand_parser(self, version, do_help=False): + parser = self.get_base_parser() + + self.subcommands = {} + subparsers = parser.add_subparsers(metavar='') + + actions_modules = shell_v1.COMMAND_MODULES + + for action_modules in actions_modules: + self._find_actions(subparsers, action_modules, version, do_help) + self._find_actions(subparsers, self, version, do_help) + + self._add_bash_completion_subparser(subparsers) + + return parser + + def _add_bash_completion_subparser(self, subparsers): + subparser = ( + subparsers.add_parser('bash_completion', + add_help=False, + formatter_class=OpenStackHelpFormatter) + ) + self.subcommands['bash_completion'] = subparser + subparser.set_defaults(func=self.do_bash_completion) + + def _find_actions(self, subparsers, actions_module, version, do_help): + msg = _(" (Supported by API versions '%(start)s' - '%(end)s')") + for attr in (a for a in dir(actions_module) if a.startswith('do_')): + # I prefer to be hyphen-separated instead of underscores. + command = attr[3:].replace('_', '-') + callback = getattr(actions_module, attr) + desc = callback.__doc__ or '' + if hasattr(callback, "versioned"): + subs = api_versions.get_substitutions(callback) + if do_help: + desc += msg % {'start': subs[0].start_version.get_string(), + 'end': subs[-1].end_version.get_string()} + else: + for versioned_method in subs: + if version.matches(versioned_method.start_version, + versioned_method.end_version): + callback = versioned_method.func + break + else: + continue + + action_help = desc.strip() + exclusive_args = getattr(callback, 'exclusive_args', {}) + arguments = getattr(callback, 'arguments', []) + + subparser = ( + subparsers.add_parser(command, + help=action_help, + description=desc, + add_help=False, + formatter_class=OpenStackHelpFormatter) + ) + subparser.add_argument('-h', '--help', + action='help', + help=argparse.SUPPRESS,) + self.subcommands[command] = subparser + + self._add_subparser_args(subparser, arguments, version, do_help, + msg) + self._add_subparser_exclusive_args(subparser, exclusive_args, + version, do_help, msg) + subparser.set_defaults(func=callback) + + def _add_subparser_exclusive_args(self, subparser, exclusive_args, + version, do_help, msg): + for group_name, arguments in exclusive_args.items(): + if group_name == '__required__': + continue + required = exclusive_args['__required__'][group_name] + exclusive_group = subparser.add_mutually_exclusive_group( + required=required) + self._add_subparser_args(exclusive_group, arguments, + version, do_help, msg) + + def _add_subparser_args(self, subparser, arguments, version, do_help, msg): + for (args, kwargs) in arguments: + start_version = kwargs.get("start_version", None) + if start_version: + start_version = api_versions.APIVersion(start_version) + end_version = kwargs.get("end_version", None) + if end_version: + end_version = api_versions.APIVersion(end_version) + else: + end_version = api_versions.APIVersion( + "%s.latest" % start_version.ver_major) + if do_help: + kwargs["help"] = kwargs.get("help", "") + (msg % { + "start": start_version.get_string(), + "end": end_version.get_string()}) + else: + if not version.matches(start_version, end_version): + continue + kw = kwargs.copy() + kw.pop("start_version", None) + kw.pop("end_version", None) + subparser.add_argument(*args, **kwargs) + + def setup_debugging(self, debug): + if debug: + streamformat = "%(levelname)s (%(module)s:%(lineno)d) %(message)s" + # Set up the root logger to debug so that the submodules can + # print debug messages + logging.basicConfig(level=logging.DEBUG, + format=streamformat) + else: + streamformat = "%(levelname)s %(message)s" + logging.basicConfig(level=logging.CRITICAL, + format=streamformat) + + def main(self, argv): + + argv = list(argv) + + # Parse args once to find version and debug settings + parser = self.get_base_parser() + (options, args) = parser.parse_known_args(argv) + self.setup_debugging(options.debug) + + api_version = api_versions.get_api_version(options.gyan_api_version) + + if '--endpoint_type' in argv: + spot = argv.index('--endpoint_type') + argv[spot] = '--endpoint-type' + + subcommand_parser = self.get_subcommand_parser( + api_version, do_help=("help" in args)) + + self.parser = subcommand_parser + + if options.help or not argv: + subcommand_parser.print_help() + return 0 + + args = subcommand_parser.parse_args(argv) + + # Short-circuit and deal with help right away. + if not hasattr(args, 'func') or args.func == self.do_help: + self.do_help(args) + return 0 + elif args.func == self.do_bash_completion: + self.do_bash_completion(args) + return 0 + + (os_username, os_project_name, os_project_id, + os_user_domain_id, os_user_domain_name, + os_project_domain_id, os_project_domain_name, + os_auth_url, os_auth_system, endpoint_type, + service_type, bypass_url, insecure, os_cacert) = ( + (args.os_username, args.os_project_name, args.os_project_id, + args.os_user_domain_id, args.os_user_domain_name, + args.os_project_domain_id, args.os_project_domain_name, + args.os_auth_url, args.os_auth_system, args.endpoint_type, + args.service_type, args.bypass_url, args.insecure, + args.os_cacert) + ) + + if os_auth_system and os_auth_system != "keystone": + auth_plugin = auth.load_plugin(os_auth_system) + else: + auth_plugin = None + + # Fetched and set later as needed + os_password = None + + if not endpoint_type: + endpoint_type = DEFAULT_ENDPOINT_TYPE + + if not service_type: + service_type = DEFAULT_SERVICE_TYPE + +# NA - there is only one service this CLI accesses +# service_type = utils.get_service_type(args.func) or service_type + + # FIXME(usrleon): Here should be restrict for project id same as + # for os_username or os_password but for compatibility it is not. + if not cliutils.isunauthenticated(args.func): + if auth_plugin: + auth_plugin.parse_opts(args) + + if not auth_plugin or not auth_plugin.opts: + if not os_username: + raise exc.CommandError("You must provide a username " + "via either --os-username or " + "env[OS_USERNAME]") + + if not os_project_name and not os_project_id: + raise exc.CommandError("You must provide a project name " + "or project id via --os-project-name, " + "--os-project-id, env[OS_PROJECT_NAME] " + "or env[OS_PROJECT_ID]") + + if not os_auth_url: + if os_auth_system and os_auth_system != 'keystone': + os_auth_url = auth_plugin.get_auth_url() + + if not os_auth_url: + raise exc.CommandError("You must provide an auth url " + "via either --os-auth-url or " + "env[OS_AUTH_URL] or specify an " + "auth_system which defines a " + "default url with --os-auth-system " + "or env[OS_AUTH_SYSTEM]") + + # Now check for the password/token of which pieces of the + # identifying keyring key can come from the underlying client + if not cliutils.isunauthenticated(args.func): + # NA - Client can't be used with SecretsHelper + if (auth_plugin and auth_plugin.opts and + "os_password" not in auth_plugin.opts): + use_pw = False + else: + use_pw = True + + if use_pw: + # Auth using token must have failed or not happened + # at all, so now switch to password mode and save + # the token when its gotten... using our keyring + # saver + os_password = args.os_password + if not os_password: + raise exc.CommandError( + 'Expecting a password provided via either ' + '--os-password, env[OS_PASSWORD], or ' + 'prompted response') + + client = base_client + + kwargs = {} + if profiler: + kwargs["profile"] = args.profile + + self.cs = client.Client(version=api_version, + username=os_username, + password=os_password, + project_id=os_project_id, + project_name=os_project_name, + user_domain_id=os_user_domain_id, + user_domain_name=os_user_domain_name, + project_domain_id=os_project_domain_id, + project_domain_name=os_project_domain_name, + auth_url=os_auth_url, + service_type=service_type, + region_name=args.os_region_name, + endpoint_override=bypass_url, + interface=endpoint_type, + insecure=insecure, + cacert=os_cacert, + **kwargs) + + args.func(self.cs, args) + + if profiler and args.profile: + trace_id = profiler.get().get_base_id() + print("To display trace use the command:\n\n" + " osprofiler trace show --html %s " % trace_id) + + def _dump_timings(self, timings): + class Tyme(object): + def __init__(self, url, seconds): + self.url = url + self.seconds = seconds + results = [Tyme(url, end - start) for url, start, end in timings] + total = 0.0 + for tyme in results: + total += tyme.seconds + results.append(Tyme("Total", total)) + cliutils.print_list(results, ["url", "seconds"], sortby_index=None) + + def do_bash_completion(self, _args): + """Prints arguments for bash-completion. + + Prints all of the commands and options to stdout so that the + gyan.bash_completion script doesn't have to hard code them. + """ + commands = set() + options = set() + for sc_str, sc in self.subcommands.items(): + commands.add(sc_str) + for option in sc._optionals._option_string_actions.keys(): + options.add(option) + + commands.remove('bash-completion') + commands.remove('bash_completion') + print(' '.join(commands | options)) + + @cliutils.arg('command', metavar='', nargs='?', + help='Display help for .') + def do_help(self, args): + """Display help about this program or one of its subcommands.""" + command = getattr(args, 'command', '') + if command: + if args.command in self.subcommands: + self.subcommands[args.command].print_help() + else: + raise exc.CommandError("'%s' is not a valid subcommand" % + args.command) + else: + self.parser.print_help() + + +# I'm picky about my shell help. +class OpenStackHelpFormatter(argparse.HelpFormatter): + def start_section(self, heading): + # Title-case the headings + heading = '%s%s' % (heading[0].upper(), heading[1:]) + super(OpenStackHelpFormatter, self).start_section(heading) + + +def main(): + try: + return OpenStackGyanShell().main( + map(encodeutils.safe_decode, sys.argv[1:])) + except Exception as e: + logger.debug(e, exc_info=1) + print("ERROR: %s" % encodeutils.safe_encode(six.text_type(e)), + file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gyanclient/v1/__init__.py b/gyanclient/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyanclient/v1/client.py b/gyanclient/v1/client.py new file mode 100644 index 0000000..9ceb463 --- /dev/null +++ b/gyanclient/v1/client.py @@ -0,0 +1,125 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keystoneauth1 import loading +from keystoneauth1 import session as ksa_session + +from gyanclient.common import httpclient +from gyanclient.v1 import nodes +from gyanclient.v1 import models +from gyanclient.v1 import versions + + +class Client(object): + """Top-level object to access the OpenStack Gyan API.""" + + def __init__(self, api_version=None, auth_token=None, + auth_type='password', auth_url=None, endpoint_override=None, + interface='public', insecure=False, password=None, + project_domain_id=None, project_domain_name=None, + project_id=None, project_name=None, region_name=None, + service_name=None, service_type='container', session=None, + user_domain_id=None, user_domain_name=None, + username=None, cacert=None, **kwargs): + """Initialization of Client object. + + :param api_version: Gyan API version + :type api_version: gyanclient.api_version.APIVersion + :param str auth_token: Auth token + :param str auth_url: Auth URL + :param str auth_type: Auth Type + :param str endpoint_override: Bypass URL + :param str interface: Interface + :param str insecure: Allow insecure + :param str password: User password + :param str project_domain_id: ID of project domain + :param str project_domain_name: Nam of project domain + :param str project_id: Project/Tenant ID + :param str project_name: Project/Tenant Name + :param str region_name: Region Name + :param str service_name: Service Name + :param str service_type: Service Type + :param str session: Session + :param str user_domain_id: ID of user domain + :param str user_id: User ID + :param str username: Username + :param str cacert: CA certificate + """ + if endpoint_override and auth_token: + auth_type = 'admin_token' + session = None + loader_kwargs = { + 'token': auth_token, + 'endpoint': endpoint_override + } + elif auth_token and not session: + auth_type = 'token' + loader_kwargs = { + 'token': auth_token, + 'auth_url': auth_url, + 'project_domain_id': project_domain_id, + 'project_domain_name': project_domain_name, + 'project_id': project_id, + 'project_name': project_name, + 'user_domain_id': user_domain_id, + 'user_domain_name': user_domain_name + } + else: + loader_kwargs = { + 'auth_url': auth_url, + 'password': password, + 'project_domain_id': project_domain_id, + 'project_domain_name': project_domain_name, + 'project_id': project_id, + 'project_name': project_name, + 'user_domain_id': user_domain_id, + 'user_domain_name': user_domain_name, + 'username': username, + } + + # Backwards compatibility for people not passing in Session + if session is None: + loader = loading.get_plugin_loader(auth_type) + # This should be able to handle v2 and v3 Keystone Auth + auth_plugin = loader.load_from_options(**loader_kwargs) + session = ksa_session.Session(auth=auth_plugin, + verify=(cacert or not insecure)) + client_kwargs = {} + if not endpoint_override: + try: + # Trigger an auth error so that we can throw the exception + # we always have + session.get_endpoint( + service_name=service_name, + service_type=service_type, + interface=interface, + region_name=region_name + ) + except Exception: + raise RuntimeError('Not authorized') + else: + client_kwargs = {'endpoint_override': endpoint_override} + + self.http_client = httpclient.SessionClient(service_type=service_type, + service_name=service_name, + interface=interface, + region_name=region_name, + session=session, + api_version=api_version, + **client_kwargs) + self.containers = nodes.NodeManager(self.http_client) + self.images = models.ModelManager(self.http_client) + self.versions = versions.VersionManager(self.http_client) + + @property + def api_version(self): + return self.http_client.api_version diff --git a/gyanclient/v1/models.py b/gyanclient/v1/models.py new file mode 100644 index 0000000..bd12388 --- /dev/null +++ b/gyanclient/v1/models.py @@ -0,0 +1,112 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from six.moves.urllib import parse + +from gyanclient import api_versions +from gyanclient.common import base +from gyanclient.common import utils +from gyanclient import exceptions + + +CREATION_ATTRIBUTES = ['name', 'image', 'command', 'cpu', 'memory', + 'environment', 'workdir', 'labels', 'image_pull_policy', + 'restart_policy', 'interactive', 'image_driver', + 'security_groups', 'hints', 'nets', 'auto_remove', + 'runtime', 'hostname', 'mounts', 'disk', + 'availability_zone', 'auto_heal', 'privileged', + 'exposed_ports', 'healthcheck'] + + +class Model(base.Resource): + def __repr__(self): + return "" % self._info + + +class ModelManager(base.Manager): + resource_class = Model + + @staticmethod + def _path(id=None): + + if id: + return '/v1/ml-models/%s' % id + else: + return '/v1/ml-models' + + def list_models(self, **kwargs): + """Retrieve a list of Models. + + :returns: A list of models. + + """ + + return self._list_pagination(self._path(''), + "models") + + def get(self, id): + try: + return self._list(self._path(id))[0] + except IndexError: + return None + + def model_train(self, **kwargs): + new = {} + new['name'] = kwargs["name"] + new['ml_file'] = kwargs["ml_file"] + return self._create(self._path(), new) + + def delete_model(self, id): + return self._delete(self._path(id)) + + def _action(self, id, action, method='POST', **kwargs): + kwargs.setdefault('headers', {}) + kwargs['headers'].setdefault('Content-Length', '0') + resp, body = self.api.json_request(method, + self._path(id) + action, + **kwargs) + return resp, body + + def deploy_model(self, id): + return self._action(id, '/deploy') + + def undeploy_model(self, id): + return self._action(id, '/unstop') + + def rebuild(self, id, **kwargs): + return self._action(id, '/rebuild', + qparams=kwargs) + + def restart(self, id, timeout): + return self._action(id, '/reboot', + qparams={'timeout': timeout}) + + def pause(self, id): + return self._action(id, '/pause') + + def unpause(self, id): + return self._action(id, '/unpause') + + def logs(self, id, **kwargs): + if kwargs['stdout'] is False and kwargs['stderr'] is False: + kwargs['stdout'] = True + kwargs['stderr'] = True + return self._action(id, '/logs', method='GET', + qparams=kwargs)[1] + + def execute(self, id, **kwargs): + return self._action(id, '/execute', + qparams=kwargs)[1] + + def execute_resize(self, id, exec_id, width, height): + self._action(id, '/execute_resize', + qparams={'exec_id': exec_id, 'w': width, 'h': height})[1] diff --git a/gyanclient/v1/models_shell.py b/gyanclient/v1/models_shell.py new file mode 100644 index 0000000..2b3ad9f --- /dev/null +++ b/gyanclient/v1/models_shell.py @@ -0,0 +1,116 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +from contextlib import closing +import io +import json +import os +import tarfile +import time +import yaml + +from gyanclient.common import cliutils as utils +from gyanclient.common import utils as gyan_utils +from gyanclient import exceptions as exc + + +def _show_model(model): + utils.print_dict(model._info) + + +@utils.arg('model-id', + metavar='', + nargs='+', + help='ID of the model to delete.') +def do_model_delete(cs, args): + """Delete specified model.""" + opts = {} + opts['id'] = args.model_id + opts = gyan_utils.remove_null_parms(**opts) + try: + cs.models.delete_model(**opts) + print("Request to delete model %s has been accepted." % + args.model_id) + except Exception as e: + print("Delete for model %(model)s failed: %(e)s" % + {'model': args.model_id, 'e': e}) + + +@utils.arg('model-id', + metavar='', + help='ID or name of the model to show.') +def do_model_show(cs, args): + """Show details of a container.""" + opts = {} + opts['model_id'] = args.model_id + opts = gyan_utils.remove_null_parms(**opts) + model = cs.models.get(**opts) + _show_model(model) + + +@utils.arg('model-id', + metavar='', + help='ID of the model to be deployed') +def do_undeploy_model(cs, args): + """Undeploy the model.""" + opts = {} + opts['model_id'] = args.model_id + opts = gyan_utils.remove_null_parms(**opts) + try: + model = cs.models.undeploy_model(**opts) + _show_model(model) + except Exception as e: + print("Undeployment of the model %(model)s " + "failed: %(e)s" % {'model': args.model_id, 'e': e}) + + +@utils.arg('model-id', + metavar='', + help='ID of the model to be deployed') +def do_deploy_model(cs, args): + """Deploy already created model.""" + opts = {} + opts['model_id'] = args.model_id + opts = gyan_utils.remove_null_parms(**opts) + try: + model = cs.models.deploy_model(**opts) + _show_model(model) + except Exception as e: + print("Deployment of the model %(model)s " + "failed: %(e)s" % {'model': args.model_id, 'e': e}) + + +def do_model_list(cs, args): + """List models""" + models = cs.models.list_models() + gyan_utils.list_models(models) + + +@utils.arg('name', + metavar='', + help='ID or name of the model to train') +@utils.arg('--ml-file', + metavar='', + help='The ML model file to be trained') +def do_train_model(cs, args): + """Remove security group for specified container.""" + opts = {} + opts['name'] = args.name + opts = gyan_utils.remove_null_parms(**opts) + try: + opts['ml_file'] = yaml.load(open(args.ml_file)) + models = cs.models.model_train(**opts) + gyan_utils.list_models(models) + except Exception as e: + print("Creation of model %(model)s " + "failed: %(e)s" % {'model': args.name, 'e': e}) diff --git a/gyanclient/v1/nodes.py b/gyanclient/v1/nodes.py new file mode 100644 index 0000000..fb5c1e2 --- /dev/null +++ b/gyanclient/v1/nodes.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from six.moves.urllib import parse + +from gyanclient import api_versions +from gyanclient.common import base +from gyanclient.common import utils +from gyanclient import exceptions + + +class Node(base.Resource): + def __repr__(self): + return "" % self._info + + +class NodeManager(base.Manager): + resource_class = Node + + @staticmethod + def _path(id=None): + + if id: + return '/v1/ml-nodes/%s' % id + else: + return '/v1/ml-nodes' + + def list_nodes(self, **kwargs): + """Retrieve a list of Nodes. + + :returns: A list of nodes. + + """ + + return self._list_pagination(self._path(''), + "nodes") + + def get(self, id): + try: + return self._list(self._path(id))[0] + except IndexError: + return None \ No newline at end of file diff --git a/gyanclient/v1/nodes_shell.py b/gyanclient/v1/nodes_shell.py new file mode 100644 index 0000000..a676874 --- /dev/null +++ b/gyanclient/v1/nodes_shell.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +from contextlib import closing +import io +import json +import os +import tarfile +import time +import yaml + +from gyanclient.common import cliutils as utils +from gyanclient.common import utils as gyan_utils +from gyanclient import exceptions as exc + + +def _show_node(node): + utils.print_dict(node._info) + + +@utils.arg('node-id', + metavar='', + help='ID or name of the node to show.') +def do_node_show(cs, args): + """Show details of a container.""" + opts = {} + opts['node_id'] = args.node_id + opts = gyan_utils.remove_null_parms(**opts) + node = cs.nodes.get(**opts) + _show_node(node) + + +def do_node_list(cs, args): + """List Nodes""" + nodes = cs.nodes.list_nodes() + gyan_utils.list_nodes(nodes) \ No newline at end of file diff --git a/gyanclient/v1/shell.py b/gyanclient/v1/shell.py new file mode 100644 index 0000000..5eeadaa --- /dev/null +++ b/gyanclient/v1/shell.py @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from gyanclient.v1 import nodes_shell +from gyanclient.v1 import models_shell +from gyanclient.v1 import versions_shell + +COMMAND_MODULES = [ + nodes_shell, + models_shell, + versions_shell +] diff --git a/gyanclient/v1/versions.py b/gyanclient/v1/versions.py new file mode 100644 index 0000000..2f56b4f --- /dev/null +++ b/gyanclient/v1/versions.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyanclient.common import base + + +class Version(base.Resource): + def __repr__(self): + return "" + + +class VersionManager(base.Manager): + resource_class = Version + + def list(self): + url = "%s" % self.api.get_endpoint() + url = "%s/" % url.rsplit("/", 1)[0] + return self._list(url, "versions") diff --git a/gyanclient/v1/versions_shell.py b/gyanclient/v1/versions_shell.py new file mode 100644 index 0000000..2a81ea7 --- /dev/null +++ b/gyanclient/v1/versions_shell.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyanclient import api_versions +from gyanclient.common import cliutils as utils + + +def do_version_list(cs, args): + """List all API versions.""" + print("Client supported API versions:") + print("Minimum version %(v)s" % + {'v': api_versions.MIN_API_VERSION}) + print("Maximum version %(v)s" % + {'v': api_versions.MAX_API_VERSION}) + + print("\nServer supported API versions:") + result = cs.versions.list() + columns = ["Id", "Status", "Min Version", "Max Version"] + utils.print_list(result, columns) diff --git a/gyanclient/version.py b/gyanclient/version.py new file mode 100644 index 0000000..50c5396 --- /dev/null +++ b/gyanclient/version.py @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pbr import version + +version_info = version.VersionInfo('python-gyanclient') diff --git a/setup.cfg b/setup.cfg index 6927e80..e0ff727 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,3 +35,19 @@ input_file = gyanclient/locale/gyanclient.pot keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = gyanclient/locale/gyanclient.pot + +[entry_points] +console_scripts = + gyan = gyanclient.shell:main + +[build_releasenotes] +all_files = 1 +build-dir = releasenotes/build +source-dir = releasenotes/source + +[wheel] +universal = 1 + +[global] +setup-hooks = + pbr.hooks.setup_hook diff --git a/setup.py b/setup.py index 056c16c..98b93eb 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at