x
/
meteos
Code Issues Proposed changes

Refactoring for pep8 

implements blueprint unit-testing

Change-Id: Ie80a1f26c167c2b36f6c06852e2d625b58f4c614
This commit is contained in:
Hiroyuki Eguchi 2017-01-22 15:16:25 +09:00
parent 10d6362b61
commit 786806e5d3
31 changed files with 150 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
meteos/api/openstack/__init__.py
View File

@ -21,7 +21,6 @@ WSGI middleware for OpenStack API controllers.
from oslo_log import log
import routes
from meteos.api.openstack import wsgi
from meteos.i18n import _, _LW
from meteos import wsgi as base_wsgi

6
meteos/api/v1/datasets.py
View File

@ -16,13 +16,7 @@
"""The datasets api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

6
meteos/api/v1/experiments.py
View File

@ -16,13 +16,7 @@
"""The experiments api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

6
meteos/api/v1/learnings.py
View File

@ -16,13 +16,7 @@
"""The learnings api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

9
meteos/api/v1/model_evaluations.py
View File

@ -16,13 +16,7 @@
"""The model_evaluations api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc
@ -52,7 +46,8 @@ class ModelEvaluationController(wsgi.Controller, wsgi.AdminActionsMixin):
context = req.environ['meteos.context']
try:
model_evaluation = self.engine_api.get_model_evaluation(context, id)
model_evaluation = self.engine_api.get_model_evaluation(context,
id)
except exception.NotFound:
raise exc.HTTPNotFound()

6
meteos/api/v1/models.py
View File

@ -16,13 +16,7 @@
"""The models api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

8
meteos/api/v1/router.py
View File

@ -20,12 +20,12 @@ WSGI middleware for OpenStack Learning API v1.
"""
import meteos.api.openstack
from meteos.api.v1 import learnings
from meteos.api.v1 import experiments
from meteos.api.v1 import templates
from meteos.api.v1 import datasets
from meteos.api.v1 import models
from meteos.api.v1 import experiments
from meteos.api.v1 import learnings
from meteos.api.v1 import model_evaluations
from meteos.api.v1 import models
from meteos.api.v1 import templates
from meteos.api import versions

6
meteos/api/v1/templates.py
View File

@ -16,13 +16,7 @@
"""The templates api."""
import ast
import re
import string
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

1
meteos/api/versions.py
View File

@ -19,7 +19,6 @@ import copy
from oslo_config import cfg
from meteos.api import openstack
from meteos.api.openstack import api_version_request
from meteos.api.openstack import wsgi
from meteos.api.views import versions as views_versions

2
meteos/api/views/datasets.py
View File

@ -47,8 +47,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, dataset):
"""Detailed view of a single dataset."""
context = request.environ['meteos.context']
dataset_dict = {
'id': dataset.get('id'),
'created_at': dataset.get('created_at'),

2
meteos/api/views/experiments.py
View File

@ -46,8 +46,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, experiment):
"""Detailed view of a single experiment."""
context = request.environ['meteos.context']
experiment_dict = {
'id': experiment.get('id'),
'created_at': experiment.get('created_at'),

2
meteos/api/views/learnings.py
View File

@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, learning):
"""Detailed view of a single learning."""
context = request.environ['meteos.context']
learning_dict = {
'id': learning.get('id'),
'created_at': learning.get('created_at'),

12
meteos/api/views/model_evaluations.py
View File

@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, model_evaluation):
"""Detailed view of a single model evaluation."""
context = request.environ['meteos.context']
model_evaluation_dict = {
'id': model_evaluation.get('id'),
'name': model_evaluation.get('display_name'),
@ -65,17 +63,19 @@ class ViewBuilder(common.ViewBuilder):
'stderr': model_evaluation.get('stderr'),
}
self.update_versioned_resource_dict(request, model_evaluation_dict, model_evaluation)
self.update_versioned_resource_dict(request,
model_evaluation_dict,
model_evaluation)
return {'model_evaluation': model_evaluation_dict}
def _list_view(self, func, request, model_evaluations):
"""Provide a view for a list of model evaluations."""
model_evaluations_list = [func(request, model_evaluation)['model_evaluation']
for model_evaluation in model_evaluations]
for model_evaluation in model_evaluations]
model_evaluations_links = self._get_collection_links(request,
model_evaluations,
self._collection_name)
model_evaluations,
self._collection_name)
model_evaluations_dict = dict(model_evaluations=model_evaluations_list)
if model_evaluations_links:

2
meteos/api/views/models.py
View File

@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, model):
"""Detailed view of a single model."""
context = request.environ['meteos.context']
model_dict = {
'id': model.get('id'),
'created_at': model.get('created_at'),

2
meteos/api/views/templates.py
View File

@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder):
def detail(self, request, template):
"""Detailed view of a single template."""
context = request.environ['meteos.context']
template_dict = {
'id': template.get('id'),
'created_at': template.get('created_at'),

43
meteos/cluster/binary/meteos-script-1.6.0.py
View File

@ -32,35 +32,40 @@
import base64
import os
import socket
import sys
import uuid
import socket
from ast import literal_eval
from numpy import array
from math import sqrt
from numpy import array
from pyspark import SparkContext
from pyspark.mllib.linalg import SparseVector
from pyspark.mllib.classification import LogisticRegressionWithSGD
from pyspark.mllib.classification import LogisticRegressionModel
from pyspark.mllib.clustering import KMeans, KMeansModel
from pyspark.mllib.classification import LogisticRegressionWithSGD
from pyspark.mllib.clustering import KMeans
from pyspark.mllib.clustering import KMeansModel
from pyspark.mllib.evaluation import BinaryClassificationMetrics
from pyspark.mllib.evaluation import MulticlassMetrics
from pyspark.mllib.evaluation import RankingMetrics
from pyspark.mllib.evaluation import RegressionMetrics
from pyspark.mllib.feature import Word2Vec
from pyspark.mllib.feature import Word2VecModel
from pyspark.mllib.fpm import FPGrowth
from pyspark.mllib.fpm import FPGrowthModel
from pyspark.mllib.evaluation import BinaryClassificationMetrics
from pyspark.mllib.evaluation import RegressionMetrics
from pyspark.mllib.evaluation import MulticlassMetrics
from pyspark.mllib.evaluation import RankingMetrics
from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating
from pyspark.mllib.linalg import SparseVector
from pyspark.mllib.recommendation import ALS
from pyspark.mllib.recommendation import MatrixFactorizationModel
from pyspark.mllib.recommendation import Rating
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.regression import LinearRegressionWithSGD
from pyspark.mllib.regression import LinearRegressionModel
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
from pyspark.mllib.regression import LinearRegressionWithSGD
from pyspark.mllib.tree import DecisionTree
from pyspark.mllib.tree import DecisionTreeModel
from pyspark.mllib.util import MLUtils
EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9'
EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9'
class ModelController(object):
@ -155,12 +160,11 @@ class RecommendationController(ModelController):
testData = ratings.map(lambda p: (p.user, p.product))
predictions = model.predictAll(testData)\
.map(lambda r: ((r.user, r.product), r.rating))
.map(lambda r: ((r.user, r.product), r.rating))
ratingsTuple = ratings.map(lambda r: ((r.user, r.product), r.rating))
scoreAndLabels = predictions.join(ratingsTuple).map(lambda tup: tup[1])
# Instantiate regression metrics to compare predicted and actual ratings
metrics = RegressionMetrics(scoreAndLabels)
result = "{}: {}".format("MAE", metrics.meanAbsoluteError) + os.linesep\
@ -287,7 +291,7 @@ class DecisionTreeModelController(ModelController):
index = ','.join(index_l)
value = ','.join(value_l)
parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])'
parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])'
return SparseVector.parse(parsed_str)
@ -378,7 +382,7 @@ class FPGrowthModelController(ModelController):
transactions = data.map(lambda line: line.strip().split(' '))
model= FPGrowth.train(transactions,
model = FPGrowth.train(transactions,
minSupport=minSupport,
numPartitions=numPartitions)
@ -431,7 +435,7 @@ class MeteosSparkController(object):
if collect:
self.data.collect()
self.data.saveAsTextFile(self.datapath)
print self.data.take(10)
print(self.data.take(10))
def load_data(self):
@ -475,7 +479,8 @@ class MeteosSparkController(object):
dataset_format = self.job_args.get('dataset_format')
if dataset_format == 'libsvm':
self.model = self.controller.create_model_libsvm(self.data, list_params)
self.model = self.controller.create_model_libsvm(self.data,
list_params)
else:
self.model = self.controller.create_model(self.data, list_params)

10
meteos/cluster/sahara.py
View File

@ -18,20 +18,14 @@
Handles all requests relating to volumes + sahara.
"""
import copy
import os
from osc_lib import exceptions as sahara_exception
from saharaclient import client as sahara_client
from keystoneauth1 import loading as ks_loading
from osc_lib import exceptions as sahara_exception
from oslo_config import cfg
import six
from saharaclient import client as sahara_client
from meteos.common import client_auth
from meteos.common.config import core_opts
import meteos.context as ctxt
from meteos.db import base
from meteos import exception
from meteos.i18n import _
SAHARA_GROUP = 'sahara'

4
meteos/common/config.py
View File

@ -30,11 +30,7 @@ from oslo_config import cfg
from oslo_log import log
from oslo_middleware import cors
from oslo_utils import netutils
import six
from meteos.common import constants
from meteos import exception
from meteos.i18n import _
CONF = cfg.CONF
log.register_options(CONF)

5
meteos/db/api.py
View File

@ -307,7 +307,8 @@ def model_evaluation_get(context, model_evaluation_id):
return IMPL.model_evaluation_get(context, model_evaluation_id)
def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None):
def model_evaluation_get_all(context, filters=None,
sort_key=None, sort_dir=None):
"""Get all model_evaluations."""
return IMPL.model_evaluation_get_all(
context, filters=filters, sort_key=sort_key, sort_dir=sort_dir,
@ -315,7 +316,7 @@ def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None
def model_evaluation_get_all_by_project(context, project_id, filters=None,
sort_key=None, sort_dir=None):
sort_key=None, sort_dir=None):
"""Returns all model_evaluations with given project ID."""
return IMPL.model_evaluation_get_all_by_project(
context, project_id, filters=filters,

4
meteos/db/migrations/alembic/versions/001_meteos_init.py
View File

@ -26,8 +26,8 @@ down_revision = None
from alembic import op
from oslo_log import log
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Text
from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint
from sqlalchemy import Boolean, Column, DateTime, Text
from sqlalchemy import Integer, MetaData, String, Table
from meteos.i18n import _LE

38
meteos/db/sqlalchemy/api.py
View File

@ -19,10 +19,8 @@
"""Implementation of SQLAlchemy backend."""
import copy
import datetime
from functools import wraps
import sys
import uuid
import warnings
# NOTE(uglide): Required to override default oslo_db Query class
@ -30,21 +28,13 @@ import meteos.db.sqlalchemy.query # noqa
from oslo_config import cfg
from oslo_db import api as oslo_db_api
from oslo_db import exception as db_exception
from oslo_db import options as db_options
from oslo_db.sqlalchemy import session
from oslo_db.sqlalchemy import utils as db_utils
from oslo_log import log
from oslo_utils import timeutils
from oslo_utils import uuidutils
import six
from sqlalchemy import and_
from sqlalchemy import or_
from sqlalchemy.orm import joinedload
from sqlalchemy.sql.expression import true
from sqlalchemy.sql import func
from meteos.common import constants
from meteos.db.sqlalchemy import models
from meteos import exception
from meteos.i18n import _, _LE, _LW
@ -239,19 +229,6 @@ def ensure_dict_has_id(model_dict):
return model_dict
def _sync_learnings(context, project_id, user_id, session):
(learnings, gigs) = learning_data_get_for_project(context,
project_id,
user_id,
session=session)
return {'learnings': learnings}
QUOTA_SYNC_FUNCTIONS = {
'_sync_learnings': _sync_learnings,
}
#
@ -857,11 +834,14 @@ def model_evaluation_create(context, model_evaluation_values):
model_evaluation_ref.save(session=session)
# NOTE(u_glide): Do so to prevent errors with relationships
return model_evaluation_get(context, model_evaluation_ref['id'], session=session)
return model_evaluation_get(context,
model_evaluation_ref['id'],
session=session)
def _model_evaluation_get_all_with_filters(context, project_id=None, filters=None,
sort_key=None, sort_dir=None):
def _model_evaluation_get_all_with_filters(context, project_id=None,
filters=None, sort_key=None,
sort_dir=None):
if not sort_key:
sort_key = 'created_at'
if not sort_dir:
@ -899,7 +879,7 @@ def _model_evaluation_get_all_with_filters(context, project_id=None, filters=Non
@require_context
def model_evaluation_get_all_by_project(context, project_id, filters=None,
sort_key=None, sort_dir=None):
sort_key=None, sort_dir=None):
"""Returns list of model_evaluations with given project ID."""
query = _model_evaluation_get_all_with_filters(
context, project_id=project_id, filters=filters,
@ -913,7 +893,9 @@ def model_evaluation_delete(context, model_evaluation_id):
session = get_session()
with session.begin():
model_evaluation_ref = model_evaluation_get(context, model_evaluation_id, session)
model_evaluation_ref = model_evaluation_get(context,
model_evaluation_id,
session)
model_evaluation_ref.soft_delete(session=session)

5
meteos/db/sqlalchemy/models.py
View File

@ -22,10 +22,9 @@ SQLAlchemy models for Meteos data.
from oslo_config import cfg
from oslo_db.sqlalchemy import models
from oslo_log import log
from sqlalchemy import Column, Integer, String, schema
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import orm
from sqlalchemy import ForeignKey, DateTime, Boolean, Enum, Text
from sqlalchemy import DateTime, Boolean, Text
from meteos.common import constants

77
meteos/engine/api.py
View File

@ -20,20 +20,16 @@
Handles all requests relating to learnings.
"""
from oslo_config import cfg
from oslo_log import log
from oslo_utils import excutils
from oslo_utils import strutils
from oslo_utils import timeutils
import six
from meteos.common import constants
from meteos.db import base
from meteos.engine import rpcapi as engine_rpcapi
from meteos import exception
from meteos.i18n import _, _LE, _LI, _LW
from meteos import policy
from meteos.engine import rpcapi as engine_rpcapi
from meteos import utils
LOG = log.getLogger(__name__)
@ -121,7 +117,7 @@ class API(base.Base):
"statuses": statuses}
raise exception.InvalidLearning(reason=msg)
result = self.engine_rpcapi.delete_template(context, id)
self.engine_rpcapi.delete_template(context, id)
def get_all_experiments(self, context, search_opts=None,
sort_key='created_at', sort_dir='desc'):
@ -199,7 +195,7 @@ class API(base.Base):
"statuses": statuses}
raise exception.InvalidLearning(reason=msg)
result = self.engine_rpcapi.delete_experiment(context, id)
self.engine_rpcapi.delete_experiment(context, id)
def get_all_datasets(self, context, search_opts=None,
sort_key='created_at', sort_dir='desc'):
@ -286,10 +282,10 @@ class API(base.Base):
"statuses": statuses}
raise exception.InvalidLearning(reason=msg)
result = self.engine_rpcapi.delete_dataset(context,
dataset['cluster_id'],
dataset['job_id'],
id)
self.engine_rpcapi.delete_dataset(context,
dataset['cluster_id'],
dataset['job_id'],
id)
def _enable_load_model(self, context):
@ -387,10 +383,10 @@ class API(base.Base):
"statuses": statuses}
raise exception.InvalidLearning(reason=msg)
result = self.engine_rpcapi.delete_model(context,
model['cluster_id'],
model['job_id'],
id)
self.engine_rpcapi.delete_model(context,
model['cluster_id'],
model['job_id'],
id)
def load_model(self, context, id, dataset_format, model_type,
job_template_id, experiment_id, cluster_id):
@ -445,20 +441,22 @@ class API(base.Base):
self.db.model_update(context, id, updates)
def get_all_model_evaluations(self, context, search_opts=None,
sort_key='created_at', sort_dir='desc'):
sort_key='created_at', sort_dir='desc'):
policy.check_policy(context, 'model_evaluation', 'get_all')
if search_opts is None:
search_opts = {}
LOG.debug("Searching for model evaluations by: %s", six.text_type(search_opts))
LOG.debug("Searching for model evaluations by: %s",
six.text_type(search_opts))
project_id = context.project_id
model_evaluations = self.db.model_evaluation_get_all_by_project(context,
project_id,
sort_key=sort_key,
sort_dir=sort_dir)
model_evaluations = self.db.model_evaluation_get_all_by_project(
context,
project_id,
sort_key=sort_key,
sort_dir=sort_dir)
if search_opts:
results = []
@ -474,22 +472,23 @@ class API(base.Base):
return rv
def create_model_evaluation(self, context, name, source_dataset_url,
model_id, model_type, dataset_format, template_id,
job_template_id, experiment_id, cluster_id,
swift_tenant, swift_username, swift_password):
model_id, model_type, dataset_format,
template_id, job_template_id, experiment_id,
cluster_id, swift_tenant, swift_username,
swift_password):
"""Create a Model Evaluation"""
policy.check_policy(context, 'model_evaluation', 'create')
model_evaluation = {'id': None,
'display_name': name,
'model_id': model_id,
'model_type': model_type,
'source_dataset_url': source_dataset_url,
'dataset_format': dataset_format,
'user_id': context.user_id,
'project_id': context.project_id,
'cluster_id': cluster_id
}
'display_name': name,
'model_id': model_id,
'model_type': model_type,
'source_dataset_url': source_dataset_url,
'dataset_format': dataset_format,
'user_id': context.user_id,
'project_id': context.project_id,
'cluster_id': cluster_id
}
try:
result = self.db.model_evaluation_create(context, model_evaluation)
@ -505,7 +504,8 @@ class API(base.Base):
result['id'],
updates)
LOG.info(_LI("Accepted creation of model evaluation %s."), result['id'])
LOG.info(_LI("Accepted creation of model evaluation %s."),
result['id'])
except Exception:
with excutils.save_and_reraise_exception():
self.db.model_evaluation_delete(context, result['id'])
@ -530,10 +530,11 @@ class API(base.Base):
raise exception.InvalidLearning(reason=msg)
if model_evaluation.job_id:
self.engine_rpcapi.delete_model_evaluation(context,
model_evaluation['cluster_id'],
model_evaluation['job_id'],
id)
self.engine_rpcapi\
.delete_model_evaluation(context,
model_evaluation['cluster_id'],
model_evaluation['job_id'],
id)
else:
self.db.model_evaluation_delete(context, id)

5
meteos/engine/driver.py
View File

@ -18,15 +18,10 @@ Drivers for learnings.
"""
import six
import time
from oslo_config import cfg
from oslo_log import log
from meteos import exception
from meteos.i18n import _, _LE
from meteos import utils
LOG = log.getLogger(__name__)

48
meteos/engine/drivers/generic.py
View File

@ -26,20 +26,16 @@ from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log
from oslo_utils import excutils
from oslo_utils import importutils
from oslo_utils import units
import retrying
import six
from meteos import cluster
from meteos.common import constants as const
from meteos import context
from meteos import exception
from meteos.i18n import _, _LE, _LI, _LW
from meteos.engine import driver
from meteos import utils
from meteos import cluster
EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9'
EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9'
LOG = log.getLogger(__name__)
learning_opts = [
@ -77,8 +73,9 @@ class GenericLearningDriver(driver.LearningDriver):
def __init__(self, *args, **kwargs):
"""Do initialization."""
super(GenericLearningDriver, self).__init__(
[False, True], *args, **kwargs)
super(GenericLearningDriver, self).__init__([False, True],
*args,
**kwargs)
self.admin_context = context.get_admin_context()
self.cluster_api = cluster.API()
self.sshpool = None
@ -162,7 +159,8 @@ class GenericLearningDriver(driver.LearningDriver):
job_args['method'] = method
# Set parameters of DataSet
job_args['source_dataset_url'] = request_specs.get('source_dataset_url')
job_args['source_dataset_url'] = request_specs\
.get('source_dataset_url')
job_args['dataset_format'] = request_specs.get('dataset_format')
dataset_args = {'params': request_specs.get('params')}
job_args['dataset'] = dataset_args
@ -420,10 +418,10 @@ class GenericLearningDriver(driver.LearningDriver):
try:
stdout, stderr = self._run_ssh(ip,
['netstat',
'-tnl',
'|',
'grep',
port])
'-tnl',
'|',
'grep',
port])
except processutils.ProcessExecutionError:
pass
@ -463,12 +461,12 @@ class GenericLearningDriver(driver.LearningDriver):
ip = self._get_master_ip(context, request_specs['cluster_id'])
port = request_specs['port']
return self._run_ssh(ip, ['echo',
base64.b64encode(EXIT_CODE),
'|',
'netcat',
'localhost',
port])
return self._run_ssh(ip, ['echo',
base64.b64encode(EXIT_CODE),
'|',
'netcat',
'localhost',
port])
self._wait_for_model_to_load(ip, port, unload=True)
@ -504,12 +502,12 @@ class GenericLearningDriver(driver.LearningDriver):
LOG.debug("Execute job with args: %s", learning_args)
return self._run_ssh(ip, ['echo',
learning_args,
'|',
'netcat',
'localhost',
port])
return self._run_ssh(ip, ['echo',
learning_args,
'|',
'netcat',
'localhost',
port])
def delete_learning(self, context, cluster_id, job_id, id):
"""Delete Learning."""

64
meteos/engine/manager.py
View File

@ -20,29 +20,16 @@
:learning_driver: Used by :class:`LearningManager`.
"""
import copy
import datetime
import functools
from oslo_config import cfg
from oslo_log import log
from oslo_serialization import jsonutils
from oslo_service import periodic_task
from oslo_utils import excutils
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import timeutils
import six
from meteos.common import constants
from meteos import context
from meteos import exception
import meteos.engine.configuration
from meteos.i18n import _, _LE, _LI, _LW
from meteos import manager
from meteos.engine import api
import meteos.engine.configuration
from meteos.engine import rpcapi as engine_rpcapi
from meteos import utils
LOG = log.getLogger(__name__)
@ -86,10 +73,10 @@ class LearningManager(manager.Manager):
if stderr:
status = constants.STATUS_ERROR
LOG.error(_LI("Fail to create %s %s."), resource_name, id)
LOG.error(_LI("Fail to create %s."), id)
else:
status = constants.STATUS_AVAILABLE
LOG.info(_LI("%s %s created successfully."), resource_name, id)
LOG.info(_LI("%s created successfully."), id)
updates = {
'status': status,
@ -124,7 +111,7 @@ class LearningManager(manager.Manager):
response = self.driver.create_template(
context, request_spec)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("template %s failed on creation."),
request_spec['id'])
@ -150,7 +137,7 @@ class LearningManager(manager.Manager):
template = self.db.template_get(context, id)
self.driver.delete_template(context, template)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Template %s failed on deletion."), id)
self.db.template_update(
@ -177,7 +164,7 @@ class LearningManager(manager.Manager):
experiment = request_spec
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Experiment %s failed on creation."),
request_spec['id'])
@ -203,7 +190,7 @@ class LearningManager(manager.Manager):
experiment = self.db.experiment_get(context, id)
self.driver.delete_experiment(context, experiment['cluster_id'])
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Experiment %s failed on deletion."), id)
self.db.experiment_update(
@ -228,7 +215,7 @@ class LearningManager(manager.Manager):
request_spec['template_id'],
request_spec['cluster_id'])
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Dataset %s failed on creation."),
request_spec['id'])
@ -247,7 +234,7 @@ class LearningManager(manager.Manager):
try:
self.driver.delete_dataset(context, cluster_id, job_id, id)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Dataset %s failed on deletion."), id)
self.db.dataset_update(
@ -272,7 +259,7 @@ class LearningManager(manager.Manager):
request_spec['template_id'],
request_spec['cluster_id'])
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model %s failed on creation."),
request_spec['id'])
@ -291,7 +278,7 @@ class LearningManager(manager.Manager):
try:
self.driver.delete_model(context, cluster_id, job_id, id)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model %s failed on deletion."), id)
self.db.model_update(
@ -314,7 +301,7 @@ class LearningManager(manager.Manager):
try:
self.driver.load_model(context, request_spec)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model %s failed on loading."),
request_spec['id'])
@ -325,7 +312,7 @@ class LearningManager(manager.Manager):
self.db.model_update(context,
request_spec['id'],
{'status' : constants.STATUS_ACTIVE})
{'status': constants.STATUS_ACTIVE})
def unload_model(self, context, request_spec=None):
"""Unload a Model."""
@ -338,7 +325,7 @@ class LearningManager(manager.Manager):
try:
self.driver.unload_model(context, request_spec)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model %s failed on unloading."),
request_spec['id'])
@ -349,7 +336,7 @@ class LearningManager(manager.Manager):
self.db.model_update(context,
request_spec['id'],
{'status' : constants.STATUS_AVAILABLE})
{'status': constants.STATUS_AVAILABLE})
def create_model_evaluation(self, context, request_spec=None):
"""Create a Model Evaluation."""
@ -366,7 +353,7 @@ class LearningManager(manager.Manager):
request_spec['template_id'],
request_spec['cluster_id'])
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model Evaluation %s failed on creation."),
model_evaluation_id)
@ -378,14 +365,18 @@ class LearningManager(manager.Manager):
self._update_status(context, 'Model Evaluation', model_evaluation_id,
job_id, stdout, stderr)
def delete_model_evaluation(self, context, cluster_id=None, job_id=None, id=None):
def delete_model_evaluation(self, context, cluster_id=None,
job_id=None, id=None):
"""Deletes a Model Evaluation."""
context = context.elevated()
try:
self.driver.delete_model_evaluation(context, cluster_id, job_id, id)
self.driver.delete_model_evaluation(context,
cluster_id,
job_id,
id)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Model Evaluation %s failed on deletion."), id)
self.db.model_evaluation_update(
@ -411,7 +402,7 @@ class LearningManager(manager.Manager):
request_spec['template_id'],
request_spec['cluster_id'])
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Learning %s failed on creation."),
learning_id)
@ -432,9 +423,10 @@ class LearningManager(manager.Manager):
LOG.debug("Create learning with request: %s", request_spec)
try:
stdout, stderr = self.driver.create_online_learning(context, request_spec)
stdout, stderr = self.driver.create_online_learning(context,
request_spec)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Learning %s failed on creation."),
request_spec['id'])
@ -453,7 +445,7 @@ class LearningManager(manager.Manager):
try:
self.driver.delete_learning(context, cluster_id, job_id, id)
except Exception as e:
except Exception:
with excutils.save_and_reraise_exception():
LOG.error(_LE("Learning %s failed on deletion."), id)
self.db.learning_update(

5
meteos/exception.py
View File

@ -171,11 +171,6 @@ class MalformedRequestBody(MeteosException):
message = _("Malformed message body: %(reason)s.")
class NotAuthorized(MeteosException):
message = _("Not authorized.")
code = 403
class AdminRequired(NotAuthorized):
message = _("User does not have admin privileges.")

6
meteos/opts.py
View File

@ -26,16 +26,16 @@ import oslo_policy.opts
import meteos.api.common
import meteos.api.middleware.auth
import meteos.cluster.sahara
import meteos.common.config
import meteos.db.api
import meteos.db.base
import meteos.exception
import meteos.service
import meteos.engine.api
import meteos.engine.driver
import meteos.engine.drivers.generic
import meteos.engine.manager
import meteos.cluster.sahara
import meteos.exception
import meteos.service
import meteos.wsgi

1
meteos/service.py
View File

@ -225,7 +225,6 @@ class Service(service.Service):
def report_state(self):
"""Update the state of this service in the datastore."""
ctxt = context.get_admin_context()
zone = CONF.storage_availability_zone
state_catalog = {}
try:
try:

10
meteos/utils.py
View File

@ -17,31 +17,22 @@
"""Utilities and helper functions."""
import contextlib
import errno
import functools
import inspect
import os
import pyclbr
import random
import re
import shutil
import socket
import sys
import tempfile
import time
from eventlet import pools
import netaddr
from oslo_concurrency import lockutils
from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log
from oslo_utils import importutils
from oslo_utils import netutils
from oslo_utils import timeutils
import paramiko
import retrying
import six
from meteos.common import constants
@ -250,7 +241,6 @@ def file_open(filename):
except IOError as e:
if e.errno != errno.ENOENT:
raise
result = False
else:
data = fd.read()
fd.close()

39
tox.ini
View File

@ -34,24 +34,7 @@ basepython = python3.4
commands = oslo_debug_helper {posargs}
[testenv:pep8]
commands =
flake8 {posargs}
# Run bashate during pep8 runs to ensure violations are caught by
# the check and gate queues.
bashate -i E006,E042,E043 \
tools/enable-pre-commit-hook.sh \
contrib/ci/pre_test_hook.sh \
contrib/ci/post_test_hook.sh \
devstack/plugin.sh \
devstack/upgrade/from-mitaka/upgrade-meteos \
devstack/upgrade/resources.sh \
devstack/upgrade/shutdown.sh \
devstack/upgrade/upgrade.sh \
tools/cover.sh \
tools/check_logging.sh \
run_tests.sh
{toxinidir}/tools/check_exec.py {toxinidir}/meteos
{toxinidir}/tools/check_logging.sh {toxinidir}/meteos
commands = flake8 {posargs}
[testenv:genconfig]
whitelist_externals = bash
@ -64,24 +47,6 @@ commands = {posargs}
[testenv:docs]
commands = python setup.py build_sphinx
[testenv:cover]
commands = {toxinidir}/tools/cover.sh {posargs}
[testenv:fast8]
commands =
{toxinidir}/tools/fast8.sh
[testenv:pylint]
deps = -r{toxinidir}/requirements.txt
pylint==0.26.0
whitelist_externals = bash
commands = bash tools/lintstack.sh
[testenv:lint]
deps = -r{toxinidir}/requirements.txt
pylint==0.26.0
commands = python tools/lintstack.py check
[testenv:api-ref]
# This environment is called from CI scripts to test and publish
# the API Ref to developer.openstack.org.
@ -96,7 +61,7 @@ commands = alembic -c meteos/db/migrations/alembic.ini revision -m ""{posargs}
[flake8]
# Following checks are ignored on purpose:
ignore =
ignore = E127,E128,E501,F401
builtins = _
exclude = .git,.tox,.testrepository,.venv,build,cover,dist,doc,*egg,api-ref/build,*/source/conf.py