From 786806e5d38cfd24b05276933ada20d703840a61 Mon Sep 17 00:00:00 2001 From: Hiroyuki Eguchi Date: Sun, 22 Jan 2017 15:16:25 +0900 Subject: [PATCH] Refactoring for pep8 implements blueprint unit-testing Change-Id: Ie80a1f26c167c2b36f6c06852e2d625b58f4c614 --- meteos/api/openstack/__init__.py | 1 - meteos/api/v1/datasets.py | 6 -- meteos/api/v1/experiments.py | 6 -- meteos/api/v1/learnings.py | 6 -- meteos/api/v1/model_evaluations.py | 9 +-- meteos/api/v1/models.py | 6 -- meteos/api/v1/router.py | 8 +- meteos/api/v1/templates.py | 6 -- meteos/api/versions.py | 1 - meteos/api/views/datasets.py | 2 - meteos/api/views/experiments.py | 2 - meteos/api/views/learnings.py | 2 - meteos/api/views/model_evaluations.py | 12 +-- meteos/api/views/models.py | 2 - meteos/api/views/templates.py | 2 - meteos/cluster/binary/meteos-script-1.6.0.py | 43 ++++++----- meteos/cluster/sahara.py | 10 +-- meteos/common/config.py | 4 - meteos/db/api.py | 5 +- .../alembic/versions/001_meteos_init.py | 4 +- meteos/db/sqlalchemy/api.py | 38 +++------ meteos/db/sqlalchemy/models.py | 5 +- meteos/engine/api.py | 77 ++++++++++--------- meteos/engine/driver.py | 5 -- meteos/engine/drivers/generic.py | 48 ++++++------ meteos/engine/manager.py | 64 +++++++-------- meteos/exception.py | 5 -- meteos/opts.py | 6 +- meteos/service.py | 1 - meteos/utils.py | 10 --- tox.ini | 39 +--------- 31 files changed, 150 insertions(+), 285 deletions(-) diff --git a/meteos/api/openstack/__init__.py b/meteos/api/openstack/__init__.py index 32b5c02..d6fd537 100644 --- a/meteos/api/openstack/__init__.py +++ b/meteos/api/openstack/__init__.py @@ -21,7 +21,6 @@ WSGI middleware for OpenStack API controllers. from oslo_log import log import routes -from meteos.api.openstack import wsgi from meteos.i18n import _, _LW from meteos import wsgi as base_wsgi diff --git a/meteos/api/v1/datasets.py b/meteos/api/v1/datasets.py index 07cf8bc..dc670a9 100644 --- a/meteos/api/v1/datasets.py +++ b/meteos/api/v1/datasets.py @@ -16,13 +16,7 @@ """The datasets api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc diff --git a/meteos/api/v1/experiments.py b/meteos/api/v1/experiments.py index 0cd6958..65cd886 100644 --- a/meteos/api/v1/experiments.py +++ b/meteos/api/v1/experiments.py @@ -16,13 +16,7 @@ """The experiments api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc diff --git a/meteos/api/v1/learnings.py b/meteos/api/v1/learnings.py index 1897567..38d6da2 100644 --- a/meteos/api/v1/learnings.py +++ b/meteos/api/v1/learnings.py @@ -16,13 +16,7 @@ """The learnings api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc diff --git a/meteos/api/v1/model_evaluations.py b/meteos/api/v1/model_evaluations.py index e2144bc..71703a4 100644 --- a/meteos/api/v1/model_evaluations.py +++ b/meteos/api/v1/model_evaluations.py @@ -16,13 +16,7 @@ """The model_evaluations api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc @@ -52,7 +46,8 @@ class ModelEvaluationController(wsgi.Controller, wsgi.AdminActionsMixin): context = req.environ['meteos.context'] try: - model_evaluation = self.engine_api.get_model_evaluation(context, id) + model_evaluation = self.engine_api.get_model_evaluation(context, + id) except exception.NotFound: raise exc.HTTPNotFound() diff --git a/meteos/api/v1/models.py b/meteos/api/v1/models.py index eefe465..44e15ad 100644 --- a/meteos/api/v1/models.py +++ b/meteos/api/v1/models.py @@ -16,13 +16,7 @@ """The models api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc diff --git a/meteos/api/v1/router.py b/meteos/api/v1/router.py index 331bf5d..a1f2880 100644 --- a/meteos/api/v1/router.py +++ b/meteos/api/v1/router.py @@ -20,12 +20,12 @@ WSGI middleware for OpenStack Learning API v1. """ import meteos.api.openstack -from meteos.api.v1 import learnings -from meteos.api.v1 import experiments -from meteos.api.v1 import templates from meteos.api.v1 import datasets -from meteos.api.v1 import models +from meteos.api.v1 import experiments +from meteos.api.v1 import learnings from meteos.api.v1 import model_evaluations +from meteos.api.v1 import models +from meteos.api.v1 import templates from meteos.api import versions diff --git a/meteos/api/v1/templates.py b/meteos/api/v1/templates.py index c5834c6..be4ea77 100644 --- a/meteos/api/v1/templates.py +++ b/meteos/api/v1/templates.py @@ -16,13 +16,7 @@ """The templates api.""" -import ast -import re -import string - from oslo_log import log -from oslo_utils import strutils -from oslo_utils import uuidutils import six import webob from webob import exc diff --git a/meteos/api/versions.py b/meteos/api/versions.py index 3596349..d993d7e 100644 --- a/meteos/api/versions.py +++ b/meteos/api/versions.py @@ -19,7 +19,6 @@ import copy from oslo_config import cfg from meteos.api import openstack -from meteos.api.openstack import api_version_request from meteos.api.openstack import wsgi from meteos.api.views import versions as views_versions diff --git a/meteos/api/views/datasets.py b/meteos/api/views/datasets.py index 229a5a0..36b6de0 100644 --- a/meteos/api/views/datasets.py +++ b/meteos/api/views/datasets.py @@ -47,8 +47,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, dataset): """Detailed view of a single dataset.""" - context = request.environ['meteos.context'] - dataset_dict = { 'id': dataset.get('id'), 'created_at': dataset.get('created_at'), diff --git a/meteos/api/views/experiments.py b/meteos/api/views/experiments.py index f6fe953..cf18720 100644 --- a/meteos/api/views/experiments.py +++ b/meteos/api/views/experiments.py @@ -46,8 +46,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, experiment): """Detailed view of a single experiment.""" - context = request.environ['meteos.context'] - experiment_dict = { 'id': experiment.get('id'), 'created_at': experiment.get('created_at'), diff --git a/meteos/api/views/learnings.py b/meteos/api/views/learnings.py index c265b12..f500857 100644 --- a/meteos/api/views/learnings.py +++ b/meteos/api/views/learnings.py @@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, learning): """Detailed view of a single learning.""" - context = request.environ['meteos.context'] - learning_dict = { 'id': learning.get('id'), 'created_at': learning.get('created_at'), diff --git a/meteos/api/views/model_evaluations.py b/meteos/api/views/model_evaluations.py index 180a200..2292166 100644 --- a/meteos/api/views/model_evaluations.py +++ b/meteos/api/views/model_evaluations.py @@ -49,8 +49,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, model_evaluation): """Detailed view of a single model evaluation.""" - context = request.environ['meteos.context'] - model_evaluation_dict = { 'id': model_evaluation.get('id'), 'name': model_evaluation.get('display_name'), @@ -65,17 +63,19 @@ class ViewBuilder(common.ViewBuilder): 'stderr': model_evaluation.get('stderr'), } - self.update_versioned_resource_dict(request, model_evaluation_dict, model_evaluation) + self.update_versioned_resource_dict(request, + model_evaluation_dict, + model_evaluation) return {'model_evaluation': model_evaluation_dict} def _list_view(self, func, request, model_evaluations): """Provide a view for a list of model evaluations.""" model_evaluations_list = [func(request, model_evaluation)['model_evaluation'] - for model_evaluation in model_evaluations] + for model_evaluation in model_evaluations] model_evaluations_links = self._get_collection_links(request, - model_evaluations, - self._collection_name) + model_evaluations, + self._collection_name) model_evaluations_dict = dict(model_evaluations=model_evaluations_list) if model_evaluations_links: diff --git a/meteos/api/views/models.py b/meteos/api/views/models.py index 00afd44..36ef135 100644 --- a/meteos/api/views/models.py +++ b/meteos/api/views/models.py @@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, model): """Detailed view of a single model.""" - context = request.environ['meteos.context'] - model_dict = { 'id': model.get('id'), 'created_at': model.get('created_at'), diff --git a/meteos/api/views/templates.py b/meteos/api/views/templates.py index 9bd8b03..6831945 100644 --- a/meteos/api/views/templates.py +++ b/meteos/api/views/templates.py @@ -48,8 +48,6 @@ class ViewBuilder(common.ViewBuilder): def detail(self, request, template): """Detailed view of a single template.""" - context = request.environ['meteos.context'] - template_dict = { 'id': template.get('id'), 'created_at': template.get('created_at'), diff --git a/meteos/cluster/binary/meteos-script-1.6.0.py b/meteos/cluster/binary/meteos-script-1.6.0.py index a56124d..b477676 100644 --- a/meteos/cluster/binary/meteos-script-1.6.0.py +++ b/meteos/cluster/binary/meteos-script-1.6.0.py @@ -32,35 +32,40 @@ import base64 import os +import socket import sys import uuid -import socket + from ast import literal_eval -from numpy import array from math import sqrt +from numpy import array from pyspark import SparkContext -from pyspark.mllib.linalg import SparseVector -from pyspark.mllib.classification import LogisticRegressionWithSGD from pyspark.mllib.classification import LogisticRegressionModel -from pyspark.mllib.clustering import KMeans, KMeansModel +from pyspark.mllib.classification import LogisticRegressionWithSGD +from pyspark.mllib.clustering import KMeans +from pyspark.mllib.clustering import KMeansModel +from pyspark.mllib.evaluation import BinaryClassificationMetrics +from pyspark.mllib.evaluation import MulticlassMetrics +from pyspark.mllib.evaluation import RankingMetrics +from pyspark.mllib.evaluation import RegressionMetrics from pyspark.mllib.feature import Word2Vec from pyspark.mllib.feature import Word2VecModel from pyspark.mllib.fpm import FPGrowth from pyspark.mllib.fpm import FPGrowthModel -from pyspark.mllib.evaluation import BinaryClassificationMetrics -from pyspark.mllib.evaluation import RegressionMetrics -from pyspark.mllib.evaluation import MulticlassMetrics -from pyspark.mllib.evaluation import RankingMetrics -from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating +from pyspark.mllib.linalg import SparseVector +from pyspark.mllib.recommendation import ALS +from pyspark.mllib.recommendation import MatrixFactorizationModel +from pyspark.mllib.recommendation import Rating from pyspark.mllib.regression import LabeledPoint -from pyspark.mllib.regression import LinearRegressionWithSGD from pyspark.mllib.regression import LinearRegressionModel -from pyspark.mllib.tree import DecisionTree, DecisionTreeModel +from pyspark.mllib.regression import LinearRegressionWithSGD +from pyspark.mllib.tree import DecisionTree +from pyspark.mllib.tree import DecisionTreeModel from pyspark.mllib.util import MLUtils -EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9' +EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9' class ModelController(object): @@ -155,12 +160,11 @@ class RecommendationController(ModelController): testData = ratings.map(lambda p: (p.user, p.product)) predictions = model.predictAll(testData)\ - .map(lambda r: ((r.user, r.product), r.rating)) + .map(lambda r: ((r.user, r.product), r.rating)) ratingsTuple = ratings.map(lambda r: ((r.user, r.product), r.rating)) scoreAndLabels = predictions.join(ratingsTuple).map(lambda tup: tup[1]) - # Instantiate regression metrics to compare predicted and actual ratings metrics = RegressionMetrics(scoreAndLabels) result = "{}: {}".format("MAE", metrics.meanAbsoluteError) + os.linesep\ @@ -287,7 +291,7 @@ class DecisionTreeModelController(ModelController): index = ','.join(index_l) value = ','.join(value_l) - parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])' + parsed_str = '(' + param_len + ', [' + index + '],[' + value + '])' return SparseVector.parse(parsed_str) @@ -378,7 +382,7 @@ class FPGrowthModelController(ModelController): transactions = data.map(lambda line: line.strip().split(' ')) - model= FPGrowth.train(transactions, + model = FPGrowth.train(transactions, minSupport=minSupport, numPartitions=numPartitions) @@ -431,7 +435,7 @@ class MeteosSparkController(object): if collect: self.data.collect() self.data.saveAsTextFile(self.datapath) - print self.data.take(10) + print(self.data.take(10)) def load_data(self): @@ -475,7 +479,8 @@ class MeteosSparkController(object): dataset_format = self.job_args.get('dataset_format') if dataset_format == 'libsvm': - self.model = self.controller.create_model_libsvm(self.data, list_params) + self.model = self.controller.create_model_libsvm(self.data, + list_params) else: self.model = self.controller.create_model(self.data, list_params) diff --git a/meteos/cluster/sahara.py b/meteos/cluster/sahara.py index 78b05fb..d163296 100644 --- a/meteos/cluster/sahara.py +++ b/meteos/cluster/sahara.py @@ -18,20 +18,14 @@ Handles all requests relating to volumes + sahara. """ -import copy -import os - -from osc_lib import exceptions as sahara_exception -from saharaclient import client as sahara_client from keystoneauth1 import loading as ks_loading +from osc_lib import exceptions as sahara_exception from oslo_config import cfg -import six +from saharaclient import client as sahara_client from meteos.common import client_auth from meteos.common.config import core_opts -import meteos.context as ctxt from meteos.db import base -from meteos import exception from meteos.i18n import _ SAHARA_GROUP = 'sahara' diff --git a/meteos/common/config.py b/meteos/common/config.py index 1897aff..a84d9ee 100644 --- a/meteos/common/config.py +++ b/meteos/common/config.py @@ -30,11 +30,7 @@ from oslo_config import cfg from oslo_log import log from oslo_middleware import cors from oslo_utils import netutils -import six -from meteos.common import constants -from meteos import exception -from meteos.i18n import _ CONF = cfg.CONF log.register_options(CONF) diff --git a/meteos/db/api.py b/meteos/db/api.py index d071573..b7cdf80 100644 --- a/meteos/db/api.py +++ b/meteos/db/api.py @@ -307,7 +307,8 @@ def model_evaluation_get(context, model_evaluation_id): return IMPL.model_evaluation_get(context, model_evaluation_id) -def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None): +def model_evaluation_get_all(context, filters=None, + sort_key=None, sort_dir=None): """Get all model_evaluations.""" return IMPL.model_evaluation_get_all( context, filters=filters, sort_key=sort_key, sort_dir=sort_dir, @@ -315,7 +316,7 @@ def model_evaluation_get_all(context, filters=None, sort_key=None, sort_dir=None def model_evaluation_get_all_by_project(context, project_id, filters=None, - sort_key=None, sort_dir=None): + sort_key=None, sort_dir=None): """Returns all model_evaluations with given project ID.""" return IMPL.model_evaluation_get_all_by_project( context, project_id, filters=filters, diff --git a/meteos/db/migrations/alembic/versions/001_meteos_init.py b/meteos/db/migrations/alembic/versions/001_meteos_init.py index 2c60734..5f425fa 100644 --- a/meteos/db/migrations/alembic/versions/001_meteos_init.py +++ b/meteos/db/migrations/alembic/versions/001_meteos_init.py @@ -26,8 +26,8 @@ down_revision = None from alembic import op from oslo_log import log -from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Text -from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint +from sqlalchemy import Boolean, Column, DateTime, Text +from sqlalchemy import Integer, MetaData, String, Table from meteos.i18n import _LE diff --git a/meteos/db/sqlalchemy/api.py b/meteos/db/sqlalchemy/api.py index a648723..406a837 100644 --- a/meteos/db/sqlalchemy/api.py +++ b/meteos/db/sqlalchemy/api.py @@ -19,10 +19,8 @@ """Implementation of SQLAlchemy backend.""" import copy -import datetime from functools import wraps import sys -import uuid import warnings # NOTE(uglide): Required to override default oslo_db Query class @@ -30,21 +28,13 @@ import meteos.db.sqlalchemy.query # noqa from oslo_config import cfg from oslo_db import api as oslo_db_api -from oslo_db import exception as db_exception from oslo_db import options as db_options from oslo_db.sqlalchemy import session from oslo_db.sqlalchemy import utils as db_utils from oslo_log import log -from oslo_utils import timeutils from oslo_utils import uuidutils -import six -from sqlalchemy import and_ -from sqlalchemy import or_ -from sqlalchemy.orm import joinedload -from sqlalchemy.sql.expression import true from sqlalchemy.sql import func -from meteos.common import constants from meteos.db.sqlalchemy import models from meteos import exception from meteos.i18n import _, _LE, _LW @@ -239,19 +229,6 @@ def ensure_dict_has_id(model_dict): return model_dict -def _sync_learnings(context, project_id, user_id, session): - (learnings, gigs) = learning_data_get_for_project(context, - project_id, - user_id, - session=session) - return {'learnings': learnings} - - -QUOTA_SYNC_FUNCTIONS = { - '_sync_learnings': _sync_learnings, -} - - # @@ -857,11 +834,14 @@ def model_evaluation_create(context, model_evaluation_values): model_evaluation_ref.save(session=session) # NOTE(u_glide): Do so to prevent errors with relationships - return model_evaluation_get(context, model_evaluation_ref['id'], session=session) + return model_evaluation_get(context, + model_evaluation_ref['id'], + session=session) -def _model_evaluation_get_all_with_filters(context, project_id=None, filters=None, - sort_key=None, sort_dir=None): +def _model_evaluation_get_all_with_filters(context, project_id=None, + filters=None, sort_key=None, + sort_dir=None): if not sort_key: sort_key = 'created_at' if not sort_dir: @@ -899,7 +879,7 @@ def _model_evaluation_get_all_with_filters(context, project_id=None, filters=Non @require_context def model_evaluation_get_all_by_project(context, project_id, filters=None, - sort_key=None, sort_dir=None): + sort_key=None, sort_dir=None): """Returns list of model_evaluations with given project ID.""" query = _model_evaluation_get_all_with_filters( context, project_id=project_id, filters=filters, @@ -913,7 +893,9 @@ def model_evaluation_delete(context, model_evaluation_id): session = get_session() with session.begin(): - model_evaluation_ref = model_evaluation_get(context, model_evaluation_id, session) + model_evaluation_ref = model_evaluation_get(context, + model_evaluation_id, + session) model_evaluation_ref.soft_delete(session=session) diff --git a/meteos/db/sqlalchemy/models.py b/meteos/db/sqlalchemy/models.py index 4a266d9..2ae25ac 100644 --- a/meteos/db/sqlalchemy/models.py +++ b/meteos/db/sqlalchemy/models.py @@ -22,10 +22,9 @@ SQLAlchemy models for Meteos data. from oslo_config import cfg from oslo_db.sqlalchemy import models from oslo_log import log -from sqlalchemy import Column, Integer, String, schema +from sqlalchemy import Column, Integer, String from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import orm -from sqlalchemy import ForeignKey, DateTime, Boolean, Enum, Text +from sqlalchemy import DateTime, Boolean, Text from meteos.common import constants diff --git a/meteos/engine/api.py b/meteos/engine/api.py index 368b886..9a91c07 100644 --- a/meteos/engine/api.py +++ b/meteos/engine/api.py @@ -20,20 +20,16 @@ Handles all requests relating to learnings. """ -from oslo_config import cfg from oslo_log import log from oslo_utils import excutils -from oslo_utils import strutils -from oslo_utils import timeutils import six from meteos.common import constants from meteos.db import base +from meteos.engine import rpcapi as engine_rpcapi from meteos import exception from meteos.i18n import _, _LE, _LI, _LW from meteos import policy -from meteos.engine import rpcapi as engine_rpcapi -from meteos import utils LOG = log.getLogger(__name__) @@ -121,7 +117,7 @@ class API(base.Base): "statuses": statuses} raise exception.InvalidLearning(reason=msg) - result = self.engine_rpcapi.delete_template(context, id) + self.engine_rpcapi.delete_template(context, id) def get_all_experiments(self, context, search_opts=None, sort_key='created_at', sort_dir='desc'): @@ -199,7 +195,7 @@ class API(base.Base): "statuses": statuses} raise exception.InvalidLearning(reason=msg) - result = self.engine_rpcapi.delete_experiment(context, id) + self.engine_rpcapi.delete_experiment(context, id) def get_all_datasets(self, context, search_opts=None, sort_key='created_at', sort_dir='desc'): @@ -286,10 +282,10 @@ class API(base.Base): "statuses": statuses} raise exception.InvalidLearning(reason=msg) - result = self.engine_rpcapi.delete_dataset(context, - dataset['cluster_id'], - dataset['job_id'], - id) + self.engine_rpcapi.delete_dataset(context, + dataset['cluster_id'], + dataset['job_id'], + id) def _enable_load_model(self, context): @@ -387,10 +383,10 @@ class API(base.Base): "statuses": statuses} raise exception.InvalidLearning(reason=msg) - result = self.engine_rpcapi.delete_model(context, - model['cluster_id'], - model['job_id'], - id) + self.engine_rpcapi.delete_model(context, + model['cluster_id'], + model['job_id'], + id) def load_model(self, context, id, dataset_format, model_type, job_template_id, experiment_id, cluster_id): @@ -445,20 +441,22 @@ class API(base.Base): self.db.model_update(context, id, updates) def get_all_model_evaluations(self, context, search_opts=None, - sort_key='created_at', sort_dir='desc'): + sort_key='created_at', sort_dir='desc'): policy.check_policy(context, 'model_evaluation', 'get_all') if search_opts is None: search_opts = {} - LOG.debug("Searching for model evaluations by: %s", six.text_type(search_opts)) + LOG.debug("Searching for model evaluations by: %s", + six.text_type(search_opts)) project_id = context.project_id - model_evaluations = self.db.model_evaluation_get_all_by_project(context, - project_id, - sort_key=sort_key, - sort_dir=sort_dir) + model_evaluations = self.db.model_evaluation_get_all_by_project( + context, + project_id, + sort_key=sort_key, + sort_dir=sort_dir) if search_opts: results = [] @@ -474,22 +472,23 @@ class API(base.Base): return rv def create_model_evaluation(self, context, name, source_dataset_url, - model_id, model_type, dataset_format, template_id, - job_template_id, experiment_id, cluster_id, - swift_tenant, swift_username, swift_password): + model_id, model_type, dataset_format, + template_id, job_template_id, experiment_id, + cluster_id, swift_tenant, swift_username, + swift_password): """Create a Model Evaluation""" policy.check_policy(context, 'model_evaluation', 'create') model_evaluation = {'id': None, - 'display_name': name, - 'model_id': model_id, - 'model_type': model_type, - 'source_dataset_url': source_dataset_url, - 'dataset_format': dataset_format, - 'user_id': context.user_id, - 'project_id': context.project_id, - 'cluster_id': cluster_id - } + 'display_name': name, + 'model_id': model_id, + 'model_type': model_type, + 'source_dataset_url': source_dataset_url, + 'dataset_format': dataset_format, + 'user_id': context.user_id, + 'project_id': context.project_id, + 'cluster_id': cluster_id + } try: result = self.db.model_evaluation_create(context, model_evaluation) @@ -505,7 +504,8 @@ class API(base.Base): result['id'], updates) - LOG.info(_LI("Accepted creation of model evaluation %s."), result['id']) + LOG.info(_LI("Accepted creation of model evaluation %s."), + result['id']) except Exception: with excutils.save_and_reraise_exception(): self.db.model_evaluation_delete(context, result['id']) @@ -530,10 +530,11 @@ class API(base.Base): raise exception.InvalidLearning(reason=msg) if model_evaluation.job_id: - self.engine_rpcapi.delete_model_evaluation(context, - model_evaluation['cluster_id'], - model_evaluation['job_id'], - id) + self.engine_rpcapi\ + .delete_model_evaluation(context, + model_evaluation['cluster_id'], + model_evaluation['job_id'], + id) else: self.db.model_evaluation_delete(context, id) diff --git a/meteos/engine/driver.py b/meteos/engine/driver.py index 05dda74..1078ab1 100644 --- a/meteos/engine/driver.py +++ b/meteos/engine/driver.py @@ -18,15 +18,10 @@ Drivers for learnings. """ -import six -import time - from oslo_config import cfg from oslo_log import log -from meteos import exception from meteos.i18n import _, _LE -from meteos import utils LOG = log.getLogger(__name__) diff --git a/meteos/engine/drivers/generic.py b/meteos/engine/drivers/generic.py index 14454c3..195b6d8 100644 --- a/meteos/engine/drivers/generic.py +++ b/meteos/engine/drivers/generic.py @@ -26,20 +26,16 @@ from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log from oslo_utils import excutils -from oslo_utils import importutils -from oslo_utils import units -import retrying -import six +from meteos import cluster from meteos.common import constants as const from meteos import context from meteos import exception from meteos.i18n import _, _LE, _LI, _LW from meteos.engine import driver from meteos import utils -from meteos import cluster -EXIT_CODE='80577372-9349-463a-bbc3-1ca54f187cc9' +EXIT_CODE = '80577372-9349-463a-bbc3-1ca54f187cc9' LOG = log.getLogger(__name__) learning_opts = [ @@ -77,8 +73,9 @@ class GenericLearningDriver(driver.LearningDriver): def __init__(self, *args, **kwargs): """Do initialization.""" - super(GenericLearningDriver, self).__init__( - [False, True], *args, **kwargs) + super(GenericLearningDriver, self).__init__([False, True], + *args, + **kwargs) self.admin_context = context.get_admin_context() self.cluster_api = cluster.API() self.sshpool = None @@ -162,7 +159,8 @@ class GenericLearningDriver(driver.LearningDriver): job_args['method'] = method # Set parameters of DataSet - job_args['source_dataset_url'] = request_specs.get('source_dataset_url') + job_args['source_dataset_url'] = request_specs\ + .get('source_dataset_url') job_args['dataset_format'] = request_specs.get('dataset_format') dataset_args = {'params': request_specs.get('params')} job_args['dataset'] = dataset_args @@ -420,10 +418,10 @@ class GenericLearningDriver(driver.LearningDriver): try: stdout, stderr = self._run_ssh(ip, ['netstat', - '-tnl', - '|', - 'grep', - port]) + '-tnl', + '|', + 'grep', + port]) except processutils.ProcessExecutionError: pass @@ -463,12 +461,12 @@ class GenericLearningDriver(driver.LearningDriver): ip = self._get_master_ip(context, request_specs['cluster_id']) port = request_specs['port'] - return self._run_ssh(ip, ['echo', - base64.b64encode(EXIT_CODE), - '|', - 'netcat', - 'localhost', - port]) + return self._run_ssh(ip, ['echo', + base64.b64encode(EXIT_CODE), + '|', + 'netcat', + 'localhost', + port]) self._wait_for_model_to_load(ip, port, unload=True) @@ -504,12 +502,12 @@ class GenericLearningDriver(driver.LearningDriver): LOG.debug("Execute job with args: %s", learning_args) - return self._run_ssh(ip, ['echo', - learning_args, - '|', - 'netcat', - 'localhost', - port]) + return self._run_ssh(ip, ['echo', + learning_args, + '|', + 'netcat', + 'localhost', + port]) def delete_learning(self, context, cluster_id, job_id, id): """Delete Learning.""" diff --git a/meteos/engine/manager.py b/meteos/engine/manager.py index 62069e9..d5e078a 100644 --- a/meteos/engine/manager.py +++ b/meteos/engine/manager.py @@ -20,29 +20,16 @@ :learning_driver: Used by :class:`LearningManager`. """ -import copy -import datetime -import functools - from oslo_config import cfg from oslo_log import log -from oslo_serialization import jsonutils -from oslo_service import periodic_task from oslo_utils import excutils from oslo_utils import importutils -from oslo_utils import strutils from oslo_utils import timeutils -import six from meteos.common import constants -from meteos import context -from meteos import exception +import meteos.engine.configuration from meteos.i18n import _, _LE, _LI, _LW from meteos import manager -from meteos.engine import api -import meteos.engine.configuration -from meteos.engine import rpcapi as engine_rpcapi -from meteos import utils LOG = log.getLogger(__name__) @@ -86,10 +73,10 @@ class LearningManager(manager.Manager): if stderr: status = constants.STATUS_ERROR - LOG.error(_LI("Fail to create %s %s."), resource_name, id) + LOG.error(_LI("Fail to create %s."), id) else: status = constants.STATUS_AVAILABLE - LOG.info(_LI("%s %s created successfully."), resource_name, id) + LOG.info(_LI("%s created successfully."), id) updates = { 'status': status, @@ -124,7 +111,7 @@ class LearningManager(manager.Manager): response = self.driver.create_template( context, request_spec) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("template %s failed on creation."), request_spec['id']) @@ -150,7 +137,7 @@ class LearningManager(manager.Manager): template = self.db.template_get(context, id) self.driver.delete_template(context, template) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Template %s failed on deletion."), id) self.db.template_update( @@ -177,7 +164,7 @@ class LearningManager(manager.Manager): experiment = request_spec - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Experiment %s failed on creation."), request_spec['id']) @@ -203,7 +190,7 @@ class LearningManager(manager.Manager): experiment = self.db.experiment_get(context, id) self.driver.delete_experiment(context, experiment['cluster_id']) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Experiment %s failed on deletion."), id) self.db.experiment_update( @@ -228,7 +215,7 @@ class LearningManager(manager.Manager): request_spec['template_id'], request_spec['cluster_id']) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Dataset %s failed on creation."), request_spec['id']) @@ -247,7 +234,7 @@ class LearningManager(manager.Manager): try: self.driver.delete_dataset(context, cluster_id, job_id, id) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Dataset %s failed on deletion."), id) self.db.dataset_update( @@ -272,7 +259,7 @@ class LearningManager(manager.Manager): request_spec['template_id'], request_spec['cluster_id']) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model %s failed on creation."), request_spec['id']) @@ -291,7 +278,7 @@ class LearningManager(manager.Manager): try: self.driver.delete_model(context, cluster_id, job_id, id) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model %s failed on deletion."), id) self.db.model_update( @@ -314,7 +301,7 @@ class LearningManager(manager.Manager): try: self.driver.load_model(context, request_spec) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model %s failed on loading."), request_spec['id']) @@ -325,7 +312,7 @@ class LearningManager(manager.Manager): self.db.model_update(context, request_spec['id'], - {'status' : constants.STATUS_ACTIVE}) + {'status': constants.STATUS_ACTIVE}) def unload_model(self, context, request_spec=None): """Unload a Model.""" @@ -338,7 +325,7 @@ class LearningManager(manager.Manager): try: self.driver.unload_model(context, request_spec) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model %s failed on unloading."), request_spec['id']) @@ -349,7 +336,7 @@ class LearningManager(manager.Manager): self.db.model_update(context, request_spec['id'], - {'status' : constants.STATUS_AVAILABLE}) + {'status': constants.STATUS_AVAILABLE}) def create_model_evaluation(self, context, request_spec=None): """Create a Model Evaluation.""" @@ -366,7 +353,7 @@ class LearningManager(manager.Manager): request_spec['template_id'], request_spec['cluster_id']) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model Evaluation %s failed on creation."), model_evaluation_id) @@ -378,14 +365,18 @@ class LearningManager(manager.Manager): self._update_status(context, 'Model Evaluation', model_evaluation_id, job_id, stdout, stderr) - def delete_model_evaluation(self, context, cluster_id=None, job_id=None, id=None): + def delete_model_evaluation(self, context, cluster_id=None, + job_id=None, id=None): """Deletes a Model Evaluation.""" context = context.elevated() try: - self.driver.delete_model_evaluation(context, cluster_id, job_id, id) + self.driver.delete_model_evaluation(context, + cluster_id, + job_id, + id) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Model Evaluation %s failed on deletion."), id) self.db.model_evaluation_update( @@ -411,7 +402,7 @@ class LearningManager(manager.Manager): request_spec['template_id'], request_spec['cluster_id']) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Learning %s failed on creation."), learning_id) @@ -432,9 +423,10 @@ class LearningManager(manager.Manager): LOG.debug("Create learning with request: %s", request_spec) try: - stdout, stderr = self.driver.create_online_learning(context, request_spec) + stdout, stderr = self.driver.create_online_learning(context, + request_spec) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Learning %s failed on creation."), request_spec['id']) @@ -453,7 +445,7 @@ class LearningManager(manager.Manager): try: self.driver.delete_learning(context, cluster_id, job_id, id) - except Exception as e: + except Exception: with excutils.save_and_reraise_exception(): LOG.error(_LE("Learning %s failed on deletion."), id) self.db.learning_update( diff --git a/meteos/exception.py b/meteos/exception.py index e82716e..b3877dd 100644 --- a/meteos/exception.py +++ b/meteos/exception.py @@ -171,11 +171,6 @@ class MalformedRequestBody(MeteosException): message = _("Malformed message body: %(reason)s.") -class NotAuthorized(MeteosException): - message = _("Not authorized.") - code = 403 - - class AdminRequired(NotAuthorized): message = _("User does not have admin privileges.") diff --git a/meteos/opts.py b/meteos/opts.py index 7a07233..c59a134 100644 --- a/meteos/opts.py +++ b/meteos/opts.py @@ -26,16 +26,16 @@ import oslo_policy.opts import meteos.api.common import meteos.api.middleware.auth +import meteos.cluster.sahara import meteos.common.config import meteos.db.api import meteos.db.base -import meteos.exception -import meteos.service import meteos.engine.api import meteos.engine.driver import meteos.engine.drivers.generic import meteos.engine.manager -import meteos.cluster.sahara +import meteos.exception +import meteos.service import meteos.wsgi diff --git a/meteos/service.py b/meteos/service.py index b4175f3..daaa725 100644 --- a/meteos/service.py +++ b/meteos/service.py @@ -225,7 +225,6 @@ class Service(service.Service): def report_state(self): """Update the state of this service in the datastore.""" ctxt = context.get_admin_context() - zone = CONF.storage_availability_zone state_catalog = {} try: try: diff --git a/meteos/utils.py b/meteos/utils.py index 18593dc..dd7d456 100644 --- a/meteos/utils.py +++ b/meteos/utils.py @@ -17,31 +17,22 @@ """Utilities and helper functions.""" -import contextlib import errno import functools import inspect import os import pyclbr -import random -import re -import shutil -import socket import sys -import tempfile import time from eventlet import pools -import netaddr from oslo_concurrency import lockutils from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log from oslo_utils import importutils -from oslo_utils import netutils from oslo_utils import timeutils import paramiko -import retrying import six from meteos.common import constants @@ -250,7 +241,6 @@ def file_open(filename): except IOError as e: if e.errno != errno.ENOENT: raise - result = False else: data = fd.read() fd.close() diff --git a/tox.ini b/tox.ini index fbd7da0..d4d315c 100644 --- a/tox.ini +++ b/tox.ini @@ -34,24 +34,7 @@ basepython = python3.4 commands = oslo_debug_helper {posargs} [testenv:pep8] -commands = - flake8 {posargs} - # Run bashate during pep8 runs to ensure violations are caught by - # the check and gate queues. - bashate -i E006,E042,E043 \ - tools/enable-pre-commit-hook.sh \ - contrib/ci/pre_test_hook.sh \ - contrib/ci/post_test_hook.sh \ - devstack/plugin.sh \ - devstack/upgrade/from-mitaka/upgrade-meteos \ - devstack/upgrade/resources.sh \ - devstack/upgrade/shutdown.sh \ - devstack/upgrade/upgrade.sh \ - tools/cover.sh \ - tools/check_logging.sh \ - run_tests.sh - {toxinidir}/tools/check_exec.py {toxinidir}/meteos - {toxinidir}/tools/check_logging.sh {toxinidir}/meteos +commands = flake8 {posargs} [testenv:genconfig] whitelist_externals = bash @@ -64,24 +47,6 @@ commands = {posargs} [testenv:docs] commands = python setup.py build_sphinx -[testenv:cover] -commands = {toxinidir}/tools/cover.sh {posargs} - -[testenv:fast8] -commands = - {toxinidir}/tools/fast8.sh - -[testenv:pylint] -deps = -r{toxinidir}/requirements.txt - pylint==0.26.0 -whitelist_externals = bash -commands = bash tools/lintstack.sh - -[testenv:lint] -deps = -r{toxinidir}/requirements.txt - pylint==0.26.0 -commands = python tools/lintstack.py check - [testenv:api-ref] # This environment is called from CI scripts to test and publish # the API Ref to developer.openstack.org. @@ -96,7 +61,7 @@ commands = alembic -c meteos/db/migrations/alembic.ini revision -m ""{posargs} [flake8] # Following checks are ignored on purpose: -ignore = +ignore = E127,E128,E501,F401 builtins = _ exclude = .git,.tox,.testrepository,.venv,build,cover,dist,doc,*egg,api-ref/build,*/source/conf.py