openstack
/
watcher
Code Issues Proposed changes

Merge "consolidation of watcher"

This commit is contained in:
Jenkins 2015-10-22 16:32:11 +00:00 committed by Gerrit Code Review
parent 970c9c0c64 74160c5e78
commit 4f0b9d7629
140 changed files with 2991 additions and 1271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -22,11 +22,12 @@ lib64
pip-log.txt
# Unit test / coverage reports
.coverage
.coverage*
.tox
nosetests.xml
.testrepository
.venv
.idea
# Translations
*.mo
@ -57,6 +58,6 @@ sftp-config.json
/cover/
.settings/
.eclipse
.project
.pydevproject
cover
/demo/

8
README.rst
View File

@ -1,6 +1,6 @@
===============================
watcher
===============================
=======
Watcher
=======
OpenStack Watcher provides a flexible and scalable resource optimization service for multi-tenant OpenStack-based clouds.
Watcher provides a complete optimization loop—including everything from a metrics receiver, complex event processor and profiler,
@ -11,4 +11,4 @@ including the reduction of data center operating costs, increased system perform
* Wiki: http://wiki.openstack.org/wiki/Watcher
* Source: http://git.openstack.org/cgit/stackforge/watcher
* Bugs: http://bugs.launchpad.net/watcher
* Documentation: http://factory.b-com.com/www/watcher/watcher/doc/build/html/

13
setup.cfg
View File

@ -1,5 +1,5 @@
[metadata]
name = watcher
name = python-watcher
summary = Watcher takes advantage of CEP and ML algorithms/metaheuristics to improve physical resources usage through better VM placement. Watcher can improve your cloud optimization by reducing energy footprint and increasing profits.
description-file =
README.rst
@ -23,11 +23,13 @@ classifier =
[files]
packages =
watcher
data_files =
etc/ = etc/*
[global]
setup-hooks =
pbr.hooks.setup_hook
[entry_points]
oslo.config.opts =
watcher = watcher.opts:list_opts
@ -41,6 +43,12 @@ console_scripts =
watcher.database.migration_backend =
sqlalchemy = watcher.db.sqlalchemy.migration
watcher_strategies =
basic = watcher.decision_engine.strategies.basic_consolidation:BasicConsolidation
watcher_metrics_collector =
influxdb = watcher.metrics_engine.framework.datasources.influxdb_collector:InfluxDBCollector
[build_sphinx]
source-dir = doc/source
build-dir = doc/build
@ -63,4 +71,3 @@ input_file = watcher/locale/watcher.pot
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = watcher/locale/watcher.pot

14
tox.ini
View File

@ -19,7 +19,7 @@ commands = flake8
commands = {posargs}
[testenv:cover]
commands = python setup.py testr --coverage --testr-args='{posargs}'
commands = python setup.py testr --coverage --omit="watcher/tests/*,watcher/openstack/*" --testr-args='{posargs}'
[testenv:docs]
commands = python setup.py build_sphinx
@ -33,7 +33,7 @@ commands =
oslo-config-generator --namespace watcher \
--namespace keystonemiddleware.auth_token \
--namespace oslo.db \
--output-file etc/watcher/watcher.conf
--output-file etc/watcher/watcher.conf.sample
[flake8]
# E123, E125 skipped as they are invalid PEP-8.
@ -41,4 +41,12 @@ commands =
show-source=True
ignore=E123,E125,H404,H405,H305
builtins= _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,*sqlalchemy/alembic/versions/*,demo/
[testenv:pypi]
commands =
python setup.py sdist bdist_wheel
twine upload --config-file .pypirc {posargs} dist/*
[testenv:wheel]
commands = python setup.py bdist_wheel

3
watcher/__init__.py
View File

@ -15,5 +15,4 @@
import pbr.version
__version__ = pbr.version.VersionInfo(
'watcher').version_string()
__version__ = pbr.version.VersionInfo('python-watcher').version_string()

2
watcher/api/controllers/v1/__init__.py
View File

@ -33,6 +33,7 @@ from watcher.api.controllers.v1 import action
from watcher.api.controllers.v1 import action_plan
from watcher.api.controllers.v1 import audit
from watcher.api.controllers.v1 import audit_template
from watcher.api.controllers.v1 import goal
class APIBase(wtypes.Base):
@ -155,6 +156,7 @@ class Controller(rest.RestController):
audit_templates = audit_template.AuditTemplatesController()
actions = action.ActionsController()
action_plans = action_plan.ActionPlansController()
goals = goal.GoalsController()
@wsme_pecan.wsexpose(V1)
def get(self):

69
watcher/api/controllers/v1/audit.py
View File

@ -48,25 +48,44 @@ class Audit(base.APIBase):
between the internal object model and the API representation of a audit.
"""
_audit_template_uuid = None
_audit_template_name = None
def _get_audit_template(self, value):
if value == wtypes.Unset:
return None
audit_template = None
try:
if utils.is_uuid_like(value) or utils.is_int_like(value):
audit_template = objects.AuditTemplate.get(
pecan.request.context, value)
else:
audit_template = objects.AuditTemplate.get_by_name(
pecan.request.context, value)
except exception.AuditTemplateNotFound:
pass
if audit_template:
self.audit_template_id = audit_template.id
return audit_template
def _get_audit_template_uuid(self):
return self._audit_template_uuid
def _set_audit_template_uuid(self, value):
if value == wtypes.Unset:
self._audit_template_uuid = wtypes.Unset
elif value and self._audit_template_uuid != value:
try:
if utils.is_uuid_like(value) or utils.is_int_like(value):
audit_template = objects.AuditTemplate.get(
pecan.request.context, value)
else:
audit_template = objects.AuditTemplate.get_by_name(
pecan.request.context, value)
if value and self._audit_template_uuid != value:
self._audit_template_uuid = None
audit_template = self._get_audit_template(value)
if audit_template:
self._audit_template_uuid = audit_template.uuid
self.audit_template_id = audit_template.id
except exception.AuditTemplateNotFound:
self._audit_template_uuid = None
def _get_audit_template_name(self):
return self._audit_template_name
def _set_audit_template_name(self, value):
if value and self._audit_template_name != value:
self._audit_template_name = None
audit_template = self._get_audit_template(value)
if audit_template:
self._audit_template_name = audit_template.name
uuid = types.uuid
"""Unique UUID for this audit"""
@ -84,7 +103,13 @@ class Audit(base.APIBase):
_get_audit_template_uuid,
_set_audit_template_uuid,
mandatory=True)
"""The UUID of the node this port belongs to"""
"""The UUID of the audit template this audit refers to"""
audit_template_name = wsme.wsproperty(wtypes.text,
_get_audit_template_name,
_set_audit_template_name,
mandatory=False)
"""The name of the audit template this audit refers to"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated audit links"""
@ -92,9 +117,7 @@ class Audit(base.APIBase):
def __init__(self, **kwargs):
self.fields = []
fields = list(objects.Audit.fields)
# audit_template_uuid is not part of objects.Audit.fields
# because it's an API-only attribute.
fields.append('audit_template_uuid')
for k in fields:
# Skip fields we do not expose.
if not hasattr(self, k):
@ -103,14 +126,22 @@ class Audit(base.APIBase):
setattr(self, k, kwargs.get(k, wtypes.Unset))
self.fields.append('audit_template_id')
# audit_template_uuid & audit_template_name are not part of
# objects.Audit.fields because they're API-only attributes.
fields.append('audit_template_uuid')
setattr(self, 'audit_template_uuid', kwargs.get('audit_template_id',
wtypes.Unset))
fields.append('audit_template_name')
setattr(self, 'audit_template_name', kwargs.get('audit_template_id',
wtypes.Unset))
@staticmethod
def _convert_with_links(audit, url, expand=True):
if not expand:
audit.unset_fields_except(['uuid', 'type', 'deadline',
'state', 'audit_template_uuid'])
'state', 'audit_template_uuid',
'audit_template_name'])
# The numeric ID should not be exposed to
# the user, it's internal only.
@ -237,7 +268,7 @@ class AuditsController(rest.RestController):
:param limit: maximum number of resources to return in a single result.
:param sort_key: column to sort results by. Default: id.
:param sort_dir: direction to sort. "asc" or "desc". Default: asc.
:param audit_template: Optional UUID or description of an audit
:param audit_template: Optional UUID or name of an audit
template, to get only audits for that audit template.
"""
return self._get_audits_collection(marker, limit, sort_key,

208
watcher/api/controllers/v1/goal.py Normal file
View File

@ -0,0 +1,208 @@
# -*- encoding: utf-8 -*-
# Copyright 2013 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
import pecan
from pecan import rest
import wsme
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from watcher.api.controllers import base
from watcher.api.controllers import link
from watcher.api.controllers.v1 import collection
from watcher.api.controllers.v1 import types
from watcher.api.controllers.v1 import utils as api_utils
from watcher.common import exception
CONF = cfg.CONF
class Goal(base.APIBase):
"""API representation of a action.
This class enforces type checking and value constraints, and converts
between the internal object model and the API representation of a action.
"""
name = wtypes.text
"""Name of the goal"""
strategy = wtypes.text
"""The strategy associated with the goal"""
uuid = types.uuid
"""Unused field"""
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link and associated action links"""
def __init__(self, **kwargs):
super(Goal, self).__init__()
self.fields = []
self.fields.append('name')
self.fields.append('strategy')
setattr(self, 'name', kwargs.get('name',
wtypes.Unset))
setattr(self, 'strategy', kwargs.get('strategy',
wtypes.Unset))
@staticmethod
def _convert_with_links(goal, url, expand=True):
if not expand:
goal.unset_fields_except(['name', 'strategy'])
goal.links = [link.Link.make_link('self', url,
'goals', goal.name),
link.Link.make_link('bookmark', url,
'goals', goal.name,
bookmark=True)]
return goal
@classmethod
def convert_with_links(cls, goal, expand=True):
goal = Goal(**goal)
return cls._convert_with_links(goal, pecan.request.host_url, expand)
@classmethod
def sample(cls, expand=True):
sample = cls(name='27e3153e-d5bf-4b7e-b517-fb518e17f34c',
strategy='action description')
return cls._convert_with_links(sample, 'http://localhost:9322', expand)
class GoalCollection(collection.Collection):
"""API representation of a collection of goals."""
goals = [Goal]
"""A list containing goals objects"""
def __init__(self, **kwargs):
self._type = 'goals'
@staticmethod
def convert_with_links(goals, limit, url=None, expand=False,
**kwargs):
collection = GoalCollection()
collection.goals = [Goal.convert_with_links(g, expand) for g in goals]
if 'sort_key' in kwargs:
reverse = False
if kwargs['sort_key'] == 'strategy':
if 'sort_dir' in kwargs:
reverse = True if kwargs['sort_dir'] == 'desc' else False
collection.goals = sorted(
collection.goals,
key=lambda goal: goal.name,
reverse=reverse)
collection.next = collection.get_next(limit, url=url, **kwargs)
return collection
@classmethod
def sample(cls):
sample = cls()
sample.actions = [Goal.sample(expand=False)]
return sample
class GoalsController(rest.RestController):
"""REST controller for Goals."""
def __init__(self):
super(GoalsController, self).__init__()
from_goals = False
"""A flag to indicate if the requests to this controller are coming
from the top-level resource Goals."""
_custom_actions = {
'detail': ['GET'],
}
def _get_goals_collection(self, limit,
sort_key, sort_dir, expand=False,
resource_url=None, goal_name=None):
limit = api_utils.validate_limit(limit)
sort_dir = api_utils.validate_sort_dir(sort_dir)
goals = []
if not goal_name and goal_name in CONF.watcher_goals.goals.keys():
goals.append({'name': goal_name, 'strategy': goals[goal_name]})
else:
for name, strategy in CONF.watcher_goals.goals.items():
goals.append({'name': name, 'strategy': strategy})
return GoalCollection.convert_with_links(goals[:limit], limit,
url=resource_url,
expand=expand,
sort_key=sort_key,
sort_dir=sort_dir)
@wsme_pecan.wsexpose(GoalCollection, int, wtypes.text, wtypes.text)
def get_all(self, limit=None,
sort_key='name', sort_dir='asc'):
"""Retrieve a list of goals.
:param limit: maximum number of resources to return in a single result.
:param sort_key: column to sort results by. Default: id.
:param sort_dir: direction to sort. "asc" or "desc". Default: asc.
to get only actions for that goal.
"""
return self._get_goals_collection(limit, sort_key, sort_dir)
@wsme_pecan.wsexpose(GoalCollection, wtypes.text, int,
wtypes.text, wtypes.text)
def detail(self, goal_name=None, limit=None,
sort_key='name', sort_dir='asc'):
"""Retrieve a list of actions with detail.
:param goal_name: name of a goal, to get only goals for that
action.
:param limit: maximum number of resources to return in a single result.
:param sort_key: column to sort results by. Default: id.
:param sort_dir: direction to sort. "asc" or "desc". Default: asc.
to get only goals for that goal.
"""
# NOTE(lucasagomes): /detail should only work agaist collections
parent = pecan.request.path.split('/')[:-1][-1]
if parent != "goals":
raise exception.HTTPNotFound
expand = True
resource_url = '/'.join(['goals', 'detail'])
return self._get_goals_collection(limit, sort_key, sort_dir,
expand, resource_url, goal_name)
@wsme_pecan.wsexpose(Goal, wtypes.text)
def get_one(self, goal_name):
"""Retrieve information about the given goal.
:param goal_name: name of the goal.
"""
if self.from_goals:
raise exception.OperationNotPermitted
goals = CONF.watcher_goals.goals
goal = {}
if goal_name in goals.keys():
goal = {'name': goal_name, 'strategy': goals[goal_name]}
return Goal.convert_with_links(goal)

2
watcher/applier/api/applier.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/api/command_mapper.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/api/messaging/applier_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

5
watcher/applier/api/primitive_command.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.applier.api.promise import Promise

2
watcher/applier/api/promise.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/framework/command/hypervisor_state_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/framework/command/migrate_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/applier/framework/command/nop_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -14,6 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.applier.api.primitive_command import PrimitiveCommand
from watcher.applier.api.promise import Promise

2
watcher/applier/framework/command/power_state_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

23
watcher/applier/framework/command/wrapper/nova_wrapper.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -568,7 +570,7 @@ class NovaWrapper(object):
def create_instance(self, hypervisor_id, inst_name="test", image_id=None,
flavor_name="m1.tiny",
sec_group_list=["default"],
network_names_list=["private"], keypair_name="mykeys",
network_names_list=["demo-net"], keypair_name="mykeys",
create_new_floating_ip=True,
block_device_mapping_v2=None):
"""This method creates a new instance.
@ -623,15 +625,16 @@ class NovaWrapper(object):
return
net_obj = {"net-id": nic_id}
net_list.append(net_obj)
s = self.nova.servers
instance = s.create(inst_name,
image, flavor=flavor,
key_name=keypair_name,
security_groups=sec_group_list,
nics=net_list,
block_device_mapping_v2=block_device_mapping_v2,
availability_zone="nova:" +
hypervisor_id)
instance = self.nova.servers. \
create(inst_name,
image, flavor=flavor,
key_name=keypair_name,
security_groups=sec_group_list,
nics=net_list,
block_device_mapping_v2=block_device_mapping_v2,
availability_zone="nova:" +
hypervisor_id)
# Poll at 5 second intervals, until the status is no longer 'BUILD'
if instance:

2
watcher/applier/framework/command_executor.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/applier/framework/default_applier.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.applier.api.applier import Applier
from watcher.applier.framework.command_executor import CommandExecutor
from watcher.objects import Action

3
watcher/applier/framework/default_command_mapper.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.applier.api.command_mapper import CommandMapper

3
watcher/applier/framework/deploy_phase.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.openstack.common import log
LOG = log.getLogger(__name__)

3
watcher/applier/framework/manager_applier.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from concurrent.futures import ThreadPoolExecutor
from oslo_config import cfg

5
watcher/applier/framework/messaging/events.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

2
watcher/applier/framework/messaging/launch_action_plan.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/framework/messaging/trigger_action_plan.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/applier/framework/rpcapi.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

5
watcher/cmd/decisionengine.py
View File

@ -27,7 +27,10 @@ from watcher.decision_engine.framework.manager_decision_engine import \
DecisionEngineManager
from watcher.openstack.common._i18n import _LI
from watcher.openstack.common import log as logging
cfg.CONF.import_opt('hostname',
'watcher.metrics_engine.framework.'
'datasources.influxdb_collector',
group='watcher_influxdb_collector')
LOG = logging.getLogger(__name__)

2
watcher/common/config.py
View File

@ -24,7 +24,7 @@ from watcher import version
def parse_args(argv, default_config_files=None):
rpc.set_defaults(control_exchange='watcher')
cfg.CONF(argv[1:],
project='watcher',
project='python-watcher',
version=version.version_info.release_string(),
default_config_files=default_config_files)
rpc.init(cfg.CONF)

56
watcher/common/exception.py
View File

@ -29,7 +29,6 @@ from watcher.common.i18n import _
from watcher.common.i18n import _LE
from watcher.openstack.common import log as logging
LOG = logging.getLogger(__name__)
exc_log_opts = [
@ -227,6 +226,61 @@ class PatchError(Invalid):
# decision engine
class BaseException(Exception):
def __init__(self, desc=""):
if (not isinstance(desc, basestring)):
raise IllegalArgumentException(
"Description must be an instance of str")
desc = desc.strip()
self._desc = desc
def get_description(self):
return self._desc
def get_message(self):
return "An exception occurred without a description."
def __str__(self):
return self.get_message()
class IllegalArgumentException(BaseException):
def __init__(self, desc):
BaseException.__init__(self, desc)
if self._desc == "":
raise IllegalArgumentException("Description cannot be empty")
def get_message(self):
return self._desc
class NoSuchMetric(BaseException):
def __init__(self, desc):
BaseException.__init__(self, desc)
if self._desc == "":
raise NoSuchMetric("No such metric")
def get_message(self):
return self._desc
class NoDataFound(BaseException):
def __init__(self, desc):
BaseException.__init__(self, desc)
if self._desc == "":
raise NoSuchMetric("no rows were returned")
def get_message(self):
return self._desc
class ClusterEmpty(WatcherException):
message = _("The list of hypervisor(s) in the cluster is empty.'")

35
watcher/decision_engine/api/collector/metrics_resource_collector.py
View File

@ -1,35 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class MetricsResourceCollector(object):
def __init__(self):
pass
def get_average_usage_vm_cpu(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_usage_vm_memory(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_virtual_machine_capacity(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_network_incomming(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_network_outcomming(self, uuid):
raise NotImplementedError("Should have implemented this")

2
watcher/decision_engine/api/messaging/decision_engine_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/decision_engine/api/messaging/event_consumer.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class EventConsumer(object):

5
watcher/decision_engine/api/planner/planner.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class Planner(object):

2
watcher/decision_engine/api/solution/solution.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/decision_engine/api/solution/solution_comparator.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/decision_engine/api/solution/solution_evaluator.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

5
watcher/decision_engine/api/strategy/meta_action.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.strategy import StrategyLevel

6
watcher/decision_engine/api/selector/selector.py → watcher/decision_engine/api/strategy/selector.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,7 +15,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class Selector(object):
pass
def define_from_goal(self, goal_name):
raise NotImplementedError("Should have implemented this")

2
watcher/decision_engine/api/strategy/strategy.py
View File

@ -24,8 +24,6 @@ from watcher.decision_engine.framework.default_solution import DefaultSolution
LOG = log.getLogger(__name__)
# todo(jed) add interface
@six.add_metaclass(abc.ABCMeta)
class Strategy(object):

5
watcher/decision_engine/api/strategy/strategy_context.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class StrategyContext(object):

5
watcher/decision_engine/api/strategy/strategy_level.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

5
watcher/decision_engine/api/strategy/strategy_state.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

59
watcher/decision_engine/framework/command/trigger_audit_command.py
View File

@ -45,41 +45,42 @@ class TriggerAuditCommand(DecisionEngineCommand):
self.messaging.topic_status.publish_event(event.get_type().name,
payload)
# todo(jed) remove params
def update_audit(self, request_context, audit_uuid, state):
LOG.debug("update audit " + str(state))
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = state
audit.save()
self.notify(audit_uuid, Events.TRIGGER_AUDIT, state)
return audit
def execute(self, audit_uuid, request_context):
LOG.debug("Execute TriggerAuditCommand ")
try:
LOG.debug("Execute TriggerAuditCommand ")
# 1 - change status to ONGOING
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = AuditStatus.ONGOING
audit.save()
# 1 - change status to ONGOING
audit = self.update_audit(request_context, audit_uuid,
AuditStatus.ONGOING)
# 2 - notify the others components of the system
self.notify(audit_uuid, Events.TRIGGER_AUDIT, AuditStatus.ONGOING)
# 3 - Retrieve metrics
cluster = self.statedb.get_latest_state_cluster()
# 3 - Retrieve metrics
cluster = self.statedb.get_latest_state_cluster()
# 4 - Select appropriate strategy
audit_template = AuditTemplate.get_by_id(request_context,
audit.audit_template_id)
# 4 - Select appropriate strategy
audit_template = AuditTemplate.get_by_id(request_context,
audit.audit_template_id)
self.strategy_context.set_goal(audit_template.goal)
self.strategy_context.set_metrics_resource_collector(
self.ressourcedb)
self.strategy_context.set_goal(audit_template.goal)
self.strategy_context.set_metrics_resource_collector(self.ressourcedb)
# 5 - compute change requests
solution = self.strategy_context.execute_strategy(cluster)
# 5 - compute change requests
solution = self.strategy_context.execute_strategy(cluster)
# 6 - create an action plan
planner = DefaultPlanner()
planner.schedule(request_context, audit.id, solution)
# 6 - create an action plan
planner = DefaultPlanner()
planner.schedule(request_context, audit.id, solution)
# 7 - change status to SUCCESS
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = AuditStatus.SUCCESS
audit.save()
# 8 - notify the others components of the system
self.notify(audit_uuid, Events.TRIGGER_AUDIT,
AuditStatus.SUCCESS)
# 7 - change status to SUCCESS and notify
self.update_audit(request_context, audit_uuid, AuditStatus.SUCCESS)
except Exception as e:
self.update_audit(request_context, audit_uuid, AuditStatus.FAILED)
LOG.error(" " + unicode(e))

18
watcher/decision_engine/framework/default_planner.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -88,11 +90,11 @@ class DefaultPlanner(Planner):
# TODO(jed) type
primitive = self.create_action(action_plan.id,
Primitives.LIVE_MIGRATE.value,
action.get_vm().get_uuid(),
action.get_vm().uuid,
action.get_source_hypervisor().
get_uuid(),
uuid,
action.get_dest_hypervisor().
get_uuid(),
uuid,
description=str(action)
)
@ -100,18 +102,16 @@ class DefaultPlanner(Planner):
primitive = self.create_action(action_plan_id=action_plan.id,
action_type=Primitives.
POWER_STATE.value,
applies_to=action.target.
get_uuid(),
applies_to=action.target.uuid,
parameter=action.
get_power_state().
powerstate.
value, description=str(action))
elif isinstance(action, ChangeHypervisorState):
primitive = self.create_action(action_plan_id=action_plan.id,
action_type=Primitives.
HYPERVISOR_STATE.value,
applies_to=action.target.
get_uuid(),
parameter=action.get_state().
applies_to=action.target.uuid,
parameter=action.state.
value,
description=str(action))

4
watcher/decision_engine/framework/default_solution.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,7 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# jed <jean-emile.dartois@b-com.com>
#
from watcher.decision_engine.api.solution.solution import Solution
from watcher.openstack.common import log

3
watcher/decision_engine/framework/manager_decision_engine.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from concurrent.futures import ThreadPoolExecutor
from oslo_config import cfg

14
watcher/decision_engine/framework/messaging/audit_endpoint.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,24 +15,24 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.framework.command.trigger_audit_command import \
TriggerAuditCommand
from watcher.decision_engine.framework.ressourcedb_collector import RessourceDB
from watcher.decision_engine.framework.statedb_collector import NovaCollector
from watcher.metrics_engine.framework.collector_manager import CollectorManager
from watcher.openstack.common import log
LOG = log.getLogger(__name__)
class AuditEndpoint(object):
def __init__(self, de):
self.de = de
self.manager = CollectorManager()
def do_trigger_audit(self, context, audit_uuid):
statedb = NovaCollector()
ressourcedb = RessourceDB()
statedb = self.manager.get_statedb_collector()
ressourcedb = self.manager.get_metric_collector()
audit = TriggerAuditCommand(self.de, statedb,
ressourcedb)
audit.execute(audit_uuid, context)

5
watcher/decision_engine/framework/messaging/events.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

25
watcher/decision_engine/framework/meta_actions/hypervisor_state.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.meta_action import MetaAction
from watcher.decision_engine.framework.model.hypervisor_state import \
HypervisorState
@ -26,14 +29,24 @@ class ChangeHypervisorState(MetaAction):
:param target:
:return:
'''
self.target = target
self.state = HypervisorState.ONLINE
self._target = target
self._state = HypervisorState.ONLINE
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state
@property
def target(self):
return self._target
@target.setter
def target(self, p):
self._target = p
def __str__(self):
return MetaAction.__str__(self) + " ChangeHypervisorState" + str(

3
watcher/decision_engine/framework/meta_actions/migrate.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

27
watcher/decision_engine/framework/meta_actions/power_state.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.meta_action import MetaAction
from watcher.decision_engine.framework.model.power_state import PowerState
@ -25,15 +28,25 @@ class ChangePowerState(MetaAction):
:param target:
:return:
"""
self.target = target
self.power_state = PowerState.g0
self._target = target
self._power_state = PowerState.g0
def set_power_state(self, state):
self.power_state = state
@property
def powerstate(self):
return self._power_state
def get_power_state(self):
return self.power_state
@powerstate.setter
def powerstate(self, p):
self._power_state = p
@property
def target(self):
return self._target
@target.setter
def target(self, p):
self._target = p
def __str__(self):
return MetaAction.__str__(self) + "ChangePowerState " + str(
self.target) + " => " + str(self.power_state)
self.target) + " => " + str(self.powerstate)

31
watcher/decision_engine/framework/model/hypervisor.py
View File

@ -21,11 +21,30 @@ from watcher.decision_engine.framework.model.power_state import PowerState
class Hypervisor(NamedElement):
def __init__(self):
self.state = HypervisorState.ONLINE
self.power_state = PowerState.g0
self._state = HypervisorState.ONLINE
self._status = HypervisorState.ENABLED
self._power_state = PowerState.g0
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state
@property
def status(self):
return self._status
@status.setter
def status(self, s):
self._status = s
@property
def powerstate(self):
return self._power_state
@powerstate.setter
def powerstate(self, p):
self._power_state = p

6
watcher/decision_engine/framework/model/hypervisor_state.py
View File

@ -18,5 +18,7 @@ from enum import Enum
class HypervisorState(Enum):
ONLINE = 'ONLINE'
OFFLINE = 'OFFLINE'
ONLINE = 'up'
OFFLINE = 'down'
ENABLED = 'enabled'
DISABLED = 'disabled'

25
watcher/decision_engine/framework/model/mapping.py
View File

@ -37,15 +37,15 @@ class Mapping(object):
self.lock.acquire()
# init first
if hypervisor.get_uuid() not in self._mapping_hypervisors.keys():
self._mapping_hypervisors[hypervisor.get_uuid()] = []
if hypervisor.uuid not in self._mapping_hypervisors.keys():
self._mapping_hypervisors[hypervisor.uuid] = []
# map node => vms
self._mapping_hypervisors[hypervisor.get_uuid()].append(
vm.get_uuid())
self._mapping_hypervisors[hypervisor.uuid].append(
vm.uuid)
# map vm => node
self.mapping_vm[vm.get_uuid()] = hypervisor.get_uuid()
self.mapping_vm[vm.uuid] = hypervisor.uuid
finally:
self.lock.release()
@ -56,16 +56,23 @@ class Mapping(object):
:param hypervisor: the hypervisor
:param vm: the virtual machine or instance
"""
self.unmap_from_id(hypervisor.get_uuid(), vm.get_uuid())
self.unmap_from_id(hypervisor.uuid, vm.uuid)
def unmap_from_id(self, node_uuid, vm_uuid):
"""
:rtype : object
"""
try:
self.lock.acquire()
if str(node_uuid) in self._mapping_hypervisors:
self._mapping_hypervisors[str(node_uuid)].remove(str(vm_uuid))
# remove vm
self.mapping_vm.pop(vm_uuid)
else:
LOG.warn("trying to delete the virtual machine " + str(
vm_uuid) + " but it was not found")
vm_uuid) + " but it was not found on hypervisor" + str(
node_uuid))
finally:
self.lock.release()
@ -76,7 +83,7 @@ class Mapping(object):
return self.mapping_vm
def get_node_from_vm(self, vm):
return self.get_node_from_vm_id(vm.get_uuid())
return self.get_node_from_vm_id(vm.uuid)
def get_node_from_vm_id(self, vm_uuid):
"""Getting host information from the guest VM
@ -93,7 +100,7 @@ class Mapping(object):
:param hypervisor:
:return:
"""
return self.get_node_vms_from_id(hypervisor.get_uuid())
return self.get_node_vms_from_id(hypervisor.uuid)
def get_node_vms_from_id(self, node_uuid):
if str(node_uuid) in self._mapping_hypervisors.keys():

17
watcher/decision_engine/framework/model/model_root.py
View File

@ -15,6 +15,7 @@
# limitations under the License.
from watcher.common.exception import HypervisorNotFound
from watcher.common.exception import IllegalArgumentException
from watcher.common.exception import VMNotFound
from watcher.decision_engine.framework.model.hypervisor import Hypervisor
from watcher.decision_engine.framework.model.mapping import Mapping
@ -33,26 +34,28 @@ class ModelRoot(object):
def assert_hypervisor(self, hypervisor):
if not isinstance(hypervisor, Hypervisor):
raise Exception("assert_vm")
raise IllegalArgumentException(
"Hypervisor must be an instance of hypervisor")
def assert_vm(self, vm):
if not isinstance(vm, VM):
raise Exception("assert_vm")
raise IllegalArgumentException(
"VM must be an instance of VM")
def add_hypervisor(self, hypervisor):
self.assert_hypervisor(hypervisor)
self._hypervisors[hypervisor.get_uuid()] = hypervisor
self._hypervisors[hypervisor.uuid] = hypervisor
def remove_hypervisor(self, hypervisor):
self.assert_hypervisor(hypervisor)
if str(hypervisor.get_uuid()) not in self._hypervisors.keys():
raise HypervisorNotFound(hypervisor.get_uuid())
if str(hypervisor.uuid) not in self._hypervisors.keys():
raise HypervisorNotFound(hypervisor.uuid)
else:
del self._hypervisors[hypervisor.get_uuid()]
del self._hypervisors[hypervisor.uuid]
def add_vm(self, vm):
self.assert_vm(vm)
self._vms[vm.get_uuid()] = vm
self._vms[vm.uuid] = vm
def get_all_hypervisors(self):
return self._hypervisors

21
watcher/decision_engine/framework/model/named_element.py
View File

@ -18,13 +18,24 @@
class NamedElement(object):
def __init__(self):
self.uuid = ""
self._uuid = ""
self._human_id = ""
def set_uuid(self, uuid):
self.uuid = uuid
@property
def uuid(self):
return self._uuid
def get_uuid(self):
return self.uuid
@uuid.setter
def uuid(self, u):
self._uuid = u
@property
def human_id(self):
return self._human_id
@human_id.setter
def human_id(self, h):
self._human_id = h
def __str__(self):
return "[" + str(self.uuid) + "]"

4
watcher/decision_engine/framework/model/resource.py
View File

@ -39,7 +39,7 @@ class Resource(object):
return self.name
def set_capacity(self, element, value):
self.mapping[element.get_uuid()] = value
self.mapping[element.uuid] = value
def get_capacity_from_id(self, uuid):
if str(uuid) in self.mapping.keys():
@ -49,4 +49,4 @@ class Resource(object):
return None
def get_capacity(self, element):
return self.get_capacity_from_id(element.get_uuid())
return self.get_capacity_from_id(element.uuid)

12
watcher/decision_engine/framework/model/vm.py
View File

@ -19,10 +19,12 @@ from watcher.decision_engine.framework.model.vm_state import VMState
class VM(NamedElement):
def __init__(self):
self.state = VMState.INIT
self._state = VMState.ACTIVE.value
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state

20
watcher/decision_engine/framework/model/vm_state.py
View File

@ -18,9 +18,17 @@ from enum import Enum
class VMState(Enum):
INIT = 1,
READY = 2,
RUNNING = 3,
SLEEPING = 4,
KILLED = 5,
LIVE_MIGRATION = 6
ACTIVE = 'active' # VM is running
BUILDING = 'building' # VM only exists in DB
PAUSED = 'paused'
SUSPENDED = 'suspended' # VM is suspended to disk.
STOPPED = 'stopped' # VM is powered off, the disk image is still there.
RESCUED = 'rescued' # A rescue image is running with the original VM image
# attached.
RESIZED = 'resized' # a VM with the new size is active.
SOFT_DELETED = 'soft-delete'
# still available to restore.
DELETED = 'deleted' # VM is permanently deleted.
ERROR = 'error'

117
watcher/decision_engine/framework/ressourcedb_collector.py
View File

@ -1,117 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import ceilometerclient.v2 as c_client
import keystoneclient.v3.client as ksclient
from oslo_config import cfg
CONF = cfg.CONF
from watcher.decision_engine.api.collector.metrics_resource_collector import \
MetricsResourceCollector
CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
class RessourceDB(MetricsResourceCollector):
def __init__(self):
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
self.keystone = ksclient.Client(**creds)
self.ceilometer = c_client.Client(
endpoint=self.get_ceilometer_uri(),
token=self.keystone.auth_token)
def make_query(user_id=None, tenant_id=None, resource_id=None,
user_ids=None, tenant_ids=None, resource_ids=None):
"""Returns query built form given parameters.
This query can be then used for querying resources, meters and
statistics.
:Parameters:
- `user_id`: user_id, has a priority over list of ids
- `tenant_id`: tenant_id, has a priority over list of ids
- `resource_id`: resource_id, has a priority over list of ids
- `user_ids`: list of user_ids
- `tenant_ids`: list of tenant_ids
- `resource_ids`: list of resource_ids
"""
user_ids = user_ids or []
tenant_ids = tenant_ids or []
resource_ids = resource_ids or []
query = []
if user_id:
user_ids = [user_id]
for u_id in user_ids:
query.append({"field": "user_id", "op": "eq", "value": u_id})
if tenant_id:
tenant_ids = [tenant_id]
for t_id in tenant_ids:
query.append({"field": "project_id", "op": "eq", "value": t_id})
if resource_id:
resource_ids = [resource_id]
for r_id in resource_ids:
query.append({"field": "resource_id", "op": "eq", "value": r_id})
return query
def get_ceilometer_uri(self):
a = self.keystone.services.list(**{'type': 'metering'})
e = self.keystone.endpoints.list()
for s in e:
if s.service_id == a[0].id and s.interface == 'internal':
return s.url
raise Exception("Ceilometer Metering Service internal not defined")
def get_average_usage_vm_cpu(self, instance_uuid):
"""The last VM CPU usage values to average
:param uuid:00
:return:
"""
# query influxdb stream
query = self.make_query(resource_id=instance_uuid)
cpu_util_sample = self.ceilometer.samples.list('cpu_util',
q=query)
cpu_usage = 0
count = len(cpu_util_sample)
for each in cpu_util_sample:
# print each.timestamp, each.counter_name, each.counter_volume
cpu_usage = cpu_usage + each.counter_volume
if count == 0:
return 0
else:
return cpu_usage / len(cpu_util_sample)
def get_average_usage_vm_memory(self, uuid):
# Obtaining Memory Usage is not implemented for LibvirtInspector
# waiting for kilo memory.resident
return 1
def get_average_usage_vm_disk(self, uuid):
# waiting for kilo disk.usage
return 1

3
watcher/decision_engine/framework/rpcapi.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_config import cfg

104
watcher/decision_engine/framework/statedb_collector.py
View File

@ -1,104 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from keystoneclient.auth.identity import v3
from keystoneclient import session
from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
from watcher.decision_engine.api.collector.cluster_state_collector import \
ClusterStateCollector
from watcher.decision_engine.framework.model.hypervisor import Hypervisor
from watcher.decision_engine.framework.model.model_root import ModelRoot
from watcher.decision_engine.framework.model.resource import Resource
from watcher.decision_engine.framework.model.resource import ResourceType
from watcher.decision_engine.framework.model.vm import VM
from watcher.openstack.common import log
from oslo_config import cfg
CONF = cfg.CONF
LOG = log.getLogger(__name__)
CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
class NovaCollector(ClusterStateCollector):
def get_latest_state_cluster(self):
try:
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
auth = v3.Password(auth_url=creds['auth_url'],
username=creds['username'],
password=creds['password'],
project_name=creds['project_name'],
user_domain_name=creds[
'user_domain_name'],
project_domain_name=creds[
'project_domain_name'])
sess = session.Session(auth=auth)
wrapper = NovaWrapper(creds, session=sess)
cluster = ModelRoot()
mem = Resource(ResourceType.memory)
num_cores = Resource(ResourceType.cpu_cores)
disk = Resource(ResourceType.disk)
cluster.create_resource(mem)
cluster.create_resource(num_cores)
cluster.create_resource(disk)
flavor_cache = {}
hypervisors = wrapper.get_hypervisors_list()
for h in hypervisors:
i = h.hypervisor_hostname.index('.')
name = h.hypervisor_hostname[0:i]
# create hypervisor in stateDB
hypervisor = Hypervisor()
hypervisor.set_uuid(name)
# set capacity
mem.set_capacity(hypervisor, h.memory_mb)
disk.set_capacity(hypervisor, h.disk_available_least)
num_cores.set_capacity(hypervisor, h.vcpus)
cluster.add_hypervisor(hypervisor)
vms = wrapper.get_vms_by_hypervisor(str(name))
for v in vms:
# create VM in stateDB
vm = VM()
vm.set_uuid(v.id)
# set capacity
wrapper.get_flavor_instance(v, flavor_cache)
mem.set_capacity(vm, v.flavor['ram'])
disk.set_capacity(vm, v.flavor['disk'])
num_cores.set_capacity(vm, v.flavor['vcpus'])
# print(dir(v))
cluster.get_mapping().map(hypervisor, vm)
cluster.add_vm(vm)
return cluster
except Exception as e:
LOG.error("nova collector " + unicode(e))
return None

12
watcher/decision_engine/framework/strategy/strategy_loader.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,7 +15,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_config import cfg
from stevedore import driver
from watcher.decision_engine.strategies.basic_consolidation import \
BasicConsolidation
from watcher.openstack.common import log
@ -52,5 +56,13 @@ class StrategyLoader(object):
"Basic offline consolidation")
}
def load_driver(self, algo):
_algo = driver.DriverManager(
namespace='watcher_strategies',
name=algo,
invoke_on_load=True,
)
return _algo
def load(self, model):
return self.strategies[model]

3
watcher/decision_engine/framework/strategy/strategy_selector.py
View File

@ -15,6 +15,7 @@
# limitations under the License.
from oslo_config import cfg
from watcher.decision_engine.api.strategy.selector import Selector
from watcher.decision_engine.framework.strategy.strategy_loader import \
StrategyLoader
from watcher.objects.audit_template import Goal
@ -40,7 +41,7 @@ CONF.register_group(goals_opt_group)
CONF.register_opts(WATCHER_GOALS_OPTS, goals_opt_group)
class StrategySelector(object):
class StrategySelector(Selector):
def __init__(self):
self.strategy_loader = StrategyLoader()

117
watcher/decision_engine/strategies/basic_consolidation.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,10 +15,15 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.framework.model.vm_state import VMState
from watcher.metrics_engine.api.metrics_resource_collector import \
AggregationFunction
from watcher.common.exception import ClusterEmpty
from watcher.common.exception import ClusteStateNotDefined
from watcher.common.exception import MetricCollectorNotDefined
from watcher.common.exception import NoDataFound
from watcher.decision_engine.api.strategy.strategy import Strategy
from watcher.decision_engine.api.strategy.strategy import StrategyLevel
from watcher.decision_engine.framework.meta_actions.hypervisor_state import \
@ -78,8 +85,8 @@ class BasicConsolidation(Strategy):
self.efficiency = 100
# TODO(jed) improve threshold overbooking ?,...
self.threshold_mem = 0.90
self.threshold_disk = 0.80
self.threshold_mem = 1
self.threshold_disk = 1
self.threshold_cores = 1
# TODO(jed) target efficiency
@ -115,6 +122,11 @@ class BasicConsolidation(Strategy):
if src_hypervisor == dest_hypervisor:
return False
LOG.debug('Migrate VM %s from %s to %s ',
str(src_hypervisor),
str(dest_hypervisor),
str(vm_to_mig))
total_cores = 0
total_disk = 0
total_mem = 0
@ -162,6 +174,13 @@ class BasicConsolidation(Strategy):
cores_available = cap_cores.get_capacity(dest_hypervisor)
disk_available = cap_disk.get_capacity(dest_hypervisor)
mem_available = cap_mem.get_capacity(dest_hypervisor)
LOG.debug("VCPU %s/%s ", str(total_cores * self.threshold_cores),
str(cores_available), )
LOG.debug("DISK %s/%s ", str(total_disk * self.threshold_disk),
str(disk_available), )
LOG.debug("MEM %s/%s ", str(total_mem * self.threshold_mem),
str(mem_available))
if cores_available >= total_cores * self.threshold_cores \
and disk_available >= total_disk * self.threshold_disk \
and mem_available >= total_mem * self.threshold_mem:
@ -232,21 +251,25 @@ class BasicConsolidation(Strategy):
metrics_collector = self.get_metrics_resource_collector()
if metrics_collector is None:
raise MetricCollectorNotDefined()
total_cores_used = 0
total_memory_used = 0
total_disk_used = 0
for vm_id in model.get_mapping().get_node_vms(hypervisor):
total_cores_used += metrics_collector.get_average_usage_vm_cpu(
vm_id)
total_memory_used += metrics_collector.get_average_usage_vm_memory(
vm_id)
total_disk_used += metrics_collector.get_average_usage_vm_disk(
vm_id)
cpu_compute_mean_16h = metrics_collector.get_measurement(
metric='compute_cpu_user_percent_gauge',
aggregation_function=AggregationFunction.MEAN,
start_time="16 hours before now",
filters=["resource_id=" + hypervisor.uuid + ""])
if len(cpu_compute_mean_16h) > 0:
cpu_capacity = model.get_resource_from_id(
ResourceType.cpu_cores).get_capacity(hypervisor)
cpu_utilization = float(cpu_compute_mean_16h[0].value)
total_cores_used = cpu_capacity * (cpu_utilization / 100)
else:
raise NoDataFound(
"No values returned for " + str(hypervisor.uuid) +
" compute_cpu_percent_gauge")
return self.calculate_weight(model, hypervisor, total_cores_used,
total_disk_used,
total_memory_used)
0,
0)
def calculate_migration_efficiency(self):
"""Calculate migration efficiency
@ -275,15 +298,25 @@ class BasicConsolidation(Strategy):
if model is None:
raise ClusteStateNotDefined()
vm = model.get_vm_from_id(vm.get_uuid())
cores_used = metric_collector.get_average_usage_vm_cpu(vm.get_uuid())
memory_used = metric_collector.get_average_usage_vm_memory(
vm.get_uuid())
disk_used = metric_collector.get_average_usage_vm_disk(vm.get_uuid())
vm = model.get_vm_from_id(vm.uuid)
instance_cpu_mean_16 = metric_collector.get_measurement(
metric='instance_cpu_percent_gauge',
aggregation_function=AggregationFunction.MEAN,
start_time="16 hours before now",
filters=["resource_id=" + vm.uuid + ""])
return self.calculate_weight(model, vm, cores_used,
disk_used,
memory_used)
if len(instance_cpu_mean_16) > 0:
cpu_capacity = model.get_resource_from_id(
ResourceType.cpu_cores).get_capacity(vm)
vm_cpu_utilization = instance_cpu_mean_16[0].value
total_cores_used = cpu_capacity * (vm_cpu_utilization / 100)
else:
raise NoDataFound("No values returned for " + str(vm.uuid) +
" instance_cpu_percent_gauge")
return self.calculate_weight(model, vm, total_cores_used,
0,
0)
def print_utilization(self, model):
if model is None:
@ -308,13 +341,27 @@ class BasicConsolidation(Strategy):
unsuccessful_migration = 0
first = True
self.print_utilization(current_model)
size_cluster = len(current_model.get_all_hypervisors())
if size_cluster == 0:
raise ClusterEmpty()
self.compute_attempts(size_cluster)
for hypevisor_id in current_model.get_all_hypervisors():
hypervisor = current_model.get_hypervisor_from_id(hypevisor_id)
count = current_model.get_mapping(). \
get_node_vms_from_id(hypevisor_id)
if len(count) == 0:
change_power = ChangePowerState(hypervisor)
change_power.powerstate = PowerState.g1_S1
change_power.set_level(StrategyLevel.conservative)
self.solution.add_change_request(change_power)
if hypervisor.state == HypervisorState.ONLINE:
h = ChangeHypervisorState(hypervisor)
h.set_level(StrategyLevel.aggressive)
h.state = HypervisorState.OFFLINE
self.solution.add_change_request(h)
while self.get_allowed_migration_attempts() >= unsuccessful_migration:
if first is not True:
self.efficiency = self.calculate_migration_efficiency()
@ -325,9 +372,16 @@ class BasicConsolidation(Strategy):
''' calculate score of nodes based on load by VMs '''
for hypevisor_id in current_model.get_all_hypervisors():
hypevisor = current_model.get_hypervisor_from_id(hypevisor_id)
result = self.calculate_score_node(hypevisor, current_model)
if result != 0:
hypervisor = current_model.get_hypervisor_from_id(hypevisor_id)
count = current_model.get_mapping(). \
get_node_vms_from_id(hypevisor_id)
if len(count) > 0:
result = self.calculate_score_node(hypervisor,
current_model)
else:
''' the hypervisor has not VMs '''
result = 0
if len(count) > 0:
score.append((hypevisor_id, result))
''' sort compute nodes by Score decreasing '''''
@ -350,8 +404,9 @@ class BasicConsolidation(Strategy):
vm_score = []
for vm_id in vms_to_mig:
vm = current_model.get_vm_from_id(vm_id)
vm_score.append(
(vm_id, self.calculate_score_vm(vm, current_model)))
if vm.state == VMState.ACTIVE.value:
vm_score.append(
(vm_id, self.calculate_score_vm(vm, current_model)))
''' sort VM's by Score '''
v = sorted(vm_score, reverse=True, key=lambda x: (x[1]))
@ -392,7 +447,7 @@ class BasicConsolidation(Strategy):
# TODO(jed) how to manage strategy level
# from conservative to aggressive
change_power = ChangePowerState(mig_src_hypervisor)
change_power.set_power_state(PowerState.g1_S1)
change_power.powerstate = PowerState.g1_S1
change_power.set_level(
StrategyLevel.conservative)
tmp_vm_migration_schedule.append(change_power)
@ -400,7 +455,7 @@ class BasicConsolidation(Strategy):
h = ChangeHypervisorState(mig_src_hypervisor)
h.set_level(StrategyLevel.aggressive)
h.set_state(HypervisorState.OFFLINE)
h.state = HypervisorState.OFFLINE
tmp_vm_migration_schedule.append(h)
self.number_of_released_nodes += 1
@ -414,7 +469,7 @@ class BasicConsolidation(Strategy):
self.solution.add_change_request(a)
else:
unsuccessful_migration += 1
self.print_utilization(current_model)
# self.print_utilization(current_model)
infos = {
"number_of_migrations": self.number_of_migrations,
"number_of_nodes_released": self.number_of_released_nodes,

3
watcher/decision_engine/strategies/dummy_strategy.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.strategy import Strategy
from watcher.openstack.common import log

0
watcher/decision_engine/api/collector/__init__.py → watcher/metrics_engine/__init__.py
View File

0
watcher/decision_engine/api/selector/__init__.py → watcher/metrics_engine/api/__init__.py
View File

12
watcher/decision_engine/api/collector/cluster_state_collector.py → watcher/metrics_engine/api/cluster_state_collector.py
View File

@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,10 +15,14 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class ClusterStateCollector(object):
@abc.abstractmethod
def get_latest_state_cluster(self):
raise NotImplementedError("Should have implemented this")
# todo(jed) think abouts needed interfaces
# todo(jed) stream incremental diff

65
watcher/metrics_engine/api/metrics_resource_collector.py Normal file
View File

@ -0,0 +1,65 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import abc
from enum import Enum
import six
class AggregationFunction(Enum):
MEAN = 'mean'
COUNT = 'count'
class Measure(object):
def __init__(self, time, value):
self.time = time
self.value = value
def __str__(self):
return str(self.time) + " " + str(self.value)
@six.add_metaclass(abc.ABCMeta)
class MetricsResourceCollector(object):
@abc.abstractmethod
def get_measurement(self,
metric,
callback=None,
start_time=None,
end_time=None,
filters=None,
aggregation_function=None,
intervals=None):
"""
:param metric: The full name of a metric in the system.
Must be the complete name. Case sensitive
:param callback: Asynchronous Callback Functions to live retrev
:param start_time:Starting time for the query.
This may be an absolute or relative time.
:param end_time: An end time for the query.
If the end time is not supplied, the current time
on the TSD will be used.
:param filters: An optional set of tags for filtering or grouping
:param aggregation_function: A mathematical function
:param intervals: An optional interval and function
to reduce the number of data points returned
:return:
"""
raise NotImplementedError("Should have implemented this")

0
watcher/tests/applier/demo/__init__.py → watcher/metrics_engine/framework/__init__.py
View File

83
watcher/metrics_engine/framework/collector_manager.py Normal file
View File

@ -0,0 +1,83 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from keystoneclient.auth.identity import v3
from keystoneclient import session
from oslo_config import cfg
from stevedore import driver
from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
from watcher.metrics_engine.framework.statedb_collector import NovaCollector
from watcher.openstack.common import log
LOG = log.getLogger(__name__)
CONF = cfg.CONF
WATCHER_METRICS_COLLECTOR_OPTS = [
cfg.StrOpt('metrics_resource',
default="influxdb",
help='The driver that collect measurements'
'of the utilization'
'of the physical and virtual resources')
]
metrics_collector_opt_group = cfg.OptGroup(
name='watcher_collector',
title='Defines Metrics collector available')
CONF.register_group(metrics_collector_opt_group)
CONF.register_opts(WATCHER_METRICS_COLLECTOR_OPTS, metrics_collector_opt_group)
CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
class CollectorManager(object):
def get_metric_collector(self):
manager = driver.DriverManager(
namespace='watcher_metrics_collector',
name=CONF.watcher_collector.metrics_resource,
invoke_on_load=True,
)
return manager.driver
def get_statedb_collector(self):
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
auth = v3.Password(auth_url=creds['auth_url'],
username=creds['username'],
password=creds['password'],
project_name=creds['project_name'],
user_domain_name=creds[
'user_domain_name'],
project_domain_name=creds[
'project_domain_name'])
sess = session.Session(auth=auth)
wrapper = NovaWrapper(creds, session=sess)
return NovaCollector(wrapper=wrapper)

20
watcher/metrics_engine/framework/datasources/__init__.py Normal file
View File

@ -0,0 +1,20 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class InvalidQuery(Exception):
pass

169
watcher/metrics_engine/framework/datasources/influxdb_collector.py Normal file
View File

@ -0,0 +1,169 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from concurrent.futures import ThreadPoolExecutor
import datetime
import parsedatetime
from influxdb import InfluxDBClient
from oslo_config import cfg
from watcher.metrics_engine.api.metrics_resource_collector import \
AggregationFunction
from watcher.metrics_engine.api.metrics_resource_collector import Measure
from watcher.metrics_engine.api.metrics_resource_collector import \
MetricsResourceCollector
from watcher.metrics_engine.framework.datasources.sql_ast.build_db_query import \
DBQuery
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import And
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import \
Condition
from watcher.openstack.common import log
LOG = log.getLogger(__name__)
CONF = cfg.CONF
WATCHER_INFLUXDB_COLLECTOR_OPTS = [
cfg.StrOpt('hostname',
default='localhost',
help='The hostname to connect to InfluxDB'),
cfg.IntOpt('port',
default='8086',
help='port to connect to InfluxDB, defaults to 8086'),
cfg.StrOpt('username',
default='root',
help='user to connect, defaults to root'),
cfg.StrOpt('password',
default='root',
help='password of the user, defaults to root'),
cfg.StrOpt('database',
default='indeed',
help='database name to connect to'),
cfg.BoolOpt('param ssl',
default=False,
help='use https instead of http to connect to InfluxDB'),
cfg.IntOpt('timeout',
default='5',
help='number of seconds Requests'
'will wait for your client to establish a connection'),
cfg.IntOpt('timeout',
default='5',
help='number of seconds Requests'
'will wait for your client to establish a connection'),
cfg.BoolOpt('use_udp',
default=False,
help='use UDP to connect to InfluxDB'),
cfg.IntOpt('udp_port',
default='4444',
help=' UDP port to connect to InfluxDB')
]
influxdb_collector_opt_group = cfg.OptGroup(
name='watcher_influxdb_collector',
title='Defines the parameters of the module collector')
CONF.register_group(influxdb_collector_opt_group)
CONF.register_opts(WATCHER_INFLUXDB_COLLECTOR_OPTS,
influxdb_collector_opt_group)
class InfluxDBCollector(MetricsResourceCollector):
def __init__(self):
self.executor = ThreadPoolExecutor(max_workers=3)
def get_client(self):
LOG.debug("InfluxDB " + str(CONF.watcher_influxdb_collector.hostname))
influx = InfluxDBClient(CONF.watcher_influxdb_collector.hostname,
CONF.watcher_influxdb_collector.port,
CONF.watcher_influxdb_collector.username,
CONF.watcher_influxdb_collector.password,
CONF.watcher_influxdb_collector.database)
if {u'name': u'' + CONF.watcher_influxdb_collector.database + ''} not \
in influx.get_list_database():
raise Exception("The selected database does not exist"
"or the user credentials supplied are wrong")
return influx
def convert(self, time):
cal = parsedatetime.Calendar()
time_struct, result = cal.parse(time)
return datetime.datetime(*time_struct[:6]).ctime()
def build_query(self,
measurement,
start_time=None,
end_time=None,
filters=None,
aggregation_function=None,
intervals=None):
query = DBQuery(measurement)
conditions = []
if start_time is not None:
c = Condition('time', '>', self.convert(start_time))
conditions.append(c)
if end_time is not None:
c = Condition('time', '>', self.convert(end_time))
conditions.append(c)
if filters is not None:
for f in filters:
c = Condition(f.split('=')[0], '=', f.split('=')[1])
conditions.append(c)
if aggregation_function is not None:
if aggregation_function == AggregationFunction.MEAN:
query.select("mean(value)")
elif aggregation_function == AggregationFunction.COUNT:
query.select("count(value)")
if intervals is not None:
query.groupby("time(" + str(intervals) + ")")
if len(conditions) == 1:
query.where(conditions[0])
elif len(conditions) != 0:
_where = And(conditions[0], conditions[1])
for i in range(2, len(conditions)):
_where = And(_where, conditions[i])
query.where(_where)
LOG.debug(query)
return query
def get_measurement(self,
metric,
callback=None,
start_time=None,
end_time=None,
filters=None,
aggregation_function=None,
intervals=None):
results = []
client = self.get_client()
query = self.build_query(metric, start_time, end_time, filters,
aggregation_function, intervals)
results_from_influx = client.query(query)
for item in results_from_influx[None]:
time = item.get('time', None)
for field in ['value', 'count', 'min', 'max', 'mean']:
value = item.get(field, None)
if value is not None:
row = Measure(time, value)
if callback is not None:
self.executor.submit(callback, row)
else:
results.append(row)
return results

0
watcher/tests/decision_engine/demo/__init__.py → watcher/metrics_engine/framework/datasources/sql_ast/__init__.py
View File

59
watcher/metrics_engine/framework/datasources/sql_ast/build_db_query.py Normal file
View File

@ -0,0 +1,59 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import From
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import \
GroupBy
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Limit
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import List
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Select
from watcher.metrics_engine.framework.datasources.sql_ast.sql_ast import Where
class DBQuery(object):
def __init__(self, _from):
self._select = Select(_from)
self.inline = False
def select_from(self, _from):
self._select._from = From(_from)
return self
def where(self, where):
self._select.where = Where(where)
return self
def groupby(self, g):
self._select.groupby = GroupBy(g)
return self
def limit(self, limit):
self._select.limit = Limit(limit)
return self
def select(self, *args):
self._select.what = List(*args)
return self
def __str__(self):
self._select.inline = self.inline
s = str(self._select)
if not self.inline:
s += ';'
return s

157
watcher/metrics_engine/framework/datasources/sql_ast/sql_ast.py Normal file
View File

@ -0,0 +1,157 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class ASTNode(object):
visited = False
inline = False
def __init__(self):
pass
def children(self):
for c in self._children:
yield c
def __str__(self):
pass
class Condition(ASTNode):
def __init__(self, what, operator, _on):
self.what = what
self._on = "'" + str(_on) + "'"
self.operator = operator
def __str__(self):
s = self.what + ' ' + self.operator + ' ' + self._on
if self.inline:
s = '' + s + ''
return s
class BinaryNode(ASTNode):
def __init__(self, left, right):
self.left = left
self.right = right
def __str__(self):
return '({left} {middle} {right})'.format(
left=self.left,
middle=self.middle,
right=self.right
)
class And(BinaryNode):
middle = 'AND'
class Or(BinaryNode):
middle = 'OR'
class List(ASTNode):
def __init__(self, *args):
for arg in args:
if hasattr(arg, 'inline'):
arg.inline = True
self.items = args
def __str__(self):
lst = ', '.join(map(lambda x: str(x), self.items))
if self.inline:
lst = '(' + lst + ')'
return lst
class Set(ASTNode):
def __init__(self, **kwargs):
self.items = kwargs
def __str__(self):
return ', '.join(
['{0}={1}'.format(key, val) for key, val in self.items.items()])
class Returning(ASTNode):
def __init__(self, _list='*'):
self._list = _list
def __str__(self):
return 'RETURNING {_list}'.format(_list=self._list)
class Limit(ASTNode):
def __init__(self, limit):
if hasattr(limit, 'inline'):
limit.inline = True
self.limit = limit
def __str__(self):
return " LIMIT " + str(self.limit)
class Where(ASTNode):
def __init__(self, logic):
if hasattr(logic, 'inline'):
logic.inline = True
self.logic = logic
def __str__(self):
return "WHERE " + str(self.logic)
class GroupBy(ASTNode):
def __init__(self, logic):
if hasattr(logic, 'inline'):
logic.inline = True
self.logic = logic
def __str__(self):
return " group by " + str(self.logic)
class From(ASTNode):
def __init__(self, _from):
if hasattr(_from, 'inline'):
_from.inline = True
self._from = _from
def __str__(self):
return 'FROM {_from}'.format(_from=self._from)
class Select(ASTNode):
def __init__(self, _from, what='*', where='', groupby='',
limit=''):
self._from = "\"" + _from + "\""
self.what = what
self.where = where and Where(where)
self.groupby = groupby
self.limit = limit and Limit(limit)
self.inlint = False
def __str__(self):
s = 'SELECT ' + str(self.what) + ' FROM ' + str(
self._from) + ' ' + str(self.where) + str(self.groupby) + str(
self.limit)
if self.inline:
s = '(' + s + ')'
return s

79
watcher/metrics_engine/framework/statedb_collector.py Normal file
View File

@ -0,0 +1,79 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_config import cfg
from watcher.decision_engine.framework.model.hypervisor import Hypervisor
from watcher.decision_engine.framework.model.model_root import ModelRoot
from watcher.decision_engine.framework.model.resource import Resource
from watcher.decision_engine.framework.model.resource import ResourceType
from watcher.decision_engine.framework.model.vm import VM
from watcher.metrics_engine.api.cluster_state_collector import \
ClusterStateCollector
from watcher.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class NovaCollector(ClusterStateCollector):
def __init__(self, wrapper):
self.wrapper = wrapper
def get_latest_state_cluster(self):
cluster = ModelRoot()
mem = Resource(ResourceType.memory)
num_cores = Resource(ResourceType.cpu_cores)
disk = Resource(ResourceType.disk)
cluster.create_resource(mem)
cluster.create_resource(num_cores)
cluster.create_resource(disk)
flavor_cache = {}
hypervisors = self.wrapper.get_hypervisors_list()
for h in hypervisors:
service = self.wrapper.nova.services.find(id=h.service['id'])
# create hypervisor in stateDB
hypervisor = Hypervisor()
hypervisor.uuid = service.host
# set capacity
mem.set_capacity(hypervisor, h.memory_mb)
disk.set_capacity(hypervisor, h.free_disk_gb)
num_cores.set_capacity(hypervisor, h.vcpus)
hypervisor.state = h.state
hypervisor.status = h.status
cluster.add_hypervisor(hypervisor)
vms = self.wrapper.get_vms_by_hypervisor(str(service.host))
for v in vms:
# create VM in stateDB
vm = VM()
vm.uuid = v.id
# nova/nova/compute/vm_states.py
vm.state = getattr(v, 'OS-EXT-STS:vm_state')
# set capacity
self.wrapper.get_flavor_instance(v, flavor_cache)
mem.set_capacity(vm, v.flavor['ram'])
disk.set_capacity(vm, v.flavor['disk'])
num_cores.set_capacity(vm, v.flavor['vcpus'])
# print(dir(v))
cluster.get_mapping().map(hypervisor, vm)
cluster.add_vm(vm)
return cluster

12
watcher/opts.py
View File

@ -20,10 +20,14 @@ import itertools
import watcher.api.app
from watcher.applier.framework import manager_applier
import watcher.common.messaging.messaging_core
import watcher.openstack.common.log
from watcher.decision_engine.framework import manager_decision_engine
from watcher.decision_engine.framework.strategy import strategy_loader
from watcher.decision_engine.framework.strategy import strategy_selector
import watcher.openstack.common.log
from watcher.metrics_engine.framework import collector_manager
from watcher.metrics_engine.framework.datasources import influxdb_collector
def list_opts():
@ -42,5 +46,9 @@ def list_opts():
('watcher_decision_engine',
manager_decision_engine.WATCHER_DECISION_ENGINE_OPTS),
('watcher_applier',
manager_applier.APPLIER_MANAGER_OPTS)
manager_applier.APPLIER_MANAGER_OPTS),
('watcher_influxdb_collector',
influxdb_collector.WATCHER_INFLUXDB_COLLECTOR_OPTS),
('watcher_metrics_collector',
collector_manager.WATCHER_METRICS_COLLECTOR_OPTS)
]

2
watcher/service.py
View File

@ -33,5 +33,5 @@ cfg.CONF.register_opts(service_opts)
def prepare_service(args=None, conf=cfg.CONF):
# log.register_options(conf)
log.setup(conf, 'watcher')
conf(args, project='watcher')
conf(args, project='python-watcher')
conf.log_opt_values(LOG, logging.DEBUG)

2
watcher/tests/api/v1/test_actions.py
View File

@ -31,7 +31,7 @@ from watcher.tests.objects import utils as obj_utils
def post_get_test_action(**kw):
action = api_utils.action_post_data(**kw)
action_plan = db_utils.get_test_action_plan()
action['action_plan_id'] = None
del action['action_plan_id']
action['action_plan_uuid'] = kw.get('action_plan_uuid',
action_plan['uuid'])
action['next'] = None

63
watcher/tests/api/v1/test_goals.py Normal file
View File

@ -0,0 +1,63 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from watcher.tests.api import base as api_base
CONF = cfg.CONF
class TestListGoal(api_base.FunctionalTest):
def setUp(self):
super(TestListGoal, self).setUp()
def _assert_goal_fields(self, goal):
goal_fields = ['name', 'strategy']
for field in goal_fields:
self.assertIn(field, goal)
def test_one(self):
response = self.get_json('/goals')
self._assert_goal_fields(response['goals'][0])
def test_get_one(self):
goal_name = CONF.watcher_goals.goals.keys()[0]
response = self.get_json('/goals/%s' % goal_name)
self.assertEqual(goal_name, response['name'])
self._assert_goal_fields(response)
def test_detail(self):
goal_name = CONF.watcher_goals.goals.keys()[0]
response = self.get_json('/goals/detail')
self.assertEqual(goal_name, response['goals'][0]["name"])
self._assert_goal_fields(response['goals'][0])
def test_detail_against_single(self):
goal_name = CONF.watcher_goals.goals.keys()[0]
response = self.get_json('/goals/%s/detail' % goal_name,
expect_errors=True)
self.assertEqual(404, response.status_int)
def test_many(self):
response = self.get_json('/goals')
self.assertEqual(len(CONF.watcher_goals.goals),
len(response['goals']))
def test_collection_links(self):
response = self.get_json('/goals/?limit=2')
self.assertEqual(2, len(response['goals']))
def test_collection_links_default_limit(self):
cfg.CONF.set_override('max_limit', 3, 'api')
response = self.get_json('/goals')
self.assertEqual(3, len(response['goals']))

65
watcher/tests/applier/demo/test_applier.py
View File

@ -1,65 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
from oslo_config import cfg
from watcher.applier.framework.default_applier import DefaultApplier
from watcher.common import utils
from watcher.decision_engine.framework.default_planner import DefaultPlanner
from watcher.decision_engine.strategies.basic_consolidation import \
BasicConsolidation
from watcher.openstack.common import log
from watcher.tests.db import base
from watcher.tests.db import utils as db_utils
from watcher.tests.decision_engine.faker_cluster_state import \
FakerStateCollector
from watcher.tests.decision_engine.faker_metrics_collector import \
FakerMetricsCollector
from oslo_config import cfg
CONF = cfg.CONF
""
class TestApplier(base.DbTestCase):
default_planner = DefaultPlanner()
def create_solution(self):
metrics = FakerMetricsCollector()
current_state_cluster = FakerStateCollector()
sercon = BasicConsolidation()
sercon.set_metrics_resource_collector(metrics)
return sercon.execute(current_state_cluster.generate_scenario_1())
def test_scheduler_w(self):
CONF.debug = True
log.setup('watcher-sercon-demo')
CONF.keystone_authtoken.auth_uri = "http://10.50.0.105:5000/v3"
CONF.keystone_authtoken.admin_user = "admin"
CONF.keystone_authtoken.admin_password = "openstacktest"
CONF.keystone_authtoken.admin_tenant_name = "test"
audit = db_utils.create_test_audit(uuid=utils.generate_uuid())
action_plan = self.default_planner.schedule(self.context,
audit.id,
self.create_solution())
applier = DefaultApplier()
applier.execute(self.context, action_plan.uuid)
"""""

99
watcher/tests/applier/demo/test_migrate.py
View File

@ -1,99 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
from keystoneclient import session
from keystoneclient.auth.identity import v3
import cinderclient.v2.client as ciclient
import glanceclient.v2.client as glclient
import keystoneclient.v3.client as ksclient
import neutronclient.neutron.client as netclient
import novaclient.v2.client as nvclient
from watcher.common.utils import CONF
from oslo_config import cfg
from watcher.applier.framework.command.migrate_command import MigrateCommand
from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
from watcher.decision_engine.framework.default_planner import Primitives
from watcher.openstack.common import log
cfg.CONF.import_opt('auth_uri', 'keystoneclient.middleware.auth_token',
group='keystone_authtoken')
cfg.CONF.import_opt('admin_user', 'keystoneclient.middleware.auth_token',
group='keystone_authtoken')
cfg.CONF.import_opt('admin_password', 'keystoneclient.middleware.auth_token',
group='keystone_authtoken')
cfg.CONF.import_opt('admin_tenant_name',
'keystoneclient.middleware.auth_token',
group='keystone_authtoken')
cfg.CONF.keystone_authtoken.auth_uri = "http://10.50.0.105:5000/v3/"
cfg.CONF.keystone_authtoken.admin_user = "admin"
cfg.CONF.keystone_authtoken.admin_password = "openstacktest"
cfg.CONF.keystone_authtoken.admin_tenant_name = "test"
try:
cfg.CONF.debug = True
log.setup('watcher-sercon-demo')
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
auth = v3.Password(auth_url=creds['auth_url'],
username=creds['username'],
password=creds['password'],
project_name=creds['project_name'],
user_domain_name=creds[
'user_domain_name'],
project_domain_name=creds[
'project_domain_name'])
sess = session.Session(auth=auth)
nova = nvclient.Client("3", session=sess)
neutron = netclient.Client('2.0', session=sess)
neutron.format = 'json'
keystone = ksclient.Client(**creds)
glance_endpoint = keystone. \
service_catalog.url_for(service_type='image',
endpoint_type='publicURL')
glance = glclient.Client(glance_endpoint,
token=keystone.auth_token)
cinder = ciclient.Client('2', session=sess)
wrapper = NovaWrapper(user=creds['username'], nova=nova,
neutron=neutron, glance=glance,
cinder=cinder)
instance = wrapper. \
create_instance(hypervisor_id='ldev-indeedsrv006',
inst_name="demo_instance_1",
keypair_name='admin',
image_id=
"2b958331-379b-4618-b2ba-fbe8a608b2bb")
cmd = MigrateCommand(instance.id, Primitives.COLD_MIGRATE,
'ldev-indeedsrv006',
'ldev-indeedsrv005')
resu = cmd.execute(cmd)
resu.result()
# wrapper.delete_instance(instance.id)
except Exception as e:
print("rollback " + unicode(e))
"""""

3
watcher/tests/applier/framework/command/test_launch_action_plan_command.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from mock import call
from mock import MagicMock

2
watcher/tests/applier/framework/command/wrapper/test_nova_wrapper.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/tests/applier/framework/messaging/test_launch_action_plan_endpoint.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from mock import MagicMock

2
watcher/tests/applier/framework/test_applier_manager.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/tests/applier/framework/test_command_executor.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/tests/applier/framework/test_command_mapper.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/tests/applier/framework/test_manager.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.framework.manager_decision_engine import \

15
watcher/tests/applier/framework/test_rpcapi.py
View File

@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import mock
import oslo.messaging as om
@ -42,12 +45,6 @@ class TestApplierAPI(base.TestCase):
'check_api_version',
api_version=ApplierAPI().API_VERSION)
def test_execute_action_plan_throw_exception(self):
action_plan_uuid = "uuid"
self.assertRaises(exception.InvalidUuidOrName,
self.api.launch_action_plan,
action_plan_uuid)
def test_execute_audit_without_error(self):
with mock.patch.object(om.RPCClient, 'call') as mock_call:
action_plan_uuid = utils.generate_uuid()
@ -56,3 +53,9 @@ class TestApplierAPI(base.TestCase):
self.context.to_dict(),
'launch_action_plan',
action_plan_uuid=action_plan_uuid)
def test_execute_action_plan_throw_exception(self):
action_plan_uuid = "uuid"
self.assertRaises(exception.InvalidUuidOrName,
self.api.launch_action_plan,
action_plan_uuid)

0
watcher/tests/collector/__init__.py Normal file
View File

85
watcher/tests/collector/test_influxdb.py Normal file
View File

@ -0,0 +1,85 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import mock
from watcher.metrics_engine.api.metrics_resource_collector import \
AggregationFunction
from watcher.metrics_engine.framework.datasources.influxdb_collector import \
InfluxDBCollector
from watcher.tests import base
class TestInfluxDB(base.TestCase):
def get_databases(self):
return {'name': 'indeed'}
def test_get_measurement(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
influx.get_client.get_list_database = self.get_databases
result = influx.get_measurement("")
self.assertEqual(result, [])
def test_build_query(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
query = influx.build_query("cpu_compute")
self.assertEqual(str(query), "SELECT * FROM \"cpu_compute\" ;")
def test_build_query_aggregate(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
query = influx.build_query("cpu_compute",
aggregation_function=AggregationFunction.
COUNT)
self.assertEqual(str(query),
"SELECT count(value) FROM \"cpu_compute\" ;")
def test_build_query_aggregate_intervals(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
query = influx.build_query("cpu_compute",
aggregation_function=AggregationFunction.
COUNT,
intervals="5m")
self.assertEqual(str(query),
"SELECT count(value) FROM \"cpu_compute\" "
"group by time(5m);")
def test_build_query_aggregate_filters(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
filters = ['host=server1']
query = influx.build_query("cpu_compute",
aggregation_function=AggregationFunction.
COUNT,
intervals="5m",
filters=filters)
self.assertEqual(str(query), 'SELECT count(value) FROM'
' \"cpu_compute" WHERE'
' host = \'server1\' group by time(5m);')
def test_get_qusurement_start(self):
influx = InfluxDBCollector()
influx.get_client = mock.MagicMock()
influx.get_client.get_list_database = self.get_databases
result = influx.get_measurement("cpu_compute", start_time='now',
end_time="now")
self.assertEqual(result, [])

43
watcher/tests/collector/test_nova_collector.py Normal file
View File

@ -0,0 +1,43 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import mock
from watcher.metrics_engine.framework.statedb_collector import NovaCollector
from watcher.tests import base
class TestNovaCollector(base.TestCase):
@mock.patch('keystoneclient.v3.client.Client')
def setUp(self, mock_ksclient):
super(TestNovaCollector, self).setUp()
self.wrapper = mock.MagicMock()
self.nova_collector = NovaCollector(self.wrapper)
def test_nova_collector(self):
hypervisor = mock.Mock()
hypervisor.hypervisor_hostname = "rdev-lannion.eu"
hypervisor.service = mock.MagicMock()
service = mock.Mock()
service.host = ""
self.wrapper.get_hypervisors_list.return_value = {hypervisor}
self.wrapper.nova.services.find.get.return_value = service
model = self.nova_collector.get_latest_state_cluster()
self.assertIsNotNone(model)

52
watcher/tests/collector/test_query.py Normal file
View File

@ -0,0 +1,52 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.metrics_engine.framework.datasources.sql_ast.build_db_query import \
DBQuery
from watcher.tests import base
class TestDBQuery(base.TestCase):
def test_query(self):
expected = "SELECT * FROM \"cpu_compute.cpu.user.percent_gauge\" ;"
query = DBQuery("cpu_compute.cpu.user.percent_gauge")
self.assertEqual(str(query), expected)
def test_query_where(self):
expected = "SELECT * FROM" \
" \"cpu_compute.cpu.user.percent_gauge\" WHERE host=jed;"
query = DBQuery("cpu_compute.cpu.user.percent_gauge").where(
"host=jed")
self.assertEqual(str(query), expected)
def test_query_filter(self):
expected = "SELECT mean(value) FROM" \
" \"cpu_compute.cpu.user.percent_gauge\" WHERE host=jed;"
query = DBQuery("cpu_compute.cpu.user.percent_gauge").where(
"host=jed").select("mean(value)")
self.assertEqual(str(query), expected)
def test_query_groupby(self):
expected = "SELECT * FROM" \
" \"cpu_compute.cpu.user.percent_gauge\" " \
"group by time(5m);"
query = DBQuery("cpu_compute.cpu.user.percent_gauge").groupby(
"time(5m)")
self.assertEqual(str(query), expected)

1
watcher/tests/common/messaging/utils/test_transport_url_builder.py
View File

@ -31,7 +31,6 @@ class TestTransportUrlBuilder(base.TestCase):
def test_transport_url_not_none(self):
url = TransportUrlBuilder().url
print(url)
self.assertIsNotNone(url, "The transport url must not be none")
def test_transport_url_valid_pattern(self):

0
watcher/tests/decision_engine/api/__init__.py Normal file
View File

0
watcher/tests/decision_engine/api/messaging/__init__.py Normal file
View File

Some files were not shown because too many files have changed in this diff Show More