openstack
/
congress
Code Issues Proposed changes

Unblock gate 

Fixes to eliminate autodoc errors coming with sphinx upgrade.

Increase nova service timeout

Change-Id: Id7c8dcb82b3529afdc22957239870e2a27cc6e1b
This commit is contained in:
Eric K 2018-06-19 11:29:59 -07:00
parent 8dbe637913
commit 2bfcbc329a
15 changed files with 184 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
congress/api/library_policy_model.py
View File

@ -72,7 +72,7 @@ class LibraryPolicyModel(base.APIModel):
def get_item(self, id_, params, context=None):
"""Retrieve item with id from model.
:param: id_: The id of the item to retrieve
:param: id\_: The id of the item to retrieve
:param: params: A dict-like object containing parameters
from the request query string and body.
:param: context: Key-values providing frame of reference of request

43
congress/cfg_validator/agent/agent.py
View File

@ -47,13 +47,13 @@ class Config(object):
"""Encapsulates a configuration file and its meta-data.
Attributes:
path: Path to the configuration on the local file system.
template: A Template object to use for parsing the configuration.
data: The normalized Namespace loaded by oslo-config, contains the
parsed values.
hash: Hash of the configuration file, salted with the hostname and the
template hash
service_name: The associated service name
:ivar path: Path to the configuration on the local file system.
:ivar template: A Template object to use for parsing the configuration.
:ivar data: The normalized Namespace loaded by oslo-config, contains
the parsed values.
:ivar hash: Hash of the configuration file, salted with the hostname
and the template hash
:ivar service_name: The associated service name
"""
# pylint: disable=protected-access
@ -177,11 +177,12 @@ class Template(object):
"""Describes a template, as defined by oslo-config-generator.
Attributes:
name: The name, as used by oslo-config-generator.
path: The path to the template configuration file, as defined by oslo-
config-generator, on the local file system.
output_file: The default output path for this template.
namespaces: A set of Namespace objects, which make up this template.
:ivar name: The name, as used by oslo-config-generator.
:ivar path: The path to the template configuration file, as defined by
oslo-config-generator, on the local file system.
:ivar output_file: The default output path for this template.
:ivar namespaces: A set of Namespace objects, which make up this
template.
"""
# pylint: disable=protected-access
@ -229,10 +230,10 @@ class Template(object):
"""Info on the template
:return: a quadruple made of:
* path: the path to the template path
* name: the name of the template
* output_fle:
* namespaces: an array of namespace hashes.
* path: the path to the template path
* name: the name of the template
* output_fle:
* namespaces: an array of namespace hashes.
"""
return {'path': self.path, 'name': self.name,
'output_file': self.output_file,
@ -243,12 +244,12 @@ class ConfigManager(object):
"""Manages the services configuration files on a node and their meta-data.
Attributes:
host: A hostname.
configs: A dict mapping config hashes to their associated Config
:ivar host: A hostname.
:ivar configs: A dict mapping config hashes to their associated Config
object.
templates: A dict mapping template hashes to their associated Template
object.
namespaces: A dict mapping namespace hashes to their associated
:ivar templates: A dict mapping template hashes to their associated
Template object.
:ivar namespaces: A dict mapping namespace hashes to their associated
Namespace object.
"""

86
congress/datalog/arithmetic_solvers.py
View File

@ -191,12 +191,12 @@ class LpLang(object):
def pure_lp(self, exp, bounds):
"""Rewrite EXP to a pure LP problem.
:param: exp: is an Expression of the form
var = (arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where the degenerate cases are permitted as well.
:param exp: is an Expression of the form
var = (arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where the degenerate cases are permitted as well.
Returns a collection of expressions each of the form:
a1*x1 + ... + an*xn [<=, ==, >=] b.
:returns: a collection of expressions each of the form:
a1*x1 + ... + an*xn [<=, ==, >=] b.
"""
flat, support = self.flatten(exp, indicator=False)
flats = support
@ -215,13 +215,13 @@ class LpLang(object):
def pure_lp_term(self, exp, bounds):
"""Rewrite term exp to a pure LP term.
:param: exp: is an Expression of the form
(arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where the degenerate cases are permitted as well.
:param exp: is an Expression of the form
(arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where the degenerate cases are permitted as well.
Returns (new-exp, support) where new-exp is a term, and support is
a expressions of the following form.
a1*x1 + ... + an*xn [<=, ==, >=] b.
:returns: (new-exp, support) where new-exp is a term, and support is
a expressions of the following form.
a1*x1 + ... + an*xn [<=, ==, >=] b.
"""
flat, support = self.flatten(exp, indicator=False)
flat_no_andor = self.remove_and_or_term(flat)
@ -233,13 +233,13 @@ class LpLang(object):
def remove_and_or(self, exp):
"""Translate and/or operators into times/plus arithmetic.
:param: exp: is an Expression that takes one of the following forms.
var [!]= term1 ^ ... ^ termn
var [!]= term1 | ... | termn
var [!]= term1
where termi is an indicator variable.
:param exp: is an Expression that takes one of the following forms.
var [!]= term1 ^ ... ^ termn
var [!]= term1 | ... | termn
var [!]= term1
where termi is an indicator variable.
Returns an expression equivalent to exp but without any ands/ors.
:returns: an expression equivalent to exp but without any ands/ors.
"""
if self.isConstant(exp) or self.isVariable(exp):
return exp
@ -260,14 +260,16 @@ class LpLang(object):
def indicator_to_pure_lp(self, exp, bounds):
"""Translate exp into LP constraints without indicator variable.
:param: exp: is an Expression of the form var = arith
:param: bounds: is a dictionary from variable to its upper bound
:param exp: is an Expression of the form var = arith
:param bounds: is a dictionary from variable to its upper bound
:returns: [EXP] if it is of the wrong form. Otherwise, translates
into the form y = x < 0, and then returns two constraints where
upper(x) is the upper bound of the expression x::
-x <= y * upper(x)
x < (1 - y) * upper(x)
Returns [EXP] if it is of the wrong form. Otherwise, translates
into the form y = x < 0, and then returns two constraints where
upper(x) is the upper bound of the expression x::
-x <= y * upper(x)
x < (1 - y) * upper(x)
Taken from section 7.4 of
http://www.aimms.com/aimms/download/manuals/
aimms3om_integerprogrammingtricks.pdf
@ -314,8 +316,8 @@ class LpLang(object):
def arith_to_lt_zero(self, expr):
"""Returns Arith expression equivalent to expr but of the form A < 0.
:param: expr is an Expression
Returns an expression equivalent to expr but of the form A < 0.
:param expr: is an Expression
:returns: an expression equivalent to expr but of the form A < 0.
"""
if not self.isArith(expr):
raise self.LpConversionFailure(
@ -353,9 +355,9 @@ class LpLang(object):
def upper_bound(self, expr, bounds):
"""Returns number giving an upper bound on the given expr.
:param: expr is an Expression
:param: bounds: is a dictionary from tuple versions of variables
to the size of their upper bound.
:param expr: is an Expression
:param bounds: is a dictionary from tuple versions of variables
to the size of their upper bound.
"""
if self.isConstant(expr):
return expr
@ -385,22 +387,22 @@ class LpLang(object):
def flatten(self, exp, indicator=True):
"""Remove toplevel embedded and/ors by creating new equalities.
:param: exp: is an Expression of the form
var = (arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where arithij is either a variable or an arithmetic expression
where the degenerate cases are permitted as well.
:param exp: is an Expression of the form
var = (arith11 ^ ... ^ arith1n) | ... | (arithk1 ^ ... ^ arithkn)
where arithij is either a variable or an arithmetic expression
where the degenerate cases are permitted as well.
:param: indicator controls whether the method Returns
a single variable (with supporting expressions) or it Returns
an expression that has operator with (flat) arguments
:param indicator: controls whether the method Returns
a single variable (with supporting expressions) or it Returns
an expression that has operator with (flat) arguments
Returns a collection of expressions each of one of the following
forms:
var1 = var2 * ... * varn
var1 = var2 + ... + varn
var1 = arith
:returns: a collection of expressions each of one of the following
forms:
var1 = var2 * ... * varn
var1 = var2 + ... + varn
var1 = arith
Returns (new-expression, supporting-expressions)
:returns: (new-expression, supporting-expressions)
"""
if self.isConstant(exp) or self.isVariable(exp):
return exp, []

4
congress/datalog/nonrecursive.py
View File

@ -261,8 +261,8 @@ class NonrecursiveRuleTheory(topdown.TopDownTheory):
def arity(self, table, modal=None):
"""Return the number of arguments TABLENAME takes.
:param: table can be either a string or a Tablename
Returns None if arity is unknown (if it does not occur in
:param table: can be either a string or a Tablename
:returns: None if arity is unknown (if it does not occur in
the head of a rule).
"""
if isinstance(table, compile.Tablename):

7
congress/datalog/topdown.py
View File

@ -69,11 +69,14 @@ class TopDownTheory(base.Theory):
"""Struct for info about the original caller of top-down evaluation.
VARIABLES is the list of variables (from the initial query)
that we want bindings for.
that we want bindings for.
BINDING is the initially empty BiUnifier.
FIND_ALL controls whether just the first or all answers are found.
ANSWERS is populated by top-down evaluation: it is the list of
VARIABLES instances that the search process proved true.
VARIABLES instances that the search process proved true.
"""
def __init__(self, variables, binding, theory,

4
congress/datasources/cfgvalidator_driver.py
View File

@ -208,7 +208,8 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
"""Handles a list of config files hashes and their retrieval.
If the driver can process the parsing and translation of the config,
it registers the configs to the driver.
it registers the configs to the driver.
:param hashes: A list of config files hashes
:param host: Name of the node hosting theses config files
"""
@ -298,6 +299,7 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
"""Translates a template infos and its namespaces infos.
Modifies tables : TEMPLATE, NAMESPACE and TEMPLATE_NS
:param template_id: Template ID
:param name: A template name
:param ns_ids: List of namespace IDs, defining this template, should

1
congress/datasources/cinder_driver.py
View File

@ -18,6 +18,7 @@
version: 2.1
date: 2016-03-27
changes:
- Added columns to the volumes table: encrypted, availability_zone,
replication_status, multiattach, snapshot_id, source_volid,
consistencygroup_id, migration_status

86
congress/datasources/datasource_driver.py
View File

@ -68,16 +68,18 @@ class DataSourceDriver(data_service.DataService):
parameters describe things like the table, column names, and
sub-translators.
HDICT parameters with example values:
{'translation-type': 'HDICT',
'table-name': 'example_table',
'parent-key': 'parent_key_column',
'id-col': 'id_col',
'selector-type': 'DOT_SELECTOR',
'field-translators': ({'fieldname': 'field1', 'col': 'col1',
'translator': {'translation-type': 'VALUE'}},
{'fieldname': 'field2', 'col': 'col2',
'translator': {'translation-type': 'VALUE'})}
::
HDICT parameters with example values:
{'translation-type': 'HDICT',
'table-name': 'example_table',
'parent-key': 'parent_key_column',
'id-col': 'id_col',
'selector-type': 'DOT_SELECTOR',
'field-translators': ({'fieldname': 'field1', 'col': 'col1',
'translator': {'translation-type': 'VALUE'}},
{'fieldname': 'field2', 'col': 'col2',
'translator': {'translation-type': 'VALUE'})}
The HDICT translator reads in a python dict and translates each key in
the dict into a column of the output table. The fields in the table
@ -125,18 +127,21 @@ class DataSourceDriver(data_service.DataService):
second table's entries derived from the primary table's row. For
example, if the translator is:
{'translation-type': 'HDICT',
'table-name': 'example_table',
'selector-type': 'DOT_SELECTOR',
'field-translators': ({'fieldname': 'field1', 'col': 'col1',
'translator': {
'translation-type': 'LIST',
'table-name': 'subtable',
'val-col': 'c',
'translator': {'translation-type': 'VALUE'}},})}
::
{'translation-type': 'HDICT',
'table-name': 'example_table',
'selector-type': 'DOT_SELECTOR',
'field-translators': ({'fieldname': 'field1', 'col': 'col1',
'translator': {
'translation-type': 'LIST',
'table-name': 'subtable',
'val-col': 'c',
'translator': {'translation-type': 'VALUE'}},})}
The object {'field1': [1, 2, 3]} will translate to one tuple in
example_table and three tuples in subtable:
example_table and three tuples in subtable::
example_table: (h(1, 2, 3))
subtable: (h(1, 2, 3), 1)
(h(1, 2, 3), 2)
@ -145,7 +150,7 @@ class DataSourceDriver(data_service.DataService):
In addition, sometimes one will have data that is structured in the
following manor (i.e a dict contained in a list within a dict):
data:
data::
{'id': '11111',
'things': [{'type': 1, 'location': 2}]}
@ -155,7 +160,7 @@ class DataSourceDriver(data_service.DataService):
explicitly, and the schema would have 3 tables. This allows you to
use two hdicts to represent the data.
For Example:
For Example::
thing_translator = {
'translation-type': 'HDICT',
@ -180,7 +185,8 @@ class DataSourceDriver(data_service.DataService):
'translator': thing_translator})}
VDICT parameters with example values:
VDICT parameters with example values::
{'translation-type': 'VDICT',
'table-name': 'table',
'parent-key': 'parent_key_column',
@ -203,7 +209,8 @@ class DataSourceDriver(data_service.DataService):
VDICT's 'val-col' column due to an implementation choice (the id column
is not available until after the subtranslator runs).
LIST parameters with example values:
LIST parameters with example values::
{'translation-type': 'LIST',
'table-name': 'table1',
'parent-key': 'parent_key_column',
@ -221,7 +228,8 @@ class DataSourceDriver(data_service.DataService):
the subtranslator of a LIST may not specify a 'parent-key' because the
LIST's table will then have no columns.
VALUE parameters with example values:
VALUE parameters with example values::
{'translation-type': 'VALUE',
'extract-fn': lambda x: x.['foo']}
@ -1406,8 +1414,8 @@ class PollingDataSourceDriver(DataSourceDriver):
Triggers polling every *poll_time* seconds or after *request_refresh*
is called.
:param: poll_time: is the amount of time (in seconds) to wait between
polling rounds.
:param poll_time: is the amount of time (in seconds) to wait between
polling rounds.
"""
LOG.debug("start to poll from datasource %s", self.name)
while self._running:
@ -1490,11 +1498,11 @@ class ExecutionDriver(object):
def add_executable_method(self, method_name, method_args, method_desc=""):
"""Add executable method information.
param method_name: The name of the method to add
param method_args: List of arguments and description of the method,
:param method_name: The name of the method to add
:param method_args: List of arguments and description of the method,
e.g. [{'name': 'arg1', 'description': 'arg1'},
{'name': 'arg2', 'description': 'arg2'}]
param method_desc: Description of the method
:param method_desc: Description of the method
"""
if method_name not in self.executable_methods:
@ -1531,12 +1539,13 @@ class ExecutionDriver(object):
Action should be a service API or a user-defined function.
This method should return a dict for all supported actions,
together with optional descriptions for each action and its
required/supported arguments. E.g.
{'results': [{'name': 'execute1',
'args': [{"name": 'arg1', "description": "None"},
{"name": 'arg2', "description": "None"}],
'description': 'execute function 1'}]
}
required/supported arguments. E.g.::
{'results': [{'name': 'execute1',
'args': [{"name": 'arg1', "description": "None"},
{"name": 'arg2', "description": "None"}],
'description': 'execute function 1'}]
}
"""
actions = []
# order by name so that use can find out actions easily
@ -1569,8 +1578,9 @@ class ExecutionDriver(object):
"""This method must be implemented by each driver.
Action can be a service API or a user-defined function
:param: action: a user-defined function or a service API call
:param: action_args: in format of
:param action: a user-defined function or a service API call
:param action_args: in format of::
{'positional': ['arg1', 'arg2'],
'named': {'key1': 'value1', 'key2': 'value2'}}
"""

29
congress/datasources/doctor_driver.py
View File

@ -39,20 +39,21 @@ class DoctorDriver(datasource_driver.PushedDataSourceDriver):
which means if you update the table with [] it will clear the table.
One {} object in the list represents one row of the table.
request body:
[
{
"time": "2016-02-22T11:48:55Z",
"type": "compute.host.down",
"details": {
"hostname": "compute1",
"status": "down",
"monitor": "zabbix1",
"monitor_event_id": "111"
}
},
.....
]
request body::
[
{
"time": "2016-02-22T11:48:55Z",
"type": "compute.host.down",
"details": {
"hostname": "compute1",
"status": "down",
"monitor": "zabbix1",
"monitor_event_id": "111"
}
},
.....
]
"""
value_trans = {'translation-type': 'VALUE'}

14
congress/datasources/nova_driver.py
View File

@ -17,6 +17,7 @@
date: 2018-03-15
changes:
- (incompatible) Removed the `hosts` table for OS hosts information because
access to the information has been removed from the latest Nova API and
client.
@ -24,6 +25,7 @@ changes:
date: 2017-10-01
changes:
- Added the `tags` table for server tags information.
"""
@ -279,12 +281,14 @@ class NovaDriver(datasource_driver.PollingDataSourceDriver,
Therefore, this function translates the positional ARGS
to optional args and call the servers.set_meta() api.
:param: <list> args: expected server ID and pairs of meta
data in positional args such as:
{'positional': ['server_id', 'meta1', 'value1', 'meta2', 'value2']}
data in positional args such as::
Usage:
execute[nova.servers_set_meta(svr_id, meta1, val1, meta2, val2) :-
triggering_table(id)
{'positional': ['server_id', 'meta1', 'value1', 'meta2', 'value2']}
Usage::
execute[nova.servers_set_meta(svr_id, meta1, val1, meta2, val2) :-
triggering_table(id)
"""
action = 'servers.set_meta'
positional_args = args.get('positional', [])

2
congress/harness.py
View File

@ -59,7 +59,7 @@ def create2(node_id=None, bus_id=None, existing_node=None,
:param: policy_engine controls whether policy_engine is included
:param: datasources controls whether datasources are included
:param: api controls whether API is included
:returns DseNode
:returns: DseNode
"""
# create DseNode if existing_node not given
if existing_node is None:

12
congress/policy_engines/agnostic.py
View File

@ -894,6 +894,7 @@ class Runtime (object):
tuple (result, trace).
:param: as_list controls whether the result is forced to be a list of
answers
Returns a list of instances of query. If query/sequence are strings
the query instance list is a single string (unless as_list is True
in which case the query instance list is a list of strings). If
@ -909,7 +910,8 @@ class Runtime (object):
that fact was added or deleted.
Example atom update: q+(1) or q-(1)
Example rule update: p+(x) :- q(x) or p-(x) :- q(x)
Example action invocation:
Example action invocation::
create_network(17), options:value(17, "name", "net1") :- true
"""
assert self.get_target(theory) is not None, "Theory must be known"
@ -1946,10 +1948,10 @@ class DseRuntime (Runtime, data_service.DataService):
def execute_action(self, service_name, action, action_args):
"""Event handler for action execution.
:param: service_name: openstack service to perform the action on,
e.g. 'nova', 'neutron'
:param: action: action to perform on service, e.g. an API call
:param: action_args: positional-args and named-args in format:
:param service_name: openstack service to perform the action on,
e.g. 'nova', 'neutron'
:param action: action to perform on service, e.g. an API call
:param action_args: positional-args and named-args in format:
{'positional': ['p_arg1', 'p_arg2'],
'named': {'name1': 'n_arg1', 'name2': 'n_arg2'}}.
"""

18
congress/dse2/test_control_bus.py → congress/tests/dse2/disabled_test_control_bus.py
View File

@ -26,14 +26,16 @@ from congress.tests import base
# For manual testing, support using rabbit driver instead of fake
USE_RABBIT = False
if len(sys.argv) > 1:
driver_flg = sys.argv[1].lower()
if driver_flg == '--rabbit':
USE_RABBIT = True
elif driver_flg != '--fake':
print("Usage: %s [--fake | --rabbit]" % sys.argv[0])
sys.exit(1)
sys.argv[1:] = sys.argv[2:]
if __name__ == '__main__':
if len(sys.argv) > 1:
driver_flg = sys.argv[1].lower()
if driver_flg == '--rabbit':
USE_RABBIT = True
elif driver_flg != '--fake':
print("Usage: %s [--fake | --rabbit]" % sys.argv[0])
sys.exit(1)
sys.argv[1:] = sys.argv[2:]
class TestControlBus(base.TestCase):

19
doc/source/conf.py
View File

@ -46,9 +46,12 @@ bug_tag = ''
apidoc_module_dir = '../../congress'
apidoc_output_dir = 'api'
apidoc_excluded_paths = [
'datalog.Python2.*',
'datalog.Python3.*',
'db.migration.alembic_migrations.*',]
'datalog/Python2/*',
'datalog/Python3/*',
'db/migration/alembic_migrations/*',
'server/*',
'tests/*',
'/dse2/disabled_test_control_bus.py']
apidoc_separate_modules = True
@ -72,10 +75,18 @@ add_module_names = True
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of glob-style patterns that should be excluded when looking for
# source files. They are matched against the source file names relative to the
# source directory, using slashes as directory separators on all platforms.
exclude_patterns = ['api/congress.db.migration.alembic_migrations.*',
'api/congress.server.*']
# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ['congress.']
autodoc_mock_imports = ['congress.datalog.Python2', 'congress.datalog.Python3']
autodoc_mock_imports = ['congress.datalog.Python2', 'congress.datalog.Python3',
'cloudfoundryclient', 'congress.dse']
# -- Options for HTML output --------------------------------------------------

1
playbooks/legacy/congress-devstack-api-base/run.yaml
View File

@ -54,6 +54,7 @@
ENABLED_SERVICES=s-proxy,s-object,s-container,s-account,
ENABLED_SERVICES+=neutron-qos
export SERVICE_TIMEOUT=120 # default too short for this job
export PYTHONUNBUFFERED=true
export DEVSTACK_GATE_TEMPEST=1
export DEVSTACK_GATE_TEMPEST_REGEX="congress_tempest_plugin"