Fix serialization of choices and reenable congress-agent.

oslo_config uses OrderedDict for choices in
options types. This serialize it as list of tuples.
Order is  preserved and types are naturally
deserialized back from those tuples.

Congress-agent is reenabled. It can be disabled
in devstack on  the value of ENABLE_CONGRESS_AGENT
(default is True).

Driver will initialize table to empty at first poll not
waiting for an initial response from agents.

Closes-Bug: #1742833
Change-Id: If1423e824fc457bab89f4bcd12be8826c50f5f69

congress/cfg_validator/agent/generator.py

@@ -15,6 +15,7 @@
 #
 
 """ Generation of JSON from oslo config options (marshalling) """
+import collections
 import json
 import logging
 
@@ -59,7 +60,14 @@ class OptionJsonEncoder(json.JSONEncoder):
             if isinstance(o, types.Number):
                 res['max'] = o.max
                 res['min'] = o.min
-                res['choices'] = o.choices
+                # When we build back the type in parsing, we can directly use
+                # the list of tuples from choices and it will be in a
+                # canonical order (not sorted but the order elements were
+                # added)
+                if isinstance(o.choices, collections.OrderedDict):
+                    res['choices'] = list(o.choices.keys())
+                else:
+                    res['choices'] = o.choices
             if isinstance(o, types.Range):
                 res['max'] = o.max
                 res['min'] = o.min
@@ -71,7 +79,10 @@ class OptionJsonEncoder(json.JSONEncoder):
                 res['max_length'] = o.max_length
                 res['quotes'] = o.quotes
                 res['ignore_case'] = o.ignore_case
-                res['choices'] = o.choices
+                if isinstance(o.choices, collections.OrderedDict):
+                    res['choices'] = list(o.choices.keys())
+                else:
+                    res['choices'] = o.choices
             if isinstance(o, types.List):
                 res['item_type'] = o.item_type
                 res['bounds'] = o.bounds

congress/datasources/cfgvalidator_driver.py

@@ -192,6 +192,13 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
 
     def poll(self):
         LOG.info("%s:: polling", self.name)
+        # Initialize published state to a sensible empty state.
+        # Avoids races with queries.
+        if self.number_of_updates == 0:
+            for tablename in set(self.get_schema()):
+                self.state[tablename] = set()
+                self.publish(tablename, self.state[tablename],
+                             use_snapshot=False)
         self.agent_api.publish_templates_hashes(self.get_context())
         self.agent_api.publish_configs_hashes(self.get_context())
         self.last_updated_time = datetime.datetime.now()
@@ -231,9 +238,10 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
         :param host: Name of the node hosting theses config files
         """
 
-        LOG.debug('Process template hashes from %s' % host)
+        LOG.info('Process template hashes from %s' % host)
 
         for t_h in set(hashes) - set(self.known_templates):
+            LOG.info('Treating template hash %s' % t_h)
             template = self.agent_api.get_template(self.get_context(), t_h,
                                                    host)
 
@@ -339,7 +347,8 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
             # choices (OrderedDict). We first convert back to simple list to
             # have consistent output regardless of oslo.config version
             if isinstance(cfg_type.choices, OrderedDict):
-                choices = list(cfg_type.choices.keys())
+                choices = list(map(lambda item: item[0],
+                                   cfg_type.choices.items()))
             else:
                 choices = cfg_type.choices
             row = (cfg_type.regex, cfg_type.max_length, cfg_type.quotes,
@@ -351,7 +360,8 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver):
             # choices (OrderedDict). We first convert back to simple list to
             # have consistent output regardless of oslo.config version
             if isinstance(cfg_type.choices, OrderedDict):
-                choices = list(cfg_type.choices.keys())
+                choices = list(map(lambda item: item[0],
+                                   cfg_type.choices.items()))
             else:
                 choices = cfg_type.choices
             row = (cfg_type.min, cfg_type.max, choices)

congress_tempest_tests/tests/scenario/congress_datasources/test_cfg_validator.py

@@ -84,7 +84,7 @@ class TestCfgValidatorDriver(manager_congress.ScenarioPolicyBase):
             if res1 is None or res2 is None:
                 return False
             row1 = next((r for r in res1
-                         if r['data'][col1_name] == u'datasource_file'),
+                         if r['data'][col1_name] == u'datasource_sync_period'),
                         None)
             row2 = next((r for r in res2
                          if r['data'][col2_name] == u'congress'),

devstack/plugin.sh

@@ -100,8 +100,9 @@ function configure_congress_datasources {
     _configure_service heat heat
     _configure_service aodh aodh
     _configure_service mistral mistral
-# FIXME(ekcs): congress-agent temporarily disabled while gate issue being resolved
+    if [[ $ENABLE_CONGRESS_AGENT == "True" ]] ; then
-    # _configure_service congress-agent config
+        _configure_service congress-agent config
+    fi
 }
 
 function _configure_service {

devstack/settings

@@ -20,6 +20,8 @@ fi
 CONGRESS_CONF_DIR=/etc/congress
 CONGRESS_CONF=$CONGRESS_CONF_DIR/congress.conf
 
+# Flag for disabling congress-agent
+ENABLE_CONGRESS_AGENT=$(trueorfalse True ENABLE_CONGRESS_AGENT)
 # Validator Agent conf
 CONGRESS_AGT_CONF=$CONGRESS_CONF_DIR/congress-agent.conf
 
@@ -57,6 +59,8 @@ CONGRESSCLIENT_BRANCH=${CONGRESSCLIENT_BRANCH:-master}
 CONGRESSDASHBOARD_REPO=${CONGRESSDASHBOARD_REPO:-${GIT_BASE}/openstack/congress-dashboard.git}
 CONGRESSDASHBOARD_BRANCH=${CONGRESSDASHBOARD_BRANCH:-master}
 
-# FIXME(ekcs): congress-agent temporarily disabled while gate issue being resolved
 enable_service congress congress-api congress-engine congress-datasources
-# enable_service congress congress-api congress-engine congress-datasources congress-agent
+if [[ $ENABLE_CONGRESS_AGENT == "True" ]] ; then
+  enable_service congress-agent
+fi
+

doc/source/install/index.rst

@@ -54,7 +54,11 @@ of ``local.conf``:
     disable_service congress congress-api congress-engine congress-datasources
     enable_service congress-agent
 
-By default, the datasource is enabled for the nova, neutron and Congress services. To enable it for other services, you can define the variable ``$VALIDATOR_SERVICES``.
+By default, the datasource is enabled for the nova, neutron and Congress services. To enable it for other services, you can define the variable ``VALIDATOR_SERVICES``.
+
+The ``ENABLE_CONGRESS_AGENT`` variable in ``local.conf`` controls the
+availability of the config datasource and its agent in devstack. Set it to
+``False`` to disable it (default value is ``True``).
 
 Separate install
 --------------------