From 2e97a2869ff818c0bb3d960e89a0d24dbf6244db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20Cr=C3=A9gut?= Date: Mon, 9 Apr 2018 10:06:45 +0200 Subject: [PATCH] Correct handling of known configs in cfgvalidator datasource cfgvalidator datasource did not update the set of known configs. Each time config hashes were retrieved, the config was analyzed although it was already registered. This bugfix simplifies the data structures used and corrects the logic of the update process. Change-Id: Ibc6098c815d2a21565f845f2d5af63bb1333f1a2 Closes-Bug: 1762330 --- congress/datasources/cfgvalidator_driver.py | 95 +++++++++++---------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/congress/datasources/cfgvalidator_driver.py b/congress/datasources/cfgvalidator_driver.py index 9d028df5c..a20eb8540 100644 --- a/congress/datasources/cfgvalidator_driver.py +++ b/congress/datasources/cfgvalidator_driver.py @@ -72,8 +72,8 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver): self.known_templates = {} # { namespace_hash -> namespace_name } self.known_namespaces = {} - # { config_hash -> valued_ns } - self.known_configs = {} + # set(config_hash) + self.known_configs = set() # { template_hash -> (conf_hash, conf)[] } self.templates_awaited_by_config = {} @@ -214,19 +214,11 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver): """ LOG.debug('Received configs list from %s' % host) - for cfg_hash in set(hashes) - set(self.known_configs): + for cfg_hash in set(hashes) - self.known_configs: config = self.agent_api.get_config(self.get_context(), cfg_hash, host) - try: - processed_config = self.process_config(cfg_hash, config, host) - except KeyError: - processed_config = None - LOG.error('Config %s from %s NOT registered' - % (cfg_hash, host)) - continue - - if processed_config: - self.known_configs[cfg_hash] = processed_config + if self.process_config(cfg_hash, config, host): + self.known_configs.add(cfg_hash) LOG.debug('Config %s from %s registered' % (cfg_hash, host)) @lockutils.synchronized('validator_process_template_hashes') @@ -238,10 +230,10 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver): :param host: Name of the node hosting theses config files """ - LOG.info('Process template hashes from %s' % host) + LOG.debug('Process template hashes from %s' % host) for t_h in set(hashes) - set(self.known_templates): - LOG.info('Treating template hash %s' % t_h) + LOG.debug('Treating template hash %s' % t_h) template = self.agent_api.get_template(self.get_context(), t_h, host) @@ -254,9 +246,10 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver): self.known_templates[t_h] = template for (c_h, config) in self.templates_awaited_by_config.pop(t_h, []): - valued_ns = self.process_config(c_h, config, host) - if valued_ns: - self.known_configs[c_h] = valued_ns + if self.process_config(c_h, config, host): + self.known_configs.add(c_h) + LOG.debug('Config %s from %s registered (late)' % + (c_h, host)) return True def translate_service(self, host_id, service, version): @@ -518,44 +511,56 @@ class ValidatorDriver(datasource_driver.PollingDataSourceDriver): :param file_hash: Hash of the configuration file :param config: object representing the configuration :param host: Remote host name + :return: True if config was processed """ - template_hash = config['template'] - template = self.known_templates.get(template_hash, None) - if template is None: - waiting = self.templates_awaited_by_config.get(template_hash, []) - waiting.append((file_hash, config)) - self.templates_awaited_by_config[template_hash] = waiting - LOG.info('Template %s not yet registered' % template_hash) - return + try: + LOG.debug("process_config hash=%s" % file_hash) + template_hash = config['template'] + template = self.known_templates.get(template_hash, None) + if template is None: + waiting = ( + self.templates_awaited_by_config.get(template_hash, [])) + waiting.append((file_hash, config)) + self.templates_awaited_by_config[template_hash] = waiting + LOG.debug('Template %s not yet registered' % template_hash) + return False - host_id = utils.compute_hash(host) + host_id = utils.compute_hash(host) - namespaces = [self.known_namespaces.get(h, None).get('data', None) - for h in template['namespaces']] + namespaces = [self.known_namespaces.get(h, None).get('data', None) + for h in template['namespaces']] - conf = parsing.construct_conf_manager(namespaces) - parsing.add_parsed_conf(conf, config['data']) + conf = parsing.construct_conf_manager(namespaces) + parsing.add_parsed_conf(conf, config['data']) - for tablename in set(self.get_schema()) - set(self.state): - self.state[tablename] = set() - self.publish(tablename, self.state[tablename], use_snapshot=False) + for tablename in set(self.get_schema()) - set(self.state): + self.state[tablename] = set() + self.publish(tablename, self.state[tablename], + use_snapshot=False) - self.translate_conf(conf, file_hash) + self.translate_conf(conf, file_hash) - self.translate_host(host_id, host) + self.translate_host(host_id, host) - self.translate_service(host_id, config['service'], config['version']) + self.translate_service( + host_id, config['service'], config['version']) - file_name = os.path.basename(config['path']) - self.translate_file(file_hash, host_id, template_hash, file_name) + file_name = os.path.basename(config['path']) + self.translate_file(file_hash, host_id, template_hash, file_name) - ns_hashes = {h: self.known_namespaces[h]['name'] - for h in template['namespaces']} - self.translate_template_namespace(template_hash, template['name'], - ns_hashes) + ns_hashes = {h: self.known_namespaces[h]['name'] + for h in template['namespaces']} + self.translate_template_namespace(template_hash, template['name'], + ns_hashes) - for tablename in self.state: - self.publish(tablename, self.state[tablename], use_snapshot=True) + for tablename in self.state: + self.publish(tablename, self.state[tablename], + use_snapshot=True) + return True + except KeyError: + LOG.error('Config %s from %s NOT registered' + % (file_hash, host)) + return False class ValidatorAgentClient(object):