Extend hiera task to improve plugin support

* Add hiera_config type to mange hiera.yaml
* Plugins can put their overrides to /etc/hiera/plugins/
* Running hiera_config again will gather override files
  and insert records for them to the main config file after
  the basic elements
* Or, it metadata_yaml_file is present, hiera_config can
  take the list of enabled plugins and add records for them
  instead of scanning the plugins directory

Closes-Bug: 1512456
DocImpact: add information about placing plugin data files to /etc/hiera/plugins/ instead of other files.
Change-Id: Ifdb27708a2457e8c1430695bb2412594806d211f

deployment/puppet/osnailyfacter/lib/puppet/provider/hiera_config/ruby.rb

@@ -0,0 +1,268 @@
+require 'yaml'
+
+Puppet::Type.type(:hiera_config).provide(:ruby) do
+  desc 'Manage Hiera configuration file'
+
+  attr_accessor :property_hash
+  attr_accessor :resource
+
+  # Getters and Setters
+
+  def logger
+    property_hash[:logger]
+  end
+
+  def logger=(value)
+    property_hash[:logger] = value
+  end
+
+  def merge_behavior
+    property_hash[:merge_behavior]
+  end
+
+  def merge_behavior=(value)
+    property_hash[:merge_behavior] = value
+  end
+
+  def hierarchy
+    property_hash[:hierarchy]
+  end
+
+  def hierarchy=(value)
+    property_hash[:hierarchy] = value
+  end
+
+  def hierarchy_override
+    property_hash[:hierarchy_override]
+  end
+
+  def hierarchy_override=(value)
+    property_hash[:hierarchy_override] = value
+  end
+
+  def data_dir
+    property_hash[:data_dir]
+  end
+
+  def data_dir=(value)
+    property_hash[:data_dir] = value
+  end
+
+  #####
+
+  # join basic and override directories to form the path to the override files directory
+  # @return [String]
+  def override_dir_path
+    File.join resource[:data_dir].to_s, override_dir_name
+  end
+
+  # base name of the override directory
+  # @return [String]
+  def override_dir_name
+    resource[:override_dir].to_s
+  end
+
+  # the path to the Hiera config file
+  # @return [String]
+  def config_file
+    resource[:name].to_s
+  end
+
+  # remove all mnemoization
+  def reset
+    @override_metadata_elements = nil
+    @override_directory_elements = nil
+  end
+
+  # try to get the list of plugin entries from the metadata file
+  # returns nil if there is no metadata file
+  # @return [Array, NilClass]
+  def metadata_plugin_entries
+    return @override_metadata_elements if @override_metadata_elements
+    data = read_metadata_yaml_file
+    return unless data.is_a? Hash
+    @override_metadata_elements = []
+    data.keys.each do |key|
+      key_value = data.fetch(key, {})
+      next unless key_value.is_a? Hash
+      metadata_value = key_value.fetch('metadata', {})
+      next unless metadata_value.is_a? Hash
+      plugin_id = metadata_value.fetch('plugin_id', nil)
+      @override_metadata_elements << File.join(override_dir_name, key) if plugin_id
+    end
+    @override_metadata_elements.sort!
+    debug "Found plugins hierarchy elements in '#{resource[:metadata_yaml_file]}': #{@override_metadata_elements.inspect}"
+    @override_metadata_elements
+  end
+
+  # scan for the override directory and get all the data entries found there
+  # sorts entries alphabeticly
+  # @return [Array]
+  def override_directory_entries
+    return @override_directory_elements if @override_directory_elements
+    @override_directory_elements = []
+    dir_entries(override_dir_path).each do |file|
+      next unless file.end_with? '.yaml'
+      file = file.gsub /\.yaml$/, ''
+      @override_directory_elements << File.join(override_dir_name, file)
+    end
+    @override_directory_elements.sort!
+    debug "Found override hierarchy elements: #{@override_directory_elements.inspect}"
+    @override_directory_elements
+  end
+
+  # read the directory entries
+  # @param dir [String]
+  # @return [Array]
+  def dir_entries(dir)
+    return [] unless dir and File.directory? dir
+    Dir.entries dir
+  end
+
+  # load this file as a YAML structure
+  # @param file [String]
+  # @return [Object]
+  def yaml_load_file(file)
+    return unless file and File.exists? file
+    YAML.load_file file
+  end
+
+  # retrieve only basic hierarhy part from the confugaration structure
+  # @return [Array]
+  def get_basic_hierarchy(data)
+    hierarchy = data[:hierarchy] || []
+    hierarchy.reject do |element|
+      element.start_with? override_dir_name
+    end
+  end
+
+  # retrieve only override hierarhy part from the confugaration structure
+  # @return [Array]
+  def get_override_hierarchy(data)
+    hierarchy = data[:hierarchy] || []
+    hierarchy.select do |element|
+      element.start_with? override_dir_name
+    end
+  end
+
+  # join both hierarhy parts to form the hierarhy structure
+  # @return [Array]
+  def generate_hierarhy
+    hierarchy = []
+    hierarchy += property_hash[:hierarchy_override] if property_hash[:hierarchy_override].is_a? Array
+    hierarchy += property_hash[:hierarchy] if property_hash[:hierarchy].is_a? Array
+    hierarchy
+  end
+
+  # try to get plugin entries from the metadata file
+  # or from the directory scan entries
+  # and don't touch entries if they are manually provided
+  def generate_override_entries
+    return if resource[:hierarchy_override].is_a? Array and resource[:hierarchy_override].any?
+    entries = metadata_plugin_entries
+    entries = override_directory_entries unless entries
+    resource[:hierarchy_override] = entries
+  end
+
+  # load parameters from the configuration structure read from the coinfig file
+  # @return [Hash]
+  def load_configuration
+    return if property_hash.is_a? Hash and property_hash.any?
+    generate_override_entries
+    data = read_configuration
+    self.property_hash = {}
+    property_hash[:logger] = data[:logger]
+    property_hash[:data_dir] = data.fetch(:yaml, {})[:datadir]
+    property_hash[:hierarchy] = get_basic_hierarchy data
+    property_hash[:hierarchy_override] = get_override_hierarchy data
+    property_hash[:merge_behavior] = data[:merge_behavior]
+
+    debug "Loaded configuration: #{property_hash.inspect}"
+    property_hash
+  end
+
+  # generate configuration data structure from the parameters
+  # @return [Hash]
+  def generate_configuration
+    config = {}
+    backends = ['yaml']
+
+    config[:logger] = property_hash[:logger]
+    config[:yaml] = {:datadir => property_hash[:data_dir]}
+    config[:hierarchy] = generate_hierarhy
+    config[:backends] = backends
+    config[:merge_behavior] = property_hash[:merge_behavior]
+
+    debug "Generated configuration: #{config.inspect}"
+    config
+  end
+
+  # readt the hiera configuratio file
+  # @return [Hash]
+  def read_configuration
+    begin
+      data = yaml_load_file config_file
+      return {} unless data.is_a? Hash
+      data
+    rescue => exception
+      debug "Error parsing config file: '#{config_file}': #{exception.message}"
+      {}
+    end
+  end
+
+  # read the metadata yaml file and return either data
+  # or nil if the file was not read or is not correct
+  # @return [Hash, NilClass]
+  def read_metadata_yaml_file
+    begin
+      data = yaml_load_file resource[:metadata_yaml_file]
+      return unless data.is_a? Hash
+      data
+    rescue => exception
+      debug "Error parsing metadata file: '#{resource[:metadata_yaml_file]}': #{exception.message}"
+      nil
+    end
+  end
+
+  # write the generated data to the hiera config
+  # @param [Hash] data
+  def write_configuration(data)
+    File.open(config_file, 'w') do |file|
+      file.puts data.to_yaml
+    end
+  end
+
+  # remove the Hiera configuration file
+  def remove_configuration
+    File.delete config_file if configuration_present?
+  end
+
+  # check if the Hiera configuration file exists
+  # @return [TrueClass,FalseClass]
+  def configuration_present?
+    File.exists? config_file
+  end
+
+  #####
+
+  def exists?
+    debug 'Call: exists?'
+    load_configuration
+    configuration_present?
+  end
+
+  def destroy
+    debug 'Call: destroy'
+    remove_configuration
+    self.property_hash = {}
+  end
+
+  def flush
+    debug 'Call: flush'
+    return unless property_hash.is_a? Hash and property_hash.any?
+    configuration = read_configuration.merge generate_configuration
+    debug "Writing configuration: #{configuration.inspect}"
+    write_configuration configuration
+  end
+
+end

deployment/puppet/osnailyfacter/lib/puppet/type/hiera_config.rb

@@ -0,0 +1,78 @@
+Puppet::Type.newtype(:hiera_config) do
+  desc 'Manage Hiera yaml configuration'
+
+  ensurable
+
+  newparam(:name) do
+    desc 'The path to the Hiera config file.'
+    isnamevar
+  end
+
+  newparam(:override_dir) do
+    desc 'Look for override files in this directory.
+         The directory should be inside the data directory and the path
+         should be relative to the basic data directory path.'
+    defaultto 'plugins'
+  end
+
+  newparam(:metadata_yaml_file) do
+    desc 'Look inside this YAML file for the list of enabled plugins.
+          If this value is not defined or no file is present the list
+          of plugins will be found from the content of the override directory
+          directly.'
+  end
+
+  newproperty(:data_dir) do
+    desc 'Basic directory with Hiera data elements'
+    defaultto '/etc/hiera'
+  end
+
+  newproperty(:hierarchy, :array_matching => :all) do
+    desc 'Basic hierarchy elements. This list of elements will be used at the
+          bottom of the hierarchy before any overrides are applied.'
+    defaultto []
+
+    def is_to_s(value)
+      value.inspect
+    end
+
+    def should_to_s(value)
+      value.inspect
+    end
+  end
+
+  newproperty(:hierarchy_override, :array_matching => :all) do
+    desc 'Override hierarchy elements. These list will be automaticly gathered
+          either from the metadata file of from the override directoiry scanning.
+          If you provide the list manually it will be used without any automatic
+          element gathering.'
+    defaultto []
+
+    def is_to_s(value)
+      value.inspect
+    end
+
+    def should_to_s(value)
+      value.inspect
+    end
+  end
+
+  newproperty(:logger) do
+    desc 'The Hiera logger type.'
+    newvalues 'noop', 'puppet', 'console'
+    defaultto 'noop'
+    munge do |value|
+      value.to_s
+    end
+  end
+
+  newproperty(:merge_behavior) do
+    desc 'Merge strategy for hash lookups.'
+    newvalues 'native', 'deep', 'deeper'
+    defaultto 'native'
+    munge do |value|
+      value.to_s
+    end
+  end
+
+end

deployment/puppet/osnailyfacter/modular/hiera/hiera.pp

@@ -6,7 +6,11 @@ $deep_merge_package_name = $::osfamily ? {
 }
 
 $data_dir            = '/etc/hiera'
-$data                = [
+$override_dir        = 'plugins'
+$override_dir_path   = "${data_dir}/${override_dir}"
+$metadata_file       = '/etc/astute.yaml'
+
+$data = [
   'override/node/%{::fqdn}',
   'override/class/%{calling_class}',
   'override/module/%{calling_module}',
@@ -19,8 +23,9 @@ $data                = [
   'module/%{calling_module}',
   'nodes',
   'globals',
-  'astute'
+  'astute',
 ]
+
 $astute_data_file    = '/etc/astute.yaml'
 $hiera_main_config   = '/etc/hiera.yaml'
 $hiera_puppet_config = '/etc/puppet/hiera.yaml'
@@ -32,31 +37,28 @@ File {
   mode  => '0644',
 }
 
-$hiera_config_content = inline_template('
----
-:backends:
-  - yaml
-
-:hierarchy:
-<% @data.each do |name| -%>
-  - <%= name %>
-<% end -%>
-
-:yaml:
-  :datadir: <%= @data_dir %>
-:merge_behavior: deeper
-:logger: noop
-')
+hiera_config { $hiera_main_config :
+  ensure             => 'present',
+  data_dir           => $data_dir,
+  hierarchy          => $data,
+  override_dir       => $override_dir,
+  metadata_yaml_file => $metadata_file,
+  merge_behavior     => 'deeper',
+}
 
 file { 'hiera_data_dir' :
   ensure => 'directory',
   path   => $data_dir,
 }
 
+file { 'hiera_data_override_dir' :
+  ensure => 'directory',
+  path   => $override_dir_path,
+}
+
 file { 'hiera_config' :
   ensure  => 'present',
   path    => $hiera_main_config,
-  content => $hiera_config_content,
 }
 
 file { 'hiera_data_astute' :

deployment/puppet/osnailyfacter/spec/fixtures/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

deployment/puppet/osnailyfacter/spec/unit/provider/hiera_config/ruby_spec.rb

@@ -0,0 +1,194 @@
+describe Puppet::Type.type(:hiera_config).provider(:ruby) do
+  let(:params) do
+    {
+        :name => '/etc/hiera.yaml',
+        :hierarchy => %w(base additional),
+    }
+  end
+
+  let (:resource) do
+    Puppet::Type::Hiera_config.new params
+  end
+
+  let (:provider) do
+    resource.provider
+  end
+
+  before(:each) do
+    puppet_debug_override
+  end
+
+  let(:config_file_data) do
+    {
+        :yaml => {
+            :datadir => "/etc/hiera",
+        },
+        :hierarchy => %w(plugins/a plugins/b plugins/c additional base),
+        :backends => ["yaml"],
+        :logger => "noop",
+        :merge_behavior => 'deeper',
+    }
+  end
+
+  let(:config_file_data_for_metadata_entries) do
+    {
+        :yaml => {
+            :datadir => "/etc/hiera",
+        },
+        :hierarchy => %w(plugins/plugin1 plugins/plugin2 additional base),
+        :backends => ["yaml"],
+        :logger => "noop",
+        :merge_behavior => 'deeper',
+    }
+  end
+
+  let(:property_hash) do
+    {
+        :logger => "noop",
+        :data_dir => "/etc/hiera",
+        :hierarchy => %w(additional base),
+        :hierarchy_override => %w(plugins/a plugins/b plugins/c),
+        :merge_behavior => 'deeper',
+    }
+  end
+
+  let(:override_dir) do
+    '/etc/hiera/plugins'
+  end
+
+  let(:hierarchy_override) do
+    %w(plugins/a plugins/b plugins/c)
+  end
+
+  let(:plugin_metadata_structure) do
+    {
+        'plugin1' => {
+            'metadata' => {
+                'plugin_id' => '1',
+            },
+        },
+        'plugin2' => {
+            'metadata' => {
+                'plugin_id' => '2',
+            },
+        },
+        'a' => 'b',
+        'c' => {
+            'metadata' => 'd'
+        },
+    }
+  end
+
+  let(:metadata_plugins_list) do
+    %w(plugins/plugin1 plugins/plugin2)
+  end
+
+  before(:each) do
+    provider.stubs(:yaml_load_file).with('/etc/hiera.yaml').returns(config_file_data)
+    provider.stubs(:yaml_load_file).with('/etc/astute.yaml').returns(plugin_metadata_structure)
+    provider.stubs(:yaml_load_file).with(nil).returns(nil)
+    provider.stubs(:dir_entries).with(override_dir).returns %w(a.yaml c.yaml b.yaml . .. 1.txt test)
+  end
+
+  context '#retreive' do
+
+    it 'can read the Hiera config file file' do
+      expect(provider.read_configuration).to eq config_file_data
+    end
+
+    it 'returns an empty hash if file was not read' do
+      provider.expects(:yaml_load_file).with('/etc/hiera.yaml').raises(Errno::ENOENT)
+      expect(provider.read_configuration).to eq({})
+    end
+
+    it 'parses the retrieved data to the property_hash' do
+      expect(provider.load_configuration).to eq property_hash
+      expect(provider.property_hash).to eq property_hash
+    end
+
+    it 'can form the full override directory path' do
+      expect(provider.override_dir_path).to eq override_dir
+    end
+
+    it 'can get the list of found override elements' do
+      expect(provider.override_directory_entries).to eq hierarchy_override
+    end
+
+    it 'can get the list of elements fro mthe metadata file' do
+      resource[:metadata_yaml_file] = '/etc/astute.yaml'
+      expect(provider.metadata_plugin_entries).to eq metadata_plugins_list
+    end
+
+    it 'will use metadata entries prior to directory entries' do
+      resource[:metadata_yaml_file] = '/etc/astute.yaml'
+      expect(provider.generate_override_entries).to eq metadata_plugins_list
+    end
+
+    it 'will use directory entries if there are is metadata file' do
+      resource[:metadata_yaml_file] = '/etc/astute.yaml'
+      expect(provider.generate_override_entries).to eq metadata_plugins_list
+    end
+
+    it 'overwrites the hiera_override property with found values' do
+      provider.load_configuration
+      expect(resource[:hierarchy_override]).to eq hierarchy_override
+    end
+
+    it 'will not rewrite the hiera_override property if it contains any data from catalog' do
+      resource[:hierarchy_override] = %w(a b)
+      provider.load_configuration
+      expect(resource[:hierarchy_override]).to eq %w(a b)
+    end
+
+  end
+
+  context '#generate' do
+    before(:each) do
+      provider.property_hash = property_hash
+    end
+
+    it 'can generate the hierarchy structure' do
+      expect(provider.generate_hierarhy).to eq config_file_data[:hierarchy]
+    end
+
+    it 'can generate a new configuration structure from the property_hash' do
+      expect(provider.generate_configuration).to eq config_file_data
+    end
+
+    it 'can create a new configuration if there is no saved one' do
+      provider.stubs(:read_configuration).returns({})
+      provider.expects(:write_configuration).with(config_file_data)
+      provider.flush
+    end
+
+    it 'can set a value and set it in the configuration file' do
+      provider.stubs(:read_configuration).returns(config_file_data)
+      provider.logger = 'console'
+      provider.expects(:write_configuration).with(config_file_data.merge(:logger => 'console'))
+      provider.flush
+    end
+
+    it 'will save any additional parameters in the Hiera config file' do
+      provider.stubs(:read_configuration).returns(config_file_data.merge(:a => 1))
+      provider.load_configuration
+      provider.expects(:write_configuration).with(config_file_data.merge(:a => 1))
+      provider.flush
+    end
+  end
+
+  context '#both retreive and generate' do
+    it 'cat generate configuration with directory entries' do
+      provider.load_configuration
+      provider.hierarchy_override = resource[:hierarchy_override]
+      expect(provider.generate_configuration).to eq(config_file_data)
+    end
+
+    it 'cat generate configuration with metadata entries' do
+      resource[:metadata_yaml_file] = '/etc/astute.yaml'
+      provider.load_configuration
+      provider.hierarchy_override = resource[:hierarchy_override]
+      expect(provider.generate_configuration).to eq(config_file_data_for_metadata_entries)
+    end
+  end
+
+end

deployment/puppet/osnailyfacter/spec/unit/type/hiera_config_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe Puppet::Type.type(:hiera_config) do
+
+  subject do
+    Puppet::Type.type(:hiera_config)
+  end
+
+  let(:params) do
+    {
+      :name => '/etc/hiera.yaml',
+      :hierarchy => %w(base additional),
+    }
+  end
+
+  it 'should be able to create an instance' do
+    expect(subject.new params).not_to be_falsey
+  end
+
+  [:logger, :data_dir, :hierarchy, :hierarchy_override, :merge_behavior, :metadata_yaml_file].each do |param|
+    it "should have a #{param} parameter" do
+      expect(subject.valid_parameter?(param)).to be_truthy
+    end
+  end
+
+end
+
+

tests/noop/spec/hosts/hiera/hiera_spec.rb

@@ -13,8 +13,20 @@ describe manifest do
         'ensure' => 'present',
         'path'   => '/etc/hiera.yaml'
       )
+
       # ensure deeper merge_behavior is being set
-      should contain_file('hiera_config').with_content(/deeper/)
+      should contain_hiera_config('/etc/hiera.yaml').with(
+        'merge_behavior' => 'deeper',
+      )
+
+      # ensure hiera_config is taking plugin overrides from the astute.yaml
+      should contain_hiera_config('/etc/hiera.yaml').with(
+                 'ensure' => 'present',
+                 'metadata_yaml_file' => '/etc/astute.yaml',
+                 'override_dir' => 'plugins',
+                 'data_dir' => '/etc/hiera',
+             )
+
       # check symlinks
       should contain_file('hiera_data_astute').with(
         'ensure' => 'symlink',