diff --git a/deployment/puppet/osnailyfacter/lib/puppet/provider/hiera_config/ruby.rb b/deployment/puppet/osnailyfacter/lib/puppet/provider/hiera_config/ruby.rb new file mode 100644 index 0000000000..af59d744e5 --- /dev/null +++ b/deployment/puppet/osnailyfacter/lib/puppet/provider/hiera_config/ruby.rb @@ -0,0 +1,268 @@ +require 'yaml' + +Puppet::Type.type(:hiera_config).provide(:ruby) do + desc 'Manage Hiera configuration file' + + attr_accessor :property_hash + attr_accessor :resource + + # Getters and Setters + + def logger + property_hash[:logger] + end + + def logger=(value) + property_hash[:logger] = value + end + + def merge_behavior + property_hash[:merge_behavior] + end + + def merge_behavior=(value) + property_hash[:merge_behavior] = value + end + + def hierarchy + property_hash[:hierarchy] + end + + def hierarchy=(value) + property_hash[:hierarchy] = value + end + + def hierarchy_override + property_hash[:hierarchy_override] + end + + def hierarchy_override=(value) + property_hash[:hierarchy_override] = value + end + + def data_dir + property_hash[:data_dir] + end + + def data_dir=(value) + property_hash[:data_dir] = value + end + + ##### + + # join basic and override directories to form the path to the override files directory + # @return [String] + def override_dir_path + File.join resource[:data_dir].to_s, override_dir_name + end + + # base name of the override directory + # @return [String] + def override_dir_name + resource[:override_dir].to_s + end + + # the path to the Hiera config file + # @return [String] + def config_file + resource[:name].to_s + end + + # remove all mnemoization + def reset + @override_metadata_elements = nil + @override_directory_elements = nil + end + + # try to get the list of plugin entries from the metadata file + # returns nil if there is no metadata file + # @return [Array, NilClass] + def metadata_plugin_entries + return @override_metadata_elements if @override_metadata_elements + data = read_metadata_yaml_file + return unless data.is_a? Hash + @override_metadata_elements = [] + data.keys.each do |key| + key_value = data.fetch(key, {}) + next unless key_value.is_a? Hash + metadata_value = key_value.fetch('metadata', {}) + next unless metadata_value.is_a? Hash + plugin_id = metadata_value.fetch('plugin_id', nil) + @override_metadata_elements << File.join(override_dir_name, key) if plugin_id + end + @override_metadata_elements.sort! + debug "Found plugins hierarchy elements in '#{resource[:metadata_yaml_file]}': #{@override_metadata_elements.inspect}" + @override_metadata_elements + end + + # scan for the override directory and get all the data entries found there + # sorts entries alphabeticly + # @return [Array] + def override_directory_entries + return @override_directory_elements if @override_directory_elements + @override_directory_elements = [] + dir_entries(override_dir_path).each do |file| + next unless file.end_with? '.yaml' + file = file.gsub /\.yaml$/, '' + @override_directory_elements << File.join(override_dir_name, file) + end + @override_directory_elements.sort! + debug "Found override hierarchy elements: #{@override_directory_elements.inspect}" + @override_directory_elements + end + + # read the directory entries + # @param dir [String] + # @return [Array] + def dir_entries(dir) + return [] unless dir and File.directory? dir + Dir.entries dir + end + + # load this file as a YAML structure + # @param file [String] + # @return [Object] + def yaml_load_file(file) + return unless file and File.exists? file + YAML.load_file file + end + + # retrieve only basic hierarhy part from the confugaration structure + # @return [Array] + def get_basic_hierarchy(data) + hierarchy = data[:hierarchy] || [] + hierarchy.reject do |element| + element.start_with? override_dir_name + end + end + + # retrieve only override hierarhy part from the confugaration structure + # @return [Array] + def get_override_hierarchy(data) + hierarchy = data[:hierarchy] || [] + hierarchy.select do |element| + element.start_with? override_dir_name + end + end + + # join both hierarhy parts to form the hierarhy structure + # @return [Array] + def generate_hierarhy + hierarchy = [] + hierarchy += property_hash[:hierarchy_override] if property_hash[:hierarchy_override].is_a? Array + hierarchy += property_hash[:hierarchy] if property_hash[:hierarchy].is_a? Array + hierarchy + end + + # try to get plugin entries from the metadata file + # or from the directory scan entries + # and don't touch entries if they are manually provided + def generate_override_entries + return if resource[:hierarchy_override].is_a? Array and resource[:hierarchy_override].any? + entries = metadata_plugin_entries + entries = override_directory_entries unless entries + resource[:hierarchy_override] = entries + end + + # load parameters from the configuration structure read from the coinfig file + # @return [Hash] + def load_configuration + return if property_hash.is_a? Hash and property_hash.any? + generate_override_entries + data = read_configuration + self.property_hash = {} + property_hash[:logger] = data[:logger] + property_hash[:data_dir] = data.fetch(:yaml, {})[:datadir] + property_hash[:hierarchy] = get_basic_hierarchy data + property_hash[:hierarchy_override] = get_override_hierarchy data + property_hash[:merge_behavior] = data[:merge_behavior] + + debug "Loaded configuration: #{property_hash.inspect}" + property_hash + end + + # generate configuration data structure from the parameters + # @return [Hash] + def generate_configuration + config = {} + backends = ['yaml'] + + config[:logger] = property_hash[:logger] + config[:yaml] = {:datadir => property_hash[:data_dir]} + config[:hierarchy] = generate_hierarhy + config[:backends] = backends + config[:merge_behavior] = property_hash[:merge_behavior] + + debug "Generated configuration: #{config.inspect}" + config + end + + # readt the hiera configuratio file + # @return [Hash] + def read_configuration + begin + data = yaml_load_file config_file + return {} unless data.is_a? Hash + data + rescue => exception + debug "Error parsing config file: '#{config_file}': #{exception.message}" + {} + end + end + + # read the metadata yaml file and return either data + # or nil if the file was not read or is not correct + # @return [Hash, NilClass] + def read_metadata_yaml_file + begin + data = yaml_load_file resource[:metadata_yaml_file] + return unless data.is_a? Hash + data + rescue => exception + debug "Error parsing metadata file: '#{resource[:metadata_yaml_file]}': #{exception.message}" + nil + end + end + + # write the generated data to the hiera config + # @param [Hash] data + def write_configuration(data) + File.open(config_file, 'w') do |file| + file.puts data.to_yaml + end + end + + # remove the Hiera configuration file + def remove_configuration + File.delete config_file if configuration_present? + end + + # check if the Hiera configuration file exists + # @return [TrueClass,FalseClass] + def configuration_present? + File.exists? config_file + end + + ##### + + def exists? + debug 'Call: exists?' + load_configuration + configuration_present? + end + + def destroy + debug 'Call: destroy' + remove_configuration + self.property_hash = {} + end + + def flush + debug 'Call: flush' + return unless property_hash.is_a? Hash and property_hash.any? + configuration = read_configuration.merge generate_configuration + debug "Writing configuration: #{configuration.inspect}" + write_configuration configuration + end + +end diff --git a/deployment/puppet/osnailyfacter/lib/puppet/type/hiera_config.rb b/deployment/puppet/osnailyfacter/lib/puppet/type/hiera_config.rb new file mode 100644 index 0000000000..2401137564 --- /dev/null +++ b/deployment/puppet/osnailyfacter/lib/puppet/type/hiera_config.rb @@ -0,0 +1,78 @@ +Puppet::Type.newtype(:hiera_config) do + desc 'Manage Hiera yaml configuration' + + ensurable + + newparam(:name) do + desc 'The path to the Hiera config file.' + isnamevar + end + + newparam(:override_dir) do + desc 'Look for override files in this directory. + The directory should be inside the data directory and the path + should be relative to the basic data directory path.' + defaultto 'plugins' + end + + newparam(:metadata_yaml_file) do + desc 'Look inside this YAML file for the list of enabled plugins. + If this value is not defined or no file is present the list + of plugins will be found from the content of the override directory + directly.' + end + + newproperty(:data_dir) do + desc 'Basic directory with Hiera data elements' + defaultto '/etc/hiera' + end + + newproperty(:hierarchy, :array_matching => :all) do + desc 'Basic hierarchy elements. This list of elements will be used at the + bottom of the hierarchy before any overrides are applied.' + defaultto [] + + def is_to_s(value) + value.inspect + end + + def should_to_s(value) + value.inspect + end + end + + newproperty(:hierarchy_override, :array_matching => :all) do + desc 'Override hierarchy elements. These list will be automaticly gathered + either from the metadata file of from the override directoiry scanning. + If you provide the list manually it will be used without any automatic + element gathering.' + defaultto [] + + def is_to_s(value) + value.inspect + end + + def should_to_s(value) + value.inspect + end + end + + newproperty(:logger) do + desc 'The Hiera logger type.' + newvalues 'noop', 'puppet', 'console' + defaultto 'noop' + munge do |value| + value.to_s + end + end + + newproperty(:merge_behavior) do + desc 'Merge strategy for hash lookups.' + newvalues 'native', 'deep', 'deeper' + defaultto 'native' + munge do |value| + value.to_s + end + end + +end diff --git a/deployment/puppet/osnailyfacter/modular/hiera/hiera.pp b/deployment/puppet/osnailyfacter/modular/hiera/hiera.pp index 135abfe346..4009b2b2f2 100644 --- a/deployment/puppet/osnailyfacter/modular/hiera/hiera.pp +++ b/deployment/puppet/osnailyfacter/modular/hiera/hiera.pp @@ -6,7 +6,11 @@ $deep_merge_package_name = $::osfamily ? { } $data_dir = '/etc/hiera' -$data = [ +$override_dir = 'plugins' +$override_dir_path = "${data_dir}/${override_dir}" +$metadata_file = '/etc/astute.yaml' + +$data = [ 'override/node/%{::fqdn}', 'override/class/%{calling_class}', 'override/module/%{calling_module}', @@ -19,8 +23,9 @@ $data = [ 'module/%{calling_module}', 'nodes', 'globals', - 'astute' + 'astute', ] + $astute_data_file = '/etc/astute.yaml' $hiera_main_config = '/etc/hiera.yaml' $hiera_puppet_config = '/etc/puppet/hiera.yaml' @@ -32,31 +37,28 @@ File { mode => '0644', } -$hiera_config_content = inline_template(' ---- -:backends: - - yaml - -:hierarchy: -<% @data.each do |name| -%> - - <%= name %> -<% end -%> - -:yaml: - :datadir: <%= @data_dir %> -:merge_behavior: deeper -:logger: noop -') +hiera_config { $hiera_main_config : + ensure => 'present', + data_dir => $data_dir, + hierarchy => $data, + override_dir => $override_dir, + metadata_yaml_file => $metadata_file, + merge_behavior => 'deeper', +} file { 'hiera_data_dir' : ensure => 'directory', path => $data_dir, } +file { 'hiera_data_override_dir' : + ensure => 'directory', + path => $override_dir_path, +} + file { 'hiera_config' : ensure => 'present', path => $hiera_main_config, - content => $hiera_config_content, } file { 'hiera_data_astute' : diff --git a/deployment/puppet/osnailyfacter/spec/fixtures/.gitignore b/deployment/puppet/osnailyfacter/spec/fixtures/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/deployment/puppet/osnailyfacter/spec/fixtures/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/deployment/puppet/osnailyfacter/spec/unit/provider/hiera_config/ruby_spec.rb b/deployment/puppet/osnailyfacter/spec/unit/provider/hiera_config/ruby_spec.rb new file mode 100644 index 0000000000..f832d6189b --- /dev/null +++ b/deployment/puppet/osnailyfacter/spec/unit/provider/hiera_config/ruby_spec.rb @@ -0,0 +1,194 @@ +describe Puppet::Type.type(:hiera_config).provider(:ruby) do + let(:params) do + { + :name => '/etc/hiera.yaml', + :hierarchy => %w(base additional), + } + end + + let (:resource) do + Puppet::Type::Hiera_config.new params + end + + let (:provider) do + resource.provider + end + + before(:each) do + puppet_debug_override + end + + let(:config_file_data) do + { + :yaml => { + :datadir => "/etc/hiera", + }, + :hierarchy => %w(plugins/a plugins/b plugins/c additional base), + :backends => ["yaml"], + :logger => "noop", + :merge_behavior => 'deeper', + } + end + + let(:config_file_data_for_metadata_entries) do + { + :yaml => { + :datadir => "/etc/hiera", + }, + :hierarchy => %w(plugins/plugin1 plugins/plugin2 additional base), + :backends => ["yaml"], + :logger => "noop", + :merge_behavior => 'deeper', + } + end + + let(:property_hash) do + { + :logger => "noop", + :data_dir => "/etc/hiera", + :hierarchy => %w(additional base), + :hierarchy_override => %w(plugins/a plugins/b plugins/c), + :merge_behavior => 'deeper', + } + end + + let(:override_dir) do + '/etc/hiera/plugins' + end + + let(:hierarchy_override) do + %w(plugins/a plugins/b plugins/c) + end + + let(:plugin_metadata_structure) do + { + 'plugin1' => { + 'metadata' => { + 'plugin_id' => '1', + }, + }, + 'plugin2' => { + 'metadata' => { + 'plugin_id' => '2', + }, + }, + 'a' => 'b', + 'c' => { + 'metadata' => 'd' + }, + } + end + + let(:metadata_plugins_list) do + %w(plugins/plugin1 plugins/plugin2) + end + + before(:each) do + provider.stubs(:yaml_load_file).with('/etc/hiera.yaml').returns(config_file_data) + provider.stubs(:yaml_load_file).with('/etc/astute.yaml').returns(plugin_metadata_structure) + provider.stubs(:yaml_load_file).with(nil).returns(nil) + provider.stubs(:dir_entries).with(override_dir).returns %w(a.yaml c.yaml b.yaml . .. 1.txt test) + end + + context '#retreive' do + + it 'can read the Hiera config file file' do + expect(provider.read_configuration).to eq config_file_data + end + + it 'returns an empty hash if file was not read' do + provider.expects(:yaml_load_file).with('/etc/hiera.yaml').raises(Errno::ENOENT) + expect(provider.read_configuration).to eq({}) + end + + it 'parses the retrieved data to the property_hash' do + expect(provider.load_configuration).to eq property_hash + expect(provider.property_hash).to eq property_hash + end + + it 'can form the full override directory path' do + expect(provider.override_dir_path).to eq override_dir + end + + it 'can get the list of found override elements' do + expect(provider.override_directory_entries).to eq hierarchy_override + end + + it 'can get the list of elements fro mthe metadata file' do + resource[:metadata_yaml_file] = '/etc/astute.yaml' + expect(provider.metadata_plugin_entries).to eq metadata_plugins_list + end + + it 'will use metadata entries prior to directory entries' do + resource[:metadata_yaml_file] = '/etc/astute.yaml' + expect(provider.generate_override_entries).to eq metadata_plugins_list + end + + it 'will use directory entries if there are is metadata file' do + resource[:metadata_yaml_file] = '/etc/astute.yaml' + expect(provider.generate_override_entries).to eq metadata_plugins_list + end + + it 'overwrites the hiera_override property with found values' do + provider.load_configuration + expect(resource[:hierarchy_override]).to eq hierarchy_override + end + + it 'will not rewrite the hiera_override property if it contains any data from catalog' do + resource[:hierarchy_override] = %w(a b) + provider.load_configuration + expect(resource[:hierarchy_override]).to eq %w(a b) + end + + end + + context '#generate' do + before(:each) do + provider.property_hash = property_hash + end + + it 'can generate the hierarchy structure' do + expect(provider.generate_hierarhy).to eq config_file_data[:hierarchy] + end + + it 'can generate a new configuration structure from the property_hash' do + expect(provider.generate_configuration).to eq config_file_data + end + + it 'can create a new configuration if there is no saved one' do + provider.stubs(:read_configuration).returns({}) + provider.expects(:write_configuration).with(config_file_data) + provider.flush + end + + it 'can set a value and set it in the configuration file' do + provider.stubs(:read_configuration).returns(config_file_data) + provider.logger = 'console' + provider.expects(:write_configuration).with(config_file_data.merge(:logger => 'console')) + provider.flush + end + + it 'will save any additional parameters in the Hiera config file' do + provider.stubs(:read_configuration).returns(config_file_data.merge(:a => 1)) + provider.load_configuration + provider.expects(:write_configuration).with(config_file_data.merge(:a => 1)) + provider.flush + end + end + + context '#both retreive and generate' do + it 'cat generate configuration with directory entries' do + provider.load_configuration + provider.hierarchy_override = resource[:hierarchy_override] + expect(provider.generate_configuration).to eq(config_file_data) + end + + it 'cat generate configuration with metadata entries' do + resource[:metadata_yaml_file] = '/etc/astute.yaml' + provider.load_configuration + provider.hierarchy_override = resource[:hierarchy_override] + expect(provider.generate_configuration).to eq(config_file_data_for_metadata_entries) + end + end + +end diff --git a/deployment/puppet/osnailyfacter/spec/unit/type/hiera_config_spec.rb b/deployment/puppet/osnailyfacter/spec/unit/type/hiera_config_spec.rb new file mode 100644 index 0000000000..8018cca4c6 --- /dev/null +++ b/deployment/puppet/osnailyfacter/spec/unit/type/hiera_config_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +describe Puppet::Type.type(:hiera_config) do + + subject do + Puppet::Type.type(:hiera_config) + end + + let(:params) do + { + :name => '/etc/hiera.yaml', + :hierarchy => %w(base additional), + } + end + + it 'should be able to create an instance' do + expect(subject.new params).not_to be_falsey + end + + [:logger, :data_dir, :hierarchy, :hierarchy_override, :merge_behavior, :metadata_yaml_file].each do |param| + it "should have a #{param} parameter" do + expect(subject.valid_parameter?(param)).to be_truthy + end + end + +end + + diff --git a/tests/noop/spec/hosts/hiera/hiera_spec.rb b/tests/noop/spec/hosts/hiera/hiera_spec.rb index 3d71f51382..84df8262b5 100644 --- a/tests/noop/spec/hosts/hiera/hiera_spec.rb +++ b/tests/noop/spec/hosts/hiera/hiera_spec.rb @@ -13,8 +13,20 @@ describe manifest do 'ensure' => 'present', 'path' => '/etc/hiera.yaml' ) + # ensure deeper merge_behavior is being set - should contain_file('hiera_config').with_content(/deeper/) + should contain_hiera_config('/etc/hiera.yaml').with( + 'merge_behavior' => 'deeper', + ) + + # ensure hiera_config is taking plugin overrides from the astute.yaml + should contain_hiera_config('/etc/hiera.yaml').with( + 'ensure' => 'present', + 'metadata_yaml_file' => '/etc/astute.yaml', + 'override_dir' => 'plugins', + 'data_dir' => '/etc/hiera', + ) + # check symlinks should contain_file('hiera_data_astute').with( 'ensure' => 'symlink',