Initial commit

deployment/puppet/concat/.fixtures.yml

@@ -0,0 +1,3 @@
+fixtures:
+  symlinks:
+    'concat': '#{source_dir}'

deployment/puppet/concat/.gemfile

@@ -0,0 +1,5 @@
+source :rubygems
+
+puppetversion = ENV.key?('PUPPET_VERSION') ? "= #{ENV['PUPPET_VERSION']}" : ['>= 2.7']
+gem 'puppet', puppetversion
+gem 'puppetlabs_spec_helper', '>= 0.1.0'

deployment/puppet/concat/.gitignore

@@ -0,0 +1 @@
+pkg

deployment/puppet/concat/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - 1.8.7
+script:
+  - "rake lint"
+  - "rake spec"
+branches:
+  only:
+    - master
+env:
+  - PUPPET_VERSION=2.7.11
+gemfile: .gemfile

deployment/puppet/concat/CHANGELOG

@@ -0,0 +1,29 @@
+KNOWN ISSUES:
+- In 0.24.8 you will see inintended notifies, if you build a file
+  in a run, the next run will also see it as changed.  This is due
+  to how 0.24.8 does the purging of unhandled files, this is improved
+  in 0.25.x and we cannot work around it in our code.
+
+CHANGELOG:
+- 2010/02/19 - initial release
+- 2010/03/12 - add support for 0.24.8 and newer
+             - make the location of sort configurable
+             - add the ability to add shell comment based warnings to
+               top of files
+             - add the ablity to create empty files
+- 2010/04/05 - fix parsing of WARN and change code style to match rest
+               of the code
+             - Better and safer boolean handling for warn and force
+             - Don't use hard coded paths in the shell script, set PATH
+               top of the script
+             - Use file{} to copy the result and make all fragments owned
+               by root.  This means we can chnage the ownership/group of the
+               resulting file at any time.
+             - You can specify ensure => "/some/other/file" in concat::fragment
+               to include the contents of a symlink into the final file.
+- 2010/04/16 - Add more cleaning of the fragment name - removing / from the $name
+- 2010/05/22 - Improve documentation and show the use of ensure =>
+- 2010/07/14 - Add support for setting the filebucket behavior of files
+- 2010/10/04 - Make the warning message configurable
+- 2010/12/03 - Add flags to make concat work better on Solaris - thanks Jonathan Boyett
+- 2011/02/03 - Make the shell script more portable and add a config option for root group

deployment/puppet/concat/LICENSE

@@ -0,0 +1,14 @@
+   Copyright 2012 R.I.Pienaar
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

deployment/puppet/concat/Modulefile

@@ -0,0 +1,8 @@
+name 'ripienaar-concat'
+version '0.1.0'
+source 'git://github.com/ripienaar/puppet-concat.git'
+author 'R.I.Pienaar'
+license 'Apache'
+summary 'Concat module'
+description 'Concat module'
+project_page 'http://github.com/ripienaar/puppet-concat'

deployment/puppet/concat/README.markdown

@@ -0,0 +1,108 @@
+What is it?
+===========
+
+A Puppet module that can construct files from fragments.
+
+Please see the comments in the various .pp files for details
+as well as posts on my blog at http://www.devco.net/
+
+Released under the Apache 2.0 licence
+
+Usage:
+------
+
+If you wanted a /etc/motd file that listed all the major modules
+on the machine.  And that would be maintained automatically even
+if you just remove the include lines for other modules you could
+use code like below, a sample /etc/motd would be:
+
+<pre>
+Puppet modules on this server:
+
+    -- Apache
+    -- MySQL
+</pre>
+
+Local sysadmins can also append to the file by just editing /etc/motd.local
+their changes will be incorporated into the puppet managed motd.
+
+<pre>
+# class to setup basic motd, include on all nodes
+class motd {
+   $motd = "/etc/motd"
+
+   concat{$motd:
+      owner => root,
+      group => root,
+      mode  => 644
+   }
+
+   concat::fragment{"motd_header":
+      target => $motd,
+      content => "\nPuppet modules on this server:\n\n",
+      order   => 01,
+   }
+
+   # local users on the machine can append to motd by just creating
+   # /etc/motd.local
+   concat::fragment{"motd_local":
+      target => $motd,
+      ensure  => "/etc/motd.local",
+      order   => 15
+   }
+}
+
+# used by other modules to register themselves in the motd
+define motd::register($content="", $order=10) {
+   if $content == "" {
+      $body = $name
+   } else {
+      $body = $content
+   }
+
+   concat::fragment{"motd_fragment_$name":
+      target  => "/etc/motd",
+      content => "    -- $body\n"
+   }
+}
+
+# a sample apache module
+class apache {
+   include apache::install, apache::config, apache::service
+
+   motd::register{"Apache": }
+}
+</pre>
+
+Known Issues:
+-------------
+* In 0.24.8 you will see inintended notifies, if you build a file
+  in a run, the next run will also see it as changed.  This is due
+  to how 0.24.8 does the purging of unhandled files, this is improved
+  in 0.25.x and we cannot work around it in our code.
+* Since puppet-concat now relies on a fact for the concat directory,
+  you will need to set up pluginsync = true for at least the first run.
+  You have this issue if puppet fails to run on the client and you have
+  a message similar to
+  "err: Failed to apply catalog: Parameter path failed: File
+  paths must be fully qualified, not 'undef' at [...]/concat/manifests/setup.pp:44".
+
+Contributors:
+-------------
+**Paul Elliot**
+
+ * Provided 0.24.8 support, shell warnings and empty file creation support.
+
+**Chad Netzer**
+
+ * Various patches to improve safety of file operations
+ * Symlink support
+
+**David Schmitt**
+
+ * Patch to remove hard coded paths relying on OS path
+ * Patch to use file{} to copy the resulting file to the final destination.  This means Puppet client will show diffs and that hopefully we can change file ownerships now
+
+Contact:
+--------
+You can contact me on rip@devco.net or follow my blog at http://www.devco.net I am also on twitter as ripienaar

deployment/puppet/concat/Rakefile

@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'puppetlabs_spec_helper/rake_tasks'

deployment/puppet/concat/files/concatfragments.sh

@@ -0,0 +1,129 @@
+#!/bin/sh
+
+# Script to concat files to a config file.
+#
+# Given a directory like this:
+# /path/to/conf.d
+# |-- fragments
+# |   |-- 00_named.conf
+# |   |-- 10_domain.net
+# |   `-- zz_footer
+#
+# The script supports a test option that will build the concat file to a temp location and 
+# use /usr/bin/cmp to verify if it should be run or not.  This would result in the concat happening
+# twice on each run but gives you the option to have an unless option in your execs to inhibit rebuilds.
+# 
+# Without the test option and the unless combo your services that depend on the final file would end up 
+# restarting on each run, or in other manifest models some changes might get missed.
+#
+# OPTIONS:
+#  -o	The file to create from the sources
+#  -d	The directory where the fragments are kept
+#  -t	Test to find out if a build is needed, basically concats the files to a temp
+#       location and compare with what's in the final location, return codes are designed
+#       for use with unless on an exec resource
+#  -w   Add a shell style comment at the top of the created file to warn users that it 
+#       is generated by puppet
+#  -f   Enables the creation of empty output files when no fragments are found
+#  -n	Sort the output numerically rather than the default alpha sort
+#
+# the command: 
+#
+#   concatfragments.sh -o /path/to/conffile.cfg -d /path/to/conf.d
+#
+# creates /path/to/conf.d/fragments.concat and copies the resulting 
+# file to /path/to/conffile.cfg.  The files will be sorted alphabetically
+# pass the -n switch to sort numerically.
+# 
+# The script does error checking on the various dirs and files to make
+# sure things don't fail.
+
+OUTFILE=""
+WORKDIR=""
+TEST=""
+FORCE=""
+WARN=""
+SORTARG=""
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+
+## Well, if there's ever a bad way to do things, Nexenta has it.
+## http://nexenta.org/projects/site/wiki/Personalities
+unset SUN_PERSONALITY
+
+while getopts "o:s:d:tnw:f" options; do
+	case $options in
+		o ) OUTFILE=$OPTARG;;
+		d ) WORKDIR=$OPTARG;;
+		n ) SORTARG="-n";;
+		w ) WARNMSG="$OPTARG";;
+		f ) FORCE="true";;
+		t ) TEST="true";;
+		* ) echo "Specify output file with -o and fragments directory with -d"
+		    exit 1;;
+	esac
+done
+
+# do we have -o?
+if [ x${OUTFILE} = "x" ]; then
+	echo "Please specify an output file with -o"
+	exit 1
+fi
+
+# do we have -d?
+if [ x${WORKDIR} = "x" ]; then
+	echo "Please fragments directory with -d"
+	exit 1
+fi
+
+# can we write to -o?
+if [ -f ${OUTFILE} ]; then
+	if [ ! -w ${OUTFILE} ]; then
+		echo "Cannot write to ${OUTFILE}"
+		exit 1
+	fi
+else
+	if [ ! -w `dirname ${OUTFILE}` ]; then
+		echo "Cannot write to `dirname ${OUTFILE}` to create ${OUTFILE}"
+		exit 1
+	fi
+fi
+
+# do we have a fragments subdir inside the work dir?
+if [ ! -d "${WORKDIR}/fragments" ]  && [ ! -x "${WORKDIR}/fragments" ]; then
+	echo "Cannot access the fragments directory"
+	exit 1
+fi
+
+# are there actually any fragments?
+if [ ! "$(ls -A ${WORKDIR}/fragments)" ]; then
+	if [ x${FORCE} = "x" ]; then
+		echo "The fragments directory is empty, cowardly refusing to make empty config files"
+		exit 1
+	fi
+fi
+
+cd ${WORKDIR}
+
+if [ x${WARNMSG} = "x" ]; then
+	: > "fragments.concat"
+else
+	printf '%s\n' "$WARNMSG" > "fragments.concat"
+fi
+
+# find all the files in the fragments directory, sort them numerically and concat to fragments.concat in the working dir
+find fragments/ -type f -follow | sort ${SORTARG} | while read fragfile; do
+	cat "$fragfile" >> "fragments.concat"
+done
+
+if [ x${TEST} = "x" ]; then
+	# This is a real run, copy the file to outfile
+	cp fragments.concat ${OUTFILE}
+	RETVAL=$?
+else
+	# Just compare the result to outfile to help the exec decide
+	cmp ${OUTFILE} fragments.concat
+	RETVAL=$?
+fi
+
+exit $RETVAL

deployment/puppet/concat/lib/facter/concat_basedir.rb

@@ -0,0 +1,5 @@
+Facter.add("concat_basedir") do
+    setcode do
+        File.join(Puppet[:vardir],"concat")
+    end
+end

deployment/puppet/concat/manifests/fragment.pp

@@ -0,0 +1,49 @@
+# Puts a file fragment into a directory previous setup using concat
+# 
+# OPTIONS:
+#   - target    The file that these fragments belong to
+#   - content   If present puts the content into the file
+#   - source    If content was not specified, use the source
+#   - order     By default all files gets a 10_ prefix in the directory
+#               you can set it to anything else using this to influence the
+#               order of the content in the file
+#   - ensure    Present/Absent or destination to a file to include another file
+#   - mode      Mode for the file
+#   - owner     Owner of the file
+#   - group     Owner of the file
+#   - backup    Controls the filebucketing behavior of the final file and
+#               see File type reference for its use.  Defaults to 'puppet'
+define concat::fragment($target, $content='', $source='', $order=10, $ensure = 'present', $mode = '0644', $owner = $::id, $group = $concat::setup::root_group, $backup = 'puppet') {
+  $safe_name = regsubst($name, '/', '_', 'G')
+  $safe_target_name = regsubst($target, '/', '_', 'G')
+  $concatdir = $concat::setup::concatdir
+  $fragdir = "${concatdir}/${safe_target_name}"
+
+  # if content is passed, use that, else if source is passed use that
+  # if neither passed, but $ensure is in symlink form, make a symlink
+  case $content {
+    '': {
+      case $source {
+        '': {
+          case $ensure {
+            '', 'absent', 'present', 'file', 'directory': {
+              crit('No content, source or symlink specified')
+            }
+          }
+        }
+        default: { File{ source => $source } }
+      }
+    }
+    default: { File{ content => $content } }
+  }
+
+  file{"${fragdir}/fragments/${order}_${safe_name}":
+    ensure => $ensure,
+    mode   => $mode,
+    owner  => $owner,
+    group  => $group,
+    backup => $backup,
+    alias  => "concat_fragment_${name}",
+    notify => Exec["concat_${target}"]
+  }
+}

deployment/puppet/concat/manifests/init.pp

@@ -0,0 +1,221 @@
+# A system to construct files using fragments from other files or templates.
+#
+# This requires at least puppet 0.25 to work correctly as we use some
+# enhancements in recursive directory management and regular expressions
+# to do the work here.
+#
+# USAGE:
+# The basic use case is as below:
+#
+# concat{"/etc/named.conf":
+#    notify => Service["named"]
+# }
+#
+# concat::fragment{"foo.com_config":
+#    target  => "/etc/named.conf",
+#    order   => 10,
+#    content => template("named_conf_zone.erb")
+# }
+#
+# # add a fragment not managed by puppet so local users
+# # can add content to managed file
+# concat::fragment{"foo.com_user_config":
+#    target  => "/etc/named.conf",
+#    order   => 12,
+#    ensure  => "/etc/named.conf.local"
+# }
+#
+# This will use the template named_conf_zone.erb to build a single
+# bit of config up and put it into the fragments dir.  The file
+# will have an number prefix of 10, you can use the order option
+# to control that and thus control the order the final file gets built in.
+#
+# SETUP:
+# The class concat::setup uses the fact concat_basedir to define the variable
+# $concatdir, where all the temporary files and fragments will be
+# durably stored. The fact concat_basedir will be set up on the client to
+# <Puppet[:vardir]>/concat, so you will be able to run different setup/flavours
+# of puppet clients.
+# However, since this requires the file lib/facter/concat_basedir.rb to be
+# deployed on the clients, so you will have to set "pluginsync = true" on
+# both the master and client, at least for the first run.
+#
+# There's some regular expression magic to figure out the puppet version but
+# if you're on an older 0.24 version just set $puppetversion = 24
+#
+# DETAIL:
+# We use a helper shell script called concatfragments.sh that gets placed
+# in <Puppet[:vardir]>/concat/bin to do the concatenation.  While this might
+# seem more complex than some of the one-liner alternatives you might find on
+# the net we do a lot of error checking and safety checks in the script to avoid
+# problems that might be caused by complex escaping errors etc.
+#
+# LICENSE:
+# Apache Version 2
+#
+# LATEST:
+# http://github.com/ripienaar/puppet-concat/
+#
+# CONTACT:
+# R.I.Pienaar <rip@devco.net>
+# Volcane on freenode
+# @ripienaar on twitter
+# www.devco.net
+
+
+# Sets up so that you can use fragments to build a final config file,
+#
+# OPTIONS:
+#  - mode       The mode of the final file
+#  - owner      Who will own the file
+#  - group      Who will own the file
+#  - force      Enables creating empty files if no fragments are present
+#  - warn       Adds a normal shell style comment top of the file indicating
+#               that it is built by puppet
+#  - backup     Controls the filebucketing behavior of the final file and
+#               see File type reference for its use.  Defaults to 'puppet'
+#
+# ACTIONS:
+#  - Creates fragment directories if it didn't exist already
+#  - Executes the concatfragments.sh script to build the final file, this
+#    script will create directory/fragments.concat.   Execution happens only
+#    when:
+#    * The directory changes
+#    * fragments.concat != final destination, this means rebuilds will happen
+#      whenever someone changes or deletes the final file.  Checking is done
+#      using /usr/bin/cmp.
+#    * The Exec gets notified by something else - like the concat::fragment
+#      define
+#  - Copies the file over to the final destination using a file resource
+#
+# ALIASES:
+#  - The exec can notified using Exec["concat_/path/to/file"] or
+#    Exec["concat_/path/to/directory"]
+#  - The final file can be referened as File["/path/to/file"] or
+#    File["concat_/path/to/file"]
+define concat(
+  $owner = $::id,
+  $group = $concat::setup::root_group,
+  $mode = '0644',
+  $warn = false,
+  $force = false,
+  $backup = 'puppet',
+  $gnu = undef,
+  $order='alpha'
+) {
+  include concat::setup
+
+  $safe_name   = regsubst($name, '/', '_', 'G')
+  $concatdir   = $concat::setup::concatdir
+  $version     = $concat::setup::majorversion
+  $fragdir     = "${concatdir}/${safe_name}"
+  $concat_name = 'fragments.concat.out'
+  $default_warn_message = '# This file is managed by Puppet. DO NOT EDIT.'
+
+  case $warn {
+    'true', true, yes, on: {
+      $warnmsg = $default_warn_message
+    }
+    'false', false, no, off: {
+      $warnmsg = ''
+    }
+    default: {
+      $warnmsg = $warn
+    }
+  }
+
+  $warnmsg_escaped = regsubst($warnmsg, "'", "'\\\\''", 'G')
+  $warnflag = $warnmsg_escaped ? {
+    ''      => '',
+    default => "-w '${warnmsg_escaped}'"
+  }
+
+  case $force {
+    'true', true, yes, on: {
+      $forceflag = '-f'
+    }
+    'false', false, no, off: {
+      $forceflag = ''
+    }
+    default: {
+      fail("Improper 'force' value given to concat: ${force}")
+    }
+  }
+
+  case $order {
+    numeric: {
+      $orderflag = '-n'
+    }
+    alpha: {
+      $orderflag = ''
+    }
+    default: {
+      fail("Improper 'order' value given to concat: ${order}")
+    }
+  }
+
+  File {
+    owner  => $::id,
+    group  => $group,
+    mode   => $mode,
+    backup => $backup
+  }
+
+  file { $fragdir:
+    ensure => directory,
+  }
+
+  $source_real = $version ? {
+    24      => 'puppet:///concat/null',
+    default => undef,
+  }
+
+  file { "${fragdir}/fragments":
+    ensure   => directory,
+    force    => true,
+    ignore   => ['.svn', '.git', '.gitignore'],
+    notify   => Exec["concat_${name}"],
+    purge    => true,
+    recurse  => true,
+    source   => $source_real,
+  }
+
+  file { "${fragdir}/fragments.concat":
+    ensure   => present,
+  }
+
+  file { "${fragdir}/${concat_name}":
+    ensure   => present,
+  }
+
+  file { $name:
+    ensure   => present,
+    alias    => "concat_${name}",
+    group    => $group,
+    mode     => $mode,
+    owner    => $owner,
+    source   => "${fragdir}/${concat_name}",
+  }
+
+  exec { "concat_${name}":
+    alias       => "concat_${fragdir}",
+    command     => "${concat::setup::concatdir}/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} ${warnflag} ${forceflag} ${orderflag}",
+    notify      => File[$name],
+    require     => [
+      File[$fragdir],
+      File["${fragdir}/fragments"],
+      File["${fragdir}/fragments.concat"],
+    ],
+    subscribe   => File[$fragdir],
+    unless      => "${concat::setup::concatdir}/bin/concatfragments.sh -o ${fragdir}/${concat_name} -d ${fragdir} -t ${warnflag} ${forceflag} ${orderflag}",
+  }
+
+  if $::id == 'root' {
+    Exec["concat_${name}"] {
+      user  => root,
+      group => $group,
+    }
+  }
+}
+
+# vim:sw=2:ts=2:expandtab:textwidth=79

deployment/puppet/concat/manifests/setup.pp

@@ -0,0 +1,49 @@
+# Sets up the concat system.
+#
+# $concatdir is where the fragments live and is set on the fact concat_basedir.
+# Since puppet should always manage files in $concatdir and they should
+# not be deleted ever, /tmp is not an option.
+#
+# $puppetversion should be either 24 or 25 to enable a 24 compatible
+# mode, in 24 mode you might see phantom notifies this is a side effect
+# of the method we use to clear the fragments directory.
+#
+# The regular expression below will try to figure out your puppet version
+# but this code will only work in 0.24.8 and newer.
+#
+# It also copies out the concatfragments.sh file to ${concatdir}/bin
+class concat::setup {
+  $id = $::id
+  $root_group = $id ? {
+    root    => 0,
+    default => $id
+  }
+
+  if $::concat_basedir {
+    $concatdir = $::concat_basedir
+  } else {
+    fail ("\$concat_basedir not defined. Try running again with pluginsync enabled")
+  }
+
+  $majorversion = regsubst($::puppetversion, '^[0-9]+[.]([0-9]+)[.][0-9]+$', '\1')
+
+  file{"${concatdir}/bin/concatfragments.sh":
+    owner  => $id,
+    group  => $root_group,
+    mode   => '0755',
+    source => $majorversion ? {
+      24      => 'puppet:///concat/concatfragments.sh',
+      default => 'puppet:///modules/concat/concatfragments.sh'
+    };
+
+  [ $concatdir, "${concatdir}/bin" ]:
+    ensure => directory,
+    owner  => $id,
+    group  => $root_group,
+    mode   => '0750';
+
+  ## Old versions of this module used a different path.
+  '/usr/local/bin/concatfragments.sh':
+    ensure => absent;
+  }
+}

deployment/puppet/concat/spec/defines/init_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe 'concat' do
+  basedir = '/var/lib/puppet/concat'
+  let(:title) { '/etc/foo.bar' }
+  let(:facts) { { 
+    :concat_basedir => '/var/lib/puppet/concat',
+    :id             => 'root',
+  } }
+  let :pre_condition do
+    'include concat::setup'
+  end
+
+  directories = [
+    "#{basedir}/_etc_foo.bar",
+    "#{basedir}/_etc_foo.bar/fragments",
+  ]
+
+  directories.each do |dirs|
+    it do
+      should contain_file(dirs).with({
+        'ensure'  => 'directory',
+        'backup'  => 'puppet',
+        'group'   => 0,
+        'mode'    => '0644',
+        'owner'   => 'root',
+      })
+    end
+  end
+
+  files = [
+    "/etc/foo.bar",
+    "#{basedir}/_etc_foo.bar/fragments.concat",
+  ]
+
+  files.each do |file|
+    it do
+      should contain_file(file).with({
+        'ensure'  => 'present',
+        'backup'  => 'puppet',
+        'group'   => 0,
+        'mode'    => '0644',
+        'owner'   => 'root',
+      })
+    end
+  end
+
+  it do
+    should contain_exec("concat_/etc/foo.bar").with_command(
+      "#{basedir}/bin/concatfragments.sh " +
+      "-o #{basedir}/_etc_foo.bar/fragments.concat.out " +
+      "-d #{basedir}/_etc_foo.bar   "
+    )
+  end
+end
+
+# vim:sw=2:ts=2:expandtab:textwidth=79

deployment/puppet/concat/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'puppetlabs_spec_helper/module_spec_helper'