Run cfn-hup via a cron job.

cfn-hup is supposed to be a daemon, but it doesn't work that way. So run
it via a wrapper in a cron job. Also run this way, we won't have
logging, so make sure os-refresh-config is run via upstart so we get the
output logged.

The wrapper must use a lock to avoid problems that might be caused when
multiple updates overlap.

Change-Id: I8613c45b33091ad6448ecddebe6a1f4337f09d7d

elements/heat-cfntools/os-config-applier/etc/cfn/hooks.conf

@@ -2,7 +2,7 @@
 [os-refresh-config-{{resource}}]
 triggers=post.add,post.delete.post.update
 path=Resources.{{resource}}.Metadata
-action=os-refresh-config
+action=service os-refresh-config start
 runas=root
 
 {{/heat.refresh}}

elements/os-refresh-config/install.d/75-cfn-hup-cronjob

@@ -0,0 +1,9 @@
+#!/bin/bash
+install -m 0755 -o root -g root $(dirname $0)/cfn-hup-wrapper /usr/local/bin/cfn-hup-wrapper
+cat > /etc/cron.d/cfn-hup <<EOF
+# This is a workaround for cfn-hup not actually being a daemon
+SHELL=/bin/sh
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+*/5 * * * * root cfn-hup-wrapper
+EOF

elements/os-refresh-config/install.d/cfn-hup-wrapper

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import fcntl
+import os
+import random
+import subprocess
+import sys
+import time
+
+if len(sys.argv) > 1:
+    max_sleep = int(sys.argv[1])
+else:
+    max_sleep = 59
+
+randskew = random.randint(0, max_sleep)
+time.sleep(randskew)
+with open('/run/os-refresh-config-cfn-hup.lock', 'a') as lock:
+    fcntl.flock(lock, fcntl.LOCK_EX)
+    lock.seek(0)
+    lock.truncate()
+    lock.write('Lock held by process id %s\n' % os.getpid())
+    lock.flush()
+    try:
+        subprocess.check_call(['cfn-hup', '--no-daemon'])
+    except subprocess.CalledProcessError as e:
+        sys.stderr.write("%s\n", e)
+        sys.exit(e.returncode)