PM: Ensure only one periodic task runs per host

Ensure multiple periodic sync/recovery tasks do not run concurrently on
a single host by using an inter process lock inside the periodic task.

This has the effect of serializing the tasks, rather than the more correct
deduplication of tasks. Bug 1445127 tracks the longer term fix.

Change-Id: Ib0041f127aacfd0f49ca84deecc2a6950594dd58
Closes-Bug: 1445125

contrib/devstack/lib/designate

@@ -159,6 +159,9 @@ function configure_designate {
     sudo cp $DESIGNATE_DIR/etc/designate/rootwrap.conf.sample $DESIGNATE_ROOTWRAP_CONF
     iniset $DESIGNATE_ROOTWRAP_CONF DEFAULT filters_path $DESIGNATE_DIR/etc/designate/rootwrap.d root-helper
 
+    # Oslo Concurrency
+    iniset $DESIGNATE_CONF oslo_concurrency lock_path "$DESIGNATE_STATE_PATH"
+
     # Set up the rootwrap sudoers for designate
     local rootwrap_sudoer_cmd="$DESIGNATE_BIN_DIR/designate-rootwrap $DESIGNATE_ROOTWRAP_CONF *"
     local tempfile=`mktemp`

designate/__init__.py

@@ -22,6 +22,7 @@ import socket
 
 from oslo.config import cfg
 from oslo_log import log
+from oslo_concurrency import lockutils
 from oslo import messaging
 
 
@@ -67,3 +68,6 @@ log.set_defaults(default_log_levels=[
 
 # Set some Oslo RPC defaults
 messaging.set_transport_defaults('designate')
+
+# Set some Oslo Oslo Concurrency defaults
+lockutils.set_defaults(lock_path='$state_path')

designate/pool_manager/service.py

@@ -158,8 +158,11 @@ class Service(service.RPCService, service.Service):
     # Periodioc Tasks
     def periodic_recovery(self):
         """
-        :return:
+        :return: None
         """
+        # TODO(kiall): Replace this inter-process-lock with a distributed
+        #              lock, likely using the tooz library - see bug 1445127.
+        with lockutils.lock('periodic_recovery', external=True, delay=30):
             context = DesignateContext.get_admin_context(all_tenants=True)
 
             LOG.debug("Starting Periodic Recovery")
@@ -184,14 +187,17 @@ class Service(service.RPCService, service.Service):
                     self.update_domain(context, domain)
 
             except Exception:
-            LOG.exception(_LE('An unhandled exception in periodic recovery '
-                              'occurred'))
+                LOG.exception(_LE('An unhandled exception in periodic '
+                                  'recovery occurred'))
 
     def periodic_sync(self):
         """
         :return: None
         """
-        context = DesignateContext.get_admin_context(all_tenants=True)  # noqa
+        # TODO(kiall): Replace this inter-process-lock with a distributed
+        #              lock, likely using the tooz library - see bug 1445127.
+        with lockutils.lock('periodic_sync', external=True, delay=30):
+            context = DesignateContext.get_admin_context(all_tenants=True)
 
             LOG.debug("Starting Periodic Synchronization")
 
@@ -213,8 +219,8 @@ class Service(service.RPCService, service.Service):
             try:
                 for domain in domains:
                     # TODO(kiall): If the domain was created within the last
-                #              periodic_sync_seconds, attempt to recreate to
-                #              fill in targets which may have failed.
+                    #              periodic_sync_seconds, attempt to recreate
+                    #              to fill in targets which may have failed.
                     self.update_domain(context, domain)
 
             except Exception:

etc/designate/designate.conf.sample

@@ -281,3 +281,11 @@ debug = False
 #rndc_key_file = /etc/rndc.key
 #zone_file_path = $state_path/zones
 #query_destination = 127.0.0.1
+
+########################
+## Library Configuration
+########################
+[oslo_concurrency]
+# Path for Oslo Concurrency to store lock files, defaults to the value
+# of the state_path setting.
+#lock_path = $state_path