Add takeover delay to quantum-ha-tool

Add a takeover delay to the quantum-ha-tool to ensure we do
not try to aggressively takeover.  We want to ensure an agent
is offline for some period of time (TAKEOVER_DELAY seconds)
before we begin a migration.  We also randomize it a bit in the
event that this script is being run in parallel from several hosts.

Change-Id: Ied9a75791b27af310f81f4cec327b3699fbae7ed

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 This file is used to list changes made in each version of cookbook-openstack-common.
 
+## 7.0.2:
+* Add delay to quantum-ha-tool.py script to prevent aggressive migrations
+
 ## 7.0.1:
 * Allow quota driver to be set dynamically (LP #1234324)
 

files/default/quantum-ha-tool.py

@@ -22,6 +22,8 @@ import os
 import sys
 import json
 import argparse
+import random
+import time
 from logging.handlers import SysLogHandler
 from collections import OrderedDict
 from random import choice
@@ -31,6 +33,8 @@ LOG = logging.getLogger('quantum-ha-tool')
 LOG_FORMAT='%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
 LOG_DATE = '%m-%d %H:%M'
 DESCRIPTION = "Quantum High Availability Tool"
+TAKEOVER_DELAY = int(random.random()*30+30)
+
 
 def parse_args():
 
@@ -208,6 +212,21 @@ def l3_agent_migrate(qclient, noop=False):
        if len(agent_alive_list) < 1:
            LOG.exception("There are no l3 agents alive to migrate routers onto")
 
+       timeout = 0
+
+       while timeout < TAKEOVER_DELAY:
+
+           agent_list_new = list_agents(qclient)
+           agent_dead_list_new = agent_dead_id_list(agent_list_new, 'L3 agent')
+           if len(agent_dead_list_new) < len(agent_dead_list):
+               LOG.info("Skipping router failover since an agent came online while ensuring agents offline for seconds=%s" % TAKEOVER_DELAY)
+               sys.exit(0)
+
+           LOG.info("Agent found offline for seconds=%s but waiting seconds=%s before migration" % (timeout, TAKEOVER_DELAY))
+           timeout += 1
+           time.sleep(1)
+
+
        for agent_id in agent_dead_list:
 
            LOG.info("Querying agent_id=%s for routers to migrate", agent_id)

metadata.rb

@@ -3,7 +3,7 @@ maintainer        "Jay Pipes <jaypipes@gmail.com>"
 license           "Apache 2.0"
 description       "Installs and configures the OpenStack Network API Service and various agents and plugins"
 long_description  IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version           "7.0.1"
+version           "7.0.2"
 recipe            "openstack-network::server", "Installs packages required for a OpenStack Network server"
 recipe            "openstack-network::openvswitch", "Installs packages required for OVS"
 recipe            "openstack-network::metadata_agent", "Installs packages required for a OpenStack Network Metadata Agent"