Add force-bootstrap feature

Two features was added:
1) Ability to set a specific node to be a initial galera node.
2) Galera 3.19 requires to set bootstrap flag in datadir, overwise it will
refuse to boostrap from this node.
3) Bugfix: Added sorted to the get_oldest_node_by_seqno() to fetch same results
on all node

Change-Id: I50e42f66ba475f809cda00c72f57787a70f66cda

service/files/defaults.yaml

@@ -10,6 +10,9 @@ configs:
     monitor_password: "password"
     gcache_size: "1G"
     cluster_size: 3
+    force_bootstrap:
+      enabled: false
+      node: null
     port:
       cont: 3306
 url:

service/files/percona_entrypoint.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import fileinput
 import functools
 import json
 import logging
@@ -30,6 +31,8 @@ logging.basicConfig(format=LOG_FORMAT, datefmt=LOG_DATEFMT)
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
 
+FORCE_BOOTSTRAP = None
+FORCE_BOOTSTRAP_NODE = None
 EXPECTED_NODES = None
 MYSQL_ROOT_PASSWORD = None
 CLUSTER_NAME = None
@@ -84,7 +87,10 @@ def set_globals():
     global MYSQL_ROOT_PASSWORD, CLUSTER_NAME, XTRABACKUP_PASSWORD
     global MONITOR_PASSWORD, CONNECTION_ATTEMPTS, CONNECTION_DELAY
     global ETCD_PATH, ETCD_HOST, ETCD_PORT, EXPECTED_NODES
+    global FORCE_BOOTSTRAP, FORCE_BOOTSTRAP_NODE
 
+    FORCE_BOOTSTRAP = config['percona']['force_bootstrap']['enabled']
+    FORCE_BOOTSTRAP_NODE = config['percona']['force_bootstrap']['node']
     MYSQL_ROOT_PASSWORD = config['db']['root_password']
     CLUSTER_NAME = config['percona']['cluster_name']
     XTRABACKUP_PASSWORD = config['percona']['xtrabackup_password']
@@ -226,8 +232,8 @@ def get_oldest_node_by_seqno(etcd_client, path):
     # We need to cut etcd path prefix like "/galera/k8scluster/seqno/" to get
     # the IP addr of the node.
     prefix = key + "/"
-    result = [(str(child.key).replace(prefix, ''), int(child.value))
-              for child in root.children]
+    result = sorted([(str(child.key).replace(prefix, ''), int(child.value))
+              for child in root.children])
     result.sort(key=lambda x: x[1])
     LOG.debug("ALL seqno is %s", result)
     LOG.info("Oldest node is %s, am %s", result[-1][0], IPADDR)
@@ -327,12 +333,8 @@ def wait_for_expected_state(etcd_client, ttl):
             wait_for_my_turn(etcd_client)
             break
 
+def wait_for_my_seqno(etcd_client):
 
-def wait_for_my_turn(etcd_client):
-
-    check_for_stale_seqno(etcd_client)
-    LOG.info("Waiting for my turn to join cluster")
-    while True:
         oldest_node = get_oldest_node_by_seqno(etcd_client, 'seqno')
         if IPADDR == oldest_node:
             LOG.info("It's my turn to join the cluster")
@@ -340,6 +342,36 @@ def wait_for_my_turn(etcd_client):
         else:
             time.sleep(5)
 
+def wait_for_my_turn(etcd_client):
+
+    check_for_stale_seqno(etcd_client)
+    LOG.info("Waiting for my turn to join cluster")
+    if FORCE_BOOTSTRAP:
+        LOG.warning("Force bootstrap flag was detected, skiping normal"
+                    " bootstrap procedure")
+        if FORCE_BOOTSTRAP_NODE is None:
+            LOG.error("Force bootstrap node wasn't set. Can't continue")
+            sys.exit(1)
+
+        LOG.debug("Force bootstrap node is %s", FORCE_BOOTSTRAP_NODE)
+        my_node_name = os.environ['CCP_NODE_NAME']
+        if my_node_name == FORCE_BOOTSTRAP_NODE:
+            LOG.info("This node is the force boostrap one.")
+            set_safe_to_bootstrap()
+            return
+        else:
+            LOG.info("This node is not the force boostrap one."
+                     " Waiting for the bootstrap one to create a cluster.")
+            while True:
+                nodes = fetch_status(etcd_client, 'nodes')
+                if nodes:
+                    wait_for_my_seqno(etcd_client)
+                    return
+                else:
+                    time.sleep(5)
+    else:
+        wait_for_my_seqno(etcd_client)
+
 
 def wait_for_sync(mysqld):
 
@@ -510,6 +542,18 @@ def release_lock(lock):
     LOG.info("Successfuly released lock")
 
 
+def set_safe_to_bootstrap():
+
+    """
+    Less wordy way to do "inplace" edit of the file
+    """
+
+    for line in fileinput.input(GRASTATE_FILE, inplace=1):
+        if line.startswith("safe_to_bootstrap"):
+            line = line.replace("safe_to_bootstrap: 0", "safe_to_bootstrap: 1")
+            sys.stdout.write(line)
+
+
 def run_create_queue(etcd_client, lock, ttl):
 
     """
@@ -550,6 +594,8 @@ def run_join_cluster(etcd_client, lock, ttl):
     state = get_cluster_state(etcd_client)
     nodes_status = fetch_status(etcd_client, 'nodes')
     available_nodes, first_one = create_join_list(nodes_status)
+    if first_one:
+        set_safe_to_bootstrap()
     mysqld = run_mysqld(available_nodes, etcd_client, lock)
     wait_for_sync(mysqld)
     etcd_register_in_path(etcd_client, 'nodes', ttl)