fix race with syslog while renaming remote log dir

When a node gets new name nailgun renmaes log directoty
which had name in for mof <IP> to <node name> and then
creates symlink <IP> -> <node name>. The problem is that
node is still active and this moment and sends some logs
to syslog on master node.

So when we renamed <IP> to <node name> and haven't created
symlink name syslog may find missing log dir and it creates
it again in this case. Then nailgun tries to create symlink
and fails.

To avoid this situation we can suspend rsyslog temporarity.
It's better to do it with SIGSTOP/SIGCONT, because process
exist in this case and remote logs will be held in kernel
tcp or odp buffers until we wake up rsyslog and it process
new messages. Fortunately centos-7 uses journald together
with rsyslog, so local logs (send by syslog function) are
handled by journald and rsyslog just get it from journald
files. So it's safe to stop rsyslog temporarily, nothing
will be droppped.

Change-Id: Icb1ed6669be2f7b448c465de33de37e0b945fb79
Closes-Bug: #1506112
(cherry picked from commit 6e49910fca)

nailgun/nailgun/utils/logs.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import shutil
+import subprocess
 import sys
 
 from nailgun import consts
@@ -59,33 +60,38 @@ def prepare_syslog_dir(node, prefix=settings.SYSLOG_DIR):
             shutil.rmtree(bak)
         os.rename(new, bak)
 
-    # rename bootstrap directory into fqdn
-    if os.path.islink(old):
-        logger.debug("Old %s exists and it is link. "
-                     "Trying to unlink", old)
-        os.unlink(old)
-    if os.path.isdir(old):
-        logger.debug("Old %s exists and it is directory. "
-                     "Trying to rename into %s", old, new)
-        os.rename(old, new)
-    else:
-        logger.debug("Creating %s", new)
-        os.makedirs(new)
+    try:
+        # rename bootstrap directory into fqdn
+        if os.path.islink(old):
+            logger.debug("Old %s exists and it is link. "
+                         "Trying to unlink", old)
+            os.unlink(old)
+        if os.path.isdir(old):
+            subprocess.check_call(["/usr/bin/pkill", "-STOP", "rsyslog"])
 
-    # creating symlinks
-    for l in links:
-        if os.path.islink(l) or os.path.isfile(l):
-            logger.debug("%s already exists. "
-                         "Trying to unlink", l)
-            os.unlink(l)
-        if os.path.isdir(l):
-            logger.debug("%s already exists and it directory. "
-                         "Trying to remove", l)
-            shutil.rmtree(l)
-        logger.debug("Creating symlink %s -> %s", l, new)
-        os.symlink(objects.Node.get_node_fqdn(node), l)
+            logger.debug("Old %s exists and it is directory. "
+                         "Trying to rename into %s", old, new)
+            os.rename(old, new)
+        else:
+            logger.debug("Creating %s", new)
+            os.makedirs(new)
 
-    os.system("/usr/bin/pkill -HUP rsyslog")
+        # creating symlinks
+        for l in links:
+            if os.path.islink(l) or os.path.isfile(l):
+                logger.debug("%s already exists. "
+                             "Trying to unlink", l)
+                os.unlink(l)
+            if os.path.isdir(l):
+                logger.debug("%s already exists and it directory. "
+                             "Trying to remove", l)
+                shutil.rmtree(l)
+            logger.debug("Creating symlink %s -> %s", l, new)
+            os.symlink(objects.Node.get_node_fqdn(node), l)
+    finally:
+        subprocess.check_call(["/usr/bin/pkill", "-CONT", "rsyslog"])
+
+    subprocess.check_call(["/usr/bin/pkill", "-HUP", "rsyslog"])
 
 
 def generate_log_paths_for_node(node, prefix):