Add time-sync functionality to dos.py

Add functionality to sync time on
specific env nodes on demand using dos.py

Related-Bug: #1318641

Change-Id: If26abc316a80e31fb575fec0ebffa1685025d556

devops/helpers/helpers.py

@@ -34,6 +34,7 @@ from devops.error import DevopsError
 from devops.error import TimeoutError
 from devops.helpers.retry import retry
 from devops import logger
+from devops.settings import SSH_CREDENTIALS
 
 
 def get_free_port():
@@ -128,6 +129,73 @@ def http(host='localhost', port=80, method='GET', url='/', waited_code=200):
         return False
 
 
+def get_private_keys(env):
+    _ssh_keys = []
+    admin_remote = get_admin_remote(env)
+    for key_string in ['/root/.ssh/id_rsa',
+                       '/root/.ssh/bootstrap.rsa']:
+        with admin_remote.open(key_string) as f:
+            _ssh_keys.append(paramiko.RSAKey.from_private_key(f))
+    return _ssh_keys
+
+
+def get_admin_remote(env):
+    wait(lambda: tcp_ping(env.node_by_name(
+        'admin').get_ip_address_by_network_name('admin'), 22), timeout=180)
+    return env.node_by_name(
+        'admin').remote(network_name=SSH_CREDENTIALS['admin_network'],
+                        login=SSH_CREDENTIALS['login'],
+                        password=SSH_CREDENTIALS['password'])
+
+
+def get_node_remote(env, node_name):
+    ip = get_slave_ip(env, env.node_by_name(
+        node_name).interfaces[0].mac_address)
+    wait(lambda: tcp_ping(ip, 22), timeout=180)
+    return SSHClient(ip,
+                     username=SSH_CREDENTIALS['login'],
+                     password=SSH_CREDENTIALS['password'],
+                     private_keys=get_private_keys(env))
+
+
+def sync_node_time(env, node_name='admin', cmd=None):
+    if cmd is None:
+        cmd = "hwclock -s && NTPD=$(find /etc/init.d/ -regex "
+        cmd += "'/etc/init.d/ntp.?'); $NTPD stop; killall ntpd;"
+        cmd += " ntpd -qg && $NTPD start"
+
+    if node_name == 'admin':
+        try:
+            # If public NTP servers aren't accessible ntpdate will fail and
+            # ntpd daemon shouldn't be restarted to avoid 'Server has gone
+            # too long without sync' error while syncing time from slaves
+            remote = get_admin_remote(env)
+            remote.execute("ntpdate -d $(awk '/^server/{print"
+                           " $2}' /etc/ntp.conf)")
+        except AssertionError as e:
+            logger.warning('Error occurred while synchronizing time on master'
+                           ': {0}'.format(e))
+        else:
+            remote = get_admin_remote(env)
+            remote.execute('service ntpd stop && ntpd -qg && '
+                           'service ntpd start')
+    else:
+        remote = get_node_remote(env, node_name)
+        remote.execute(cmd)
+    remote.execute('hwclock -w')
+    remote_date = remote.execute('date')['stdout']
+    logger.info("Node time: {0}".format(remote_date))
+
+
+def get_slave_ip(env, node_mac_address):
+    remote = get_admin_remote(env)
+    ip = remote.execute(
+        "fuel nodes --node-id {0} | awk -F'|' "
+        "'END{{gsub(\" \", \"\", $5); print $5}}'".
+        format(node_mac_address))['stdout']
+    return ip[0].rstrip()
+
+
 class KeyPolicy(paramiko.WarningPolicy):
     def missing_host_key(self, client, hostname, key):
         return

devops/settings.py

@@ -36,6 +36,12 @@ DATABASES = {
     }
 }
 
+SSH_CREDENTIALS = {
+    'admin_network': environ.get('ENV_ADMIN_NETWORK', 'admin'),
+    'login': environ.get('ENV_FUEL_LOGIN', 'root'),
+    'password': environ.get('ENV_FUEL_PASSWORD', 'r00tme')
+}
+
 SECRET_KEY = 'dummykey'
 
 VNC_PASSWORD = environ.get('VNC_PASSWORD', None)

devops/shell.py

@@ -17,6 +17,8 @@ from os import environ
 
 from devops.manager import Manager
 
+from helpers.helpers import sync_node_time
+
 
 class Shell(object):
     def __init__(self):
@@ -105,6 +107,24 @@ class Shell(object):
         for network in networks:
             print("%15s  %10s" % (network.name, network.ip_network))
 
+    def do_timesync(self):
+        env = self.manager.environment_get(self.params.name)
+        if not self.params.node_name:
+            _nodes = {node.name: node.get_vnc_port() for node in env.nodes}
+            for node_name in sorted(_nodes.keys()):
+                if _nodes[node_name] != '-1':
+                    sync_node_time(env, node_name)
+        else:
+            sync_node_time(env, self.params.node_name)
+
+    def do_revert_resume(self):
+        self.manager.environment_get(self.params.name).revert(
+            self.params.snapshot_name)
+        self.manager.environment_get(self.params.name).resume(verbose=False)
+        if not self.params.no_timesync:
+            print('time synchronization is starting')
+            self.do_timesync()
+
     commands = {
         'list': do_list,
         'show': do_show,
@@ -119,6 +139,8 @@ class Shell(object):
         'snapshot-list': do_snapshot_list,
         'snapshot-delete': do_snapshot_delete,
         'net-list': do_net_list,
+        'time-sync': do_timesync,
+        'revert-resume': do_revert_resume
     }
 
     def get_params(self):
@@ -130,6 +152,15 @@ class Shell(object):
         snapshot_name_parser.add_argument('--snapshot-name',
                                           help='snapshot name',
                                           default=environ.get('SNAPSHOT_NAME'))
+        node_name_parser = argparse.ArgumentParser(add_help=False)
+        node_name_parser.add_argument('--node-name',
+                                      help='node name',
+                                      default=None)
+        no_timesync_parser = argparse.ArgumentParser(add_help=False)
+        no_timesync_parser.add_argument('--no-timesync', dest='no_timesync',
+                                        action='store_const', const=True,
+                                        help='revert without timesync',
+                                        default=False)
         parser = argparse.ArgumentParser(
             description="Manage virtual environments. "
                         "For addional help use command with -h/--help")
@@ -186,4 +217,17 @@ class Shell(object):
                               help="Show networks in environment",
                               description="Display allocated networks for "
                               "environment")
+        subparsers.add_parser('time-sync',
+                              parents=[name_parser, node_name_parser],
+                              help="Sync time on all env nodes",
+                              description="Sync time on all active nodes "
+                                          "of environment starting from "
+                                          "admin")
+        subparsers.add_parser('revert-resume',
+                              parents=[name_parser, snapshot_name_parser,
+                                       node_name_parser, no_timesync_parser],
+                              help="Revert, resume, sync time on VMs",
+                              description="Revert and resume VMs in selected"
+                                          "environment, then"
+                                          " sync time on VMs")
         return parser.parse_args()