summaryrefslogtreecommitdiff
path: root/deployment_scripts/compute_post_deployment.py
blob: f04bdeabe93b7f5b0a7cdcb29d2b998883f8f3c0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
#!/usr/bin/env python

import ConfigParser
import logging
import netifaces
import os
import re
from socket import inet_ntoa
from struct import pack
import subprocess
import sys
import yaml


ASTUTE_PATH = '/etc/astute.yaml'
ASTUTE_SECTION = 'fuel-plugin-xenserver'
LOG_ROOT = '/var/log/fuel-plugin-xenserver'
LOG_FILE = 'compute_post_deployment.log'
HIMN_IP = '169.254.0.1'
INT_BRIDGE = 'br-int'
XS_PLUGIN_ISO = 'xenserverplugins-liberty.iso'
DIST_PACKAGES_DIR = '/usr/lib/python2.7/dist-packages/'

if not os.path.exists(LOG_ROOT):
    os.mkdir(LOG_ROOT)

logging.basicConfig(filename=os.path.join(LOG_ROOT, LOG_FILE),
                    level=logging.DEBUG)


def reportError(err):
    logging.warning(err)
    raise Exception(err)


def execute(*cmd, **kwargs):
    cmd = map(str, cmd)
    logging.info(' '.join(cmd))
    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE)

    if 'prompt' in kwargs:
        prompt = kwargs.get('prompt')
        proc.stdout.flush()
        (out, err) = proc.communicate(prompt)
    else:
        out = proc.stdout.readlines()
        err = proc.stderr.readlines()
        (out, err) = map(' '.join, [out, err])

    # Both if/else need to deal with "\n" scenario
    (out, err) = (out.replace('\n', ''), err.replace('\n', ''))

    if out:
        logging.debug(out)

    if proc.returncode is not None and proc.returncode != 0:
        reportError(err)

    return out


def ssh(host, username, password, *cmd, **kwargs):
    cmd = map(str, cmd)

    return execute('sshpass', '-p', password, 'ssh',
                   '-o', 'StrictHostKeyChecking=no',
                   '%s@%s' % (username, host), *cmd,
                   prompt=kwargs.get('prompt'))


def scp(host, username, password, target_path, filename):
    return execute('sshpass', '-p', password, 'scp',
                   '-o', 'StrictHostKeyChecking=no', filename,
                   '%s@%s:%s' % (username, host, target_path))


def get_astute(astute_path):
    """Return the root object read from astute.yaml"""
    if not os.path.exists(astute_path):
        reportError('%s not found' % astute_path)

    astute = yaml.load(open(astute_path))
    return astute


def astute_get(dct, keys, default=None, fail_if_missing=True):
    """A safe dictionary getter"""
    for key in keys:
        if key in dct:
            dct = dct[key]
        else:
            if fail_if_missing:
                reportError('Value of "%s" is missing' % key)
            return default
    return dct


def get_options(astute, astute_section):
    """Return username and password filled in plugin."""
    if astute_section not in astute:
        reportError('%s not found' % astute_section)

    options = astute[astute_section]
    logging.info('username: {username}'.format(**options))
    logging.info('password: {password}'.format(**options))
    logging.info('install_xapi: {install_xapi}'.format(**options))
    return options['username'], options['password'], \
        options['install_xapi']


def get_endpoints(astute):
    """Return the IP addresses of the endpoints connected to
    storage/mgmt network.
    """
    endpoints = astute['network_scheme']['endpoints']
    endpoints = dict([(
        k.replace('br-', ''),
        endpoints[k]['IP'][0]
    ) for k in endpoints])

    logging.info('storage network: {storage}'.format(**endpoints))
    logging.info('mgmt network: {mgmt}'.format(**endpoints))
    return endpoints


def init_eth():
    """Initialize the net interface connected to HIMN

    Returns:
        the IP addresses of local host and XenServer.
    """

    domid = execute('xenstore-read', 'domid')
    himn_mac = execute(
        'xenstore-read',
        '/local/domain/%s/vm-data/himn_mac' % domid)
    logging.info('himn_mac: %s' % himn_mac)

    _mac = lambda eth: \
        netifaces.ifaddresses(eth).get(netifaces.AF_LINK)[0]['addr']
    eths = [eth for eth in netifaces.interfaces() if _mac(eth) == himn_mac]
    if len(eths) != 1:
        reportError('Cannot find eth matches himn_mac')

    eth = eths[0]
    logging.info('himn_eth: %s' % eth)

    ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)

    if not ip:
        execute('dhclient', eth)
        fname = '/etc/network/interfaces.d/ifcfg-' + eth
        s = ('auto {eth}\n'
             'iface {eth} inet dhcp\n'
             'post-up route del default dev {eth}').format(eth=eth)
        with open(fname, 'w') as f:
            f.write(s)
        logging.info('%s created' % fname)
        execute('ifdown', eth)
        execute('ifup', eth)
        ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)

    if ip:
        himn_local = ip[0]['addr']
        himn_xs = '.'.join(himn_local.split('.')[:-1] + ['1'])
        if HIMN_IP == himn_xs:
            logging.info('himn_local: %s' % himn_local)
            return eth, himn_local

    reportError('HIMN failed to get IP address from XenServer')


def check_host_compatibility(himn, username, password):
    hotfix = 'XS65ESP1013'
    installed = ssh(himn, username, password,
                    'xe patch-list name-label=%s --minimal' % hotfix)
    ver = ssh(himn, username, password,
              ('xe host-param-get uuid=$(xe host-list --minimal) '
               'param-name=software-version param-key=product_version_text'))

    if not installed and ver == "6.5":
        reportError(('Hotfix %s has not been installed '
                     'and product version is %s') % (hotfix, ver))


def install_xenapi_sdk():
    """Install XenAPI Python SDK"""
    execute('cp', 'XenAPI.py', DIST_PACKAGES_DIR)


def create_novacompute_conf(himn, username, password, public_ip, services_ssl):
    """Fill nova-compute.conf with HIMN IP and root password. """
    mgmt_if = netifaces.ifaddresses('br-mgmt')
    if mgmt_if and mgmt_if.get(netifaces.AF_INET) \
            and mgmt_if.get(netifaces.AF_INET)[0]['addr']:
        mgmt_ip = mgmt_if.get(netifaces.AF_INET)[0]['addr']
    else:
        reportError('Cannot get IP Address on Management Network')

    filename = '/etc/nova/nova-compute.conf'
    cf = ConfigParser.ConfigParser()
    try:
        cf.read(filename)
        cf.set('DEFAULT', 'compute_driver', 'xenapi.XenAPIDriver')
        cf.set('DEFAULT', 'force_config_drive', 'True')
        scheme = "https" if services_ssl else "http"
        cf.set('DEFAULT', 'novncproxy_base_url',
               '%s://%s:6080/vnc_auto.html' % (scheme, public_ip))
        cf.set('DEFAULT', 'vncserver_proxyclient_address', mgmt_ip)
        if not cf.has_section('xenserver'):
            cf.add_section('xenserver')
        cf.set('xenserver', 'connection_url', 'http://%s' % himn)
        cf.set('xenserver', 'connection_username', username)
        cf.set('xenserver', 'connection_password', password)
        cf.set('xenserver', 'vif_driver',
               'nova.virt.xenapi.vif.XenAPIOpenVswitchDriver')
        cf.set('xenserver', 'ovs_integration_bridge', INT_BRIDGE)
        cf.write(open(filename, 'w'))
    except Exception:
        reportError('Cannot set configurations to %s' % filename)
    logging.info('%s created' % filename)


def route_to_compute(endpoints, himn_xs, himn_local, username, password):
    """Route storage/mgmt requests to compute nodes. """
    out = ssh(himn_xs, username, password, 'route', '-n')
    _net = lambda ip: '.'.join(ip.split('.')[:-1] + ['0'])
    _mask = lambda cidr: inet_ntoa(pack(
        '>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1))
    _routed = lambda net, mask, gw: re.search(r'%s\s+%s\s+%s\s+' % (
        net.replace('.', r'\.'),
        gw.replace('.', r'\.'),
        mask
    ), out)

    endpoint_names = ['storage', 'mgmt']
    for endpoint_name in endpoint_names:
        endpoint = endpoints.get(endpoint_name)
        if endpoint:
            ip, cidr = endpoint.split('/')
            net, mask = _net(ip), _mask(cidr)
            if not _routed(net, mask, himn_local):
                params = ['route', 'add', '-net', net, 'netmask',
                          mask, 'gw', himn_local]
                ssh(himn_xs, username, password, *params)
                sh = 'echo \'%s\' >> /etc/sysconfig/static-routes' \
                    % ' '.join(params)
                ssh(himn_xs, username, password, sh)
        else:
            logging.info('%s network ip is missing' % endpoint_name)


def install_suppack(himn, username, password):
    """Install xapi driver supplemental pack. """
    # TODO(Johnhua): check if installed
    scp(himn, username, password, '/tmp/', XS_PLUGIN_ISO)
    ssh(
        himn, username, password, 'xe-install-supplemental-pack',
        '/tmp/%s' % XS_PLUGIN_ISO, prompt='Y\n')
    ssh(himn, username, password, 'rm', '/tmp/%s' % XS_PLUGIN_ISO)


def forward_from_himn(eth):
    """Forward packets from HIMN to storage/mgmt network. """
    execute('sed', '-i', 's/#net.ipv4.ip_forward/net.ipv4.ip_forward/g',
            '/etc/sysctl.conf')
    execute('sysctl', '-p', '/etc/sysctl.conf')

    endpoint_names = ['br-storage', 'br-mgmt']
    for endpoint_name in endpoint_names:
        execute('iptables', '-t', 'nat', '-A', 'POSTROUTING',
                '-o', endpoint_name, '-j', 'MASQUERADE')
        execute('iptables', '-A', 'FORWARD',
                '-i', endpoint_name, '-o', eth,
                '-m', 'state', '--state', 'RELATED,ESTABLISHED',
                '-j', 'ACCEPT')
        execute('iptables', '-A', 'FORWARD',
                '-i', eth, '-o', endpoint_name,
                '-j', 'ACCEPT')

    execute('iptables', '-A', 'INPUT', '-i', eth, '-j', 'ACCEPT')
    execute('iptables', '-t', 'filter', '-S', 'FORWARD')
    execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
    execute('service', 'iptables-persistent', 'save')


def forward_port(eth_in, eth_out, target_host, target_port):
    """Forward packets from eth_in to eth_out on target_host:target_port. """
    execute('iptables', '-t', 'nat', '-A', 'PREROUTING',
            '-i', eth_in, '-p', 'tcp', '--dport', target_port,
            '-j', 'DNAT', '--to', target_host)
    execute('iptables', '-A', 'FORWARD',
            '-i', eth_out, '-o', eth_in,
            '-m', 'state', '--state', 'RELATED,ESTABLISHED',
            '-j', 'ACCEPT')
    execute('iptables', '-A', 'FORWARD',
            '-i', eth_in, '-o', eth_out,
            '-j', 'ACCEPT')

    execute('iptables', '-t', 'filter', '-S', 'FORWARD')
    execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
    execute('service', 'iptables-persistent', 'save')


def install_logrotate_script(himn, username, password):
    "Install console logrotate script"
    scp(himn, username, password, '/root/', 'rotate_xen_guest_logs.sh')
    ssh(himn, username, password, 'mkdir -p /var/log/xen/guest')
    ssh(himn, username, password, '''crontab - << CRONTAB
* * * * * /root/rotate_xen_guest_logs.sh
CRONTAB''')


def modify_neutron_rootwrap_conf(himn, username, password):
    """Set xenapi configurations"""
    filename = '/etc/neutron/rootwrap.conf'
    cf = ConfigParser.ConfigParser()
    try:
        cf.read(filename)
        cf.set('xenapi', 'xenapi_connection_url', 'http://%s' % himn)
        cf.set('xenapi', 'xenapi_connection_username', username)
        cf.set('xenapi', 'xenapi_connection_password', password)
        cf.write(open(filename, 'w'))
    except Exception:
        reportError("Fail to modify file %s", filename)
    logging.info('Modify file %s successfully', filename)


def modify_neutron_ovs_agent_conf(int_br, br_mappings):
    filename = '/etc/neutron/plugins/ml2/ml2_conf.ini'
    cf = ConfigParser.ConfigParser()
    try:
        cf.read(filename)
        cf.set('agent', 'root_helper',
               'neutron-rootwrap-xen-dom0 /etc/neutron/rootwrap.conf')
        cf.set('agent', 'root_helper_daemon', '')
        cf.set('agent', 'minimize_polling', False)
        cf.set('ovs', 'integration_bridge', int_br)
        cf.set('ovs', 'bridge_mappings', br_mappings)
        cf.write(open(filename, 'w'))
    except Exception:
        reportError("Fail to modify %s", filename)
    logging.info('Modify %s successfully', filename)


def get_private_network_ethX():
    # find out bridge which is used for private network
    values = astute['network_scheme']['transformations']
    for item in values:
        if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
            return item['name']


def find_bridge_mappings(astute, himn, username, password):
    ethX = get_private_network_ethX()
    if not ethX:
        reportError("Cannot find eth used for private network")

    # find the ethX mac in /sys/class/net/ethX/address
    fo = open('/sys/class/net/%s/address' % ethX, 'r')
    mac = fo.readline()
    fo.close()
    network_uuid = ssh(himn, username, password,
            'xe vif-list params=network-uuid minimal=true MAC=%s' % mac)
    bridge = ssh(himn, username, password,
            'xe network-param-get param-name=bridge uuid=%s' % network_uuid)

    # find physical network name
    phynet_setting = astute['quantum_settings']['L2']['phys_nets']
    physnet = phynet_setting.keys()[0]
    return physnet + ':' + bridge


def restart_services(service_name):
    execute('stop', service_name)
    execute('start', service_name)


def enable_linux_bridge(himn, username, password):
    # When using OVS under XS6.5, it will prevent use of Linux bridge in
    # Dom0, but neutron-openvswitch-agent in compute node will use Linux
    # bridge, so we remove this restriction here
    ssh(himn, username, password, 'rm -f /etc/modprobe.d/blacklist-bridge*')


def patch_compute_xenapi():
    """replace folder xenapi to add patches which are not merged to upstream"""
    # TODO(huanxie): need to confirm the overall patchset list
    patchset_dir = sys.path[0]
    patchfile_list = ['%s/patchset/vif-plug.patch' % patchset_dir,
            '%s/patchset/nova-neutron-race-condition.patch' % patchset_dir,
            '%s/patchset/ovs-interim-bridge.patch' % patchset_dir,
            '%s/patchset/neutron-security-group.patch' % patchset_dir,
            '%s/patchset/speed-up-writing-config-drive.patch' % patchset_dir]
    for patch_file in patchfile_list:
        execute('patch', '-d', DIST_PACKAGES_DIR, '-p1', '-i', patch_file)


def patch_neutron_ovs_agent():
    patchset_dir = sys.path[0]
    patch_file = '%s/patchset/neutron-rootwrap-xen-dom0.patch' % patchset_dir
    execute('patch', '-d', '/usr/', '-p1', '-i', patch_file)


def apply_sm_patch(himn, username, password):
    ver = ssh(himn, username, password,
              ('xe host-param-get uuid=$(xe host-list --minimal) '
               'param-name=software-version param-key=product_version_text'))
    if ver == "6.5":
        ssh(himn, username, password,
            "sed -i s/\\'phy\\'/\\'aio\\'/g /opt/xensource/sm/ISCSISR.py")


if __name__ == '__main__':
    install_xenapi_sdk()
    astute = get_astute(ASTUTE_PATH)
    if astute:
        username, password, install_xapi = get_options(astute, ASTUTE_SECTION)
        endpoints = get_endpoints(astute)
        himn_eth, himn_local = init_eth()

        public_ip = astute_get(
            astute, ('network_metadata', 'vips', 'public', 'ipaddr'))

        services_ssl = astute_get(
            astute, ('public_ssl', 'services'))

        if username and password and endpoints and himn_local:
            check_host_compatibility(HIMN_IP, username, password)
            route_to_compute(
                endpoints, HIMN_IP, himn_local, username, password)
            if install_xapi:
                install_suppack(HIMN_IP, username, password)
            enable_linux_bridge(HIMN_IP, username, password)
            forward_from_himn(himn_eth)

            # port forwarding for novnc
            forward_port('br-mgmt', himn_eth, HIMN_IP, '80')

            # apply sm patch
            apply_sm_patch(HIMN_IP, username, password)

            create_novacompute_conf(HIMN_IP, username, password, public_ip, services_ssl)
            patch_compute_xenapi()
            restart_services('nova-compute')

            install_logrotate_script(HIMN_IP, username, password)

            # neutron-l2-agent in compute node
            modify_neutron_rootwrap_conf(HIMN_IP, username, password)
            br_mappings = find_bridge_mappings(astute, HIMN_IP,
                                               username, password)
            modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
            patch_neutron_ovs_agent()
            restart_services('neutron-plugin-openvswitch-agent')