433 lines
16 KiB
Python
433 lines
16 KiB
Python
# Copyright 2014 DreamHost, LLC
|
|
#
|
|
# Author: DreamHost, LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
|
|
import logging
|
|
|
|
import mock
|
|
import unittest2 as unittest
|
|
from datetime import datetime, timedelta
|
|
|
|
from akanda.rug import vm_manager
|
|
|
|
vm_manager.RETRY_DELAY = 0.4
|
|
vm_manager.BOOT_WAIT = 1
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class TestVmManager(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.ctx = mock.Mock()
|
|
self.quantum = self.ctx.neutron
|
|
self.conf = mock.patch.object(vm_manager.cfg, 'CONF').start()
|
|
self.conf.boot_timeout = 1
|
|
self.conf.akanda_mgt_service_port = 5000
|
|
self.conf.max_retries = 3
|
|
self.addCleanup(mock.patch.stopall)
|
|
|
|
self.log = mock.Mock()
|
|
self.update_state_p = mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'update_state'
|
|
)
|
|
|
|
self.mock_update_state = self.update_state_p.start()
|
|
self.vm_mgr = vm_manager.VmManager('the_id', 'tenant_id',
|
|
self.log, self.ctx)
|
|
mock.patch.object(self.vm_mgr, '_ensure_cache', mock.Mock)
|
|
|
|
self.next_state = None
|
|
|
|
def next_state(*args, **kwargs):
|
|
if self.next_state:
|
|
self.vm_mgr.state = self.next_state
|
|
return self.vm_mgr.state
|
|
self.mock_update_state.side_effect = next_state
|
|
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_update_state_is_alive(self, get_mgt_addr, router_api):
|
|
self.update_state_p.stop()
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
router_api.is_alive.return_value = True
|
|
|
|
self.assertEqual(self.vm_mgr.update_state(self.ctx), vm_manager.UP)
|
|
router_api.is_alive.assert_called_once_with('fe80::beef', 5000)
|
|
|
|
@mock.patch('time.sleep')
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_boot_timeout_still_booting(self, get_mgt_addr, router_api, sleep):
|
|
self.vm_mgr.last_boot = datetime.utcnow()
|
|
self.update_state_p.stop()
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
router_api.is_alive.return_value = False
|
|
|
|
self.assertEqual(
|
|
self.vm_mgr.update_state(self.ctx),
|
|
vm_manager.BOOTING
|
|
)
|
|
router_api.is_alive.assert_has_calls([
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000)
|
|
])
|
|
|
|
@mock.patch('time.sleep')
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_boot_timeout(self, get_mgt_addr, router_api, sleep):
|
|
self.vm_mgr.last_boot = datetime.utcnow() - timedelta(minutes=5)
|
|
self.update_state_p.stop()
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
router_api.is_alive.return_value = False
|
|
|
|
self.assertEqual(self.vm_mgr.update_state(self.ctx), vm_manager.DOWN)
|
|
router_api.is_alive.assert_has_calls([
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000)
|
|
])
|
|
self.vm_mgr.log.info.assert_called_once_with(
|
|
mock.ANY,
|
|
self.conf.boot_timeout
|
|
)
|
|
|
|
@mock.patch('time.sleep')
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_update_state_is_down(self, get_mgt_addr, router_api, sleep):
|
|
self.update_state_p.stop()
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
router_api.is_alive.return_value = False
|
|
|
|
self.assertEqual(self.vm_mgr.update_state(self.ctx), vm_manager.DOWN)
|
|
router_api.is_alive.assert_has_calls([
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000),
|
|
mock.call('fe80::beef', 5000)
|
|
])
|
|
|
|
@mock.patch('time.sleep')
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_update_state_retry_delay(self, get_mgt_addr, router_api, sleep):
|
|
self.update_state_p.stop()
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
router_api.is_alive.side_effect = [False, False, True]
|
|
max_retries = 5
|
|
self.conf.max_retries = max_retries
|
|
self.vm_mgr.update_state(self.ctx, silent=False)
|
|
self.assertEqual(sleep.call_count, 2)
|
|
self.log.debug.assert_has_calls([
|
|
mock.call('Alive check failed. Attempt %d of %d', 0, max_retries),
|
|
mock.call('Alive check failed. Attempt %d of %d', 1, max_retries)
|
|
])
|
|
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_update_state_no_mgt_port(self, get_mgt_addr):
|
|
with mock.patch.object(self.ctx.neutron, 'get_router_detail') as grd:
|
|
r = mock.Mock()
|
|
r.management_port = None
|
|
grd.return_value = r
|
|
get_mgt_addr.side_effect = AssertionError('Should never be called')
|
|
self.update_state_p.stop()
|
|
self.assertEqual(self.vm_mgr.update_state(self.ctx),
|
|
vm_manager.DOWN)
|
|
|
|
@mock.patch('time.sleep')
|
|
def test_boot_success(self, sleep):
|
|
self.next_state = vm_manager.UP
|
|
rtr = mock.sentinel.router
|
|
self.ctx.neutron.get_router_detail.return_value = rtr
|
|
rtr.id = 'ROUTER1'
|
|
rtr.management_port = None
|
|
rtr.external_port = None
|
|
rtr.ports = mock.MagicMock()
|
|
rtr.ports.__iter__.return_value = []
|
|
self.vm_mgr.boot(self.ctx)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.DOWN) # async
|
|
self.ctx.nova_client.reboot_router_instance.assert_called_once_with(
|
|
self.vm_mgr.router_obj
|
|
)
|
|
|
|
@mock.patch('time.sleep')
|
|
def test_boot_fail(self, sleep):
|
|
self.next_state = vm_manager.DOWN
|
|
rtr = mock.sentinel.router
|
|
self.ctx.neutron.get_router_detail.return_value = rtr
|
|
rtr.id = 'ROUTER1'
|
|
rtr.management_port = None
|
|
rtr.external_port = None
|
|
rtr.ports = mock.MagicMock()
|
|
rtr.ports.__iter__.return_value = []
|
|
self.vm_mgr.boot(self.ctx)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.DOWN)
|
|
self.ctx.nova_client.reboot_router_instance.assert_called_once_with(
|
|
self.vm_mgr.router_obj
|
|
)
|
|
|
|
@mock.patch('time.sleep')
|
|
def test_boot_with_port_cleanup(self, sleep):
|
|
self.next_state = vm_manager.UP
|
|
|
|
management_port = mock.Mock(id='mgmt', device_id='INSTANCE1')
|
|
external_port = mock.Mock(id='ext', device_id='INSTANCE1')
|
|
internal_port = mock.Mock(id='int', device_id='INSTANCE1')
|
|
|
|
rtr = mock.sentinel.router
|
|
instance = mock.sentinel.instance
|
|
self.ctx.neutron.get_router_detail.return_value = rtr
|
|
self.ctx.nova_client.get_instance.return_value = instance
|
|
rtr.id = 'ROUTER1'
|
|
instance.id = 'INSTANCE1'
|
|
rtr.management_port = management_port
|
|
rtr.external_port = external_port
|
|
rtr.ports = mock.MagicMock()
|
|
rtr.ports.__iter__.return_value = [management_port, external_port,
|
|
internal_port]
|
|
self.vm_mgr.boot(self.ctx)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.DOWN) # async
|
|
self.ctx.nova_client.reboot_router_instance.assert_called_once_with(
|
|
self.vm_mgr.router_obj
|
|
)
|
|
assert self.ctx.neutron.clear_device_id.call_count == 3
|
|
self.ctx.neutron.clear_device_id.assert_has_calls([
|
|
mock.call(management_port),
|
|
mock.call(external_port),
|
|
mock.call(internal_port)
|
|
], any_order=True)
|
|
|
|
def test_boot_check_up(self):
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'update_state'
|
|
) as update_state:
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'configure'
|
|
) as configure:
|
|
update_state.return_value = vm_manager.UP
|
|
configure.side_effect = lambda *a, **kw: setattr(
|
|
self.vm_mgr,
|
|
'state',
|
|
vm_manager.CONFIGURED
|
|
)
|
|
assert self.vm_mgr.check_boot(self.ctx) is True
|
|
update_state.assert_called_once_with(self.ctx, silent=True)
|
|
configure.assert_called_once_with(
|
|
self.ctx,
|
|
vm_manager.BOOTING,
|
|
attempts=1
|
|
)
|
|
|
|
def test_boot_check_configured(self):
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'update_state'
|
|
) as update_state:
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'configure'
|
|
) as configure:
|
|
update_state.return_value = vm_manager.CONFIGURED
|
|
configure.side_effect = lambda *a, **kw: setattr(
|
|
self.vm_mgr,
|
|
'state',
|
|
vm_manager.CONFIGURED
|
|
)
|
|
assert self.vm_mgr.check_boot(self.ctx) is True
|
|
update_state.assert_called_once_with(self.ctx, silent=True)
|
|
configure.assert_called_once_with(
|
|
self.ctx,
|
|
vm_manager.BOOTING,
|
|
attempts=1
|
|
)
|
|
|
|
def test_boot_check_still_booting(self):
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'update_state'
|
|
) as update_state:
|
|
update_state.return_value = vm_manager.BOOTING
|
|
assert self.vm_mgr.check_boot(self.ctx) is False
|
|
update_state.assert_called_once_with(self.ctx, silent=True)
|
|
|
|
def test_boot_check_unsuccessful_initial_config_update(self):
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'update_state'
|
|
) as update_state:
|
|
with mock.patch.object(
|
|
vm_manager.VmManager,
|
|
'configure'
|
|
) as configure:
|
|
update_state.return_value = vm_manager.CONFIGURED
|
|
configure.side_effect = lambda *a, **kw: setattr(
|
|
self.vm_mgr,
|
|
'state',
|
|
vm_manager.BOOTING
|
|
)
|
|
assert self.vm_mgr.check_boot(self.ctx) is False
|
|
update_state.assert_called_once_with(self.ctx, silent=True)
|
|
configure.assert_called_once_with(
|
|
self.ctx,
|
|
vm_manager.BOOTING,
|
|
attempts=1
|
|
)
|
|
|
|
@mock.patch('time.sleep')
|
|
def test_stop_success(self, sleep):
|
|
self.vm_mgr.state = vm_manager.UP
|
|
self.ctx.nova_client.get_router_instance_status.return_value = None
|
|
self.vm_mgr.stop(self.ctx)
|
|
self.ctx.nova_client.destroy_router_instance.assert_called_once_with(
|
|
self.vm_mgr.router_obj
|
|
)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.DOWN)
|
|
|
|
@mock.patch('time.sleep')
|
|
def test_stop_fail(self, sleep):
|
|
self.vm_mgr.state = vm_manager.UP
|
|
self.ctx.nova_client.get_router_instance_status.return_value = 'UP'
|
|
self.vm_mgr.stop(self.ctx)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.UP)
|
|
self.ctx.nova_client.destroy_router_instance.assert_called_once_with(
|
|
self.vm_mgr.router_obj
|
|
)
|
|
self.log.error.assert_called_once_with(mock.ANY, 1)
|
|
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
@mock.patch('akanda.rug.api.configuration.build_config')
|
|
def test_configure_success(self, config, get_mgt_addr, router_api):
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
rtr = mock.sentinel.router
|
|
|
|
self.ctx.neutron.get_router_detail.return_value = rtr
|
|
|
|
with mock.patch.object(self.vm_mgr, '_verify_interfaces') as verify:
|
|
verify.return_value = True
|
|
self.vm_mgr.configure(self.ctx)
|
|
|
|
interfaces = router_api.get_interfaces.return_value
|
|
|
|
verify.assert_called_once_with(rtr, interfaces)
|
|
config.assert_called_once_with(self.ctx.neutron, rtr, interfaces)
|
|
router_api.update_config.assert_called_once_with(
|
|
'fe80::beef',
|
|
5000,
|
|
config.return_value
|
|
)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.CONFIGURED)
|
|
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
def test_configure_mismatched_interfaces(self, get_mgt_addr, router_api):
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
rtr = mock.sentinel.router
|
|
|
|
self.quantum.get_router_detail.return_value = rtr
|
|
|
|
with mock.patch.object(self.vm_mgr, '_verify_interfaces') as verify:
|
|
verify.return_value = False
|
|
self.vm_mgr.configure(self.ctx)
|
|
|
|
interfaces = router_api.get_interfaces.return_value
|
|
|
|
verify.assert_called_once_with(rtr, interfaces)
|
|
|
|
self.assertFalse(router_api.update_config.called)
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.RESTART)
|
|
|
|
@mock.patch('time.sleep')
|
|
@mock.patch('akanda.rug.vm_manager.router_api')
|
|
@mock.patch('akanda.rug.vm_manager._get_management_address')
|
|
@mock.patch('akanda.rug.api.configuration.build_config')
|
|
def test_configure_failure(self, config, get_mgt_addr, router_api, sleep):
|
|
get_mgt_addr.return_value = 'fe80::beef'
|
|
rtr = {'id': 'the_id'}
|
|
|
|
self.quantum.get_router_detail.return_value = rtr
|
|
|
|
router_api.update_config.side_effect = Exception
|
|
|
|
with mock.patch.object(self.vm_mgr, '_verify_interfaces') as verify:
|
|
verify.return_value = True
|
|
self.vm_mgr.configure(self.ctx)
|
|
|
|
interfaces = router_api.get_interfaces.return_value
|
|
|
|
verify.assert_called_once_with(rtr, interfaces)
|
|
config.assert_called_once_with(self.quantum, rtr, interfaces)
|
|
router_api.update_config.assert_has_calls([
|
|
mock.call('fe80::beef', 5000, config.return_value),
|
|
mock.call('fe80::beef', 5000, config.return_value),
|
|
mock.call('fe80::beef', 5000, config.return_value),
|
|
])
|
|
self.assertEqual(self.vm_mgr.state, vm_manager.RESTART)
|
|
|
|
def test_verify_interfaces(self):
|
|
rtr = mock.Mock()
|
|
rtr.management_port.mac_address = 'a:b:c:d'
|
|
rtr.external_port.mac_address = 'd:c:b:a'
|
|
p = mock.Mock()
|
|
p.mac_address = 'a:a:a:a'
|
|
rtr.internal_ports = [p]
|
|
rtr.ports = [p, rtr.management_port, rtr.external_port]
|
|
|
|
interfaces = [
|
|
{'lladdr': 'a:b:c:d'},
|
|
{'lladdr': 'd:c:b:a'},
|
|
{'lladdr': 'a:a:a:a'}
|
|
]
|
|
|
|
self.assertTrue(self.vm_mgr._verify_interfaces(rtr, interfaces))
|
|
|
|
def test_verify_interfaces_with_cleared_gateway(self):
|
|
rtr = mock.Mock()
|
|
rtr.management_port = mock.MagicMock(spec=[])
|
|
rtr.external_port.mac_address = 'd:c:b:a'
|
|
p = mock.Mock()
|
|
p.mac_address = 'a:a:a:a'
|
|
rtr.internal_ports = [p]
|
|
rtr.ports = [p, rtr.management_port, rtr.external_port]
|
|
|
|
interfaces = [
|
|
{'lladdr': 'a:b:c:d'},
|
|
{'lladdr': 'd:c:b:a'},
|
|
{'lladdr': 'a:a:a:a'}
|
|
]
|
|
|
|
self.assertFalse(self.vm_mgr._verify_interfaces(rtr, interfaces))
|
|
|
|
def test_ensure_provider_ports(self):
|
|
rtr = mock.Mock()
|
|
rtr.id = 'id'
|
|
rtr.management_port = None
|
|
rtr.external_port = None
|
|
|
|
self.vm_mgr._ensure_provider_ports(rtr, self.ctx)
|
|
self.quantum.create_router_management_port.assert_called_once_with(
|
|
'id'
|
|
)
|
|
|
|
self.assertEqual(self.vm_mgr._ensure_provider_ports(rtr, self.ctx),
|
|
rtr)
|
|
self.quantum.create_router_external_port.assert_called_once_with(rtr)
|