# Copyright 2015 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import random import time import mock from oslo_config import cfg from oslo_config import fixture as oslo_fixture from oslo_utils import uuidutils import six import sqlalchemy from octavia.common import constants from octavia.common import data_models from octavia.controller.healthmanager.health_drivers import update_db from octavia.db import models as db_models from octavia.tests.unit import base class TestException(Exception): def __init__(self, value): self.value = value def __str__(self): return repr(self.value) class TestUpdateHealthDb(base.TestCase): FAKE_UUID_1 = uuidutils.generate_uuid() def setUp(self): super(TestUpdateHealthDb, self).setUp() self.conf = self.useFixture(oslo_fixture.Config(cfg.CONF)) self.conf.config(group="health_manager", event_streamer_driver='queue_event_streamer') session_patch = mock.patch('octavia.db.api.get_session') self.addCleanup(session_patch.stop) self.mock_session = session_patch.start() self.session_mock = mock.MagicMock() self.mock_session.return_value = self.session_mock self.hm = update_db.UpdateHealthDb() self.event_client = mock.MagicMock() self.hm.event_streamer.client = self.event_client self.amphora_repo = mock.MagicMock() self.amphora_health_repo = mock.MagicMock() self.listener_repo = mock.MagicMock() self.loadbalancer_repo = mock.MagicMock() self.member_repo = mock.MagicMock() self.pool_repo = mock.MagicMock() self.hm.amphora_repo = self.amphora_repo self.hm.amphora_health_repo = self.amphora_health_repo self.hm.listener_repo = self.listener_repo self.hm.listener_repo.count.return_value = 1 self.hm.loadbalancer_repo = self.loadbalancer_repo self.hm.member_repo = self.member_repo self.hm.pool_repo = self.pool_repo def _make_mock_lb_tree(self, listener=True, pool=True, health_monitor=True, members=1, lb_prov_status=constants.ACTIVE): mock_lb = mock.Mock() mock_lb.id = self.FAKE_UUID_1 mock_lb.pools = [] mock_lb.listeners = [] mock_lb.provisioning_status = lb_prov_status mock_lb.operating_status = 'blah' mock_listener1 = None mock_pool1 = None mock_members = None if listener: mock_listener1 = mock.Mock() mock_listener1.id = 'listener-id-1' mock_lb.listeners = [mock_listener1] if pool: mock_pool1 = mock.Mock() mock_pool1.id = "pool-id-1" mock_pool1.members = [] if health_monitor: mock_hm1 = mock.Mock() mock_pool1.health_monitor = mock_hm1 else: mock_pool1.health_monitor = None mock_lb.pools = [mock_pool1] if mock_listener1: mock_listener1.pools = [mock_pool1] mock_listener1.default_pool = mock_pool1 for i in range(members): mock_member_x = mock.Mock() mock_member_x.id = 'member-id-%s' % (i + 1) if health_monitor: mock_member_x.operating_status = 'NOTHING_MATCHABLE' else: mock_member_x.operating_status = constants.NO_MONITOR mock_pool1.members.append(mock_member_x) mock_members = mock_pool1.members return mock_lb, mock_listener1, mock_pool1, mock_members def _make_fake_lb_health_dict(self, listener=True, pool=True, health_monitor=True, members=1, lb_prov_status=constants.ACTIVE): lb_ref = {'enabled': True, 'id': self.FAKE_UUID_1, constants.OPERATING_STATUS: 'bogus', constants.PROVISIONING_STATUS: lb_prov_status} if pool: members_dict = {} if health_monitor: member_operating_status = 'NOTHING_MATCHABLE' else: member_operating_status = constants.NO_MONITOR for i in range(members): member_id = 'member-id-%s' % (i + 1) members_dict[member_id] = { constants.OPERATING_STATUS: member_operating_status} pool_ref = {'pool-id-1': {'members': members_dict, constants.OPERATING_STATUS: 'bogus'}} lb_ref['pools'] = pool_ref if listener: listener_ref = {'listener-id-1': { constants.OPERATING_STATUS: 'bogus'}} lb_ref['listeners'] = listener_ref return lb_ref def test_update_health_event_stream(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP, "member-id-2": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.event_client.cast.assert_any_call( {}, 'update_info', container={ 'info_type': 'listener', 'info_id': 'listener-id-1', 'info_payload': {'operating_status': 'ONLINE'}}) self.event_client.cast.assert_any_call( {}, 'update_info', container={ 'info_type': 'member', 'info_id': 'member-id-1', 'info_payload': {'operating_status': 'ONLINE'}}) self.event_client.cast.assert_any_call( {}, 'update_info', container={ 'info_type': 'pool', 'info_id': 'pool-id-1', 'info_payload': {'operating_status': 'ONLINE'}}) def test_update_health_no_listener(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_lb_pending_no_listener(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict( listener=True, pool=False, lb_prov_status=constants.PENDING_UPDATE) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_health_missing_listener(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(listener=True, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.assertFalse(self.amphora_health_repo.replace.called) def test_update_health_recv_time_stale(self): hb_interval = cfg.CONF.health_manager.heartbeat_interval health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() - hb_interval - 1 # extra -1 for buffer } lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) # Receive time is stale, so we shouldn't see this called self.assertFalse(self.loadbalancer_repo.update.called) def test_update_health_replace_error(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } self.session_mock.commit.side_effect = TestException('boom') lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) self.session_mock.rollback.assert_called_once() def test_update_health_online(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) # If the listener count is wrong, make sure we don't update lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus'} self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health, '192.0.2.1') self.assertTrue(not self.amphora_health_repo.replace.called) def test_update_lb_pool_health_offline(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": {}} }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) self.pool_repo.update.assert_any_call( self.session_mock, 'pool-id-1', operating_status=constants.OFFLINE ) def test_update_lb_multiple_listeners_one_error_pool(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.ERROR}} }}, "listener-id-2": {"status": constants.OPEN, "pools": { "pool-id-2": {"status": constants.UP, "members": {"member-id-2": constants.UP}} }} }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = { constants.OPERATING_STATUS: 'bogus', 'members': {'member-id-2': {constants.OPERATING_STATUS: 'bogus'}}} lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus'} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners')): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) # Call count should be exactly 2, as each pool should be processed once self.assertEqual(2, self.pool_repo.update.call_count) self.pool_repo.update.assert_has_calls([ mock.call(self.session_mock, 'pool-id-1', operating_status=constants.ERROR), mock.call(self.session_mock, 'pool-id-2', operating_status=constants.ONLINE) ], any_order=True) def test_update_lb_and_list_pool_health_online(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): # We should not double process a shared pool self.hm.pool_repo.update.assert_called_once_with( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) def test_update_pool_offline(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-5": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = { constants.OPERATING_STATUS: constants.OFFLINE} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) self.hm.pool_repo.update.assert_any_call( self.session_mock, "pool-id-1", operating_status=constants.OFFLINE) def test_update_health_member_drain(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.DRAIN}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.DRAINING) def test_update_health_member_maint(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.MAINT}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.OFFLINE) def test_update_health_member_unknown(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": "blah"}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.assertTrue(not self.member_repo.update.called) def test_update_health_member_down(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.DEGRADED) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) def test_update_health_member_missing_no_hm(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(health_monitor=False) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.member_repo.update.assert_not_called() def test_update_health_member_down_no_hm(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.MAINT} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict(health_monitor=False) member1 = lb_ref['pools']['pool-id-1']['members']['member-id-1'] member1[constants.OPERATING_STATUS] = constants.NO_MONITOR self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) self.member_repo.update.assert_any_call( self.session_mock, 'member-id-1', operating_status=constants.OFFLINE) def test_update_health_member_no_check(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.NO_CHECK} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.NO_MONITOR) def test_update_health_member_admin_down(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": { "member-id-1": constants.UP}}}}}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict(members=2) self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) self.member_repo.update.assert_any_call( self.session_mock, 'member-id-2', operating_status=constants.OFFLINE) def test_update_health_list_full_member_down(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.FULL, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.DEGRADED) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.DEGRADED) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) lb_ref['listeners']['listener-id-2'] = { constants.OPERATING_STATUS: 'bogus'} self.amphora_health_repo.replace.reset_mock() self.hm.update_health(health, '192.0.2.1') self.assertTrue(not self.amphora_health_repo.replace.called) def test_update_health_error(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.DOWN} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ERROR) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ERROR) # Test the logic code paths def test_update_health_full(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.FULL, "pools": { "pool-id-1": {"status": constants.DOWN, "members": {"member-id-1": constants.DOWN} } } }, "listener-id-2": {"status": constants.FULL, "pools": { "pool-id-2": {"status": constants.UP, "members": {"member-id-2": constants.UP} } } }, "listener-id-3": {"status": constants.OPEN, "pools": { "pool-id-3": {"status": constants.UP, "members": {"member-id-3": constants.UP, "member-id-31": constants.DOWN} } } }, "listener-id-4": { "status": constants.OPEN, "pools": { "pool-id-4": { "status": constants.UP, "members": {"member-id-4": constants.DRAINING} } } }, "listener-id-5": { "status": "bogus", "pools": { "pool-id-5": { "status": "bogus", "members": {"member-id-5": "bogus"} } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() # Build our own custom listeners/pools/members for i in [1, 2, 3, 4, 5]: lb_ref['listeners']['listener-id-%s' % i] = { constants.OPERATING_STATUS: 'bogus'} if i == 3: members_dict = {'member-id-3': { constants.OPERATING_STATUS: 'bogus'}, 'member-id-31': { constants.OPERATING_STATUS: 'bogus'}} else: members_dict = {'member-id-%s' % i: { constants.OPERATING_STATUS: 'bogus'}} lb_ref['pools']['pool-id-%s' % i] = { 'members': members_dict, constants.OPERATING_STATUS: 'bogus'} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') # test listener self.listener_repo.update.assert_any_call( self.session_mock, "listener-id-1", operating_status=constants.DEGRADED) self.listener_repo.update.assert_any_call( self.session_mock, "listener-id-2", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-1", operating_status=constants.ERROR) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-2", operating_status=constants.ONLINE) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-3", operating_status=constants.DEGRADED) self.pool_repo.update.assert_any_call( self.session_mock, "pool-id-4", operating_status=constants.ONLINE) # Test code paths where objects are not found in the database def test_update_health_not_found(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": {"status": constants.OPEN, "pools": { "pool-id-1": {"status": constants.UP, "members": {"member-id-1": constants.UP} } } } }, "recv_time": time.time() } self.hm.listener_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) self.hm.member_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) self.hm.pool_repo.update.side_effect = ( sqlalchemy.orm.exc.NoResultFound) self.hm.loadbalancer_repo.update.side_effect = ( [sqlalchemy.orm.exc.NoResultFound]) lb_ref = self._make_fake_lb_health_dict() lb_ref['pools']['pool-id-2'] = {constants.OPERATING_STATUS: 'bogus'} self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_health_repo.replace.called) # test listener, member for listener_id, listener in six.iteritems( health.get('listeners', {})): self.listener_repo.update.assert_any_call( self.session_mock, listener_id, operating_status=constants.ONLINE) for pool_id, pool in six.iteritems(listener.get('pools', {})): self.hm.pool_repo.update.assert_any_call( self.session_mock, pool_id, operating_status=constants.ONLINE) for member_id, member in six.iteritems( pool.get('members', {})): self.member_repo.update.assert_any_call( self.session_mock, member_id, operating_status=constants.ONLINE) @mock.patch('stevedore.driver.DriverManager.driver') def test_update_health_zombie(self, mock_driver): health = {"id": self.FAKE_UUID_1, "listeners": {}} self.amphora_repo.get_lb_for_health_update.return_value = None amp_mock = mock.MagicMock() self.amphora_repo.get.return_value = amp_mock self.hm.update_health(health, '192.0.2.1') mock_driver.delete.assert_called_once_with( amp_mock.compute_id) def test_update_health_no_status_change(self): health = { "id": self.FAKE_UUID_1, "listeners": { "listener-id-1": { "status": constants.OPEN, "pools": { "pool-id-1": { "status": constants.UP, "members": { "member-id-1": constants.UP } } } } }, "recv_time": time.time() } lb_ref = self._make_fake_lb_health_dict() # Start everything ONLINE lb_ref[constants.OPERATING_STATUS] = constants.ONLINE listener1 = lb_ref['listeners']['listener-id-1'] listener1[constants.OPERATING_STATUS] = constants.ONLINE pool1 = lb_ref['pools']['pool-id-1'] pool1[constants.OPERATING_STATUS] = constants.ONLINE member1 = pool1['members']['member-id-1'] member1[constants.OPERATING_STATUS] = constants.ONLINE self.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.event_client.cast.assert_not_called() self.loadbalancer_repo.update.assert_not_called() self.listener_repo.update.assert_not_called() self.pool_repo.update.assert_not_called() self.member_repo.update.assert_not_called() def test_update_health_lb_admin_down(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time()} lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) lb_ref['enabled'] = False self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), self.FAKE_UUID_1, operating_status='OFFLINE') def test_update_health_lb_admin_up(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time(), "ver": 1} lb_ref = self._make_fake_lb_health_dict(listener=False, pool=False) self.hm.amphora_repo.get_lb_for_health_update.return_value = lb_ref self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.loadbalancer_repo.update.called) self.loadbalancer_repo.update.assert_called_with( self.mock_session(), self.FAKE_UUID_1, operating_status='ONLINE') def test_update_health_no_db_lb(self): health = { "id": self.FAKE_UUID_1, "listeners": {}, "recv_time": time.time() } self.hm.amphora_repo.get_lb_for_health_update.return_value = {} self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertFalse(self.amphora_health_repo.replace.called) # Test missing amp in addition to missing lb DB record self.amphora_repo.get_lb_for_health_update.reset_mock() self.amphora_health_repo.replace.reset_mock() mock_amphora = mock.MagicMock() mock_amphora.load_balancer_id = None self.amphora_repo.get.return_value = mock_amphora self.hm.update_health(health, '192.0.2.1') self.assertTrue(self.amphora_repo.get_lb_for_health_update.called) self.assertTrue(self.amphora_repo.get.called) self.assertTrue(self.amphora_health_repo.replace.called) def test_update_status_and_emit_event(self): # Test update with the same operating status self.conf.config(group="health_manager", event_streamer_driver=constants.NOOP_EVENT_STREAMER) self.hm._update_status_and_emit_event( 'fake_session', self.loadbalancer_repo, constants.LOADBALANCER, 1, 'ONLINE', 'ONLINE') self.assertFalse(self.loadbalancer_repo.update.called) self.assertFalse(self.event_client.cast.called) self.conf.config(group="health_manager", event_streamer_driver='queue_event_streamer', sync_provisioning_status=True) self.loadbalancer_repo.update.reset_mock() self.event_client.reset_mock() # Test stream with provisioning sync self.hm._update_status_and_emit_event( 'fake_session', self.loadbalancer_repo, constants.LOADBALANCER, 1, 'ONLINE', 'OFFLINE') self.assertTrue(self.loadbalancer_repo.update.called) self.assertTrue(self.event_client.cast.called) self.conf.config(group="health_manager", sync_provisioning_status=False) self.loadbalancer_repo.update.reset_mock() self.event_client.reset_mock() # Test stream with no provisioning sync self.hm._update_status_and_emit_event( 'fake_session', self.loadbalancer_repo, constants.LOADBALANCER, 1, 'ONLINE', 'ONLINE') self.assertFalse(self.loadbalancer_repo.update.called) self.assertFalse(self.event_client.cast.called) class TestUpdateStatsDb(base.TestCase): def setUp(self): super(TestUpdateStatsDb, self).setUp() self.conf = self.useFixture(oslo_fixture.Config(cfg.CONF)) self.conf.config(group="health_manager", event_streamer_driver='queue_event_streamer') self.sm = update_db.UpdateStatsDb() self.event_client = mock.MagicMock() self.sm.event_streamer.client = self.event_client self.listener_stats_repo = mock.MagicMock() self.sm.listener_stats_repo = self.listener_stats_repo self.loadbalancer_id = uuidutils.generate_uuid() self.listener_id = uuidutils.generate_uuid() self.listener_stats = data_models.ListenerStatistics( listener_id=self.listener_id, bytes_in=random.randrange(1000000000), bytes_out=random.randrange(1000000000), active_connections=random.randrange(1000000000), total_connections=random.randrange(1000000000), request_errors=random.randrange(1000000000)) self.sm.get_listener_stats = mock.MagicMock() self.sm.get_listener_stats.return_value = self.listener_stats self.loadbalancer_id = uuidutils.generate_uuid() self.amphora_id = uuidutils.generate_uuid() self.listener_id = uuidutils.generate_uuid() self.listener = db_models.Listener( load_balancer_id=self.loadbalancer_id) self.listener_repo = mock.MagicMock() self.sm.repo_listener = self.listener_repo self.sm.repo_listener.get.return_value = self.listener self.loadbalancer_repo = mock.MagicMock() self.sm.repo_loadbalancer = self.loadbalancer_repo self.loadbalancer = db_models.LoadBalancer( id=self.loadbalancer_id, listeners=[self.listener]) self.loadbalancer_repo.get.return_value = self.loadbalancer @mock.patch('octavia.db.api.get_session') def test_update_stats(self, mock_session): health = { "id": self.amphora_id, "listeners": { self.listener_id: { "status": constants.OPEN, "stats": { "ereq": self.listener_stats.request_errors, "conns": self.listener_stats.active_connections, "totconns": self.listener_stats.total_connections, "rx": self.listener_stats.bytes_in, "tx": self.listener_stats.bytes_out, }, "pools": { "pool-id-1": { "status": constants.UP, "members": {"member-id-1": constants.ONLINE} } } } } } mock_session.return_value = 'blah' self.sm.update_stats(health, '192.0.2.1') self.listener_stats_repo.replace.assert_called_once_with( 'blah', self.listener_id, self.amphora_id, bytes_in=self.listener_stats.bytes_in, bytes_out=self.listener_stats.bytes_out, active_connections=self.listener_stats.active_connections, total_connections=self.listener_stats.total_connections, request_errors=self.listener_stats.request_errors) self.event_client.cast.assert_any_call( {}, 'update_info', container={ 'info_type': 'listener_stats', 'info_id': self.listener_id, 'info_payload': { 'bytes_in': self.listener_stats.bytes_in, 'total_connections': self.listener_stats.total_connections, 'active_connections': self.listener_stats.active_connections, 'bytes_out': self.listener_stats.bytes_out, 'request_errors': self.listener_stats.request_errors}}) self.event_client.cast.assert_any_call( {}, 'update_info', container={ 'info_type': 'loadbalancer_stats', 'info_id': self.loadbalancer_id, 'info_payload': { 'bytes_in': self.listener_stats.bytes_in, 'total_connections': self.listener_stats.total_connections, 'active_connections': self.listener_stats.active_connections, 'bytes_out': self.listener_stats.bytes_out, 'request_errors': self.listener_stats.request_errors}}) # Test with noop streamer self.event_client.cast.reset_mock() self.conf.config(group="health_manager", event_streamer_driver=constants.NOOP_EVENT_STREAMER) self.sm.update_stats(health, '192.0.2.1') self.conf.config(group="health_manager", event_streamer_driver='queue_event_streamer') self.assertFalse(self.event_client.cast.called) # Test with missing DB listener self.event_client.cast.reset_mock() self.sm.repo_listener.get.return_value = None self.sm.update_stats(health, '192.0.2.1') self.event_client.cast.assert_called_once_with( {}, 'update_info', container={ 'info_type': 'listener_stats', 'info_id': self.listener_id, 'info_payload': { 'bytes_in': self.listener_stats.bytes_in, 'total_connections': self.listener_stats.total_connections, 'active_connections': self.listener_stats.active_connections, 'bytes_out': self.listener_stats.bytes_out, 'request_errors': self.listener_stats.request_errors}}) # Test with update failure self.event_client.cast.reset_mock() mock_session.side_effect = Exception self.sm.update_stats(health, '192.0.2.1') self.assertFalse(self.event_client.cast.called)