From c483fcdf93e96772ad840dd13f2a2944dab1d323 Mon Sep 17 00:00:00 2001 From: xiaodongwang Date: Thu, 23 Oct 2014 08:44:28 -0700 Subject: [PATCH] move reset PROXY env from regtest.conf to regtest.sh Change-Id: I36131c433c1c30cb8a2fbc5ac60100de5ac774c5 --- compass/hdsdiscovery/hdmanager.py | 5 +- compass/tests/hdsdiscovery/__init__.py | 13 + compass/tests/hdsdiscovery/test_base.py | 129 ++++ .../tests/hdsdiscovery/test_hdsdiscovery.py | 246 +++++++ compass/tests/hdsdiscovery/test_utils.py | 105 +++ compass/tests/log_analyzor/data/config | 97 +++ .../log_analyzor/test_adapter_matcher.py | 629 ++++++++++++++++++ .../tests/log_analyzor/test_file_matcher.py | 436 ++++++++++++ .../tests/log_analyzor/test_line_matcher.py | 229 +++++++ regtest/regtest.conf | 3 - regtest/regtest.sh | 4 + 11 files changed, 1891 insertions(+), 5 deletions(-) create mode 100644 compass/tests/hdsdiscovery/__init__.py create mode 100755 compass/tests/hdsdiscovery/test_base.py create mode 100755 compass/tests/hdsdiscovery/test_hdsdiscovery.py create mode 100755 compass/tests/hdsdiscovery/test_utils.py create mode 100644 compass/tests/log_analyzor/data/config create mode 100755 compass/tests/log_analyzor/test_adapter_matcher.py create mode 100755 compass/tests/log_analyzor/test_file_matcher.py create mode 100755 compass/tests/log_analyzor/test_line_matcher.py diff --git a/compass/hdsdiscovery/hdmanager.py b/compass/hdsdiscovery/hdmanager.py index 81108141..38a19dbc 100644 --- a/compass/hdsdiscovery/hdmanager.py +++ b/compass/hdsdiscovery/hdmanager.py @@ -102,9 +102,10 @@ class HDManager(object): """ switch_lists = util.load_configs(setting.SWITCH_LIST_DIR) - switch_list = None + switch_list = [] for item in switch_lists: - switch_list = item['SWITCH_LIST'] + if item and 'SWITCH_LIST' in item and item['SWITCH_LIST']: + switch_list.extend(item['SWITCH_LIST']) if host in switch_list: return ("appliance", "Found", "") diff --git a/compass/tests/hdsdiscovery/__init__.py b/compass/tests/hdsdiscovery/__init__.py new file mode 100644 index 00000000..4ee55a4c --- /dev/null +++ b/compass/tests/hdsdiscovery/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/compass/tests/hdsdiscovery/test_base.py b/compass/tests/hdsdiscovery/test_base.py new file mode 100755 index 00000000..be096f9c --- /dev/null +++ b/compass/tests/hdsdiscovery/test_base.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test hdsdiscovery base module.""" +import os +import unittest2 + +from mock import patch + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.hdsdiscovery.base import BaseSnmpMacPlugin +from compass.hdsdiscovery.base import BaseSnmpVendor +from compass.hdsdiscovery.error import TimeoutError +from compass.utils import flags +from compass.utils import logsetting + + +class MockSnmpVendor(BaseSnmpVendor): + """snmp vendor mock class.""" + + def __init__(self): + BaseSnmpVendor.__init__(self, ["MockVendor", "FakeVendor"]) + + +class TestBaseSnmpMacPlugin(unittest2.TestCase): + """teset base snmp plugin class.""" + + def setUp(self): + super(TestBaseSnmpMacPlugin, self).setUp() + logsetting.init() + self.test_plugin = BaseSnmpMacPlugin('127.0.0.1', + {'version': '2c', + 'community': 'public'}) + + def tearDown(self): + del self.test_plugin + super(TestBaseSnmpMacPlugin, self).tearDown() + + @patch('compass.hdsdiscovery.utils.snmpget_by_cl') + def test_get_port(self, mock_snmpget): + """test snmp get port.""" + # Successfully get port number + mock_snmpget.return_value = 'IF-MIB::ifName.4 = STRING: ge-1/1/4' + result = self.test_plugin.get_port('4') + self.assertEqual('4', result) + + # Failed to get port number, switch is timeout + mock_snmpget.side_effect = TimeoutError("Timeout") + result = self.test_plugin.get_port('4') + self.assertIsNone(result) + + @patch('compass.hdsdiscovery.utils.snmpget_by_cl') + def test_get_vlan_id(self, mock_snmpget): + """test snmp get vlan.""" + # Port is None + self.assertIsNone(self.test_plugin.get_vlan_id(None)) + + # Port is not None + mock_snmpget.return_value = 'Q-BRIDGE-MIB::dot1qPvid.4 = Gauge32: 100' + result = self.test_plugin.get_vlan_id('4') + self.assertEqual('100', result) + + # Faild to query switch due to timeout + mock_snmpget.side_effect = TimeoutError("Timeout") + result = self.test_plugin.get_vlan_id('4') + self.assertIsNone(result) + + def test_get_mac_address(self): + """tet snmp get mac address.""" + # Correct input for mac numbers + mac_numbers = '0.224.129.230.57.173'.split('.') + mac = self.test_plugin.get_mac_address(mac_numbers) + self.assertEqual('00:e0:81:e6:39:ad', mac) + + # Incorrct input for mac numbers + mac_numbers = '0.224.129.230.57'.split('.') + mac = self.test_plugin.get_mac_address(mac_numbers) + self.assertIsNone(mac) + + +class BaseTest(unittest2.TestCase): + """base test class.""" + + def setUp(self): + super(BaseTest, self).setUp() + logsetting.init() + + def tearDown(self): + super(BaseTest, self).tearDown() + + def test_base_snmp_vendor(self): + """test base snmp vendor.""" + fake = MockSnmpVendor() + + is_vendor = fake.is_this_vendor("FakeVendor 1.1") + + self.assertTrue(is_vendor) + + # check case-insensitive match + self.assertFalse(fake.is_this_vendor("fakevendor1.1")) + + # breaks word-boudary match + self.assertFalse(fake.is_this_vendor("FakeVendor1.1")) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/compass/tests/hdsdiscovery/test_hdsdiscovery.py b/compass/tests/hdsdiscovery/test_hdsdiscovery.py new file mode 100755 index 00000000..f93745ac --- /dev/null +++ b/compass/tests/hdsdiscovery/test_hdsdiscovery.py @@ -0,0 +1,246 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test hdsdiscovery module.""" +import os +import unittest2 + +from mock import Mock +from mock import patch + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.hdsdiscovery.hdmanager import HDManager +from compass.hdsdiscovery.vendors.huawei.huawei import Huawei +from compass.hdsdiscovery.vendors.huawei.plugins.mac import Mac +from compass.hdsdiscovery.vendors.ovswitch.plugins.mac import Mac as OVSMac +from compass.utils import flags +from compass.utils import logsetting + + +SNMP_V2_CREDENTIALS = {'version': '2c', + 'community': 'public'} + + +class HuaweiTest(unittest2.TestCase): + """test huawei switch snmp get.""" + + def setUp(self): + super(HuaweiTest, self).setUp() + logsetting.init() + self.huawei = Huawei() + self.sys_info = 'Huawei Technologies' + + def tearDown(self): + del self.huawei + super(HuaweiTest, self).tearDown() + + def test_is_this_vendor(self): + """test device vendor is haiwei.""" + #Incorrect system information + incorrect_sys_info = "xxx" + self.assertFalse( + self.huawei.is_this_vendor(incorrect_sys_info)) + + #Correct vendor + self.assertTrue( + self.huawei.is_this_vendor(self.sys_info)) + + +class HuaweiMacTest(unittest2.TestCase): + """test get mac from huawei device.""" + + def setUp(self): + super(HuaweiMacTest, self).setUp() + logsetting.init() + host = '192.168.1.1' + credential = {'version': '2c', 'community': 'public'} + self.mac_plugin = Mac(host, credential) + + def tearDown(self): + del self.mac_plugin + super(HuaweiMacTest, self).tearDown() + + @patch("compass.hdsdiscovery.utils.snmpwalk_by_cl") + def test_process_data(self, mock_snmpwalk): + """get progress data function.""" + # GET operation haven't been implemeneted. + self.assertIsNone(self.mac_plugin.process_data('GET')) + + # SNMP Walk Timeout + #utils.snmpwalk_by_cl = Mock(return_value=None) + mock_snmpwalk.return_value = None + self.assertIsNone(self.mac_plugin.process_data()) + + # Successfully get MAC addresses from the switch + mock_snmp_walk_result = [ + {"iid": "40.110.212.77.198.190.88.1.48", "value": "10"}, + {"iid": "40.110.212.100.199.74.88.1.48", "value": "11"}, + {"iid": "0.12.41.53.220.2.88.1.48", "value": "12"} + ] + expected_mac_info = [ + {"mac": "28:6e:d4:4d:c6:be", "port": "1", "vlan": "88"}, + {"mac": "28:6e:d4:64:c7:4a", "port": "2", "vlan": "88"}, + {"mac": "00:0c:29:35:dc:02", "port": "3", "vlan": "88"} + ] + #utils.snmpwalk_by_cl = Mock(return_value=mock_snmp_walk_result) + mock_snmpwalk.return_value = mock_snmp_walk_result + self.mac_plugin.get_port = Mock() + self.mac_plugin.get_port.side_effect = ["1", "2", "3"] + result = self.mac_plugin.process_data() + self.assertEqual(expected_mac_info, result) + + +class OVSMacTest(unittest2.TestCase): + """ovs switch test.""" + + def setUp(self): + super(OVSMacTest, self).setUp() + logsetting.init() + self.host = '10.145.88.160' + self.credential = {'username': 'root', 'password': 'huawei'} + + @patch('compass.hdsdiscovery.utils.ssh_remote_execute') + def test_scan(self, ovs_mock): + """test scan ovs switch.""" + ovs_mock.return_value = [] + mac_instance = OVSMac(self.host, self.credential) + self.assertIsNone(mac_instance.scan()) + del mac_instance + + ovs_mock.return_value = ['\n', '\n', '\n'] + mac_instance = OVSMac(self.host, self.credential) + self.assertEqual([], mac_instance.scan()) + del mac_instance + + +class HDManagerTest(unittest2.TestCase): + """test HDManager.""" + + def setUp(self): + super(HDManagerTest, self).setUp() + logsetting.init() + self.manager = HDManager() + self.correct_host = '33.33.33.1' + self.correct_host_2 = '127.0.0.1' + self.correct_credential = {'version': '2c', 'community': 'public'} + + def tearDown(self): + del self.manager + super(HDManagerTest, self).tearDown() + + @patch('compass.hdsdiscovery.hdmanager.HDManager.get_sys_info') + def test_get_vendor(self, sys_info_mock): + """test get_vendor.""" + # Incorrect ip + vendor, state, err = self.manager.get_vendor('1234.1.1.1', + self.correct_credential) + self.assertIsNone(vendor) + self.assertEqual('error', state) + + # Incorrect credential + incorr_cred = {'version': '1v', 'community': 'private'} + vendor, state, err = self.manager.get_vendor(self.correct_host, + incorr_cred) + self.assertIsNone(vendor) + self.assertEqual('error', state) + + # SNMP get system description Timeout + excepted_err_msg = 'Timeout: No Response from 127.0.0.1.' + sys_info_mock.return_value = (None, excepted_err_msg) + result, state, err = self.manager.get_vendor(self.correct_host, + self.correct_credential) + self.assertIsNone(result) + self.assertEqual(state, 'unreachable') + self.assertEqual(err, excepted_err_msg) + + # No vendor plugin supported + excepted_err_msg = 'Not supported switch vendor!' + sys_info_mock.return_value = ('xxxxxx', excepted_err_msg) + result, state, err = self.manager.get_vendor(self.correct_host, + self.correct_credential) + self.assertIsNone(result) + self.assertEqual(state, 'notsupported') + self.assertEqual(err, excepted_err_msg) + + # Found the correct vendor + sys_info = ['Huawei Versatile Routing Platform Software', + 'ProCurve J9089A Switch 2610-48-PWR, revision R.11.25', + 'Pica8 XorPlus Platform Software'] + expected_vendor_names = ['huawei', 'hp', 'pica8', 'appliance'] + for info, expected_vendor in zip(sys_info, expected_vendor_names): + sys_info_mock.return_value = (info, '') + # the result is a tuple ($vendor, $state, $error_message) + result = self.manager.get_vendor(self.correct_host, + self.correct_credential) + self.assertEqual(result[0], expected_vendor) + + @patch('compass.hdsdiscovery.hdmanager.HDManager.get_sys_info') + def test_is_valid_vendor(self, sys_info_mock): + """test is_valid_vendor.""" + #non-exsiting vendor under vendors directory + self.assertFalse( + self.manager.is_valid_vendor(self.correct_host, + self.correct_credential, + 'xxxx') + ) + + #No system description retrieved + sys_info_mock.return_value = (None, 'TIMEOUT') + self.assertFalse( + self.manager.is_valid_vendor(self.correct_host, + self.correct_credential, + 'pica8') + ) + + #Incorrect vendor name + sys_info = 'Pica8 XorPlus Platform Software' + sys_info_mock.return_value = (sys_info, '') + self.assertFalse( + self.manager.is_valid_vendor(self.correct_host, + self.correct_credential, + 'huawei') + ) + + #Correct vendor name + self.assertTrue( + self.manager.is_valid_vendor(self.correct_host, + self.correct_credential, + 'pica8') + ) + + def test_learn(self): + """test learn.""" + #non-exsiting plugin + self.assertIsNone(self.manager.learn(self.correct_host, + self.correct_credential, + 'huawei', 'xxx')) + + #non-existing vendor + self.assertIsNone(self.manager.learn(self.correct_host, + self.correct_credential, + 'xxxx', 'mac')) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/compass/tests/hdsdiscovery/test_utils.py b/compass/tests/hdsdiscovery/test_utils.py new file mode 100755 index 00000000..46778ba1 --- /dev/null +++ b/compass/tests/hdsdiscovery/test_utils.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test hdsdiscovery.utils module.""" +from mock import Mock +from mock import patch +import os +import unittest2 + + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.hdsdiscovery.error import TimeoutError +from compass.hdsdiscovery import utils +from compass.utils import flags +from compass.utils import logsetting + + +SNMP_V2_CREDENTIALS = {'version': '2c', + 'community': 'public'} + + +class UtilsTest(unittest2.TestCase): + """test huawei switch snmp get.""" + + def setUp(self): + super(UtilsTest, self).setUp() + logsetting.init() + self.host = "127.0.0.1" + self.credentials = SNMP_V2_CREDENTIALS + + def tearDown(self): + super(UtilsTest, self).tearDown() + + def test_load_module(self): + """get load_module function.""" + # Successfully load HUAWEI module + huawei_vendor_path = "/".join( + (os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "hdsdiscovery/vendors/huawei") + ) + + # No module found + self.assertIsNone(utils.load_module("xxx", huawei_vendor_path)) + + @patch("compass.hdsdiscovery.utils.exec_command") + def test_snmpget_by_cl(self, mock_exec_command): + oid = "sysDescr.0" + # Incorrect credentials + incorr_credentials = {"version": "1v", "community": "public"} + self.assertIsNone(utils.snmpget_by_cl(self.host, + incorr_credentials, + oid)) + # Switch timeout, failed to execute SNMPGET + mock_exec_command.return_value = (1, None, "Timeout") + with self.assertRaises(TimeoutError): + utils.snmpget_by_cl(self.host, self.credentials, oid) + + # Successfully get system information + mock_exec_command.return_value = (0, "Huawei Technologies", None) + result = utils.snmpget_by_cl(self.host, self.credentials, oid) + self.assertEqual("Huawei Technologies", result) + + def test_snmpwalk_by_cl(self): + oid = "BRIDGE-MIB::dot1dTpFdbPort" + # the result of SNMPWALK is None + utils.exec_command = Mock(return_value=(0, None, None)) + result = utils.snmpwalk_by_cl(self.host, self.credentials, oid) + self.assertEqual([], result) + + # Successfully execute SNMPWALK + return_value = ("xxx.0.12.41.112.143.193 = INTEGER: 47\n" + "xxx.0.12.41.139.17.124 = INTEGER: 47\n") + expected_result = [ + {"iid": "0.12.41.112.143.193", "value": "47"}, + {"iid": "0.12.41.139.17.124", "value": "47"} + ] + utils.exec_command = Mock(return_value=(0, return_value, None)) + result = utils.snmpwalk_by_cl(self.host, self.credentials, oid) + self.assertEqual(expected_result, result) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/compass/tests/log_analyzor/data/config b/compass/tests/log_analyzor/data/config new file mode 100644 index 00000000..aab65d49 --- /dev/null +++ b/compass/tests/log_analyzor/data/config @@ -0,0 +1,97 @@ +ADAPTERS = [ + {'name': 'CentOS_openstack', 'os': 'CentOS', 'target_system': 'openstack'}, +] +ROLES = [ + {'name': 'os-single-controller', 'target_system': 'openstack'}, + {'name': 'os-network', 'target_system': 'openstack'}, + {'name': 'os-compute', 'target_system': 'openstack'}, +] +SWITCHES = [ + {'ip': '1.2.3.4', 'vendor': 'huawei', 'credential': {'version': 'v2c', 'community': 'public'}}, +] +MACHINES_BY_SWITCH = { + '1.2.3.4': [ + {'mac': '00:00:01:02:03:04', 'port': 1, 'vlan': 1}, + ], +} +CLUSTERS = [ + { + 'name': 'cluster1', + 'adapter': 'CentOS_openstack', + 'mutable': False, + 'security': { + 'server_credentials': { + 'username': 'root', 'password': 'huawei' + }, + 'service_credentials': { + 'username': 'service', 'password': 'huawei' + }, + 'console_credentials': { + 'username': 'admin', 'password': 'huawei' + } + }, + 'networking': { + 'interfaces': { + 'management': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.255.0', + 'ip_end': '192.168.20.200', + 'gateway': '', + 'ip_start': '192.168.20.100' + }, + 'storage': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.200', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + }, + 'public': { + 'nic': 'eth2', + 'promisc': 1, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.255', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + }, + 'tenant': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.120', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + } + }, + 'global': { + 'nameservers': '192.168.20.254', + 'proxy': 'http://192.168.20.254:3128', + 'ntp_server': '192.168.20.254', + 'search_path': 'ods.com', + 'gateway': '10.145.88.1' + }, + }, + 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }, +] +HOSTS_BY_CLUSTER = { + 'cluster1': [ + { + 'hostname': 'server1', + 'mac': '00:00:01:02:03:04', + 'mutable': False, + 'config': { + 'networking': { + 'interfaces': { + 'management': { + 'ip': '192.168.20.100', + }, + }, + }, + 'roles': ["os-single-controller", "os-network", "os-compute"], + }, + }, + ], +} diff --git a/compass/tests/log_analyzor/test_adapter_matcher.py b/compass/tests/log_analyzor/test_adapter_matcher.py new file mode 100755 index 00000000..cb5fe08a --- /dev/null +++ b/compass/tests/log_analyzor/test_adapter_matcher.py @@ -0,0 +1,629 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test adapter matcher module""" + +import os +import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.db import database +from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import Machine +from compass.db.model import Role +from compass.db.model import Switch + +from compass.log_analyzor import adapter_matcher +from compass.log_analyzor.file_matcher import FileMatcher +from compass.log_analyzor.line_matcher import IncrementalProgress +from compass.log_analyzor.line_matcher import LineMatcher +from compass.log_analyzor import progress_calculator + +from compass.utils import flags +from compass.utils import logsetting + + +class TestAdapterItemMatcher(unittest2.TestCase): + def setUp(self): + super(TestAdapterItemMatcher, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestAdapterItemMatcher, self).tearDown() + + def test_update_progress(self): + test_update_progress_range = { + 'min_progress': 0.3, + 'max_progress': 0.7, + } + expected = ['sys.log', 0.0, 0.1] + file_matchers = [ + FileMatcher( + filename='sys.log', + min_progress=0.0, + max_progress=0.1, + line_matchers={ + 'start': LineMatcher( + pattern=r'NOTICE (?P.*)', + progress=IncrementalProgress(.1, .9, .1), + message_template='%(message)s', + unmatch_nextline_next_matcher_name='start', + match_nextline_next_matcher_name='exit' + ), + } + ), + ] + matcher = adapter_matcher.AdapterItemMatcher( + file_matchers=file_matchers) + matcher.update_progress_range( + **test_update_progress_range) + file_matcher = matcher.file_matchers_[0] + result = [] + result.append(file_matcher.filename_) + result.append(file_matcher.min_progress_) + result.append(file_matcher.max_progress_) + self.assertEqual(expected, result) + + +class TestOSMatcher(unittest2.TestCase): + def setUp(self): + super(TestOSMatcher, self).setUp() + self.item_matcher = progress_calculator\ + .OS_INSTALLER_CONFIGURATIONS[ + 'CentOS' + ] + logsetting.init() + + def tearDown(self): + super(TestOSMatcher, self).tearDown() + + def test_min_larger_than_max(self): + test_min_larger_than_max = { + 'os_installer_name': 'os_installer', + 'os_pattern': r'.*.', + 'item_matcher': None, + 'min_progress': 1.0, + 'max_progress': 0.0, + } + self.assertRaises( + IndexError, + adapter_matcher.OSMatcher, + **test_min_larger_than_max + ) + + def test_progress_exceed_one(self): + test_progress_exceed_one = { + 'os_installer_name': 'os_installer', + 'os_pattern': r'.*.', + 'item_matcher': None, + 'min_progress': 1.1, + 'max_progress': 1.1, + } + self.assertRaises( + IndexError, + adapter_matcher.OSMatcher, + **test_progress_exceed_one + ) + + def test_match(self): + test_match = { + 'os_installer_name': 'cobbler', + 'os_pattern': r'CentOS.*', + 'item_matcher': self.item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6, + } + matcher = adapter_matcher.OSMatcher( + **test_match) + self.assertTrue(matcher.match( + 'cobbler', + 'CentOS6.4')) + + def test_installer_unmatch(self): + test_installer_unmatch = { + 'os_installer_name': 'razor', + 'os_pattern': r'CentOS.*', + 'item_matcher': self.item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6, + } + matcher = adapter_matcher.OSMatcher( + **test_installer_unmatch) + self.assertFalse(matcher.match( + 'cobbler', + 'CentOS6.4')) + + def test_os_unmatch(self): + test_os_unmatch = { + 'os_installer_name': 'cobbler', + 'os_pattern': r'Ubuntu.*', + 'item_matcher': self.item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6, + } + matcher = adapter_matcher.OSMatcher( + **test_os_unmatch) + self.assertFalse(matcher.match( + 'cobbler', + 'CentOS6.4')) + + def test_both_unmatch(self): + test_both_unmatch = { + 'os_installer_name': 'razor', + 'os_pattern': r'Ubuntu.*', + 'item_matcher': self.item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6, + } + matcher = adapter_matcher.OSMatcher( + **test_both_unmatch) + self.assertFalse(matcher.match( + 'cobbler', + 'CentOS6.4')) + + +class TestPackageMatcher(unittest2.TestCase): + def setUp(self): + super(TestPackageMatcher, self).setUp() + self.item_matcher = progress_calculator\ + .PACKAGE_INSTALLER_CONFIGURATIONS[ + 'openstack' + ] + logsetting.init() + + def tearDown(self): + super(TestPackageMatcher, self).tearDown() + + def test_match(self): + test_match = { + 'package_installer_name': 'chef', + 'target_system': 'openstack', + 'item_matcher': self.item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0, + } + matcher = adapter_matcher.PackageMatcher( + **test_match) + self.assertTrue(matcher.match( + 'chef', + 'openstack')) + + def test_installer_unmatch(self): + test_installer_unmatch = { + 'package_installer_name': 'puppet', + 'target_system': 'openstack', + 'item_matcher': self.item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0, + } + matcher = adapter_matcher.PackageMatcher( + **test_installer_unmatch) + self.assertFalse(matcher.match( + 'chef', + 'openstack')) + + def test_target_system_unmatch(self): + test_target_system_unmatch = { + 'package_installer_name': 'chef', + 'target_system': 'hadoop', + 'item_matcher': self.item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0, + } + matcher = adapter_matcher.PackageMatcher( + **test_target_system_unmatch) + self.assertFalse(matcher.match( + 'chef', + 'openstack')) + + def test_both_unmatch(self): + test_both_unmatch = { + 'package_installer_name': 'puppet', + 'target_system': 'hadoop', + 'item_matcher': self.item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0, + } + matcher = adapter_matcher.PackageMatcher( + **test_both_unmatch) + self.assertFalse(matcher.match( + 'chef', + 'openstack')) + + +class TestAdapterMatcher(unittest2.TestCase): + def setUp(self): + super(TestAdapterMatcher, self).setUp() + self.os_item_matcher = progress_calculator\ + .OS_INSTALLER_CONFIGURATIONS[ + 'CentOS' + ] + self.package_item_matcher = progress_calculator\ + .PACKAGE_INSTALLER_CONFIGURATIONS[ + 'openstack' + ] + logsetting.init() + database.create_db() + + def tearDown(self): + super(TestAdapterMatcher, self).tearDown() + database.drop_db() + + def test_match(self): + test_match = { + 'os_matcher': { + 'os_installer_name': 'cobbler', + 'os_pattern': 'CentOS.*', + 'item_matcher': self.os_item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6 + }, + 'package_matcher': { + 'package_installer_name': 'chef', + 'target_system': 'openstack', + 'item_matcher': self.package_item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0 + } + } + os_matcher = adapter_matcher.OSMatcher( + **test_match['os_matcher']) + package_matcher = adapter_matcher.PackageMatcher( + **test_match['package_matcher']) + matcher = adapter_matcher.AdapterMatcher( + os_matcher, package_matcher) + + self.assertTrue( + matcher.match( + 'cobbler', 'CentOS6.4', + 'chef', 'openstack')) + + def test_os_unmatch(self): + test_os_unmatch = { + 'os_matcher': { + 'os_installer_name': 'razor', + 'os_pattern': 'CentOS.*', + 'item_matcher': self.os_item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6 + }, + 'package_matcher': { + 'package_installer_name': 'chef', + 'target_system': 'openstack', + 'item_matcher': self.package_item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0 + } + } + os_matcher = adapter_matcher.OSMatcher( + **test_os_unmatch['os_matcher']) + package_matcher = adapter_matcher.PackageMatcher( + **test_os_unmatch['package_matcher']) + matcher = adapter_matcher.AdapterMatcher( + os_matcher, package_matcher) + + self.assertFalse( + matcher.match( + 'cobbler', 'CentOS6.4', + 'chef', 'openstack')) + + def test_package_unmatch(self): + test_package_unmatch = { + 'os_matcher': { + 'os_installer_name': 'cobbler', + 'os_pattern': 'CentOS.*', + 'item_matcher': self.os_item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6 + }, + 'package_matcher': { + 'package_installer_name': 'puppet', + 'target_system': 'openstack', + 'item_matcher': self.package_item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0 + } + } + os_matcher = adapter_matcher.OSMatcher( + **test_package_unmatch['os_matcher']) + package_matcher = adapter_matcher.PackageMatcher( + **test_package_unmatch['package_matcher']) + matcher = adapter_matcher.AdapterMatcher( + os_matcher, package_matcher) + + self.assertFalse( + matcher.match( + 'cobbler', 'CentOS6.4', + 'chef', 'openstack')) + + def test_both_unmatch(self): + test_both_unmatch = { + 'os_matcher': { + 'os_installer_name': 'cobbler', + 'os_pattern': 'Ubuntu*', + 'item_matcher': self.os_item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6 + }, + 'package_matcher': { + 'package_installer_name': 'chef', + 'target_system': 'hadoop', + 'item_matcher': self.package_item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0 + } + } + os_matcher = adapter_matcher.OSMatcher( + **test_both_unmatch['os_matcher']) + package_matcher = adapter_matcher.PackageMatcher( + **test_both_unmatch['package_matcher']) + matcher = adapter_matcher.AdapterMatcher( + os_matcher, package_matcher) + + self.assertFalse( + matcher.match( + 'cobbler', 'CentOS6.4', + 'chef', 'openstack')) + + def test_update_progress(self): + config = { + 'ADAPTERS': [ + { + 'name': 'CentOS_openstack', + 'os': 'CentOS', + 'target_system': 'openstack', + }, + ], + 'ROLES': [ + { + 'name': 'os-single-controller', + 'target_system': 'openstack', + }, + { + 'name': 'os-network', + 'target_system': 'openstack', + }, + { + 'name': 'os-compute', + 'target_system': 'openstack', + }, + ], + 'SWITCHES': [ + { + 'ip': '1.2.3.4', + 'vendor': 'huawei', + 'credential': { + 'version': 'v2c', + 'comunity': 'public', + } + }, + ], + 'MACHINES_BY_SWITCH': { + '1.2.3.4': [ + { + 'mac': '00:00:01:02:03:04', + 'port': 1, + 'vlan': 1 + }, + ], + }, + 'CLUSTERS': [ + { + 'name': 'cluster1', + 'adapter': 'CentOS_openstack', + 'mutable': False, + 'security': { + 'server_credentials': { + 'username': 'root', + 'password': 'huawei' + }, + 'service_credentials': { + 'username': 'service', + 'password': 'huawei' + }, + 'console_credentials': { + 'username': 'admin', + 'password': 'huawei' + } + }, + 'networking': { + 'interfaces': { + 'management': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.255.0', + 'ip_end': '192.168.20.200', + 'gateway': '', + 'ip_start': '192.168.20.100' + }, + 'storage': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.200', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + }, + 'public': { + 'nic': 'eth2', + 'promisc': 1, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.255', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + }, + 'tenant': { + 'nic': 'eth0', + 'promisc': 0, + 'netmask': '255.255.254.0', + 'ip_end': '10.145.88.120', + 'gateway': '10.145.88.1', + 'ip_start': '10.145.88.100' + } + }, + 'global': { + 'nameservers': '192.168.20.254', + 'proxy': 'http://192.168.20.254:3128', + 'ntp_server': '192.168.20.254', + 'search_path': 'ods.com', + 'gateway': '10.145.88.1' + }, + }, + 'partition': '/home 20%%;/tmp 10%%;/var 30%%;', + }, + ], + 'HOSTS_BY_CLUSTER': { + 'cluster1': [ + { + 'hostname': 'server1', + 'mac': '00:00:01:02:03:04', + 'mutable': False, + 'config': { + 'networking': { + 'interfaces': { + 'management': { + 'ip': '192.168.20.100', + }, + }, + }, + 'roles': [ + "os-single-controller", + "os-network", + "os-compute" + ], + }, + }, + ], + }, + } + self._prepare_database(config) + cluster_hosts = {} + with database.session() as session: + clusters = session.query(Cluster).all() + for cluster in clusters: + cluster_hosts[cluster.id] = [ + host.id for host in cluster.hosts] + + test_update_progress = { + 'os_matcher': { + 'os_installer_name': 'cobbler', + 'os_pattern': 'CentOS.*', + 'item_matcher': self.os_item_matcher, + 'min_progress': 0.0, + 'max_progress': 0.6 + }, + 'package_matcher': { + 'package_installer_name': 'chef', + 'target_system': 'openstack', + 'item_matcher': self.package_item_matcher, + 'min_progress': 0.6, + 'max_progress': 1.0 + } + } + os_matcher = adapter_matcher.OSMatcher( + **test_update_progress['os_matcher']) + package_matcher = adapter_matcher.PackageMatcher( + **test_update_progress['package_matcher']) + matcher = adapter_matcher.AdapterMatcher( + os_matcher, package_matcher) + for cluster_id in cluster_hosts.keys(): + matcher.update_progress( + cluster_id, + cluster_hosts[cluster_id]) + + expected_cluster_state = { + 'state': 'INSTALLING', + 'progress': 0.0 + } + cluster = {} + host = {} + with database.session(): + cluster_state = session.query(ClusterState).all() + cluster['state'] = cluster_state[0].state + cluster['progress'] = cluster_state[0].progress + self.assertEqual(expected_cluster_state, + cluster) + + def _prepare_database(self, config): + with database.session() as session: + adapters = {} + for adapter_config in config['ADAPTERS']: + adapter = Adapter(**adapter_config) + session.add(adapter) + adapters[adapter_config['name']] = adapter + + roles = {} + for role_config in config['ROLES']: + role = Role(**role_config) + session.add(role) + roles[role_config['name']] = role + + switches = {} + for switch_config in config['SWITCHES']: + switch = Switch(**switch_config) + session.add(switch) + switches[switch_config['ip']] = switch + + machines = {} + for switch_ip, machine_configs in ( + config['MACHINES_BY_SWITCH'].items() + ): + for machine_config in machine_configs: + machine = Machine(**machine_config) + machines[machine_config['mac']] = machine + machine.switch = switches[switch_ip] + session.add(machine) + + clusters = {} + for cluster_config in config['CLUSTERS']: + adapter_name = cluster_config['adapter'] + del cluster_config['adapter'] + cluster = Cluster(**cluster_config) + clusters[cluster_config['name']] = cluster + cluster.adapter = adapters[adapter_name] + cluster.state = ClusterState( + state="INSTALLING", progress=0.0, message='') + session.add(cluster) + + hosts = {} + for cluster_name, host_configs in ( + config['HOSTS_BY_CLUSTER'].items() + ): + for host_config in host_configs: + mac = host_config['mac'] + del host_config['mac'] + host = ClusterHost(**host_config) + hosts['%s.%s' % ( + host_config['hostname'], cluster_name)] = host + host.machine = machines[mac] + host.cluster = clusters[cluster_name] + host.state = HostState( + state="INSTALLING", progress=0.0, message='') + session.add(host) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/compass/tests/log_analyzor/test_file_matcher.py b/compass/tests/log_analyzor/test_file_matcher.py new file mode 100755 index 00000000..cce33299 --- /dev/null +++ b/compass/tests/log_analyzor/test_file_matcher.py @@ -0,0 +1,436 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""test file matcher module""" + +import datetime +import mock +import os +import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.db import database +from compass.db.model import Adapter +from compass.db.model import Cluster +from compass.db.model import ClusterHost +from compass.db.model import ClusterState +from compass.db.model import HostState +from compass.db.model import LogProgressingHistory +from compass.db.model import Machine +from compass.db.model import Role +from compass.db.model import Switch + +from compass.log_analyzor import file_matcher + +from compass.log_analyzor.line_matcher import IncrementalProgress +from compass.log_analyzor.line_matcher import LineMatcher +from compass.log_analyzor.line_matcher import Progress + +from compass.utils import flags +from compass.utils import logsetting + + +def prepare_database(config): + with database.session() as session: + adapters = {} + for adapter_config in config['ADAPTERS']: + adapter = Adapter(**adapter_config) + session.add(adapter) + adapters[adapter_config['name']] = adapter + + roles = {} + for role_config in config['ROLES']: + role = Role(**role_config) + session.add(role) + roles[role_config['name']] = role + + switches = {} + for switch_config in config['SWITCHES']: + switch = Switch(**switch_config) + session.add(switch) + switches[switch_config['ip']] = switch + + machines = {} + for switch_ip, machine_configs in ( + config['MACHINES_BY_SWITCH'].items() + ): + for machine_config in machine_configs: + machine = Machine(**machine_config) + machines[machine_config['mac']] = machine + machine.switch = switches[switch_ip] + session.add(machine) + + clusters = {} + for cluster_config in config['CLUSTERS']: + adapter_name = cluster_config['adapter'] + del cluster_config['adapter'] + cluster = Cluster(**cluster_config) + clusters[cluster_config['name']] = cluster + cluster.adapter = adapters[adapter_name] + cluster.state = ClusterState( + state="INSTALLING", progress=0.0, message='') + session.add(cluster) + + hosts = {} + for cluster_name, host_configs in ( + config['HOSTS_BY_CLUSTER'].items() + ): + for host_config in host_configs: + mac = host_config['mac'] + del host_config['mac'] + host = ClusterHost(**host_config) + hosts['%s.%s' % ( + host_config['hostname'], cluster_name)] = host + host.machine = machines[mac] + host.cluster = clusters[cluster_name] + host.state = HostState( + state="INSTALLING", progress=0.0, message='') + session.add(host) + + +class TestFilterFileExist(unittest2.TestCase): + def setUp(self): + super(TestFilterFileExist, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestFilterFileExist, self).tearDown() + + def test_filter(self): + pathname = 'NeverExists' + filter = file_matcher.FilterFileExist() + res = filter.filter(pathname) + self.assertFalse(res) + + +class TestCompositeFileFilter(unittest2.TestCase): + def setUp(self): + super(TestCompositeFileFilter, self).setUp() + self.file_path = os.path.dirname(os.path.abspath(__file__)) + logsetting.init() + + def tearDown(self): + super(TestCompositeFileFilter, self).tearDown() + + def test_filter(self): + filter_list = [ + file_matcher.FilterFileExist(), + file_matcher.FilterFileExist(), + ] + exists = self.file_path + '/test_file_matcher.py' + non_exists = '/nonexists' + composite_filter = file_matcher.CompositeFileFilter( + filter_list) + self.assertTrue( + composite_filter.filter(exists)) + self.assertFalse( + composite_filter.filter(non_exists)) + + def test_append_filter(self): + filter_list = [ + file_matcher.FilterFileExist(), + file_matcher.FilterFileExist(), + ] + composite_filter = file_matcher.CompositeFileFilter( + filter_list) + new_filter = file_matcher.FilterFileExist() + composite_filter.append_filter(new_filter) + self.assertEqual(3, len(composite_filter.filters_)) + + +class TestFileReader(unittest2.TestCase): + def setUp(self): + super(TestFileReader, self).setUp() + logsetting.init() + database.create_db() + self.config_file = '%s/data/config' % ( + os.path.dirname(os.path.abspath(__file__))) + + def tearDown(self): + super(TestFileReader, self).tearDown() + database.drop_db() + + def test_get_empty_history(self): + config_locals = {} + config_globals = {} + execfile(self.config_file, config_globals, config_locals) + prepare_database(config_locals) + expected = { + 'matcher_name': 'start', + 'progress': 0.0, + 'message': '', + 'severity': None + } + res = {} + reader = file_matcher.FileReader('dummy') + history = reader.get_history() + res.update( + { + 'matcher_name': history[0], + 'progress': history[1].progress, + 'message': history[1].message, + 'severity': history[1].severity + }) + self.assertEqual(expected, res) + + def test_get_existing_history(self): + config_locals = {} + config_globals = {} + execfile(self.config_file, config_globals, config_locals) + prepare_database(config_locals) + with database.session() as session: + history = LogProgressingHistory( + line_matcher_name='start', + progress=0.5, + message='dummy', + severity='INFO', + position=0, + partial_line='', + pathname='dummy') + session.add(history) + + expected = { + 'matcher_name': 'start', + 'progress': 0.5, + 'message': 'dummy', + 'severity': 'INFO' + } + res = {} + reader = file_matcher.FileReader('dummy') + history = reader.get_history() + res.update({ + 'matcher_name': history[0], + 'progress': history[1].progress, + 'message': history[1].message, + 'severity': history[1].severity + }) + self.assertEqual(expected, res) + + def test_update_history_from_none(self): + config_locals = {} + config_globals = {} + execfile(self.config_file, config_globals, config_locals) + prepare_database(config_locals) + + expected = { + 'progress': 0.5, + 'line_matcher_name': 'start' + } + reader = file_matcher.FileReader('dummy') + reader.update_history( + expected['line_matcher_name'], + Progress( + progress=expected['progress'], + message='', + severity='INFO')) + res = {} + with database.session() as session: + history = session.query( + LogProgressingHistory).first() + res.update({ + 'line_matcher_name': history.line_matcher_name, + 'progress': history.progress + }) + self.assertEqual(expected, res) + + def test_update_history_from_existing(self): + config_locals = {} + config_globals = {} + execfile(self.config_file, config_globals, config_locals) + prepare_database(config_locals) + + with database.session() as session: + history = LogProgressingHistory( + line_matcher_name='start', + progress=0.5, + message='dummy', + severity='INFO', + position=0, + partial_line='', + pathname='dummy') + session.add(history) + + expected = { + 'progress': 0.8, + 'line_matcher_name': 'start' + } + reader = file_matcher.FileReader('dummy') + reader.position_ = 1 + reader.update_history( + expected['line_matcher_name'], + Progress( + progress=expected['progress'], + message='', + severity='INFO')) + res = {} + with database.session() as session: + history = session.query( + LogProgressingHistory).first() + res.update({ + 'line_matcher_name': history.line_matcher_name, + 'progress': history.progress + }) + self.assertEqual(expected, res) + + def test_update_history_failure(self): + config_locals = {} + config_globals = {} + execfile(self.config_file, config_globals, config_locals) + prepare_database(config_locals) + + with database.session() as session: + history = LogProgressingHistory( + line_matcher_name='start', + progress=0.5, + message='dummy', + severity='INFO', + position=0, + partial_line='', + pathname='dummy') + session.add(history) + + expected = { + 'progress': 0.8, + 'line_matcher_name': 'start' + } + reader = file_matcher.FileReader('dummy') + reader.update_history( + expected['line_matcher_name'], + Progress( + progress=expected['progress'], + message='', + severity='INFO')) + res = {} + with database.session() as session: + history = session.query( + LogProgressingHistory).first() + res.update({ + 'line_matcher_name': history.line_matcher_name, + 'progress': history.progress + }) + self.assertNotEqual(expected, res) + + +class TestFileReaderFactory(unittest2.TestCase): + def setUp(self): + super(TestFileReaderFactory, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestFileReaderFactory, self).tearDown() + + def test_get_file_reader_None(self): + reader_factory = file_matcher.FileReaderFactory( + 'dummy', + file_matcher.get_file_filter()) + + reader = reader_factory.get_file_reader('dummy', 'dummy') + self.assertIsNone(reader) + + +class TestFileMatcher(unittest2.TestCase): + def setUp(self): + super(TestFileMatcher, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestFileMatcher, self).tearDown() + + def test_update_absolute_progress_range(self): + matcher = file_matcher.FileMatcher( + filename='sys.log', + min_progress=0.0, + max_progress=0.1, + line_matchers={ + 'start': LineMatcher( + pattern=r'NOTICE (?P.*)', + progress=IncrementalProgress(.1, .9, .1), + message_template='%(message)s', + unmatch_nextline_next_matcher_name='start', + match_nextline_next_matcher_name='exit' + ), + } + ) + matcher.update_absolute_progress_range(0.5, 1.0) + expected = [0.5, 0.55] + res = [] + res.append(matcher.absolute_min_progress_) + res.append(matcher.absolute_max_progress_) + self.assertEqual(expected, res) + + def test_update_total_progress_none(self): + file_progress = Progress( + message=None, + progress=0.5, + severity='info') + + total_progress = file_progress + matcher = file_matcher.FileMatcher( + filename='sys.log', + min_progress=0.0, + max_progress=0.1, + line_matchers={ + 'start': LineMatcher( + pattern=r'NOTICE (?P.*)', + progress=IncrementalProgress(.1, .9, .1), + message_template='%(message)s', + unmatch_nextline_next_matcher_name='start', + match_nextline_next_matcher_name='exit' + ), + } + ) + res = matcher.update_total_progress(file_progress, total_progress) + self.assertIsNone(res) + + def test_update_total_progress(self): + file_progress = Progress( + message='dummy', + progress=0.5, + severity='info') + + total_progress = Progress( + message='dummy', + progress=0.4, + severity='info') + + matcher = file_matcher.FileMatcher( + filename='sys.log', + min_progress=0.0, + max_progress=0.1, + line_matchers={ + 'start': LineMatcher( + pattern=r'NOTICE (?P.*)', + progress=IncrementalProgress(.1, .9, .1), + message_template='%(message)s', + unmatch_nextline_next_matcher_name='start', + match_nextline_next_matcher_name='exit' + ), + } + ) + matcher.update_total_progress(file_progress, total_progress) + self.assertEqual( + 0.5, + total_progress.progress) + + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/compass/tests/log_analyzor/test_line_matcher.py b/compass/tests/log_analyzor/test_line_matcher.py new file mode 100755 index 00000000..d68aed4e --- /dev/null +++ b/compass/tests/log_analyzor/test_line_matcher.py @@ -0,0 +1,229 @@ +#!/usr/bin/python +# +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest2 + +os.environ['COMPASS_IGNORE_SETTING'] = 'true' + + +from compass.utils import setting_wrapper as setting +reload(setting) + + +from compass.log_analyzor import line_matcher +from compass.utils import flags +from compass.utils import logsetting + + +class TestProgress(unittest2.TestCase): + """test class for Progress class.""" + + def setUp(self): + super(TestProgress, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestProgress, self).tearDown() + + def test_repr(self): + config = { + 'progress': 0.5, + 'message': 'dummy', + 'severity': '' + } + expected = 'Progress[progress:0.5, message:dummy, severity:]' + test_line_matcher = line_matcher.Progress(**config) + self.assertEqual(expected, test_line_matcher.__repr__()) + + +class TestProgressCalculator(unittest2.TestCase): + def setUp(self): + super(TestProgressCalculator, self).setUp() + logsetting.init() + self._mock_progress() + + def tearDown(self): + super(TestProgressCalculator, self).tearDown() + + def _mock_progress(self): + self.progress = line_matcher.Progress( + progress=0.5, + message='', + severity='') + + def test_update_progress_progress(self): + test_1 = { + 'progress_data': 0.7, + 'message': '', + 'severity': '', + 'progress': self.progress + } + expected_1 = 0.7 + line_matcher.ProgressCalculator.update_progress( + **test_1) + self.assertEqual(expected_1, self.progress.progress) + + def test_update_progress_other(self): + test = { + 'progress_data': 0.5, + 'message': 'dummy', + 'severity': 'dummy', + 'progress': self.progress + } + expected_message = test['message'] + expected_severity = test['severity'] + line_matcher.ProgressCalculator.update_progress( + **test) + self.assertEqual(expected_message, self.progress.message) + self.assertEqual(expected_severity, self.progress.severity) + + +class TestIncrementalProgress(unittest2.TestCase): + def setUp(self): + super(TestIncrementalProgress, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestIncrementalProgress, self).tearDown() + + def test_init(self): + test_exceed_one = { + 'min_progress': 1.1, + 'max_progress': 1.1, + 'incremental_ratio': 0.5, + } + self.assertRaises( + IndexError, + line_matcher.IncrementalProgress, + **test_exceed_one) + + def test_min_larger_than_max(self): + test_min_larger_than_max = { + 'min_progress': 0.7, + 'max_progress': 0.3, + 'incremental_ratio': 0.5, + } + self.assertRaises( + IndexError, + line_matcher.IncrementalProgress, + **test_min_larger_than_max) + + def test_invalid_ratio(self): + test_invalid_ratio = { + 'min_progress': 0.3, + 'max_progress': 0.7, + 'incremental_ratio': 1.1, + } + self.assertRaises( + IndexError, + line_matcher.IncrementalProgress, + **test_invalid_ratio) + + +class TestRelativeProgress(unittest2.TestCase): + def setUp(self): + super(TestRelativeProgress, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestRelativeProgress, self).tearDown() + + def test_init(self): + self.assertRaises( + IndexError, + line_matcher.RelativeProgress, + progress=1.1) + + +class TestLineMatcher(unittest2.TestCase): + def setUp(self): + super(TestLineMatcher, self).setUp() + logsetting.init() + + def tearDown(self): + super(TestLineMatcher, self).tearDown() + + def test_progress_unsupported(self): + test_progress_unsupported = { + 'pattern': r' ', + 'progress': 'dummy', + } + self.assertRaises( + TypeError, + line_matcher.LineMatcher, + **test_progress_unsupported) + + def test_regex_not_match(self): + line = 'abc' + regex_ = r'^s' + progress = line_matcher.Progress( + progress=1, message='a', severity=' ') + test_regex_not_match = { + 'pattern': regex_, + 'unmatch_sameline_next_matcher_name': 'usn', + 'unmatch_nextline_next_matcher_name': 'unn', + 'match_sameline_next_matcher_name': 'msn', + 'match_nextline_next_matcher_name': 'mnn', + } + matcher = line_matcher.LineMatcher( + **test_regex_not_match) + expected = ('usn', 'unn') + self.assertEqual( + expected, + matcher.update_progress( + line, progress)) + + def test_regex_match(self): + line = 'abc' + regex_ = r'^a' + progress = line_matcher.Progress( + progress=1, message='a', severity=' ') + test_regex_match = { + 'pattern': regex_, + 'unmatch_sameline_next_matcher_name': 'usn', + 'unmatch_nextline_next_matcher_name': 'unn', + 'match_sameline_next_matcher_name': 'msn', + 'match_nextline_next_matcher_name': 'mnn', + } + matcher = line_matcher.LineMatcher( + **test_regex_match) + expected = ('msn', 'mnn') + self.assertEqual( + expected, + matcher.update_progress( + line, progress)) + + def test_wrong_message(self): + line = 'abc' + progress = line_matcher.Progress( + progress=1, message='a', severity=' ') + test_wrong_message = { + 'pattern': r'.*.', + 'message_template': 'Installing %(package)s' + } + matcher = line_matcher.LineMatcher( + **test_wrong_message) + self.assertRaises( + KeyError, + matcher.update_progress, + line=line, + progress=progress) + +if __name__ == '__main__': + flags.init() + logsetting.init() + unittest2.main() diff --git a/regtest/regtest.conf b/regtest/regtest.conf index b20b22b6..c22b2b7d 100644 --- a/regtest/regtest.conf +++ b/regtest/regtest.conf @@ -38,9 +38,6 @@ export TMP_PERCENTAGE=${TMP_PERCENTAGE:-'5'} export VAR_PERCENTAGE=${VAR_PERCENTAGE:-'10'} export PARTITION=${PARTITION:-"/home=${HOME_PERCENTAGE}%,/tmp=${TMP_PERCENTAGE}%,/var=${VAR_PERCENTAGE}%"} export LOCAL_REPO_URL=${LOCAL_REPO_URL:-"http://$IPADDR"} -if [ -n "$LOCAL_REPO_URL" ]; then - export PROXY= -fi export OS_CONFIG_FILENAME=${OS_CONFIG_FILENAME:-} export PACKAGE_CONFIG_FILENAME=${PACKAGE_CONFIG_FILENAME:-} diff --git a/regtest/regtest.sh b/regtest/regtest.sh index 57fd5d3c..2aa9498b 100755 --- a/regtest/regtest.sh +++ b/regtest/regtest.sh @@ -160,6 +160,10 @@ else POLL_SWITCHES_FLAG="poll_switches" fi +if [ -n "$LOCAL_REPO_URL" ]; then + PROXY= +fi + ${CLIENT_SCRIPT} --logfile= --loglevel=debug --logdir= --compass_server="${COMPASS_SERVER_URL}" --compass_user_email="${COMPASS_USER_EMAIL}" --compass_user_password="${COMPASS_USER_PASSWORD}" --cluster_name="${CLUSTER_NAME}" --language="${LANGUAGE}" --timezone="${TIMEZONE}" --hostnames="${HOSTNAMES}" --partitions="${PARTITIONS}" --subnets="${SUBNETS}" --adapter_os_pattern="${ADAPTER_OS_PATTERN}" --adapter_name="${ADAPTER_NAME}" --adapter_target_system_pattern="${ADAPTER_TARGET_SYSTEM_PATTERN}" --adapter_flavor_pattern="${ADAPTER_FLAVOR_PATTERN}" --http_proxy="${PROXY}" --https_proxy="${PROXY}" --no_proxy="${IGNORE_PROXY}" --ntp_server="${NTP_SERVER}" --dns_servers="${NAMESERVERS}" --domain="${DOMAIN}" --search_path="${SEARCH_PATH}" --default_gateway="${GATEWAY}" --server_credential="${SERVER_CREDENTIAL}" --local_repo_url="${LOCAL_REPO_URL}" --os_config_json_file="${OS_CONFIG_FILENAME}" --service_credentials="${SERVICE_CREDENTIALS}" --console_credentials="${CONSOLE_CREDENTIALS}" --host_networks="${HOST_NETWORKS}" --network_mapping="${NETWORK_MAPPING}" --package_config_json_file="${PACKAGE_CONFIG_FILENAME}" --host_roles="${HOST_ROLES}" --default_roles="${DEFAULT_ROLES}" --switch_ips="${SWITCH_IPS}" --machines="${machines}" --switch_credential="${SWITCH_CREDENTIAL}" --deployment_timeout="${DEPLOYMENT_TIMEOUT}" --${POLL_SWITCHES_FLAG} --dashboard_url="${DASHBOARD_URL}" rc=$? deactivate