Fix for excluding gbpservice/tests/contrib/nfp_service/ from UTs

Excluding the gbpservice/tests/contrib/nfp_service/ from python27 tests
by moving out the directory permanently to gbpservice/contrib/.
Fixed file path references in few files as part of this.

Enabled whole gbpservice tests instead of just gbpservice/neutron in
.testr.conf. Fixed FW and VPN driver UTs as part of this.

Fixed pep8 warnings in reference controller. As part of this, added
oslo.i18n installation to nfp reference configurator vm, and used
gbpservice._i18n.py for i18n in logs.

Fix to mock rmq connection in the advanced controller unit tests

Change-Id: I4743e44f86e78b7d19ccd824a7c65195f12f856d
Closes-Bug: 1684076

.testr.conf

@@ -2,7 +2,6 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-120} \
-# Temporarily exclude gbpservice/neutron contrib tests until nfp tests pass
-             ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./gbpservice/neutron} $LISTOPT $IDOPTION
+             ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./gbpservice} $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list

gbpservice/contrib/nfp/tools/image_builder/disk_image_create.py

@@ -81,12 +81,14 @@ def dib():
     # element for creating configurator image
     if 'nfp-reference-configurator' in dib['elements']:
         image_name = 'nfp_reference_service'
-        service_dir = "%s/../../../../tests/contrib/nfp_service/" % cur_dir
+        service_dir = "%s/../../../../contrib/nfp_service/" % cur_dir
         service_dir = os.path.realpath(service_dir)
         pecan_dir = "%s/../../../../nfp/" % cur_dir
         pecan_dir = os.path.realpath(pecan_dir)
+        gbpservice_i18n_file = "%s/../../../../_i18n.py" % cur_dir
         os.environ['PECAN_GIT_PATH'] = pecan_dir
         os.environ['SERVICE_GIT_PATH'] = service_dir
+        os.environ['GBPSERVICE_I18N_FILE'] = gbpservice_i18n_file
         if 'devuser' in dib['elements']:
             os.environ['DIB_DEV_USER_USERNAME'] = 'ubuntu'
             os.environ['DIB_DEV_USER_SHELL'] = '/bin/bash'

gbpservice/contrib/nfp/tools/image_builder/elements/nfp-reference-configurator/extra-data.d/99-nfp-service

@@ -4,6 +4,7 @@ set -eu
 # copy the reference_configurator and pecan folders to VM at /root/
 cp -rL ${SERVICE_GIT_PATH}/reference_configurator ${TMP_MOUNT_PATH}/root/
 cp -rL ${PECAN_GIT_PATH}/pecan ${TMP_MOUNT_PATH}/root/
+cp -rL ${GBPSERVICE_I18N_FILE} ${TMP_MOUNT_PATH}/root/
 
 ls -lR ${TMP_MOUNT_PATH}/root/
 

gbpservice/contrib/nfp/tools/image_builder/elements/nfp-reference-configurator/install.d/99-nfp-service

@@ -6,7 +6,7 @@ set -eu
 
 cd /usr/local/lib/python2.7/dist-packages/
 
-service_path='gbpservice/tests/contrib/nfp_service'
+service_path='gbpservice/contrib/nfp_service'
 pecan_path='gbpservice/nfp'
 mkdir -p $service_path
 mkdir -p $pecan_path
@@ -14,6 +14,7 @@ mkdir -p $pecan_path
 find gbpservice/ -type d -exec touch {}/__init__.py \;
 mv /root/reference_configurator $service_path/
 mv /root/pecan $pecan_path/
+mv /root/_i18n.py gbpservice/
 
 cd $service_path
 PWD=`pwd`

gbpservice/contrib/nfp/tools/image_builder/elements/nfp-reference-configurator/post-install.d/99-nfp-service

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 apt-get install -y --force-yes python-pecan
-apt-get install -y --force-yes oslo.serialization oslo.log python-yaml
+apt-get install -y --force-yes oslo.serialization oslo.log oslo.i18n python-yaml
 
 ls -lR /home/ubuntu/
 cd /usr/local/lib/python2.7/dist-packages/gbpservice/nfp/pecan/api &&  python setup.py develop

gbpservice/contrib/nfp_service/reference_configurator/config/pecan.service

@@ -5,7 +5,7 @@ After=network.target auditd.service
 [Service]
 Type=forking
 Restart=always
-ExecStart=/usr/local/lib/python2.7/dist-packages/gbpservice/tests/contrib/nfp_service/reference_configurator/bin/nfp-pecan
+ExecStart=/usr/local/lib/python2.7/dist-packages/gbpservice/contrib/nfp_service/reference_configurator/bin/nfp-pecan
 KillMode=process
 Restart=on-failure
 

gbpservice/contrib/nfp_service/reference_configurator/controllers/controller.py

@@ -10,26 +10,27 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import subprocess
-
 import netaddr
 import netifaces
-from oslo_log._i18n import _LE
-from oslo_log._i18n import _LI
-from oslo_log import log as logging
-import oslo_serialization.jsonutils as jsonutils
 import pecan
 from pecan import rest
+import subprocess
 import time
 import yaml
 
+from gbpservice._i18n import _LE
+from gbpservice._i18n import _LI
+
+from oslo_log import log as logging
+import oslo_serialization.jsonutils as jsonutils
+
 LOG = logging.getLogger(__name__)
 SUCCESS = 'SUCCESS'
 FAILED = 'FAILED'
 
 notifications = []
 FW_SCRIPT_PATH = ("/usr/local/lib/python2.7/dist-packages/" +
-                  "gbpservice/tests/contrib/nfp_service/" +
+                  "gbpservice/contrib/nfp_service/" +
                   "reference_configurator/scripts/configure_fw_rules.py")
 
 
@@ -199,8 +200,8 @@ class Controller(rest.RestController):
             try:
                 source_interface = self._get_if_name_by_cidr(cidr)
             except Exception:
-                raise Exception("Some of the interfaces do not have "
-                                "IP Address")
+                raise Exception(_("Some of the interfaces do not have "
+                                  "IP Address"))
             try:
                 interface_number_string = source_interface.split("eth", 1)[1]
             except IndexError:
@@ -224,7 +225,7 @@ class Controller(rest.RestController):
                 LOG.info(_LI("Static route configuration result: %(output)s"),
                          {'output': output})
             except Exception as ex:
-                raise Exception("Failed to add static routes: %(ex)s" % {
+                raise Exception(_("Failed to add static routes: %(ex)s") % {
                                 'ex': str(ex)})
         for command in default_route_commands:
             try:
@@ -233,7 +234,7 @@ class Controller(rest.RestController):
                 LOG.info(_LI("Static route configuration result: %(output)s"),
                          {'output': out})
             except Exception as ex:
-                raise Exception("Failed to add static routes: %(ex)s" % {
+                raise Exception(_("Failed to add static routes: %(ex)s") % {
                                 'ex': str(ex)})
 
     def _get_if_name_by_cidr(self, cidr):
@@ -264,13 +265,13 @@ class Controller(rest.RestController):
                     retry_count = retry_count + 1
                     continue
                 else:
-                    raise Exception("Some of the interfaces do not have "
-                                    "IP Address")
+                    raise Exception(_("Some of the interfaces do not have "
+                                      "IP Address"))
 
     def _apply_user_config(self, config_data):
         LOG.info(_LI("Applying user config with configuration "
-                 "type : %(config_type)s and "
-                 "configuration data : %(config_data)s "),
+                     "type : %(config_type)s and "
+                     "configuration data : %(config_data)s "),
                  {'config_type': config_data['resource'],
                   'config_data': config_data['resource_data']})
         service_config = config_data['resource_data'][

gbpservice/contrib/tests/unit/nfp/configurator/advanced_controller/test_controller.py

@@ -15,6 +15,7 @@ import mock
 import os
 import oslo_serialization.jsonutils as jsonutils
 import pecan
+import pika
 PECAN_CONFIG_FILE = (os.getcwd() +
                      "/gbpservice/nfp/pecan/api/config.py")
 
@@ -33,6 +34,7 @@ setattr(pecan, 'mode', constants.advanced)
 from gbpservice.contrib.nfp.configurator.advanced_controller import controller
 from gbpservice.nfp.pecan.api import root_controller
 reload(root_controller)
+pika.BlockingConnection = mock.MagicMock(return_value=None)
 
 
 class ControllerTestCase(base.BaseTestCase, rest.RestController):

gbpservice/contrib/tests/unit/nfp/configurator/drivers/vpn/test_vpn_driver.py

@@ -45,6 +45,7 @@ class VpnaasIpsecDriverTestCase(base.BaseTestCase):
         self.conf = 'conf'
         self.test_dict = vpn_test_data.VPNTestData()
         self.context = self.test_dict.make_service_context()
+        self.headers = self.test_dict.get_header()
         self.plugin_rpc = vpn.VpnaasRpcSender(self.test_dict.sc)
         self.driver = vyos_vpn_driver.VpnaasIpsecDriver(self.conf)
         self.svc_validate = (
@@ -90,6 +91,7 @@ class VpnaasIpsecDriverTestCase(base.BaseTestCase):
             mock_post.assert_called_with(
                 self.test_dict.url_create_ipsec_conn,
                 data=jsonutils.dumps(self.test_dict.svc_context),
+                headers=self.headers,
                 timeout=self.test_dict.timeout)
             mock_update_status.assert_called_with(
                 context,
@@ -129,6 +131,7 @@ class VpnaasIpsecDriverTestCase(base.BaseTestCase):
             mock_delete.assert_called_with(
                 url.encode('ascii', 'ignore'),
                 timeout=self.test_dict.timeout,
+                headers=self.headers,
                 data=None)
 
     def test_check_status(self):
@@ -158,6 +161,7 @@ class VpnGenericConfigDriverTestCase(base.BaseTestCase):
         self.conf = 'conf'
         self.test_dict = vpn_test_data.VPNTestData()
         self.context = self.test_dict.make_service_context()
+        self.headers = self.test_dict.get_header()
         self.plugin_rpc = vpn.VpnaasRpcSender(self.test_dict.sc)
         self.rest_apt = vyos_vpn_driver.RestApi(self.test_dict.vm_mgmt_ip)
         self.driver = vyos_vpn_driver.VpnGenericConfigDriver()
@@ -194,6 +198,7 @@ class VpnGenericConfigDriverTestCase(base.BaseTestCase):
                 self.test_dict.url_for_add_inte,
                 jsonutils.dumps(
                     self.test_dict.data_for_interface),
+                headers=self.headers,
                 timeout=self.test_dict.timeout)
 
     def test_clear_interfaces(self):
@@ -217,6 +222,7 @@ class VpnGenericConfigDriverTestCase(base.BaseTestCase):
                 self.test_dict.url_for_del_inte,
                 data=jsonutils.dumps(
                     self.test_dict.data_for_interface),
+                headers=self.headers,
                 timeout=self.test_dict.timeout)
 
     def test_configure_source_routes(self):
@@ -239,6 +245,7 @@ class VpnGenericConfigDriverTestCase(base.BaseTestCase):
                 self.test_dict.url_for_add_src_route,
                 data=jsonutils.dumps(
                     self.test_dict.data_for_add_src_route),
+                headers=self.headers,
                 timeout=self.test_dict.timeout)
 
     def test_delete_source_routes(self):
@@ -260,6 +267,7 @@ class VpnGenericConfigDriverTestCase(base.BaseTestCase):
                 self.test_dict.url_for_del_src_route,
                 data=jsonutils.dumps(
                     self.test_dict.data_for_del_src_route),
+                headers=self.headers,
                 timeout=self.test_dict.timeout)
 
 
@@ -318,6 +326,7 @@ class RestApiTestCase(base.BaseTestCase):
         self.resp = mock.Mock()
         self.resp = mock.Mock(status_code=200)
         self.test_dict = vpn_test_data.VPNTestData()
+        self.headers = self.test_dict.get_header()
         self.args = {'peer_address': '1.103.2.2'}
         self.fake_resp_dict = {'status': None}
         self.timeout = self.rest_obj.timeout
@@ -336,10 +345,12 @@ class RestApiTestCase(base.BaseTestCase):
             mock_post), (
             mock.patch.object(jsonutils, 'loads',
                               return_value=self.fake_resp_dict)):
-            self.rest_obj.post('create-ipsec-site-conn', self.data)
+            self.rest_obj.post('create-ipsec-site-conn',
+                               self.data, self.headers)
             mock_post.assert_called_with(
                 self.test_dict.url_create_ipsec_conn,
                 data=self.j_data,
+                headers=self.headers,
                 timeout=self.timeout)
 
     def test_put_success(self):
@@ -351,10 +362,12 @@ class RestApiTestCase(base.BaseTestCase):
         self.resp = mock.Mock(status_code=200)
         with mock.patch.object(requests, 'put', return_value=self.resp) as (
                 mock_put):
-            self.rest_obj.put('create-ipsec-site-conn', self.data)
+            self.rest_obj.put('create-ipsec-site-conn',
+                              self.data, self.headers)
             mock_put.assert_called_with(
                 self.test_dict.url_create_ipsec_conn,
                 data=self.j_data,
+                headers=self.headers,
                 timeout=self.timeout)
 
     def test_put_fail(self):
@@ -367,10 +380,12 @@ class RestApiTestCase(base.BaseTestCase):
         with mock.patch.object(requests, 'put', return_value=self.resp) as (
                 mock_put):
 
-            self.rest_obj.put('create-ipsec-site-conn', self.data)
+            self.rest_obj.put('create-ipsec-site-conn',
+                              self.data, self.headers)
             mock_put.assert_called_with(
                 self.test_dict.url_create_ipsec_conn,
                 data=jsonutils.dumps(self.data),
+                headers=self.headers,
                 timeout=self.timeout)
 
     def test_delete_success(self):
@@ -386,11 +401,12 @@ class RestApiTestCase(base.BaseTestCase):
                               return_value=self.fake_resp_dict)):
             self.rest_obj.delete('delete-ipsec-site-conn',
                                  self.args,
-                                 self.data)
+                                 self.headers)
             mock_delete.assert_called_with(
                 self.test_dict.url_delete_ipsec_conn,
                 timeout=self.timeout,
-                data=self.j_data)
+                data=None,
+                headers=self.headers)
 
     def test_get_success(self):
         """
@@ -401,10 +417,12 @@ class RestApiTestCase(base.BaseTestCase):
         self.resp = mock.Mock(status_code=200)
         with mock.patch.object(requests, 'get', return_value=self.resp) as (
                 mock_get):
-            self.rest_obj.get('create-ipsec-site-tunnel', self.data)
+            self.rest_obj.get('create-ipsec-site-tunnel',
+                              self.data, self.headers)
             mock_get.assert_called_with(
                 self.test_dict.url_create_ipsec_tunnel,
                 params=self.data,
+                headers=self.headers,
                 timeout=self.timeout)
 
     def test_get_fail(self):
@@ -416,8 +434,10 @@ class RestApiTestCase(base.BaseTestCase):
         self.resp = mock.Mock(status_code=404)
         with mock.patch.object(requests, 'get', return_value=self.resp) as (
                 mock_get):
-            self.rest_obj.get('create-ipsec-site-tunnel', self.data)
+            self.rest_obj.get('create-ipsec-site-tunnel',
+                              self.data, self.headers)
             mock_get.assert_called_with(
                 self.test_dict.url_create_ipsec_tunnel,
                 params=self.data,
+                headers=self.headers,
                 timeout=self.timeout)

gbpservice/contrib/tests/unit/nfp/configurator/test_data/fw_test_data.py

@@ -470,7 +470,9 @@ class FakeEventGenericConfig(object):
                             'notification_data': {},
                             'resource_type': 'firewall',
                             'service_vendor': 'vyos',
-                            'context': 'APIcontext'},
+                            'context': {'service_vm_context': {'vyos': {
+                                'username': 'name',
+                                'password': 'password'}}}},
                     'firewall': fo._fake_firewall_obj(),
                     'host': fo.host,
                     'resource_data': kwargs}

gbpservice/contrib/tests/unit/nfp/configurator/test_data/vpn_test_data.py

@@ -27,7 +27,10 @@ class VPNTestData(object):
     def __init__(self):
 
         self.context_device = {'notification_data': {},
-                               'resource': 'interfaces'}
+                               'resource': 'interfaces',
+                               'service_vm_context': {'vyos': {
+                                             'username': 'username',
+                                             'password': 'password'}}}
         self.sc = 'sc'
         self.conf = 'conf'
         self.msg = 'msg'
@@ -384,6 +387,12 @@ class VPNTestData(object):
                                  "floating_ip": ""}]}
                                         ]}]}}
 
+    def get_header(self):
+        header = {'username': 'username',
+                  'password': 'password',
+                  'Content-Type': 'application/json'}
+        return header
+
     def make_service_context(self, operation_type=None):
         '''
         Prepares a simple service_info dictionary and appends it to
@@ -391,6 +400,10 @@ class VPNTestData(object):
 
         '''
 
+        agent_info = {'context': {'service_vm_context': {
+                                'vyos': {
+                                    'username': 'username',
+                                    'password': 'password'}}}}
         self.service_info = {}
         self.service_info.update({'vpnservices': self.vpnservice})
         if operation_type is None:
@@ -401,6 +414,7 @@ class VPNTestData(object):
 
         self.service_info.update({'subnets': self.subnet})
         self.context.update({'service_info': self.service_info})
+        self.context.update({'agent_info': agent_info})
         return self.context
 
     def _create_vpnservice_obj(self):

gbpservice/nfp/pecan/constants.py

@@ -28,7 +28,7 @@ controller_mode_map = {
 
 controllers = {
     controller_mode_map[base]: 'gbpservice.nfp.base_configurator.controllers',
-    controller_mode_map[base_with_vm]: ('gbpservice.tests.contrib'
+    controller_mode_map[base_with_vm]: ('gbpservice.contrib'
                            '.nfp_service.reference_configurator.controllers'),
     controller_mode_map[advanced]: ('gbpservice.contrib.nfp.configurator'
                           '.advanced_controller.controller_loader')