Enable Vertica load via curl

Vertica comes with a User Defined Load function that takes a URL as a
load source. This can be used to load files that are stored in Swift. As
this is a common use case, it is valuable to enable this by default.

This can be done in the post-prepare method for Vertica. A new UDL_LIBS
list has been added that describes any UDLs to be loaded into the
database. This change only has one entry - the curl function.

Change-Id: I0d03f509629298e68f2a2b83bb061e3c27b949ef
Depends-On: Icb7d9b1551124a700ad767eeba10be939ffa9aaa
Closes-Bug: #1531510

trove/guestagent/datastore/experimental/vertica/manager.py

@@ -59,6 +59,7 @@ class Manager(manager.Manager):
         if cluster_config is None:
             self.app.install_vertica()
             self.app.create_db()
+            self.app.add_udls()
         elif cluster_config['instance_type'] not in ["member", "master"]:
             raise RuntimeError(_("Bad cluster configuration: instance type "
                                "given as %s.") %
@@ -102,6 +103,7 @@ class Manager(manager.Manager):
         try:
             LOG.debug("Installing cluster on members: %s." % members)
             self.app.install_cluster(members)
+            self.app.add_udls()
             LOG.debug("install_cluster call has finished.")
         except Exception:
             LOG.exception(_('Cluster installation failed.'))

trove/guestagent/datastore/experimental/vertica/service.py

@@ -182,6 +182,44 @@ class VerticaApp(object):
         self._generate_database_password()
         LOG.info(_("install_vertica completed."))
 
+    def add_udls(self):
+        """Load the user defined load libraries into the database."""
+        LOG.info(_("Adding configured user defined load libraries."))
+        password = self._get_database_password()
+        loaded_udls = []
+        for lib in system.UDL_LIBS:
+            func_name = lib['func_name']
+            lib_name = lib['lib_name']
+            language = lib['language']
+            factory = lib['factory']
+            path = lib['path']
+            if os.path.isfile(path):
+                LOG.debug("Adding the %s library as %s." %
+                          (func_name, lib_name))
+                out, err = system.exec_vsql_command(
+                    password,
+                    system.CREATE_LIBRARY % (lib_name, path)
+                )
+                if err:
+                    LOG.error(err)
+                    raise RuntimeError(_("Failed to create library %s.")
+                                       % lib_name)
+                out, err = system.exec_vsql_command(
+                    password,
+                    system.CREATE_SOURCE % (func_name, language,
+                                            factory, lib_name)
+                )
+                if err:
+                    LOG.error(err)
+                    raise RuntimeError(_("Failed to create source %s.")
+                                       % func_name)
+                loaded_udls.append(func_name)
+            else:
+                LOG.warning("Skipping %s as path %s not found." %
+                            (func_name, path))
+        LOG.info(_("The following UDL functions are available for use: %s")
+                 % loaded_udls)
+
     def _generate_database_password(self):
         """Generate and write the password to vertica.cnf file."""
         config = ConfigParser.ConfigParser()

trove/guestagent/datastore/experimental/vertica/system.py

@@ -39,6 +39,17 @@ USER_EXISTS = ("/opt/vertica/bin/vsql -w '%s' -c "
 VERTICA_AGENT_SERVICE_COMMAND = "service vertica_agent %s"
 VERTICA_CONF = "/etc/vertica.cnf"
 INSTALL_TIMEOUT = 1000
+CREATE_LIBRARY = "CREATE LIBRARY %s AS '%s'"
+CREATE_SOURCE = "CREATE SOURCE %s AS LANGUAGE '%s' NAME '%s' LIBRARY %s"
+UDL_LIBS = [
+    {
+        'func_name': "curl",
+        'lib_name': "curllib",
+        'language': "C++",
+        'factory': "CurlSourceFactory",
+        'path': "/opt/vertica/sdk/examples/build/cURLLib.so"
+    },
+]
 
 
 def shell_execute(command, command_executor="root"):

trove/tests/unittests/guestagent/test_vertica_manager.py

@@ -24,6 +24,7 @@ from trove.guestagent.datastore.experimental.vertica.manager import Manager
 from trove.guestagent.datastore.experimental.vertica.service import (
     VerticaAppStatus)
 from trove.guestagent.datastore.experimental.vertica.service import VerticaApp
+from trove.guestagent.datastore.experimental.vertica import system
 from trove.guestagent import dbaas
 from trove.guestagent import volume
 from trove.guestagent.volume import VolumeDevice
@@ -51,6 +52,7 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
         self.origin_is_root_enabled = VerticaApp.is_root_enabled
         self.origin_prepare_for_install_vertica = (
             VerticaApp.prepare_for_install_vertica)
+        self.origin_add_udls = VerticaApp.add_udls
 
     def tearDown(self):
         super(GuestAgentManagerTest, self).tearDown()
@@ -70,6 +72,7 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
         VerticaApp.is_root_enabled = self.origin_is_root_enabled
         VerticaApp.prepare_for_install_vertica = (
             self.origin_prepare_for_install_vertica)
+        VerticaApp.add_udls = self.origin_add_udls
 
     def test_update_status(self):
         mock_status = MagicMock()
@@ -108,6 +111,7 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
         VerticaApp.install_vertica = MagicMock(return_value=None)
         VerticaApp.create_db = MagicMock(return_value=None)
         VerticaApp.prepare_for_install_vertica = MagicMock(return_value=None)
+        VerticaApp.add_udls = MagicMock()
         # invocation
         self.manager.prepare(context=self.context, packages=packages,
                              config_contents=config_content,
@@ -133,6 +137,7 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
         VerticaApp.prepare_for_install_vertica.assert_any_call()
         VerticaApp.install_vertica.assert_any_call()
         VerticaApp.create_db.assert_any_call()
+        VerticaApp.add_udls.assert_any_call()
 
     def test_prepare_pkg(self):
         self._prepare_dynamic(['vertica'])
@@ -161,12 +166,15 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
     @patch.object(VerticaApp, 'install_vertica')
     @patch.object(VerticaApp, '_export_conf_to_members')
     @patch.object(VerticaApp, 'create_db')
-    def test_install_cluster(self, mock_install, mock_export, mock_create_db):
+    @patch.object(VerticaApp, 'add_udls')
+    def test_install_cluster(self, mock_udls, mock_install, mock_export,
+                             mock_create_db):
         members = ['test1', 'test2']
         self.manager.install_cluster(self.context, members)
         mock_install.assert_called_with('test1,test2')
         mock_export.assert_called_with(members)
         mock_create_db.assert_called_with('test1,test2')
+        mock_udls.assert_any_call()
 
     @patch.object(VerticaAppStatus, 'set_status')
     @patch.object(VerticaApp, 'install_cluster',
@@ -179,6 +187,26 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
                           self.context, members)
         mock_set_status.assert_called_with(rd_instance.ServiceStatuses.FAILED)
 
+    @patch.object(VerticaApp, '_get_database_password')
+    @patch.object(path, 'isfile')
+    @patch.object(system, 'exec_vsql_command')
+    def test_add_udls(self, mock_vsql, mock_isfile, mock_pwd):
+        mock_vsql.return_value = (None, None)
+        password = 'password'
+        mock_pwd.return_value = password
+        mock_isfile.return_value = True
+        self.manager.app.add_udls()
+        mock_vsql.assert_any_call(
+            password,
+            "CREATE LIBRARY curllib AS "
+            "'/opt/vertica/sdk/examples/build/cURLLib.so'"
+        )
+        mock_vsql.assert_any_call(
+            password,
+            "CREATE SOURCE curl AS LANGUAGE 'C++' NAME 'CurlSourceFactory' "
+            "LIBRARY curllib"
+        )
+
     @patch.object(volume.VolumeDevice, 'mount_points', return_value=[])
     @patch.object(volume.VolumeDevice, 'unmount_device', return_value=None)
     @patch.object(volume.VolumeDevice, 'mount', return_value=None)
@@ -186,6 +214,7 @@ class GuestAgentManagerTest(trove_testtools.TestCase):
     @patch.object(volume.VolumeDevice, 'format', return_value=None)
     @patch.object(VerticaApp, 'prepare_for_install_vertica')
     @patch.object(VerticaApp, 'install_if_needed')
+    @patch.object(VerticaApp, 'add_udls')
     @patch.object(VerticaAppStatus, 'begin_install')
     def _prepare_method(self, instance_id, instance_type, *args):
         cluster_config = {"id": instance_id,