Add a custom filter method to scan wrapped dict results

Basic filtering is available via Resource._filter_fields and
Resource._children but partial filtering of fields is not supported.

This change allows a Resource to define custom sanitization methods
for individual fields. The specified method is applied after the
field has been generated.

For nested structures, particularly when elements are not Resource
types, it is easier to scan the field after generation and remove
specific elements than it is to modify the recursive descent already
implemented in to_wrapped_dict().

JobExecutions are filtered with this mechanism to set swift credential
values to empty strings and to remove the 'conf' section reported
for running Oozie actions.

Closes-Bug: #1300291
Change-Id: Ifb0b99a3e13d40306139b0d0021aead69e870205
(cherry picked from commit 3367d9c363)

sahara/conductor/resource.py

@@ -28,6 +28,7 @@ import datetime
 import six
 
 from sahara.conductor import objects
+from sahara.swift import swift_helper
 from sahara.utils import types
 
 
@@ -81,6 +82,7 @@ class Resource(types.FrozenDict):
     _resource_name = 'resource'
     _children = {}
     _filter_fields = []
+    _sanitize_fields = {}
 
     def __init__(self, dct):
         super(Resource, self).__setattr__('_initial_dict', dct)
@@ -147,7 +149,9 @@ class Resource(types.FrozenDict):
                 if refname in self._children:
                     childs_backref = self._children[refname][1]
                 dct[refname] = self._entity_to_dict(entity, childs_backref)
-
+                sanitize = self._sanitize_fields.get(refname)
+                if sanitize is not None:
+                    dct[refname] = sanitize(self, dct[refname])
         return dct
 
     def _entity_to_dict(self, entity, childs_backref):
@@ -213,8 +217,27 @@ class DataSource(Resource, objects.DataSource):
 
 
 class JobExecution(Resource, objects.JobExecution):
+
+    def sanitize_job_configs(self, job_configs):
+        if 'configs' in job_configs:
+            configs = job_configs['configs']
+            if swift_helper.HADOOP_SWIFT_USERNAME in configs:
+                configs[swift_helper.HADOOP_SWIFT_USERNAME] = ""
+            if swift_helper.HADOOP_SWIFT_PASSWORD in configs:
+                configs[swift_helper.HADOOP_SWIFT_PASSWORD] = ""
+        return job_configs
+
+    def sanitize_info(self, info):
+        if 'actions' in info:
+            for d in info['actions']:
+                if 'conf' in d:
+                    del d['conf']
+        return info
+
     _resource_name = "job_execution"
-    _filter_fields = ['extra', 'job_configs']
+    _filter_fields = ['extra']
+    _sanitize_fields = {'job_configs': sanitize_job_configs,
+                        'info': sanitize_info}
 
 
 class JobBinary(Resource, objects.JobBinary):

sahara/swift/swift_helper.py

@@ -26,6 +26,8 @@ LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 HADOOP_SWIFT_AUTH_URL = 'fs.swift.service.sahara.auth.url'
 HADOOP_SWIFT_TENANT = 'fs.swift.service.sahara.tenant'
+HADOOP_SWIFT_USERNAME = 'fs.swift.service.sahara.username'
+HADOOP_SWIFT_PASSWORD = 'fs.swift.service.sahara.password'
 
 
 def _retrieve_tenant():

sahara/tests/unit/conductor/test_resource.py

@@ -18,6 +18,7 @@ import copy
 import unittest2
 
 from sahara.conductor import resource as r
+from sahara.swift import swift_helper
 from sahara.utils import types
 
 
@@ -100,6 +101,8 @@ SAMPLE_JOB_EXECUTION = {
     "extra": {},
     "id": "1b0b1874-a261-4d1f-971a-a2cebadeba6c",
     "info": {
+        "actions": [{"conf": "some stuff"},
+                    {"conf": "more stuff"}],
         "status": "Pending"
     },
     "input_id": "b5ddde55-594e-428f-9040-028be81eb3c2",
@@ -109,8 +112,9 @@ SAMPLE_JOB_EXECUTION = {
             "bill"
         ],
         "configs": {
-            "fs.swift.service.savanna.password": "openstack",
-            "fs.swift.service.savanna.username": "admin"
+            swift_helper.HADOOP_SWIFT_PASSWORD: "openstack",
+            swift_helper.HADOOP_SWIFT_USERNAME: "admin",
+            "myfavoriteconfig": 1
         }
     },
     "job_id": "d0f3e397-7bef-42f9-a4db-e5a96059246e",
@@ -203,6 +207,22 @@ class TestResource(unittest2.TestCase):
 
     def test_job_execution_filter_credentials(self):
         job_exec = r.JobExecution(SAMPLE_JOB_EXECUTION)
-        wrapped_dict = job_exec.to_wrapped_dict()
+        self.assertIn('extra', job_exec)
+        self.assertIn(swift_helper.HADOOP_SWIFT_PASSWORD,
+                      job_exec['job_configs']['configs'])
+        self.assertIn(swift_helper.HADOOP_SWIFT_USERNAME,
+                      job_exec['job_configs']['configs'])
+        for a in job_exec['info']['actions']:
+            self.assertIn('conf', a)
+
+        wrapped_dict = job_exec.to_wrapped_dict()['job_execution']
         self.assertNotIn('extra', wrapped_dict)
-        self.assertNotIn('job_configs', wrapped_dict)
+
+        configs = wrapped_dict['job_configs']['configs']
+        self.assertEqual("",
+                         configs[swift_helper.HADOOP_SWIFT_PASSWORD])
+        self.assertEqual("",
+                         configs[swift_helper.HADOOP_SWIFT_USERNAME])
+
+        for a in wrapped_dict['info']['actions']:
+            self.assertNotIn('conf', a)