From 0d97c2988295b4a5ed918297575fe254008dc7d9 Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Wed, 2 Apr 2014 18:03:24 -0400 Subject: [PATCH] Add a custom filter method to scan wrapped dict results Basic filtering is available via Resource._filter_fields and Resource._children but partial filtering of fields is not supported. This change allows a Resource to define custom sanitization methods for individual fields. The specified method is applied after the field has been generated. For nested structures, particularly when elements are not Resource types, it is easier to scan the field after generation and remove specific elements than it is to modify the recursive descent already implemented in to_wrapped_dict(). JobExecutions are filtered with this mechanism to set swift credential values to empty strings and to remove the 'conf' section reported for running Oozie actions. Closes-Bug: #1300291 Change-Id: Ifb0b99a3e13d40306139b0d0021aead69e870205 (cherry picked from commit 3367d9c3632629b900b618c80261d8b6de8086ee) --- sahara/conductor/resource.py | 27 +++++++++++++++++-- sahara/swift/swift_helper.py | 2 ++ sahara/tests/unit/conductor/test_resource.py | 28 +++++++++++++++++--- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/sahara/conductor/resource.py b/sahara/conductor/resource.py index 99c14e2c2a..228898134f 100644 --- a/sahara/conductor/resource.py +++ b/sahara/conductor/resource.py @@ -28,6 +28,7 @@ import datetime import six from sahara.conductor import objects +from sahara.swift import swift_helper from sahara.utils import types @@ -81,6 +82,7 @@ class Resource(types.FrozenDict): _resource_name = 'resource' _children = {} _filter_fields = [] + _sanitize_fields = {} def __init__(self, dct): super(Resource, self).__setattr__('_initial_dict', dct) @@ -147,7 +149,9 @@ class Resource(types.FrozenDict): if refname in self._children: childs_backref = self._children[refname][1] dct[refname] = self._entity_to_dict(entity, childs_backref) - + sanitize = self._sanitize_fields.get(refname) + if sanitize is not None: + dct[refname] = sanitize(self, dct[refname]) return dct def _entity_to_dict(self, entity, childs_backref): @@ -213,8 +217,27 @@ class DataSource(Resource, objects.DataSource): class JobExecution(Resource, objects.JobExecution): + + def sanitize_job_configs(self, job_configs): + if 'configs' in job_configs: + configs = job_configs['configs'] + if swift_helper.HADOOP_SWIFT_USERNAME in configs: + configs[swift_helper.HADOOP_SWIFT_USERNAME] = "" + if swift_helper.HADOOP_SWIFT_PASSWORD in configs: + configs[swift_helper.HADOOP_SWIFT_PASSWORD] = "" + return job_configs + + def sanitize_info(self, info): + if 'actions' in info: + for d in info['actions']: + if 'conf' in d: + del d['conf'] + return info + _resource_name = "job_execution" - _filter_fields = ['extra', 'job_configs'] + _filter_fields = ['extra'] + _sanitize_fields = {'job_configs': sanitize_job_configs, + 'info': sanitize_info} class JobBinary(Resource, objects.JobBinary): diff --git a/sahara/swift/swift_helper.py b/sahara/swift/swift_helper.py index f5bf4d0c47..472dcc9bd6 100644 --- a/sahara/swift/swift_helper.py +++ b/sahara/swift/swift_helper.py @@ -26,6 +26,8 @@ LOG = logging.getLogger(__name__) CONF = cfg.CONF HADOOP_SWIFT_AUTH_URL = 'fs.swift.service.sahara.auth.url' HADOOP_SWIFT_TENANT = 'fs.swift.service.sahara.tenant' +HADOOP_SWIFT_USERNAME = 'fs.swift.service.sahara.username' +HADOOP_SWIFT_PASSWORD = 'fs.swift.service.sahara.password' def _retrieve_tenant(): diff --git a/sahara/tests/unit/conductor/test_resource.py b/sahara/tests/unit/conductor/test_resource.py index 6d3fd22466..1bf9b1ed79 100644 --- a/sahara/tests/unit/conductor/test_resource.py +++ b/sahara/tests/unit/conductor/test_resource.py @@ -18,6 +18,7 @@ import copy import unittest2 from sahara.conductor import resource as r +from sahara.swift import swift_helper from sahara.utils import types @@ -100,6 +101,8 @@ SAMPLE_JOB_EXECUTION = { "extra": {}, "id": "1b0b1874-a261-4d1f-971a-a2cebadeba6c", "info": { + "actions": [{"conf": "some stuff"}, + {"conf": "more stuff"}], "status": "Pending" }, "input_id": "b5ddde55-594e-428f-9040-028be81eb3c2", @@ -109,8 +112,9 @@ SAMPLE_JOB_EXECUTION = { "bill" ], "configs": { - "fs.swift.service.savanna.password": "openstack", - "fs.swift.service.savanna.username": "admin" + swift_helper.HADOOP_SWIFT_PASSWORD: "openstack", + swift_helper.HADOOP_SWIFT_USERNAME: "admin", + "myfavoriteconfig": 1 } }, "job_id": "d0f3e397-7bef-42f9-a4db-e5a96059246e", @@ -203,6 +207,22 @@ class TestResource(unittest2.TestCase): def test_job_execution_filter_credentials(self): job_exec = r.JobExecution(SAMPLE_JOB_EXECUTION) - wrapped_dict = job_exec.to_wrapped_dict() + self.assertIn('extra', job_exec) + self.assertIn(swift_helper.HADOOP_SWIFT_PASSWORD, + job_exec['job_configs']['configs']) + self.assertIn(swift_helper.HADOOP_SWIFT_USERNAME, + job_exec['job_configs']['configs']) + for a in job_exec['info']['actions']: + self.assertIn('conf', a) + + wrapped_dict = job_exec.to_wrapped_dict()['job_execution'] self.assertNotIn('extra', wrapped_dict) - self.assertNotIn('job_configs', wrapped_dict) + + configs = wrapped_dict['job_configs']['configs'] + self.assertEqual("", + configs[swift_helper.HADOOP_SWIFT_PASSWORD]) + self.assertEqual("", + configs[swift_helper.HADOOP_SWIFT_USERNAME]) + + for a in wrapped_dict['info']['actions']: + self.assertNotIn('conf', a)