Add Exif plugin.

Adds Exif plugin for analysis of Exif tags in TIFF and JPEG file types.

Change-Id: I1a83e919eba7d84676cbc411e3c339c27080ff63

python_nemesis/api/v1/__init__.py

@@ -15,6 +15,7 @@
 from flask import Blueprint
 from flask import jsonify
 from flask import request
+import json
 import magic
 import os
 from oslo_config.cfg import CONF
@@ -29,7 +30,9 @@ from python_nemesis.exceptions import NotFoundException
 from python_nemesis.extensions import log
 from python_nemesis.file_hasher import get_all_hashes
 from python_nemesis.notifications import submit_worker_notification
+from python_nemesis.swift import download_raw_content
 from python_nemesis.swift import upload_to_swift
+from swiftclient.exceptions import ClientException
 import uuid
 from werkzeug.utils import secure_filename
 
@@ -47,10 +50,23 @@ def api_definition():
     return ""
 
 
+def get_artifacts(file_id):
+    artifacts = {}
+    for plugin in CONF.analysis_plugins:
+        try:
+            artifact_name = "%s_%s" % (file_id, plugin)
+            artifact = download_raw_content('artifacts', artifact_name)
+            artifacts[plugin] = json.loads(artifact[1])
+        except ClientException:
+            artifacts[plugin] = None
+    return artifacts
+
+
 @V1_API.route('/v1/file/<string:req_hash>')
 def lookup_hash(req_hash):
     try:
         result = search_by_hash(req_hash)
+
     except Exception as err:
         log.logger.error(str(err))
         raise NemesisException(str(err))
@@ -66,6 +82,12 @@ def lookup_hash(req_hash):
     else:
         add_request(req_hash, 'multiple_found')
 
+    for curr_result in result:
+        if curr_result['status'] == 'complete':
+            file = get_file_by_sha512_hash(curr_result['sha512'])
+            artifacts = get_artifacts(file.file_id)
+            curr_result['artifacts'] = artifacts
+
     return jsonify(result)
 
 
@@ -108,7 +130,12 @@ def post_file():
     os.remove(filename)
 
     # Send message to worker queue with file details.
-    worker_msg = {"file_uuid": file_uuid, "file_id": file_id}
+    worker_msg = {"file_uuid": file_uuid,
+                  "file_id": file_id,
+                  "file_size": file_size,
+                  "file_type": file_type,
+                  "file_hashes": file_hashes}
+
     submit_worker_notification(worker_msg)
 
     return jsonify(file_dict)

python_nemesis/plugins/__init__.py

@@ -0,0 +1,20 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class BasePlugin(object):
+
+    def __init__(self, file_data):
+        self.file_data = file_data
+
+    def analyse(self):
+        return None

python_nemesis/plugins/clamav.py

@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-class NemesisPlugin(object):
-
-    def __init__(self):
-        pass

python_nemesis/plugins/exif.py.plugin

@@ -0,0 +1,47 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import exifread
+
+
+class NemesisPlugin(object):
+    plugin_name = 'Exif'
+
+    def __init__(self, file_data):
+        self.file_data = file_data
+
+    def analyse(self):
+        exif_formats = ['image/jpeg', 'image/tiff']
+        file_type = self.file_data['file_type']
+        if file_type in exif_formats:
+            tags = self._extract_exif_tags()
+            result = {"success": True,
+                      "result": {'exif_tags': tags},
+                      "message": None}
+            return result
+
+        else:
+            result = {"success": False,
+                      "result": None,
+                      "message": ("File format %s is not an EXIF compatible "
+                                  "format such as JPEG or TIFF, unable to "
+                                  "perform EXIF extraction." % file_type)}
+            return result
+
+    def _extract_exif_tags(self):
+        with open('/tmp/%s' % self.file_data['file_uuid'], 'rb') as f:
+            tags = exifread.process_file(f)
+
+        ret_tags = {}
+        for k, v in tags.items():
+            ret_tags[k] = v.printable
+        return ret_tags

python_nemesis/plugins/null_plugin.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-class NemesisPlugin(object):
-    plugin_name = 'null_plugin'
-
-    def __init__(self, file_path):
-        self.file_path = file_path
-
-    def analyse(self):
-        return None

python_nemesis/swift.py

@@ -27,6 +27,17 @@ def get_swift_session():
     return swift_session
 
 
+def upload_raw_content(container, object_name, content):
+    swift_session = get_swift_session()
+    swift_session.put_object(container, object_name, content)
+
+
+def download_raw_content(container, object_name):
+    swift_session = get_swift_session()
+    obj = swift_session.get_object(container, object_name)
+    return obj
+
+
 def upload_to_swift(container, filename, file_id):
     swift_session = get_swift_session()
     with open(os.path.join(filename), 'rb') as upload_file:

python_nemesis/worker/__init__.py

@@ -12,12 +12,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import json
 from oslo_config.cfg import CONF
 import oslo_messaging
 from python_nemesis.db.utilities import update_status_by_file_id
 from python_nemesis.extensions import log
 from python_nemesis.swift import delete_from_swift
 from python_nemesis.swift import download_from_swift
+from python_nemesis.swift import upload_raw_content
+from python_nemesis.worker.loader import load_class
 from python_nemesis.worker_app import create_worker_app
 
 
@@ -39,7 +42,23 @@ class NewFileEndpoint(object):
             log.logger.info("Fetched file to /tmp/%s" % file_uuid)
 
             log.logger.info("Running analysis plugins.")
-            log.logger.info("Updating file analysis ")
+            for plugin in CONF.analysis_plugins:
+                log.logger.info("Loading %s analysis plugin." % plugin)
+                plugin_class = load_class('%s.NemesisPlugin' % plugin)
+                plugin_obj = plugin_class(payload)
+                plugin_result = plugin_obj.analyse()
+
+                try:
+                    json_result = json.dumps(plugin_result)
+                except Exception:
+                    msg = "Failed to parse results from %s plugin." % plugin
+                    json_result = json.dumps({"success": False,
+                                              "message": msg})
+
+                obj_name = '%s_%s' % (file_id, plugin)
+                upload_raw_content('artifacts', obj_name, json_result)
+
+                log.logger.info("Plugin returned %s result." % plugin_result)
 
             log.logger.info("Cleaning up analysis subject.")
             delete_from_swift('incoming_files', file_uuid)