Add Exif plugin.

Adds Exif plugin for analysis of Exif tags in TIFF and JPEG file types.

Change-Id: I1a83e919eba7d84676cbc411e3c339c27080ff63
This commit is contained in:
Robert Putt 2018-03-08 22:56:42 +00:00
parent 2341f2acaf
commit 819c104d9e
7 changed files with 126 additions and 44 deletions

View File

@ -15,6 +15,7 @@
from flask import Blueprint
from flask import jsonify
from flask import request
import json
import magic
import os
from oslo_config.cfg import CONF
@ -29,7 +30,9 @@ from python_nemesis.exceptions import NotFoundException
from python_nemesis.extensions import log
from python_nemesis.file_hasher import get_all_hashes
from python_nemesis.notifications import submit_worker_notification
from python_nemesis.swift import download_raw_content
from python_nemesis.swift import upload_to_swift
from swiftclient.exceptions import ClientException
import uuid
from werkzeug.utils import secure_filename
@ -47,10 +50,23 @@ def api_definition():
return ""
def get_artifacts(file_id):
artifacts = {}
for plugin in CONF.analysis_plugins:
try:
artifact_name = "%s_%s" % (file_id, plugin)
artifact = download_raw_content('artifacts', artifact_name)
artifacts[plugin] = json.loads(artifact[1])
except ClientException:
artifacts[plugin] = None
return artifacts
@V1_API.route('/v1/file/<string:req_hash>')
def lookup_hash(req_hash):
try:
result = search_by_hash(req_hash)
except Exception as err:
log.logger.error(str(err))
raise NemesisException(str(err))
@ -66,6 +82,12 @@ def lookup_hash(req_hash):
else:
add_request(req_hash, 'multiple_found')
for curr_result in result:
if curr_result['status'] == 'complete':
file = get_file_by_sha512_hash(curr_result['sha512'])
artifacts = get_artifacts(file.file_id)
curr_result['artifacts'] = artifacts
return jsonify(result)
@ -108,7 +130,12 @@ def post_file():
os.remove(filename)
# Send message to worker queue with file details.
worker_msg = {"file_uuid": file_uuid, "file_id": file_id}
worker_msg = {"file_uuid": file_uuid,
"file_id": file_id,
"file_size": file_size,
"file_type": file_type,
"file_hashes": file_hashes}
submit_worker_notification(worker_msg)
return jsonify(file_dict)

View File

@ -0,0 +1,20 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class BasePlugin(object):
def __init__(self, file_data):
self.file_data = file_data
def analyse(self):
return None

View File

@ -1,19 +0,0 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class NemesisPlugin(object):
def __init__(self):
pass

View File

@ -0,0 +1,47 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import exifread
class NemesisPlugin(object):
plugin_name = 'Exif'
def __init__(self, file_data):
self.file_data = file_data
def analyse(self):
exif_formats = ['image/jpeg', 'image/tiff']
file_type = self.file_data['file_type']
if file_type in exif_formats:
tags = self._extract_exif_tags()
result = {"success": True,
"result": {'exif_tags': tags},
"message": None}
return result
else:
result = {"success": False,
"result": None,
"message": ("File format %s is not an EXIF compatible "
"format such as JPEG or TIFF, unable to "
"perform EXIF extraction." % file_type)}
return result
def _extract_exif_tags(self):
with open('/tmp/%s' % self.file_data['file_uuid'], 'rb') as f:
tags = exifread.process_file(f)
ret_tags = {}
for k, v in tags.items():
ret_tags[k] = v.printable
return ret_tags

View File

@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class NemesisPlugin(object):
plugin_name = 'null_plugin'
def __init__(self, file_path):
self.file_path = file_path
def analyse(self):
return None

View File

@ -27,6 +27,17 @@ def get_swift_session():
return swift_session
def upload_raw_content(container, object_name, content):
swift_session = get_swift_session()
swift_session.put_object(container, object_name, content)
def download_raw_content(container, object_name):
swift_session = get_swift_session()
obj = swift_session.get_object(container, object_name)
return obj
def upload_to_swift(container, filename, file_id):
swift_session = get_swift_session()
with open(os.path.join(filename), 'rb') as upload_file:

View File

@ -12,12 +12,15 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
from oslo_config.cfg import CONF
import oslo_messaging
from python_nemesis.db.utilities import update_status_by_file_id
from python_nemesis.extensions import log
from python_nemesis.swift import delete_from_swift
from python_nemesis.swift import download_from_swift
from python_nemesis.swift import upload_raw_content
from python_nemesis.worker.loader import load_class
from python_nemesis.worker_app import create_worker_app
@ -39,7 +42,23 @@ class NewFileEndpoint(object):
log.logger.info("Fetched file to /tmp/%s" % file_uuid)
log.logger.info("Running analysis plugins.")
log.logger.info("Updating file analysis ")
for plugin in CONF.analysis_plugins:
log.logger.info("Loading %s analysis plugin." % plugin)
plugin_class = load_class('%s.NemesisPlugin' % plugin)
plugin_obj = plugin_class(payload)
plugin_result = plugin_obj.analyse()
try:
json_result = json.dumps(plugin_result)
except Exception:
msg = "Failed to parse results from %s plugin." % plugin
json_result = json.dumps({"success": False,
"message": msg})
obj_name = '%s_%s' % (file_id, plugin)
upload_raw_content('artifacts', obj_name, json_result)
log.logger.info("Plugin returned %s result." % plugin_result)
log.logger.info("Cleaning up analysis subject.")
delete_from_swift('incoming_files', file_uuid)