Add Exif plugin.
Adds Exif plugin for analysis of Exif tags in TIFF and JPEG file types. Change-Id: I1a83e919eba7d84676cbc411e3c339c27080ff63
This commit is contained in:
parent
2341f2acaf
commit
819c104d9e
|
@ -15,6 +15,7 @@
|
|||
from flask import Blueprint
|
||||
from flask import jsonify
|
||||
from flask import request
|
||||
import json
|
||||
import magic
|
||||
import os
|
||||
from oslo_config.cfg import CONF
|
||||
|
@ -29,7 +30,9 @@ from python_nemesis.exceptions import NotFoundException
|
|||
from python_nemesis.extensions import log
|
||||
from python_nemesis.file_hasher import get_all_hashes
|
||||
from python_nemesis.notifications import submit_worker_notification
|
||||
from python_nemesis.swift import download_raw_content
|
||||
from python_nemesis.swift import upload_to_swift
|
||||
from swiftclient.exceptions import ClientException
|
||||
import uuid
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
|
@ -47,10 +50,23 @@ def api_definition():
|
|||
return ""
|
||||
|
||||
|
||||
def get_artifacts(file_id):
|
||||
artifacts = {}
|
||||
for plugin in CONF.analysis_plugins:
|
||||
try:
|
||||
artifact_name = "%s_%s" % (file_id, plugin)
|
||||
artifact = download_raw_content('artifacts', artifact_name)
|
||||
artifacts[plugin] = json.loads(artifact[1])
|
||||
except ClientException:
|
||||
artifacts[plugin] = None
|
||||
return artifacts
|
||||
|
||||
|
||||
@V1_API.route('/v1/file/<string:req_hash>')
|
||||
def lookup_hash(req_hash):
|
||||
try:
|
||||
result = search_by_hash(req_hash)
|
||||
|
||||
except Exception as err:
|
||||
log.logger.error(str(err))
|
||||
raise NemesisException(str(err))
|
||||
|
@ -66,6 +82,12 @@ def lookup_hash(req_hash):
|
|||
else:
|
||||
add_request(req_hash, 'multiple_found')
|
||||
|
||||
for curr_result in result:
|
||||
if curr_result['status'] == 'complete':
|
||||
file = get_file_by_sha512_hash(curr_result['sha512'])
|
||||
artifacts = get_artifacts(file.file_id)
|
||||
curr_result['artifacts'] = artifacts
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
|
@ -108,7 +130,12 @@ def post_file():
|
|||
os.remove(filename)
|
||||
|
||||
# Send message to worker queue with file details.
|
||||
worker_msg = {"file_uuid": file_uuid, "file_id": file_id}
|
||||
worker_msg = {"file_uuid": file_uuid,
|
||||
"file_id": file_id,
|
||||
"file_size": file_size,
|
||||
"file_type": file_type,
|
||||
"file_hashes": file_hashes}
|
||||
|
||||
submit_worker_notification(worker_msg)
|
||||
|
||||
return jsonify(file_dict)
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
class BasePlugin(object):
|
||||
|
||||
def __init__(self, file_data):
|
||||
self.file_data = file_data
|
||||
|
||||
def analyse(self):
|
||||
return None
|
|
@ -1,19 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
class NemesisPlugin(object):
|
||||
|
||||
def __init__(self):
|
||||
pass
|
|
@ -0,0 +1,47 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import exifread
|
||||
|
||||
|
||||
class NemesisPlugin(object):
|
||||
plugin_name = 'Exif'
|
||||
|
||||
def __init__(self, file_data):
|
||||
self.file_data = file_data
|
||||
|
||||
def analyse(self):
|
||||
exif_formats = ['image/jpeg', 'image/tiff']
|
||||
file_type = self.file_data['file_type']
|
||||
if file_type in exif_formats:
|
||||
tags = self._extract_exif_tags()
|
||||
result = {"success": True,
|
||||
"result": {'exif_tags': tags},
|
||||
"message": None}
|
||||
return result
|
||||
|
||||
else:
|
||||
result = {"success": False,
|
||||
"result": None,
|
||||
"message": ("File format %s is not an EXIF compatible "
|
||||
"format such as JPEG or TIFF, unable to "
|
||||
"perform EXIF extraction." % file_type)}
|
||||
return result
|
||||
|
||||
def _extract_exif_tags(self):
|
||||
with open('/tmp/%s' % self.file_data['file_uuid'], 'rb') as f:
|
||||
tags = exifread.process_file(f)
|
||||
|
||||
ret_tags = {}
|
||||
for k, v in tags.items():
|
||||
ret_tags[k] = v.printable
|
||||
return ret_tags
|
|
@ -1,23 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
class NemesisPlugin(object):
|
||||
plugin_name = 'null_plugin'
|
||||
|
||||
def __init__(self, file_path):
|
||||
self.file_path = file_path
|
||||
|
||||
def analyse(self):
|
||||
return None
|
|
@ -27,6 +27,17 @@ def get_swift_session():
|
|||
return swift_session
|
||||
|
||||
|
||||
def upload_raw_content(container, object_name, content):
|
||||
swift_session = get_swift_session()
|
||||
swift_session.put_object(container, object_name, content)
|
||||
|
||||
|
||||
def download_raw_content(container, object_name):
|
||||
swift_session = get_swift_session()
|
||||
obj = swift_session.get_object(container, object_name)
|
||||
return obj
|
||||
|
||||
|
||||
def upload_to_swift(container, filename, file_id):
|
||||
swift_session = get_swift_session()
|
||||
with open(os.path.join(filename), 'rb') as upload_file:
|
||||
|
|
|
@ -12,12 +12,15 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
from oslo_config.cfg import CONF
|
||||
import oslo_messaging
|
||||
from python_nemesis.db.utilities import update_status_by_file_id
|
||||
from python_nemesis.extensions import log
|
||||
from python_nemesis.swift import delete_from_swift
|
||||
from python_nemesis.swift import download_from_swift
|
||||
from python_nemesis.swift import upload_raw_content
|
||||
from python_nemesis.worker.loader import load_class
|
||||
from python_nemesis.worker_app import create_worker_app
|
||||
|
||||
|
||||
|
@ -39,7 +42,23 @@ class NewFileEndpoint(object):
|
|||
log.logger.info("Fetched file to /tmp/%s" % file_uuid)
|
||||
|
||||
log.logger.info("Running analysis plugins.")
|
||||
log.logger.info("Updating file analysis ")
|
||||
for plugin in CONF.analysis_plugins:
|
||||
log.logger.info("Loading %s analysis plugin." % plugin)
|
||||
plugin_class = load_class('%s.NemesisPlugin' % plugin)
|
||||
plugin_obj = plugin_class(payload)
|
||||
plugin_result = plugin_obj.analyse()
|
||||
|
||||
try:
|
||||
json_result = json.dumps(plugin_result)
|
||||
except Exception:
|
||||
msg = "Failed to parse results from %s plugin." % plugin
|
||||
json_result = json.dumps({"success": False,
|
||||
"message": msg})
|
||||
|
||||
obj_name = '%s_%s' % (file_id, plugin)
|
||||
upload_raw_content('artifacts', obj_name, json_result)
|
||||
|
||||
log.logger.info("Plugin returned %s result." % plugin_result)
|
||||
|
||||
log.logger.info("Cleaning up analysis subject.")
|
||||
delete_from_swift('incoming_files', file_uuid)
|
||||
|
|
Loading…
Reference in New Issue