diff --git a/nemesis.conf.sample b/nemesis.conf.sample index a67e641..7cc17f6 100644 --- a/nemesis.conf.sample +++ b/nemesis.conf.sample @@ -13,3 +13,13 @@ delay_auth_decision = True [sqlalchemy] database_uri = sqlite:////tmp/nemesis.db echo = true + +[swift] +user = +password = +domain = +project = +auth_version = +auth_uri = +region = +container = diff --git a/python_nemesis/api/v1/__init__.py b/python_nemesis/api/v1/__init__.py index 710c5ad..1c29571 100644 --- a/python_nemesis/api/v1/__init__.py +++ b/python_nemesis/api/v1/__init__.py @@ -28,6 +28,7 @@ from python_nemesis.exceptions import NotFoundException from python_nemesis.extensions import db from python_nemesis.extensions import log from python_nemesis.file_hasher import get_all_hashes +from python_nemesis.swift import upload_to_swift import uuid from werkzeug.utils import secure_filename @@ -94,6 +95,7 @@ def post_file(): current_file.last_updated = datetime.datetime.now() current_file.status = 'analysing' db.session.commit() + file_id = current_file.file_id file_dict = current_file.to_dict() else: @@ -103,6 +105,11 @@ def post_file(): file_hashes['sha512'], file_size, current_user.user_id) + file_id = file.file_id file_dict = file.to_dict() + # Upload to swift and remove the local temp file. + upload_to_swift(filename, file_id) + os.remove(filename) + return jsonify(file_dict) diff --git a/python_nemesis/config.py b/python_nemesis/config.py index ffa9d97..a6dd748 100644 --- a/python_nemesis/config.py +++ b/python_nemesis/config.py @@ -35,6 +35,18 @@ IDENTITY_OPTS = [ cfg.StrOpt('password') ] +SWIFT_OPT_GRP = cfg.OptGroup(name='swift') +SWIFT_OPTS = [ + cfg.StrOpt('user'), + cfg.StrOpt('password'), + cfg.StrOpt('domain'), + cfg.StrOpt('project'), + cfg.IntOpt('auth_version'), + cfg.StrOpt('auth_uri'), + cfg.StrOpt('region'), + cfg.StrOpt('container') +] + def register_opts(conf, config_file): '''Register Oslo Configuration Options from a provided config file. @@ -49,6 +61,8 @@ def register_opts(conf, config_file): conf.register_opts(SQLALCHEMY_OPTS, SQLALCHEMY_OPT_GRP) conf.register_group(IDENTITY_OPT_GRP) conf.register_opts(IDENTITY_OPTS, IDENTITY_OPT_GRP) + conf.register_group(SWIFT_OPT_GRP) + conf.register_opts(SWIFT_OPTS, SWIFT_OPT_GRP) def collect_sqlalchemy_opts(app, conf): diff --git a/python_nemesis/db/models.py b/python_nemesis/db/models.py index d66a043..eaf8511 100644 --- a/python_nemesis/db/models.py +++ b/python_nemesis/db/models.py @@ -29,8 +29,7 @@ class Files(db.Model): file_lookup = db.relationship("FileLookupRequest") def to_dict(self): - return {"file_id": self.file_id, - "sha512": self.sha512_hash, + return {"sha512": self.sha512_hash, "sha256": self.sha256_hash, "sha1": self.sha1_hash, "md5": self.md5_hash, diff --git a/python_nemesis/db/utilities.py b/python_nemesis/db/utilities.py index 7fa5b71..b6e8c06 100644 --- a/python_nemesis/db/utilities.py +++ b/python_nemesis/db/utilities.py @@ -33,8 +33,7 @@ def add_request(lookup_hash, result, file_id=None): def search_by_hash(lookup_hash): results = db.session.query(Files). \ - filter(or_(Files.file_id == lookup_hash, - Files.sha512_hash == lookup_hash, + filter(or_(Files.sha512_hash == lookup_hash, Files.sha256_hash == lookup_hash, Files.sha1_hash == lookup_hash, Files.md5_hash == lookup_hash)) diff --git a/python_nemesis/swift.py b/python_nemesis/swift.py new file mode 100644 index 0000000..5b404f9 --- /dev/null +++ b/python_nemesis/swift.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from flask import current_app +import os +import swiftclient.client as swiftclient + + +def upload_to_swift(filename, file_id): + config = current_app.config['cfg'] + auth_version = config.swift.auth_version + swift_session = swiftclient.Connection(authurl=config.swift.auth_uri, + user=config.swift.user, + key=config.swift.password, + tenant_name=config.swift.project, + auth_version=auth_version) + + with open(os.path.join(filename), 'rb') as upload_file: + container = config.swift.container.encode('utf-8') + file_id = str(file_id).encode('utf-8') + swift_session.put_object(container, file_id, upload_file) diff --git a/requirements.txt b/requirements.txt index 4a8ebcd..08374db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ flask-keystone>=0.2 # Apache-2.0 oslo.config>=5.1.0 # Apache-2.0 oslo.messaging>=5.29.0 # Apache-2.0 oslo.log>=3.30.0 # Apache-2.0 +python-swiftclient>=3.2.0 # Apache-2.0 keystonemiddleware>=4.17.0 # Apache-2.0