Upload files sent to API to swift.

Upload files sent to API to swift container.

Change-Id: I6f7d541952fc0c6376e913990ba287b1030bc10c

nemesis.conf.sample

@@ -13,3 +13,13 @@ delay_auth_decision = True
 [sqlalchemy]
 database_uri = sqlite:////tmp/nemesis.db
 echo = true
+
+[swift]
+user =
+password =
+domain =
+project =
+auth_version =
+auth_uri =
+region =
+container =

python_nemesis/api/v1/__init__.py

@@ -28,6 +28,7 @@ from python_nemesis.exceptions import NotFoundException
 from python_nemesis.extensions import db
 from python_nemesis.extensions import log
 from python_nemesis.file_hasher import get_all_hashes
+from python_nemesis.swift import upload_to_swift
 import uuid
 from werkzeug.utils import secure_filename
 
@@ -94,6 +95,7 @@ def post_file():
         current_file.last_updated = datetime.datetime.now()
         current_file.status = 'analysing'
         db.session.commit()
+        file_id = current_file.file_id
         file_dict = current_file.to_dict()
 
     else:
@@ -103,6 +105,11 @@ def post_file():
                                file_hashes['sha512'],
                                file_size,
                                current_user.user_id)
+        file_id = file.file_id
         file_dict = file.to_dict()
 
+    # Upload to swift and remove the local temp file.
+    upload_to_swift(filename, file_id)
+    os.remove(filename)
+
     return jsonify(file_dict)

python_nemesis/config.py

@@ -35,6 +35,18 @@ IDENTITY_OPTS = [
     cfg.StrOpt('password')
 ]
 
+SWIFT_OPT_GRP = cfg.OptGroup(name='swift')
+SWIFT_OPTS = [
+    cfg.StrOpt('user'),
+    cfg.StrOpt('password'),
+    cfg.StrOpt('domain'),
+    cfg.StrOpt('project'),
+    cfg.IntOpt('auth_version'),
+    cfg.StrOpt('auth_uri'),
+    cfg.StrOpt('region'),
+    cfg.StrOpt('container')
+]
+
 
 def register_opts(conf, config_file):
     '''Register Oslo Configuration Options from a provided config file.
@@ -49,6 +61,8 @@ def register_opts(conf, config_file):
     conf.register_opts(SQLALCHEMY_OPTS, SQLALCHEMY_OPT_GRP)
     conf.register_group(IDENTITY_OPT_GRP)
     conf.register_opts(IDENTITY_OPTS, IDENTITY_OPT_GRP)
+    conf.register_group(SWIFT_OPT_GRP)
+    conf.register_opts(SWIFT_OPTS, SWIFT_OPT_GRP)
 
 
 def collect_sqlalchemy_opts(app, conf):

python_nemesis/db/models.py

@@ -29,8 +29,7 @@ class Files(db.Model):
     file_lookup = db.relationship("FileLookupRequest")
 
     def to_dict(self):
-        return {"file_id": self.file_id,
-                "sha512": self.sha512_hash,
+        return {"sha512": self.sha512_hash,
                 "sha256": self.sha256_hash,
                 "sha1": self.sha1_hash,
                 "md5": self.md5_hash,

python_nemesis/db/utilities.py

@@ -33,8 +33,7 @@ def add_request(lookup_hash, result, file_id=None):
 
 def search_by_hash(lookup_hash):
     results = db.session.query(Files). \
-        filter(or_(Files.file_id == lookup_hash,
-                   Files.sha512_hash == lookup_hash,
+        filter(or_(Files.sha512_hash == lookup_hash,
                    Files.sha256_hash == lookup_hash,
                    Files.sha1_hash == lookup_hash,
                    Files.md5_hash == lookup_hash))

python_nemesis/swift.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from flask import current_app
+import os
+import swiftclient.client as swiftclient
+
+
+def upload_to_swift(filename, file_id):
+    config = current_app.config['cfg']
+    auth_version = config.swift.auth_version
+    swift_session = swiftclient.Connection(authurl=config.swift.auth_uri,
+                                           user=config.swift.user,
+                                           key=config.swift.password,
+                                           tenant_name=config.swift.project,
+                                           auth_version=auth_version)
+
+    with open(os.path.join(filename), 'rb') as upload_file:
+        container = config.swift.container.encode('utf-8')
+        file_id = str(file_id).encode('utf-8')
+        swift_session.put_object(container, file_id, upload_file)

requirements.txt

@@ -11,4 +11,5 @@ flask-keystone>=0.2 # Apache-2.0
 oslo.config>=5.1.0 # Apache-2.0
 oslo.messaging>=5.29.0 # Apache-2.0
 oslo.log>=3.30.0 # Apache-2.0
+python-swiftclient>=3.2.0  # Apache-2.0
 keystonemiddleware>=4.17.0 # Apache-2.0