crypto - use random iv when wrapping body key

Change-Id: Ia32a7b1cbafd5f593d0609310e4a38de6c52f220

swift/common/crypto_utils.py

@@ -88,11 +88,16 @@ def dump_crypto_meta(crypto_meta):
     :param crypto_meta: a dict containing crypto meta items
     :returns: a string serialization of a crypto meta dict
     """
+    def b64_encode_meta(crypto_meta):
+        return {
+            name: (base64.b64encode(value).decode() if name in ('iv', 'key')
+                   else b64_encode_meta(value) if isinstance(value, dict)
+                   else value)
+            for name, value in crypto_meta.items()}
+
     # use sort_keys=True to make serialized form predictable for testing
-    return urllib.quote_plus(json.dumps({
+    return urllib.quote_plus(
-        name: (base64.b64encode(value).decode() if name in ('iv', 'key')
+        json.dumps(b64_encode_meta(crypto_meta), sort_keys=True))
-               else value)
-        for name, value in crypto_meta.items()}, sort_keys=True))
 
 
 def load_crypto_meta(value):
@@ -110,12 +115,16 @@ def load_crypto_meta(value):
     :raises EncryptionException: if an error occurs while parsing the
                                  crypto meta
     """
+    def b64_decode_meta(crypto_meta):
+        return {
+            str(name): (base64.b64decode(val) if name in ('iv', 'key')
+                        else b64_decode_meta(val) if isinstance(val, dict)
+                        else str(val))
+            for name, val in crypto_meta.items()}
+
     try:
         value = urllib.unquote_plus(value)
-        crypto_meta = {str(name): (base64.b64decode(value)
+        return b64_decode_meta(json.loads(value))
-                                   if name in ('iv', 'key') else str(value))
-                       for name, value in json.loads(value).items()}
-        return crypto_meta
     except (KeyError, ValueError, TypeError) as err:
         msg = 'Bad crypto meta %s: %s' % (value, err)
         raise EncryptionException(msg)

swift/common/middleware/crypto.py

@@ -131,21 +131,23 @@ class Crypto(object):
         # helper method to create random key of correct length
         return os.urandom(KEY_LENGTH)
 
-    def wrap_key(self, wrapping_key, key_to_wrap, iv):
+    def wrap_key(self, wrapping_key, key_to_wrap):
         # we don't use an RFC 3394 key wrap algorithm such as cryptography's
         # aes_wrap_key because it's slower and we have iv material readily
         # available so don't need a deterministic algorithm
+        iv = self._get_random_iv()
         encryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv),
                            backend=default_backend()).encryptor()
-        return encryptor.update(key_to_wrap)
+        return {'key': encryptor.update(key_to_wrap), 'iv': iv}
 
-    def unwrap_key(self, wrapping_key, wrapped_key, iv):
+    def unwrap_key(self, wrapping_key, context):
+        # unwrap a key from dict of form returned by wrap_key
         # check the key length early - unwrapping won't change the length
-        self.check_key(wrapped_key)
+        self.check_key(context['key'])
-        decryptor = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv),
+        decryptor = Cipher(algorithms.AES(wrapping_key),
+                           modes.CTR(context['iv']),
                            backend=default_backend()).decryptor()
-        key = decryptor.update(wrapped_key)
+        return decryptor.update(context['key'])
-        return key
 
     def check_key(self, key):
         if len(key) != KEY_LENGTH:

swift/common/middleware/decrypter.py

@@ -70,8 +70,7 @@ class BaseDecrypterContext(CryptoWSGIContext):
         """
         try:
             return self.crypto.unwrap_key(wrapping_key,
-                                          crypto_meta['key'],
+                                          crypto_meta['body_key'])
-                                          crypto_meta['iv'])
         except KeyError as err:
             err = 'Missing %s' % err
         except ValueError as err:

swift/common/middleware/encrypter.py

@@ -68,15 +68,12 @@ class EncInputWrapper(object):
         # do this once when body is first read
         if self.body_crypto_ctxt is None:
             self.body_crypto_meta = self.crypto.create_crypto_meta()
-            self.body_key = self.crypto.create_random_key()
+            body_key = self.crypto.create_random_key()
-            # wrap the body key with object key re-using body iv
+            # wrap the body key with object key
-            self.body_crypto_meta['key'] = self.crypto.wrap_key(
+            self.body_crypto_meta['body_key'] = self.crypto.wrap_key(
-                self.keys['object'],
+                self.keys['object'], body_key)
-                self.body_key,
-                self.body_crypto_meta['iv']
-            )
             self.body_crypto_ctxt = self.crypto.create_encryption_ctxt(
-                self.body_key, self.body_crypto_meta.get('iv'))
+                body_key, self.body_crypto_meta.get('iv'))
             self.plaintext_md5 = md5()
             self.ciphertext_md5 = md5()
 

test/unit/common/middleware/test_crypto.py

@@ -12,8 +12,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import mock
-
 import unittest
 import os
 
@@ -194,24 +193,26 @@ class TestCrypto(unittest.TestCase):
         wrapping_key = os.urandom(32)
         key_to_wrap = os.urandom(32)
         iv = os.urandom(16)
-        wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap, iv)
+        with mock.patch('swift.common.middleware.crypto.Crypto._get_random_iv',
+                        return_value=iv):
+            wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap)
         cipher = Cipher(algorithms.AES(wrapping_key), modes.CTR(iv),
                         backend=default_backend())
-        expected = cipher.encryptor().update(key_to_wrap)
+        expected = {'key': cipher.encryptor().update(key_to_wrap),
+                    'iv': iv}
         self.assertEqual(expected, wrapped)
 
-        unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped, iv)
+        unwrapped = self.crypto.unwrap_key(wrapping_key, wrapped)
         self.assertEqual(key_to_wrap, unwrapped)
 
     def test_unwrap_bad_key(self):
         # verify that ValueError is raised if unwrapped key is invalid
         wrapping_key = os.urandom(32)
-        iv = os.urandom(16)
         for length in (0, 16, 24, 31, 33):
             key_to_wrap = os.urandom(length)
-            wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap, iv)
+            wrapped = self.crypto.wrap_key(wrapping_key, key_to_wrap)
             with self.assertRaises(ValueError) as cm:
-                self.crypto.unwrap_key(wrapping_key, wrapped, iv)
+                self.crypto.unwrap_key(wrapping_key, wrapped)
             self.assertEqual(
                 cm.exception.message, 'Key must be length 32 bytes')
 

test/unit/common/middleware/test_crypto_utils.py

@@ -149,11 +149,14 @@ class TestModuleMethods(unittest.TestCase):
                       'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22%7D'
 
     meta_with_key = {'iv': '0123456789abcdef', 'cipher': 'AES_CTR_256',
-                     'key': '0123456789abcdef0123456789abcdef'}
+                     'body_key': {'key': 'fedcba9876543210fedcba9876543210',
-    serialized_meta_with_key = '%7B%22cipher%22%3A+%22AES_CTR_256%22%2C+%22' \
+                                  'iv': 'fedcba9876543210'}}
-                               'iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg%3D%3D%22' \
+    serialized_meta_with_key = '%7B%22body_key%22%3A+%7B%22iv%22%3A+%22ZmVkY' \
-                               '%2C+%22key%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZjA' \
+                               '2JhOTg3NjU0MzIxMA%3D%3D%22%2C+%22key%22%3A+%' \
-                               'xMjM0NTY3ODlhYmNkZWY%3D%22%7D'
+                               '22ZmVkY2JhOTg3NjU0MzIxMGZlZGNiYTk4NzY1NDMyMT' \
+                               'A%3D%22%7D%2C+%22cipher%22%3A+%22AES_CTR_256' \
+                               '%22%2C+%22iv%22%3A+%22MDEyMzQ1Njc4OWFiY2RlZg' \
+                               '%3D%3D%22%7D'
 
     def test_dump_crypto_meta(self):
         actual = crypto_utils.dump_crypto_meta(self.meta)

test/unit/common/middleware/test_decrypter.py

@@ -61,8 +61,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
         enc_body = encrypt(body, body_key, FAKE_IV)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         hdrs = {
             'Etag': 'hashOfCiphertext',
             'content-type': 'text/plain',
@@ -313,7 +314,8 @@ class TestDecrypterObjectRequests(unittest.TestCase):
                       self.decrypter.logger.get_lines_for_level('error')[0])
 
     def test_GET_with_bad_body_key_for_object_body(self):
-        bad_crypto_meta = fake_get_crypto_meta(key='wrapped too short key')
+        body_key_meta = {'key': 'wrapped too short key', 'iv': FAKE_IV}
+        bad_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
         self.assertIn('Key must be length 32',
                       self.decrypter.logger.get_lines_for_level('error')[0])
@@ -321,7 +323,7 @@ class TestDecrypterObjectRequests(unittest.TestCase):
     def test_GET_with_missing_body_key_for_object_body(self):
         bad_crypto_meta = fake_get_crypto_meta()  # no key by default
         self._test_GET_with_bad_crypto_meta_for_object_body(bad_crypto_meta)
-        self.assertIn("Missing 'key'",
+        self.assertIn("Missing 'body_key'",
                       self.decrypter.logger.get_lines_for_level('error')[0])
 
     def test_HEAD_success(self):
@@ -368,8 +370,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
         enc_body = encrypt(body, body_key, FAKE_IV)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         hdrs = {
             'Etag': 'hashOfCiphertext',
             'etag': 'hashOfCiphertext',
@@ -405,8 +408,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
         enc_body = encrypt(body, body_key, FAKE_IV)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         hdrs = {
             'Etag': 'hashOfCiphertext',
             'etag': 'hashOfCiphertext',
@@ -471,8 +475,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         cont_key = fetch_crypto_keys()['container']
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
         enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
         hdrs = {
@@ -503,8 +508,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         cont_key = fetch_crypto_keys()['container']
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         ctxt = Crypto().create_encryption_ctxt(body_key, FAKE_IV)
         enc_body = [encrypt(chunk, ctxt=ctxt) for chunk in chunks]
         enc_body = [enc_body[0][3:], enc_body[1], enc_body[2][:2]]
@@ -539,8 +545,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         cont_key = fetch_crypto_keys()['container']
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         plaintext = 'Cwm fjord veg balks nth pyx quiz'
         plaintext_etag = md5hex(plaintext)
         ciphertext = encrypt(plaintext, body_key, FAKE_IV)
@@ -605,8 +612,9 @@ class TestDecrypterObjectRequests(unittest.TestCase):
         cont_key = fetch_crypto_keys()['container']
         object_key = fetch_crypto_keys()['object']
         body_key = os.urandom(32)
-        body_crypto_meta = fake_get_crypto_meta(
+        body_key_meta = {'key': encrypt(body_key, object_key, FAKE_IV),
-            key=encrypt(body_key, object_key, FAKE_IV))
+                         'iv': FAKE_IV}
+        body_crypto_meta = fake_get_crypto_meta(body_key=body_key_meta)
         plaintext = 'Cwm fjord veg balks nth pyx quiz'
         plaintext_etag = md5hex(plaintext)
         ciphertext = encrypt(plaintext, body_key, FAKE_IV)

test/unit/common/middleware/test_encrypter.py

@@ -75,10 +75,15 @@ class TestEncrypter(unittest.TestCase):
         # verify body crypto meta
         actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta']
         actual = json.loads(urllib.unquote_plus(actual))
-        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
         self.assertEqual(Crypto().get_cipher(), actual['cipher'])
         self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
-        self.assertEqual(expected_wrapped_key, base64.b64decode(actual['key']))
+
+        # verify wrapped body key
+        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
+        self.assertEqual(expected_wrapped_key,
+                         base64.b64decode(actual['body_key']['key']))
+        self.assertEqual(FAKE_IV,
+                         base64.b64decode(actual['body_key']['iv']))
 
         # verify etag
         self.assertEqual(ciphertext_etag, req_hdrs['Etag'])
@@ -285,10 +290,15 @@ class TestEncrypter(unittest.TestCase):
         # verify body crypto meta
         actual = req_hdrs['X-Object-Sysmeta-Crypto-Meta']
         actual = json.loads(urllib.unquote_plus(actual))
-        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
         self.assertEqual(Crypto().get_cipher(), actual['cipher'])
         self.assertEqual(FAKE_IV, base64.b64decode(actual['iv']))
-        self.assertEqual(expected_wrapped_key, base64.b64decode(actual['key']))
+
+        # verify wrapped body key
+        expected_wrapped_key = encrypt(body_key, object_key, FAKE_IV)
+        self.assertEqual(expected_wrapped_key,
+                         base64.b64decode(actual['body_key']['key']))
+        self.assertEqual(FAKE_IV,
+                         base64.b64decode(actual['body_key']['iv']))
 
     def test_PUT_with_etag_override_in_headers(self):
         # verify handling of another middleware's

test/unit/common/middleware/test_encrypter_decrypter.py

@@ -300,11 +300,10 @@ class TestCryptoPipelineChanges(unittest.TestCase):
                 # verify on disk data - body
                 body_iv = load_crypto_meta(
                     metadata['x-object-sysmeta-crypto-meta'])['iv']
-                wrapped_body_key = load_crypto_meta(
+                body_key_meta = load_crypto_meta(
-                    metadata['x-object-sysmeta-crypto-meta'])['key']
+                    metadata['x-object-sysmeta-crypto-meta'])['body_key']
                 obj_key = self.km.create_key('/a/%s/o' % self.container_name)
-                body_key = crypto.Crypto({}).unwrap_key(
+                body_key = crypto.Crypto({}).unwrap_key(obj_key, body_key_meta)
-                    obj_key, wrapped_body_key, body_iv)
                 exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
                 self.assertEqual(exp_enc_body, contents)
                 # verify on disk user metadata