Add ftps subclass of BaseFtpStorage

The patch add ftps subclass of BaseFtpStorage.
ref: https://storyboard.openstack.org/#!/story/2004332
Story: #2004332
Task: #27917

Change-Id: I36dc6286ec0b8dd5bd5663d974e48942536822ff

freezer/common/config.py

@@ -89,6 +89,7 @@ DEFAULT_PARAMS = {
     'cindernative_backup_id': None, 'sync': True, 'engine_name': 'tar',
     'timeout': 120, 'project_id': None, 'ftp_username': '',
     'ftp_password': '', 'ftp_host': '', 'ftp_port': DEFAULT_FTP_PORT,
+    'ftp_keyfile': '', 'ftp_certfile': '',
 }
 
 _COMMON = [
@@ -564,6 +565,16 @@ _COMMON = [
                default=DEFAULT_PARAMS['ftp_port'],
                help="Remote port for FTP, FTPS storage (default 21)"
                ),
+    cfg.StrOpt('ftp-keyfile',
+               dest='ftp_keyfile',
+               default=DEFAULT_PARAMS['ftp_keyfile'],
+               help="Required if ftps server requires client certificate"
+               ),
+    cfg.StrOpt('ftp-certfile',
+               dest='ftp_certfile',
+               default=DEFAULT_PARAMS['ftp_certfile'],
+               help="Required if ftps server requires client certificate"
+               ),
 ]
 
 

freezer/engine/nova/nova.py

@@ -82,6 +82,15 @@ class NovaEngine(engine.BackupEngine):
                                            'project_' + project_id)
             with self.storage.open(backup_basepath, 'rb') as backup_file:
                 data = backup_file.readline()
+        elif self.storage._type in ['ftp', 'ftps']:
+            backup_basepath = os.path.join(self.storage.storage_path,
+                                           'project_' + project_id)
+            file = tempfile.NamedTemporaryFile('wb', delete=True)
+            self.storage.get_file(backup_basepath, file.name)
+            with open(file.name) as f:
+                data = f.readline()
+            LOG.info("get_nova_tenant download {0}".format(data))
+            file.close()
 
         return json.loads(data)
 
@@ -186,11 +195,20 @@ class NovaEngine(engine.BackupEngine):
                 key=object_name,
                 data=data
             )
-        elif self.storage._type in ['local', 'ssh', 'ftp', 'ftps']:
+        elif self.storage._type in ['local', 'ssh']:
             backup_basepath = os.path.join(self.storage.storage_path,
                                            "project_" + project_id)
             with self.storage.open(backup_basepath, 'wb') as backup_file:
                 backup_file.write(data)
+        elif self.storage._type in ['ftp', 'ftps']:
+            backup_basepath = os.path.join(self.storage.storage_path,
+                                           'project_' + project_id)
+            file = tempfile.NamedTemporaryFile('wb', delete=True)
+            with open(file.name, 'wb') as f:
+                f.write(data)
+            LOG.info("backup_nova_tenant data={0}".format(data))
+            self.storage.put_file(file.name, backup_basepath)
+            file.close()
 
         executor = futures.ThreadPoolExecutor(
             max_workers=len(instance_ids))

freezer/main.py

@@ -235,9 +235,13 @@ def storage_from_dict(backup_args, max_segment_size):
                 backup_args['ftp_username'],
                 backup_args['ftp_host'], int(backup_args['ftp_port']),
                 max_segment_size]
+        if storage_name == 'ftps':
+            args.append(backup_args['ftp_keyfile'])
+            args.append(backup_args['ftp_certfile'])
+        LOG.info('args=%s' % args)
         storage = importutils.import_object(
-            "freezer.storage.{0}.{1}Storage".format(
-                storage_name, storage_name.capitalize()), *args)
+            "freezer.storage.ftp.{0}Storage".format(
+                storage_name.capitalize()), *args)
     else:
         raise Exception("No storage found for name {0}".format(
             backup_args['storage']))

freezer/storage/ftp.py

@@ -15,7 +15,6 @@ limitations under the License.
 
 """
 
-import errno
 import ftplib
 import json
 import os
@@ -198,9 +197,9 @@ class BaseFtpStorage(fslike.FsLikeStorage):
             res = self.ftp.nlst()
             LOG.info('ftp listdir res=%s' % res)
             return sorted(res)
-        except IOError as e:
+        except ftplib.error_perm as e:
             LOG.info("ftp listdir error %s" % e)
-            if e.errno == errno.ENOENT:
+            if '550' in e[0]:
                 return list()
             else:
                 raise
@@ -208,31 +207,6 @@ class BaseFtpStorage(fslike.FsLikeStorage):
     def open(self, path, mode):
         pass
 
-    def read_metadata_file(self, path):
-        # files = self.ftp.mlsd(path) # 3.3 support
-        LOG.info("ftp read_metadta_file path=%s" % path)
-        tmpdir = self._create_tempdir()
-        try:
-            data_down = utils.path_join(tmpdir, "data_down")
-            LOG.info("read metada datadown=%s" % data_down)
-            self.get_file(path, data_down)
-            file_size = self.ftp.size(path)
-            data = ""
-            received_size = 0
-            with open(data_down, 'r') as reader:
-                reader.prefetch(file_size)
-                chunk = reader.read(CHUNK_SIZE)
-                while chunk:
-                    received_size += len(chunk)
-                    data += chunk
-                    chunk = reader.read(CHUNK_SIZE)
-            if file_size != received_size:
-                raise IOError('Size mismatch: expected {} received {}'
-                              .format(file_size, received_size))
-            return data
-        finally:
-            shutil.rmtree(tmpdir)
-
     def backup_blocks(self, backup):
         LOG.info("ftp backup_blocks ")
         self.init()
@@ -260,6 +234,7 @@ class BaseFtpStorage(fslike.FsLikeStorage):
         :return:
         """
         tmpdir = self._create_tempdir()
+        LOG.info('add stream')
         try:
             split = package_name.rsplit('/', 1)
             # create backup_basedir
@@ -316,7 +291,7 @@ class FtpStorage(BaseFtpStorage):
             LOG.info("ftp nlst result=%s" % nfiles)
         except socket.error as e:
             LOG.info("ftp socket error=%s" % e)
-            self.ftpclient.set_pasv(False)
+            self.ftp.set_pasv(False)
         except ftplib.all_errors as e:  # socket.error
             msg = "create ftp failed error=%s" % e
             LOG.info(msg)
@@ -332,10 +307,36 @@ class FtpsStorage(BaseFtpStorage):
     _type = 'ftps'
 
     def __init__(self, storage_path, remote_pwd,
-                 remote_username, remote_ip, port, max_segment_size):
+                 remote_username, remote_ip, port, max_segment_size,
+                 keyfile, certfile):
         """
             :param storage_path: directory of storage
             :type storage_path: str
             :return:
             """
-        pass
+        self.keyfile = keyfile
+        self.certfile = certfile
+        LOG.info("key=%s cer=%s" % (self.keyfile, self.certfile))
+        super(FtpsStorage, self).__init__(storage_path, remote_pwd,
+                                          remote_username, remote_ip,
+                                          port, max_segment_size)
+
+    def init(self):
+        try:
+            ftps = ftplib.FTP_TLS(keyfile=self.keyfile,
+                                  certfile=self.certfile)
+            ftps.set_pasv(True)
+            ftps.connect(self.remote_ip, self.port, 60)
+            ftps.login(self.remote_username, self.remote_pwd)
+            msg = ftps.prot_p()
+            LOG.info("ftps encrypt %s, ret=%s" % (self.remote_ip, msg))
+            nfiles = ftps.nlst()
+            LOG.info("ftps nlst result=%s" % nfiles)
+        except socket.error as e:
+            LOG.info("ftps socket error=%s" % e)
+            self.ftp.set_pasv(False)
+        except ftplib.all_errors as e:
+            msg = "create ftps failed error=%s" % e
+            LOG.info(msg)
+            raise Exception(msg)
+        self.ftp = ftps