From d3854dde75db00550782dc6f5bdd8e03128a6411 Mon Sep 17 00:00:00 2001 From: gengchc2 Date: Mon, 26 Nov 2018 16:38:45 -0800 Subject: [PATCH] Add ftps subclass of BaseFtpStorage The patch add ftps subclass of BaseFtpStorage. ref: https://storyboard.openstack.org/#!/story/2004332 Story: #2004332 Task: #27917 Change-Id: I36dc6286ec0b8dd5bd5663d974e48942536822ff --- freezer/common/config.py | 11 +++++++ freezer/engine/nova/nova.py | 20 +++++++++++- freezer/main.py | 8 +++-- freezer/storage/ftp.py | 63 +++++++++++++++++++------------------ 4 files changed, 68 insertions(+), 34 deletions(-) diff --git a/freezer/common/config.py b/freezer/common/config.py index 889943d4..893bd9de 100644 --- a/freezer/common/config.py +++ b/freezer/common/config.py @@ -89,6 +89,7 @@ DEFAULT_PARAMS = { 'cindernative_backup_id': None, 'sync': True, 'engine_name': 'tar', 'timeout': 120, 'project_id': None, 'ftp_username': '', 'ftp_password': '', 'ftp_host': '', 'ftp_port': DEFAULT_FTP_PORT, + 'ftp_keyfile': '', 'ftp_certfile': '', } _COMMON = [ @@ -564,6 +565,16 @@ _COMMON = [ default=DEFAULT_PARAMS['ftp_port'], help="Remote port for FTP, FTPS storage (default 21)" ), + cfg.StrOpt('ftp-keyfile', + dest='ftp_keyfile', + default=DEFAULT_PARAMS['ftp_keyfile'], + help="Required if ftps server requires client certificate" + ), + cfg.StrOpt('ftp-certfile', + dest='ftp_certfile', + default=DEFAULT_PARAMS['ftp_certfile'], + help="Required if ftps server requires client certificate" + ), ] diff --git a/freezer/engine/nova/nova.py b/freezer/engine/nova/nova.py index 501144ca..cda69cf6 100644 --- a/freezer/engine/nova/nova.py +++ b/freezer/engine/nova/nova.py @@ -82,6 +82,15 @@ class NovaEngine(engine.BackupEngine): 'project_' + project_id) with self.storage.open(backup_basepath, 'rb') as backup_file: data = backup_file.readline() + elif self.storage._type in ['ftp', 'ftps']: + backup_basepath = os.path.join(self.storage.storage_path, + 'project_' + project_id) + file = tempfile.NamedTemporaryFile('wb', delete=True) + self.storage.get_file(backup_basepath, file.name) + with open(file.name) as f: + data = f.readline() + LOG.info("get_nova_tenant download {0}".format(data)) + file.close() return json.loads(data) @@ -186,11 +195,20 @@ class NovaEngine(engine.BackupEngine): key=object_name, data=data ) - elif self.storage._type in ['local', 'ssh', 'ftp', 'ftps']: + elif self.storage._type in ['local', 'ssh']: backup_basepath = os.path.join(self.storage.storage_path, "project_" + project_id) with self.storage.open(backup_basepath, 'wb') as backup_file: backup_file.write(data) + elif self.storage._type in ['ftp', 'ftps']: + backup_basepath = os.path.join(self.storage.storage_path, + 'project_' + project_id) + file = tempfile.NamedTemporaryFile('wb', delete=True) + with open(file.name, 'wb') as f: + f.write(data) + LOG.info("backup_nova_tenant data={0}".format(data)) + self.storage.put_file(file.name, backup_basepath) + file.close() executor = futures.ThreadPoolExecutor( max_workers=len(instance_ids)) diff --git a/freezer/main.py b/freezer/main.py index e11d97ce..4c79d549 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -235,9 +235,13 @@ def storage_from_dict(backup_args, max_segment_size): backup_args['ftp_username'], backup_args['ftp_host'], int(backup_args['ftp_port']), max_segment_size] + if storage_name == 'ftps': + args.append(backup_args['ftp_keyfile']) + args.append(backup_args['ftp_certfile']) + LOG.info('args=%s' % args) storage = importutils.import_object( - "freezer.storage.{0}.{1}Storage".format( - storage_name, storage_name.capitalize()), *args) + "freezer.storage.ftp.{0}Storage".format( + storage_name.capitalize()), *args) else: raise Exception("No storage found for name {0}".format( backup_args['storage'])) diff --git a/freezer/storage/ftp.py b/freezer/storage/ftp.py index 7dc0ca9d..9d090a3a 100644 --- a/freezer/storage/ftp.py +++ b/freezer/storage/ftp.py @@ -15,7 +15,6 @@ limitations under the License. """ -import errno import ftplib import json import os @@ -198,9 +197,9 @@ class BaseFtpStorage(fslike.FsLikeStorage): res = self.ftp.nlst() LOG.info('ftp listdir res=%s' % res) return sorted(res) - except IOError as e: + except ftplib.error_perm as e: LOG.info("ftp listdir error %s" % e) - if e.errno == errno.ENOENT: + if '550' in e[0]: return list() else: raise @@ -208,31 +207,6 @@ class BaseFtpStorage(fslike.FsLikeStorage): def open(self, path, mode): pass - def read_metadata_file(self, path): - # files = self.ftp.mlsd(path) # 3.3 support - LOG.info("ftp read_metadta_file path=%s" % path) - tmpdir = self._create_tempdir() - try: - data_down = utils.path_join(tmpdir, "data_down") - LOG.info("read metada datadown=%s" % data_down) - self.get_file(path, data_down) - file_size = self.ftp.size(path) - data = "" - received_size = 0 - with open(data_down, 'r') as reader: - reader.prefetch(file_size) - chunk = reader.read(CHUNK_SIZE) - while chunk: - received_size += len(chunk) - data += chunk - chunk = reader.read(CHUNK_SIZE) - if file_size != received_size: - raise IOError('Size mismatch: expected {} received {}' - .format(file_size, received_size)) - return data - finally: - shutil.rmtree(tmpdir) - def backup_blocks(self, backup): LOG.info("ftp backup_blocks ") self.init() @@ -260,6 +234,7 @@ class BaseFtpStorage(fslike.FsLikeStorage): :return: """ tmpdir = self._create_tempdir() + LOG.info('add stream') try: split = package_name.rsplit('/', 1) # create backup_basedir @@ -316,7 +291,7 @@ class FtpStorage(BaseFtpStorage): LOG.info("ftp nlst result=%s" % nfiles) except socket.error as e: LOG.info("ftp socket error=%s" % e) - self.ftpclient.set_pasv(False) + self.ftp.set_pasv(False) except ftplib.all_errors as e: # socket.error msg = "create ftp failed error=%s" % e LOG.info(msg) @@ -332,10 +307,36 @@ class FtpsStorage(BaseFtpStorage): _type = 'ftps' def __init__(self, storage_path, remote_pwd, - remote_username, remote_ip, port, max_segment_size): + remote_username, remote_ip, port, max_segment_size, + keyfile, certfile): """ :param storage_path: directory of storage :type storage_path: str :return: """ - pass + self.keyfile = keyfile + self.certfile = certfile + LOG.info("key=%s cer=%s" % (self.keyfile, self.certfile)) + super(FtpsStorage, self).__init__(storage_path, remote_pwd, + remote_username, remote_ip, + port, max_segment_size) + + def init(self): + try: + ftps = ftplib.FTP_TLS(keyfile=self.keyfile, + certfile=self.certfile) + ftps.set_pasv(True) + ftps.connect(self.remote_ip, self.port, 60) + ftps.login(self.remote_username, self.remote_pwd) + msg = ftps.prot_p() + LOG.info("ftps encrypt %s, ret=%s" % (self.remote_ip, msg)) + nfiles = ftps.nlst() + LOG.info("ftps nlst result=%s" % nfiles) + except socket.error as e: + LOG.info("ftps socket error=%s" % e) + self.ftp.set_pasv(False) + except ftplib.all_errors as e: + msg = "create ftps failed error=%s" % e + LOG.info(msg) + raise Exception(msg) + self.ftp = ftps