From 2e389f1ddb3765dd4e1f718220fdf8d2ec22dc92 Mon Sep 17 00:00:00 2001 From: SamYaple Date: Mon, 18 Jan 2016 20:44:07 +0000 Subject: [PATCH] Add storage code This code will be the basis of writing out the data to the backing storage with the initial driver being on disk storage. Additionally move compression/encryption code to storage rather than backup module ./backup.py --backup /dev/loop0 --manifest mani --location /path/to/store/backup/ Change-Id: I22d291ea3ac03b4d5ddb447915c972db4cdc195d --- .../_compression => storage}/__init__.py | 0 .../_compression}/__init__.py | 0 ekko/storage/_drivers/__init__.py | 0 ekko/storage/_drivers/local.py | 43 +++++++++++++++++++ ekko/storage/_encryption/__init__.py | 0 ekko/storage/drivers.py | 39 +++++++++++++++++ setup.cfg | 2 + tools/backup.py | 22 ++++++++-- 8 files changed, 102 insertions(+), 4 deletions(-) rename ekko/{backup/_compression => storage}/__init__.py (100%) rename ekko/{backup/_encryption => storage/_compression}/__init__.py (100%) create mode 100644 ekko/storage/_drivers/__init__.py create mode 100644 ekko/storage/_drivers/local.py create mode 100644 ekko/storage/_encryption/__init__.py create mode 100644 ekko/storage/drivers.py diff --git a/ekko/backup/_compression/__init__.py b/ekko/storage/__init__.py similarity index 100% rename from ekko/backup/_compression/__init__.py rename to ekko/storage/__init__.py diff --git a/ekko/backup/_encryption/__init__.py b/ekko/storage/_compression/__init__.py similarity index 100% rename from ekko/backup/_encryption/__init__.py rename to ekko/storage/_compression/__init__.py diff --git a/ekko/storage/_drivers/__init__.py b/ekko/storage/_drivers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ekko/storage/_drivers/local.py b/ekko/storage/_drivers/local.py new file mode 100644 index 0000000..03cb544 --- /dev/null +++ b/ekko/storage/_drivers/local.py @@ -0,0 +1,43 @@ +# Copyright 2016 Sam Yaple +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distutils.dir_util import mkpath +import os +import uuid + +from ekko.storage import drivers + + +class LocalStorage(drivers.BaseStorage): + + def put_data(self, data_segment): + data = data_segment[0] + segment = data_segment[1] + file_path = os.path.join( + self.storage_location, + str(uuid.UUID(bytes=segment.backupset_id)), + str(segment.incremental) + ) + + mkpath(file_path) + + file_output = os.path.join( + file_path, + str(segment.segment) + ) + + with open(file_output, 'wb') as f: + f.write(data) + + return segment diff --git a/ekko/storage/_encryption/__init__.py b/ekko/storage/_encryption/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ekko/storage/drivers.py b/ekko/storage/drivers.py new file mode 100644 index 0000000..3e62152 --- /dev/null +++ b/ekko/storage/drivers.py @@ -0,0 +1,39 @@ +# Copyright 2016 Sam Yaple +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class BaseStorage(object): + """Base class for storage drivers + + :params storage_location: Location to store data + """ + + def __init__(self, storage_location): + self.storage_location = storage_location + + @abc.abstractmethod + def put_data(self, data_segment): + """Write data to backing location + + :params data_segment: A tuple with a raw chunk of data to write out and + An object of type manifest.structure.Segment with the metadata + to pair with the raw chunk of data + :returns: A generator with the segment objects + """ + raise NotImplementedError() diff --git a/setup.cfg b/setup.cfg index 41c29bf..461e815 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,8 @@ ekko.backup.backend = ekko.manifest.drivers = osdk = ekko.manifest._drivers.osdk:OSDKManifest sqlite = ekko.manifest._drivers.sqlite:SQLiteManifest +ekko.storage.drivers = + local = ekko.storage._drivers.local:LocalStorage [files] packages = diff --git a/tools/backup.py b/tools/backup.py index 3071ffc..8268683 100755 --- a/tools/backup.py +++ b/tools/backup.py @@ -38,6 +38,9 @@ def parse_args(): help='change block tracking info') parser.add_argument('--backend', required=False, default='raw', choices=['raw'], help='backend driver') + parser.add_argument('--storage', required=False, default='local', + choices=['local'], help='storage driver') + parser.add_argument('--location', required=True, help='storage path') parser.add_argument('--driver', required=False, default='sqlite', choices=['osdk', 'sqlite'], help='manifest driver') return parser.parse_args() @@ -65,7 +68,7 @@ def read_segments(segments, metadata, backend): for start, data in backend.get_data(reads): if data == zero_blob: continue - yield manifest_structure.Segment( + yield data, manifest_structure.Segment( metadata.backupset_id, metadata.incremental, start / size, @@ -92,14 +95,21 @@ def main(): invoke_args=[args.manifest] ).driver - backend = driver.DriverManager( + disk_backend = driver.DriverManager( namespace='ekko.backup.backend', name=args.backend, invoke_on_load=True, invoke_args=[args.backup] ).driver - size_of_disk = backend.get_size() + storage = driver.DriverManager( + namespace='ekko.storage.drivers', + name=args.storage, + invoke_on_load=True, + invoke_args=[args.location] + ).driver + + size_of_disk = disk_backend.get_size() incremental = 0 metadata = manifest_structure.Metadata(incremental, size_of_disk) @@ -108,7 +118,11 @@ def main(): segments_list = list(range(0, int(math.ceil( float(size_of_disk)/metadata.segment_size)))) - segments = read_segments(segments_list, metadata, backend) + data_segments = read_segments(segments_list, metadata, disk_backend) + + segments = [ + storage.put_data(data_segment) for data_segment in data_segments + ] manifest.put_segments(segments, metadata)