Merge "Add storage code"
This commit is contained in:
commit
361c190a4c
|
@ -0,0 +1,43 @@
|
|||
# Copyright 2016 Sam Yaple
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from distutils.dir_util import mkpath
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from ekko.storage import drivers
|
||||
|
||||
|
||||
class LocalStorage(drivers.BaseStorage):
|
||||
|
||||
def put_data(self, data_segment):
|
||||
data = data_segment[0]
|
||||
segment = data_segment[1]
|
||||
file_path = os.path.join(
|
||||
self.storage_location,
|
||||
str(uuid.UUID(bytes=segment.backupset_id)),
|
||||
str(segment.incremental)
|
||||
)
|
||||
|
||||
mkpath(file_path)
|
||||
|
||||
file_output = os.path.join(
|
||||
file_path,
|
||||
str(segment.segment)
|
||||
)
|
||||
|
||||
with open(file_output, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
return segment
|
|
@ -0,0 +1,39 @@
|
|||
# Copyright 2016 Sam Yaple
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
|
||||
import six
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseStorage(object):
|
||||
"""Base class for storage drivers
|
||||
|
||||
:params storage_location: Location to store data
|
||||
"""
|
||||
|
||||
def __init__(self, storage_location):
|
||||
self.storage_location = storage_location
|
||||
|
||||
@abc.abstractmethod
|
||||
def put_data(self, data_segment):
|
||||
"""Write data to backing location
|
||||
|
||||
:params data_segment: A tuple with a raw chunk of data to write out and
|
||||
An object of type manifest.structure.Segment with the metadata
|
||||
to pair with the raw chunk of data
|
||||
:returns: A generator with the segment objects
|
||||
"""
|
||||
raise NotImplementedError()
|
|
@ -26,6 +26,8 @@ ekko.backup.backend =
|
|||
ekko.manifest.drivers =
|
||||
osdk = ekko.manifest._drivers.osdk:OSDKManifest
|
||||
sqlite = ekko.manifest._drivers.sqlite:SQLiteManifest
|
||||
ekko.storage.drivers =
|
||||
local = ekko.storage._drivers.local:LocalStorage
|
||||
|
||||
[files]
|
||||
packages =
|
||||
|
|
|
@ -38,6 +38,9 @@ def parse_args():
|
|||
help='change block tracking info')
|
||||
parser.add_argument('--backend', required=False, default='raw',
|
||||
choices=['raw'], help='backend driver')
|
||||
parser.add_argument('--storage', required=False, default='local',
|
||||
choices=['local'], help='storage driver')
|
||||
parser.add_argument('--location', required=True, help='storage path')
|
||||
parser.add_argument('--driver', required=False, default='sqlite',
|
||||
choices=['osdk', 'sqlite'], help='manifest driver')
|
||||
return parser.parse_args()
|
||||
|
@ -65,7 +68,7 @@ def read_segments(segments, metadata, backend):
|
|||
for start, data in backend.get_data(reads):
|
||||
if data == zero_blob:
|
||||
continue
|
||||
yield manifest_structure.Segment(
|
||||
yield data, manifest_structure.Segment(
|
||||
metadata.backupset_id,
|
||||
metadata.incremental,
|
||||
start / size,
|
||||
|
@ -92,14 +95,21 @@ def main():
|
|||
invoke_args=[args.manifest]
|
||||
).driver
|
||||
|
||||
backend = driver.DriverManager(
|
||||
disk_backend = driver.DriverManager(
|
||||
namespace='ekko.backup.backend',
|
||||
name=args.backend,
|
||||
invoke_on_load=True,
|
||||
invoke_args=[args.backup]
|
||||
).driver
|
||||
|
||||
size_of_disk = backend.get_size()
|
||||
storage = driver.DriverManager(
|
||||
namespace='ekko.storage.drivers',
|
||||
name=args.storage,
|
||||
invoke_on_load=True,
|
||||
invoke_args=[args.location]
|
||||
).driver
|
||||
|
||||
size_of_disk = disk_backend.get_size()
|
||||
incremental = 0
|
||||
metadata = manifest_structure.Metadata(incremental, size_of_disk)
|
||||
|
||||
|
@ -108,7 +118,11 @@ def main():
|
|||
|
||||
segments_list = list(range(0, int(math.ceil(
|
||||
float(size_of_disk)/metadata.segment_size))))
|
||||
segments = read_segments(segments_list, metadata, backend)
|
||||
data_segments = read_segments(segments_list, metadata, disk_backend)
|
||||
|
||||
segments = [
|
||||
storage.put_data(data_segment) for data_segment in data_segments
|
||||
]
|
||||
|
||||
manifest.put_segments(segments, metadata)
|
||||
|
||||
|
|
Loading…
Reference in New Issue