diff --git a/.gitignore b/.gitignore index 043c504a..596446eb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ tmp/ #vim *.swp + +state/ +clients.json +rs/ diff --git a/cli.py b/cli.py index 9f8b5a6a..1ad50520 100755 --- a/cli.py +++ b/cli.py @@ -11,6 +11,7 @@ import subprocess from solar.core import actions as xa from solar.core import resource as xr from solar.core import signals as xs +from solar import operations @click.group() @@ -137,6 +138,28 @@ def init_cli_connect(): cli.add_command(disconnect) +def init_changes(): + @click.group() + def changes(): + pass + + cli.add_command(changes) + + @click.command() + @click.argument('path') + def stage(path): + log = operations.stage_changes(path) + print log.show() + + changes.add_command(stage) + + @click.command() + def commit(): + operations.commit_changes() + + changes.add_command(commit) + + def init_cli_connections(): @click.group() def connections(): @@ -185,5 +208,6 @@ if __name__ == '__main__': init_cli_connect() init_cli_connections() init_cli_deployment_config() + init_changes() cli() diff --git a/requirements.txt b/requirements.txt index a13f9914..de631f81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ PyYAML==3.11 jsonschema==2.4.0 requests==2.7.0 mock +dictdiffer==0.4.0 +enum34==1.0.4 diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index 70215007..68d2a0aa 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -4,6 +4,7 @@ import os from copy import deepcopy +<<<<<<< HEAD import yaml import solar @@ -118,6 +119,8 @@ class Resource(object): metadata['input'][k]['value'] = v db.add_resource(self.name, metadata) + meta_file = os.path.join(self.base_dir, 'meta.yaml') + utils.yaml_dump_to(metadata, meta_file) def create(name, base_path, args, tags=[], connections={}): @@ -127,7 +130,7 @@ def create(name, base_path, args, tags=[], connections={}): base_meta_file = os.path.join(base_path, 'meta.yaml') actions_path = os.path.join(base_path, 'actions') - meta = yaml.load(open(base_meta_file).read()) + meta = utils.yaml_load(base_meta_file) meta['id'] = name meta['version'] = '1.0.0' meta['actions'] = {} diff --git a/solar/solar/operations.py b/solar/solar/operations.py index 0cfba81c..078b0b81 100644 --- a/solar/solar/operations.py +++ b/solar/solar/operations.py @@ -3,33 +3,38 @@ from solar import state from solar.core import signals from solar.core import resource +from solar import utils -from dictdiffer import diff +from dictdiffer import diff, patch import networkx as nx def connections(res, graph): - + result = [] for pred in graph.predecessors(res.name): - edge = graph.get_edge_edge(pred, res.name) - if ':' in edge['label']: - parent, child = edge['label'].split(':') - yield pred, res.name, {parent: child} + edge = graph.get_edge_data(pred, res.name) + if 'label' in edge: + if ':' in edge['label']: + parent, child = edge['label'].split(':') + mapping = {parent: child} + else: + mapping = {edge['label']: edge['label']} else: - yield pred, res.name, {edge['label']: edge['label']} + mapping = None + result.append((pred, res.name, mapping)) + return result def to_dict(resource, graph): return {'uid': resource.name, - 'path': resource.dest_path, - 'meta': resource.metadata, + 'path': resource.base_dir, 'tags': resource.tags, 'args': resource.args_dict(), 'connections': connections(resource, graph)} -def stage_changes(): - resources = resource.load_all() +def stage_changes(path): + resources = resource.load_all(path) conn_graph = signals.detailed_connection_graph() commited = state.CD() @@ -38,7 +43,7 @@ def stage_changes(): for res_uid in nx.topological_sort(conn_graph): commited_data = commited.get(res_uid, {}) staged_data = to_dict(resources[res_uid], conn_graph) - df = diff(commited_data, staged_data) + df = list(diff(commited_data, staged_data)) if df: log_item = state.LogItem( @@ -48,3 +53,15 @@ def stage_changes(): log.add(log_item) return log + +def commit_changes(): + # just shortcut to test stuff + commited = state.CD() + history = state.CL() + staged = state.SL() + + while staged.items: + l = staged.popleft() + commited[l.res] = patch(commited.get(l.res, {}), l.diff) + l.state = state.states.success + history.add(l) diff --git a/solar/solar/state.py b/solar/solar/state.py index 8614ebb9..66ca6617 100644 --- a/solar/solar/state.py +++ b/solar/solar/state.py @@ -12,24 +12,50 @@ # License for the specific language governing permissions and limitations # under the License. +import os import collections from collections import deque +from functools import partial + +from solar import utils + +from enum import Enum + + +states = Enum('States', 'pending inprogress error success') + + +def state_file(filename): + filepath = os.path.join(utils.read_config()['state'], filename) + if 'log' in filename: + return Log(filepath) + elif 'data' in filename: + return Data(filepath) + + +CD = partial(state_file, 'commited_data') +SD = partial(state_file, 'staged_data') +SL = partial(state_file, 'stage_log') +IL = partial(state_file, 'inprogress_log') +CL = partial(state_file, 'commit_log') class LogItem(object): - def __init__(self, uid, res_uid, diff): + def __init__(self, uid, res_uid, diff, state=None): self.uid = uid self.res = res_uid self.diff = diff + self.state = state or states.pending def to_yaml(self): return utils.yaml_dump(self.to_dict()) def to_dict(self): return {'uid': self.uid, - 'res': self.res_uid, - 'diff': self.diff} + 'res': self.res, + 'diff': self.diff, + 'state': self.state.name} def __str__(self): return self.to_yaml() @@ -42,18 +68,27 @@ class Log(object): def __init__(self, path): self.path = path - self.items = deque([LogItem(**l) for - l in utils.yaml_load(path)]) + items = utils.yaml_load(path) or [] + self.items = deque([LogItem( + l['uid'], l['res'], + l['diff'], getattr(states, l['state'])) for l in items]) + + def sync(self): + utils.yaml_dump_to([i.to_dict() for i in self.items], self.path) def add(self, logitem): self.items.append(logitem) - utils.yaml_dump_to(self.items, path) + self.sync() def popleft(self): item = self.items.popleft() - utils.yaml_dump_to(self.items, path) + self.sync() return item + def show(self, verbose=False): + return ['L(uuid={0}, res={1})'.format(l.uid, l.res) + for l in self.items] + def __repr__(self): return 'Log({0})'.format(self.path) @@ -62,15 +97,22 @@ class Data(collections.MutableMapping): def __init__(self, path): self.path = path - self.store = utils.yaml_load(path) + self.store = utils.yaml_load(path) or {} def __getitem__(self, key): return self.store[key] def __setitem__(self, key, value): self.store[key] = value - utils.yaml_dump_to(self.store, path) + utils.yaml_dump_to(self.store, self.path) def __delitem__(self, key): self.store.pop(key) - utils.yaml_dump_to(self.store, path) + utils.yaml_dump_to(self.store, self.path) + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) +