From c59cd180aff37c01be0f48f84d90c76067189a41 Mon Sep 17 00:00:00 2001 From: Vladimir Kuklin Date: Fri, 29 Jul 2016 21:57:55 +0300 Subject: [PATCH] Ressurect --start|--end options for graph execution Change-Id: I22d96ed91a63d498e1e4ace69cdd50d3f6246dc3 CLoses-bug: #1612616 --- bin/astute-simulator | 1 + lib/astute.rb | 2 + lib/astute/ext/array.rb | 28 ++++++ lib/astute/ext/hash.rb | 12 ++- lib/astute/task_deployment.rb | 58 +++++++++++++ lib/astute/task_node.rb | 6 +- lib/fuel_deployment/cluster.rb | 72 +++++++++++++--- lib/fuel_deployment/graph.rb | 2 +- lib/fuel_deployment/node.rb | 8 +- lib/fuel_deployment/simulator.rb | 5 +- lib/fuel_deployment/task.rb | 45 +++------- spec/unit/fuel_deployment/cluster_spec.rb | 30 ++++--- spec/unit/fuel_deployment/graph_spec.rb | 4 +- spec/unit/fuel_deployment/node_spec.rb | 10 +-- spec/unit/fuel_deployment/task_spec.rb | 4 +- spec/unit/task_deployment_spec.rb | 94 ++++++++++++++++++++ spec/unit/task_node_spec.rb | 2 +- tests/deployment.rb | 5 +- tests/loop.rb | 2 +- tests/node_concurrency.rb | 2 +- tests/scale.rb | 2 +- tests/subgraph.rb | 100 ++++++++++++++++++++++ tests/subgraph_scale.rb | 76 ++++++++++++++++ tests/task_concurrency.rb | 2 +- 24 files changed, 487 insertions(+), 85 deletions(-) create mode 100644 lib/astute/ext/array.rb create mode 100755 tests/subgraph.rb create mode 100755 tests/subgraph_scale.rb diff --git a/bin/astute-simulator b/bin/astute-simulator index d7ba7aba..d95661ce 100755 --- a/bin/astute-simulator +++ b/bin/astute-simulator @@ -14,6 +14,7 @@ # under the License. require 'fuel_deployment/simulator' +require 'astute' simulator = Astute::Simulator.new simulator.run diff --git a/lib/astute.rb b/lib/astute.rb index 55685701..ef80bd2b 100644 --- a/lib/astute.rb +++ b/lib/astute.rb @@ -23,8 +23,10 @@ require 'pp' require 'bunny' require 'zlib' +require 'astute/ext/array' require 'astute/ext/exception' require 'astute/ext/deep_copy' +require 'astute/ext/hash' require 'astute/exceptions' require 'astute/config' require 'astute/logparser' diff --git a/lib/astute/ext/array.rb b/lib/astute/ext/array.rb new file mode 100644 index 00000000..a7ed0764 --- /dev/null +++ b/lib/astute/ext/array.rb @@ -0,0 +1,28 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Array + + def compact_blank + reject do |val| + case val + when Hash then val.compact_blank.blank? + when Array then val.map { |v| v.respond_to?(:compact_blank) ? v.compact_blank : v }.blank? + when String then val.blank? + else val.blank? + end + end + end +end diff --git a/lib/astute/ext/hash.rb b/lib/astute/ext/hash.rb index 16213259..b5597571 100644 --- a/lib/astute/ext/hash.rb +++ b/lib/astute/ext/hash.rb @@ -18,5 +18,15 @@ class Hash def absent_keys(array) array.select { |key| self[key].blank? } end - + + def compact_blank + delete_if do |_key, val| + case val + when Hash then val.compact_blank.blank? + when Array then val.map { |v| v.respond_to?(:compact_blank) ? v.compact_blank : v }.blank? + when String then val.blank? + else val.blank? + end + end + end end diff --git a/lib/astute/task_deployment.rb b/lib/astute/task_deployment.rb index 5020ab06..01b0e09d 100644 --- a/lib/astute/task_deployment.rb +++ b/lib/astute/task_deployment.rb @@ -31,6 +31,60 @@ module Astute @node_class = node_class end + def self.munge_task(tasks_names, tasks_graph) + result = Set.new + tasks_names.each do |task| + if task.is_a? Deployment::Task + result.add task + next + end + Astute.logger.debug("munging task #{task}") + parts = task.split('/') + task_name = parts[0] + task_range = parts[1] + if task_range + Astute.logger.debug("expanding task #{task} range to specific nodes #{task_range}") + node_ids = expand_node_ids(task_range).flatten + Astute.logger.debug("expanded task #{task} range to #{node_ids.to_a}") + else + Astute.logger.debug("expanding task #{task} range to all_nodes") + node_ids = tasks_graph.each_node.collect {|node| node.uid} + end + exp_t = tasks_graph.each_task.select do |_task| + #Astute.logger.debug("task node id comparison is #{_task.node in? node_ids}") + rv = (_task.name == task_name and _task.node.uid.in? node_ids) + rv + end + exp_t.each do |t| + result.add t + end + end + result + end + + def self.expand_node_ids(interval) + interval.split(',').collect do |part| + if part =~ /^(\d+)-(\d+)$/ + ($1.to_i .. $2.to_i).to_a + else + part + end + end + end + + def self.munge_list_of_start_end(tasks_graph, subgraphs) + subgraphs.each do |subgraph| + subgraph['start'] ||= [] + subgraph['end'] ||= [] + Astute.logger.debug("munging start tasks #{subgraph['start'].to_a} ") + subgraph['start'] = munge_task(subgraph['start'], tasks_graph) unless subgraph['start'].blank? + Astute.logger.debug("munged start tasks to #{subgraph['start'].to_a}") + Astute.logger.debug("munging end tasks #{subgraph['end'].to_a} ") + subgraph['end'] = munge_task(subgraph['end'], tasks_graph) unless subgraph['end'].blank? + Astute.logger.debug("munged end tasks to #{subgraph['end'].to_a} ") + end + end + def create_cluster(deployment_options={}) tasks_graph = deployment_options.fetch(:tasks_graph, {}) tasks_directory = deployment_options.fetch(:tasks_directory, {}) @@ -75,6 +129,10 @@ module Astute setup_tasks(tasks_graph, cluster) setup_task_depends(tasks_graph, cluster) setup_task_concurrency(tasks_graph, cluster) + subgraphs = self.class.munge_list_of_start_end(cluster, tasks_metadata.fetch('subgraphs', [])) + cluster.subgraphs = subgraphs unless subgraphs.compact_blank.blank? + Astute.logger.debug(cluster.subgraphs) + cluster.setup_start_end unless cluster.subgraphs.blank? cluster end diff --git a/lib/astute/task_node.rb b/lib/astute/task_node.rb index ff416363..53f5cc49 100644 --- a/lib/astute/task_node.rb +++ b/lib/astute/task_node.rb @@ -32,7 +32,7 @@ module Astute def poll return unless busy? - debug("Node #{id}: task #{task.name}, task status #{task.status}") + debug("Node #{uid}: task #{task.name}, task status #{task.status}") # Please be informed that this code define special method # of Deployment::Node class. We use special method `task` @@ -41,7 +41,7 @@ module Astute if @task.running? @ctx.report({ 'nodes' => [{ - 'uid' => id, + 'uid' => uid, 'deployment_graph_task_name' => task.name, 'progress' => current_progress_bar, 'task_status' => task.status.to_s, @@ -55,7 +55,7 @@ module Astute def report_node_status node_status = { - 'uid' => id, + 'uid' => uid, 'progress' => current_progress_bar, } node_status.merge!(node_report_status) diff --git a/lib/fuel_deployment/cluster.rb b/lib/fuel_deployment/cluster.rb index 942e6f9a..d8ca9828 100644 --- a/lib/fuel_deployment/cluster.rb +++ b/lib/fuel_deployment/cluster.rb @@ -29,12 +29,12 @@ module Deployment # @param [String] id Cluster name def initialize(id=nil) @nodes = {} - @id = id + @uid = id @node_concurrency = Deployment::Concurrency::Counter.new @task_concurrency = Deployment::Concurrency::Group.new @emergency_brake = false @fault_tolerance_groups = [] - + @subgraphs = [] @dot_task_filter = nil @dot_node_filter = nil @dot_plot_number = 0 @@ -43,8 +43,9 @@ module Deployment include Enumerable include Deployment::Log - attr_accessor :id + attr_accessor :uid attr_accessor :gracefully_stop_mark + attr_accessor :subgraphs attr_reader :emergency_brake attr_reader :nodes attr_reader :node_concurrency @@ -139,6 +140,47 @@ module Deployment end end + # Sets up subgraphs for execution + # e.g. user might want to run only a subset + # of tasks: in this case he sends + # an array of subgraphs to be executed. + # Each array consists of starting vertices + # and ending vertices. These vertices are then + # traversed forward or backward + def setup_start_end + cluster_tasks_set = Set.new each_task + tasks_to_include = Set.new + self.subgraphs.each do |subgraph| + setup_start_end_piece(subgraph, cluster_tasks_set).each do |piece| + tasks_to_include.add piece + end + end + to_skip_tasks = cluster_tasks_set - tasks_to_include + to_skip_tasks.each do |task| + warn "Skipping task #{task} due to subgraph evaluation" + task.skip! + end + end + + def setup_start_end_piece(subgraph, cluster) + start_tasks = Set.new + end_tasks = Set.new + subgraph.fetch('start', []).each do |task| + visit(task).each do |t| + start_tasks.add t + end + end + subgraph.fetch('end', []).each do |task| + visit(task, direction: :backward).each do |t| + end_tasks.add t + end + end + start_tasks = start_tasks.empty? ? cluster : start_tasks + end_tasks = end_tasks.empty? ? cluster : end_tasks + start_tasks & end_tasks + end + + # Iterates through the task that are ready to be run # @yield Deployment::Task def each_ready_task @@ -172,7 +214,9 @@ module Deployment not (permanently_visited.include? task or temporary_visited.include? task) end return topology unless next_task - visit next_task, topology, permanently_visited, temporary_visited + visit(next_task, permanently_visited, temporary_visited).each do |task| + topology.insert 0, task + end end topology end @@ -184,7 +228,12 @@ module Deployment # @param [Array] topology A list of topologically sorted tasks # @param [Set] permanently_visited Set of permanently visited tasks # @param [Array] temporary_visited List of temporary visited tasks - def visit(task, topology = [], permanently_visited = Set.new, temporary_visited = []) + # @param [Symbol direction] direction Which direction to traverse things: :forward or :backward + # @yield [Deployment::Task] + def visit(task, permanently_visited = Set.new, temporary_visited = [], direction: :forward, &block) + unless block_given? + return to_enum(method=:visit, task, permanently_visited, temporary_visited, direction: direction) + end if temporary_visited.include? task # This node have already been visited in this small iteration and # it means that there is a loop. @@ -200,8 +249,9 @@ module Deployment # add this node to the last iteration visit list and run recursion # on the forward dependencies temporary_visited << task - task.each_forward_dependency do |forward_task| - visit forward_task, topology, permanently_visited, temporary_visited + task_method = "each_#{direction}_dependency" + task.send(task_method.to_sym) do |_task| + visit(_task, permanently_visited, temporary_visited, direction: direction, &block) end # Small iteration have completed without loops. # We add this node to the list of permanently marked nodes and @@ -209,7 +259,7 @@ module Deployment permanently_visited.add task temporary_visited.delete task # Insert this node to the head of topology sort list and return it. - topology.insert 0, task + yield task end # Process a single node when it's visited. @@ -448,7 +498,7 @@ module Deployment # @return [String] def to_dot template = <<-eos -digraph "<%= id || 'graph' %>" { +digraph "<%= uid || 'graph' %>" { node[ style = "filled, solid"]; <% each_task do |task| -%> <% next unless task.name =~ dot_task_filter if dot_task_filter -%> @@ -488,7 +538,7 @@ digraph "<%= id || 'graph' %>" { if suffix.is_a? Integer suffix = suffix.to_s.rjust 5, '0' end - graph_name = id || 'graph' + graph_name = uid || 'graph' file = "#{graph_name}-#{suffix}.#{type}" end info "Writing the graph image: '#{suffix}' to the file: '#{file}'" @@ -586,7 +636,7 @@ digraph "<%= id || 'graph' %>" { # @return [String] def to_s - "Cluster[#{id}]" + "Cluster[#{uid}]" end # @return [String] diff --git a/lib/fuel_deployment/graph.rb b/lib/fuel_deployment/graph.rb index 8c1eed19..be9123fd 100644 --- a/lib/fuel_deployment/graph.rb +++ b/lib/fuel_deployment/graph.rb @@ -92,7 +92,7 @@ module Deployment # @param [Object] data The task data payload # @param [Class] task_class Optional custom task class # @return [Deployment::Task] - def task_create(task, data=nil, task_class=Deployment::Task) + def task_create(task, data={}, task_class=Deployment::Task) if task_present? task task = task_get task elsif task.is_a? Deployment::Task diff --git a/lib/fuel_deployment/node.rb b/lib/fuel_deployment/node.rb index 38725ef2..e64d2fff 100644 --- a/lib/fuel_deployment/node.rb +++ b/lib/fuel_deployment/node.rb @@ -45,7 +45,7 @@ module Deployment @status = :online @task = nil @critical = false - @id = id || self.name + @uid = id || self.name self.cluster = cluster cluster.node_add self create_new_graph @@ -60,7 +60,7 @@ module Deployment attr_reader :cluster alias :current_task :task attr_reader :graph - attr_accessor :id + attr_accessor :uid attr_reader :critical alias :critical? :critical attr_reader :sync_point @@ -252,8 +252,8 @@ module Deployment # @return [String] def to_s - return "Node[#{id}]" if id == name - "Node[#{id}/#{name}]" + return "Node[#{uid}]" if uid == name + "Node[#{uid}/#{name}]" end # @return [String] diff --git a/lib/fuel_deployment/simulator.rb b/lib/fuel_deployment/simulator.rb index ee3197c5..0d7e5cdf 100644 --- a/lib/fuel_deployment/simulator.rb +++ b/lib/fuel_deployment/simulator.rb @@ -17,6 +17,7 @@ require_relative '../astute/exceptions' require_relative '../astute/config' require_relative '../fuel_deployment' require_relative '../astute/task_deployment' +require_relative '../astute/task_cluster' require 'active_support/all' require 'yaml' require 'optparse' @@ -44,7 +45,7 @@ module Deployment attr_accessor :context end - class TestCluster < Cluster + class TestCluster < Astute::TaskCluster def tasks_to_fail return @tasks_to_fail if @tasks_to_fail @tasks_to_fail = [] @@ -317,7 +318,7 @@ graph much more readable. deployment = Astute::TaskDeployment.new context, Deployment::TestCluster, Deployment::TestNode cluster = deployment.create_cluster yaml_file_data Deployment::Log.logger.level = Logger::INFO - cluster.id = 'simulator' + cluster.uid = 'simulator' cluster end diff --git a/lib/fuel_deployment/task.rb b/lib/fuel_deployment/task.rb index c23c0807..cf32b566 100644 --- a/lib/fuel_deployment/task.rb +++ b/lib/fuel_deployment/task.rb @@ -49,7 +49,7 @@ module Deployment # @param [Deployment::Node] node The task will be assigned to this node # @param [Object] data The data payload. It can be any object and contain any # information that will be required to actually run the task. - def initialize(name, node, data=nil) + def initialize(name, node, data={}) self.name = name @status = :pending @backward_dependencies = Set.new @@ -69,40 +69,6 @@ module Deployment attr_reader :forward_dependencies attr_accessor :data - # Walk the task graph forward using DFS algorithm - # @param [Array] visited The list of visited tasks for loop detection - # @yield [Deployment::Task] - def dfs_forward(visited = [], &block) - return to_enum(:dfs_forward) unless block_given? - if visited.include? self - visited << self - raise Deployment::LoopDetected.new self, 'Loop detected!', visited - end - visited << self - yield self - each_forward_dependency do |task| - task.dfs_forward visited, &block - end - visited.delete self - end - - # Walk the task graph backward using DFS algorithm - # @param [Array] visited The list of visited tasks for loop detection - # @yield [Deployment::Task] - def dfs_backward(visited = [], &block) - return to_enum(:dfs_backward) unless block_given? - if visited.include? self - visited << self - raise Deployment::LoopDetected.new self, 'Loop detected!', visited - end - visited << self - yield self - each_backward_dependency do |task| - task.dfs_backward visited, &block - end - visited.delete self - end - # Set this task's Node object # @param [Deployment::Node] node The new node object # @raise [Deployment::InvalidArgument] if the object is not a Node @@ -119,6 +85,10 @@ module Deployment @name = name.to_s end + def skip! + @data['type'] = 'skipped' + end + # Set the new task status. The task status can influence the dependency # status of the tasks that depend on this task then they should be reset to allow them to update # their status too. @@ -431,6 +401,10 @@ module Deployment status == :dep_failed end + def is_skipped? + @data.fetch('type', nil) == 'skipped' + end + # # This task failed # # @return [true, false] # def abortive? @@ -481,6 +455,7 @@ module Deployment case status when :pending; sync_point? ? :cyan : :white + is_skipped? ? :magenta : :white when :ready :yellow when :successful; diff --git a/spec/unit/fuel_deployment/cluster_spec.rb b/spec/unit/fuel_deployment/cluster_spec.rb index d1ae136b..082a3b04 100644 --- a/spec/unit/fuel_deployment/cluster_spec.rb +++ b/spec/unit/fuel_deployment/cluster_spec.rb @@ -19,7 +19,7 @@ describe Deployment::Cluster do let(:cluster) do cluster = Deployment::Cluster.new - cluster.id = 'test' + cluster.uid = 'test' node1 = cluster.create_node 'node1' node2 = cluster.create_node 'node2' node1.create_task 'task1' @@ -67,12 +67,12 @@ describe Deployment::Cluster do context '#attributes' do it 'has an id' do - expect(subject.id).to eq 'test' + expect(subject.uid).to eq 'test' end it 'can set an id' do - subject.id = 1 - expect(subject.id).to eq 1 + subject.uid = 1 + expect(subject.uid).to eq 1 end it 'has nodes' do @@ -328,13 +328,19 @@ describe Deployment::Cluster do end it 'can walk forward' do - visited = task1_1.dfs_forward.to_a - expect(visited).to eq [task1_1, task1_2, task1_4, task2_1, task2_2, task1_3, task1_4, task2_1, task2_2] + visited = Set.new + cluster.visit(task1_1).each do |t| + visited.add t + end + expect(visited).to eq [task1_1, task1_2, task1_4, task2_1, task2_2, task1_3].to_set end it 'can walk backward' do - visited = task2_2.dfs_backward.to_a - expect(visited).to eq [task2_2, task2_1, task1_4, task1_2, task1_1, task1_3, task1_1] + visited = Set.new + cluster.visit(task2_2, direction: :backward).each do |t| + visited.add t + end + expect(visited).to eq [task2_2, task2_1, task1_4, task1_2, task1_1, task1_3].to_set end it 'can topology sort' do @@ -361,16 +367,16 @@ describe Deployment::Cluster do end it 'can walk forward' do - message = 'Task[task1/node1]: Loop detected! Path: Task[task1/node1], Task[task2/node1], Task[task3/node1], Task[task4/node1], Task[task1/node1]' + message = 'Cluster[test]: Loop detected! Path: Task[task1/node1], Task[task2/node1], Task[task3/node1], Task[task4/node1], Task[task1/node1]' expect do - task1_1.dfs_forward.to_a + cluster.visit(task1_1).to_a end.to raise_error Deployment::LoopDetected, message end it 'can walk backward' do - message = 'Task[task1/node1]: Loop detected! Path: Task[task1/node1], Task[task4/node1], Task[task3/node1], Task[task2/node1], Task[task1/node1]' + message = 'Cluster[test]: Loop detected! Path: Task[task1/node1], Task[task4/node1], Task[task3/node1], Task[task2/node1], Task[task1/node1]' expect do - task1_1.dfs_backward.to_a + cluster.visit(task1_1, direction: :backward).to_a end.to raise_error Deployment::LoopDetected, message end diff --git a/spec/unit/fuel_deployment/graph_spec.rb b/spec/unit/fuel_deployment/graph_spec.rb index 73289ac6..b0141fdd 100644 --- a/spec/unit/fuel_deployment/graph_spec.rb +++ b/spec/unit/fuel_deployment/graph_spec.rb @@ -18,7 +18,7 @@ describe Deployment::Graph do let(:cluster) do cluster = Deployment::Cluster.new - cluster.id = 'test' + cluster.uid = 'test' node1 = cluster.create_node 'node1' node2 = cluster.create_node 'node2' node1.create_task 'task1' @@ -97,7 +97,7 @@ describe Deployment::Graph do it 'creating an existing task will return it and update the data payload' do task1 = subject.task_create 'new_task' - expect(task1.data).to be_nil + expect(task1.data).to eq Hash.new task2 = subject.task_create 'new_task', 'my_data' expect(task2.data).to eq 'my_data' expect(task1).to eq task2 diff --git a/spec/unit/fuel_deployment/node_spec.rb b/spec/unit/fuel_deployment/node_spec.rb index 89f5520a..33bc50e2 100644 --- a/spec/unit/fuel_deployment/node_spec.rb +++ b/spec/unit/fuel_deployment/node_spec.rb @@ -18,7 +18,7 @@ describe Deployment::Node do let(:cluster) do cluster = Deployment::Cluster.new - cluster.id = 'test' + cluster.uid = 'test' node1 = cluster.create_node 'node1' node1.create_task 'task1' cluster @@ -52,7 +52,7 @@ describe Deployment::Node do end it 'should have an id' do - expect(subject.id).to eq 'node1' + expect(subject.uid).to eq 'node1' end it 'should have critical' do @@ -114,8 +114,8 @@ describe Deployment::Node do end it 'can set an id' do - subject.id = 2 - expect(subject.id).to eq 2 + subject.uid = 2 + expect(subject.uid).to eq 2 end it 'will not set task to an invalid object' do @@ -236,7 +236,7 @@ describe Deployment::Node do it 'can to_s' do expect(subject.to_s).to eq 'Node[node1]' - subject.id = 1 + subject.uid = 1 expect(subject.to_s).to eq 'Node[1/node1]' end diff --git a/spec/unit/fuel_deployment/task_spec.rb b/spec/unit/fuel_deployment/task_spec.rb index 4bc29527..7d94a254 100644 --- a/spec/unit/fuel_deployment/task_spec.rb +++ b/spec/unit/fuel_deployment/task_spec.rb @@ -18,7 +18,7 @@ describe Deployment::Task do let(:cluster) do cluster = Deployment::Cluster.new - cluster.id = 'test' + cluster.uid = 'test' node1 = cluster.create_node 'node1' node2 = cluster.create_node 'node2' node1.create_task 'task1' @@ -77,7 +77,7 @@ describe Deployment::Task do end it 'should have a data' do - expect(subject.data).to eq nil + expect(subject.data).to eq Hash.new end it 'should set name as a string' do diff --git a/spec/unit/task_deployment_spec.rb b/spec/unit/task_deployment_spec.rb index 4abae160..264447b9 100644 --- a/spec/unit/task_deployment_spec.rb +++ b/spec/unit/task_deployment_spec.rb @@ -80,6 +80,47 @@ describe Astute::TaskDeployment do "null"=> [] } end + let(:tasks_graph_3) do + { + "null" => + [ + {"id" => "sync_task", + "requires" =>[] + } + ], + "1" => + [ + {"id" => "14", "requires" => [{"node_id" => nil, "name" => "sync_task"}], "required_for" => [{"name" => 15, "node_id" => "1"}]}, + {"id" => "15", "requires" => [{"node_id" => "2", "name" => "6"}]}, + {"id" => "0", "required_for" => [{"name" => 1, "node_id" => "1"}]}, + {"id" => "1", "required_for" => [{"name" => 2, "node_id" => "1"}, {"name" => 3, "node_id" => "1"}]}, + {"id" => "2", "required_for" => [{"name" => 4, "node_id" => "1"}, {"name" => 5, "node_id" => "1"}]}, + {"id" => "3", "required_for" => [{"name" => 6, "node_id" => "1"}, {"name" => 7, "node_id" => "1"}]}, + {"id" => "4", "required_for" => [{"name" => 8, "node_id" => "1"}]}, + {"id" => "5", "required_for" => [{"name" => 10, "node_id" => "1"}]}, + {"id" => "6", "required_for" => [{"name" => 11, "node_id" => "1"}]}, + {"id" => "7", "required_for" => [{"name" => 12, "node_id" => "1"}]}, + {"id" => "8", "required_for" => [{"name" => 9, "node_id" => "1"}]}, + {"id" => "9"}, + {"id" => "10", "required_for" => [{"name" => 9, "node_id" => "1"}]}, + {"id" => "11", "required_for" => [{"name" => 13, "node_id" => "1"}]}, + {"id" => "12", "required_for" => [{"name" => 13, "node_id" => "1"}]}, + {"id" => "13", "required_for" => [{"name" => 9, "node_id" => "1"}]}], + "2" => [ + {"id" => "0", "required_for" => [{"name" => 1, "node_id" => "2"}, + {"name" => 3, "node_id" => "2"}]}, + {"id" => "1", "required_for" => [{"name" => 2, "node_id" => "2"}]}, + {"id" => "2"}, + {"id" => "3", "required_for" => [{"name" => 4, "node_id" => "2"}]}, + {"id" => "4", "requires" => [{"node_id" => 1, "name" => "3"}], "required_for" => [{"name" => 5, "node_id" => "2"}]}, + {"id" => "5", "requires" => [{"node_id" => 1, "name" => "13"}], "required_for" => [{"name" => 7, "node_id" => "2"}]}, + {"id" => "6", "requires" => [{"node_id" => nil, "name" => "sync_task"}], "required_for" => [{"name" => 8, "node_id" => "2"}]}, + {"id" => "7"}, + {"id" => "8"} + ] + } + end + let(:tasks_directory) do {"ironic_post_swift_key"=>{ @@ -343,6 +384,59 @@ describe Astute::TaskDeployment do end end + context 'subgraphs' do + it 'should call subgraph set up if subgraphs are present' do + task_deployment.stubs(:fail_offline_nodes).returns([]) + task_deployment.stubs(:write_graph_to_file) + Astute::TaskCluster.any_instance.expects(:run).returns({:success => true}) + + + ctx.stubs(:report) + Astute::TaskCluster.any_instance.expects(:setup_start_end).once + + subgraphs = [ + { + 'start' => [ + "3", + ], + 'end' => [ + "9" + ] + }, + { + 'start' => [ "4" ] + } + ] + tasks_metadata.merge!("subgraphs" => subgraphs) + task_deployment.deploy( + tasks_metadata: tasks_metadata, + tasks_graph: tasks_graph_3, + tasks_directory: tasks_directory) + end + it 'should not call subgraph setup if subgraphs are not present' do + task_deployment.stubs(:fail_offline_nodes).returns([]) + task_deployment.stubs(:write_graph_to_file) + ctx.stubs(:report) + Astute::TaskCluster.any_instance.expects(:run).returns({:success => true}) + Astute::TaskCluster.any_instance.expects(:setup_start_end).never + + subgraphs = [ + { + 'start' => [], + 'end' => nil + }, + {'start'=>['task99']} + ] + tasks_metadata.merge!("subgraphs" => subgraphs) + task_deployment.deploy( + tasks_metadata: tasks_metadata, + tasks_graph: tasks_graph, + tasks_directory: tasks_directory) + end + end + + + context 'should report final status' do it 'succeed status and 100 progress for all nodes' do diff --git a/spec/unit/task_node_spec.rb b/spec/unit/task_node_spec.rb index 598d00d5..33ed297c 100644 --- a/spec/unit/task_node_spec.rb +++ b/spec/unit/task_node_spec.rb @@ -68,7 +68,7 @@ describe Astute::TaskNode do it 'should run noop puppet task' do cluster_new = Astute::TaskCluster.new - cluster_new.id = 'test2' + cluster_new.uid = 'test2' cluster_new.noop_run = true task_node_new = Astute::TaskNode.new('node_id', cluster_new) task_node_new.context = ctx diff --git a/tests/deployment.rb b/tests/deployment.rb index a4a35973..1219e1a1 100755 --- a/tests/deployment.rb +++ b/tests/deployment.rb @@ -17,7 +17,7 @@ require_relative '../lib/fuel_deployment/simulator' simulator = Astute::Simulator.new cluster = Deployment::TestCluster.new -cluster.id = 'deployment' +cluster.uid = 'deployment' node1_data = [ [0, 1], @@ -52,7 +52,7 @@ node2_data = [ ] cluster = Deployment::TestCluster.new -cluster.id = 'deployment' +cluster.uid = 'deployment' node1 = cluster.node_create 'node1', Deployment::TestNode node2 = cluster.node_create 'node2', Deployment::TestNode @@ -61,6 +61,7 @@ node2.set_critical sync_node.set_as_sync_point sync_node.create_task 'sync_task' + node1_data.each do |task_from, task_to| task_from = node1.graph.create_task "task#{task_from}" task_to = node1.graph.create_task "task#{task_to}" diff --git a/tests/loop.rb b/tests/loop.rb index a5b46d68..df9bf48b 100755 --- a/tests/loop.rb +++ b/tests/loop.rb @@ -17,7 +17,7 @@ require_relative '../lib/fuel_deployment/simulator' simulator = Astute::Simulator.new cluster = Deployment::TestCluster.new -cluster.id = 'loop' +cluster.uid = 'loop' cluster.plot = true if simulator.options[:plot] node1 = Deployment::TestNode.new 'node1', cluster diff --git a/tests/node_concurrency.rb b/tests/node_concurrency.rb index 5ebf2c6a..cfda3d81 100755 --- a/tests/node_concurrency.rb +++ b/tests/node_concurrency.rb @@ -17,7 +17,7 @@ require_relative '../lib/fuel_deployment/simulator' simulator = Astute::Simulator.new cluster = Deployment::TestCluster.new -cluster.id = 'node_concurrency' +cluster.uid = 'node_concurrency' node1 = Deployment::TestNode.new 'node1', cluster node2 = Deployment::TestNode.new 'node2', cluster diff --git a/tests/scale.rb b/tests/scale.rb index 60e4b5f6..f6a654be 100755 --- a/tests/scale.rb +++ b/tests/scale.rb @@ -21,7 +21,7 @@ cluster = Deployment::TestCluster.new TASK_NUMBER = 100 NODE_NUMBER = 100 -cluster.id = 'scale' +cluster.uid = 'scale' cluster.plot = true if simulator.options[:plot] def make_nodes(cluster) diff --git a/tests/subgraph.rb b/tests/subgraph.rb new file mode 100755 index 00000000..c8af99db --- /dev/null +++ b/tests/subgraph.rb @@ -0,0 +1,100 @@ +#!/usr/bin/env ruby +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require_relative '../lib/fuel_deployment/simulator' +require 'astute' + +simulator = Astute::Simulator.new +cluster = Deployment::TestCluster.new +cluster.uid = 'deployment' + +node1_data = [ + [0, 1], + [1, 2], + [1, 3], + [2, 4], + [2, 5], + [3, 6], + [3, 7], + [4, 8], + [5, 10], + [6, 11], + [7, 12], + [8, 9], + [10, 9], + [11, 13], + [12, 13], + [13, 9], + # [9, 14], + [14, 15], +] + +node2_data = [ + [0, 1], + [1, 2], + [0, 3], + [3, 4], + [4, 5], + # [5, 6], + [5, 7], + [6, 8], +] + +cluster = Deployment::TestCluster.new +cluster.uid = 'deployment' + +node1 = cluster.node_create '1', Deployment::TestNode +node2 = cluster.node_create '2', Deployment::TestNode +sync_node = cluster.node_create 'sync_node', Deployment::TestNode +node2.set_critical +sync_node.set_as_sync_point +sync_node.create_task('sync_task', data={}) + +node1_data.each do |task_from, task_to| + task_from = node1.graph.create_task("task#{task_from}", data={}) + task_to = node1.graph.create_task("task#{task_to}", data={}) + node1.graph.add_dependency task_from, task_to +end + +node2_data.each do |task_from, task_to| + task_from = node2.graph.create_task("task#{task_from}", data={}) + task_to = node2.graph.create_task("task#{task_to}", data={}) + node2.graph.add_dependency task_from, task_to +end + +node2['task4'].depends node1['task3'] +node2['task5'].depends node1['task13'] +node1['task15'].depends node2['task6'] + +sync_node['sync_task'].depends node2['task5'] +sync_node['sync_task'].depends node1['task9'] +node2['task6'].depends sync_node['sync_task'] +node1['task14'].depends sync_node['sync_task'] +subgraphs = [ + { + 'start' => [ + "task3", + ], + 'end' => [ + "task9" + ] + }, + { + 'start' => [ "task4" ] + } +] +cluster.subgraphs = Astute::TaskDeployment.munge_list_of_start_end(cluster, subgraphs) +cluster.setup_start_end +simulator.run cluster diff --git a/tests/subgraph_scale.rb b/tests/subgraph_scale.rb new file mode 100755 index 00000000..c4d56205 --- /dev/null +++ b/tests/subgraph_scale.rb @@ -0,0 +1,76 @@ +#!/usr/bin/env ruby +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require_relative '../lib/fuel_deployment/simulator' +require 'astute' + +simulator = Astute::Simulator.new +cluster = Deployment::TestCluster.new + +TASK_NUMBER = 100 +NODE_NUMBER = 1000 + +cluster.uid = 'scale' +cluster.plot = true if simulator.options[:plot] + +def make_nodes(cluster) + 1.upto(NODE_NUMBER).map do |node| + Deployment::TestNode.new "node#{node}", cluster + end +end + +def make_tasks(node) + previous_task = nil + 1.upto(TASK_NUMBER).each do |number| + task = "task#{number}" + unless previous_task + previous_task = task + next + end + task_from = node.graph.create_task previous_task + task_to = node.graph.create_task task + node.graph.add_dependency task_from, task_to + previous_task = task + end +end + +make_nodes cluster + +cluster.each_node do |node| + puts "Make tasks for: #{node}" + make_tasks node + nil +end + +cluster.each_node do |node| + next if node.name == 'node1' + node['task10'].depends cluster['node1']['task50'] +end +subgraphs = [ + { + 'start' => [ + "task3", + ], + 'end' => [ + "task29" + ] + }, + { + 'start' => [ "task4" ] + } +] +cluster.subgraphs = Astute::TaskDeployment.munge_list_of_start_end(cluster, subgraphs) +cluster.setup_start_end +simulator.run cluster diff --git a/tests/task_concurrency.rb b/tests/task_concurrency.rb index e96d0297..ac0a8ce1 100755 --- a/tests/task_concurrency.rb +++ b/tests/task_concurrency.rb @@ -17,7 +17,7 @@ require_relative '../lib/fuel_deployment/simulator' simulator = Astute::Simulator.new cluster = Deployment::TestCluster.new -cluster.id = 'task_concurrency' +cluster.uid = 'task_concurrency' node1 = Deployment::TestNode.new 'node1', cluster node2 = Deployment::TestNode.new 'node2', cluster