openstack
/
fuel-astute
Code Issues Proposed changes

Merge "Add Simulator class and command"

This commit is contained in:
Jenkins 2016-07-29 15:45:28 +00:00 committed by Gerrit Code Review
parent 86fc3de327 b106c0557d
commit d31ce10481
21 changed files with 504 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -17,3 +17,7 @@ docs/_build
raemon/
*.svg
*.png
*.yaml
!examples/example_astute_config.yaml

2
astute.gemspec
View File

@ -28,7 +28,7 @@ Gem::Specification.new do |s|
s.add_development_dependency 'simplecov-rcov', '~> 0.2.3'
s.files = Dir.glob("{bin,lib,spec,examples}/**/*")
s.executables = ['astuted']
s.executables = %w(astuted astute-simulator)
s.require_path = 'lib'
end

26
tests/mini.rb → bin/astute-simulator
View File

@ -1,5 +1,5 @@
#!/usr/bin/env ruby
# Copyright 2015 Mirantis, Inc.
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -13,25 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require 'fuel_deployment/simulator'
cluster = Deployment::TestCluster.new 'mini'
cluster.plot = true if options[:plot]
node1 = Deployment::TestNode.new 'node1', cluster
node1.graph.add_new_task 'task1'
node1.graph.add_new_task 'task2'
node1.graph.add_new_task 'task3'
node1['task1'].before node1['task2']
node1['task2'].before node1['task3']
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end
simulator = Astute::Simulator.new
simulator.run

33
lib/astute/orchestrator.rb
View File

@ -74,12 +74,8 @@ module Astute
Astute.logger.info "Task based deployment will be used"
deployment_engine = TaskDeployment.new(context)
deployment_engine.deploy(
tasks_graph: deployment_options[:tasks_graph],
tasks_directory: deployment_options[:tasks_directory],
tasks_metadata: deployment_options[:tasks_metadata],
dry_run: deployment_options.fetch(:dry_run, false)
)
write_input_data_to_file(context, deployment_options) if Astute.config.enable_graph_file
deployment_engine.deploy(deployment_options)
ensure
Astute.logger.info "Deployment summary: time was spent " \
"#{time_summary(time_start)}"
@ -287,5 +283,30 @@ module Astute
Time.at(amount_time).utc.strftime("%H:%M:%S")
end
# Dump the task graph data to a file
# @param [Astute::Context] context
# @param [Hash] data
def write_input_data_to_file(context, data={})
yaml_file = File.join(
Astute.config.graph_dot_dir,
"graph-#{context.task_id}.yaml"
)
data = filter_sensitive_data(data)
File.open(yaml_file, 'w') { |f| f.write(YAML.dump(data)) }
Astute.logger.info("Check inpute data file #{yaml_file}")
end
# Remove the potentially sensitive data
# from the task parameters before dumping the graph
# @param [Hash] data
# @return [Hash]
def filter_sensitive_data(data)
data = data.deep_dup
data[:tasks_graph].each do |_node_id, tasks|
tasks.each { |task| task.delete('parameters') }
end
data
end
end # class
end # module

28
lib/astute/task_deployment.rb
View File

@ -11,24 +11,28 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require 'fuel_deployment'
require_relative '../fuel_deployment'
module Astute
class TaskDeployment
def initialize(context)
def initialize(context, cluster_class=TaskCluster, node_class=TaskNode)
@ctx = context
@cluster_class = cluster_class
@node_class = node_class
end
def deploy(tasks_graph: {}, tasks_directory: {} , tasks_metadata: {}, dry_run: false)
raise DeploymentEngineError, "Deployment graph was not provided!" if
tasks_graph.blank?
def create_cluster(deployment_options={})
tasks_graph = deployment_options.fetch(:tasks_graph, {})
tasks_directory = deployment_options.fetch(:tasks_directory, {})
tasks_metadata = deployment_options.fetch(:tasks_metadata, {})
raise DeploymentEngineError, 'Deployment graph was not provided!' if tasks_graph.blank?
support_virtual_node(tasks_graph)
unzip_graph(tasks_graph, tasks_directory)
Deployment::Log.logger = Astute.logger
cluster = TaskCluster.new
cluster = @cluster_class.new
cluster.node_concurrency.maximum = Astute.config.max_nodes_per_call
cluster.stop_condition { Thread.current[:gracefully_stop] }
cluster.fault_tolerance_groups = tasks_metadata.fetch(
@ -40,7 +44,7 @@ module Astute
critical_uids = critical_node_uids(cluster.fault_tolerance_groups)
tasks_graph.keys.each do |node_id|
node = TaskNode.new(node_id, cluster)
node = @node_class.new(node_id, cluster)
node.context = @ctx
node.set_critical if critical_uids.include?(node_id)
node.set_as_sync_point if sync_point?(node_id)
@ -50,7 +54,13 @@ module Astute
setup_tasks(tasks_graph, cluster)
setup_task_depends(tasks_graph, cluster)
setup_task_concurrency(tasks_graph, cluster)
cluster
end
def deploy(deployment_options={})
cluster = create_cluster(deployment_options)
dry_run = deployment_options.fetch(:dry_run, false)
Deployment::Log.logger = Astute.logger if Astute.respond_to? :logger
write_graph_to_file(cluster)
if dry_run
result = Hash.new
@ -174,7 +184,7 @@ module Astute
tasks_graph['virtual_sync_node'] = tasks_graph['null']
tasks_graph.delete('null')
tasks_graph.each do |node_id, tasks|
tasks_graph.each do |_node_id, tasks|
tasks.each do |task|
task.fetch('requires',[]).each do |d_t|
d_t['node_id'] = 'virtual_sync_node' if d_t['node_id'].nil?

18
lib/fuel_deployment.rb
View File

@ -12,13 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
require 'fuel_deployment/error'
require 'fuel_deployment/log'
require 'fuel_deployment/version'
require_relative 'fuel_deployment/error'
require_relative 'fuel_deployment/log'
require_relative 'fuel_deployment/version'
require 'fuel_deployment/task'
require 'fuel_deployment/graph'
require 'fuel_deployment/node'
require 'fuel_deployment/cluster'
require 'fuel_deployment/concurrency/group'
require 'fuel_deployment/concurrency/counter'
require_relative 'fuel_deployment/task'
require_relative 'fuel_deployment/graph'
require_relative 'fuel_deployment/node'
require_relative 'fuel_deployment/cluster'
require_relative 'fuel_deployment/concurrency/group'
require_relative 'fuel_deployment/concurrency/counter'

42
lib/fuel_deployment/cluster.rb
View File

@ -33,6 +33,11 @@ module Deployment
@node_concurrency = Deployment::Concurrency::Counter.new
@task_concurrency = Deployment::Concurrency::Group.new
@emergency_brake = false
@fault_tolerance_groups = []
@dot_task_filter = nil
@dot_node_filter = nil
@dot_plot_number = 0
end
include Enumerable
@ -45,6 +50,9 @@ module Deployment
attr_reader :node_concurrency
attr_reader :task_concurrency
attr_reader :fault_tolerance_groups
attr_accessor :dot_node_filter
attr_accessor :dot_task_filter
attr_accessor :dot_plot_number
# Add an existing node object to the cluster
# @param [Deployment::Node] node a new node object
@ -258,7 +266,7 @@ module Deployment
topology_sort
result = loop do
if all_nodes_are_successful?
status = 'All nodes are deployed successfully.'\
status = 'All nodes are deployed successfully. '\
'Stopping the deployment process!'
result = {
:success => true,
@ -443,11 +451,17 @@ module Deployment
digraph "<%= id || 'graph' %>" {
node[ style = "filled, solid"];
<% each_task do |task| -%>
<% next unless task.name =~ dot_task_filter if dot_task_filter -%>
<% next unless task.node.name =~ dot_node_filter if dot_node_filter and task.node -%>
"<%= task %>" [label = "<%= task %>", fillcolor = "<%= task.color %>"];
<% end -%>
<% each_task do |task| -%>
<% task.each_forward_dependency do |forward_task| -%>
<% next unless task.name =~ dot_task_filter if dot_task_filter -%>
<% next unless task.node.name =~ dot_node_filter if dot_node_filter and task.node -%>
<% next unless forward_task.name =~ dot_task_filter if dot_task_filter -%>
<% next unless forward_task.node.name =~ dot_node_filter if dot_node_filter and forward_task.node -%>
"<%= task %>" -> "<%= forward_task %>";
<% end -%>
<% end -%>
@ -457,18 +471,19 @@ digraph "<%= id || 'graph' %>" {
end
# Plot the graph using the 'dot' binary
# @param [Integer,String] suffix File name index or suffix.
# Will use incrementing value unless provided.
# @param [String] type The type of image produced
# @param [String] file Save image to this file
# @param [Hash] options
# Will use autogenerated name in the current folder unless provided
# @return [true, false] Successful?
def make_image(suffix=nil, file=nil, type='svg')
def make_image(options={})
file = options.fetch :file, nil
suffix = options.fetch :suffix, nil
type = options.fetch :type, 'svg'
unless file
unless suffix
@plot_number = 0 unless @plot_number
suffix = @plot_number
@plot_number += 1
suffix = dot_plot_number
self.dot_plot_number += 1
end
if suffix.is_a? Integer
suffix = suffix.to_s.rjust 5, '0'
@ -476,8 +491,9 @@ digraph "<%= id || 'graph' %>" {
graph_name = id || 'graph'
file = "#{graph_name}-#{suffix}.#{type}"
end
command = "dot -T#{type} -o#{file}"
Open3.popen2e(command) do |stdin, out, process|
info "Writing the graph image: '#{suffix}' to the file: '#{file}'"
command = ['dot', '-T', type, '-o', file]
Open3.popen2e(*command) do |stdin, out, process|
stdin.puts to_dot
stdin.close
output = out.read
@ -550,10 +566,10 @@ digraph "<%= id || 'graph' %>" {
end
def count_tolerance_fail(node)
@fault_tolerance_groups.select do |g|
fault_tolerance_groups.select do |g|
g['node_ids'].include?(node.name)
end.each do |group|
debug "Count faild node #{node.name} for group #{group['name']}"
debug "Count failed node #{node.name} for group #{group['name']}"
group['fault_tolerance'] -= 1
group['node_ids'].delete(node.name)
group['failed_node_ids'] << node.name
@ -561,7 +577,7 @@ digraph "<%= id || 'graph' %>" {
end
def fault_tolerance_excess?
is_failed = @fault_tolerance_groups.select { |group| group['fault_tolerance'] < 0 }
is_failed = fault_tolerance_groups.select { |group| group['fault_tolerance'] < 0 }
return false if is_failed.empty?
warn "Fault tolerance exceeded the stop conditions #{is_failed}"

350
lib/fuel_deployment/simulator.rb Normal file
View File

@ -0,0 +1,350 @@
#!/usr/bin/env ruby
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require_relative '../astute/exceptions'
require_relative '../astute/config'
require_relative '../fuel_deployment'
require_relative '../astute/task_deployment'
require 'active_support/all'
require 'yaml'
require 'optparse'
require 'find'
module Deployment
class TestNode < Node
def run(task)
debug "Run task: #{task}"
self.task = task
self.status = :busy
end
def poll
debug 'Poll node status'
if busy?
status = :successful
status = :failed if cluster.tasks_to_fail.include? "#{task.name}/#{task.node.name}"
debug "#{task} finished with: #{status}"
self.task.status = status
self.status = :online
end
end
attr_accessor :context
end
class TestCluster < Cluster
def tasks_to_fail
return @tasks_to_fail if @tasks_to_fail
@tasks_to_fail = []
end
def tasks_to_fail=(value)
@tasks_to_fail = value
end
attr_accessor :plot_post_node
attr_accessor :plot_pre_node
def hook_post_node(*args)
return unless plot_post_node
make_image
end
def hook_pre_node(*args)
return unless plot_pre_node
make_image
end
end
end
module Astute
class SimulatorContext
attr_accessor :reporter, :deploy_log_parser
attr_reader :task_id, :status
def initialize(task_id, reporter=nil, deploy_log_parser=nil)
@task_id = task_id
@reporter = reporter
@status = {}
@deploy_log_parser = deploy_log_parser
end
def report_and_update_status(_data)
end
def report(_msg)
end
end
end
module Astute
class TaskDeployment
def fail_offline_nodes(*_args)
[]
end
end
end
module Astute
class Simulator
# Parse the simulator CLI options
# @return [Hash]
def options
return @options if @options
@options = {}
parser = OptionParser.new do |opts|
opts.on('-l', '--list-yamls', 'List all deployment YAML files') do
options[:list_yaml_files] = true
end
opts.on('-L', '--list-images', 'List all generated image files') do
options[:list_image_files] = true
end
opts.on('-r', '--remove-yamls', 'Remove all deployment YAML files') do
options[:remove_yaml_files] = true
end
opts.on('-R', '--remove-images', 'Remove all generated image files') do
options[:remove_image_files] = true
end
opts.on('-y', '--yaml FILE', 'Load this file as the tasks graph from Astute') do |value|
options[:yaml] = value
end
opts.on('-f', '--fail task/node,...', Array, 'Set the tasks that will fail during the simulation') do |value|
options[:tasks_to_fail] = value
end
opts.on('-p', '--plot-first', 'Plot the first deployment step') do
options[:plot_first] = true
end
opts.on('-P', '--plot-last', 'Plot the last deployment step') do
options[:plot_last] = true
end
opts.on('-d', '--debug', 'Show debug messages') do
options[:debug] = true
end
opts.on('-N', '--plot-pre-node', 'Make an image snapshot om every node visit before doing anything. It will create a lot of images!') do
options[:plot_pre_node] = true
end
opts.on('-n', '--plot-post-node', 'Make an image snapshot after a task on a node have been run. Only impotent steps will be saved.') do
options[:plot_post_node] = true
end
opts.on('-g', '--graph-task-filter REGEXP', 'Plot only tasks with matching name.') do |value|
options[:graph_task_filter] = Regexp.new value
end
opts.on('-G', '--graph-node-filter REGEXP', 'Plot only tasks with matching node name.') do |value|
options[:graph_node_filter] = Regexp.new value
end
end
parser.banner = <<-eof
Usage: astute-simulator [options]
This tool uses the Astute task deployment libraries to simulate the deployment process.
It should load the YAML file with the tasks data dumps produced by Astute during the
deployment and run the simulated deployment. The output can be produced either as a
deployment log or as a set of graph image snapshots using Graphviz to render them.
You can use 'list-images' to locate the dumped YAML files and then
run the simulation like this:
# astute-simulator -y /path/to/yaml/file.yaml
The simulation should produce the log output displaying the which tasks and
on which nodes are being run as well as the deployment result status.
You can grep this log to determine the order the task will be started without
creating any graph images.
You can use 'plot-first' and 'plot-last' options to generate an SVG image of
the initial graph status and the result status of the deployment. It will
generate the 'start' and the 'end' images in the current directory.
You may have to download them from the Fuel master node in order to view them
on your system. The helper tools 'list-images' and 'remove-images' can be used
to manage the generated images.
Options 'plot-pre-node' and 'plot-post-node' will enable taking snapshots of the
graph status either before the node is processed or after the task have been
run on a node. These images can be very useful for debugging graph ordering
issues but it will take a lot of time to generate them if there are many tasks
in the graph.
Option 'fail' can be used to simulate a task failure.
The argument is a comma-separated list of task/node pairs like this:
# astute-simulator -y /path/to/yaml/file.yaml -f ntp-client/2,heat-db/1 -P
This command will simulate the fail of two task on the node1 and node2.
The final state of the graph will be plotted so the consequences of these failure
can be clearly seen.
The task status is color coded in the graph image:
* white - A task has not been run yet.
* cyan - A task is a sync-point (ignores dependency failures) and have not been run yet.
* yellow - A task has all dependencies satisfied or has none and can be started right now.
* blue - A task is running right now.
* green - A task has already been run and was successful.
* red - A task has already been run and have failed.
* orange - A task has some failed dependencies or its node is failed and a task will not run.
* violet - A task is skipped or its node is skipped and a task will not run.
There are two filtering options 'graph-task-filter' and 'graph-node-filter'. You can use them
to limit the scope of plotted graph only to tasks with name or node name matching the provided
regular expressions. For example, plot only 'openstack' related tasks on the second node:
# astute-simulator -y /path/to/yaml/file.yaml -g openstack -G '^2$' -p
Limiting the number of plotted task will speed up the image generation and will make the
graph much more readable.
eof
parser.parse!
@options[:yaml] = ARGV[0] unless @options[:yaml]
@options
end
# Output the list of deployment YAML files
def list_yaml_files
yaml_files do |file|
puts file
end
end
# Output the list of generated image files
def list_image_files
image_files do |file|
puts file
end
end
# Remove all deployment YAML files
def remove_yaml_files
yaml_files do |file|
puts "Remove: #{file}"
File.unlink file if File.file? file
end
end
# Remove all generated image files
def remove_image_files
image_files do |file|
puts "Remove: #{file}"
File.unlink file if File.file? file
end
end
# Find all image files starting from the current folder
def image_files
return to_enum(:image_files) unless block_given?
Find.find('.') do |file|
next unless File.file? file
next unless file.end_with?('.svg') or file.end_with?('.png')
yield file
end
end
# Find all deployment yaml files in the dot files dir
def yaml_files
return to_enum(:yaml_files) unless block_given?
return unless File.directory? Astute.config.graph_dot_dir
Find.find(Astute.config.graph_dot_dir) do |file|
next unless File.file? file
next unless file.end_with? '.yaml'
yield file
end
end
# Set the cluster options and run the simulation
# @param cluster [Deployment::Cluster]
def deploy(cluster)
unless cluster.is_a? Deployment::Cluster
raise Astute::DeploymentEngineError, "Argument should be a Cluster object! Got: #{cluster.class}"
end
if options[:debug]
Deployment::Log.logger.level = Logger::DEBUG
end
if options[:tasks_to_fail]
cluster.tasks_to_fail = options[:tasks_to_fail]
end
if options[:graph_task_filter]
cluster.dot_task_filter = options[:graph_task_filter]
end
if options[:graph_node_filter]
cluster.dot_node_filter = options[:graph_node_filter]
end
if options[:plot_first]
cluster.make_image(suffix: 'start')
end
if options[:plot_pre_node]
cluster.plot_pre_node = true
end
if options[:plot_post_node]
cluster.plot_post_node = true
end
result = cluster.run
if options[:plot_last]
cluster.make_image(suffix: 'end')
end
cluster.info "Result: #{result.inspect}"
end
# Create a cluster object from the dumped YAML file
# using the TaskDeployment class from Astute
# @param yaml_file [String] Path to the YAML file
# @return [Deployment::Cluster]
def cluster_from_yaml(yaml_file=nil)
raise Astute::DeploymentEngineError, 'No task YAML file have been provided!' unless yaml_file
raise Astute::DeploymentEngineError, "No such file: #{yaml_file}" unless File.exists? yaml_file
yaml_file_data = YAML.load_file yaml_file
raise Astute::DeploymentEngineError, 'Wrong data! YAML should contain Hash!' unless yaml_file_data.is_a? Hash
context = Astute::SimulatorContext.new 'simulator'
deployment = Astute::TaskDeployment.new context, Deployment::TestCluster, Deployment::TestNode
cluster = deployment.create_cluster yaml_file_data
Deployment::Log.logger.level = Logger::INFO
cluster.id = 'simulator'
cluster
end
# Run an action based on options
# @param cluster [Deployment::Cluster]
def run(cluster=nil)
if options[:list_yaml_files]
list_yaml_files
elsif options[:list_image_files]
list_image_files
elsif options[:remove_yaml_files]
remove_yaml_files
elsif options[:remove_image_files]
remove_image_files
else
unless cluster
if options[:yaml]
cluster = cluster_from_yaml options[:yaml]
else
puts 'You have neither provided a task YAML file nor used a helper action!'
puts 'Please, point the simulator at the task graph YAML dump using the "-y" option.'
exit(1)
end
end
deploy cluster
end
end
end
end

13
lib/fuel_deployment/task.rb
View File

@ -341,8 +341,13 @@ module Deployment
# set the pending tasks to dep_failed if the node have failed
def check_for_node_status
return unless node
if Deployment::Node::FAILED_STATUSES.include? node.status and NOT_RUN_STATUSES.include? status
self.status = :dep_failed
if NOT_RUN_STATUSES.include? status
if Deployment::Node::FAILED_STATUSES.include? node.status
self.status = :dep_failed
end
if node.status == :skipped
self.status = :skipped
end
end
end
@ -483,9 +488,9 @@ module Deployment
when :failed;
:red
when :dep_failed;
:magenta
:orange
when :skipped;
:purple
:violet
when :running;
:blue
else

4
spec/unit/fuel_deployment/task_spec.rb
View File

@ -121,7 +121,7 @@ describe Deployment::Task do
expect(task2_2.color).to eq :white
task2_1.status = :failed
expect(task2_1.color).to eq :red
expect(task2_2.color).to eq :magenta
expect(task2_2.color).to eq :orange
task2_1.status = :running
task2_2.status = :pending
expect(task2_1.color).to eq :blue
@ -132,7 +132,7 @@ describe Deployment::Task do
expect(task2_2.color).to eq :yellow
task2_1.status = :skipped
task2_2.status = :pending
expect(task2_1.color).to eq :purple
expect(task2_1.color).to eq :violet
expect(task2_2.color).to eq :yellow
end

8
spec/unit/orchestrator_spec.rb
View File

@ -20,6 +20,7 @@ describe Astute::Orchestrator do
before(:each) do
@orchestrator = Astute::Orchestrator.new
@orchestrator.stubs(:write_input_data_to_file)
@reporter = mock('reporter')
@reporter.stub_everything
end
@ -66,12 +67,12 @@ describe Astute::Orchestrator do
it 'should run task deployment' do
Astute::TaskDeployment.any_instance.expects(:deploy).with(
Astute::TaskDeployment.any_instance.expects(:deploy).with({
:tasks_metadata => tasks_metadata,
:tasks_graph => tasks_graph,
:tasks_directory => tasks_directory,
:dry_run => false
)
})
@orchestrator.task_deploy(
@reporter,
@ -79,7 +80,8 @@ describe Astute::Orchestrator do
{
:tasks_metadata => tasks_metadata,
:tasks_graph => tasks_graph,
:tasks_directory => tasks_directory
:tasks_directory => tasks_directory,
:dry_run => false
}
)
end

1
specs/astute.spec
View File

@ -113,6 +113,7 @@ install -D -m644 %{_builddir}/%{rbname}-%{version}/%{rbname}.service %{buildroot
%dir /var/lib/astute
%dir /var/lib/astute/graphs
%config(noreplace) %{_bindir}/astuted
%config(noreplace) %{_bindir}/astute-simulator
%config(noreplace) %{_sysconfdir}/sysconfig/%{rbname}
%doc %{gem_dir}/doc/%{rbname}-%{version}

60
tests/concurrency.rb
View File

@ -1,60 +0,0 @@
#!/usr/bin/env ruby
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
cluster = Deployment::TestCluster.new
cluster.id = 'concurrency'
cluster.plot = true if options[:plot]
node1 = Deployment::TestNode.new 'node1', cluster
node2 = Deployment::TestNode.new 'node2', cluster
node3 = Deployment::TestNode.new 'node3', cluster
node4 = Deployment::TestNode.new 'node4', cluster
node5 = Deployment::TestNode.new 'node5', cluster
node1.add_new_task('task1')
node1.add_new_task('final')
node2.add_new_task('task1')
node2.add_new_task('final')
node3.add_new_task('task1')
node3.add_new_task('final')
node4.add_new_task('task1')
node4.add_new_task('final')
node5.add_new_task('task1')
node5.add_new_task('final')
node1['final'].after node1['task1']
node2['final'].after node2['task1']
node3['final'].after node3['task1']
node4['final'].after node4['task1']
node5['final'].after node5['task1']
node1['task1'].maximum_concurrency = 2
node1['final'].maximum_concurrency = 1
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end

25
tests/deployment.rb
View File

@ -13,7 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require_relative '../lib/fuel_deployment/simulator'
simulator = Astute::Simulator.new
cluster = Deployment::TestCluster.new
cluster.id = 'deployment'
node1_data = [
[0, 1],
@ -49,17 +53,12 @@ node2_data = [
cluster = Deployment::TestCluster.new
cluster.id = 'deployment'
cluster.plot = true if options[:plot]
node1 = cluster.node_create 'node1', Deployment::TestNode
node2 = cluster.node_create 'node2', Deployment::TestNode
sync_node = cluster.node_create 'sync_node', Deployment::TestNode
node2.set_critical if options[:critical]
node2.set_critical
sync_node.set_as_sync_point
sync_node.create_task 'sync_task'
node1_data.each do |task_from, task_to|
@ -74,8 +73,6 @@ node2_data.each do |task_from, task_to|
node2.graph.add_dependency task_from, task_to
end
node2.fail_tasks << node2['task4'] if options[:fail]
node2['task4'].depends node1['task3']
node2['task5'].depends node1['task13']
node1['task15'].depends node2['task6']
@ -85,12 +82,4 @@ sync_node['sync_task'].depends node1['task9']
node2['task6'].depends sync_node['sync_task']
node1['task14'].depends sync_node['sync_task']
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
p cluster.run
end
simulator.run cluster

77
tests/fuel.rb
View File

@ -1,77 +0,0 @@
#!/usr/bin/env ruby
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require 'yaml'
file = if File.file? ARGV[0].to_s
ARGV[0].to_s
else
File.join File.dirname(__FILE__), 'fuel.yaml'
end
deployment_tasks = YAML.load_file file
fail 'Wrong data! YAML should contain Hash!' unless deployment_tasks.is_a? Hash
cluster = Deployment::TestCluster.new
cluster.id = 'fuel'
cluster.plot = true if options[:plot]
deployment_tasks.each do |node_name, node_tasks|
node = cluster.node_create node_name, Deployment::TestNode
node_tasks.each do |task_data|
cluster[node].create_task task_data['id'], task_data
end
end
deployment_tasks.each do |node_name, node_tasks|
node_tasks.each do |task_data|
task_name = task_data['id']
task = cluster[node_name][task_name]
requires = task_data.fetch 'requires', []
requires.each do |requirement|
next unless requirement.is_a? Hash
required_task = cluster[requirement['node_id']][requirement['name']]
unless required_task
warn "Task: #{requirement['name']} is not found on node: #{cluster[requirement['node_id']]}"
next
end
task.requires required_task
end
required_for = task_data.fetch 'required_for', []
required_for.each do |requirement|
next unless requirement.is_a? Hash
required_by_task = cluster[requirement['node_id']][requirement['name']]
task = cluster[node_name][task_data['id']]
unless required_by_task
warn "Task: #{requirement['name']} is not found on node: #{cluster[requirement['node_id']]}"
next
end
task.is_required required_by_task
end
end
end
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end

21
tests/loop.rb
View File

@ -13,11 +13,13 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require_relative '../lib/fuel_deployment/simulator'
cluster = Deployment::Cluster.new
simulator = Astute::Simulator.new
cluster = Deployment::TestCluster.new
cluster.id = 'loop'
cluster.plot = true if options[:plot]
cluster.plot = true if simulator.options[:plot]
node1 = Deployment::TestNode.new 'node1', cluster
task1 = node1.graph.add_new_task 'task1'
@ -26,15 +28,6 @@ task3 = node1.graph.add_new_task 'task3'
task2.after task1
task3.after task2
task1.after task3
task1.after task3 if options[:fail]
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end
simulator.run cluster

14
tests/node_concurrency.rb
View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require_relative '../lib/fuel_deployment/simulator'
simulator = Astute::Simulator.new
cluster = Deployment::TestCluster.new
cluster.id = 'node_concurrency'
cluster.plot = true if options[:plot]
node1 = Deployment::TestNode.new 'node1', cluster
node2 = Deployment::TestNode.new 'node2', cluster
@ -53,12 +53,4 @@ node6['final'].after node6['task1']
cluster.node_concurrency.maximum = 2
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end
simulator.run cluster

18
tests/scale.rb
View File

@ -13,14 +13,16 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require_relative '../lib/fuel_deployment/simulator'
simulator = Astute::Simulator.new
cluster = Deployment::TestCluster.new
TASK_NUMBER = 100
NODE_NUMBER = 100
cluster = Deployment::TestCluster.new
cluster.id = 'scale'
cluster.plot = true if options[:plot]
cluster.plot = true if simulator.options[:plot]
def make_nodes(cluster)
1.upto(NODE_NUMBER).map do |node|
@ -56,12 +58,4 @@ cluster.each_node do |node|
node['task10'].depends cluster['node1']['task50']
end
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end
simulator.run cluster

18
tests/simulator.rb Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env ruby
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require_relative '../lib/fuel_deployment/simulator'
simulator = Astute::Simulator.new
simulator.run

14
tests/task_concurrency.rb
View File

@ -13,11 +13,11 @@
# License for the specific language governing permissions and limitations
# under the License.
require File.absolute_path File.join File.dirname(__FILE__), 'test_node.rb'
require_relative '../lib/fuel_deployment/simulator'
simulator = Astute::Simulator.new
cluster = Deployment::TestCluster.new
cluster.id = 'task_concurrency'
cluster.plot = true if options[:plot]
node1 = Deployment::TestNode.new 'node1', cluster
node2 = Deployment::TestNode.new 'node2', cluster
@ -49,12 +49,4 @@ node5['final'].after node5['task1']
cluster.task_concurrency['task1'].maximum = 3
cluster.task_concurrency['final'].maximum = 2
if options[:plot]
cluster.make_image 'start'
end
if options[:interactive]
binding.pry
else
cluster.run
end
simulator.run cluster

88
tests/test_node.rb
View File

@ -1,88 +0,0 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
lib_dir = File.join File.dirname(__FILE__), '../lib'
lib_dir = File.absolute_path File.expand_path lib_dir
$LOAD_PATH << lib_dir
require 'rubygems'
require 'fuel_deployment'
require 'optparse'
require 'pry'
Deployment::Log.logger.level = Logger::INFO
def options
return $options if $options
$options = {}
OptionParser.new do |opts|
opts.on('-p', '--plot') do |value|
options[:plot] = value
end
opts.on('-f', '--fail') do |value|
options[:fail] = value
end
opts.on('-c', '--critical') do |value|
options[:critical] = value
end
opts.on('-i', '--interactive') do |value|
options[:interactive] = value
end
opts.on('-d', '--debug') do
Deployment::Log.logger.level = Logger::DEBUG
end
end.parse!
$options
end
module Deployment
class TestNode < Node
def fail_tasks
return @fail_tasks if @fail_tasks
@fail_tasks = []
end
def fail_tasks=(value)
@fail_tasks = value
end
def run(task)
fail Deployment::InvalidArgument, "#{self}: Node can run only tasks" unless task.is_a? Deployment::Task
debug "Run task: #{task}"
self.task = task
self.status = :busy
end
def poll
debug 'Poll node status'
if busy?
status = :successful
status = :failed if fail_tasks.include? task
debug "#{task} finished with: #{status}"
self.task.status = status
self.status = :online
self.status = :skipped if task.status == :dep_failed
self.status = :failed if task.status == :failed
end
end
end
class TestCluster < Cluster
attr_accessor :plot
def hook_pre_node(*args)
make_image if plot
end
end
end