diff --git a/astute.gemspec b/astute.gemspec index 567dce29..c0486062 100644 --- a/astute.gemspec +++ b/astute.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |s| s.add_dependency 'amqp', '1.4.1' s.add_dependency 'raemon', '0.3.0' + s.add_development_dependency 'facter' s.add_development_dependency 'rake', '10.0.4' s.add_development_dependency 'rspec', '2.13.0' s.add_development_dependency 'mocha', '0.13.3' diff --git a/lib/astute.rb b/lib/astute.rb index 33f88843..00f065de 100644 --- a/lib/astute.rb +++ b/lib/astute.rb @@ -34,10 +34,14 @@ require 'astute/cobbler_manager' require 'astute/image_provision' require 'astute/dump' require 'astute/deploy_actions' -require 'astute/post_deploy_actions/update_no_quorum_policy' -require 'astute/post_deploy_actions/restart_radosgw' -require 'astute/post_deploy_actions/update_cluster_hosts_info' -require 'astute/post_deploy_actions/upload_cirros_image' +['/astute/pre_deploy_actions/*.rb', + '/astute/pre_node_actions/*.rb', + '/astute/post_deploy_actions/*.rb', + '/astute/post_deployment_actions/*.rb', + '/astute/common_actions/*.rb' + ].each do |path| + Dir[File.dirname(__FILE__) + path].each{ |f| require f } +end require 'astute/ssh' require 'astute/ssh_actions/ssh_erase_nodes' require 'astute/ssh_actions/ssh_hard_reboot' diff --git a/lib/astute/common_actions/pacemaker.rb b/lib/astute/common_actions/pacemaker.rb new file mode 100644 index 00000000..d4255e6d --- /dev/null +++ b/lib/astute/common_actions/pacemaker.rb @@ -0,0 +1,90 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +module Astute + class Pacemaker + + def self.commands(behavior, deployment_info) + return [] if deployment_info.first['deployment_mode'] !~ /ha/i + + controller_nodes = deployment_info.select{ |n| n['role'] =~ /controller/i }.map{ |n| n['uid'] } + return [] if controller_nodes.empty? + + ha_size = deployment_info.first['nodes'].count { |n| + ['controller', 'primary-controller'].include? n['role'] + } + + action = if ha_size < 3 + case behavior + when 'stop' then 'stop' + when 'start' then 'start' + end + else + case behavior + when 'stop' then 'ban' + when 'start' then 'clear' + end + end + + cmds = pacemaker_services_list(deployment_info).inject([]) do |cmds, pacemaker_service| + if ha_size < 3 + cmds << "crm resource #{action} #{pacemaker_service} && sleep 3" + else + cmds << "pcs resource #{action} #{pacemaker_service} `crm_node -n` && sleep 3" + end + end + + cmds + end + + private + + def self.pacemaker_services_list(deployment_info) + services_list = [] + #Heat engine service is present everywhere + services_list += heat_service_name(deployment_info) + + if deployment_info.first['quantum'] + services_list << 'p_neutron-openvswitch-agent' + services_list << 'p_neutron-metadata-agent' + services_list << 'p_neutron-l3-agent' + services_list << 'p_neutron-dhcp-agent' + end + + if deployment_info.first.fetch('ceilometer', {})['enabled'] + services_list += ceilometer_service_names(deployment_info) + end + return services_list + end + + def self.ceilometer_service_names(deployment_info) + case deployment_info.first['cobbler']['profile'] + when /centos/i + ['p_openstack-ceilometer-compute','p_openstack-ceilometer-central'] + when /ubuntu/i + ['p_ceilometer-agent-central','p_ceilometer-agent-compute'] + end + end + + def self.heat_service_name(deployment_info) + case deployment_info.first['cobbler']['profile'] + when /centos/i + ['openstack-heat-engine', 'p_openstack-heat-engine'] + when /ubuntu/i + ['heat-engine', 'p_heat-engine'] + end + end + + end #class +end diff --git a/lib/astute/deploy_actions.rb b/lib/astute/deploy_actions.rb index 05a482a5..fd421544 100644 --- a/lib/astute/deploy_actions.rb +++ b/lib/astute/deploy_actions.rb @@ -27,16 +27,43 @@ module Astute end class PreDeployActions < DeployActions - def initialize(deployment_info, context) super @actions = [] end - end class PostDeployActions < DeployActions + def initialize(deployment_info, context) + super + @actions = [ + PostPatchingHa.new + ] + end + end + class PreNodeActions + + def initialize(context) + @node_uids = [] + @context = context + @actions = [ + PrePatchingHa.new, + StopOSTServices.new, + PrePatching.new + ] + end + + def process(deployment_info) + nodes_to_process = deployment_info.select { |n| !@node_uids.include?(n['uid']) } + return if nodes_to_process.empty? + + @actions.each { |action| action.process(nodes_to_process, @context) } + @node_uids += nodes_to_process.map { |n| n['uid'] } + end + end + + class PostDeploymentActions < DeployActions def initialize(deployment_info, context) super @actions = [ @@ -46,7 +73,6 @@ module Astute UpdateClusterHostsInfo.new ] end - end @@ -56,12 +82,12 @@ module Astute raise "Should be implemented!" end - def run_shell_command(context, node_uids, cmd) + def run_shell_command(context, node_uids, cmd, timeout=60) shell = MClient.new(context, 'execute_shell_command', node_uids, check_result=true, - timeout=60, + timeout=timeout, retries=1) #TODO: return result for all nodes not only for first @@ -80,5 +106,9 @@ module Astute class PreDeployAction < DeployAction; end class PostDeployAction < DeployAction; end + class PreNodeAction < DeployAction; end + class PostNodeAction < DeployAction; end + class PreDeploymentAction < DeployAction; end + class PostDeploymentAction < DeployAction; end end diff --git a/lib/astute/deployment_engine.rb b/lib/astute/deployment_engine.rb index cf5b2625..0a9c3d4a 100644 --- a/lib/astute/deployment_engine.rb +++ b/lib/astute/deployment_engine.rb @@ -56,15 +56,14 @@ module Astute # Sync time sync_time(part.map{ |n| n['uid'] }) - - # Pre deploy hooks - PreDeployActions.new(part, @ctx).process end rescue => e Astute.logger.error("Unexpected error #{e.message} traceback #{e.format_backtrace}") raise e end + pre_node_actions = PreNodeActions.new(@ctx) + fail_deploy = false # Sort by priority (the lower the number, the higher the priority) # and send groups to deploy @@ -76,7 +75,16 @@ module Astute # Prevent deploy too many nodes at once nodes_group.each_slice(Astute.config[:MAX_NODES_PER_CALL]) do |part| if !fail_deploy + + # Pre deploy hooks + pre_node_actions.process(part) + PreDeployActions.new(part, @ctx).process + deploy_piece(part) + + # Post deploy hook + PostDeployActions.new(part, @ctx).process + fail_deploy = fail_critical_node?(part) else nodes_to_report = part.map do |n| diff --git a/lib/astute/orchestrator.rb b/lib/astute/orchestrator.rb index 5f1c4327..a91e28cf 100644 --- a/lib/astute/orchestrator.rb +++ b/lib/astute/orchestrator.rb @@ -41,8 +41,8 @@ module Astute deploy_engine_instance.deploy(deployment_info) - # Post deploy hooks - PostDeployActions.new(deployment_info, context).process + # Post deployment hooks + PostDeploymentActions.new(deployment_info, context).process context.status end diff --git a/lib/astute/post_deploy_actions/post_patching_ha.rb b/lib/astute/post_deploy_actions/post_patching_ha.rb new file mode 100644 index 00000000..9d88e81a --- /dev/null +++ b/lib/astute/post_deploy_actions/post_patching_ha.rb @@ -0,0 +1,40 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +module Astute + class PostPatchingHa < PostDeployAction + + def process(deployment_info, context) + return if deployment_info.first['openstack_version_prev'].nil? || + deployment_info.first['deployment_mode'] !~ /ha/i + + controller_nodes = deployment_info.select{ |n| n['role'] =~ /controller/i }.map{ |n| n['uid'] } + return if controller_nodes.empty? + + Astute.logger.info "Starting unmigration of pacemaker services from " \ + "nodes #{controller_nodes.inspect}" + + Astute::Pacemaker.commands(action='start', deployment_info).each do |pcmk_unban_cmd| + response = run_shell_command(context, controller_nodes, pcmk_unban_cmd) + + if response[:data][:exit_code] != 0 + Astute.logger.warn "#{context.task_id}: Failed to unban service, "\ + "check the debugging output for details" + end + end + + Astute.logger.info "#{context.task_id}: Finished post-patching-ha hook" + end #process + end #class +end diff --git a/lib/astute/post_deploy_actions/restart_radosgw.rb b/lib/astute/post_deployment_actions/restart_radosgw.rb similarity index 97% rename from lib/astute/post_deploy_actions/restart_radosgw.rb rename to lib/astute/post_deployment_actions/restart_radosgw.rb index 0c45d84f..576aa862 100644 --- a/lib/astute/post_deploy_actions/restart_radosgw.rb +++ b/lib/astute/post_deployment_actions/restart_radosgw.rb @@ -13,7 +13,7 @@ # under the License. module Astute - class RestartRadosgw < PostDeployAction + class RestartRadosgw < PostDeploymentAction def process(deployment_info, context) ceph_node = deployment_info.find { |n| n['role'] == 'ceph-osd' } diff --git a/lib/astute/post_deploy_actions/update_cluster_hosts_info.rb b/lib/astute/post_deployment_actions/update_cluster_hosts_info.rb similarity index 97% rename from lib/astute/post_deploy_actions/update_cluster_hosts_info.rb rename to lib/astute/post_deployment_actions/update_cluster_hosts_info.rb index 605bb83d..7182d60f 100644 --- a/lib/astute/post_deploy_actions/update_cluster_hosts_info.rb +++ b/lib/astute/post_deployment_actions/update_cluster_hosts_info.rb @@ -14,7 +14,7 @@ module Astute - class UpdateClusterHostsInfo < PostDeployAction + class UpdateClusterHostsInfo < PostDeploymentAction def process(deployment_info, context) Astute.logger.info "Updating /etc/hosts in all cluster nodes" diff --git a/lib/astute/post_deploy_actions/update_no_quorum_policy.rb b/lib/astute/post_deployment_actions/update_no_quorum_policy.rb similarity index 98% rename from lib/astute/post_deploy_actions/update_no_quorum_policy.rb rename to lib/astute/post_deployment_actions/update_no_quorum_policy.rb index 0653bdb0..3aaa2422 100644 --- a/lib/astute/post_deploy_actions/update_no_quorum_policy.rb +++ b/lib/astute/post_deployment_actions/update_no_quorum_policy.rb @@ -13,7 +13,7 @@ # under the License. module Astute - class UpdateNoQuorumPolicy < PostDeployAction + class UpdateNoQuorumPolicy < PostDeploymentAction def process(deployment_info, context) # NOTE(bogdando) use 'suicide' if fencing is enabled in corosync diff --git a/lib/astute/post_deploy_actions/upload_cirros_image.rb b/lib/astute/post_deployment_actions/upload_cirros_image.rb similarity index 98% rename from lib/astute/post_deploy_actions/upload_cirros_image.rb rename to lib/astute/post_deployment_actions/upload_cirros_image.rb index bb1e400b..20022dfa 100644 --- a/lib/astute/post_deploy_actions/upload_cirros_image.rb +++ b/lib/astute/post_deployment_actions/upload_cirros_image.rb @@ -16,7 +16,7 @@ module Astute class CirrosError < AstuteError; end - class UploadCirrosImage < PostDeployAction + class UploadCirrosImage < PostDeploymentAction def process(deployment_info, context) #FIXME: update context status to multirole support: possible situation where one of the diff --git a/lib/astute/pre_node_actions/pre_patching.rb b/lib/astute/pre_node_actions/pre_patching.rb new file mode 100644 index 00000000..edcc8ab9 --- /dev/null +++ b/lib/astute/pre_node_actions/pre_patching.rb @@ -0,0 +1,77 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +module Astute + class PrePatching < PreNodeAction + + def process(deployment_info, context) + return unless deployment_info.first['openstack_version_prev'] + + # We should stop services with SIGTERM or even SIGKILL. + # StopOSTServices do this and should be run before. + + remove_cmd = getremovepackage_cmd(deployment_info) + + nodes = deployment_info.map { |n| n['uid'] } + + Astute.logger.info "Starting removal of error-prone packages" + Astute.logger.info "Executing command #{remove_cmd}" + Astute.logger.info "On nodes #{nodes.inspect}" + + response = run_shell_command(context, nodes, remove_cmd, 600) + + if response[:data][:exit_code] != 0 + Astute.logger.error "#{context.task_id}: Fail to remove packages, "\ + "check the debugging output for details" + end + + Astute.logger.info "#{context.task_id}: Finished pre-patching hook" + end #process + + def getremovepackage_cmd(deployment_info) + os = deployment_info.first['cobbler']['profile'] + case os + when /centos/i then "yum -y remove #{centos_packages}" + when /ubuntu/i then "aptitude -y remove #{ubuntu_packages}" + else + raise DeploymentEngineError, "Unknown system #{os}" + end + end + + def centos_packages + packages = <<-Packages + python-oslo-messaging python-oslo-config openstack-heat-common + python-nova python-routes python-routes1.12 python-neutron + python-django-horizon murano-api sahara sahara-dashboard + python-ceilometer openstack-swift openstack-utils + python-glance python-glanceclient python-cinder + python-sqlalchemy python-testtools + Packages + packages.tr!("\n"," ") + end + + def ubuntu_packages + packages = <<-Packages + python-oslo.messaging python-oslo.config python-heat python-nova + python-routes python-routes1.13 python-neutron python-django-horizon + murano-common murano-api sahara sahara-dashboard python-ceilometer + python-swift python-cinder python-keystoneclient python-neutronclient + python-novaclient python-swiftclient python-troveclient + python-sqlalchemy python-testtools + Packages + packages.tr!("\n"," ") + end + + end #class +end diff --git a/lib/astute/pre_node_actions/pre_patching_ha.rb b/lib/astute/pre_node_actions/pre_patching_ha.rb new file mode 100644 index 00000000..7e3847c5 --- /dev/null +++ b/lib/astute/pre_node_actions/pre_patching_ha.rb @@ -0,0 +1,50 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +module Astute + class PrePatchingHa < PreNodeAction + + def process(deployment_info, context) + return if deployment_info.first['openstack_version_prev'].nil? || + deployment_info.first['deployment_mode'] !~ /ha/i + + # Run only once for node. If one of role is controller or primary-controller + # generate new deployment_info block. + # Important for 'mongo' role which run early then 'controller' + current_uids = deployment_info.map{ |n| n['uid'] } + controllers = deployment_info.first['nodes'].select{ |n| current_uids.include?(n['uid']) && n['role'] =~ /controller/i } + c_deployment_info = deployment_info.select { |d_i| controllers.map{ |n| n['uid'] }.include? d_i['uid'] } + + return if c_deployment_info.empty? + c_deployment_info.each do |c_d_i| + c_d_i['role'] = controllers.find{ |c| c['uid'] == c_d_i['uid'] }['role'] + end + controller_nodes = c_deployment_info.map{ |n| n['uid'] } + + Astute.logger.info "Starting migration of pacemaker services from " \ + "nodes #{controller_nodes.inspect}" + + Astute::Pacemaker.commands(action='stop', c_deployment_info).each do |pcmk_ban_cmd| + response = run_shell_command(context, controller_nodes, pcmk_ban_cmd) + + if response[:data][:exit_code] != 0 + Astute.logger.warn "#{context.task_id}: Failed to ban service, "\ + "check the debugging output for details" + end + end + + Astute.logger.info "#{context.task_id}: Finished pre-patching-ha hook" + end #process + end #class +end diff --git a/lib/astute/pre_node_actions/stop_ost_services.rb b/lib/astute/pre_node_actions/stop_ost_services.rb new file mode 100644 index 00000000..f60e0eb9 --- /dev/null +++ b/lib/astute/pre_node_actions/stop_ost_services.rb @@ -0,0 +1,65 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +module Astute + class StopOSTServices < PreNodeAction + + def process(deployment_info, context) + old_env = deployment_info.first['openstack_version_prev'] + return unless old_env + + Astute.logger.info "Stop all Openstack services hook start" + + node_uids = deployment_info.collect { |n| n['uid'] } + file_content = get_file + target_file = '/tmp/stop_services.rb' + + upload_script(context, node_uids, target_file, file_content) + + Astute.logger.info "Running file: #{target_file} on node uids: #{node_uids.join ', '}" + + response = run_shell_command(context, node_uids, "/usr/bin/ruby #{target_file} |tee /tmp/stop_services.log") + + if response[:data][:exit_code] != 0 + Astute.logger.warn "#{context.task_id}: Script returned error code #{response[:data][:exit_code]}" + end + + Astute.logger.info "#{context.task_id}: Finished stop services pre-patching hook" + end #process + + private + + def get_file + File.read File.join(File.dirname(__FILE__), 'stop_services.script') + end + + def upload_script(context, node_uids, target_file, file_content) + target_file = '/tmp/stop_services.rb' + Astute.logger.info "Uploading file: #{target_file} to nodes uids: #{node_uids.join ', '}" + + MClient.new(context, "uploadfile", node_uids).upload( + :path => target_file, + :content => file_content, + :user_owner => 'root', + :group_owner => 'root', + :permissions => '0700', + :overwrite => true, + :parents => true + ) + rescue MClientTimeout, MClientError => e + Astute.logger.error("#{context.task_id}: mcollective error: #{e.message}") + end + + end #class +end #module diff --git a/lib/astute/pre_node_actions/stop_services.script b/lib/astute/pre_node_actions/stop_services.script new file mode 100644 index 00000000..0481bade --- /dev/null +++ b/lib/astute/pre_node_actions/stop_services.script @@ -0,0 +1,213 @@ +#!/usr/bin/env ruby +require 'rubygems' +require 'facter' + +# pre-deploy hook library +module PreDeploy + @dry_run = false + @process_tree = nil + @osfamily = nil + @stop_services_regexp = %r{nova|cinder|glance|keystone|neutron|sahara|murano|ceilometer|heat|swift|apache2|httpd} + + # get regexp that selects services and processes to stop + # @return [Regexp] + def self.stop_services_regexp + @stop_services_regexp + end + + # set regexp that selects services and processes to stop + # @param value [Regexp] + def self.stop_services_regexp=(value) + @stop_services_regexp = value + end + + # get osfamily from facter + # @return [String] + def self.osfamily + return @osfamily if @osfamily + @osfamily = Facter.value 'osfamily' + end + + # get dry run without doing anything switch + # @return [TrueClass,FalseClass] + def self.dry_run + @dry_run + end + + # set dry run without doing anything switch + # @param value [TrueClass,FalseClass] + def self.dry_run=(value) + @dry_run = value + end + + # get ps from shell command + # @return [String] + def self.ps + `ps haxo pid,ppid,cmd` + end + + # get service statu from shell command + # @return String + def self.services + `service --status-all 2>&1` + end + + # same as process_tree but reset mnemoization + # @return [Hash Hash String,Integer>>] + def self.process_tree_with_renew + @process_tree = nil + self.process_tree + end + + # build process tree from process list + # @return [Hash Hash String,Integer>>] + def self.process_tree + return @process_tree if @process_tree + @process_tree = {} + self.ps.split("\n").each do |p| + f = p.split + pid = f.shift.to_i + ppid = f.shift.to_i + cmd = f.join ' ' + + # create entry for this pid if not present + @process_tree[pid] = { + :children => [] + } unless @process_tree.key? pid + + # fill this entry + @process_tree[pid][:ppid] = ppid + @process_tree[pid][:pid] = pid + @process_tree[pid][:cmd] = cmd + + # create entry for parent process if not present + @process_tree[ppid] = { + :children => [] + } unless @process_tree.key? ppid + + # fill parent's children + @process_tree[ppid][:children] << pid + end + @process_tree + end + + # kill selected pid or array of them + # @param pids [Integer,String] Pids to kill + # @param signal [Integer,String] Which signal? + # @param recursive [TrueClass,FalseClass] Kill children too? + # @return [TrueClass,FalseClass] Was the signal sent? Process may still be present even on success. + def self.kill_pids(pids, signal = 9, recursive = true) + pids = Array pids + + pids_to_kill = pids.inject([]) do |all_pids, pid| + pid = pid.to_i + if recursive + all_pids + self.get_children_pids(pid) + else + all_pids << pid + end + end + + pids_to_kill.uniq! + pids_to_kill.sort! + + return false unless pids_to_kill.any? + puts "Kill these pids: #{pids_to_kill.join ', '} with signal #{signal}" + self.run "kill -#{signal} #{pids_to_kill.join ' '}" + end + + # recursion to find all children pids + # @return [Array] + def self.get_children_pids(pid) + pid = pid.to_i + unless self.process_tree.key? pid + puts "No such pid: #{pid}" + return [] + end + self.process_tree[pid][:children].inject([pid]) do |all_children_pids, child_pid| + all_children_pids + self.get_children_pids(child_pid) + end + end + + # same as services_to_stop but reset mnemoization + # @return Array[String] + def self.services_to_stop_with_renew + @services_to_stop = nil + self.services_to_stop + end + + # find running services that should be stopped + # uses service status and regex to filter + # @return [Array] + def self.services_to_stop + return @services_to_stop if @services_to_stop + @services_to_stop = self.services.split("\n").inject([]) do |services_to_stop, service| + fields = service.chomp.split + running = if fields[4] == 'running...' + fields[0] + elsif fields[1] == '+' + fields[3] + else + nil + end + + if running =~ @stop_services_regexp + # replace wrong service name + running = 'httpd' if running == 'httpd.event' and self.osfamily == 'RedHat' + running = 'openstack-keystone' if running == 'keystone' and self.osfamily == 'RedHat' + services_to_stop << running + else + services_to_stop + end + end + end + + # stop services that match stop_services_regex + def self.stop_services + self.services_to_stop.each do |service| + puts "Try to stop service: #{service}" + self.run "service #{service} stop" + end + end + + # filter pids which cmd match regexp + # @param regexp Search pids by this regexp + # @return [Hash Hash String,Integer>>] + def self.pids_by_regexp(regexp) + matched = {} + self.process_tree.each do |pid,process| + matched[pid] = process if process[:cmd] =~ regexp + end + matched + end + + # kill pids that match stop_services_regexp + # @return + def self.kill_pids_by_stop_regexp + pids = self.pids_by_regexp(@stop_services_regexp).keys + self.kill_pids pids + end + + # here be other fixes + # TODO: not needed anymore? + def self.misc_fixes + if self.osfamily == 'Debian' + puts 'Enabling WSGI module' + self.run 'a2enmod wsgi' + end + end + + # run the shell command with dry_run support + # @param cmd [String] Command to run + def self.run(cmd) + command = "#{self.dry_run ? 'echo' : ''} #{cmd} 2>&1" + system command + end +end # class + +if __FILE__ == $0 + # PreDeploy.dry_run = true + PreDeploy.misc_fixes + PreDeploy.stop_services + PreDeploy.kill_pids_by_stop_regexp +end diff --git a/spec/unit/common_actions/pacemaker_spec.rb b/spec/unit/common_actions/pacemaker_spec.rb new file mode 100644 index 00000000..3112588f --- /dev/null +++ b/spec/unit/common_actions/pacemaker_spec.rb @@ -0,0 +1,108 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require File.join(File.dirname(__FILE__), '../../spec_helper') + +describe Astute::Pacemaker do + include SpecHelpers + + let(:ctx) do + ctx = mock('context') + ctx.stubs(:task_id) + ctx.stubs(:reporter) + ctx.stubs(:status).returns('1' => 'success', '2' => 'success') + ctx + end + + let(:pacemaker) { Astute::Pacemaker } + + let(:deploy_data) { [ + {'uid' => '1', + 'role' => 'controller', + 'openstack_version_prev' => 'old_version', + 'deployment_mode' => 'ha_compact', + 'cobbler' => { + 'profile' => 'centos-x86_64' + }, + 'nodes' => [ + {'uid' => '1', 'role' => 'controller'}, + {'uid' => '2', 'role' => 'compute'} + ] + }, + {'uid' => '2', + 'role' => 'compute' + } + ] + } + + it 'should return empty array if deployment mode not HA' do + deploy_data.first['deployment_mode'] = 'simple' + expect(pacemaker.commands(behavior='stop', deploy_data)).to eql([]) + end + + it 'should return empty array if no controllers' do + deploy_data.first['role'] = 'cinder' + expect(pacemaker.commands(behavior='stop', deploy_data)).to eql([]) + end + + context 'controller < 3' do + it 'should return stop service commands for pacemaker' do + expect(pacemaker.commands(behavior='stop', deploy_data)).to eql( + ['crm resource stop openstack-heat-engine && sleep 3', + 'crm resource stop p_openstack-heat-engine && sleep 3']) + end + + it 'should return start service commands for HA pacemaker' do + expect(pacemaker.commands(behavior='start', deploy_data)).to eql( + ['crm resource start openstack-heat-engine && sleep 3', + 'crm resource start p_openstack-heat-engine && sleep 3']) + end + end + + context 'controller >= 3' do + + let(:ha_deploy_data) { + deploy_data.first['nodes'] = [ + {'uid' => '1', 'role' => 'controller'}, + {'uid' => '2', 'role' => 'compute'}, + {'uid' => '3', 'role' => 'primary-controller'}, + {'uid' => '4', 'role' => 'controller'}, + ] + deploy_data + } + + it 'should return stop service commands for pacemaker' do + expect(pacemaker.commands(behavior='stop', ha_deploy_data)).to eql( + ['pcs resource ban openstack-heat-engine `crm_node -n` && sleep 3', + 'pcs resource ban p_openstack-heat-engine `crm_node -n` && sleep 3']) + end + + it 'should return start service commands for pacemaker' do + expect(pacemaker.commands(behavior='start', ha_deploy_data)).to eql( + ['pcs resource clear openstack-heat-engine `crm_node -n` && sleep 3', + 'pcs resource clear p_openstack-heat-engine `crm_node -n` && sleep 3']) + end + end + + it 'should return quantum service commands if quantum enable' do + deploy_data.first['quantum'] = [] + expect(pacemaker.commands(behavior='stop', deploy_data).size).to eql(6) + end + + it 'should return ceilometer service commands if ceilometer enable' do + deploy_data.first['ceilometer'] = { 'enabled' => true } + expect(pacemaker.commands(behavior='stop', deploy_data).size).to eql(4) + end + +end \ No newline at end of file diff --git a/spec/unit/post_deploy_actions_spec.rb b/spec/unit/deploy_actions_spec.rb similarity index 55% rename from spec/unit/post_deploy_actions_spec.rb rename to spec/unit/deploy_actions_spec.rb index 548cabc0..5e707a2f 100644 --- a/spec/unit/post_deploy_actions_spec.rb +++ b/spec/unit/deploy_actions_spec.rb @@ -1,4 +1,4 @@ -# Copyright 2013 Mirantis, Inc. +# Copyright 2014 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,12 +14,12 @@ require File.join(File.dirname(__FILE__), '../spec_helper') -describe Astute::PostDeployActions do +describe Astute::PostDeploymentActions do include SpecHelpers let(:deploy_data) {[]} let(:ctx) { mock } - let(:post_deploy_actions) { Astute::PostDeployActions.new(deploy_data, ctx) } + let(:post_deployment_actions) { Astute::PostDeploymentActions.new(deploy_data, ctx) } it 'should run post hooks' do Astute::UpdateNoQuorumPolicy.any_instance.expects(:process) @@ -31,10 +31,53 @@ describe Astute::PostDeployActions do Astute::UpdateClusterHostsInfo.any_instance.expects(:process) .with(deploy_data, ctx) - post_deploy_actions.process + post_deployment_actions.process end end +describe Astute::PreNodeActions do + include SpecHelpers + + let(:deploy_data) {[{'uid' => '1'}, {'uid' => '2'}]} + let(:ctx) { mock } + let(:pre_node_actions) { Astute::PreNodeActions.new(ctx) } + + it 'should pre node hooks' do + Astute::PrePatchingHa.any_instance.expects(:process) + .with(deploy_data, ctx) + Astute::StopOSTServices.any_instance.expects(:process) + .with(deploy_data, ctx) + Astute::PrePatching.any_instance.expects(:process) + .with(deploy_data, ctx) + + pre_node_actions.process(deploy_data) + end +end + +describe Astute::PreNodeActions do + include SpecHelpers + + let(:deploy_data1) {[{'uid' => '1'}, {'uid' => '2'}]} + let(:deploy_data2) {[{'uid' => '1'}]} + let(:ctx) { mock } + let(:pre_node_actions) { Astute::PreNodeActions.new(ctx) } + + it 'should process nodes sending first' do + Astute::PrePatching.any_instance.expects(:process) + .with(deploy_data1, ctx) + pre_node_actions.process(deploy_data1) + end + + it 'should not process repeated nodes' do + Astute::PrePatching.any_instance.expects(:process) + .with(deploy_data1, ctx) + pre_node_actions.process(deploy_data1) + Astute::PrePatching.any_instance.expects(:process).never + pre_node_actions.process(deploy_data2) + end + +end + describe Astute::PostDeployAction do include SpecHelpers diff --git a/spec/unit/deployment_engine_spec.rb b/spec/unit/deployment_engine_spec.rb index 486a2d70..f569012c 100644 --- a/spec/unit/deployment_engine_spec.rb +++ b/spec/unit/deployment_engine_spec.rb @@ -42,6 +42,8 @@ describe Astute::DeploymentEngine do before(:each) do Astute::PreDeployActions.any_instance.stubs(:process).returns(nil) + Astute::PostDeployActions.any_instance.stubs(:process).returns(nil) + Astute::PreNodeActions.any_instance.stubs(:process).returns(nil) deployer.stubs(:generate_ssh_keys) deployer.stubs(:upload_ssh_keys) deployer.stubs(:sync_puppet_manifests) diff --git a/spec/unit/nailyfact_deploy_spec.rb b/spec/unit/nailyfact_deploy_spec.rb index 3ef43d3d..ae8529d0 100644 --- a/spec/unit/nailyfact_deploy_spec.rb +++ b/spec/unit/nailyfact_deploy_spec.rb @@ -55,6 +55,8 @@ describe "NailyFact DeploymentEngine" do deploy_engine.stubs(:enable_puppet_deploy).with(uniq_nodes_uid) deploy_engine.stubs(:sync_time) Astute::PreDeployActions.any_instance.stubs(:process).returns(nil) + Astute::PreNodeActions.any_instance.stubs(:process).returns(nil) + Astute::PreDeployActions.any_instance.stubs(:process).returns(nil) end context 'log parsing' do diff --git a/spec/unit/orchestrator_spec.rb b/spec/unit/orchestrator_spec.rb index ff03cae9..628b0dbe 100644 --- a/spec/unit/orchestrator_spec.rb +++ b/spec/unit/orchestrator_spec.rb @@ -125,7 +125,7 @@ describe Astute::Orchestrator do nodes = [{'uid' => 1, 'role' => 'controller'}] Astute::DeploymentEngine::NailyFact.any_instance.expects(:deploy). with(nodes) - Astute::PostDeployActions.any_instance.expects(:process).returns(nil) + Astute::PostDeploymentActions.any_instance.expects(:process).returns(nil) @orchestrator.deploy(@reporter, 'task_uuid', nodes) end diff --git a/spec/unit/post_deploy_actions/post_patching_hook_spec.rb b/spec/unit/post_deploy_actions/post_patching_hook_spec.rb new file mode 100644 index 00000000..66d67a24 --- /dev/null +++ b/spec/unit/post_deploy_actions/post_patching_hook_spec.rb @@ -0,0 +1,106 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require File.join(File.dirname(__FILE__), '../../spec_helper') + +describe Astute::PostPatchingHa do + include SpecHelpers + + let(:ctx) do + ctx = mock('context') + ctx.stubs(:task_id) + ctx.stubs(:reporter) + ctx.stubs(:status).returns('1' => 'success', '2' => 'success') + ctx + end + + let(:deploy_data) { [ + {'uid' => '1', + 'role' => 'controller', + 'openstack_version_prev' => 'old_version', + 'deployment_mode' => 'ha_compact', + 'cobbler' => { + 'profile' => 'centos-x86_64' + } + }, + {'uid' => '2', + 'role' => 'compute' + } + ] + } + + let(:post_patching_ha) { Astute::PostPatchingHa.new } + + it 'should run if upgrade/downgrade env' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + post_patching_ha.expects(:run_shell_command).returns(:data => {:exit_code => 0}) + post_patching_ha.process(deploy_data, ctx) + end + + it 'should not run if deploy new env' do + deploy_data.first.delete('openstack_version_prev') + + Astute::Pacemaker.expects(:commands).never + post_patching_ha.expects(:run_shell_command).never + + post_patching_ha.process(deploy_data, ctx) + end + + it 'should run if upgrade/downgrade not HA env' do + deploy_data.first['deployment_mode'] = 'simple' + + Astute::Pacemaker.expects(:commands).never + post_patching_ha.expects(:run_shell_command).never + + post_patching_ha.process(deploy_data, ctx) + end + + it 'should not change deployment status if command fail' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + post_patching_ha.expects(:run_shell_command).once.returns(:data => {:exit_code => 1}) + ctx.expects(:report_and_update_status).never + + post_patching_ha.process(deploy_data, ctx) + end + + it 'should not change deployment status if shell exec using mcollective fail' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + post_patching_ha.expects(:run_shell_command).once.returns(:data => {}) + + post_patching_ha.process(deploy_data, ctx) + ctx.expects(:report_and_update_status).never + end + + it 'should run command for every pacemaker services' do + Astute::Pacemaker.expects(:commands).returns(['command1', 'command2']) + post_patching_ha.expects(:run_shell_command).twice.returns(:data => {:exit_code => 1}) + + post_patching_ha.process(deploy_data, ctx) + end + + it 'should get commands for service ban' do + Astute::Pacemaker.expects(:commands).with('start', deploy_data).returns(['basic command']) + post_patching_ha.expects(:run_shell_command).returns(:data => {:exit_code => 0}) + post_patching_ha.process(deploy_data, ctx) + end + + it 'should not run if no controllers in cluster' do + deploy_data.first['role'] = 'cinder' + + Astute::Pacemaker.expects(:commands).never + post_patching_ha.expects(:run_shell_command).never + post_patching_ha.process(deploy_data, ctx) + end + +end \ No newline at end of file diff --git a/spec/unit/restart_radosgw_hook_spec.rb b/spec/unit/post_deployment_actions/restart_radosgw_hook_spec.rb similarity index 98% rename from spec/unit/restart_radosgw_hook_spec.rb rename to spec/unit/post_deployment_actions/restart_radosgw_hook_spec.rb index 5910b442..84de0568 100644 --- a/spec/unit/restart_radosgw_hook_spec.rb +++ b/spec/unit/post_deployment_actions/restart_radosgw_hook_spec.rb @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -require File.join(File.dirname(__FILE__), '../spec_helper') +require File.join(File.dirname(__FILE__), '../../spec_helper') describe Astute::RestartRadosgw do include SpecHelpers diff --git a/spec/unit/update_cluster_hosts_info_hook_spec.rb b/spec/unit/post_deployment_actions/update_cluster_hosts_info_hook_spec.rb similarity index 98% rename from spec/unit/update_cluster_hosts_info_hook_spec.rb rename to spec/unit/post_deployment_actions/update_cluster_hosts_info_hook_spec.rb index 03453ace..9b8b92d0 100644 --- a/spec/unit/update_cluster_hosts_info_hook_spec.rb +++ b/spec/unit/post_deployment_actions/update_cluster_hosts_info_hook_spec.rb @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -require File.join(File.dirname(__FILE__), '../spec_helper') +require File.join(File.dirname(__FILE__), '../../spec_helper') describe Astute::UpdateClusterHostsInfo do include SpecHelpers diff --git a/spec/unit/update_no_quorum_policy_spec.rb b/spec/unit/post_deployment_actions/update_no_quorum_policy_spec.rb similarity index 98% rename from spec/unit/update_no_quorum_policy_spec.rb rename to spec/unit/post_deployment_actions/update_no_quorum_policy_spec.rb index baf8e229..611e832e 100644 --- a/spec/unit/update_no_quorum_policy_spec.rb +++ b/spec/unit/post_deployment_actions/update_no_quorum_policy_spec.rb @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -require File.join(File.dirname(__FILE__), '../spec_helper') +require File.join(File.dirname(__FILE__), '../../spec_helper') describe Astute::UpdateNoQuorumPolicy do include SpecHelpers diff --git a/spec/unit/upload_cirros_image_hook_spec.rb b/spec/unit/post_deployment_actions/upload_cirros_image_hook_spec.rb similarity index 98% rename from spec/unit/upload_cirros_image_hook_spec.rb rename to spec/unit/post_deployment_actions/upload_cirros_image_hook_spec.rb index 5e0eaa72..4b8f717d 100644 --- a/spec/unit/upload_cirros_image_hook_spec.rb +++ b/spec/unit/post_deployment_actions/upload_cirros_image_hook_spec.rb @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -require File.join(File.dirname(__FILE__), '../spec_helper') +require File.join(File.dirname(__FILE__), '../../spec_helper') describe Astute::UploadCirrosImage do include SpecHelpers diff --git a/spec/unit/pre_node_actions/pre_patching_hook_spec.rb b/spec/unit/pre_node_actions/pre_patching_hook_spec.rb new file mode 100644 index 00000000..3c1dfc0e --- /dev/null +++ b/spec/unit/pre_node_actions/pre_patching_hook_spec.rb @@ -0,0 +1,106 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require File.join(File.dirname(__FILE__), '../../spec_helper') + +describe Astute::PrePatching do + include SpecHelpers + + let(:ctx) do + ctx = mock('context') + ctx.stubs(:task_id) + ctx.stubs(:reporter) + ctx.stubs(:status).returns('1' => 'success', '2' => 'success') + ctx + end + + let(:deploy_data) { [ + {'uid' => '1', + 'role' => 'controller', + 'openstack_version_prev' => 'old_version', + 'cobbler' => { + 'profile' => 'centos-x86_64' + } + }, + {'uid' => '2', + 'role' => 'compute' + } + ] + } + + let(:pre_patching) { Astute::PrePatching.new } + + it 'should run if upgrade/downgrade env' do + pre_patching.expects(:run_shell_command).once.returns(:data => {:exit_code => 0}) + pre_patching.process(deploy_data, ctx) + end + + it 'should not run if deploy new env' do + deploy_data.first.delete('openstack_version_prev') + pre_patching.process(deploy_data, ctx) + pre_patching.expects(:run_shell_command).never + + pre_patching.process(deploy_data, ctx) + end + + it 'should not change deployment status if command fail' do + pre_patching.expects(:run_shell_command).once.returns(:data => {:exit_code => 1}) + ctx.expects(:report_and_update_status).never + + pre_patching.process(deploy_data, ctx) + end + + it 'should not change deployment status if shell exec using mcollective fail' do + pre_patching.expects(:run_shell_command).once.returns(:data => {}) + + pre_patching.process(deploy_data, ctx) + ctx.expects(:report_and_update_status).never + end + + describe '#getremovepackage_cmd' do + + it 'should use yum command for CenoOS system' do + pre_patching.expects(:run_shell_command).once.with( + ctx, + ['1', '2'], + regexp_matches(/yum/), + is_a(Integer)) + .returns(:data => {:exit_code => 0}) + + pre_patching.process(deploy_data, ctx) + end + + it 'should use aptitude command for Ubuntu system' do + new_deploy_data = deploy_data.clone + new_deploy_data.first['cobbler']['profile'] = 'ubuntu_1204_x86_64' + pre_patching.expects(:run_shell_command).once.with( + ctx, + ['1', '2'], + regexp_matches(/aptitude/), + is_a(Integer)) + .returns(:data => {:exit_code => 0}) + + pre_patching.process(new_deploy_data, ctx) + end + + it 'raise error if target system unknown' do + new_deploy_data = deploy_data.clone + new_deploy_data.first['cobbler']['profile'] = 'unknown' + pre_patching.expects(:run_shell_command).never + expect { pre_patching.process(new_deploy_data, ctx) }.to raise_error(Astute::DeploymentEngineError, /Unknown system/) + end + + end # getremovepackage_cmd + +end \ No newline at end of file diff --git a/spec/unit/pre_node_actions/pre_pathing_ha_hook_spec.rb b/spec/unit/pre_node_actions/pre_pathing_ha_hook_spec.rb new file mode 100644 index 00000000..df8e6cd1 --- /dev/null +++ b/spec/unit/pre_node_actions/pre_pathing_ha_hook_spec.rb @@ -0,0 +1,202 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require File.join(File.dirname(__FILE__), '../../spec_helper') + +describe Astute::PrePatchingHa do + include SpecHelpers + + let(:ctx) do + ctx = mock('context') + ctx.stubs(:task_id) + ctx.stubs(:reporter) + ctx.stubs(:status).returns('1' => 'success', '2' => 'success') + ctx + end + + let(:deploy_data) { [ + {'uid' => '1', + 'role' => 'cinder', + 'openstack_version_prev' => 'old_version', + 'deployment_mode' => 'ha_compact', + 'cobbler' => { + 'profile' => 'centos-x86_64' + }, + 'nodes' => [ + { + 'uid' => '1', + 'slave_name' => 'node-1', + 'role' => 'cinder' + }, + { + 'uid' => '1', + 'slave_name' => 'node-1', + 'role' => 'controller' + }, + { + 'uid' => '2', + 'slave_name' => 'node-2', + 'role' => 'ceph-osd' + }, + { + 'uid' => '3', + 'slave_name' => 'node-3', + 'role' => 'primary-controller' + } + ], + }, + {'uid' => '2', + 'role' => 'compute' + }, + {'uid' => '3', + 'role' => 'primary-controller' + } + ] + } + + let(:pre_patching_ha) { Astute::PrePatchingHa.new } + + it 'should run if upgrade/downgrade env' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + pre_patching_ha.expects(:run_shell_command).returns(:data => {:exit_code => 0}) + pre_patching_ha.process(deploy_data, ctx) + end + + it 'should not run if deploy new env' do + deploy_data.first.delete('openstack_version_prev') + + Astute::Pacemaker.expects(:commands).never + pre_patching_ha.expects(:run_shell_command).never + + pre_patching_ha.process(deploy_data, ctx) + end + + it 'should run if upgrade/downgrade not HA env' do + deploy_data.first['deployment_mode'] = 'simple' + + Astute::Pacemaker.expects(:commands).never + pre_patching_ha.expects(:run_shell_command).never + + pre_patching_ha.process(deploy_data, ctx) + end + + it 'should not change deployment status if command fail' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + pre_patching_ha.expects(:run_shell_command).once.returns(:data => {:exit_code => 1}) + ctx.expects(:report_and_update_status).never + + pre_patching_ha.process(deploy_data, ctx) + end + + it 'should not change deployment status if shell exec using mcollective fail' do + Astute::Pacemaker.expects(:commands).returns(['basic command']) + pre_patching_ha.expects(:run_shell_command).once.returns(:data => {}) + + pre_patching_ha.process(deploy_data, ctx) + ctx.expects(:report_and_update_status).never + end + + it 'should run command for every pacemaker services' do + Astute::Pacemaker.expects(:commands).returns(['command1', 'command2']) + pre_patching_ha.expects(:run_shell_command).twice.returns(:data => {:exit_code => 1}) + + pre_patching_ha.process(deploy_data, ctx) + end + + context 'Pacemaker stuff' do + + let(:transormed_data) { + [{'uid' => '1', + 'role' => 'controller', + 'openstack_version_prev' => 'old_version', + 'deployment_mode' => 'ha_compact', + 'cobbler' => { + 'profile' => 'centos-x86_64' + }, + 'nodes' => [ + { + 'uid' => '1', + 'slave_name' => 'node-1', + 'role' => 'cinder' + }, + { + 'uid' => '1', + 'slave_name' => 'node-1', + 'role' => 'controller' + }, + { + 'uid' => '2', + 'slave_name' => 'node-2', + 'role' => 'ceph-osd' + }, + { + 'uid' => '3', + 'slave_name' => 'node-3', + 'role' => 'primary-controller' + } + ], + }, + {'uid' => '3', + 'role' => 'primary-controller' + }] + } + + let(:no_controllers) { + [{'uid' => '1', + 'role' => 'compute', + 'openstack_version_prev' => 'old_version', + 'deployment_mode' => 'ha_compact', + 'cobbler' => { + 'profile' => 'centos-x86_64' + }, + 'nodes' => [ + { + 'uid' => '1', + 'slave_name' => 'node-1', + 'role' => 'cinder' + }, + { + 'uid' => '2', + 'slave_name' => 'node-2', + 'role' => 'controller' + }, + { + 'uid' => '3', + 'slave_name' => 'node-3', + 'role' => 'mongo' + } + ], + }, + {'uid' => '3', + 'role' => 'cinder' + }] + } + + it 'should get commands for service ban' do + Astute::Pacemaker.expects(:commands).with('stop', transormed_data).returns(['basic command']) + pre_patching_ha.expects(:run_shell_command).returns(:data => {:exit_code => 0}) + pre_patching_ha.process(deploy_data, ctx) + end + + it 'should not run if no controllers in cluster' do + deploy_data.first['role'] = 'cinder' + + Astute::Pacemaker.expects(:commands).never + pre_patching_ha.expects(:run_shell_command).never + pre_patching_ha.process(no_controllers, ctx) + end + + end + +end \ No newline at end of file diff --git a/spec/unit/pre_node_actions/stop_ost_services_hook_spec.rb b/spec/unit/pre_node_actions/stop_ost_services_hook_spec.rb new file mode 100644 index 00000000..3573f525 --- /dev/null +++ b/spec/unit/pre_node_actions/stop_ost_services_hook_spec.rb @@ -0,0 +1,102 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +require File.join(File.dirname(__FILE__), '../../spec_helper') + +describe Astute::StopOSTServices do + include SpecHelpers + + let(:ctx) do + ctx = mock('context') + ctx.stubs(:task_id) + ctx.stubs(:reporter) + ctx.stubs(:status).returns('1' => 'success', '2' => 'success') + ctx + end + + let(:deploy_data) { [ + {'uid' => '1', + 'role' => 'controller', + 'openstack_version_prev' => 'old_version', + 'nodes' => [ + { + 'uid' => 1, + 'slave_name' => 'node-1', + 'role' => 'controller' + }, + { + 'uid' => 2, + 'slave_name' => 'node-2', + 'role' => 'ceph-osd' + } + ], + }, + { + 'uid' => '2', + 'role' => 'compute' + } + ] + } + + let(:stop_ost_services) { Astute::StopOSTServices.new } + + it 'should run if upgrade/downgrade env' do + stop_ost_services.expects(:upload_script).once + stop_ost_services.expects(:run_shell_command).once.returns(:data => {:exit_code => 0}) + stop_ost_services.process(deploy_data, ctx) + end + + it 'should not run if deploy new env' do + deploy_data.first.delete('openstack_version_prev') + stop_ost_services.process(deploy_data, ctx) + stop_ost_services.expects(:upload_script).never + stop_ost_services.expects(:run_shell_command).never + + stop_ost_services.process(deploy_data, ctx) + end + + it 'should not change deployment status if command fail' do + stop_ost_services.stubs(:upload_script).once + stop_ost_services.expects(:run_shell_command).once.returns(:data => {:exit_code => 1}) + ctx.expects(:report_and_update_status).never + + stop_ost_services.process(deploy_data, ctx) + end + + it 'should not change deployment status if shell exec using mcollective fail' do + stop_ost_services.stubs(:upload_script).once + stop_ost_services.expects(:run_shell_command).once.returns(:data => {}) + + stop_ost_services.process(deploy_data, ctx) + ctx.expects(:report_and_update_status).never + end + + it 'should raise exception if shell exec using mcollective fail' do + stop_ost_services.expects(:upload_script).once.returns('test_script.rb') + stop_ost_services.stubs(:run_shell_command).once.returns(:data => {:exit_code => 42}) + + stop_ost_services.process(deploy_data, ctx) + ctx.expects(:report_and_update_status).never + end + + it 'should upload target script and run it' do + script_content = 'script content' + target_file = '/tmp/stop_services.rb' + stop_ost_services.stubs(:get_file).once.returns script_content + stop_ost_services.expects(:upload_script).with(ctx, deploy_data.map{ |n| n['uid'] }, target_file, script_content).once + stop_ost_services.expects(:run_shell_command).once.returns(:data => {:exit_code => 0}) + stop_ost_services.process(deploy_data, ctx) + end + +end diff --git a/spec/unit/pre_node_actions/stop_services_script_spec.rb b/spec/unit/pre_node_actions/stop_services_script_spec.rb new file mode 100644 index 00000000..c3161a17 --- /dev/null +++ b/spec/unit/pre_node_actions/stop_services_script_spec.rb @@ -0,0 +1,311 @@ +# Copyright 2013 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +#require File.join(File.dirname(__FILE__), '../../spec_helper') + +RSpec.configure do |c| + c.mock_with :mocha +end + +load File.join(File.dirname(__FILE__), '../../../lib/astute/pre_node_actions/stop_services.script') + +describe PreDeploy do +# include SpecHelpers + + let(:redhat_ps) do + <<-eos +100 1 /usr/bin/python nova-api.py +101 100 /usr/bin/python nova-api.py +102 100 /usr/bin/python nova-api.py +103 100 /usr/bin/python nova-api.py +104 1 /usr/bin/python cinder-volume.py +105 104 /usr/sbin/tgtd +106 1 /usr/bin/python neutron.py +107 106 /usr/sbin/dnsmasq +108 1 /usr/sbin/httpd +109 1 /usr/bin/python keystone.py + eos + end + + let(:debian_pstree) do + { + 104 => { + :children => [105], + :ppid => 1, + :cmd => "/usr/bin/python cinder-volume.py", + :pid => 104 + }, + 105 => { + :children => [], + :ppid => 104, + :cmd => "/usr/sbin/tgtd", + :pid => 105 + }, + 100 => { + :children => [101, 102, 103], + :ppid => 1, + :cmd => "/usr/bin/python nova-api.py", + :pid => 100 + }, + 1 => { + :children => [100, 104, 106, 108, 109] + }, + 106 => { + :children => [107], + :ppid => 1, + :cmd => "/usr/bin/python neutron.py", + :pid => 106 + }, + 101 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 101 + }, + 107 => { + :children => [], + :ppid => 106, + :cmd => "/usr/sbin/dnsmasq", + :pid => 107 + }, + 102 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 102 + }, + 108 => { + :children => [], + :ppid => 1, + :cmd => "/usr/sbin/apache2", + :pid => 108 + }, + 103 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 103 + }, + 109 => { + :children => [], + :ppid => 1, + :cmd => "/usr/bin/python keystone.py", + :pid => 109 + } + } + end + + let(:redhat_pstree) do + { + 104 => { + :children => [105], + :ppid => 1, + :cmd => "/usr/bin/python cinder-volume.py", + :pid => 104 + }, + 105 => { + :children => [], + :ppid => 104, + :cmd => "/usr/sbin/tgtd", + :pid => 105 + }, + 100 => { + :children => [101, 102, 103], + :ppid => 1, + :cmd => "/usr/bin/python nova-api.py", + :pid => 100 + }, + 1 => { + :children => [100, 104, 106, 108, 109] + }, + 106 => { + :children => [107], + :ppid => 1, + :cmd => "/usr/bin/python neutron.py", + :pid => 106 + }, + 101 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 101 + }, + 107 => { + :children => [], + :ppid => 106, + :cmd => "/usr/sbin/dnsmasq", + :pid => 107 + }, + 102 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 102 + }, + 108 => { + :children => [], + :ppid => 1, + :cmd => "/usr/sbin/httpd", + :pid => 108 + }, + 103 => { + :children => [], + :ppid => 100, + :cmd => "/usr/bin/python nova-api.py", + :pid => 103 + }, + 109 => { + :children => [], + :ppid => 1, + :cmd => "/usr/bin/python keystone.py", + :pid => 109 + } + } + end + + let(:debian_ps) do + <<-eos +100 1 /usr/bin/python nova-api.py +101 100 /usr/bin/python nova-api.py +102 100 /usr/bin/python nova-api.py +103 100 /usr/bin/python nova-api.py +104 1 /usr/bin/python cinder-volume.py +105 104 /usr/sbin/tgtd +106 1 /usr/bin/python neutron.py +107 106 /usr/sbin/dnsmasq +108 1 /usr/sbin/apache2 +109 1 /usr/bin/python keystone.py + eos + end + + let(:debian_services) do + <<-eos + [ ? ] ntpd + [ ? ] neutron + [ + ] cinder-volume + [ - ] nginx + [ - ] smbd + [ + ] sshd + [ + ] nova-api + [ + ] apache2 + [ + ] keystone + eos + end + + let(:redhat_services) do + <<-eos +ntpd is stopped +neutron is stopped +sshd (pid 50) is running... +cinder-volume (pid 104) is running... +nova-api (pid 100) is running... +nginx is stopped +smbd is stopped +httpd.event (pid 108) is running... +keystone (pid 109) is running... + eos + end + + let(:debian_services_to_stop) do + ["cinder-volume", "nova-api", "apache2", "keystone"] + end + + let(:redhat_services_to_stop) do + ["cinder-volume", "nova-api", "httpd", "openstack-keystone"] + end +################################################################### + + it 'should correctly parse ps output on Debian system' do + subject.stubs(:ps).returns(debian_ps) + subject.stubs(:osfamily).returns 'Debian' + subject.process_tree_with_renew + expect(subject.process_tree).to eq debian_pstree + end + + it 'should correctly parse ps output on RedHat system' do + subject.stubs(:ps).returns(redhat_ps) + subject.stubs(:osfamily).returns 'RedHat' + subject.process_tree_with_renew + expect(subject.process_tree).to eq redhat_pstree + end + + it 'should find services to stop on Debian system' do + subject.stubs(:services).returns debian_services + subject.stubs(:osfamily).returns 'Debian' + subject.services_to_stop_with_renew + expect(subject.services_to_stop).to eq debian_services_to_stop + end + + it 'should find services to stop on RedHat system' do + subject.stubs(:services).returns redhat_services + subject.stubs(:osfamily).returns 'RedHat' + subject.services_to_stop_with_renew + expect(subject.services_to_stop).to eq redhat_services_to_stop + end + + it 'should find processes by regexp' do + subject.stubs(:ps).returns(debian_ps) + subject.stubs(:osfamily).returns 'Debian' + subject.process_tree_with_renew + dnsmasq = {107 => { + :children => [], + :ppid => 106, + :cmd => "/usr/sbin/dnsmasq", + :pid => 107 + }} + expect(subject.pids_by_regexp /dnsmasq/).to eq dnsmasq + end + + it 'should kill correct processes on Debian system' do + subject.stubs(:ps).returns(debian_ps) + subject.stubs(:osfamily).returns 'Debian' + subject.stubs(:dry_run).returns true + subject.expects(:run).with 'kill -9 100 101 102 103 104 105 106 107 108 109' + subject.process_tree_with_renew + subject.kill_pids_by_stop_regexp + end + + it 'should kill correct processes on RedHat system' do + subject.stubs(:ps).returns(redhat_ps) + subject.stubs(:osfamily).returns 'RedHat' + subject.stubs(:dry_run).returns true + subject.expects(:run).with 'kill -9 100 101 102 103 104 105 106 107 108 109' + subject.process_tree_with_renew + subject.kill_pids_by_stop_regexp + end + + it 'should stop correct services on Debian system' do + subject.stubs(:services).returns debian_services + subject.stubs(:osfamily).returns 'Debian' + subject.stubs(:dry_run).returns true + subject.expects(:run).with 'service cinder-volume stop' + subject.expects(:run).with 'service nova-api stop' + subject.expects(:run).with 'service apache2 stop' + subject.expects(:run).with 'service keystone stop' + subject.services_to_stop_with_renew + subject.stop_services + end + + it 'should stop correct services on RedHat system' do + subject.stubs(:services).returns redhat_services + subject.stubs(:osfamily).returns 'RedHat' + subject.stubs(:dry_run).returns true + subject.expects(:run).with 'service cinder-volume stop' + subject.expects(:run).with 'service nova-api stop' + subject.expects(:run).with 'service httpd stop' + subject.expects(:run).with 'service openstack-keystone stop' + subject.services_to_stop_with_renew + subject.stop_services + end +end \ No newline at end of file