From 9fdb07ec6653c751d79e66e7f0ab4a8ce68d120c Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Wed, 11 Jul 2018 11:01:13 +0200 Subject: [PATCH] Enforce proper ordering when applying firewall rules The tripleo firewall module has fundamentally three pieces: 1) firewall::pre (allows existing connections/ssh/icmp) 2) firewall::rule (allows services traffic) 3) firewall::post (drops all traffic) One of the assumptions coded in the module is the following line: Service<||> -> Class['tripleo::firewall::post'] Which has been added so that (see also bug LP#1643575): """ use ordering to make sure we start all Services in catalog before post rules. It ensure that we don't drop all traffic before starting the services, which could lead to services errors (e.g. trying to reach database or amqp) """ The problem is that there is nothing specifying that the firewall rules created by tripleo services need to be implemented between the pre and post classes. So the following can happen: Jul 10 05:04:13 overcloud-controller-1 systemd: Started OpenSSH server daemon. Jul 10 05:04:13 overcloud-controller-1 puppet-user[32418]: (/Stage[main]/Ssh::Server::Service/Service[sshd]) Triggered 'refresh' from 2 events Jul 10 05:04:13 overcloud-controller-1 puppet-user[32418]: (/Stage[main]/Tripleo::Firewall::Pre/Tripleo::Firewall::Rule[000 accept related established rules]/Firewall[000 accept related established rules ipv4]/ensure) created ... Jul 10 05:04:13 overcloud-controller-1 puppet-user[32418]: (/Stage[main]/Tripleo::Firewall::Post/Tripleo::Firewall::Rule[998 log all]/Firewall[998 log all ipv4]/ensure) created ... Jul 10 05:04:14 overcloud-controller-1 puppet-user[32418]: (/Stage[main]/Tripleo::Firewall/Tripleo::Firewall::Service_rules[cinder_api]/Tripleo::Firewall::Rule[119 cinder]/Firewall[119 cinder ipv4]/ensure) created This means that we can actually open the traffic for our services *after* said traffic has been completely blocked. In order to fix this we tag the pre/post rules with a different tag and add resource collectors to actually enforce proper ordering. We now get: ... Jul 11 08:54:43 overcloud-controller-0 puppet-user[32554]: (/Stage[main]/Tripleo::Firewall::Pre/Tripleo::Firewall::Rule[000 accept related established rules]/Firewall[000 accept related established rules ipv4]/ensure) created ... Jul 11 08:54:43 overcloud-controller-0 puppet-user[32554]: (/Stage[main]/Tripleo::Firewall/Tripleo::Firewall::Service_rules[cinder_api]/Tripleo::Firewall::Rule[119 cinder]/Firewall[119 cinder ipv4]/ensure) created ... Jul 11 08:54:52 overcloud-controller-0 puppet-user[32554]: (/Stage[main]/Tripleo::Firewall::Post/Tripleo::Firewall::Rule[998 log all]/Firewall[998 log all ipv4]/ensure) created Tested this by doing 20 deploys of 1ctrl+1cmp and then scaling up the overcloud to 3ctrl+2cmp. The reason this change, besides being semantically correct, is needed for scaling up the controller role is the following: 1) When we detect a scaleup situation, we call 'pcs cluster node add ' from the bootstrap node 2) It can happen that 1) succeeds because the node still had no iptables at all, so pcs communication succeeds. So at this point the cluster has added a new node and all is well 3) Now we apply the pre and post rules and all traffic is blocked. And slowly we start adding rules to let services be accessible over the network. 4) pacemaker is unable to talk to other nodes and assumes a network split. Eventually it will start fencing node to guarantee the state of the remote nodes and so the deploy will fail. With this change if the 'pcs cluster node add' call succeeds we are guaranteed to not interrupt network traffic afterwards. NB: cherry-pick had a slight conflict due Ic9a2626e73d132c3be7ff14a1f4cdba0c16c5b53 Change-Id: I01e681a6305e2708bf364781a2032265b146d065 Closes-Bug: #1781147 (cherry picked from commit c525c64f6a9535b0b040cca1a7813e93dfaa797c) --- manifests/firewall.pp | 5 ++++- manifests/firewall/post.pp | 2 ++ manifests/firewall/pre.pp | 5 +++++ manifests/firewall/rule.pp | 8 ++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/manifests/firewall.pp b/manifests/firewall.pp index 3898e75de..9e87c5bce 100644 --- a/manifests/firewall.pp +++ b/manifests/firewall.pp @@ -110,7 +110,10 @@ class tripleo::firewall( 'firewall_settings' => $firewall_post_extras, }) - Class['tripleo::firewall::pre'] -> Class['tripleo::firewall::post'] + Class['tripleo::firewall::pre'] + -> Firewall<|tag == 'tripleo-firewall-rule'|> + -> Class['tripleo::firewall::post'] + Service<||> -> Class['tripleo::firewall::post'] # Allow composable services to load their own custom diff --git a/manifests/firewall/post.pp b/manifests/firewall/post.pp index 7b5f56391..820a0f6f1 100644 --- a/manifests/firewall/post.pp +++ b/manifests/firewall/post.pp @@ -39,11 +39,13 @@ class tripleo::firewall::post( tripleo::firewall::rule{ '998 log all': proto => 'all', jump => 'LOG', + tag => 'tripleo-firewall-postrule', } tripleo::firewall::rule{ '999 drop all': proto => 'all', action => 'drop', extras => $firewall_settings, + tag => 'tripleo-firewall-postrule', } notice('At this stage, all network traffic is blocked.') } diff --git a/manifests/firewall/pre.pp b/manifests/firewall/pre.pp index 39120d92d..db7540a2b 100644 --- a/manifests/firewall/pre.pp +++ b/manifests/firewall/pre.pp @@ -36,22 +36,26 @@ class tripleo::firewall::pre( proto => 'all', state => ['RELATED', 'ESTABLISHED'], extras => $firewall_settings, + tag => 'tripleo-firewall-prerule', } tripleo::firewall::rule{ '001 accept all icmp': proto => 'icmp', extras => $firewall_settings, + tag => 'tripleo-firewall-prerule', } tripleo::firewall::rule{ '002 accept all to lo interface': proto => 'all', iniface => 'lo', extras => $firewall_settings, + tag => 'tripleo-firewall-prerule', } tripleo::firewall::rule{ '003 accept ssh': dport => '22', extras => $firewall_settings, + tag => 'tripleo-firewall-prerule', } tripleo::firewall::rule{ '004 accept ipv6 dhcpv6': @@ -59,5 +63,6 @@ class tripleo::firewall::pre( proto => 'udp', state => ['NEW'], destination => 'fe80::/64', + tag => 'tripleo-firewall-prerule', } } diff --git a/manifests/firewall/rule.pp b/manifests/firewall/rule.pp index f1ea0c9d0..eecedb118 100644 --- a/manifests/firewall/rule.pp +++ b/manifests/firewall/rule.pp @@ -63,6 +63,12 @@ # (optional) The destination cidr associated to the rule. # Defaults to undef # +# [*tag*] +# (optional) tag to add to the resource definition. +# Used to order any rule that is not pre and post to happen in between +# pre and post rules +# Defaults to 'tripleo-firewall-rule' +# # [*extras*] # (optional) Hash of any puppetlabs-firewall supported parameters. # Defaults to {} @@ -80,6 +86,7 @@ define tripleo::firewall::rule ( $destination = undef, $extras = {}, $jump = undef, + $tag = 'tripleo-firewall-rule', ) { if $port == 'all' { @@ -109,6 +116,7 @@ define tripleo::firewall::rule ( 'chain' => $chain, 'destination' => $destination, 'jump' => $jump_real, + 'tag' => $tag, } if $proto == 'icmp' { $ipv6 = {