Merge "cron job to clean up rabbitmq connections" into stable/5.0

deployment/puppet/openstack/manifests/controller_ha.pp

@@ -167,6 +167,8 @@ class openstack::controller_ha (
       idle_timeout                 => $idle_timeout,
     }
 
+    include openstack::ha::rabbitmq_connections_cleanup
+
     if $quantum and $quantum_network_node {
       class { '::openstack::neutron_router':
         #service_endpoint      => $internal_virtual_ip,

deployment/puppet/openstack/manifests/ha/rabbitmq_connections_cleanup.pp

@@ -0,0 +1,40 @@
+# RabbitMQ connections cleanup
+class openstack::ha::rabbitmq_connections_cleanup (
+  $rabbit_virtual_host = '/',
+  $rabbit_userid       = $::openstack::controller_ha::amqp_user,
+  $rabbit_password     = $::openstack::controller_ha::amqp_password,
+) {
+
+  exec {'rabbitmq-plugins enable rabbitmq_management':
+    path        => '/sbin:/bin:/usr/sbin:/usr/bin',
+    environment => 'HOME=/root',
+    require     => Package[$::rabbitmq::server::package_name],
+    notify      => Service[$::rabbitmq::service::service_name],
+  }
+
+  package {'python-rabbit': }
+
+  file {'rabbitmq-connections-cleanup':
+    path    => '/usr/local/bin/rabbitmq-connections-cleanup',
+    content => template('openstack/rabbitmq-connections-cleanup'),
+    owner   => '0',
+    group   => '0',
+    mode    => '0755',
+    require => Package['python-rabbit'],
+  }
+
+  file {'rabbitmq-connections-cleanup.conf':
+    path    => '/root/rabbitmq-connections-cleanup.conf',
+    content => template('openstack/rabbitmq-connections-cleanup.conf.erb'),
+  }
+
+  cron {'rabbitmq-connections-cleanup':
+    command => '/usr/local/bin/rabbitmq-connections-cleanup /root/rabbitmq-connections-cleanup.conf',
+    user    => 'root',
+    minute  => '*/1',
+    require => [File['rabbitmq-connections-cleanup',
+                     'rabbitmq-connections-cleanup.conf'],
+                Exec['rabbitmq-plugins enable rabbitmq_management'],
+                Service[$::rabbitmq::service::service_name]],
+  }
+}

deployment/puppet/openstack/templates/rabbitmq-connections-cleanup

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2014 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import ConfigParser
+import pyrabbit
+import pyrabbit.api
+import sys
+import time
+import traceback
+
+
+def collect_suspected_connections(host, port, login, password, vhost):
+    client = pyrabbit.api.Client('{0}:{1}'.format(host, port), login, password)
+    connections = client.get_connections()
+    suspected_connections = set()
+    for connection in connections or []:
+        if connection['vhost'] == vhost and connection.get(
+                'channels', 1) == 0 and connection.get('timeout', 0) == 0:
+            print 'Connection {0} has no channels'.format(connection['name'])
+            suspected_connections.add(connection['name'])
+    if len(suspected_connections) > 0:
+        yield (client, suspected_connections)
+
+
+def kill_connections(collected, vhost):
+    if len(collected) == 0:
+        print 'No problem found'
+        return
+
+    print 'Sleeping for 10s'
+    time.sleep(10)
+
+    for client, suspected_connections in collected:
+        connections = client.get_connections()
+        for connection in connections:
+            name = connection['name']
+            if connection['vhost'] == vhost and connection.get(
+                    'channels', 1) == 0 and name in suspected_connections:
+                print 'Terminating connection', name
+                client.delete_connection(name)
+
+
+class FakeSecHead(object):
+    def __init__(self, fp):
+        self.fp = fp
+        self.header = '[DEFAULT]\n'
+
+    def readline(self):
+        if self.header:
+            try:
+                return self.header
+            finally:
+                self.header = None
+        else:
+            return self.fp.readline()
+
+
+def main():
+    cfg = ConfigParser.ConfigParser(
+        defaults={
+            'rabbit_hosts': '127.0.0.1',
+            'rabbit_virtual_host': '/',
+            'rabbit_userid': 'guest',
+            'rabbit_password': 'guest'
+        })
+    cfg.readfp(FakeSecHead(open(sys.argv[1])))
+    hosts = cfg.get('DEFAULT', 'rabbit_hosts').split(',')
+    vhost = cfg.get('DEFAULT', 'rabbit_virtual_host')
+    login = cfg.get('DEFAULT', 'rabbit_userid')
+    password = cfg.get('DEFAULT', 'rabbit_password')
+
+    suspected_connections = []
+    for record in hosts:
+        hostname = record.split(':')[0]
+        port = int(sys.argv[2]) if len(sys.argv) == 3 else 15672
+        try:
+            print 'Accessing RabbitMQ management plugin on ' \
+                  '{0}:{1}'.format(hostname, port)
+            suspected_connections.extend(collect_suspected_connections(
+                hostname, port, login, password, vhost))
+            break
+        except Exception as e:
+            traceback.print_exc(e, file=sys.stdout)
+            print
+    kill_connections(suspected_connections, vhost)
+
+
+if __name__ == "__main__":
+    if 2 < len(sys.argv) > 3:
+        print 'Usage: rabbitmq-connections-cleanup config-file.conf [port]'
+    else:
+        main()

deployment/puppet/openstack/templates/rabbitmq-connections-cleanup.conf.erb

@@ -0,0 +1,5 @@
+[DEFAULT]
+#rabbit_hosts = '127.0.0.1'
+rabbit_virtual_host = <%= @rabbit_virtual_host %>
+rabbit_userid = <%= @rabbit_userid %>
+rabbit_password = <%= @rabbit_password %>