Support deletion of keypairs from AWS

Deletion of keypairs from OpenStack does not trigger a key pair deletion
from AWS. Listen to the keypair deletion notifications and delete the
key from AWS when one is received.

Also pulling in some of the local changes.

Change-Id: Iea466533a8a12c0acccf5b6bf08d99b4e7a1b622
Closes-Bug: #1716454

nova/tests/unit/virt/ec2/test_ec2.py

@@ -47,6 +47,7 @@ class EC2DriverTestCase(test.NoDBTestCase):
                    region_name=self.region_name,
                    group='AWS')
         self.flags(api_servers=['http://localhost:9292'], group='glance')
+        self.flags(rabbit_port='5672')
         self.conn = EC2Driver(None, False)
         self.type_data = None
         self.project_id = 'fake'

nova/tests/unit/virt/ec2/test_keypair.py

@@ -0,0 +1,83 @@
+"""
+Copyright 2016 Platform9 Systems Inc.
+All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+"""
+
+from moto import mock_ec2_deprecated
+from nova import test
+from nova.virt.ec2.keypair import KeyPairNotifications
+import boto
+import mock
+
+
+class KeyPairNotificationsTestCase(test.NoDBTestCase):
+    @mock_ec2_deprecated
+    def setUp(self):
+        super(KeyPairNotificationsTestCase, self).setUp()
+        fake_access_key = 'aws_access_key'
+        fake_secret_key = 'aws_secret_key'
+        region_name = 'us-west-1'
+        region = boto.ec2.get_region(region_name)
+        self.fake_aws_conn = boto.ec2.EC2Connection(
+            aws_access_key_id=fake_access_key,
+            aws_secret_access_key=fake_secret_key,
+            region=region)
+        self.flags(rabbit_port=5672)
+        self.conn = KeyPairNotifications(self.fake_aws_conn,
+                                         transport='memory')
+
+    def test_handle_notification_create_event(self):
+        body = {'event_type': 'keypair.create.start'}
+        with mock.patch.object(boto.ec2.EC2Connection, 'delete_key_pair') \
+                as mock_delete:
+            self.conn.handle_notification(body, None)
+            mock_delete.assert_not_called()
+
+    def test_handle_notifications_no_event_type(self):
+        body = {}
+        with mock.patch.object(boto.ec2.EC2Connection, 'delete_key_pair') \
+                as mock_delete:
+            self.conn.handle_notification(body, None)
+            mock_delete.assert_not_called()
+
+    @mock_ec2_deprecated
+    def test_handle_notifications_delete_key(self):
+        fake_key_name = 'fake_key'
+        fake_key_data = 'fake_key_data'
+        self.fake_aws_conn.import_key_pair(fake_key_name, fake_key_data)
+        body = {'event_type': 'keypair.delete.start',
+                'payload': {
+                    'key_name': fake_key_name
+                    }
+                }
+        self.conn.handle_notification(body, None)
+        aws_keypairs = self.fake_aws_conn.get_all_key_pairs()
+        self.assertEqual(len(aws_keypairs), 0)
+
+    @mock_ec2_deprecated
+    def test_handle_notifications_delete_key_with_multiple_keys_in_aws(self):
+        fake_key_name_1 = 'fake_key_1'
+        fake_key_data_1 = 'fake_key_data_1'
+        fake_key_name_2 = 'fake_key_2'
+        fake_key_data_2 = 'fake_key_data_2'
+        self.fake_aws_conn.import_key_pair(fake_key_name_1, fake_key_data_1)
+        self.fake_aws_conn.import_key_pair(fake_key_name_2, fake_key_data_2)
+        body = {'event_type': 'keypair.delete.start',
+                'payload': {
+                    'key_name': fake_key_name_1
+                    }
+                }
+        self.conn.handle_notification(body, None)
+        aws_keypairs = self.fake_aws_conn.get_all_key_pairs()
+        self.assertEqual(len(aws_keypairs), 1)
+        self.assertEqual(aws_keypairs[0].name, fake_key_name_2)
+

nova/virt/ec2/ec2driver.py

@@ -1,6 +1,6 @@
 """
 Copyright (c) 2014 Thoughtworks.
-Copyright (c) 2016 Platform9 Systems Inc.
+Copyright (c) 2017 Platform9 Systems Inc.
 All Rights reserved
 Licensed under the Apache License, Version 2.0 (the "License"); you may
 not use this file except in compliance with the License. You may obtain
@@ -15,6 +15,7 @@ under the License.
 
 import base64
 import datetime
+import eventlet
 import hashlib
 import json
 import time
@@ -34,12 +35,14 @@ from nova.i18n import _
 from nova.image import glance
 from nova.virt import driver
 from nova.virt.ec2.exception_handler import Ec2ExceptionHandler
+from nova.virt.ec2.keypair import KeyPairNotifications
 from nova.virt import hardware
 from nova.virt import virtapi
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_service import loopingcall
 
+eventlet.monkey_patch()
 LOG = logging.getLogger(__name__)
 
 aws_group = cfg.OptGroup(name='AWS',
@@ -63,11 +66,12 @@ aws_opts = [
     # 1 TB Storage
     cfg.IntOpt('max_disk_gb',
                default=1024,
-               help='Max storage in GB that can be used')
+               help='Max storage in GB that can be used'),
+    cfg.BoolOpt('enable_keypair_notifications', default=True,
+                help='Listen to keypair delete notifications and act on them')
 ]
 
 CONF = cfg.CONF
-# CONF.import_opt('my_ip', 'nova.netconf')
 
 CONF.register_group(aws_group)
 CONF.register_opts(aws_opts, group=aws_group)
@@ -121,7 +125,8 @@ EC2_FLAVOR_MAP = {
     't2.micro': {'memory_mb': 1024.0, 'vcpus': 1},
     't2.nano': {'memory_mb': 512.0, 'vcpus': 1},
     't2.small': {'memory_mb': 2048.0, 'vcpus': 1},
-    'x1.32xlarge': {'memory_mb': 1998848.0, 'vcpus': 128}
+    'x1.32xlarge': {'memory_mb': 1998848.0, 'vcpus': 128},
+    't1.micro': {'memory_mb': 613.0, 'vcpus': 1},
 }
 _EC2_NODES = None
 
@@ -190,6 +195,9 @@ class EC2Driver(driver.ComputeDriver):
             aws_region, aws_access_key_id=CONF.AWS.access_key,
             aws_secret_access_key=CONF.AWS.secret_key)
 
+        # Allow keypair deletion to be controlled by conf
+        if CONF.AWS.enable_keypair_notifications:
+            eventlet.spawn(KeyPairNotifications(self.ec2_conn).run)
         LOG.info("EC2 driver init with %s region" % aws_region)
         if _EC2_NODES is None:
             set_nodes([CONF.host])
@@ -257,7 +265,7 @@ class EC2Driver(driver.ComputeDriver):
         """
         image_api = glance.get_default_image_service()
         image_meta = image_api._client.call(context, 2, 'get',
-                                            image_lacking_meta['id'])
+                                            image_lacking_meta.id)
         LOG.info("Calling _get_image_ami_id_from_meta Meta: %s" % image_meta)
         try:
             return image_meta['aws_image_id']
@@ -372,6 +380,8 @@ class EC2Driver(driver.ComputeDriver):
             instance['metadata'].update({'ec2_id': ec2_id})
             ec2_instance_obj.add_tag("Name", instance['display_name'])
             ec2_instance_obj.add_tag("openstack_id", instance['uuid'])
+            ec2_instance_obj.add_tag("openstack_project_id", context.project_id)
+            ec2_instance_obj.add_tag("openstack_user_id", context.user_id)
             self._uuid_to_ec2_instance[instance.uuid] = ec2_instance_obj
 
             # Fetch Public IP of the instance if it has one
@@ -684,13 +694,13 @@ class EC2Driver(driver.ComputeDriver):
         return True
 
     def attach_interface(self, instance, image_meta, vif):
-        LOG.debug("******* ATTTACH INTERFACE *******")
+        LOG.debug("AWS: Attaching interface", instance=instance)
         if vif['id'] in self._interfaces:
             raise exception.InterfaceAttachFailed('duplicate')
         self._interfaces[vif['id']] = vif
 
     def detach_interface(self, instance, vif):
-        LOG.debug("******* DETACH INTERFACE *******")
+        LOG.debug("AWS: Detaching interface", instance=instance)
         try:
             del self._interfaces[vif['id']]
         except KeyError:

nova/virt/ec2/keypair.py

@@ -0,0 +1,70 @@
+"""
+Copyright (c) 2014 Thoughtworks.
+Copyright (c) 2017 Platform9 Systems Inc.
+All Rights reserved
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+"""
+
+import eventlet
+eventlet.monkey_patch()
+from kombu import Connection
+from kombu import Exchange
+from kombu import Queue
+from kombu.mixins import ConsumerMixin
+from oslo_config import cfg
+from oslo_log import log as logging
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+rabbit_opts = [
+    cfg.StrOpt('rabbit_userid'),
+    cfg.StrOpt('rabbit_password'),
+    cfg.StrOpt('rabbit_host'),
+    cfg.StrOpt('rabbit_port'),
+]
+
+CONF.register_opts(rabbit_opts)
+
+
+class KeyPairNotifications(ConsumerMixin):
+    nova_exchange = 'nova'
+    routing_key = 'notifications.info'
+    queue_name = 'notifications.omni.keypair'
+    events_of_interest = ['keypair.delete.start', 'keypair.delete.end']
+
+    def __init__(self, aws_connection, transport='amqp'):
+        self.ec2_conn = aws_connection
+        self.broker_uri = \
+            "{transport}://{username}:{password}@{rabbit_host}:{rabbit_port}"\
+            .format(transport=transport,
+                    username=CONF.rabbit_userid,
+                    password=CONF.rabbit_password,
+                    rabbit_host=CONF.rabbit_host,
+                    rabbit_port=CONF.rabbit_port)
+        self.connection = Connection(self.broker_uri)
+
+    def get_consumers(self, consumer, channel):
+        exchange = Exchange(self.nova_exchange, type="topic", durable=False)
+        queue = Queue(self.queue_name, exchange, routing_key=self.routing_key,
+                      durable=False, auto_delete=True, no_ack=True)
+        return [consumer(queue, callbacks=[self.handle_notification])]
+
+    def handle_notification(self, body, message):
+        if 'event_type' in body and body['event_type'] in \
+                self.events_of_interest:
+            LOG.debug('Body: %r' % body)
+            key_name = body['payload']['key_name']
+            try:
+                LOG.info('Deleting %s keypair', key_name)
+                self.ec2_conn.delete_key_pair(key_name)
+            except:
+                LOG.exception('Could not delete %s', key_name)

run_tests.sh

@@ -25,6 +25,7 @@ DIRECTORY="$WORKSPACE/openstack"
 GCE_TEST="test_gce"
 AWS_TEST="test_ec2"
 AWS_NOVA_TEST="test_ec2.EC2DriverTestCase"
+AWS_KEYPAIR_TEST="test_keypair.KeyPairNotificationsTestCase"
 declare -A results
 declare -i fail
 declare -i pass
@@ -87,7 +88,7 @@ copy_neutron_files
 
 echo "============Running tests============"
 run_tests cinder "$GCE_TEST|$AWS_TEST" &
-run_tests nova "$GCE_TEST|$AWS_NOVA_TEST" &
+run_tests nova "$GCE_TEST|$AWS_NOVA_TEST|$AWS_KEYPAIR_TEST" &
 run_tests glance_store "$GCE_TEST" &
 run_tests neutron "$GCE_TEST|$AWS_TEST" &
 wait