From ca3370eedc55751205ce1d4f8c4e40067baca2c5 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Thu, 16 Oct 2014 17:00:10 -0500 Subject: [PATCH 01/20] Cam & Ed | Set up Moto (mock Boto) server for testing --- tests/moto_test.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/moto_test.py diff --git a/tests/moto_test.py b/tests/moto_test.py new file mode 100644 index 0000000..5ef8757 --- /dev/null +++ b/tests/moto_test.py @@ -0,0 +1,34 @@ +import os +import unittest +from boto import ec2 +from boto.regioninfo import RegionInfo +from ..ec2driver_config import * + +START_MOTO_SERVER_COMMAND = 'moto_server ec2 -p1234' +END_MOTO_SERVER_COMMAND = "ps aux | grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -9"\ + % START_MOTO_SERVER_COMMAND + +class MotoTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + os.system(START_MOTO_SERVER_COMMAND + "&") + moto_region = RegionInfo(name='moto_region', endpoint="localhost:1234") + cls.ec2_conn = ec2.EC2Connection(aws_access_key_id='the_key', aws_secret_access_key='the_secret', + host='0.0.0.0', port='1234', + region = moto_region, + is_secure=False) + print cls.ec2_conn + + def test_should_create_an_instance(self): + reservation = self.ec2_conn.run_instances('ami-abcd1234') + ec2_instances = reservation.instances + self.assertEqual(len(ec2_instances), 1) + + @classmethod + def tearDownClass(cls): + os.system(END_MOTO_SERVER_COMMAND) + +if __name__ == '__main__': + unittest.main() + + From 8d09039ef4b3471bec5aec0db858f4fe0bd289e3 Mon Sep 17 00:00:00 2001 From: Kashyap Kopparam Date: Tue, 21 Oct 2014 12:46:33 +0530 Subject: [PATCH 02/20] Kashyap | added test for attach volume and modified the volume_map to support default values --- credentials.py | 1 - ec2driver_config.py | 8 +++++--- tests/test_ec2driver.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/credentials.py b/credentials.py index 9341342..307f012 100644 --- a/credentials.py +++ b/credentials.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os - def get_nova_creds(): d = {} d['username'] = os.environ['OS_USERNAME'] diff --git a/ec2driver_config.py b/ec2driver_config.py index 603bc7a..37f0e0e 100644 --- a/ec2driver_config.py +++ b/ec2driver_config.py @@ -14,7 +14,7 @@ # This is the config file which is going to hold the values for being able # to connect to the AWS Public cloud. - +from collections import defaultdict aws_region = "us-east-1" aws_access_key_id = "AKIAIZJDDRNNJUWZ3LXA" @@ -29,8 +29,10 @@ instance_type = "t2.micro" flavor_map = {2: 't2.micro', 5: 't2.small', 1: 't2.medium', 3: 'c3.xlarge', 4: 'c3.2xlarge'} #Add image maps key: image in openstack, Value: EC2_AMI_ID image_map = {} -volume_map = {'3df37a34-662e-4aa8-b71d-b8313d2e945b': 'vol-83db57cb', - '7d63c661-7e93-445b-b3cb-765f1c8ae4c0': 'vol-1eea8a56'} +# Using defaultdict as we need to get a default EBS volume to be returned if we access this map with an unknown key +volume_map_no_default = {'ed6fcf64-8c74-49a0-a30c-76128c7bda47': 'vol-83db57cb', + 'ac28d216-6dda-4a7b-86c4-d95209ae8181': 'vol-1eea8a56'} +volume_map = defaultdict(lambda: 'vol-83db57cb', volume_map_no_default) keypair_map = {} # The limit on maximum resources you could have in the AWS EC2. diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index 6ddd503..a88c867 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -204,6 +204,17 @@ class EC2DriverTest(unittest.TestCase): self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') self.assertEqual(diagnostics['instance._state'], 'running(16)') + def test_attach_volume(self): + creds = get_nova_creds() + creds['service_type'] = 'volume' + nova = client.Client(**creds) + volume = nova.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) + instance, instance_ref = self.spawn_ec2_instance() + self.nova.volumes.create_server_volume(instance_ref, volume.id, "/dev/sdb") + time.sleep(30) + volumes = self.nova.volumes.get_server_volumes(instance.id) + self.assertIn(volume, volumes) + @classmethod def tearDown(self): print "Cleanup: Destroying the instance used for testing" From 4d74624edc60f89c5af38ca0d9f6bfa3da94840a Mon Sep 17 00:00:00 2001 From: Kashyap Kopparam Date: Tue, 21 Oct 2014 14:15:56 +0530 Subject: [PATCH 03/20] Kashyap | refactored the test class to provide nova_volume client instead of attach_volume test creating it --- tests/test_ec2driver.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index a88c867..cb850dc 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -16,8 +16,13 @@ class EC2DriverTest(unittest.TestCase): print "Establishing connection with AWS" self.ec2_conn = ec2.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) + # nova client for nova self.creds = get_nova_creds() self.nova = client.Client(**self.creds) + + # nova client for cinder + self.creds['service_type'] = 'volume' + self.nova_volume = client.Client(**self.creds) self.servers = [] def spawn_ec2_instance(self): @@ -205,10 +210,7 @@ class EC2DriverTest(unittest.TestCase): self.assertEqual(diagnostics['instance._state'], 'running(16)') def test_attach_volume(self): - creds = get_nova_creds() - creds['service_type'] = 'volume' - nova = client.Client(**creds) - volume = nova.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) + volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) instance, instance_ref = self.spawn_ec2_instance() self.nova.volumes.create_server_volume(instance_ref, volume.id, "/dev/sdb") time.sleep(30) From b154d352fc4102b4bf9e0864edbbd5829f25cbd3 Mon Sep 17 00:00:00 2001 From: Kashyap Kopparam Date: Tue, 21 Oct 2014 14:55:30 +0530 Subject: [PATCH 04/20] Kashyap | tear down now takes care of deleting volumes --- tests/test_ec2driver.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index cb850dc..9aeb336 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -24,6 +24,7 @@ class EC2DriverTest(unittest.TestCase): self.creds['service_type'] = 'volume' self.nova_volume = client.Client(**self.creds) self.servers = [] + self.volumes = [] def spawn_ec2_instance(self): print "Spawning an instance" @@ -211,6 +212,7 @@ class EC2DriverTest(unittest.TestCase): def test_attach_volume(self): volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) + self.volumes.append(volume) instance, instance_ref = self.spawn_ec2_instance() self.nova.volumes.create_server_volume(instance_ref, volume.id, "/dev/sdb") time.sleep(30) @@ -222,6 +224,10 @@ class EC2DriverTest(unittest.TestCase): print "Cleanup: Destroying the instance used for testing" for instance in self.servers: instance.delete() + # wait for all instances to completely shut down and detach volumes if any + time.sleep(120) + for volume in self.volumes: + volume.delete() if __name__ == '__main__': unittest.main() From 7c7a8540d24ab5d4b338f3b72d0cd1aed0f42fa6 Mon Sep 17 00:00:00 2001 From: new Date: Tue, 21 Oct 2014 15:41:22 +0530 Subject: [PATCH 05/20] Venu | adding docStrings and refactoring --- .gitignore | 1 + ec2driver.py | 65 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 7d9478c..d6fbf63 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc .vagrant +.idea diff --git a/ec2driver.py b/ec2driver.py index d2d0a3e..e7a728d 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -16,15 +16,15 @@ """Connection to the Amazon Web Services - EC2 service""" from threading import Lock import base64 - +from ec2driver_config import * from boto import ec2 import boto.ec2.cloudwatch from boto import exception as boto_exc from boto.exception import EC2ResponseError +from credentials import get_nova_creds + from oslo.config import cfg from novaclient.v1_1 import client - -from ec2driver_config import * from nova import block_device from nova.compute import power_state from nova.compute import task_states @@ -37,10 +37,6 @@ from nova.openstack.common import loopingcall from nova.virt import driver from nova.virt import virtapi from nova.compute import flavors -import base64 -import time -from novaclient.v1_1 import client -from credentials import get_nova_creds LOG = logging.getLogger(__name__) @@ -118,9 +114,9 @@ class EC2Driver(driver.ComputeDriver): def __init__(self, virtapi, read_only=False): super(EC2Driver, self).__init__(virtapi) self.host_status_base = { - 'vcpus': 100000, - 'memory_mb': 8000000000, - 'local_gb': 600000000000, + 'vcpus': VCPUS, + 'memory_mb': MEMORY_IN_MBS, + 'local_gb': DISK_IN_GB, 'vcpus_used': 0, 'memory_mb_used': 0, 'local_gb_used': 100000000000, @@ -150,9 +146,15 @@ class EC2Driver(driver.ComputeDriver): set_nodes([CONF.host]) def init_host(self, host): + """Initialize anything that is necessary for the driver to function, + including catching up with currently running VM's on the given host. + """ return def list_instances(self): + """Return the names of all the instances known to the virtualization + layer, as a list. + """ all_instances = self.ec2_conn.get_all_instances() instance_ids = [] for instance in all_instances: @@ -169,6 +171,27 @@ class EC2Driver(driver.ComputeDriver): def spawn(self, context, instance, image_meta, injected_files, admin_password, network_info=None, block_device_info=None): + """Create a new instance/VM/domain on the virtualization platform. + Once this successfully completes, the instance should be + running (power_state.RUNNING). + + If this fails, any partial instance should be completely + cleaned up, and the virtualization platform should be in the state + that it was before this call began. + + :param context: security context + :param instance: nova.objects.instance.Instance + This function should use the data there to guide + the creation of the new instance. + :param image_meta: image object returned by nova.image.glance that + defines the image from which to boot this instance + :param injected_files: User files to inject into instance. + :param admin_password: set in instance. + :param network_info: + :py:meth:`~nova.network.manager.NetworkManager.get_instance_nw_info` + :param block_device_info: Information about block devices to be + attached to the instance. + """ LOG.info("***** Calling SPAWN *******************") LOG.info("****** %s" % instance._user_data) LOG.info("****** Allocating an elastic IP *********") @@ -194,24 +217,21 @@ class EC2Driver(driver.ComputeDriver): self.ec2_conn.associate_address(instance_id=ec2_id, allocation_id=elastic_ip_address.allocation_id) def snapshot(self, context, instance, image_id, update_task_state): + """Snapshot an image of the specified instance + on EC2 and create an Image which gets stored in AMI (internally in EBS Snapshot) + :param context: security context + :param instance: nova.objects.instance.Instance + :param image_id: Reference to a pre-created image that will hold the snapshot. """ - Snapshot an image on EC2 and create an Image which gets stored in AMI (internally in EBS Snapshot) - - """ - LOG.info("***** Calling SNAPSHOT *******************") - if(instance['metadata']['ec2_id'] is None): + if instance['metadata']['ec2_id'] is None: raise exception.InstanceNotRunning(instance_id=instance['uuid']) # Adding the below line only alters the state of the instance and not # its image in OpenStack. update_task_state( task_state=task_states.IMAGE_UPLOADING, expected_state=task_states.IMAGE_SNAPSHOT) - - # TODO change the image status to Active instead of in saving or - # queuing - ec2_id = instance['metadata']['ec2_id'] ec_instance_info = self.ec2_conn.get_only_instances( instance_ids=[ec2_id], filters=None, dry_run=False, max_results=None) @@ -220,15 +240,14 @@ class EC2Driver(driver.ComputeDriver): ec2_image_id = ec2_instance.create_image(name=str( image_id), description="Image from OpenStack", no_reboot=False, dry_run=False) LOG.info("Image has been created state to %s." % ec2_image_id) - # The instance will be in pending state when it comes up, waiting for - # it to be in available + + # The instance will be in pending state when it comes up, waiting forit to be in available self._wait_for_image_state(ec2_image_id, "available") image_api = glance.get_default_image_service() image_ref = glance.generate_image_url(image_id) metadata = {'is_public': False, - # 'checksum': '4eada48c2843d2a262c814ddc92ecf2c', #Hard-coded value for now 'location': image_ref, 'properties': { 'kernel_id': instance['kernel_id'], @@ -236,7 +255,7 @@ class EC2Driver(driver.ComputeDriver): 'owner_id': instance['project_id'], 'ramdisk_id': instance['ramdisk_id'], 'ec2_image_id': ec2_image_id - } + } } image_api.update(context, image_id, metadata) From 57c2fa582cdcaf7e48aee334e5426c98651fe631 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Mon, 20 Oct 2014 17:00:15 -0500 Subject: [PATCH 06/20] Ed & Cam | Add separate configuration for tests --- ec2driver_config.py | 48 ++++++------------------------------ ec2driver_standard_config.py | 42 +++++++++++++++++++++++++++++++ ec2driver_test_config.py | 48 ++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 ec2driver_standard_config.py create mode 100644 ec2driver_test_config.py diff --git a/ec2driver_config.py b/ec2driver_config.py index 37f0e0e..4fc24f5 100644 --- a/ec2driver_config.py +++ b/ec2driver_config.py @@ -1,42 +1,8 @@ -# Copyright (c) 2014 Thoughtworks. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +import os -# This is the config file which is going to hold the values for being able -# to connect to the AWS Public cloud. -from collections import defaultdict - -aws_region = "us-east-1" -aws_access_key_id = "AKIAIZJDDRNNJUWZ3LXA" -aws_secret_access_key = "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" - -#Adding a Red Hat Linux image below -aws_ami = "ami-785bae10" -#aws_ami = "ami-864d84ee" -instance_type = "t2.micro" - -# Mapping OpenStack's flavor IDs(which seems to be randomly assigned) to EC2's flavor names -flavor_map = {2: 't2.micro', 5: 't2.small', 1: 't2.medium', 3: 'c3.xlarge', 4: 'c3.2xlarge'} -#Add image maps key: image in openstack, Value: EC2_AMI_ID -image_map = {} -# Using defaultdict as we need to get a default EBS volume to be returned if we access this map with an unknown key -volume_map_no_default = {'ed6fcf64-8c74-49a0-a30c-76128c7bda47': 'vol-83db57cb', - 'ac28d216-6dda-4a7b-86c4-d95209ae8181': 'vol-1eea8a56'} -volume_map = defaultdict(lambda: 'vol-83db57cb', volume_map_no_default) -keypair_map = {} - -# The limit on maximum resources you could have in the AWS EC2. - -VCPUS = 100 -MEMORY_IN_MBS = 88192 -DISK_IN_GB = 1028 +if os.environ.get('TEST'): + print "test environment" + from ec2driver_test_config import * +else: + print "prod env" + from ec2driver_standard_config import * \ No newline at end of file diff --git a/ec2driver_standard_config.py b/ec2driver_standard_config.py new file mode 100644 index 0000000..e14ae58 --- /dev/null +++ b/ec2driver_standard_config.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014 Thoughtworks. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is the config file which is going to hold the values for being able +# to connect to the AWS Public cloud. +from collections import defaultdict + +aws_region = "us-east-1" +aws_access_key_id = "AKIAIZJDDRNNJUWZ3LXA" +aws_secret_access_key = "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" + +#Adding a Red Hat Linux image below +aws_ami = "ami-785bae10" +#aws_ami = "ami-864d84ee" +instance_type = "t2.micro" + +# Mapping OpenStack's flavor IDs(which seems to be randomly assigned) to EC2's flavor names +flavor_map = {2: 't2.micro', 5: 't2.small', 1: 't2.medium', 3: 'c3.xlarge', 4: 'c3.2xlarge'} +#Add image maps key: image in openstack, Value: EC2_AMI_ID +image_map = {} +# Using defaultdict as we need to get a default EBS volume to be returned if we access this map with an unknown key +volume_map_no_default = {'ed6fcf64-8c74-49a0-a30c-76128c7bda47': 'vol-83db57cb', + 'ac28d216-6dda-4a7b-86c4-d95209ae8181': 'vol-1eea8a56'} +volume_map = defaultdict(lambda: 'vol-83db57cb', volume_map_no_default) +keypair_map = {} + +# The limit on maximum resources you could have in the AWS EC2. + +VCPUS = 100 +MEMORY_IN_MBS = 88192 +DISK_IN_GB = 1028 \ No newline at end of file diff --git a/ec2driver_test_config.py b/ec2driver_test_config.py new file mode 100644 index 0000000..0b17eeb --- /dev/null +++ b/ec2driver_test_config.py @@ -0,0 +1,48 @@ +# Copyright (c) 2014 Thoughtworks. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is the config file which is going to hold the values for being able +# to connect to the AWS Public cloud. +from collections import defaultdict + +aws_region = "moto_region" +aws_endpoint = "localhost:1234" +aws_access_key_id = 'the_key' +aws_secret_access_key = 'the_secret' +host = '0.0.0.0' +port = '1234' +secure = False + +#Adding a Red Hat Linux image below +aws_ami = "ami-785bae10" +#aws_ami = "ami-864d84ee" +instance_type = "t2.micro" + +# Mapping OpenStack's flavor IDs(which seems to be randomly assigned) to EC2's flavor names +flavor_map = {2: 't2.micro', 5: 't2.small', 1: 't2.medium', 3: 'c3.xlarge', 4: 'c3.2xlarge'} +#Add image maps key: image in openstack, Value: EC2_AMI_ID +image_map = {} +# Using defaultdict as we need to get a default EBS volume to be returned if we access this map with an unknown key +volume_map_no_default = {'ed6fcf64-8c74-49a0-a30c-76128c7bda47': 'vol-83db57cb', + 'ac28d216-6dda-4a7b-86c4-d95209ae8181': 'vol-1eea8a56'} +volume_map = defaultdict(lambda: 'vol-83db57cb', volume_map_no_default) +keypair_map = {} + +# The limit on maximum resources you could have in the AWS EC2. + +VCPUS = 100 +MEMORY_IN_MBS = 88192 +DISK_IN_GB = 1028 + + From 0cac93e178b5a619fea0562fabdf5c97687180f3 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Mon, 20 Oct 2014 17:10:59 -0500 Subject: [PATCH 07/20] Cam & Ed | Modify driver and tests to work with moto --- ec2driver.py | 29 +++++-- tests/test_ec2driver.py | 169 +++++++++++++++++++++------------------- 2 files changed, 111 insertions(+), 87 deletions(-) diff --git a/ec2driver.py b/ec2driver.py index e7a728d..abdc124 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -23,6 +23,7 @@ from boto import exception as boto_exc from boto.exception import EC2ResponseError from credentials import get_nova_creds +from boto.regioninfo import RegionInfo from oslo.config import cfg from novaclient.v1_1 import client from nova import block_device @@ -133,12 +134,22 @@ class EC2Driver(driver.ComputeDriver): self.nova = client.Client(**self.creds) # To connect to EC2 - self.ec2_conn = ec2.connect_to_region( - aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( - aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) + # self.ec2_conn = ec2.connect_to_region( + # aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - self.reservation = self.ec2_conn.get_all_reservations() + moto_region = RegionInfo(name=aws_region, endpoint=aws_endpoint) + self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + host=host, + port=port, + region = moto_region, + is_secure=secure) + self.cloudwatch_conn = None + + # self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( + # aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) + + # self.reservation = self.ec2_conn.get_all_reservations() self.security_group_lock = Lock() @@ -368,15 +379,17 @@ class EC2Driver(driver.ComputeDriver): else: # Deleting the instance from EC2 ec2_id = instance['metadata']['ec2_id'] - ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id]) + try: + ec2_instances = self.ec2_conn.get_only_instances(instance_ids=[ec2_id]) + except Exception: + return if ec2_instances.__len__() == 0: LOG.warning(_("EC2 instance with ID %s not found") % ec2_id, instance=instance) return else: # get the elastic ip associated with the instance & disassociate # it, and release it - ec2_instance = ec2_instances[0] - elastic_ip_address = self.ec2_conn.get_all_addresses(addresses=[ec2_instance.ip_address])[0] + elastic_ip_address = self.ec2_conn.get_all_addresses(addresses=instance['metadata']['public_ip_address'])[0] LOG.info("****** Disassociating the elastic IP *********") self.ec2_conn.disassociate_address(elastic_ip_address.public_ip) diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index 9aeb336..55be1fe 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -1,5 +1,6 @@ import unittest import time +from boto.regioninfo import RegionInfo from novaclient.v1_1 import client from ..credentials import get_nova_creds @@ -8,15 +9,25 @@ from boto import ec2 from ..ec2driver_config import * import urllib2 + class EC2DriverTest(unittest.TestCase): _multiprocess_shared_ = True @classmethod def setUp(self): print "Establishing connection with AWS" - self.ec2_conn = ec2.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key) - # nova client for nova + # self.ec2_conn = ec2.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, + # aws_secret_access_key=aws_secret_access_key) + + moto_region = RegionInfo(name=aws_region, endpoint=aws_endpoint) + self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + host=host, + port=port, + region = moto_region, + is_secure=secure) + + self.creds = get_nova_creds() self.nova = client.Client(**self.creds) @@ -27,6 +38,9 @@ class EC2DriverTest(unittest.TestCase): self.volumes = [] def spawn_ec2_instance(self): + + print "aws_region: " + aws_region + print "Spawning an instance" image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") flavor = self.nova.flavors.find(name="m1.tiny") @@ -34,7 +48,7 @@ class EC2DriverTest(unittest.TestCase): name="cirros-test", image=image.id, flavor=flavor.id) instance = self.nova.servers.get(server.id) while instance.status != 'ACTIVE': - time.sleep(10) + # time.sleep(10) instance = self.nova.servers.get(server.id) self.servers.append(instance) return instance, server.id @@ -43,11 +57,12 @@ class EC2DriverTest(unittest.TestCase): print "******* Spawn Test ***********" instance, instance_ref = self.spawn_ec2_instance() - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None) + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']]) + ec2_eip = self.ec2_conn.get_all_addresses(addresses=instance.metadata['public_ip_address'])[0] self.assertEqual(ec2_instance[0].id, instance.metadata['ec2_id']) - self.assertEqual(ec2_instance[0].ip_address, instance.metadata['public_ip_address']) + self.assertEqual(ec2_eip.instance_id, instance.metadata['ec2_id']) + def test_destroy(self): print "******* Destroy Test ***********" @@ -55,27 +70,23 @@ class EC2DriverTest(unittest.TestCase): ec2_id = instance.metadata['ec2_id'] - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down - while ec2_instance[0].state != "running": - time.sleep(10) - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) + while ec2_instance.state != "running": + # time.sleep(10) + print "1" + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] instance.delete() - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down - while ec2_instance[0].state != "shutting-down": - time.sleep(10) - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) + # while ec2_instance[0].state != "shutting-down" or ec2_instance[0].state != "terminated": + while ec2_instance.state not in ("shutting-down", "terminated"): + # time.sleep(10) + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id], filters=None, dry_run=False, - max_results=None) + self.assertTrue(ec2_instance.state in ("shutting-down", "terminated")) - self.assertEquals(ec2_instance[0].state, "shutting-down") def test_power_off(self): print "******* Power Off Test ***********" @@ -84,12 +95,11 @@ class EC2DriverTest(unittest.TestCase): self.nova.servers.stop(instance) while instance.status != 'SHUTOFF': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) # assert power off - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None)[0] + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] self.assertEqual(ec2_instance.state, "stopped") def test_soft_reboot(self): @@ -106,12 +116,11 @@ class EC2DriverTest(unittest.TestCase): instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) #assert restarted - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None)[0] + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] self.assertEqual(ec2_instance.state, "running") def test_hard_reboot(self): @@ -123,26 +132,27 @@ class EC2DriverTest(unittest.TestCase): # we are waiting for the status to actually get to 'Hard Reboot' before # beginning to wait for it to go to 'Active' status while instance.status != 'HARD_REBOOT': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) #assert restarted - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None)[0] + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] self.assertEqual(ec2_instance.state, "running") def test_resize(self): print "******* Resize Test ***********" instance, instance_ref = self.spawn_ec2_instance() - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None)[0] + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] + + # ip_before_resize = ec2_instance.ip_address + + ip_before_resize = self.ec2_conn.get_all_addresses(addresses=instance.metadata['public_ip_address'])[0] - ip_before_resize = ec2_instance.ip_address self.assertEqual(ec2_instance.instance_type, "t2.micro") new_flavor = self.nova.flavors.find(name="m1.small") @@ -153,62 +163,63 @@ class EC2DriverTest(unittest.TestCase): # wait for the status to actually go to Verify_Resize, before # confirming the resize. while instance.status != 'VERIFY_RESIZE': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) # Confirm the resize self.nova.servers.confirm_resize(instance) while instance.status != 'ACTIVE': - time.sleep(5) + # time.sleep(5) instance = self.nova.servers.get(instance.id) - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None)[0] - ip_after_resize = ec2_instance.ip_address + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] + # ip_after_resize = ec2_instance.ip_address + ip_after_resize = self.ec2_conn.get_all_addresses(addresses=instance.metadata['public_ip_address'])[0] + self.assertEqual(ec2_instance.instance_type, "t2.small") - self.assertEqual(ip_before_resize, ip_after_resize, - "Public IP Address should be same before and after the resize") - def test_user_data(self): - """To test the spawn method by providing a file user_data for config drive. - Will bring up a LAMP server. - """ - content = open('user_data', 'r') - user_data_content = content.read() - image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") - flavor = self.nova.flavors.find(name="m1.tiny") - server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, - userdata=user_data_content) - instance = self.nova.servers.get(server.id) - while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: - time.sleep(10) - instance = self.nova.servers.get(server.id) - self.servers.append(instance) + self.assertEqual(ip_before_resize.public_ip, ip_after_resize.public_ip) - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None) - print ec2_instance - print ec2_instance[0].ip_address - #getting the public ip of the ec2 instance - url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" + # def test_user_data(self): + # """To test the spawn method by providing a file user_data for config drive. + # Will bring up a LAMP server. + # """ + # content = open('user_data', 'r') + # user_data_content = content.read() + # image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") + # flavor = self.nova.flavors.find(name="m1.tiny") + # server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, + # userdata=user_data_content) + # instance = self.nova.servers.get(server.id) + # while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: + # time.sleep(10) + # instance = self.nova.servers.get(server.id) + # self.servers.append(instance) + # + # ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, + # dry_run=False, max_results=None) + # print ec2_instance + # print ec2_instance[0].ip_address + # #getting the public ip of the ec2 instance + # url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" + # + # #wait for the instance to downalod all the dependencies for a LAMP server + # time.sleep(300) + # print url + # raw_response = urllib2.urlopen(url) + # print raw_response + # self.assertEqual(raw_response.code, 200) - #wait for the instance to downalod all the dependencies for a LAMP server - time.sleep(300) - print url - raw_response = urllib2.urlopen(url) - print raw_response - self.assertEqual(raw_response.code, 200) - - def test_diagnostics(self): - print "******* Diagnostics Test ***********" - instance, instance_ref = self.spawn_ec2_instance() - print "instance_ref: ", instance_ref - - diagnostics = instance.diagnostics()[1] - - self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') - self.assertEqual(diagnostics['instance._state'], 'running(16)') + # def test_diagnostics(self): + # print "******* Diagnostics Test ***********" + # instance, instance_ref = self.spawn_ec2_instance() + # print "instance_ref: ", instance_ref + # + # diagnostics = instance.diagnostics()[1] + # + # self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') + # self.assertEqual(diagnostics['instance._state'], 'running(16)') def test_attach_volume(self): volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) From 3dc04a0a187cb26984a2f59425f08e9e25055621 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Mon, 20 Oct 2014 17:19:48 -0500 Subject: [PATCH 08/20] Cam & Ed | Script to run tests with moto and openstack started --- start_openstack.sh | 21 +++++++++++++++++++++ tests/run_tests.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 start_openstack.sh create mode 100755 tests/run_tests.sh diff --git a/start_openstack.sh b/start_openstack.sh new file mode 100644 index 0000000..67c1642 --- /dev/null +++ b/start_openstack.sh @@ -0,0 +1,21 @@ +cd /opt/stack/keystone && /opt/stack/keystone/bin/keystone-all --config-file /etc/keystone/keystone.conf --debug & +cd /opt/stack/horizon && sudo tail -f /var/log/apache2/horizon_error.log & +cd /opt/stack/glance; /usr/local/bin/glance-registry --config-file=/etc/glance/glance-registry.conf & +cd /opt/stack/glance; /usr/local/bin/glance-api --config-file=/etc/glance/glance-api.conf & +cd /opt/stack/nova && /usr/local/bin/nova-api & +cd /opt/stack/nova && sg libvirtd '/usr/local/bin/nova-compute --config-file /etc/nova/nova.conf' & +cd /opt/stack/nova && /usr/local/bin/nova-conductor --config-file /etc/nova/nova.conf & +cd /opt/stack/nova && /usr/local/bin/nova-cert --config-file /etc/nova/nova.conf & +cd /opt/stack/nova && /usr/local/bin/nova-network --config-file /etc/nova/nova.conf & +cd /opt/stack/nova && /usr/local/bin/nova-scheduler --config-file /etc/nova/nova.conf & +cd /opt/stack/nova && /usr/local/bin/nova-novncproxy --config-file /etc/nova/nova.conf --web /opt/stack/noVNC +cd /opt/stack/nova && /usr/local/bin/nova-xvpvncproxy --config-file /etc/nova/nova.conf +cd /opt/stack/nova && /usr/local/bin/nova-consoleauth --config-file /etc/nova/nova.conf +cd /opt/stack/nova && /usr/local/bin/nova-objectstore --config-file /etc/nova/nova.conf & +cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-api --config-file /etc/cinder/cinder.conf & +cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-scheduler --config-file /etc/cinder/cinder.conf & +cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-volume --config-file /etc/cinder/cinder.conf & +cd /opt/stack/heat; bin/heat-engine --config-file=/etc/heat/heat.conf +cd /opt/stack/heat; bin/heat-api --config-file=/etc/heat/heat.conf +cd /opt/stack/heat; bin/heat-api-cfn --config-file=/etc/heat/heat.conf +cd /opt/stack/heat; bin/heat-api-cloudwatch --config-file=/etc/heat/heat.conf diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 0000000..be4a03c --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +#source ~/devstack/openrc admin +sh ~/devstack/openrc admin + +export TEST=True +ps aux | grep 'screen\|nova\|cinder\|horizon' | grep -v grep | awk '{print $2}' | xargs kill -9 +ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 + +echo "START MOTO" +moto_server ec2 -p1234 & + +echo "START OPENSTACK" +bash ../start_openstack.sh +#sh ../start_openstack.sh > testlog.txt 2>&1 & + +#/usr/bin/env bash ~/devstack/rejoin-stack.sh & + +sleep 10 + +echo "START TESTS" +nosetests -s test_ec2driver.py > test_output.txt 2>&1 +#nosetests -s test_ec2driver.py + +echo "KILL EVERYTHING" +ps aux | grep 'screen\|nova\|cinder\|horizon' | grep -v grep | awk '{print $2}' | xargs kill -9 +#ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 +unset TEST +echo 'DONE' \ No newline at end of file From 8760d36da52f0b421021f2f4b538079b906a5bf9 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Mon, 20 Oct 2014 17:31:10 -0500 Subject: [PATCH 09/20] Cam & Ed | Remove superfluous moto test --- tests/moto_test.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 tests/moto_test.py diff --git a/tests/moto_test.py b/tests/moto_test.py deleted file mode 100644 index 5ef8757..0000000 --- a/tests/moto_test.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import unittest -from boto import ec2 -from boto.regioninfo import RegionInfo -from ..ec2driver_config import * - -START_MOTO_SERVER_COMMAND = 'moto_server ec2 -p1234' -END_MOTO_SERVER_COMMAND = "ps aux | grep '%s' | grep -v grep | awk '{print $2}' | xargs kill -9"\ - % START_MOTO_SERVER_COMMAND - -class MotoTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - os.system(START_MOTO_SERVER_COMMAND + "&") - moto_region = RegionInfo(name='moto_region', endpoint="localhost:1234") - cls.ec2_conn = ec2.EC2Connection(aws_access_key_id='the_key', aws_secret_access_key='the_secret', - host='0.0.0.0', port='1234', - region = moto_region, - is_secure=False) - print cls.ec2_conn - - def test_should_create_an_instance(self): - reservation = self.ec2_conn.run_instances('ami-abcd1234') - ec2_instances = reservation.instances - self.assertEqual(len(ec2_instances), 1) - - @classmethod - def tearDownClass(cls): - os.system(END_MOTO_SERVER_COMMAND) - -if __name__ == '__main__': - unittest.main() - - From 159964e2f1ceca3e157759341a06394a44614302 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 10:35:01 -0500 Subject: [PATCH 10/20] Ed & Cam | Don't sleep in tests and skip tests that don't work with moto if using moto --- ec2driver.py | 7 +-- tests/test_ec2driver.py | 122 ++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 66 deletions(-) diff --git a/ec2driver.py b/ec2driver.py index abdc124..5af4e34 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -144,12 +144,9 @@ class EC2Driver(driver.ComputeDriver): port=port, region = moto_region, is_secure=secure) - self.cloudwatch_conn = None - # self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( - # aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - - # self.reservation = self.ec2_conn.get_all_reservations() + self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( + aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) self.security_group_lock = Lock() diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index 55be1fe..6954900 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -1,14 +1,19 @@ import unittest import time -from boto.regioninfo import RegionInfo - -from novaclient.v1_1 import client -from ..credentials import get_nova_creds - -from boto import ec2 -from ..ec2driver_config import * import urllib2 +from boto.regioninfo import RegionInfo +from novaclient.v1_1 import client +from boto import ec2 + +from ..credentials import get_nova_creds +from ..ec2driver_config import * + + +def sleep_if_ec2_not_mocked(seconds): + if not os.environ.get('TEST'): + time.sleep(seconds) + class EC2DriverTest(unittest.TestCase): _multiprocess_shared_ = True @@ -16,8 +21,6 @@ class EC2DriverTest(unittest.TestCase): @classmethod def setUp(self): print "Establishing connection with AWS" - # self.ec2_conn = ec2.connect_to_region(aws_region, aws_access_key_id=aws_access_key_id, - # aws_secret_access_key=aws_secret_access_key) moto_region = RegionInfo(name=aws_region, endpoint=aws_endpoint) self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, @@ -27,7 +30,6 @@ class EC2DriverTest(unittest.TestCase): region = moto_region, is_secure=secure) - self.creds = get_nova_creds() self.nova = client.Client(**self.creds) @@ -48,7 +50,7 @@ class EC2DriverTest(unittest.TestCase): name="cirros-test", image=image.id, flavor=flavor.id) instance = self.nova.servers.get(server.id) while instance.status != 'ACTIVE': - # time.sleep(10) + sleep_if_ec2_not_mocked(10) instance = self.nova.servers.get(server.id) self.servers.append(instance) return instance, server.id @@ -73,16 +75,14 @@ class EC2DriverTest(unittest.TestCase): ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down while ec2_instance.state != "running": - # time.sleep(10) - print "1" + sleep_if_ec2_not_mocked(10) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] instance.delete() ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down - # while ec2_instance[0].state != "shutting-down" or ec2_instance[0].state != "terminated": while ec2_instance.state not in ("shutting-down", "terminated"): - # time.sleep(10) + sleep_if_ec2_not_mocked(10) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] self.assertTrue(ec2_instance.state in ("shutting-down", "terminated")) @@ -95,7 +95,7 @@ class EC2DriverTest(unittest.TestCase): self.nova.servers.stop(instance) while instance.status != 'SHUTOFF': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) # assert power off @@ -116,7 +116,7 @@ class EC2DriverTest(unittest.TestCase): instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) #assert restarted @@ -132,11 +132,11 @@ class EC2DriverTest(unittest.TestCase): # we are waiting for the status to actually get to 'Hard Reboot' before # beginning to wait for it to go to 'Active' status while instance.status != 'HARD_REBOOT': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) #assert restarted @@ -149,8 +149,6 @@ class EC2DriverTest(unittest.TestCase): ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] - # ip_before_resize = ec2_instance.ip_address - ip_before_resize = self.ec2_conn.get_all_addresses(addresses=instance.metadata['public_ip_address'])[0] self.assertEqual(ec2_instance.instance_type, "t2.micro") @@ -163,14 +161,14 @@ class EC2DriverTest(unittest.TestCase): # wait for the status to actually go to Verify_Resize, before # confirming the resize. while instance.status != 'VERIFY_RESIZE': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) # Confirm the resize self.nova.servers.confirm_resize(instance) while instance.status != 'ACTIVE': - # time.sleep(5) + sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] @@ -181,45 +179,47 @@ class EC2DriverTest(unittest.TestCase): self.assertEqual(ip_before_resize.public_ip, ip_after_resize.public_ip) - # def test_user_data(self): - # """To test the spawn method by providing a file user_data for config drive. - # Will bring up a LAMP server. - # """ - # content = open('user_data', 'r') - # user_data_content = content.read() - # image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") - # flavor = self.nova.flavors.find(name="m1.tiny") - # server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, - # userdata=user_data_content) - # instance = self.nova.servers.get(server.id) - # while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: - # time.sleep(10) - # instance = self.nova.servers.get(server.id) - # self.servers.append(instance) - # - # ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - # dry_run=False, max_results=None) - # print ec2_instance - # print ec2_instance[0].ip_address - # #getting the public ip of the ec2 instance - # url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" - # - # #wait for the instance to downalod all the dependencies for a LAMP server - # time.sleep(300) - # print url - # raw_response = urllib2.urlopen(url) - # print raw_response - # self.assertEqual(raw_response.code, 200) + @unittest.skipIf(os.environ.get('TEST'), 'Not supported by moto') + def test_user_data(self): + """To test the spawn method by providing a file user_data for config drive. + Will bring up a LAMP server. + """ + content = open('user_data', 'r') + user_data_content = content.read() + image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") + flavor = self.nova.flavors.find(name="m1.tiny") + server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, + userdata=user_data_content) + instance = self.nova.servers.get(server.id) + while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: + sleep_if_ec2_not_mocked(10) + instance = self.nova.servers.get(server.id) + self.servers.append(instance) - # def test_diagnostics(self): - # print "******* Diagnostics Test ***********" - # instance, instance_ref = self.spawn_ec2_instance() - # print "instance_ref: ", instance_ref - # - # diagnostics = instance.diagnostics()[1] - # - # self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') - # self.assertEqual(diagnostics['instance._state'], 'running(16)') + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, + dry_run=False, max_results=None) + print ec2_instance + print ec2_instance[0].ip_address + #getting the public ip of the ec2 instance + url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" + + #wait for the instance to downalod all the dependencies for a LAMP server + sleep_if_ec2_not_mocked(300) + print url + raw_response = urllib2.urlopen(url) + print raw_response + self.assertEqual(raw_response.code, 200) + + @unittest.skipIf(os.environ.get('TEST'), 'Not supported by moto') + def test_diagnostics(self): + print "******* Diagnostics Test ***********" + instance, instance_ref = self.spawn_ec2_instance() + print "instance_ref: ", instance_ref + + diagnostics = instance.diagnostics()[1] + + self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') + self.assertEqual(diagnostics['instance._state'], 'running(16)') def test_attach_volume(self): volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) From dc298201e807912f1b67a30cbc1e34e704ae12e0 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 11:05:57 -0500 Subject: [PATCH 11/20] Ed & Cam | Scripts to start and end moto and setup environment for running moto tests --- .gitignore | 2 ++ start_openstack.sh | 21 --------------------- tests/run_tests.sh | 29 ----------------------------- tests/setup_moto.sh | 8 ++++++++ tests/shutdown_moto.sh | 7 +++++++ 5 files changed, 17 insertions(+), 50 deletions(-) delete mode 100644 start_openstack.sh delete mode 100755 tests/run_tests.sh create mode 100755 tests/setup_moto.sh create mode 100755 tests/shutdown_moto.sh diff --git a/.gitignore b/.gitignore index d6fbf63..453b431 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc .vagrant .idea +*.swp +tests/logs/ \ No newline at end of file diff --git a/start_openstack.sh b/start_openstack.sh deleted file mode 100644 index 67c1642..0000000 --- a/start_openstack.sh +++ /dev/null @@ -1,21 +0,0 @@ -cd /opt/stack/keystone && /opt/stack/keystone/bin/keystone-all --config-file /etc/keystone/keystone.conf --debug & -cd /opt/stack/horizon && sudo tail -f /var/log/apache2/horizon_error.log & -cd /opt/stack/glance; /usr/local/bin/glance-registry --config-file=/etc/glance/glance-registry.conf & -cd /opt/stack/glance; /usr/local/bin/glance-api --config-file=/etc/glance/glance-api.conf & -cd /opt/stack/nova && /usr/local/bin/nova-api & -cd /opt/stack/nova && sg libvirtd '/usr/local/bin/nova-compute --config-file /etc/nova/nova.conf' & -cd /opt/stack/nova && /usr/local/bin/nova-conductor --config-file /etc/nova/nova.conf & -cd /opt/stack/nova && /usr/local/bin/nova-cert --config-file /etc/nova/nova.conf & -cd /opt/stack/nova && /usr/local/bin/nova-network --config-file /etc/nova/nova.conf & -cd /opt/stack/nova && /usr/local/bin/nova-scheduler --config-file /etc/nova/nova.conf & -cd /opt/stack/nova && /usr/local/bin/nova-novncproxy --config-file /etc/nova/nova.conf --web /opt/stack/noVNC -cd /opt/stack/nova && /usr/local/bin/nova-xvpvncproxy --config-file /etc/nova/nova.conf -cd /opt/stack/nova && /usr/local/bin/nova-consoleauth --config-file /etc/nova/nova.conf -cd /opt/stack/nova && /usr/local/bin/nova-objectstore --config-file /etc/nova/nova.conf & -cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-api --config-file /etc/cinder/cinder.conf & -cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-scheduler --config-file /etc/cinder/cinder.conf & -cd /opt/stack/cinder && /opt/stack/cinder/bin/cinder-volume --config-file /etc/cinder/cinder.conf & -cd /opt/stack/heat; bin/heat-engine --config-file=/etc/heat/heat.conf -cd /opt/stack/heat; bin/heat-api --config-file=/etc/heat/heat.conf -cd /opt/stack/heat; bin/heat-api-cfn --config-file=/etc/heat/heat.conf -cd /opt/stack/heat; bin/heat-api-cloudwatch --config-file=/etc/heat/heat.conf diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100755 index be4a03c..0000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,29 +0,0 @@ -#! /usr/bin/env bash - -#source ~/devstack/openrc admin -sh ~/devstack/openrc admin - -export TEST=True -ps aux | grep 'screen\|nova\|cinder\|horizon' | grep -v grep | awk '{print $2}' | xargs kill -9 -ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 - -echo "START MOTO" -moto_server ec2 -p1234 & - -echo "START OPENSTACK" -bash ../start_openstack.sh -#sh ../start_openstack.sh > testlog.txt 2>&1 & - -#/usr/bin/env bash ~/devstack/rejoin-stack.sh & - -sleep 10 - -echo "START TESTS" -nosetests -s test_ec2driver.py > test_output.txt 2>&1 -#nosetests -s test_ec2driver.py - -echo "KILL EVERYTHING" -ps aux | grep 'screen\|nova\|cinder\|horizon' | grep -v grep | awk '{print $2}' | xargs kill -9 -#ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 -unset TEST -echo 'DONE' \ No newline at end of file diff --git a/tests/setup_moto.sh b/tests/setup_moto.sh new file mode 100755 index 0000000..5c6075a --- /dev/null +++ b/tests/setup_moto.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash + +echo "Setting environment variable TEST=True" +export TEST=True + +echo "Restarting moto" +ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 +moto_server ec2 -p1234 > logs/moto_log.txt 2>&1 & diff --git a/tests/shutdown_moto.sh b/tests/shutdown_moto.sh new file mode 100755 index 0000000..3df2178 --- /dev/null +++ b/tests/shutdown_moto.sh @@ -0,0 +1,7 @@ +#! /usr/bin/env bash + +echo "Unsetting environment variable TEST" +unset TEST + +echo "Killing moto" +ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 From 0fb3336b7c1348d6ac65edeb339592540076aa0e Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 11:16:10 -0500 Subject: [PATCH 12/20] Cam & Ed | Add section on running tests to README --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d54d93c..b2167c7 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,14 @@ Using the native OpenStack Dashboard or APIs you would be able to manage the EC2 The driver should now be loaded. The contents of the repository is mapped to `/opt/stack/nova/nova/virt/ec2/`, and you can edit it directly from your host computer with an IDE of your choice. -###Important notes +###Running Tests +1. To optionally mock the EC2 server with Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/setup_moto.sh` +2. `~/devstack/rejoin-stack.sh` +3. `cd /opt/stack/nova/nova/virt/ec2/tests` +4. `nosetests -s test_ec2driver.py` +5. To stop Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/shutdown_moto.sh` + +###Important Notes In Amazon’s EC2 there is no concept of suspend and resume on instances. Therefore, we simply stop EC2 instances when suspended and start the instances when resumed, we do the same on pause and un-pause. ##To Be Continued From e62357739ac4da63cb58265d8f49b2546cf7ad25 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 12:39:42 -0500 Subject: [PATCH 13/20] Ed & Cam | Update README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b2167c7..df4af6c 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,12 @@ Using the native OpenStack Dashboard or APIs you would be able to manage the EC2 The driver should now be loaded. The contents of the repository is mapped to `/opt/stack/nova/nova/virt/ec2/`, and you can edit it directly from your host computer with an IDE of your choice. ###Running Tests -1. To optionally mock the EC2 server with Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/setup_moto.sh` +1. Moto can be used to mock the EC2 server. To install moto, run `pip install moto`. +1. To optionally use Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/setup_moto.sh`. 2. `~/devstack/rejoin-stack.sh` 3. `cd /opt/stack/nova/nova/virt/ec2/tests` -4. `nosetests -s test_ec2driver.py` -5. To stop Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/shutdown_moto.sh` +4. Use `nosetests -s test_ec2driver.py` +5. To stop Moto, run `source /opt/stack/nova/nova/virt/ec2/tests/shutdown_moto.sh`. ###Important Notes In Amazon’s EC2 there is no concept of suspend and resume on instances. Therefore, we simply stop EC2 instances when suspended and start the instances when resumed, we do the same on pause and un-pause. From 11f9afe4fc2d579a23e9758833f4c23518ef9f88 Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 12:46:07 -0500 Subject: [PATCH 14/20] Cam & Ed | Fix endpoint/host/port config for ec2 connection --- ec2driver.py | 8 ++------ ec2driver_standard_config.py | 12 ++++++++---- ec2driver_test_config.py | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ec2driver.py b/ec2driver.py index 5af4e34..af034c9 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -133,16 +133,12 @@ class EC2Driver(driver.ComputeDriver): self.creds = get_nova_creds() self.nova = client.Client(**self.creds) - # To connect to EC2 - # self.ec2_conn = ec2.connect_to_region( - # aws_region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - - moto_region = RegionInfo(name=aws_region, endpoint=aws_endpoint) + region = RegionInfo(name=aws_region, endpoint=aws_endpoint) self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, host=host, port=port, - region = moto_region, + region=region, is_secure=secure) self.cloudwatch_conn = ec2.cloudwatch.connect_to_region( diff --git a/ec2driver_standard_config.py b/ec2driver_standard_config.py index e14ae58..984e425 100644 --- a/ec2driver_standard_config.py +++ b/ec2driver_standard_config.py @@ -16,9 +16,13 @@ # to connect to the AWS Public cloud. from collections import defaultdict -aws_region = "us-east-1" -aws_access_key_id = "AKIAIZJDDRNNJUWZ3LXA" -aws_secret_access_key = "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" +aws_region = 'us-east-1' +aws_endpoint = 'ec2.us-east-1.amazonaws.com' +aws_access_key_id = 'AKIAIZJDDRNNJUWZ3LXA' +aws_secret_access_key = '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' +port = 443 +host = str(port) + ":" + aws_endpoint +secure = True #Adding a Red Hat Linux image below aws_ami = "ami-785bae10" @@ -39,4 +43,4 @@ keypair_map = {} VCPUS = 100 MEMORY_IN_MBS = 88192 -DISK_IN_GB = 1028 \ No newline at end of file +DISK_IN_GB = 1028 diff --git a/ec2driver_test_config.py b/ec2driver_test_config.py index 0b17eeb..1f4fd51 100644 --- a/ec2driver_test_config.py +++ b/ec2driver_test_config.py @@ -17,11 +17,11 @@ from collections import defaultdict aws_region = "moto_region" -aws_endpoint = "localhost:1234" +aws_endpoint = "localhost" aws_access_key_id = 'the_key' aws_secret_access_key = 'the_secret' -host = '0.0.0.0' -port = '1234' +port = 1234 +host = str(port) + ":" + aws_endpoint secure = False #Adding a Red Hat Linux image below From 6c9b81765be7868d766d8f19477b736faf0ae2cd Mon Sep 17 00:00:00 2001 From: cameron-r Date: Tue, 21 Oct 2014 12:47:01 -0500 Subject: [PATCH 15/20] Cam & Ed | Change name of TEST environment variable to MOCK_EC2 --- ec2driver_config.py | 2 +- tests/setup_moto.sh | 4 ++-- tests/shutdown_moto.sh | 4 ++-- tests/test_ec2driver.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ec2driver_config.py b/ec2driver_config.py index 4fc24f5..b8e4186 100644 --- a/ec2driver_config.py +++ b/ec2driver_config.py @@ -1,6 +1,6 @@ import os -if os.environ.get('TEST'): +if os.environ.get('MOCK_EC2'): print "test environment" from ec2driver_test_config import * else: diff --git a/tests/setup_moto.sh b/tests/setup_moto.sh index 5c6075a..4c7afd5 100755 --- a/tests/setup_moto.sh +++ b/tests/setup_moto.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash -echo "Setting environment variable TEST=True" -export TEST=True +echo "Setting environment variable MOCK_EC2=True" +export MOCK_EC2=True echo "Restarting moto" ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 diff --git a/tests/shutdown_moto.sh b/tests/shutdown_moto.sh index 3df2178..a953e38 100755 --- a/tests/shutdown_moto.sh +++ b/tests/shutdown_moto.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash -echo "Unsetting environment variable TEST" -unset TEST +echo "Unsetting environment variable MOCK_EC2" +unset MOCK_EC2 echo "Killing moto" ps aux | grep moto_server | grep -v grep | awk '{print $2}' | xargs kill -9 diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index 6954900..e688920 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -11,7 +11,7 @@ from ..ec2driver_config import * def sleep_if_ec2_not_mocked(seconds): - if not os.environ.get('TEST'): + if not os.environ.get('MOCK_EC2'): time.sleep(seconds) From 06f56da8c65ec068930be10d590effbd9c0c8b14 Mon Sep 17 00:00:00 2001 From: Ed Thome Date: Tue, 21 Oct 2014 13:56:37 -0500 Subject: [PATCH 16/20] Ed & Cam | Create logs directory when starting moto --- tests/setup_moto.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/setup_moto.sh b/tests/setup_moto.sh index 4c7afd5..88ac5fa 100755 --- a/tests/setup_moto.sh +++ b/tests/setup_moto.sh @@ -1,5 +1,10 @@ #! /usr/bin/env bash +if [ ! -d "logs" ]; then + echo "Making logs directory" + mkdir logs +fi + echo "Setting environment variable MOCK_EC2=True" export MOCK_EC2=True From 8d0078e352b4d1cd7deaa99c98acb17aebe6976a Mon Sep 17 00:00:00 2001 From: Ed Thome Date: Tue, 21 Oct 2014 13:57:10 -0500 Subject: [PATCH 17/20] Cam & Ed | Fix skipped tests to use new environment variable --- tests/test_ec2driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index e688920..81fc445 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -179,7 +179,7 @@ class EC2DriverTest(unittest.TestCase): self.assertEqual(ip_before_resize.public_ip, ip_after_resize.public_ip) - @unittest.skipIf(os.environ.get('TEST'), 'Not supported by moto') + @unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto') def test_user_data(self): """To test the spawn method by providing a file user_data for config drive. Will bring up a LAMP server. @@ -210,7 +210,7 @@ class EC2DriverTest(unittest.TestCase): print raw_response self.assertEqual(raw_response.code, 200) - @unittest.skipIf(os.environ.get('TEST'), 'Not supported by moto') + @unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto') def test_diagnostics(self): print "******* Diagnostics Test ***********" instance, instance_ref = self.spawn_ec2_instance() From d5abe2973eac43c673bfd792b736c5b5cf66c0c7 Mon Sep 17 00:00:00 2001 From: Ed Thome Date: Tue, 21 Oct 2014 19:42:11 +0000 Subject: [PATCH 18/20] Cam & Ed | Extract setup/teardown/helper methods for tests into superclass --- ec2driver.py | 1 + tests/ec2_test_base.py | 66 ++++++++++++++++++ tests/test_ec2driver.py | 144 ++++++++++++---------------------------- 3 files changed, 110 insertions(+), 101 deletions(-) create mode 100644 tests/ec2_test_base.py diff --git a/ec2driver.py b/ec2driver.py index af034c9..0060f4b 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -16,6 +16,7 @@ """Connection to the Amazon Web Services - EC2 service""" from threading import Lock import base64 +import time from ec2driver_config import * from boto import ec2 import boto.ec2.cloudwatch diff --git a/tests/ec2_test_base.py b/tests/ec2_test_base.py new file mode 100644 index 0000000..527fcff --- /dev/null +++ b/tests/ec2_test_base.py @@ -0,0 +1,66 @@ +import os +import unittest +import time + +from boto.regioninfo import RegionInfo +from novaclient.v1_1 import client +from boto import ec2 + +from ..credentials import get_nova_creds +from ..ec2driver_config import * + +class EC2TestBase(unittest.TestCase): + @staticmethod + def sleep_if_ec2_not_mocked(seconds): + if not os.environ.get('MOCK_EC2'): + time.sleep(seconds) + + + @classmethod + def setUp(self): + print "Establishing connection with AWS" + + region = RegionInfo(name=aws_region, endpoint=aws_endpoint) + self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + host=host, + port=port, + region = region, + is_secure=secure) + + self.creds = get_nova_creds() + self.nova = client.Client(**self.creds) + + # nova client for cinder + self.creds['service_type'] = 'volume' + self.nova_volume = client.Client(**self.creds) + + self.servers = [] + self.volumes = [] + + @classmethod + def tearDown(self): + print "Cleanup: Destroying the instance used for testing" + for instance in self.servers: + instance.delete() + + # wait for all instances to completely shut down and detach volumes if any + self.sleep_if_ec2_not_mocked(120) + + for volume in self.volumes: + volume.delete() + + def spawn_ec2_instance(self): + print "aws_region: " + aws_region + + print "Spawning an instance" + image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") + flavor = self.nova.flavors.find(name="m1.tiny") + server = self.nova.servers.create( + name="cirros-test", image=image.id, flavor=flavor.id) + instance = self.nova.servers.get(server.id) + while instance.status != 'ACTIVE': + EC2TestBase.sleep_if_ec2_not_mocked(10) + instance = self.nova.servers.get(server.id) + self.servers.append(instance) + return instance, server.id \ No newline at end of file diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index 81fc445..cb3be18 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -1,59 +1,12 @@ import unittest -import time import urllib2 -from boto.regioninfo import RegionInfo from novaclient.v1_1 import client -from boto import ec2 - -from ..credentials import get_nova_creds from ..ec2driver_config import * +from ec2_test_base import EC2TestBase -def sleep_if_ec2_not_mocked(seconds): - if not os.environ.get('MOCK_EC2'): - time.sleep(seconds) - - -class EC2DriverTest(unittest.TestCase): - _multiprocess_shared_ = True - - @classmethod - def setUp(self): - print "Establishing connection with AWS" - - moto_region = RegionInfo(name=aws_region, endpoint=aws_endpoint) - self.ec2_conn = ec2.EC2Connection(aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - host=host, - port=port, - region = moto_region, - is_secure=secure) - - self.creds = get_nova_creds() - self.nova = client.Client(**self.creds) - - # nova client for cinder - self.creds['service_type'] = 'volume' - self.nova_volume = client.Client(**self.creds) - self.servers = [] - self.volumes = [] - - def spawn_ec2_instance(self): - - print "aws_region: " + aws_region - - print "Spawning an instance" - image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") - flavor = self.nova.flavors.find(name="m1.tiny") - server = self.nova.servers.create( - name="cirros-test", image=image.id, flavor=flavor.id) - instance = self.nova.servers.get(server.id) - while instance.status != 'ACTIVE': - sleep_if_ec2_not_mocked(10) - instance = self.nova.servers.get(server.id) - self.servers.append(instance) - return instance, server.id +class TestEC2Driver(EC2TestBase): def test_spawn(self): print "******* Spawn Test ***********" @@ -75,14 +28,14 @@ class EC2DriverTest(unittest.TestCase): ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down while ec2_instance.state != "running": - sleep_if_ec2_not_mocked(10) + self.sleep_if_ec2_not_mocked(10) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] instance.delete() ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] # EC2 statecode: 16->Running, 32->Shutting Down while ec2_instance.state not in ("shutting-down", "terminated"): - sleep_if_ec2_not_mocked(10) + self.sleep_if_ec2_not_mocked(10) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[ec2_id])[0] self.assertTrue(ec2_instance.state in ("shutting-down", "terminated")) @@ -95,7 +48,7 @@ class EC2DriverTest(unittest.TestCase): self.nova.servers.stop(instance) while instance.status != 'SHUTOFF': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) # assert power off @@ -116,7 +69,7 @@ class EC2DriverTest(unittest.TestCase): instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) #assert restarted @@ -132,11 +85,11 @@ class EC2DriverTest(unittest.TestCase): # we are waiting for the status to actually get to 'Hard Reboot' before # beginning to wait for it to go to 'Active' status while instance.status != 'HARD_REBOOT': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) while instance.status != 'ACTIVE': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) #assert restarted @@ -161,14 +114,14 @@ class EC2DriverTest(unittest.TestCase): # wait for the status to actually go to Verify_Resize, before # confirming the resize. while instance.status != 'VERIFY_RESIZE': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) # Confirm the resize self.nova.servers.confirm_resize(instance) while instance.status != 'ACTIVE': - sleep_if_ec2_not_mocked(5) + self.sleep_if_ec2_not_mocked(5) instance = self.nova.servers.get(instance.id) ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']])[0] @@ -181,64 +134,53 @@ class EC2DriverTest(unittest.TestCase): @unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto') def test_user_data(self): - """To test the spawn method by providing a file user_data for config drive. - Will bring up a LAMP server. - """ - content = open('user_data', 'r') - user_data_content = content.read() - image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") - flavor = self.nova.flavors.find(name="m1.tiny") - server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, - userdata=user_data_content) - instance = self.nova.servers.get(server.id) - while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: - sleep_if_ec2_not_mocked(10) - instance = self.nova.servers.get(server.id) - self.servers.append(instance) + """To test the spawn method by providing a file user_data for config drive. + Will bring up a LAMP server. + """ + content = open('user_data', 'r') + user_data_content = content.read() + image = self.nova.images.find(name="cirros-0.3.1-x86_64-uec") + flavor = self.nova.flavors.find(name="m1.tiny") + server = self.nova.servers.create(name="cirros-test", image=image.id, flavor=flavor.id, + userdata=user_data_content) + instance = self.nova.servers.get(server.id) + while instance.status != 'ACTIVE' and 'ec2_id' not in instance.metadata: + self.sleep_if_ec2_not_mocked(10) + instance = self.nova.servers.get(server.id) + self.servers.append(instance) - ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, - dry_run=False, max_results=None) - print ec2_instance - print ec2_instance[0].ip_address - #getting the public ip of the ec2 instance - url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" + ec2_instance = self.ec2_conn.get_only_instances(instance_ids=[instance.metadata['ec2_id']], filters=None, + dry_run=False, max_results=None) + print ec2_instance + print ec2_instance[0].ip_address + #getting the public ip of the ec2 instance + url = "http://"+ec2_instance[0].ip_address+"/phpinfo.php" - #wait for the instance to downalod all the dependencies for a LAMP server - sleep_if_ec2_not_mocked(300) - print url - raw_response = urllib2.urlopen(url) - print raw_response - self.assertEqual(raw_response.code, 200) + #wait for the instance to downalod all the dependencies for a LAMP server + self.sleep_if_ec2_not_mocked(300) + print url + raw_response = urllib2.urlopen(url) + print raw_response + self.assertEqual(raw_response.code, 200) @unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto') def test_diagnostics(self): - print "******* Diagnostics Test ***********" - instance, instance_ref = self.spawn_ec2_instance() - print "instance_ref: ", instance_ref + print "******* Diagnostics Test ***********" + instance, instance_ref = self.spawn_ec2_instance() + print "instance_ref: ", instance_ref - diagnostics = instance.diagnostics()[1] - - self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') - self.assertEqual(diagnostics['instance._state'], 'running(16)') + diagnostics = instance.diagnostics()[1] + self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') + self.assertEqual(diagnostics['instance._state'], 'running(16)') def test_attach_volume(self): volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) self.volumes.append(volume) instance, instance_ref = self.spawn_ec2_instance() self.nova.volumes.create_server_volume(instance_ref, volume.id, "/dev/sdb") - time.sleep(30) + self.sleep_if_ec2_not_mocked(30) volumes = self.nova.volumes.get_server_volumes(instance.id) self.assertIn(volume, volumes) - @classmethod - def tearDown(self): - print "Cleanup: Destroying the instance used for testing" - for instance in self.servers: - instance.delete() - # wait for all instances to completely shut down and detach volumes if any - time.sleep(120) - for volume in self.volumes: - volume.delete() - if __name__ == '__main__': unittest.main() From 0a43cf2b3d32829692c2c882d601775671f32b27 Mon Sep 17 00:00:00 2001 From: Ed Thome Date: Tue, 21 Oct 2014 21:55:39 +0000 Subject: [PATCH 19/20] Cam & Ed | Renaming, cleaning up imports --- ec2driver.py | 31 ++++++++++++++++++++----------- tests/ec2_test_base.py | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ec2driver.py b/ec2driver.py index 0060f4b..e834ce4 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -19,7 +19,6 @@ import base64 import time from ec2driver_config import * from boto import ec2 -import boto.ec2.cloudwatch from boto import exception as boto_exc from boto.exception import EC2ResponseError from credentials import get_nova_creds @@ -27,6 +26,7 @@ from credentials import get_nova_creds from boto.regioninfo import RegionInfo from oslo.config import cfg from novaclient.v1_1 import client +from ec2driver_config import * from nova import block_device from nova.compute import power_state from nova.compute import task_states @@ -39,6 +39,8 @@ from nova.openstack.common import loopingcall from nova.virt import driver from nova.virt import virtapi from nova.compute import flavors +from novaclient.v1_1 import client +from credentials import get_nova_creds LOG = logging.getLogger(__name__) @@ -533,10 +535,10 @@ class EC2Driver(driver.ComputeDriver): 'username': 'EC2user', 'password': 'EC2password'} - def _get_ec2_instance_ids_for_security_group(self, ec2_security_group): + def _get_ec2_instance_ids_with_security_group(self, ec2_security_group): return [instance.id for instance in ec2_security_group.instances()] - def _get_openstack_instances_for_security_group(self, openstack_security_group): + def _get_openstack_instances_with_security_group(self, openstack_security_group): return [instance for instance in (self.nova.servers.list()) if openstack_security_group.name in [group['name'] for group in instance.security_groups]] @@ -574,21 +576,28 @@ class EC2Driver(driver.ComputeDriver): openstack_security_group = self.nova.security_groups.get(security_group_id) ec2_security_group = self._get_or_create_ec2_security_group(openstack_security_group) - ec2_instance_ids_for_security_group = self._get_ec2_instance_ids_for_security_group(ec2_security_group) - ec2_ids_for_openstack_instances_for_security_group = [ + ec2_ids_for_ec2_instances_with_security_group = self._get_ec2_instance_ids_with_security_group(ec2_security_group) + ec2_ids_for_openstack_instances_with_security_group = [ instance.metadata['ec2_id'] for instance - in self._get_openstack_instances_for_security_group(openstack_security_group) + in self._get_openstack_instances_with_security_group(openstack_security_group) ] self.security_group_lock.acquire() try: - if self._should_add_security_group_to_instance(ec2_instance_ids_for_security_group, ec2_ids_for_openstack_instances_for_security_group): - ec2_instance_id_to_add_security_group = self._get_id_of_ec2_instance_to_update_security_group(ec2_instance_ids_for_security_group, ec2_ids_for_openstack_instances_for_security_group) - self._add_security_group_to_instance(ec2_instance_id_to_add_security_group, ec2_security_group) + ec2_instance_to_update = self._get_id_of_ec2_instance_to_update_security_group( + ec2_ids_for_ec2_instances_with_security_group, + ec2_ids_for_openstack_instances_with_security_group + ) + + should_add_security_group = self._should_add_security_group_to_instance( + ec2_ids_for_ec2_instances_with_security_group, + ec2_ids_for_openstack_instances_with_security_group) + + if should_add_security_group: + self._add_security_group_to_instance(ec2_instance_to_update, ec2_security_group) else: - ec2_instance_id_to_remove_security_group = self._get_id_of_ec2_instance_to_update_security_group(ec2_instance_ids_for_security_group, ec2_ids_for_openstack_instances_for_security_group) - self._remove_security_group_from_instance(ec2_instance_id_to_remove_security_group, ec2_security_group) + self._remove_security_group_from_instance(ec2_instance_to_update, ec2_security_group) finally: self.security_group_lock.release() diff --git a/tests/ec2_test_base.py b/tests/ec2_test_base.py index 527fcff..72717c5 100644 --- a/tests/ec2_test_base.py +++ b/tests/ec2_test_base.py @@ -1,4 +1,3 @@ -import os import unittest import time @@ -9,6 +8,7 @@ from boto import ec2 from ..credentials import get_nova_creds from ..ec2driver_config import * + class EC2TestBase(unittest.TestCase): @staticmethod def sleep_if_ec2_not_mocked(seconds): From be513ea8cef86114b149fd0533b8065f32c5ad0e Mon Sep 17 00:00:00 2001 From: Ed Thome Date: Wed, 22 Oct 2014 19:32:31 +0000 Subject: [PATCH 20/20] Cam & Ed | Fix cloudwatch import, skip attach volume test with moto --- ec2driver.py | 9 ++++----- tests/test_ec2driver.py | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ec2driver.py b/ec2driver.py index e834ce4..0e3ea35 100644 --- a/ec2driver.py +++ b/ec2driver.py @@ -17,15 +17,15 @@ from threading import Lock import base64 import time -from ec2driver_config import * + from boto import ec2 +import boto.ec2.cloudwatch from boto import exception as boto_exc from boto.exception import EC2ResponseError -from credentials import get_nova_creds - from boto.regioninfo import RegionInfo from oslo.config import cfg from novaclient.v1_1 import client + from ec2driver_config import * from nova import block_device from nova.compute import power_state @@ -38,10 +38,9 @@ from nova.openstack.common import log as logging from nova.openstack.common import loopingcall from nova.virt import driver from nova.virt import virtapi -from nova.compute import flavors -from novaclient.v1_1 import client from credentials import get_nova_creds + LOG = logging.getLogger(__name__) ec2driver_opts = [ diff --git a/tests/test_ec2driver.py b/tests/test_ec2driver.py index cb3be18..2441d62 100644 --- a/tests/test_ec2driver.py +++ b/tests/test_ec2driver.py @@ -173,8 +173,10 @@ class TestEC2Driver(EC2TestBase): self.assertEqual(diagnostics['instance.instance_type'], 't2.micro') self.assertEqual(diagnostics['instance._state'], 'running(16)') + @unittest.skipIf(os.environ.get('MOCK_EC2'), 'Not supported by moto') def test_attach_volume(self): - volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, volume_type=None, availability_zone=None, imageRef=None) + volume = self.nova_volume.volumes.create(1, snapshot_id=None, display_name='test', display_description=None, + volume_type=None, availability_zone=None, imageRef=None) self.volumes.append(volume) instance, instance_ref = self.spawn_ec2_instance() self.nova.volumes.create_server_volume(instance_ref, volume.id, "/dev/sdb")