openstack
/
ec2-api
Code Issues Proposed changes

add metadata test and fix metadata logic 

added ssh.py file is temporary solution until such class
is absent in tempest-lib

Change-Id: I6e4638ed969a30311db03a9c72cbdcae47c179e8
This commit is contained in:
Andrey Pavlov 2015-03-22 22:12:13 +03:00
parent e31c1c5c10
commit 4b2924c02e
11 changed files with 350 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ec2api/api/instance.py
View File

@ -76,6 +76,9 @@ def run_instances(context, image_id, min_count, max_count,
if idempotent_run:
return idempotent_run
if user_data:
user_data = base64.b64decode(user_data)
return instance_engine.run_instances(
context, image_id, min_count, max_count,
key_name, security_group_id,
@ -355,8 +358,7 @@ def describe_instance_attribute(context, instance_id, attribute):
raise exception.InvalidAttribute(attr=attribute)
user_data = getattr(os_instance, 'OS-EXT-SRV-ATTR:user_data')
if user_data:
value = base64.b64decode(user_data)
result['userData'] = {'value': value}
result['userData'] = {'value': user_data}
attribute_formatter = {
'blockDeviceMapping': _format_attr_block_device_mapping,

9
ec2api/metadata/api.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import itertools
from novaclient import exceptions as nova_exception
@ -179,17 +180,17 @@ def _build_metadata(context, ec2_instance, ec2_reservation,
# meta-data/public-keys/0/ : 'openssh-key'
# meta-data/public-keys/0/openssh-key : '%s' % publickey
if ec2_instance['keyName']:
keypair = clients.nova(context).keypairs.get(ec2_instance['keyName'])
metadata['public-keys'] = {
'0': {'_name': "0=" + keypair.name,
'openssh-key': keypair.public_key}}
# TODO(andrey-mp): public key should be added in the future
'0': {'_name': "0=" + ec2_instance['keyName']}}
full_metadata = {'meta-data': metadata}
userdata = instance_api.describe_instance_attribute(
context, ec2_instance['instanceId'], 'userData')
if 'userData' in userdata:
full_metadata['user-data'] = userdata['userData']['value']
userdata = userdata['userData']['value']
full_metadata['user-data'] = base64.b64decode(userdata)
return full_metadata

3
ec2api/tests/functional/api/test_addresses.py
View File

@ -379,6 +379,8 @@ class AddressTest(base.EC2TestCase):
resp, data = self.client.AssociateAddress(InstanceId=instance_id,
PublicIp=ip)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
clean_aa = self.addResourceCleanUp(self.client.DisassociateAddress,
PublicIp=ip)
resp, data = self.client.DescribeAddresses(*[], **{})
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
@ -386,6 +388,7 @@ class AddressTest(base.EC2TestCase):
resp, data = self.client.DisassociateAddress(PublicIp=ip)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
self.cancelResourceCleanUp(clean_aa)
# NOTE(andrey-mp): Amazon needs some time to diassociate
time.sleep(2)

3
ec2api/tests/functional/config.py
View File

@ -62,6 +62,9 @@ AWSGroup = [
cfg.StrOpt('ebs_image_id',
default=None,
help="EBS Image ID for testing snapshots, volumes, instances"),
cfg.StrOpt('image_user',
default='cirros',
help="User for sshing into instance based on configured image"),
cfg.BoolOpt('run_incompatible_tests',
default=False,
help='Will run all tests plus incompatible with Amazon.'),

101
ec2api/tests/functional/scenario/base.py Normal file
View File

@ -0,0 +1,101 @@
# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo_log import log
from tempest_lib.common.utils import data_utils
from ec2api.tests.functional import base
from ec2api.tests.functional import config
CONF = config.CONF
LOG = log.getLogger(__name__)
class BaseScenarioTest(base.EC2TestCase):
def run_instance(self, **kwargs):
kwargs.setdefault('ImageId', CONF.aws.image_id)
kwargs.setdefault('InstanceType', CONF.aws.instance_type)
kwargs.setdefault('Placement', {'AvailabilityZone': CONF.aws.aws_zone})
kwargs['MinCount'] = 1
kwargs['MaxCount'] = 1
resp, data = self.client.RunInstances(*[], **kwargs)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
instance_id = data['Instances'][0]['InstanceId']
self.addResourceCleanUp(self.client.TerminateInstances,
InstanceIds=[instance_id])
self.get_instance_waiter().wait_available(instance_id,
final_set=('running'))
return instance_id
def get_instance_ip(self, instance_id):
instance = self.get_instance(instance_id)
public_ip = instance.get('PublicIpAddress')
if public_ip:
return public_ip
resp, data = self.client.AllocateAddress(*[], **{})
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
public_ip = data['PublicIp']
self.addResourceCleanUp(self.client.ReleaseAddress, PublicIp=public_ip)
resp, data = self.client.AssociateAddress(InstanceId=instance_id,
PublicIp=public_ip)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
self.addResourceCleanUp(self.client.DisassociateAddress,
PublicIp=public_ip)
return public_ip
def create_key_pair(self, key_name):
resp, data = self.client.CreateKeyPair(KeyName=key_name)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
self.addResourceCleanUp(self.client.DeleteKeyPair, KeyName=key_name)
return data.get('KeyMaterial')
def prepare_security_group(self):
name = data_utils.rand_name('sgName')
desc = data_utils.rand_name('sgDesc')
resp, data = self.client.CreateSecurityGroup(GroupName=name,
Description=desc)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
self.addResourceCleanUp(self.client.DeleteSecurityGroup,
GroupName=name)
time.sleep(2)
kwargs = {
'GroupName': name,
'IpPermissions': [{
'IpProtocol': 'icmp',
'FromPort': -1,
'ToPort': -1,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}],
}, {
'IpProtocol': 'tcp',
'FromPort': 22,
'ToPort': 22,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}],
}]
}
resp, data = self.client.AuthorizeSecurityGroupIngress(*[], **kwargs)
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
return name

58
ec2api/tests/functional/scenario/test_metadata.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import base64
from oslo_log import log
from tempest_lib.common.utils import data_utils
from ec2api.tests.functional import base
from ec2api.tests.functional import config
from ec2api.tests.functional.scenario import base as scenario_base
from ec2api.tests.functional import ssh
CONF = config.CONF
LOG = log.getLogger(__name__)
class MetadataTest(scenario_base.BaseScenarioTest):
def test_metadata(self):
if not CONF.aws.image_id:
raise self.skipException('aws image_id does not provided')
key_name = data_utils.rand_name('testkey')
pkey = self.create_key_pair(key_name)
sec_group_name = self.prepare_security_group()
user_data = data_utils.rand_uuid()
instance_id = self.run_instance(KeyName=key_name, UserData=user_data,
SecurityGroups=[sec_group_name])
resp, data = self.client.DescribeInstanceAttribute(
InstanceId=instance_id, Attribute='userData')
self.assertEqual(200, resp.status_code, base.EC2ErrorConverter(data))
self.assertEqual(data['UserData']['Value'],
base64.b64encode(user_data))
ip_address = self.get_instance_ip(instance_id)
ssh_client = ssh.Client(ip_address, CONF.aws.image_user, pkey=pkey)
url = 'http://169.254.169.254'
data = ssh_client.exec_command('curl %s/latest/user-data' % url)
self.assertEqual(user_data, data)
data = ssh_client.exec_command('curl %s/latest/meta-data/ami-id' % url)
self.assertEqual(CONF.aws.image_id, data)

161
ec2api/tests/functional/ssh.py Normal file
View File

@ -0,0 +1,161 @@
# Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import cStringIO
import select
import socket
import time
import warnings
from oslo_log import log as logging
import six
from tempest_lib import exceptions
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import paramiko
LOG = logging.getLogger(__name__)
class SSHTimeout(exceptions.TempestException):
message = ("Connection to the %(host)s via SSH timed out.\n"
"User: %(user)s, Password: %(password)s")
class SSHExecCommandFailed(exceptions.TempestException):
"""Raised when remotely executed command returns nonzero status."""
message = ("Command '%(command)s', exit status: %(exit_status)d, "
"Error:\n%(strerror)s")
class Client(object):
def __init__(self, host, username, password=None, timeout=300, pkey=None,
channel_timeout=10, look_for_keys=False, key_filename=None):
self.host = host
self.username = username
self.password = password
if isinstance(pkey, six.string_types):
pkey = paramiko.RSAKey.from_private_key(
cStringIO.StringIO(str(pkey)))
self.pkey = pkey
self.look_for_keys = look_for_keys
self.key_filename = key_filename
self.timeout = int(timeout)
self.channel_timeout = float(channel_timeout)
self.buf_size = 1024
def _get_ssh_connection(self, sleep=1.5, backoff=1):
"""Returns an ssh connection to the specified host."""
bsleep = sleep
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
_start_time = time.time()
if self.pkey is not None:
LOG.info("Creating ssh connection to '%s' as '%s'"
" with public key authentication",
self.host, self.username)
else:
LOG.info("Creating ssh connection to '%s' as '%s'"
" with password %s",
self.host, self.username, str(self.password))
attempts = 0
while True:
try:
ssh.connect(self.host, username=self.username,
password=self.password,
look_for_keys=self.look_for_keys,
key_filename=self.key_filename,
timeout=self.channel_timeout, pkey=self.pkey)
LOG.info("ssh connection to %s@%s successfuly created",
self.username, self.host)
return ssh
except (socket.error,
paramiko.SSHException) as e:
if self._is_timed_out(_start_time):
LOG.exception("Failed to establish authenticated ssh"
" connection to %s@%s after %d attempts",
self.username, self.host, attempts)
raise SSHTimeout(host=self.host,
user=self.username,
password=self.password)
bsleep += backoff
attempts += 1
LOG.warning("Failed to establish authenticated ssh"
" connection to %s@%s (%s). Number attempts: %s."
" Retry after %d seconds.",
self.username, self.host, e, attempts, bsleep)
time.sleep(bsleep)
def _is_timed_out(self, start_time):
return (time.time() - self.timeout) > start_time
def exec_command(self, cmd):
"""Execute the specified command on the server
Note that this method is reading whole command outputs to memory, thus
shouldn't be used for large outputs.
:returns: data read from standard output of the command.
:raises: SSHExecCommandFailed if command returns nonzero
status. The exception contains command status stderr content.
"""
ssh = self._get_ssh_connection()
transport = ssh.get_transport()
channel = transport.open_session()
channel.fileno() # Register event pipe
channel.exec_command(cmd)
channel.shutdown_write()
out_data = []
err_data = []
poll = select.poll()
poll.register(channel, select.POLLIN)
start_time = time.time()
while True:
ready = poll.poll(self.channel_timeout)
if not any(ready):
if not self._is_timed_out(start_time):
continue
raise exceptions.TimeoutException(
"Command: '{0}' executed on host '{1}'.".format(
cmd, self.host))
if not ready[0]: # If there is nothing to read.
continue
out_chunk = err_chunk = None
if channel.recv_ready():
out_chunk = channel.recv(self.buf_size)
out_data += out_chunk,
if channel.recv_stderr_ready():
err_chunk = channel.recv_stderr(self.buf_size)
err_data += err_chunk,
if channel.closed and not err_chunk and not out_chunk:
break
exit_status = channel.recv_exit_status()
if 0 != exit_status:
raise SSHExecCommandFailed(
command=cmd, exit_status=exit_status,
strerror=''.join(err_data))
return ''.join(out_data)
def test_connection_auth(self):
"""Raises an exception when we can not connect to server via ssh."""
connection = self._get_ssh_connection()
connection.close()

4
ec2api/tests/unit/fakes.py
View File

@ -115,7 +115,7 @@ ROOT_DEVICE_NAME_INSTANCE_1 = '/dev/vda'
ROOT_DEVICE_NAME_INSTANCE_2 = '/dev/sdb1'
IPV6_INSTANCE_2 = 'fe80:b33f::a8bb:ccff:fedd:eeff'
CLIENT_TOKEN_INSTANCE_2 = 'client-token-2'
USER_DATA_INSTANCE_2 = 'fake-user data'
USER_DATA_INSTANCE_2 = base64.b64encode('fake-user data')
# DHCP options constants
@ -660,7 +660,7 @@ OS_INSTANCE_2 = {
'root_device_name': ROOT_DEVICE_NAME_INSTANCE_2,
'volumes_attached': [{'id': ID_OS_VOLUME_2,
'delete_on_termination': False}],
'user_data': base64.b64encode(USER_DATA_INSTANCE_2),
'user_data': USER_DATA_INSTANCE_2,
'hostname': 'Server %s' % ID_OS_INSTANCE_2,
}

8
ec2api/tests/unit/test_instance.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import copy
import datetime
import itertools
@ -324,6 +325,8 @@ class InstanceTestCase(base.ApiTestCase):
def do_check(engine, extra_kwargs={}, extra_db_instance={}):
instance_api.instance_engine = engine
user_data = base64.b64decode(fakes.USER_DATA_INSTANCE_2)
self.execute(
'RunInstances',
{'ImageId': fakes.ID_EC2_IMAGE_1,
@ -337,11 +340,12 @@ class InstanceTestCase(base.ApiTestCase):
'BlockDeviceMapping.1.DeviceName': '/dev/vdd',
'BlockDeviceMapping.1.Ebs.SnapshotId': (
fakes.ID_EC2_SNAPSHOT_1),
'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'False'})
'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'False',
'UserData': fakes.USER_DATA_INSTANCE_2})
self.nova.servers.create.assert_called_once_with(
mock.ANY, mock.ANY, mock.ANY, min_count=1, max_count=1,
userdata=None, kernel_id=fakes.ID_OS_IMAGE_AKI_1,
userdata=user_data, kernel_id=fakes.ID_OS_IMAGE_AKI_1,
ramdisk_id=fakes.ID_OS_IMAGE_ARI_1, key_name=None,
block_device_mapping='fake_bdm',
availability_zone='fake_zone', security_groups=['default'],

12
ec2api/tests/unit/test_metadata_api.py
View File

@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import mock
from novaclient import exceptions as nova_exception
@ -39,7 +41,7 @@ class MetadataApiTestCase(base.ApiTestCase):
'reservationSet': [fakes.EC2_RESERVATION_1]}
self.instance_api.describe_instance_attribute.return_value = {
'instanceId': fakes.ID_EC2_INSTANCE_1,
'userData': {'value': 'fake_user_data'}}
'userData': {'value': base64.b64encode('fake_user_data')}}
self.fake_context = self._create_context()
@ -166,17 +168,15 @@ class MetadataApiTestCase(base.ApiTestCase):
fakes.ID_OS_INSTANCE_2, fakes.IP_NETWORK_INTERFACE_1)
self.assertEqual(fakes.IP_NETWORK_INTERFACE_1, retval)
@mock.patch('novaclient.client.Client')
def test_pubkey(self, nova):
keypair = mock.Mock(public_key=fakes.PUBLIC_KEY_KEY_PAIR)
keypair.configure_mock(name=fakes.NAME_KEY_PAIR)
nova.return_value.keypairs.get.return_value = keypair
def test_pubkey_name(self):
retval = api.get_metadata_item(
self.fake_context,
['2009-04-04', 'meta-data', 'public-keys'],
fakes.ID_OS_INSTANCE_1, fakes.IP_NETWORK_INTERFACE_2)
self.assertEqual('0=%s' % fakes.NAME_KEY_PAIR, retval)
@base.skip_not_implemented
def test_pubkey(self):
retval = api.get_metadata_item(
self.fake_context,
['2009-04-04', 'meta-data', 'public-keys', '0', 'openssh-key'],

1
requirements.txt
View File

@ -15,6 +15,7 @@ oslo.log>=0.1.0 # Apache-2.0
oslo.utils>=1.2.0 # Apache-2.0
oslo.serialization>=1.2.0 # Apache-2.0
oslo.db>=1.4.1 # Apache-2.0
paramiko>=1.13.0
Paste
PasteDeploy>=1.5.0
pbr>=0.6,!=0.7,<1.0