openstack
/
fuel-devops
Code Issues Proposed changes

Reformat code

This commit is contained in:
vic 2013-01-22 17:50:48 +04:00
parent e641646dac
commit c602a04fc0
15 changed files with 105 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
samples/node_ip_address_integration.py
View File

@ -1,42 +0,0 @@
import unittest
from src import devops
from src.devops.helpers import wait, TimeoutError
import time
ISO_URL = 'http://mc0n1-srt.srt.mirantis.net/livecd.iso'
class TestNodeIpAddress(unittest.TestCase):
def setUp(self):
self.env = devops.load("""
networks:
- network: net1
dhcp_server: True
nodes:
- node: foo
networks: net1
cdrom: '%s'
vnc: True
""" % ISO_URL)
devops.build(self.env)
def tearDown(self):
devops.destroy(self.env)
def test_ip_address_detection(self):
node = self.env.nodes[0]
network = self.env.networks[0]
node.start()
# Wait for ISOLINUX to boot
time.sleep(10)
# Trigger ISOLINUX menu selection
node.send_keys('<Enter>')
try:
wait(lambda: len(node.ip_addresses) > 0, timeout=60)
except TimeoutError:
self.fail("Node didn't get ip_address in specified amount of time")
self.assertEqual(node.ip_address, node.ip_addresses[0])
self.assertTrue(node.ip_address in network.ip_addresses)

10
samples/one.py
View File

@ -1,12 +1,12 @@
import ipaddr
import os
def one(manager):
environment = manager.environment_create('test_env7')
internal_pool=manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
internal_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
)
private_pool=manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
private_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('10.108.0.0/16')], prefix=24
)
external_pool=manager.create_network_pool(networks=[ipaddr.IPNetwork('172.18.95.0/24')], prefix=27
external_pool = manager.create_network_pool(networks=[ipaddr.IPNetwork('172.18.95.0/24')], prefix=27
)
internal = manager.network_create(
environment=environment, name='internal', pool=internal_pool)
@ -27,8 +27,8 @@ def one(manager):
environment.start()
if __name__ == '__main__':
from devops.manager import Manager
one(Manager())

96
samples/run.py
View File

@ -1,96 +0,0 @@
import time
from src import devops
from src.devops.helpers import wait, tcp_ping
import logging
MASTER_AND_SLAVE_CONFIG = """
name: 'Sample environment'
networks:
- network: internal
- network: external
nodes:
- node: master
disk: '5Gb'
cdrom: http://mc0n1-srt.srt.mirantis.net/nailgun-ubuntu-12.04-amd64.last.iso
networks: ['external', 'internal']
vnc: True
- node: slave
networks: ['internal']
vnc: True
"""
def main():
logging.basicConfig(level=logging.WARN)
logger = logging.getLogger('test.integration')
logger.setLevel(logging.INFO)
environment = devops.load(MASTER_AND_SLAVE_CONFIG)
logger.info("Building environment")
devops.build(environment)
logger.info("Environment ready")
try:
external_network = environment.network['external']
master_node = environment.node['master']
slave_node = environment.node['slave']
logger.info("Starting master node")
master_node.start()
logger.info("VNC to master is available on %d" % master_node.vnc_port)
logger.info("Waiting master node to boot")
time.sleep(15)
logger.info("Sending user input")
ip = external_network.ip_addresses
host_ip = ip[1]
master_ip = ip[2]
netmask = ip.netmask
master_node.send_keys("""<Esc><Enter>
<Wait>
/install/vmlinuz initrd=/install/initrd.gz
priority=critical
locale=en_US
file=/cdrom/preseed/manual.seed
vga=788
netcfg/get_ipaddress=%s
netcfg/get_netmask=%s
netcfg/get_gateway=%s
netcfg/get_nameservers=%s
netcfg/confirm_static=true
<Enter>""" % (master_ip, netmask, host_ip, host_ip))
logger.info("Finished sending user input")
logger.info("Waiting master node to install")
wait(lambda: tcp_ping(master_ip, 22))
logger.info("Starting slave node")
slave_node.start()
logger.info("VNC to slave node at port %d" % slave_node.vnc_port)
logger.info("Waiting slave node to configure network")
wait(lambda: len(slave_node.ip_addresses) > 0, timeout=120)
logger.info(
"Slave node has IP address %s" % slave_node.ip_addresses[0])
except:
devops.save(environment)
logger.warn("Environment has been saved as %s" % environment.id)
raise
if __name__ == '__main__':
main()

13
src/devops/driver/libvirt/libvirt_driver.py
View File

@ -68,7 +68,7 @@ class LibvirtDriver(object):
self.conn.networkLookupByUUIDString(network.uuid)
return True
except libvirt.libvirtError, e:
if e.message == 'virNetworkLookupByUUIDString() failed':
if e.message == 'virNetworkLookupByUUIDString() failed':
return False
raise
@ -82,7 +82,7 @@ class LibvirtDriver(object):
self.conn.lookupByUUIDString(node.uuid)
return True
except libvirt.libvirtError, e:
if e.message == 'virDomainLookupByUUIDString() failed':
if e.message == 'virDomainLookupByUUIDString() failed':
return False
raise
@ -96,7 +96,7 @@ class LibvirtDriver(object):
self.conn.storageVolLookupByKey(volume.uuid)
return True
except libvirt.libvirtError, e:
if e.message == 'virStorageVolLookupByKey() failed':
if e.message == 'virStorageVolLookupByKey() failed':
return False
raise
@ -234,7 +234,7 @@ class LibvirtDriver(object):
"""
self.conn.lookupByUUIDString(node.uuid).destroy()
# @retry()
@retry()
def node_get_snapshots(self, node):
"""
:rtype : List
@ -251,6 +251,7 @@ class LibvirtDriver(object):
:rtype : None
"""
xml = self.xml_builder.build_snapshot_xml(name, description)
print xml
self.conn.lookupByUUIDString(node.uuid).snapshotCreateXML(xml, 0)
def _get_snapshot(self, domain, name):
@ -308,7 +309,7 @@ class LibvirtDriver(object):
:rtype : None
"""
libvirt_volume = self.conn.storagePoolLookupByName(pool).createXML(
self.xml_builder.build_volume_xml(volume),0)
self.xml_builder.build_volume_xml(volume), 0)
volume.uuid = libvirt_volume.key()
@retry()
@ -376,5 +377,5 @@ class LibvirtDriver(object):
prefix_or_netmask = ip.get('prefix') or ip.get('netmask')
allocated_networks.append(ipaddr.IPNetwork(
"{0:>s}/{1:>s}".format(address, prefix_or_netmask)))
self.allocated_networks=allocated_networks
self.allocated_networks = allocated_networks
return self.allocated_networks

7
src/devops/driver/libvirt/libvirt_xml_builder.py
View File

@ -3,7 +3,6 @@ from xmlbuilder import XMLBuilder
class LibvirtXMLBuilder(object):
def __init__(self, driver):
super(LibvirtXMLBuilder, self).__init__()
self.driver = driver
@ -14,7 +13,7 @@ class LibvirtXMLBuilder(object):
name = '_'.join(list(args))
if len(name) > self.NAME_SIZE:
hash_str = str(hash(name))
name=hash_str+name[len(name)-self.NAME_SIZE+len(hash_str):]
name = hash_str + name[len(name) - self.NAME_SIZE + len(hash_str):]
return name
def build_network_xml(self, network):
@ -80,7 +79,7 @@ class LibvirtXMLBuilder(object):
def _build_disk_device(self, device_xml, disk_device):
with device_xml.disk(type=disk_device.type, device=disk_device.device):
device_xml.driver(type = disk_device.volume.format)
device_xml.driver(type=disk_device.volume.format)
device_xml.source(file=self.driver.volume_path(disk_device.volume))
device_xml.target(dev=disk_device.target_dev, bus=disk_device.bus)
@ -88,7 +87,7 @@ class LibvirtXMLBuilder(object):
if interface.type != 'network':
raise NotImplementedError()
with device_xml.interface(type=interface.type):
device_xml.mac(address = interface.mac_address)
device_xml.mac(address=interface.mac_address)
device_xml.source(network=self.driver.network_name(interface.network))
if not (interface.type is None):
device_xml.model(type=interface.model)

2
src/devops/error.py
View File

@ -1,2 +1,2 @@
class DevopsError(Exception):
message ="Devops Error"
message = "Devops Error"

1
src/devops/helpers/helpers.py
View File

@ -307,6 +307,7 @@ def xmlrpcmethod(uri, method):
except:
raise AttributeError, "Error occured while getting server method"
def generate_mac():
return "64:{0:02x}:{1:02x}:{2:02x}:{3:02x}:{4:02x}".format(*bytearray(os.urandom(5)))

5
src/devops/helpers/network.py
View File

@ -2,11 +2,11 @@ class IpNetworksPool(object):
def __init__(self, networks, prefix):
self.networks = networks
self.prefix = prefix
self.allocated_networks=[]
self.allocated_networks = []
self._initialize_generator()
def set_allocated_networks(self, allocated_networks):
self.allocated_networks=allocated_networks
self.allocated_networks = allocated_networks
self._initialize_generator()
def _overlaps(self, network, allocated_networks):
@ -18,6 +18,7 @@ class IpNetworksPool(object):
for sub_net in network.iter_subnets(new_prefix=self.prefix):
if not self._overlaps(sub_net, self.allocated_networks):
yield sub_net
self._generator = _get_generator()
def __iter__(self):

4
src/devops/helpers/retry.py
View File

@ -1,7 +1,7 @@
import functools
from time import sleep
def retry(count=1, delay=1):
def retry(count=10, delay=1):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
@ -15,5 +15,7 @@ def retry(count=1, delay=1):
if i >= count:
raise
sleep(delay)
return wrapper
return decorator

16
src/devops/manager.py
View File

@ -1,4 +1,5 @@
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "devops.settings")
import ipaddr
from devops.helpers.helpers import generate_mac
@ -8,7 +9,7 @@ from devops.models import Address, Interface, Node, Network, Environment, Volume
class Manager(object):
def __init__(self):
super(Manager, self).__init__()
self.default_pool=None
self.default_pool = None
def environment_create(self, name):
return Environment.objects.create(name=name)
@ -25,12 +26,13 @@ class Manager(object):
return pool
def _get_default_pool(self):
self.default_pool = self.default_pool or self.create_network_pool(networks=[ipaddr.IPNetwork('10.0.0.0/16')], prefix=24)
self.default_pool = self.default_pool or self.create_network_pool(networks=[ipaddr.IPNetwork('10.0.0.0/16')],
prefix=24)
return self.default_pool
def network_create(
self, name, environment=None, ip_network=None, pool=None, has_dhcp_server=True, has_pxe_server=False,
forward='route'):
forward='nat'):
allocated_network = ip_network or ExternalModel.allocate_network(pool or self._get_default_pool())
return Network.objects.create(environment=environment, name=name, ip_network=ip_network or allocated_network,
has_pxe_server=has_pxe_server, has_dhcp_server=has_dhcp_server, forward=forward)
@ -41,7 +43,7 @@ class Manager(object):
if not boot: boot = ['network', 'cdrom', 'hd']
node = Node.objects.create(name=name, environment=environment, role=role, vcpu=vcpu, memory=memory,
has_vnc=has_vnc, metadata=metadata, hypervisor=hypervisor, os_type=os_type, architecture=architecture,
)
)
node.boot = boot
return node
@ -56,7 +58,8 @@ class Manager(object):
def volume_create_child(self, name, backing_store, format=None, environment=None):
return Volume.objects.create(name=name, environment=environment, capacity=backing_store.capacity, format=format or backing_store.format, backing_store=backing_store)
return Volume.objects.create(name=name, environment=environment, capacity=backing_store.capacity,
format=format or backing_store.format, backing_store=backing_store)
def volume_create(self, name, capacity, format='qcow2', environment=None):
return Volume.objects.create(name=name, environment=environment, capacity=capacity, format=format)
@ -77,5 +80,6 @@ class Manager(object):
Address.objects.create(ip_address=ip_address, interface=interface)
def node_attach_volume(self, node, volume, device='disk', type='file', bus='virtio', target_dev=None):
DiskDevice.objects.create(device=device, type=type, bus=bus, target_dev=target_dev or node.next_disk_name(), volume=volume, node=node)
DiskDevice.objects.create(device=device, type=type, bus=bus, target_dev=target_dev or node.next_disk_name(),
volume=volume, node=node)

38
src/devops/models.py
View File

@ -3,17 +3,19 @@ from devops.driver.libvirt.libvirt_driver import LibvirtDriver
from django.db import models
def choices(*args, **kwargs):
defaults = {'max_length':255, 'null':False}
defaults = {'max_length': 255, 'null': False}
defaults.update(kwargs)
defaults.update(choices=double_tuple(*args))
return models.CharField(**defaults)
def double_tuple(*args):
dict = []
for arg in args:
dict.append((arg,arg))
dict.append((arg, arg))
return tuple(dict)
class Environment(models.Model):
name = models.CharField(max_length=255, unique=True, null=False)
@ -81,8 +83,8 @@ class Environment(models.Model):
for node in self.nodes:
node.revert(name)
class ExternalModel(models.Model):
class ExternalModel(models.Model):
_driver = None
@classmethod
@ -145,7 +147,7 @@ class Network(ExternalModel):
while True:
self._iterhosts = self._iterhosts or IPNetwork(self.ip_network).iterhosts()
ip = self._iterhosts.next()
if ip<self.ip_pool_start or ip>self.ip_pool_end:
if ip < self.ip_pool_start or ip > self.ip_pool_end:
continue
if not Address.objects.filter(interface__network=self, ip_address=str(ip)).exists():
return ip
@ -160,7 +162,7 @@ class Network(ExternalModel):
def start(self):
self.create(verbose=False)
def create(self, verbose = True):
def create(self, verbose=True):
if verbose or not self.driver.network_active(self):
self.driver.network_create(self)
@ -170,7 +172,7 @@ class Network(ExternalModel):
def erase(self):
self.remove(verbose=False)
def remove(self, verbose = True):
def remove(self, verbose=True):
if verbose or self.uuid:
if verbose or self.driver.network_exists(self):
if self.driver.network_active(self):
@ -178,10 +180,11 @@ class Network(ExternalModel):
self.driver.network_undefine(self)
self.delete()
class Node(ExternalModel):
hypervisor = choices('kvm')
os_type = choices('hvm')
architecture = choices('x86_64','i686')
architecture = choices('x86_64', 'i686')
boot = ['network', 'cdrom', 'hd']
metadata = models.CharField(max_length=255, null=True)
role = models.CharField(max_length=255, null=True)
@ -204,9 +207,9 @@ class Node(ExternalModel):
def interfaces(self):
return Interface.objects.filter(node=self).order_by('id')
# @property
# def networks(self):
# return Network.objects.filter(interface__node=self)
# @property
# def networks(self):
# return Network.objects.filter(interface__node=self)
def interface_by_name(self, name):
self.interfaces.filter(name=name)
@ -228,11 +231,13 @@ class Node(ExternalModel):
def erase(self):
self.remove(verbose=False)
def remove(self, verbose = True):
def remove(self, verbose=True):
if verbose or self.uuid:
if verbose or self.driver.node_exists(self):
if self.driver.node_active(self):
self.driver.node_destroy(self)
for snapshot in self.driver.node_get_snapshots(self):
self.driver.node_delete_snapshot(node=self, name=snapshot)
self.driver.node_undefine(self)
self.delete()
@ -261,7 +266,7 @@ class Volume(ExternalModel):
def erase(self):
self.remove(verbose=False)
def remove(self, verbose = True):
def remove(self, verbose=True):
if verbose or self.uuid:
if verbose or self.driver.volume_exists(self):
self.driver.volume_delete(self)
@ -277,17 +282,19 @@ class Volume(ExternalModel):
return self.driver.volume_path(self)
def fill_from_exist(self):
self.capacity=self.get_capacity()
self.format=self.get_format()
self.capacity = self.get_capacity()
self.format = self.get_format()
class DiskDevice(models.Model):
device = choices('disk', 'cdrom')
type = choices('file')
bus = choices('virtio')
target_dev = models.CharField(max_length=255, null=False)
target_dev = models.CharField(max_length=255, null=False)
node = models.ForeignKey(Node, null=False)
volume = models.ForeignKey(Volume, null=True)
class Interface(models.Model):
mac_address = models.CharField(max_length=255, unique=True, null=False)
network = models.ForeignKey(Network)
@ -303,6 +310,7 @@ class Interface(models.Model):
def add_address(self, address):
Address.objects.create(ip_address=address, interface=self)
class Address(models.Model):
ip_address = models.GenericIPAddressField()
interface = models.ForeignKey(Interface)

8
src/devops/shell.py
View File

@ -3,7 +3,6 @@ import os
from devops.manager import Manager
class Shell(object):
def __init__(self):
super(Shell, self).__init__()
self.params = self.get_params()
@ -18,8 +17,8 @@ class Shell(object):
def do_show(self):
environment = self.manager.environment_get(self.params.name)
print {
'name' : environment.name ,
'nodes': map(lambda x: 'node', environment.nodes.values('name'))
'name': environment.name,
'nodes': map(lambda x: 'node', environment.nodes.values('name'))
}
def do_erase(self):
@ -44,9 +43,8 @@ class Shell(object):
self.manager.environment_get(self.params.name).snapshot(self.params.snapshot_name)
commands = {
'list' : do_list,
'list': do_list,
'show': do_show,
'erase': do_erase,
'start': do_start,

105
src/devops/tests/test_manager.py
View File

@ -5,13 +5,16 @@ from devops.manager import Manager
class TestManager(TestCase):
manager = Manager()
def tearDown(self):
for environment in self.manager.environment_list():
environment.erase()
def test_getting_subnetworks(self):
pool = IpNetworksPool(networks=[IPNetwork('10.1.0.0/22')], prefix=24)
pool.set_allocated_networks([IPv4Network('10.1.1.0/24')])
networks = list(pool)
networks = list(pool)
self.assertTrue(IPv4Network('10.1.0.0/24') in networks)
self.assertFalse(IPv4Network('10.1.1.0/24') in networks)
self.assertTrue(IPv4Network('10.1.2.0/24') in networks)
@ -26,9 +29,9 @@ class TestManager(TestCase):
network = self.manager.network_create(
environment=environment, name='internal', ip_network='10.1.0.0/24')
interface = self.manager.interface_create(network=network, node=node)
self.manager.network_create_address(str('10.1.0.1'),interface=interface)
self.manager.network_create_address(str('10.1.0.1'), interface=interface)
ip = network.next_ip()
self.manager.network_create_address(str('10.1.0.3'),interface=interface)
self.manager.network_create_address(str('10.1.0.3'), interface=interface)
ip = network.next_ip()
self.assertEquals('10.1.0.4', str(ip))
@ -37,7 +40,7 @@ class TestManager(TestCase):
print environment.volumes
def test_network_pool(self):
environment = self.manager.environment_create('test_env')
environment = self.manager.environment_create('test_env2')
self.assertEqual('10.0.0.0/24', str(self.manager.network_create(
environment=environment, name='internal', pool=None).ip_network))
self.assertEqual('10.0.1.0/24', str(self.manager.network_create(
@ -54,73 +57,53 @@ class TestManager(TestCase):
def test_node_creationw(self):
node = self.manager.node_create(name='test_node', environment=None)
environment = self.manager.environment_create('test_env55')
node = self.manager.node_create(name='test_node4', environment=environment)
node.define()
def test_node_creation(self):
environment = None
try:
print 1
environment = self.manager.environment_create('test_env2')
internal = self.manager.network_create(
environment=environment, name='internal', pool=None)
# external = self.manager.create_network(
# environment=environment, name='external', pool=None)
# private = self.manager.create_network(
# environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
self.manager.interface_create(node=node, network=internal)
# self.manager.create_interface(node=node, network=external)
# self.manager.create_interface(node=node, network=private)
environment.define()
except:
if environment:
environment.erase()
raise
environment = self.manager.environment_create('test_env3')
internal = self.manager.network_create(
environment=environment, name='internal', pool=None)
# external = self.manager.create_network(
# environment=environment, name='external', pool=None)
# private = self.manager.create_network(
# environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
self.manager.interface_create(node=node, network=internal)
# self.manager.create_interface(node=node, network=external)
# self.manager.create_interface(node=node, network=private)
environment.define()
def test_use_exist_volume(self):
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135871063107.qcow2')
print volume.format
print volume.capacity
print volume.uuid
def test_create_volume(self):
environment = self.manager.environment_create('test_env3')
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume, environment=environment)
v3.define()
def test_create_volume(self):
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume)
v3.define()
def test_create_volume(self):
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp89', backing_store=volume)
v3.define()
def test_create_node3(self):
environment = None
try:
print 1
environment = self.manager.environment_create('test_env3')
internal = self.manager.network_create(
environment=environment, name='internal', pool=None)
external = self.manager.network_create(
environment=environment, name='external', pool=None)
private = self.manager.network_create(
environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
self.manager.interface_create(node=node, network=internal)
self.manager.interface_create(node=node, network=external)
self.manager.interface_create(node=node, network=private)
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp892', backing_store=volume, environment=environment)
v4 = self.manager.volume_create_child('test_vp891', backing_store=volume, environment=environment)
self.manager.node_attach_volume(node=node, volume=v3)
self.manager.node_attach_volume(node, v4)
environment.define()
except:
if environment:
environment.erase()
raise
environment = self.manager.environment_create('test_env3')
internal = self.manager.network_create(
environment=environment, name='internal', pool=None)
external = self.manager.network_create(
environment=environment, name='external', pool=None)
private = self.manager.network_create(
environment=environment, name='private', pool=None)
node = self.manager.node_create(name='test_node', environment=environment)
self.manager.interface_create(node=node, network=internal)
self.manager.interface_create(node=node, network=external)
self.manager.interface_create(node=node, network=private)
volume = self.manager.volume_get_predefined('/var/lib/libvirt/images/disk-135824657433.qcow2')
v3 = self.manager.volume_create_child('test_vp892', backing_store=volume, environment=environment)
v4 = self.manager.volume_create_child('test_vp891', backing_store=volume, environment=environment)
self.manager.node_attach_volume(node=node, volume=v3)
self.manager.node_attach_volume(node, v4)
environment.define()

4
src/devops/tests/test_models.py
View File

@ -3,8 +3,6 @@ from devops.models import double_tuple
class TestModels(unittest.TestCase):
def test_django_choices(self):
self.assertEquals((('a','a'), ('b','b')),double_tuple('a', 'b'))
self.assertEquals((('a', 'a'), ('b', 'b')), double_tuple('a', 'b'))

1
src/dos.py
View File

@ -3,4 +3,5 @@ import os
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "devops.settings")
from devops.shell import Shell
Shell().execute()