Evacuation is working now and node will be evacuated but ?
We need to contact nova guys to ask why when we disable or enable a node the node state is always going up even if the node is powereded off. We had to wait and retry many times for 1 minute at least to give nova compute sometime to update nova api with it's state. Change-Id: If2ca244a79b930a37cd8e676fb4fe7a65ae34bfc
This commit is contained in:
parent
043a677238
commit
144067babf
|
@ -32,6 +32,7 @@ class OSClient:
|
||||||
"""
|
"""
|
||||||
self.authmethod = authmethod
|
self.authmethod = authmethod
|
||||||
self.authurl = authurl
|
self.authurl = authurl
|
||||||
|
self.auth_session = None
|
||||||
if authmethod == 'password':
|
if authmethod == 'password':
|
||||||
self.username = kwargs.get('username', None)
|
self.username = kwargs.get('username', None)
|
||||||
self.password = kwargs.get('password')
|
self.password = kwargs.get('password')
|
||||||
|
@ -56,10 +57,10 @@ class OSClient:
|
||||||
user_domain_id=self.user_domain_id,
|
user_domain_id=self.user_domain_id,
|
||||||
user_domain_name=self.user_domain_name,
|
user_domain_name=self.user_domain_name,
|
||||||
project_domain_name=self.project_domain_name)
|
project_domain_name=self.project_domain_name)
|
||||||
self.authSession = session.Session(auth=auth)
|
self.auth_session = session.Session(auth=auth)
|
||||||
|
|
||||||
def novacomputes(self):
|
def novacomputes(self):
|
||||||
nova = novaclient.Client(session=self.authSession,
|
nova = novaclient.Client(session=self.auth_session,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
services = nova.services.list()
|
services = nova.services.list()
|
||||||
compute_nodes = []
|
compute_nodes = []
|
||||||
|
@ -73,7 +74,7 @@ class OSClient:
|
||||||
return compute_nodes
|
return compute_nodes
|
||||||
|
|
||||||
def novahypervisors(self):
|
def novahypervisors(self):
|
||||||
nova = novaclient.Client(session=self.authSession,
|
nova = novaclient.Client(session=self.auth_session,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
hypervisors = nova.hypervisors.list()
|
hypervisors = nova.hypervisors.list()
|
||||||
nova_hypervisors = []
|
nova_hypervisors = []
|
||||||
|
@ -85,10 +86,10 @@ class OSClient:
|
||||||
def neutronagents(self, hosts=[]):
|
def neutronagents(self, hosts=[]):
|
||||||
if not hosts:
|
if not hosts:
|
||||||
hosts = self.compute_hosts
|
hosts = self.compute_hosts
|
||||||
new_sess = session.Session(auth=self.authSession.auth)
|
new_sess = session.Session(auth=self.auth_session.auth)
|
||||||
neutron = neutronclient.Client(session=new_sess,
|
neutron = neutronclient.Client(session=new_sess,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
self.authSession = new_sess
|
self.auth_session = new_sess
|
||||||
agents = neutron.list_agents()
|
agents = neutron.list_agents()
|
||||||
neutron_agents = []
|
neutron_agents = []
|
||||||
for agent in agents.get('agents'):
|
for agent in agents.get('agents'):
|
||||||
|
@ -99,41 +100,37 @@ class OSClient:
|
||||||
return neutron_agents
|
return neutron_agents
|
||||||
|
|
||||||
def evacuate(self, nodes):
|
def evacuate(self, nodes):
|
||||||
new_sess = session.Session(auth=self.authSession.auth)
|
"""
|
||||||
|
Will get the hypervisors and list all running VMs on it and then start
|
||||||
|
Evacuating one by one ...
|
||||||
|
:param nodes: List of nodes to be evacuated !
|
||||||
|
:return: List of nodes with VMs that were running on that node
|
||||||
|
"""
|
||||||
|
new_sess = session.Session(auth=self.auth_session.auth)
|
||||||
nova = novaclient.Client(session=new_sess,
|
nova = novaclient.Client(session=new_sess,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
self.authSession = new_sess
|
self.auth_session = new_sess
|
||||||
evacuated_nodes = []
|
evacuated_nodes = []
|
||||||
print "Nodes", nodes
|
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
hypervisors = nova.hypervisors.search(node.get('host'), True)
|
hypervisors = nova.hypervisors.search(node.get('host'), True)
|
||||||
print "Hypervisor found is:", hypervisors
|
|
||||||
for hypervisor in hypervisors:
|
for hypervisor in hypervisors:
|
||||||
host = {'host': node.get('host'), 'servers': hypervisor.servers}
|
if not hasattr(hypervisor, 'servers'):
|
||||||
evacuated_nodes.append(host)
|
break
|
||||||
for server in hypervisor.servers:
|
for server in hypervisor.servers:
|
||||||
try:
|
try:
|
||||||
output = nova.servers.evacuate(server.get('uuid'),
|
nova.servers.evacuate(server.get('uuid'),
|
||||||
on_shared_storage=True)
|
on_shared_storage=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print "ERRORORRRROROROROROROROROROROROROROROROROROROROR"
|
LOG.error(e)
|
||||||
print e
|
host = {'host': node.get('host'), 'servers': hypervisor.servers}
|
||||||
for i in range(0, 100):
|
evacuated_nodes.append(host)
|
||||||
print "-",
|
|
||||||
if i == 50:
|
|
||||||
print "Evacuation Result !",
|
|
||||||
print
|
|
||||||
|
|
||||||
print output
|
|
||||||
exit()
|
|
||||||
|
|
||||||
return evacuated_nodes
|
return evacuated_nodes
|
||||||
|
|
||||||
def set_in_maintance(self, nodes):
|
def set_in_maintance(self, nodes):
|
||||||
new_sess = session.Session(auth=self.authSession.auth)
|
new_sess = session.Session(auth=self.auth_session.auth)
|
||||||
nova = novaclient.Client(session=new_sess,
|
nova = novaclient.Client(session=new_sess,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
self.authSession = new_sess
|
self.auth_session = new_sess
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
output = []
|
output = []
|
||||||
host = nova.hosts.get(node)[0]
|
host = nova.hosts.get(node)[0]
|
||||||
|
@ -141,11 +138,11 @@ class OSClient:
|
||||||
try:
|
try:
|
||||||
output.append(host.update(values))
|
output.append(host.update(values))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print "ERROR ::: ", e
|
LOG.error(e)
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def get_session(self):
|
def get_session(self):
|
||||||
auth_session = session.Session(auth=self.authSession.auth)
|
auth_session = session.Session(auth=self.auth_session.auth)
|
||||||
return auth_session
|
return auth_session
|
||||||
|
|
||||||
def get_node_status(self, node):
|
def get_node_status(self, node):
|
||||||
|
@ -155,51 +152,52 @@ class OSClient:
|
||||||
:return: True or False. True => node disabled, False => node is enabled
|
:return: True or False. True => node disabled, False => node is enabled
|
||||||
or unknow status !
|
or unknow status !
|
||||||
"""
|
"""
|
||||||
nova = novaclient.Client(session=self.authSession,
|
nova = novaclient.Client(session=self.auth_session,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
try:
|
try:
|
||||||
node = nova.services.find(host=node.get('host'))
|
node_service = nova.services.find(host=node.get('host'))
|
||||||
print node
|
del nova
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.error(e)
|
LOG.error(e)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if not node:
|
if not node_service:
|
||||||
return False
|
return False
|
||||||
node = node.to_dict()
|
node = node_service.to_dict()
|
||||||
if node.get('status') == 'disabled':
|
if node.get('status') == 'disabled':
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def disable_node(self, node):
|
def disable_node(self, node):
|
||||||
auth_session = session.Session(auth=self.authSession.auth)
|
auth_session = session.Session(auth=self.auth_session.auth)
|
||||||
nova = novaclient.Client(session=auth_session,
|
nova = novaclient.Client(session=auth_session,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
try:
|
try:
|
||||||
node = nova.services.find(host=node.get('host'))
|
node_service = nova.services.find(host=node.get('host'))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.error(e)
|
LOG.error(e)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if not node:
|
if not node_service:
|
||||||
return False
|
return False
|
||||||
node = node.to_dict()
|
node = node_service.to_dict()
|
||||||
|
del node_service
|
||||||
try:
|
try:
|
||||||
res = nova.services.disable_log_reason(
|
nova.services.disable_log_reason(
|
||||||
host=node.get('host'),
|
host=node.get('host'),
|
||||||
binary=node.get('binary'),
|
binary=node.get('binary'),
|
||||||
reason='Host Failed and node evacuated.'
|
reason='Host Failed and needs to be evacuated.'
|
||||||
)
|
)
|
||||||
|
del nova
|
||||||
LOG.info('Compute host: %s has been disabled to be evacuated. '
|
LOG.info('Compute host: %s has been disabled to be evacuated. '
|
||||||
'Host details: %s' % (node.get('host'), str(node)))
|
'Host details: %s' % (node.get('host'), str(node)))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
LOG.error(e)
|
LOG.error(e)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def get_hypervisor_instances(self, node):
|
def get_hypervisor_instances(self, node):
|
||||||
auth_session = session.Session(auth=self.authSession.auth)
|
auth_session = session.Session(auth=self.auth_session.auth)
|
||||||
nova = novaclient.Client(session=auth_session,
|
nova = novaclient.Client(session=auth_session,
|
||||||
endpoint_type=self.endpoint_type)
|
endpoint_type=self.endpoint_type)
|
||||||
hypervisors = nova.hypervisors.search(node.get('host'), True)
|
hypervisors = nova.hypervisors.search(node.get('host'), True)
|
||||||
|
|
|
@ -70,9 +70,15 @@ class EvacuationManager(object):
|
||||||
succeeded_nodes.append(node)
|
succeeded_nodes.append(node)
|
||||||
|
|
||||||
nodes = succeeded_nodes
|
nodes = succeeded_nodes
|
||||||
|
|
||||||
|
# Start evacuation calls ...
|
||||||
from time import sleep
|
from time import sleep
|
||||||
sleep(30)
|
for i in range(0, 10):
|
||||||
evacuated_nodes = self.driver.evacuate_nodes(nodes)
|
try:
|
||||||
|
sleep(30)
|
||||||
|
evacuated_nodes = self.driver.evacuate_nodes(nodes)
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error(e)
|
||||||
return evacuated_nodes
|
return evacuated_nodes
|
||||||
|
|
||||||
def _disable_node(self, node):
|
def _disable_node(self, node):
|
||||||
|
@ -81,3 +87,12 @@ class EvacuationManager(object):
|
||||||
else:
|
else:
|
||||||
True
|
True
|
||||||
|
|
||||||
|
def reinitialize_driver(self):
|
||||||
|
evcuation_conf = CONF.get('evacuation')
|
||||||
|
self.driver = importutils.import_object(
|
||||||
|
evcuation_conf.get('driver'),
|
||||||
|
evcuation_conf.get('wait'),
|
||||||
|
evcuation_conf.get('retries'),
|
||||||
|
**evcuation_conf.get('options')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue