l3-agent-evacuate.py: evacuate a network node

Change-Id: I1dc363ef3d36fa92c7122852029a9ee81bd3d7af
This commit is contained in:
Saverio Proto 2016-11-23 21:15:32 +01:00
parent 38a70fda49
commit cc3ed99079
3 changed files with 358 additions and 0 deletions

147
lib/openstackapi.py Normal file
View File

@ -0,0 +1,147 @@
# Copyright (c) 2016 SWITCH http://www.switch.ch
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Author: Valery Tschopp <valery.tschopp@switch.ch>
# Date: 2016-07-05
import keystoneclient
from cinderclient.v2 import client as cinder_client
from glanceclient.v2 import client as glance_client
from keystoneauth1.identity import v3 as identity_v3
from keystoneauth1 import session
from keystoneclient.v3 import client as keystone_v3
from neutronclient.v2_0 import client as neutron_client
from novaclient import client as nova_client
class OpenstackAPI():
"""Openstack API clients
Initialize all the necessary Openstack clients for all available regions.
"""
def __init__(self, os_auth_url, os_username, os_password, os_project_name,
user_domain_name='default',
project_domain_name='default'):
# keystone_V3 client requires a /v3 auth url
if '/v2.0' in os_auth_url:
self.auth_url = os_auth_url.replace('/v2.0', '/v3')
else:
self.auth_url = os_auth_url
_auth = identity_v3.Password(auth_url=self.auth_url,
username=os_username,
password=os_password,
project_name=os_project_name,
user_domain_name=user_domain_name,
project_domain_name=project_domain_name)
self._auth_session = session.Session(auth=_auth)
self._keystone = keystone_v3.Client(session=self._auth_session)
# all regions available
self.all_region_names = []
for region in self.keystone.regions.list():
self.all_region_names.append(region.id)
self._nova = {}
self._cinder = {}
self._neutron = {}
self._glance = {}
@property
def keystone(self):
"""Get Keystone client"""
return self._keystone
def nova(self, region):
"""Get Nova client for the region."""
if region not in self._nova:
# Nova client lazy initialisation
_nova = nova_client.Client('2',
session=self._auth_session,
region_name=region)
self._nova[region] = _nova
return self._nova[region]
def cinder(self, region):
"""Get Cinder client for the region."""
if region not in self._cinder:
# Cinder client lazy initialisation
_cinder = cinder_client.Client(session=self._auth_session,
region_name=region)
self._cinder[region] = _cinder
return self._cinder[region]
def neutron(self, region):
"""Get Neutron client for the region."""
if region not in self._neutron:
# Neutron client lazy initialisation
_neutron = neutron_client.Client(session=self._auth_session,
region_name=region)
self._neutron[region] = _neutron
return self._neutron[region]
def glance(self, region):
"""Get Glance client for the region."""
if region not in self._glance:
# Glance client lazy initialisation
_glance = glance_client.Client(session=self._auth_session,
region_name=region)
self._glance[region] = _glance
return self._glance[region]
def get_all_regions(self):
"""Get list of all region names"""
return self.all_region_names
def get_user(self, user_name_or_id):
"""Get a user by name or id"""
user = None
try:
# try by name
user = self._keystone.users.find(name=user_name_or_id)
except keystoneclient.exceptions.NotFound as e:
# try by ID
user = self._keystone.users.get(user_name_or_id)
return user
def get_user_projects(self, user):
"""Get all user projects"""
projects = self._keystone.projects.list(user=user)
return projects
def get_project(self, project_name_or_id):
"""Get a project by name or id"""
project = None
try:
# try by name
project = self._keystone.projects.find(name=project_name_or_id)
except keystoneclient.exceptions.NotFound as e:
# try by ID
project = self._keystone.projects.get(project_name_or_id)
return project
def get_project_users(self, project):
"""Get all users in project"""
assignments = self._keystone.role_assignments.list(project=project)
user_ids = set()
for assignment in assignments:
if hasattr(assignment, 'user'):
user_ids.add(assignment.user['id'])
users = []
for user_id in user_ids:
users.append(self._keystone.users.get(user_id))
return users

64
neutron/README.md Normal file
View File

@ -0,0 +1,64 @@
# Neutron folder
this folder contains scripts that are related to Neutron
## L3 Agent Evacuate
Migrate away the OpenStack routers from a L3 Agent
```
./l3-agent-evacuate.py --help
usage: l3-agent-evacuate.py [-h] [-f FROM_L3AGENT] [-t TO_L3AGENT] [-r ROUTER]
[-l LIMIT] [-v]
Evacuate a neutron l3-agent
optional arguments:
-h, --help show this help message and exit
-f FROM_L3AGENT, --from-l3agent FROM_L3AGENT
l3agent uuid
-t TO_L3AGENT, --to-l3agent TO_L3AGENT
l3agent uuid
-r ROUTER, --router ROUTER
specific router
-l LIMIT, --limit LIMIT
max number of routers to migrate
-v, --verbose verbose
```
First of all we should have clear in mind that when we create a router in
Openstack, that router is just a network namespace on one of the network nodes,
with name qrouter-<uuid>. For each namespace there is a qr (downstream) and a
qg (upstream) interface. In some situations an operation might want to migrate
away all the routers from a network node, to be able for example to reboot the
node without impacting the user traffic. The neutron component responsible for
creating the namespaces and cabling them with openvswitch is the l3 agent. You
can check the uuid of the l3 agents currently running with:
``` openstack network agent list ```
When you are running multiple l3 agents, if you create a new router Openstack
will schedule the namespace to be created on one of the available network
nodes. Given a specific router, with this command you can find out on which
network node the namespace has been created:
```
neutron l3-agent-list-hosting-router <router_uuid>
```
To list instead all the routers scheduled on a specific network node ```
neutron router-list-on-l3-agent <l3agent-uuid> ``` Using the neutron commands
`l3-agent-router-add` and `l3-agent-router-remove` is then possible to move a
router from a l3 agent to another one.
The tool `l3-agent-evacuate.py` will create a list of all the routers present
on the `from-agent` and will move 1 router every 10 seconds to the `to-agent`.
It is better to add a 10 seconds delay because Openvswitch has to make a lot of
operations when the namespace is created, and moving many routers at once will
cause openvswitch to blow up with unpredictable behavior.
The script has also a `--router` option if you want to migrate a specific
router, or a `--limit` option if you want to migrate just a few routers.
While migrating the routers, you can check (especially on the target l3-agent)
the openvswitch operations going on in `/var/log/syslog`.

147
neutron/l3-agent-evacuate.py Executable file
View File

@ -0,0 +1,147 @@
#!/usr/bin/env python
#
# Copyright (c) 2016 SWITCH http://www.switch.ch
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Author: Saverio Proto <saverio.proto@switch.ch>
"""
Example usage:
python l3-agent-evacuate.py --from-l3agent 19f59173-68eb-49e3-a078-10831935a8f7 --to-l3agent f00dddd0-b944-4eeb-80d1-fa0811725196
python l3-agent-evacuate.py --from-l3agent f00dddd0-b944-4eeb-80d1-fa0811725196 --to-l3agent 19f59173-68eb-49e3-a078-10831935a8f7
"""
import os
import sys
sys.path.append('../lib')
import argparse
import openstackapi
import keystoneclient
import time
def get_environ(key, verbose=False):
if key not in os.environ:
print "ERROR:", key, "not define in environment"
sys.exit(1)
if verbose:
if 'password' in key.lower():
key_value = '*' * len(os.environ[key])
else:
key_value = os.environ[key]
print "{}: {}".format(key, key_value)
return os.environ[key]
def main():
"""
Evacuate a neutron l3-agent
"""
parser = argparse.ArgumentParser(
description="Evacuate a neutron l3-agent")
parser.add_argument('-f', '--from-l3agent', help='l3agent uuid', required=True)
parser.add_argument('-t', '--to-l3agent', help='l3agent uuid', required=True)
parser.add_argument('-r', '--router', help='specific router')
parser.add_argument('-l', '--limit', help='max number of routers to migrate')
parser.add_argument('-v', '--verbose', help='verbose', action='store_true')
args = parser.parse_args()
# get OS_* environment variables
os_auth_url = get_environ('OS_AUTH_URL', args.verbose)
os_username = get_environ('OS_USERNAME', args.verbose)
os_password = get_environ('OS_PASSWORD', args.verbose)
os_tenant_name = get_environ('OS_TENANT_NAME', args.verbose)
os_region_name = get_environ('OS_REGION_NAME', args.verbose)
api = openstackapi.OpenstackAPI(os_auth_url, os_username, os_password, os_project_name=os_tenant_name)
if args.limit:
limit=int(args.limit)
else:
limit = 0
#Validate agent's UUID
validateargs(api, os_region_name, args.from_l3agent, args.to_l3agent, args.router)
if args.router:
moverouter(api, os_region_name, args.from_l3agent, args.to_l3agent, args.router)
else:
evacuate_l3_agent(api, os_region_name, args.from_l3agent, args.to_l3agent, limit)
def validateargs(api, region, from_agent, to_agent, router):
neutron = api.neutron(region)
l3_agents_uuids=[]
routers_uuids=[]
for agent in neutron.list_agents()['agents']:
if agent['agent_type'] == u"L3 agent":
l3_agents_uuids.append(agent['id'])
for r in neutron.list_routers()['routers']:
routers_uuids.append(r['id'])
if from_agent not in l3_agents_uuids:
print "%s not a valid agent" % from_agent
sys.exit(1)
if to_agent not in l3_agents_uuids:
print "%s not a valid agent" % to_agent
sys.exit(1)
if router:
if router not in routers_uuids:
print "%s not a valid router" % router
sys.exit(1)
if neutron.list_l3_agent_hosting_routers(router)['agents'][0]['id'] != from_agent:
print "Wrong from_agent for specified router"
sys.exit(1)
def moverouter(api, region, from_agent, to_agent, router):
neutron = api.neutron(region)
r_id = {'router_id': router}
print "Removing router %s" % router
neutron.remove_router_from_l3_agent(from_agent, router)
print "Adding router %s" % router
neutron.add_router_to_l3_agent(to_agent, r_id)
def evacuate_l3_agent(api, region, from_agent, to_agent, limit):
"""Evacuate"""
neutron = api.neutron(region)
routers = neutron.list_routers_on_l3_agent(from_agent)["routers"]
#filter out from the list ha routers
ha_false_routers=[]
for r in routers:
if not r["ha"]:
ha_false_routers.append(r)
if not len(ha_false_routers):
print "Warning: l3 agent was already evacuated"
sys.exit(1)
if limit and (len(ha_false_routers) > limit):
ha_false_routers = ha_false_routers[0:limit]
print "Starting ... Moving a router every 10 seconds\n"
for r in ha_false_routers:
r_id = {'router_id': r['id']}
print "Removing router %s" % r['id']
neutron.remove_router_from_l3_agent(from_agent, r['id'])
print "Adding router %s" % r['id']
neutron.add_router_to_l3_agent(to_agent, r_id)
time.sleep(10)
if __name__ == '__main__':
main()