From cc3ed9907997e3d59d146f8d09b6ab6f498fdd20 Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Wed, 23 Nov 2016 21:15:32 +0100 Subject: [PATCH] l3-agent-evacuate.py: evacuate a network node Change-Id: I1dc363ef3d36fa92c7122852029a9ee81bd3d7af --- lib/openstackapi.py | 147 +++++++++++++++++++++++++++++++++++ neutron/README.md | 64 +++++++++++++++ neutron/l3-agent-evacuate.py | 147 +++++++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 lib/openstackapi.py create mode 100644 neutron/README.md create mode 100755 neutron/l3-agent-evacuate.py diff --git a/lib/openstackapi.py b/lib/openstackapi.py new file mode 100644 index 0000000..23e5d82 --- /dev/null +++ b/lib/openstackapi.py @@ -0,0 +1,147 @@ +# Copyright (c) 2016 SWITCH http://www.switch.ch +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Author: Valery Tschopp +# Date: 2016-07-05 + +import keystoneclient + +from cinderclient.v2 import client as cinder_client +from glanceclient.v2 import client as glance_client +from keystoneauth1.identity import v3 as identity_v3 +from keystoneauth1 import session +from keystoneclient.v3 import client as keystone_v3 +from neutronclient.v2_0 import client as neutron_client +from novaclient import client as nova_client + +class OpenstackAPI(): + """Openstack API clients + + Initialize all the necessary Openstack clients for all available regions. + """ + + def __init__(self, os_auth_url, os_username, os_password, os_project_name, + user_domain_name='default', + project_domain_name='default'): + # keystone_V3 client requires a /v3 auth url + if '/v2.0' in os_auth_url: + self.auth_url = os_auth_url.replace('/v2.0', '/v3') + else: + self.auth_url = os_auth_url + + _auth = identity_v3.Password(auth_url=self.auth_url, + username=os_username, + password=os_password, + project_name=os_project_name, + user_domain_name=user_domain_name, + project_domain_name=project_domain_name) + self._auth_session = session.Session(auth=_auth) + self._keystone = keystone_v3.Client(session=self._auth_session) + + # all regions available + self.all_region_names = [] + for region in self.keystone.regions.list(): + self.all_region_names.append(region.id) + + self._nova = {} + self._cinder = {} + self._neutron = {} + self._glance = {} + + @property + def keystone(self): + """Get Keystone client""" + return self._keystone + + def nova(self, region): + """Get Nova client for the region.""" + if region not in self._nova: + # Nova client lazy initialisation + _nova = nova_client.Client('2', + session=self._auth_session, + region_name=region) + self._nova[region] = _nova + return self._nova[region] + + + def cinder(self, region): + """Get Cinder client for the region.""" + if region not in self._cinder: + # Cinder client lazy initialisation + _cinder = cinder_client.Client(session=self._auth_session, + region_name=region) + self._cinder[region] = _cinder + return self._cinder[region] + + def neutron(self, region): + """Get Neutron client for the region.""" + if region not in self._neutron: + # Neutron client lazy initialisation + _neutron = neutron_client.Client(session=self._auth_session, + region_name=region) + self._neutron[region] = _neutron + return self._neutron[region] + + def glance(self, region): + """Get Glance client for the region.""" + if region not in self._glance: + # Glance client lazy initialisation + _glance = glance_client.Client(session=self._auth_session, + region_name=region) + self._glance[region] = _glance + return self._glance[region] + + def get_all_regions(self): + """Get list of all region names""" + return self.all_region_names + + def get_user(self, user_name_or_id): + """Get a user by name or id""" + user = None + try: + # try by name + user = self._keystone.users.find(name=user_name_or_id) + except keystoneclient.exceptions.NotFound as e: + # try by ID + user = self._keystone.users.get(user_name_or_id) + return user + + def get_user_projects(self, user): + """Get all user projects""" + projects = self._keystone.projects.list(user=user) + return projects + + def get_project(self, project_name_or_id): + """Get a project by name or id""" + project = None + try: + # try by name + project = self._keystone.projects.find(name=project_name_or_id) + except keystoneclient.exceptions.NotFound as e: + # try by ID + project = self._keystone.projects.get(project_name_or_id) + return project + + def get_project_users(self, project): + """Get all users in project""" + assignments = self._keystone.role_assignments.list(project=project) + user_ids = set() + for assignment in assignments: + if hasattr(assignment, 'user'): + user_ids.add(assignment.user['id']) + users = [] + for user_id in user_ids: + users.append(self._keystone.users.get(user_id)) + return users + diff --git a/neutron/README.md b/neutron/README.md new file mode 100644 index 0000000..6e1391e --- /dev/null +++ b/neutron/README.md @@ -0,0 +1,64 @@ +# Neutron folder + +this folder contains scripts that are related to Neutron + +## L3 Agent Evacuate + +Migrate away the OpenStack routers from a L3 Agent + +``` +./l3-agent-evacuate.py --help +usage: l3-agent-evacuate.py [-h] [-f FROM_L3AGENT] [-t TO_L3AGENT] [-r ROUTER] + [-l LIMIT] [-v] + +Evacuate a neutron l3-agent + +optional arguments: + -h, --help show this help message and exit + -f FROM_L3AGENT, --from-l3agent FROM_L3AGENT + l3agent uuid + -t TO_L3AGENT, --to-l3agent TO_L3AGENT + l3agent uuid + -r ROUTER, --router ROUTER + specific router + -l LIMIT, --limit LIMIT + max number of routers to migrate + -v, --verbose verbose +``` + +First of all we should have clear in mind that when we create a router in +Openstack, that router is just a network namespace on one of the network nodes, +with name qrouter-. For each namespace there is a qr (downstream) and a +qg (upstream) interface. In some situations an operation might want to migrate +away all the routers from a network node, to be able for example to reboot the +node without impacting the user traffic. The neutron component responsible for +creating the namespaces and cabling them with openvswitch is the l3 agent. You +can check the uuid of the l3 agents currently running with: + +``` openstack network agent list ``` + +When you are running multiple l3 agents, if you create a new router Openstack +will schedule the namespace to be created on one of the available network +nodes. Given a specific router, with this command you can find out on which +network node the namespace has been created: +``` +neutron l3-agent-list-hosting-router +``` + +To list instead all the routers scheduled on a specific network node ``` +neutron router-list-on-l3-agent ``` Using the neutron commands +`l3-agent-router-add` and `l3-agent-router-remove` is then possible to move a +router from a l3 agent to another one. + +The tool `l3-agent-evacuate.py` will create a list of all the routers present +on the `from-agent` and will move 1 router every 10 seconds to the `to-agent`. +It is better to add a 10 seconds delay because Openvswitch has to make a lot of +operations when the namespace is created, and moving many routers at once will +cause openvswitch to blow up with unpredictable behavior. + +The script has also a `--router` option if you want to migrate a specific +router, or a `--limit` option if you want to migrate just a few routers. + +While migrating the routers, you can check (especially on the target l3-agent) +the openvswitch operations going on in `/var/log/syslog`. + diff --git a/neutron/l3-agent-evacuate.py b/neutron/l3-agent-evacuate.py new file mode 100755 index 0000000..98e886f --- /dev/null +++ b/neutron/l3-agent-evacuate.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 SWITCH http://www.switch.ch +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Author: Saverio Proto + +""" +Example usage: +python l3-agent-evacuate.py --from-l3agent 19f59173-68eb-49e3-a078-10831935a8f7 --to-l3agent f00dddd0-b944-4eeb-80d1-fa0811725196 +python l3-agent-evacuate.py --from-l3agent f00dddd0-b944-4eeb-80d1-fa0811725196 --to-l3agent 19f59173-68eb-49e3-a078-10831935a8f7 +""" + +import os +import sys +sys.path.append('../lib') + +import argparse +import openstackapi +import keystoneclient +import time + +def get_environ(key, verbose=False): + if key not in os.environ: + print "ERROR:", key, "not define in environment" + sys.exit(1) + if verbose: + if 'password' in key.lower(): + key_value = '*' * len(os.environ[key]) + else: + key_value = os.environ[key] + print "{}: {}".format(key, key_value) + return os.environ[key] + + +def main(): + """ + Evacuate a neutron l3-agent + """ + parser = argparse.ArgumentParser( + description="Evacuate a neutron l3-agent") + parser.add_argument('-f', '--from-l3agent', help='l3agent uuid', required=True) + parser.add_argument('-t', '--to-l3agent', help='l3agent uuid', required=True) + parser.add_argument('-r', '--router', help='specific router') + parser.add_argument('-l', '--limit', help='max number of routers to migrate') + parser.add_argument('-v', '--verbose', help='verbose', action='store_true') + args = parser.parse_args() + + # get OS_* environment variables + os_auth_url = get_environ('OS_AUTH_URL', args.verbose) + os_username = get_environ('OS_USERNAME', args.verbose) + os_password = get_environ('OS_PASSWORD', args.verbose) + os_tenant_name = get_environ('OS_TENANT_NAME', args.verbose) + os_region_name = get_environ('OS_REGION_NAME', args.verbose) + + + api = openstackapi.OpenstackAPI(os_auth_url, os_username, os_password, os_project_name=os_tenant_name) + if args.limit: + limit=int(args.limit) + else: + limit = 0 + + #Validate agent's UUID + validateargs(api, os_region_name, args.from_l3agent, args.to_l3agent, args.router) + + if args.router: + moverouter(api, os_region_name, args.from_l3agent, args.to_l3agent, args.router) + else: + evacuate_l3_agent(api, os_region_name, args.from_l3agent, args.to_l3agent, limit) + +def validateargs(api, region, from_agent, to_agent, router): + neutron = api.neutron(region) + l3_agents_uuids=[] + routers_uuids=[] + + for agent in neutron.list_agents()['agents']: + if agent['agent_type'] == u"L3 agent": + l3_agents_uuids.append(agent['id']) + + for r in neutron.list_routers()['routers']: + routers_uuids.append(r['id']) + + if from_agent not in l3_agents_uuids: + print "%s not a valid agent" % from_agent + sys.exit(1) + + if to_agent not in l3_agents_uuids: + print "%s not a valid agent" % to_agent + sys.exit(1) + + if router: + if router not in routers_uuids: + print "%s not a valid router" % router + sys.exit(1) + if neutron.list_l3_agent_hosting_routers(router)['agents'][0]['id'] != from_agent: + print "Wrong from_agent for specified router" + sys.exit(1) + + +def moverouter(api, region, from_agent, to_agent, router): + neutron = api.neutron(region) + r_id = {'router_id': router} + print "Removing router %s" % router + neutron.remove_router_from_l3_agent(from_agent, router) + print "Adding router %s" % router + neutron.add_router_to_l3_agent(to_agent, r_id) + +def evacuate_l3_agent(api, region, from_agent, to_agent, limit): + """Evacuate""" + neutron = api.neutron(region) + routers = neutron.list_routers_on_l3_agent(from_agent)["routers"] + + #filter out from the list ha routers + ha_false_routers=[] + for r in routers: + if not r["ha"]: + ha_false_routers.append(r) + + if not len(ha_false_routers): + print "Warning: l3 agent was already evacuated" + sys.exit(1) + if limit and (len(ha_false_routers) > limit): + ha_false_routers = ha_false_routers[0:limit] + print "Starting ... Moving a router every 10 seconds\n" + for r in ha_false_routers: + r_id = {'router_id': r['id']} + print "Removing router %s" % r['id'] + neutron.remove_router_from_l3_agent(from_agent, r['id']) + print "Adding router %s" % r['id'] + neutron.add_router_to_l3_agent(to_agent, r_id) + time.sleep(10) + + + +if __name__ == '__main__': + main()