From e62bf6009640c4182c23b6efec6f18ed6467e9aa Mon Sep 17 00:00:00 2001 From: Roopak Parikh Date: Tue, 22 Mar 2016 21:52:08 +0000 Subject: [PATCH] First version of Mors - Lease Manager What works: - All the add/delete/update APIs integrated with UI and tested end to end - Basic unit tests that test the above metioned APIs. What (may) not be working or in other words is not fully tested - Actual deletes of the VM, it used to work, but code has gone through major changes so need to test again. - Cases: -- Making sure Override of the lease works. -- Cases where VM changes tenants or is deleted before the lease expiry -- Removal of the tenant (not tested at all) Next steps: - Better unit test cases - better verification - Deployment scripts (Ansible playbooks) Adding a manage script for managing database upgrade script Adding manage.py --- .idea/vcs.xml | 6 + .reviewboardrc | 6 + README.md | 5 + build.sh | 6 + etc/init.d/pf9-mors | 100 ++++++++ etc/nginx/conf.d/locations/mors.conf | 4 + etc/pf9/pf9-mors-api-paste.ini | 14 ++ etc/pf9/pf9-mors.ini | 16 ++ mors/__init__.py | 0 mors/context_util.py | 69 ++++++ mors/lease_manager.py | 186 +++++++++++++++ mors/leasehandler/__init__.py | 12 + mors/leasehandler/constants.py | 5 + mors/leasehandler/fake_lease_handler.py | 38 +++ mors/leasehandler/nova_lease_handler.py | 70 ++++++ mors/mors_wsgi.py | 130 ++++++++++ mors/persistence.py | 108 +++++++++ mors_manage.py | 27 +++ mors_repo/README | 4 + mors_repo/__init__.py | 0 mors_repo/__init__.pyc | Bin 0 -> 245 bytes mors_repo/manage.py | 5 + mors_repo/migrate.cfg | 25 ++ mors_repo/versions/001_Add_initial_tables.py | 36 +++ mors_repo/versions/__init__.py | 0 mors_repo/versions/__init__.pyc | Bin 0 -> 254 bytes pf9_mors.py | 48 ++++ setup.py | 36 +++ support/Makefile | 65 +++++ support/mors.expect | 9 + support/sign_packages.sh | 5 + test.sh | 19 ++ test/api-paste.ini | 14 ++ test/pf9-mors.ini | 16 ++ test/run_tests.py | 15 ++ test/test_api.py | 236 +++++++++++++++++++ test/test_persistence.py | 56 +++++ 37 files changed, 1391 insertions(+) create mode 100644 .idea/vcs.xml create mode 100644 .reviewboardrc create mode 100755 build.sh create mode 100755 etc/init.d/pf9-mors create mode 100644 etc/nginx/conf.d/locations/mors.conf create mode 100644 etc/pf9/pf9-mors-api-paste.ini create mode 100644 etc/pf9/pf9-mors.ini create mode 100644 mors/__init__.py create mode 100644 mors/context_util.py create mode 100644 mors/lease_manager.py create mode 100644 mors/leasehandler/__init__.py create mode 100644 mors/leasehandler/constants.py create mode 100644 mors/leasehandler/fake_lease_handler.py create mode 100644 mors/leasehandler/nova_lease_handler.py create mode 100644 mors/mors_wsgi.py create mode 100644 mors/persistence.py create mode 100644 mors_manage.py create mode 100644 mors_repo/README create mode 100644 mors_repo/__init__.py create mode 100644 mors_repo/__init__.pyc create mode 100644 mors_repo/manage.py create mode 100644 mors_repo/migrate.cfg create mode 100644 mors_repo/versions/001_Add_initial_tables.py create mode 100644 mors_repo/versions/__init__.py create mode 100644 mors_repo/versions/__init__.pyc create mode 100644 pf9_mors.py create mode 100644 setup.py create mode 100644 support/Makefile create mode 100644 support/mors.expect create mode 100755 support/sign_packages.sh create mode 100755 test.sh create mode 100644 test/api-paste.ini create mode 100644 test/pf9-mors.ini create mode 100644 test/run_tests.py create mode 100644 test/test_api.py create mode 100644 test/test_persistence.py diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.reviewboardrc b/.reviewboardrc new file mode 100644 index 0000000..fdb9cb7 --- /dev/null +++ b/.reviewboardrc @@ -0,0 +1,6 @@ +REVIEWBOARD_URL = 'https://rbcommons.com/s/platform9/' +REPOSITORY = 'pf9-mors' +GUESS_DESCRIPTION = True +GUESS_SUMMARY = True +TARGET_GROUPS = 'platform9' +TRACKING_BRANCH = 'origin/atherton' diff --git a/README.md b/README.md index 6917964..9df5164 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,7 @@ # pf9-mors Mors is the Roman God of death. Mors helps us implement leases. + +The functionality is described here in details: +https://platform9.atlassian.net/wiki/pages/viewpage.action?pageId=58490897 + + diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..4150a58 --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -e +source ../pf9-version/pf9-version.rc +export ROOT_DIR=`pwd` +cd $ROOT_DIR/support/ +make all diff --git a/etc/init.d/pf9-mors b/etc/init.d/pf9-mors new file mode 100755 index 0000000..c31cd78 --- /dev/null +++ b/etc/init.d/pf9-mors @@ -0,0 +1,100 @@ +#!/bin/sh +# +# chkconfig: - 98 02 +# description: Platform9 Lease Manager (code -named Mors) + +### BEGIN INIT INFO +# Provides: pf9-mors +# Required-Start: $remote_fs $network $syslog +# Required-Stop: $remote_fs $syslog +# Default-Stop: 0 1 6 +# Short-Description: Mors Server +# Description: Platform9 Lease Manager (code -named Mors) +### END INIT INFO + +. /etc/rc.d/init.d/functions + +name=pf9-mors +prog=pf9-mors +bindir=/opt/pf9/$name/bin +python=$bindir/python +exec="$python $bindir/pf9_mors.py" +pidfile="/var/run/$name.pid" +log_stdout="/var/log/pf9/$name-out.log" + +[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog + +lockfile=/var/lock/subsys/$prog + +start() { + [ -x $python ] || exit 5 + echo -n $"Starting $prog: " + daemon --pidfile $pidfile "$exec >> $log_stdout 2>&1 & echo \$! > $pidfile" + retval=$? + echo + [ $retval -eq 0 ] && touch $lockfile + return $retval +} + +stop() { + echo -n $"Stopping $prog: " + killproc -p $pidfile $prog + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + status -p $pidfile $prog +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac +exit $? diff --git a/etc/nginx/conf.d/locations/mors.conf b/etc/nginx/conf.d/locations/mors.conf new file mode 100644 index 0000000..123b0c7 --- /dev/null +++ b/etc/nginx/conf.d/locations/mors.conf @@ -0,0 +1,4 @@ +location /mors/ { + proxy_pass http://127.0.0.1:8989/; + include /etc/nginx/conf.d/pf9/cors.conf; +} \ No newline at end of file diff --git a/etc/pf9/pf9-mors-api-paste.ini b/etc/pf9/pf9-mors-api-paste.ini new file mode 100644 index 0000000..2c0d723 --- /dev/null +++ b/etc/pf9/pf9-mors-api-paste.ini @@ -0,0 +1,14 @@ +[app:myService] +paste.app_factory = mors.mors_wsgi:app_factory + +[pipeline:main] +pipeline = authtoken myService + +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory +auth_host = 127.0.0.1 +auth_port = 35357 +auth_protocol = http +admin_token = RzvUwrEgiOaQFTXV +auth_uri = http://127.0.0.1:8080/keystone +identity_uri = http://127.0.0.1:8080/keystone_admin diff --git a/etc/pf9/pf9-mors.ini b/etc/pf9/pf9-mors.ini new file mode 100644 index 0000000..daf725f --- /dev/null +++ b/etc/pf9/pf9-mors.ini @@ -0,0 +1,16 @@ +[DEFAULT] +db_conn= +context_factory= +lease_handler= +listen_port=8989 +sleep_seconds=300 +paste-ini=/etc/pf9/pf9-mors-api-paste.ini +log_file=/var/log/pf9/pf9-mors.log +repo=/opt/pf9/pf9-mors/lib/python2.7/site-packages/mors_repo +[nova] +user_name= +password= +version=2 +auth_url= +region_name= + diff --git a/mors/__init__.py b/mors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mors/context_util.py b/mors/context_util.py new file mode 100644 index 0000000..1a1e82f --- /dev/null +++ b/mors/context_util.py @@ -0,0 +1,69 @@ +# Copyright (c) 2016 Platform9 Systems Inc. +# All Rights reserved + +from flask import request, jsonify +import functools, os + + +def get_context(): + return Context(request.headers['X-User-Id'], + request.headers['X-User'], + request.headers['X-Roles'], + request.headers['X-Tenant-Id']) + + +def error_handler(func): + from sqlalchemy.exc import IntegrityError + import traceback,sys + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except ValueError as exc: + traceback.print_exc(file=sys.stdout) + return jsonify({'error': 'Invalid input'}), 422, {'ContentType': 'application/json'} + + except IntegrityError as exc: + traceback.print_exc(file=sys.stdout) + return jsonify({'error': 'Already exists'}), 409, {'ContentType': 'application/json'} + + return inner + +def enforce(required=[]): + """ + Generates a decorator that checks permissions before calling the + contained pecan handler function. + :param list[str] required: Roles require to run function. + """ + + def _enforce(fun): + + @functools.wraps(fun) + def newfun(self, *args, **kwargs): + + if not (required): + return fun(*args, **kwargs) + else: + roles_hdr = request.headers('X-Roles') + if roles_hdr: + roles = roles_hdr.split(',') + else: + roles = [] + + if set(roles) & set(required): + + return fun( *args, **kwargs) + else: + return jsonify({'error': 'Unauthorized'}), 403, {'ContentType': 'application/json'} + + return newfun + + return _enforce + + +class Context: + def __init__(self, user_id, user_name, roles_str, tenant_id): + self.user_id = user_id + self.user_name = user_name + self.roles = roles_str.split(',') + self.tenant_id = tenant_id diff --git a/mors/lease_manager.py b/mors/lease_manager.py new file mode 100644 index 0000000..b590f2b --- /dev/null +++ b/mors/lease_manager.py @@ -0,0 +1,186 @@ +# Copyright Platform9 Systems Inc. 2016 + +from datetime import datetime, timedelta + +from leasehandler import get_lease_handler +from persistence import DbPersistence +from eventlet.greenthread import spawn_after +import logging +from leasehandler.constants import SUCCESS_OK, ERR_UNKNOWN, ERR_NOT_FOUND + +logger = logging.getLogger(__name__) + + +def get_tenant_lease_data(data): + """ + Simple function to transform tenant database proxy object into an externally + consumable dictionary. + :param data: database row object + """ + return {'vm_lease_policy': {'tenant_uuid': data['tenant_uuid'], + 'expiry_days': data['expiry_days'], + 'created_at': data['created_at'], + 'created_by': data['created_by'], + 'updated_at': data['updated_at'], + 'updated_by': data['updated_by']}} + + +def get_vm_lease_data(data): + """ + Simple function to transform instance database proxy object into an externally + consumable dictionary. + :param data: database row object + """ + return {'instance_uuid': data['instance_uuid'], + 'tenant_uuid': data['tenant_uuid'], + 'expiry': data['expiry'], + 'created_at': data['created_at'], + 'created_by': data['created_by'], + 'updated_at': data['updated_at'], + 'updated_by': data['updated_by']} + + +class LeaseManager: + """ + Lease Manager is the main class for mors dealing with CRUD operations for the REST API + as well as the actual deletion of the Instances. Instance deletion and discovery is achieved + through an object 'leasehandler'. + """ + def __init__(self, conf): + self.domain_mgr = DbPersistence(conf.get("DEFAULT", "db_conn")) + self.lease_handler = get_lease_handler(conf) + self.sleep_seconds = conf.getint("DEFAULT", "sleep_seconds") + + def add_tenant_lease(self, context, tenant_obj): + logger.info("Adding tenant lease %s", tenant_obj) + self.domain_mgr.add_tenant_lease( + tenant_obj['tenant_uuid'], + tenant_obj['expiry_days'], + context.user_id, + datetime.utcnow()) + + def update_tenant_lease(self, context, tenant_obj): + logger.info("Update tenant lease %s", tenant_obj) + self.domain_mgr.update_tenant_lease( + tenant_obj['tenant_uuid'], + tenant_obj['expiry_days'], + context.user_id, + datetime.utcnow()) + + def delete_tenant_lease(self, context, tenant_id): + logger.info("Delete tenant lease %s", tenant_id) + return self.domain_mgr.delete_tenant_lease(tenant_id) + + def get_tenant_leases(self, context): + logger.debug("Getting all tenant lease") + all_tenants = self.domain_mgr.get_all_tenant_leases() + all_tenants = map(lambda x: get_tenant_lease_data(x), all_tenants) + logger.debug("Getting all tenant lease %s", all_tenants) + return all_tenants + + def get_tenant_lease(self, context, tenant_id): + data = self.domain_mgr.get_tenant_lease(tenant_id) + logger.debug("Getting tenant lease %s", data) + if data: + return get_tenant_lease_data(data) + return {} + + def get_tenant_and_associated_instance_leases(self, context, tenant_uuid): + logger.debug("Getting tenant and instances leases %s", tenant_uuid) + return { + 'tenant_lease': self.get_tenant_lease(context, tenant_uuid), + 'all_vms': + map(lambda x: get_vm_lease_data(x), self.domain_mgr.get_instance_leases_by_tenant(tenant_uuid)) + } + + # To Be Implemented + def check_instance_lease_violation(self, instance_lease, tenant_lease): + return True + + def get_instance_lease(self, context, instance_id): + data = self.domain_mgr.get_instance_lease(instance_id) + if data: + data = get_vm_lease_data(data) + logger.debug("Get instance lease %s %s", instance_id, data) + return data + + def add_instance_lease(self, context, tenant_uuid, instance_lease_obj): + logger.info("Add instance lease %s", instance_lease_obj) + tenant_lease = self.domain_mgr.get_tenant_lease(tenant_uuid) + self.check_instance_lease_violation(instance_lease_obj, tenant_lease) + self.domain_mgr.add_instance_lease(instance_lease_obj['instance_uuid'], + tenant_uuid, + instance_lease_obj['expiry'], + context.user_id, + datetime.utcnow()) + + def update_instance_lease(self, context, tenant_uuid, instance_lease_obj): + logger.info("Update instance lease %s", instance_lease_obj) + self.domain_mgr.update_instance_lease(instance_lease_obj['instance_uuid'], + tenant_uuid, + instance_lease_obj['expiry'], + context.user_id, + datetime.utcnow()) + + def delete_instance_lease(self, context, instance_uuid): + logger.info("Delete instance lease %s", instance_uuid) + self.domain_mgr.delete_instance_leases([instance_uuid]) + + def start(self): + spawn_after(self.sleep_seconds, self.run) + + # Could have used a generator here, would save memory but wonder if it is a good idea given the error conditions + # This is a simple implementation which goes and deletes VMs one by one + def _get_vms_to_delete_for_tenant(self, tenant_uuid, expiry_days): + vms_to_delete = [] + vm_ids_to_delete = set() + now = datetime.utcnow() + add_days = timedelta(days=expiry_days) + instance_leases = self.get_tenant_and_associated_instance_leases(None, tenant_uuid)['all_vms'] + for i_lease in instance_leases: + if now > i_lease['expiry']: + logger.info("Explicit lease for %s queueing for deletion", i_lease['instance_uuid']) + vms_to_delete.append(i_lease) + vm_ids_to_delete.add(i_lease['instance_uuid']) + else: + logger.debug("Ignoring vm, vm not expired yet %s", i_lease['instance_uuid']) + + tenant_vms = self.lease_handler.get_all_vms(tenant_uuid) + for vm in tenant_vms: + expiry_date = vm['created_at'] + add_days + if now > expiry_date and not (vm['instance_uuid'] in vm_ids_to_delete): + logger.info("Instance %s queued up for deletion creation date %s", vm['instance_uuid'], + vm['created_at']) + vms_to_delete.append(vm) + else: + logger.debug("Ignoring vm, vm not expired yet or already deleted %s, %s", vm['instance_uuid'], + vm['created_at']) + + return vms_to_delete + + def _delete_vms_for_tenant(self, t_lease): + tenant_vms_to_delete = self._get_vms_to_delete_for_tenant(t_lease['tenant_uuid'], t_lease['expiry_days']) + + # Keep it simple and delete them serially + result = self.lease_handler.delete_vms(tenant_vms_to_delete) + + remove_from_db = [] + + for vm_result in result.items(): + # If either the VM has been successfully deleted or has already been deleted + # Remove from our database + if vm_result[1] == SUCCESS_OK or vm_result[1] == ERR_NOT_FOUND: + remove_from_db.append(vm_result[0]) + + if len(remove_from_db) > 0: + logger.info("Removing vms %s from db", remove_from_db) + self.domain_mgr.delete_instance_leases(remove_from_db) + + def run(self): + # Delete the cleanup + tenant_leases = self.domain_mgr.get_all_tenant_leases() + for t_lease in tenant_leases: + self._delete_vms_for_tenant(t_lease) + + # Sleep again for sleep_seconds + spawn_after(self.sleep_seconds, self.run) diff --git a/mors/leasehandler/__init__.py b/mors/leasehandler/__init__.py new file mode 100644 index 0000000..e353ef6 --- /dev/null +++ b/mors/leasehandler/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2016 Platform9 Systems Inc. + +from nova_lease_handler import NovaLeaseHandler +from fake_lease_handler import FakeLeaseHandler +import constants + + +def get_lease_handler(conf): + if conf.get("DEFAULT", "lease_handler") == "test": + return FakeLeaseHandler(conf) + else: + return NovaLeaseHandler(conf) diff --git a/mors/leasehandler/constants.py b/mors/leasehandler/constants.py new file mode 100644 index 0000000..85d0e24 --- /dev/null +++ b/mors/leasehandler/constants.py @@ -0,0 +1,5 @@ +# Copyright Platform9 Systems Inc. 2016 + +SUCCESS_OK = 0 +ERR_NOT_FOUND = 1 +ERR_UNKNOWN = 2 diff --git a/mors/leasehandler/fake_lease_handler.py b/mors/leasehandler/fake_lease_handler.py new file mode 100644 index 0000000..2d69e12 --- /dev/null +++ b/mors/leasehandler/fake_lease_handler.py @@ -0,0 +1,38 @@ +# Copyright Platform9 Systems Inc. 2016 +import constants +import logging +from datetime import datetime + +# @TODO: Need to move this to a test folder + +class FakeLeaseHandler: + # Singleton tenants + tenants = {} + + def __init__(self,conf): + self.logger = logging.getLogger("test-lease-handler") + pass + + def add_tenant_data(self, tenant_id, instances): + FakeLeaseHandler.tenants[tenant_id] = instances + print FakeLeaseHandler.tenants + + def get_tenant_data(self, tenant_id): + return FakeLeaseHandler.tenants[tenant_id] + + def get_all_vms(self, tenant_uuid): + return FakeLeaseHandler.tenants[tenant_uuid] + + def delete_vm(self, tenant_uuid, vm_id): + vms = FakeLeaseHandler.tenants[tenant_uuid] + new_vm_data = filter(lambda x: x['instance_uuid'] != vm_id, vms) + FakeLeaseHandler.tenants[tenant_uuid] = new_vm_data + + + def delete_vms(self, vms): + result = {} + for vm in vms: + self.logger.info("Deleting VM vm %s", vm) + self.delete_vm(vm['tenant_uuid'], vm['instance_uuid']) + result[vm['instance_uuid']] = constants.SUCCESS_OK + return result diff --git a/mors/leasehandler/nova_lease_handler.py b/mors/leasehandler/nova_lease_handler.py new file mode 100644 index 0000000..8d8fd67 --- /dev/null +++ b/mors/leasehandler/nova_lease_handler.py @@ -0,0 +1,70 @@ +# Copyright 2016 Platform9 Systems Inc. +from novaclient import client +import logging +import novaclient +from datetime import datetime +from constants import SUCCESS_OK, ERR_NOT_FOUND, ERR_UNKNOWN +logger = logging.getLogger(__name__) + +DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + +def get_vm_data(data): + return {'instance_uuid': data.id, + 'tenant_uuid': data.tenant_id, + 'created_at': datetime.strptime(data.created)} + +class NovaLeaseHandler: + def __init__(self, conf): + self.conf = conf + + def _get_nova_client(self): + return client.Client(self.conf.get("nova", "version"), + username=self.conf.get("nova", "user_name"), + region_name=self.conf.get("nova", "region_name"), + tenant_id=self.conf.get("nova", "tenant_uuid"), + api_key=self.conf.get("nova", "password"), + auth_url=self.conf.get("nova", "auth_url"), + connection_pool=False) + + def get_all_vms(self, tenant_uuid): + """ + Get all vms for a given tenant + :param tenant_uuid: + :return: an iteratble that returns a set of vms (each vm has a UUID and a created_at field) + """ + try: + with self._get_nova_client() as nova: + vms = nova.servers.list(search_opts={'all_tenants':1, 'tenant_id':tenant_uuid}) + return map(lambda x: get_vm_data(x), vms) + except Exception as e: + logger.exception("Error getting list of vms for tenant %s", tenant_uuid) + + return [] + + def _delete_vm(self, nova, vm_uuid): + try: + logger.info("Deleting VM %s", vm_uuid) + nova.server.delete(vm_uuid) + return SUCCESS_OK + except novaclient.exceptions.NotFound: + return ERR_NOT_FOUND + except Exception as e: + logger.exception("Error deleting vm %s", vm_uuid) + return ERR_UNKNOWN + + def delete_vms(self, vms): + """ + Delete a VM on a given tenant + :param tenant_uuid: + :param vm_uuid: + :return: dictionary of vm_id to result + """ + result = {} + try: + with self._get_nova_client() as nova: + for vm in vms: + result[vm['instance_uuid']] = self._delete_vm(nova, vm['instance_uuid']) + return result + except Exception as e: + logger.exception("Error deleting vm %s", vms) + return result \ No newline at end of file diff --git a/mors/mors_wsgi.py b/mors/mors_wsgi.py new file mode 100644 index 0000000..35a1ce5 --- /dev/null +++ b/mors/mors_wsgi.py @@ -0,0 +1,130 @@ +# Copyright (c) 2016 Platform9 Systems Inc. +# All Rights reserved + +from flask import Flask, request, jsonify +from lease_manager import LeaseManager +from context_util import enforce, get_context, error_handler +from flask.json import JSONEncoder +from datetime import datetime + +DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +APP_NAME = "MORS" + + +class CustomJSONEncoder(JSONEncoder): + def default(self, obj): + try: + if isinstance(obj, datetime): + return obj.strftime(DATE_FORMAT) + iterable = iter(obj) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, obj) + + +app = Flask(__name__) +app.debug = True +app.json_encoder = CustomJSONEncoder + +lease_manager = None + +@enforce(required=['admin']) +@app.route("/v1/tenant", methods=['GET']) +@app.route("/v1/tenant/", methods=['GET']) +@error_handler +def get_all_tenants(): + all_tenants = lease_manager.get_tenant_leases(get_context()) + if all_tenants: + return jsonify({"all_tenants":all_tenants}) + else: + return jsonify({}), 200, {'ContentType': 'application/json'} + +@enforce(required=['_member_']) +@app.route("/v1/tenant/", methods=['GET']) +@error_handler +def get_tenant(tenant_id): + tenant_lease = lease_manager.get_tenant_lease(get_context(), tenant_id) + if not tenant_lease: + return jsonify({'success': False}), 404, {'ContentType': 'application/json'} + return jsonify(tenant_lease) + + +@enforce(required=['admin']) +@app.route("/v1/tenant/", methods=['PUT', 'POST']) +@error_handler +def add_update_tenant(tenant_id): + tenant_lease = request.get_json()["vm_lease_policy"] + if request.method == "POST": + lease_manager.add_tenant_lease(get_context(), tenant_lease) + else: + lease_manager.update_tenant_lease(get_context(), tenant_lease) + return jsonify({'success': True}), 200, {'ContentType': 'application/json'} + +@enforce(required=['admin']) +@app.route("/v1/tenant/", methods=['DELETE']) +@error_handler +def delete_tenant_lease(tenant_id): + lease_manager.delete_tenant_lease(get_context(), tenant_id) + return jsonify({'success': True}), 200, {'ContentType': 'application/json'} + +#-- Instnace - tenant related + +@enforce(required=['_member_']) +@app.route("/v1/tenant//instances/", methods=['GET']) +@error_handler +def get_tenant_and_instances(tenant_id): + instances = lease_manager.get_tenant_and_associated_instance_leases(get_context(), tenant_id) + if not instances: + return jsonify({'success': False}), 404, {'ContentType': 'application/json'} + return jsonify(instances) + +# --- Instance related --- +@enforce(required=['_member_']) +@app.route("/v1/tenant//instance/", methods=['GET']) +@error_handler +def get_vm_lease(tenant_id, instance_id): + lease_info = lease_manager.get_instance_lease(get_context(), instance_id) + if lease_info: + return jsonify(lease_info), 200, {'ContentType': 'application/json'} + else: + return jsonify({'error': 'Not found'}), 404, {'ContentType': 'application/json'} + +@enforce(required=['_member_']) +@app.route("/v1/tenant//instance/", methods=['DELETE']) +@error_handler +def delete_vm_lease(tenant_id, instance_id): + lease_manager.delete_instance_lease(get_context(), instance_id) + return jsonify({'success': True}), 200, {'ContentType': 'application/json'} + +@enforce(required=['_member_']) +@app.route("/v1/tenant//instance/", methods=['PUT', 'POST']) +@error_handler +def add_update_vm_lease(tenant_id, instance_id): + lease_obj = request.get_json() + # ds = '2012-03-01T10:00:00Z' # or any date sting of differing formats. + date = datetime.strptime(lease_obj['expiry'], DATE_FORMAT) + lease_obj['expiry'] = date + if request.method == "POST": + lease_manager.add_instance_lease(get_context(), tenant_id, lease_obj) + else: + lease_manager.update_tenant_lease(get_context(), tenant_id, lease_obj) + return jsonify({'success': True}), 200, {'ContentType': 'application/json'} + + +def start_server(conf): + global lease_manager + lease_manager = LeaseManager(conf) + lease_manager.start() + + +def shutdown_server(): + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + + +def app_factory(global_config, **local_conf): + return app diff --git a/mors/persistence.py b/mors/persistence.py new file mode 100644 index 0000000..04bd4e8 --- /dev/null +++ b/mors/persistence.py @@ -0,0 +1,108 @@ +# Copyright Platform9 Systems Inc. 2016 + +from sqlalchemy.pool import QueuePool +from sqlalchemy import create_engine, text +from sqlalchemy import Table, Column, Integer, String, MetaData, DateTime +import logging, functools + +logger = logging.getLogger(__name__) + + +def db_connect(transaction=False): + """ + Generates a decorator that get connection from a pool and returns + it to the pool when the internal function is done + :param transaction bool: should this function create and end transaction. + """ + + def _db_connect(fun): + if hasattr(fun, '__name__'): + fun.__name__ = 'method_decorator(%s)' % fun.__name__ + else: + fun.__name__ = 'method_decorator(%s)' % fun.__class__.__name__ + + @functools.wraps(fun) + def newfun(self, *args, **kwargs): + conn = self.engine.connect() + if transaction: + trans = conn.begin() + try: + ret = fun(self, conn, *args, **kwargs) + if transaction: + trans.commit() + return ret + except Exception as e: + if transaction: + trans.rollback() + logger.exception("Error during transaction ") + raise + finally: + conn.close() + + return newfun + + return _db_connect + + +class DbPersistence: + def __init__(self, db_conn_string): + self.engine = create_engine(db_conn_string, poolclass=QueuePool) + self.metadata = MetaData(bind=self.engine) + self.tenant_lease = Table('tenant_lease', self.metadata, autoload=True) + self.instance_lease = Table('instance_lease', self.metadata, autoload=True) + + @db_connect(transaction=False) + def get_all_tenant_leases(self, conn): + return conn.execute(self.tenant_lease.select()).fetchall() + + @db_connect(transaction=False) + def get_tenant_lease(self, conn, tenant_uuid): + return conn.execute(self.tenant_lease.select(tenant_uuid == tenant_uuid)).first() + + @db_connect(transaction=True) + def add_tenant_lease(self, conn, tenant_uuid, expiry_days, created_by, created_at): + logger.debug("Adding tenant lease %s %d %s %s", tenant_uuid, expiry_days, str(created_at), created_by) + conn.execute(self.tenant_lease.insert(), tenant_uuid=tenant_uuid, expiry_days=expiry_days, + created_at=created_at, created_by=created_by) + + @db_connect(transaction=True) + def update_tenant_lease(self, conn, tenant_uuid, expiry_days, updated_by, updated_at): + logger.debug("Updating tenant lease %s %d %s %s", tenant_uuid, expiry_days, str(updated_at), updated_by) + conn.execute(self.tenant_lease.update(tenant_uuid == tenant_uuid), expiry_days=expiry_days, + updated_at=updated_at, updated_by=updated_by) + + @db_connect(transaction=True) + def delete_tenant_lease(self, conn, tenant_uuid): + # Should we just soft delete ? + logger.debug("Deleting tenant lease %s", tenant_uuid) + conn.execute(self.tenant_lease.delete().where(tenant_uuid == tenant_uuid)) + conn.execute(self.instance_lease.delete().where(tenant_uuid == tenant_uuid)) + + @db_connect(transaction=False) + def get_instance_leases_by_tenant(self, conn, tenant_uuid): + return conn.execute(self.instance_lease.select(tenant_uuid == tenant_uuid)).fetchall() + + @db_connect(transaction=False) + def get_instance_lease(self, conn, instance_uuid): + return conn.execute( + self.instance_lease.select((instance_uuid == instance_uuid))).first() + + @db_connect(transaction=True) + def add_instance_lease(self, conn, instance_uuid, tenant_uuid, expiry, created_by, created_at): + logger.debug("Adding instance lease %s %s %s %s", instance_uuid, tenant_uuid, expiry, created_by) + conn.execute(self.instance_lease.insert(), instance_uuid=instance_uuid, tenant_uuid=tenant_uuid, + expiry=expiry, + created_at=created_at, created_by=created_by) + + @db_connect(transaction=True) + def update_instance_lease(self, conn, instance_uuid, tenant_uuid, expiry, updated_by, updated_at): + logger.debug("Updating instance lease %s %s %s %s", instance_uuid, tenant_uuid, expiry, updated_by) + conn.execute(self.instance_lease.update(instance_uuid == instance_uuid), tenant_uuid=tenant_uuid, + expiry=expiry, + updated_at=updated_at, updated_by=updated_by) + + @db_connect(transaction=True) + def delete_instance_leases(self, conn, instance_uuids): + # Delete 10 at a time, should we soft delete + logger.debug("Deleting instance leases %s", str(instance_uuids)) + conn.execute(self.instance_lease.delete().where(self.instance_lease.c.instance_uuid.in_(instance_uuids))) diff --git a/mors_manage.py b/mors_manage.py new file mode 100644 index 0000000..d0422a7 --- /dev/null +++ b/mors_manage.py @@ -0,0 +1,27 @@ +#!/opt/pf9/pf9-mors/bin/python +# Copyright (c) 2016 Platform9 Systems Inc. +# All Rights reserved +import argparse, logging +import ConfigParser +from migrate.versioning.api import upgrade, create, version_control + +def _get_arg_parser(): + parser = argparse.ArgumentParser(description="Lease Manager for VirtualMachines") + parser.add_argument('--config-file', dest='config_file', default='/etc/pf9/pf9-mors.ini') + parser.add_argument('--command', dest='command', default='db_sync') + return parser.parse_args() + +if __name__ == '__main__': + parser = _get_arg_parser() + conf = ConfigParser.ConfigParser() + conf.readfp(open(parser.config_file)) + if 'db_sync' == parser.command: + version_control(conf.get("DEFAULT", "db_conn"), conf.get("DEFAULT", "repo")) + upgrade(conf.get("DEFAULT", "db_conn"), , conf.get("DEFAULT", "repo")) + exit(0) + else: + print 'Unknown command' + exit(1) + + + diff --git a/mors_repo/README b/mors_repo/README new file mode 100644 index 0000000..6218f8c --- /dev/null +++ b/mors_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/mors_repo/__init__.py b/mors_repo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mors_repo/__init__.pyc b/mors_repo/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bc8428d8a2a201a9119aec531fe9cd89f23ad2d8 GIT binary patch literal 245 zcmZ8c%WA_g5L9Rmq0s;EC0V;kOb#LR3wmkDRR~Fuh4u1EG5BZtYyE*gM!mek2pw56#|2L;wH) literal 0 HcmV?d00001 diff --git a/mors_repo/manage.py b/mors_repo/manage.py new file mode 100644 index 0000000..39fa389 --- /dev/null +++ b/mors_repo/manage.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main + +if __name__ == '__main__': + main(debug='False') diff --git a/mors_repo/migrate.cfg b/mors_repo/migrate.cfg new file mode 100644 index 0000000..fc44185 --- /dev/null +++ b/mors_repo/migrate.cfg @@ -0,0 +1,25 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=Mors + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + +# When creating new change scripts, Migrate will stamp the new script with +# a version number. By default this is latest_version + 1. You can set this +# to 'true' to tell Migrate to use the UTC timestamp instead. +use_timestamp_numbering=False diff --git a/mors_repo/versions/001_Add_initial_tables.py b/mors_repo/versions/001_Add_initial_tables.py new file mode 100644 index 0000000..053690a --- /dev/null +++ b/mors_repo/versions/001_Add_initial_tables.py @@ -0,0 +1,36 @@ +# Copyright Platform9 Systems Inc. 2016 +from sqlalchemy import Table, Column, Integer, String, MetaData, DateTime + +meta = MetaData() + +tenant_lease = Table( + 'tenant_lease', meta, + Column('tenant_uuid', String(40), primary_key=True), + Column('expiry_days', Integer), + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('created_by', String(40)), + Column('updated_by', String(40)) +) + +vm_lease = Table( + 'instance_lease', meta, + Column('instance_uuid', String(40), primary_key=True), + Column('tenant_uuid', String(40)), + Column('expiry', DateTime), + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('created_by', String(40)), + Column('updated_by', String(40)) +) + +def upgrade(migrate_engine): + meta.bind = migrate_engine + tenant_lease.create() + vm_lease.create() + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + vm_lease.drop() + tenant_lease.drop() diff --git a/mors_repo/versions/__init__.py b/mors_repo/versions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mors_repo/versions/__init__.pyc b/mors_repo/versions/__init__.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81291aca91e97b2b18d54e6a5c6fa893d4e38c19 GIT binary patch literal 254 zcmZ8c%WA_g5L9RnrO^NIC0V;^O%5gG3wmkDRR~Fuh4u1EG5Cw}S^a@rLVh4?NG~0j zS(qJ$-KX2z>+j`yknDj4%^fj6lPJc1s@eIjNZ3EgXYwGS9YxVF3WQ^HN@E7ufy4Kp zaQ)d14VQh>2DARyw8N$*m;if7lk5cgWfHO%#s*WbicgRSC!FlGpO)SloSXmVNYyq5 vibByoIof4Fonp2n;si<`#mNgDsq|(o|mP)-NB`$?5I