From 88d5be79304869090ffa1ea9f5aa59520395ce82 Mon Sep 17 00:00:00 2001 From: Fabio Verboso Date: Tue, 6 Feb 2018 11:03:40 +0100 Subject: [PATCH] Moved to python3.5 Now Iotronic requires python3 and python2 is not supported anymore Change-Id: I2d2fdccb1788412d21d216052fb843ff24061040 --- etc/iotronic/iotronic.conf | 2 +- iotronic/conductor/endpoints.py | 7 +- iotronic/db/sqlalchemy/models.py | 2 - iotronic/openstack/common/fileutils.py | 4 +- iotronic/wamp/agent.py | 203 +++++++++++-------------- iotronic/wamp/functions.py | 2 +- requirements.txt | 5 +- setup.cfg | 9 +- test-requirements.txt | 11 +- tox.ini | 48 +++--- 10 files changed, 138 insertions(+), 155 deletions(-) diff --git a/etc/iotronic/iotronic.conf b/etc/iotronic/iotronic.conf index 222add3..8501952 100644 --- a/etc/iotronic/iotronic.conf +++ b/etc/iotronic/iotronic.conf @@ -25,7 +25,7 @@ wamp_realm = s4t [database] -connection = mysql://:@/iotronic +connection = mysql+pymsql://:@/iotronic [keystone_authtoken] auth_uri = http://:5000 diff --git a/iotronic/conductor/endpoints.py b/iotronic/conductor/endpoints.py index 0bb4cf9..3ff35d7 100644 --- a/iotronic/conductor/endpoints.py +++ b/iotronic/conductor/endpoints.py @@ -1,5 +1,6 @@ -# coding=utf-8 - +# Copyright 2017 MDSLAB - University of Messina +# All Rights Reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -import cPickle as cpickle +import _pickle as cpickle from iotronic.common import exception from iotronic.common import states from iotronic.conductor.provisioner import Provisioner diff --git a/iotronic/db/sqlalchemy/models.py b/iotronic/db/sqlalchemy/models.py index b80c34d..ae4a837 100644 --- a/iotronic/db/sqlalchemy/models.py +++ b/iotronic/db/sqlalchemy/models.py @@ -21,7 +21,6 @@ SQLAlchemy models for iot data. import json from oslo_config import cfg -from oslo_db import options as db_options from oslo_db.sqlalchemy import models import six.moves.urllib.parse as urlparse from sqlalchemy import Boolean @@ -43,7 +42,6 @@ _DEFAULT_SQL_CONNECTION = 'sqlite:///' + \ paths.state_path_def('iotronic.sqlite') cfg.CONF.register_opts(sql_opts, 'database') -db_options.set_defaults(cfg.CONF, _DEFAULT_SQL_CONNECTION, 'iotronic.sqlite') def table_args(): diff --git a/iotronic/openstack/common/fileutils.py b/iotronic/openstack/common/fileutils.py index e1bdb93..30a7f3b 100644 --- a/iotronic/openstack/common/fileutils.py +++ b/iotronic/openstack/common/fileutils.py @@ -16,11 +16,11 @@ import contextlib import errno import os +from oslo_log import log as logging +from oslo_utils import excutils import stat import tempfile -from oslo_utils import excutils -from oslo_log import log as logging LOG = logging.getLogger(__name__) diff --git a/iotronic/wamp/agent.py b/iotronic/wamp/agent.py index 2d7f5f3..bce6547 100644 --- a/iotronic/wamp/agent.py +++ b/iotronic/wamp/agent.py @@ -13,10 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. -from autobahn.twisted import wamp -from autobahn.twisted import websocket -from autobahn.wamp import types -from twisted.internet.defer import inlineCallbacks +import asyncio +import txaio from iotronic.common import exception from iotronic.common.i18n import _LI @@ -26,14 +24,15 @@ from oslo_config import cfg from oslo_log import log as logging import oslo_messaging from oslo_messaging.rpc import dispatcher -import threading + from threading import Thread -from twisted.internet.protocol import ReconnectingClientFactory -from twisted.internet import reactor + import os import signal +from autobahn.asyncio.component import Component + LOG = logging.getLogger(__name__) wamp_opts = [ @@ -58,114 +57,38 @@ wamp_opts = [ CONF = cfg.CONF CONF.register_opts(wamp_opts, 'wamp') -shared_result = {} +txaio.start_logging(level="info") + wamp_session_caller = None AGENT_HOST = None +LOOP = None +connected = False -def wamp_request(e, kwarg, session): - id = threading.current_thread().ident - shared_result[id] = {} - shared_result[id]['result'] = None - - def success(d): - shared_result[id]['result'] = d - LOG.debug("DEVICE sent: %s", str(d)) - e.set() - return shared_result[id]['result'] - - def fail(failure): - shared_result[id]['result'] = failure - LOG.error("WAMP FAILURE: %s", str(failure)) - e.set() - return shared_result[id]['result'] - - LOG.debug("Calling %s...", kwarg['wamp_rpc_call']) - d = session.wamp_session.call(wamp_session_caller, - kwarg['wamp_rpc_call'], *kwarg['data']) - d.addCallback(success) - d.addErrback(fail) +async def wamp_request(kwarg): + LOG.debug("calling: " + kwarg['wamp_rpc_call']) + d = await wamp_session_caller.call(kwarg['wamp_rpc_call'], *kwarg['data']) + return d # OSLO ENDPOINT class WampEndpoint(object): - def __init__(self, wamp_session, agent_uuid): - self.wamp_session = wamp_session + def __init__(self, agent_uuid): setattr(self, agent_uuid + '.s4t_invoke_wamp', self.s4t_invoke_wamp) def s4t_invoke_wamp(self, ctx, **kwarg): - e = threading.Event() - LOG.debug("CONDUCTOR sent me:", kwarg) + LOG.debug("CONDUCTOR sent me: " + kwarg['wamp_rpc_call']) - th = threading.Thread(target=wamp_request, args=(e, kwarg, self)) - th.start() + r = asyncio.run_coroutine_threadsafe(wamp_request(kwarg), LOOP) - e.wait() - LOG.debug("result received from wamp call: %s", - str(shared_result[th.ident]['result'])) - - result = shared_result[th.ident]['result'] - del shared_result[th.ident]['result'] - return result - - -class WampFrontend(wamp.ApplicationSession): - @inlineCallbacks - def onJoin(self, details): - global wamp_session_caller, AGENT_HOST - wamp_session_caller = self - - import iotronic.wamp.functions as fun - - self.subscribe(fun.board_on_leave, 'wamp.session.on_leave') - self.subscribe(fun.board_on_join, 'wamp.session.on_join') - - try: - if CONF.wamp.register_agent: - self.register(fun.registration, u'stack4things.register') - LOG.info("I have been set as registration agent") - self.register(fun.connection, - AGENT_HOST + u'.stack4things.connection') - self.register(fun.echo, - AGENT_HOST + u'.stack4things.echo') - LOG.info("procedure registered") - except Exception as e: - LOG.error("could not register procedure: {0}".format(e)) - - LOG.info("WAMP session ready.") - - session_l = yield self.call(u'wamp.session.list') - session_l.remove(details.session) - fun.update_sessions(session_l) - - def onDisconnect(self): - LOG.info("disconnected") - - -class WampClientFactory(websocket.WampWebSocketClientFactory, - ReconnectingClientFactory): - maxDelay = 30 - - def clientConnectionFailed(self, connector, reason): - # print "reason:", reason - LOG.warning("Wamp Connection Failed.") - ReconnectingClientFactory.clientConnectionFailed(self, - connector, reason) - - def clientConnectionLost(self, connector, reason): - # print "reason:", reason - LOG.warning("Wamp Connection Lost.") - ReconnectingClientFactory.clientConnectionLost(self, - connector, reason) + return r.result() class RPCServer(Thread): def __init__(self): - global AGENT_HOST - # AMQP CONFIG endpoints = [ - WampEndpoint(WampFrontend, AGENT_HOST), + WampEndpoint(AGENT_HOST), ] Thread.__init__(self) @@ -191,28 +114,84 @@ class RPCServer(Thread): class WampManager(object): def __init__(self): - component_config = types.ComponentConfig( - realm=unicode(CONF.wamp.wamp_realm)) - session_factory = wamp.ApplicationSessionFactory( - config=component_config) - session_factory.session = WampFrontend - transport_factory = WampClientFactory(session_factory, - url=CONF.wamp.wamp_transport_url) - - transport_factory.autoPingInterval = CONF.wamp.autoPingInterval - transport_factory.autoPingTimeout = CONF.wamp.autoPingTimeout LOG.debug("wamp url: %s wamp realm: %s", CONF.wamp.wamp_transport_url, CONF.wamp.wamp_realm) - websocket.connectWS(transport_factory) + + self.loop = asyncio.get_event_loop() + global LOOP + LOOP = self.loop + + comp = Component( + transports=CONF.wamp.wamp_transport_url, + realm=CONF.wamp.wamp_realm + ) + + self.comp = comp + + @comp.on_join + async def onJoin(session, details): + + global connected + connected = True + + global wamp_session_caller, AGENT_HOST + wamp_session_caller = session + + import iotronic.wamp.functions as fun + + session.subscribe(fun.board_on_leave, + 'wamp.session.on_leave') + session.subscribe(fun.board_on_join, + 'wamp.session.on_join') + + try: + if CONF.wamp.register_agent: + session.register(fun.registration, + u'stack4things.register') + LOG.info("I have been set as registration agent") + session.register(fun.connection, + AGENT_HOST + + + u'.stack4things.connection') + session.register(fun.echo, + AGENT_HOST + + + u'.stack4things.echo') + LOG.debug("procedure registered") + + except Exception as e: + LOG.error("could not register procedure: {0}".format(e)) + + LOG.info("WAMP session ready.") + + session_l = await session.call(u'wamp.session.list') + session_l.remove(details.session) + fun.update_sessions(session_l) + + @comp.on_leave + async def onLeave(session, details): + LOG.warning('WAMP Session Left: ' + str(details)) + + @comp.on_disconnect + async def onDisconnect(session, was_clean): + LOG.warning('WAMP Transport Left: ' + str(was_clean)) + + global connected + connected = False + if not connected: + comp.start(self.loop) def start(self): LOG.info("Starting WAMP server...") - reactor.run() + self.comp.start(self.loop) + self.loop.run_forever() def stop(self): - LOG.info("Stopping WAMP-agent server...") - reactor.stop() + LOG.info("Stopping WAMP server...") + + # Canceling pending tasks and stopping the loop + asyncio.gather(*asyncio.Task.all_tasks()).cancel() + # Stopping the loop + self.loop.stop() LOG.info("WAMP server stopped.") @@ -222,9 +201,13 @@ class WampAgent(object): signal.signal(signal.SIGINT, self.stop_handler) logging.register_options(CONF) + CONF(project='iotronic') logging.setup(CONF, "iotronic-wamp-agent") + if CONF.debug: + txaio.start_logging(level="debug") + # to be removed asap self.host = host self.dbapi = dbapi.get_instance() diff --git a/iotronic/wamp/functions.py b/iotronic/wamp/functions.py index 736053f..22717e9 100644 --- a/iotronic/wamp/functions.py +++ b/iotronic/wamp/functions.py @@ -87,7 +87,7 @@ def board_on_leave(session_id): board = objects.Board.get_by_uuid(ctxt, old_session.board_uuid) board.status = states.OFFLINE board.save() - LOG.debug('Board %s is now %s', old_session.uuid, states.OFFLINE) + LOG.debug('Board %s is now %s', board.uuid, states.OFFLINE) def connection(uuid, session): diff --git a/requirements.txt b/requirements.txt index f5ccd25..3980c3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,8 +12,9 @@ oslo.policy>=1.15.0 # Apache-2.0 oslo.messaging>=5.2.0 # Apache-2.0 oslo.db!=4.13.1,!=4.13.2,>=4.11.0 # Apache-2.0 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD -#paramiko>=2.0 # LGPLv2.1+ +paramiko>=2.0.0 # LGPLv2.1+ +PyMySQL>=0.7.6 # MIT License +SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0 autobahn>=0.10.1 # MIT License -#Twisted>=16.5.0 # MIT WSME>=0.8 # MIT diff --git a/setup.cfg b/setup.cfg index d8d1906..e6f81c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,8 +14,6 @@ classifier = License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.5 @@ -23,8 +21,11 @@ classifier = console_scripts = iotronic-conductor = iotronic.cmd.conductor:main iotronic-wamp-agent = iotronic.cmd.wamp_agent:main - - + +[options] +build_scripts = + executable= /usr/bin/env python + [files] packages = iotronic diff --git a/test-requirements.txt b/test-requirements.txt index fe73c84..0390e34 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,16 +12,19 @@ oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT +trollius>=1.0 # Apache-2.0 eventlet!=0.18.3,>=0.18.2 # MIT oslo.config!=3.18.0,>=3.14.0 # Apache-2.0 oslo.log>=3.11.0 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.policy>=1.15.0 # Apache-2.0 oslo.messaging>=5.2.0 # Apache-2.0 -oslo.db!=4.13.1,!=4.13.2,>=4.11.0 # Apache-2.0 +oslo.db!=4.13.1,!=4.13.2,>=4.11.0 # Apache-2.0 +paramiko>=2.0.0 # LGPLv2.1+ pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD -#paramiko>=2.0 # LGPLv2.1+ +PyMySQL>=0.7.6 # MIT License +SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT keystonemiddleware!=4.5.0,>=4.2.0 # Apache-2.0 autobahn>=0.10.1 # MIT License -#Twisted>=16.5.0 # MIT -WSME>=0.8 # MIT \ No newline at end of file +WSME>=0.8 # MIT + diff --git a/tox.ini b/tox.ini index 9f4d32d..99020df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,43 +1,39 @@ [tox] -minversion = 2.0 -envlist = py35,py34,py27,pypy,pep8 +minversion = 2.3.1 +envlist = py35,pep8 skipsdist = True [testenv] -usedevelop = True -# tox is silly... these need to be separated by a newline.... +setenv = + VIRTUAL_ENV={envdir} + PYTHONWARNINGS=default::DeprecationWarning + LANGUAGE=en_US + LC_ALL=en_US.utf-8 whitelist_externals = bash - find - rm + find + rm +usedevelop = True install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} -setenv = VIRTUAL_ENV={envdir} - LANGUAGE=en_US - LC_ALL=en_US.utf-8 deps = -r{toxinidir}/test-requirements.txt commands = - find . -type f -name "*.pyc" -delete - -[testenv:py27] -commands = - {[testenv]commands} + find . -type f -name "*.pyc" -delete [testenv:pep8] +basepython = python2.7 commands = flake8 {posargs} -[testenv:venv] -commands = {posargs} +[testenv:py35] +basepython = python3.5 -[testenv:cover] -commands = python setup.py test --coverage --testr-args='{posargs}' - -[testenv:docs] -commands = python setup.py build_sphinx - -[testenv:debug] -commands = oslo_debug_helper {posargs} [flake8] - +# TODO(dmllr): Analyze or fix the warnings blacklisted below +# E711 comparison to None should be 'if cond is not None:' +# E712 comparison to True should be 'if cond is True:' or 'if cond:' +# H404 multi line docstring should start with a summary +# H405 multi line docstring summary not separated with an empty line +# E123, E125 skipped as they are invalid PEP-8. show-source = True builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build +ignore = E711,E712,H404,H405,E123,E125,E901,H301 +exclude = .venv,.git,.tox,dist,doc,etc,*lib/python*,*egg,build