From 76f9b5372b6472ade18f48e8a1052cac39e1e31b Mon Sep 17 00:00:00 2001 From: Nicola Peditto Date: Tue, 23 Oct 2018 13:00:53 +0200 Subject: [PATCH] NEW WebService module; Device Manager improved; installation procedures updated. Change-Id: I40a70948b830edc387cef6032f6f9fea036ec66f --- .gitignore | 3 +- MANIFEST.in | 2 + etc/iotronic/iotronic.conf | 5 +- iotronic_lightningrod/Board.py | 6 +- iotronic_lightningrod/common/exception.py | 10 +- iotronic_lightningrod/config.py | 2 +- iotronic_lightningrod/devices/Device.py | 3 +- iotronic_lightningrod/devices/gpio/Gpio.py | 2 +- .../devices/gpio/raspberry.py | 37 ++ iotronic_lightningrod/devices/gpio/server.py | 6 +- iotronic_lightningrod/devices/gpio/yun.py | 2 +- iotronic_lightningrod/devices/raspberry.py | 56 +++ iotronic_lightningrod/devices/server.py | 2 +- iotronic_lightningrod/devices/yun.py | 2 +- iotronic_lightningrod/lightningrod.py | 68 +++- iotronic_lightningrod/modules/Module.py | 2 +- .../modules/device_manager.py | 67 +++- .../modules/plugin_manager.py | 27 +- .../modules/service_manager.py | 2 +- iotronic_lightningrod/modules/test.py | 4 +- iotronic_lightningrod/modules/utils.py | 2 +- iotronic_lightningrod/modules/vfs_library.py | 2 +- iotronic_lightningrod/modules/vfs_manager.py | 2 +- .../modules/webservice_manager.py | 151 +++++++ iotronic_lightningrod/plugins/Plugin.py | 22 +- .../plugins/PluginSerializer.py | 2 +- iotronic_lightningrod/plugins/pluginApis.py | 2 +- .../plugins_examples/generics/echo.json | 1 + .../generics/{zero.py => echo.py} | 4 +- .../plugins_examples/generics/zero.json | 1 - iotronic_lightningrod/proxies/Proxy.py | 35 ++ iotronic_lightningrod/proxies/__init__.py | 0 iotronic_lightningrod/proxies/configs/default | 7 + .../proxies/configs/iotronic | 7 + iotronic_lightningrod/proxies/nginx.py | 374 ++++++++++++++++++ iotronic_lightningrod/wampmessage.py | 2 +- scripts/device_bkp_rest | 39 ++ scripts/lr_configure | 8 +- scripts/lr_install | 12 +- setup.cfg | 9 +- 40 files changed, 899 insertions(+), 91 deletions(-) create mode 100644 iotronic_lightningrod/devices/gpio/raspberry.py create mode 100644 iotronic_lightningrod/devices/raspberry.py create mode 100644 iotronic_lightningrod/modules/webservice_manager.py create mode 100644 iotronic_lightningrod/plugins/plugins_examples/generics/echo.json rename iotronic_lightningrod/plugins/plugins_examples/generics/{zero.py => echo.py} (86%) delete mode 100644 iotronic_lightningrod/plugins/plugins_examples/generics/zero.json create mode 100644 iotronic_lightningrod/proxies/Proxy.py create mode 100644 iotronic_lightningrod/proxies/__init__.py create mode 100644 iotronic_lightningrod/proxies/configs/default create mode 100644 iotronic_lightningrod/proxies/configs/iotronic create mode 100644 iotronic_lightningrod/proxies/nginx.py create mode 100755 scripts/device_bkp_rest diff --git a/.gitignore b/.gitignore index 96313bb..5761d21 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ ChangeLog *.md .eggs dist -__pycache__ +iotronic_lightningrod/modules/test.py +iotronic_lightningrod/modules/vfs_* \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 01e1d51..7ce3dfe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,6 +6,8 @@ include etc/systemd/system/s4t-lightning-rod.service include etc/logrotate.d/lightning-rod.log include scripts/lr_install include scripts/lr_configure +include scripts/device_bkp_rest +include iotronic_lightningrod/proxies/configs/* include AUTHORS include ChangeLog diff --git a/etc/iotronic/iotronic.conf b/etc/iotronic/iotronic.conf index 7c38e02..6dcfbc6 100644 --- a/etc/iotronic/iotronic.conf +++ b/etc/iotronic/iotronic.conf @@ -1,3 +1,6 @@ [DEFAULT] +lightningrod_home = /var/lib/iotronic +skip_cert_verify = True debug = True -log_file = /var/log/iotronic/lightning-rod.log \ No newline at end of file +proxy = nginx +log_file = /var/log/iotronic/lightning-rod.log diff --git a/iotronic_lightningrod/Board.py b/iotronic_lightningrod/Board.py index c29589c..106dc14 100644 --- a/iotronic_lightningrod/Board.py +++ b/iotronic_lightningrod/Board.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -__author__ = "Nicola Peditto " + str(meth[0]) + " registered!") + + async def DevicePing(self): + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + return datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f') + + async def DeviceReboot(self): + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + command = "reboot" + subprocess.call(command, shell=True) + + return datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f') + + async def DeviceHostname(self): + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + command = "hostname" + # subprocess.call(command, shell=True) + + out = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE + ) + + output = out.communicate()[0].decode('utf-8').strip() + print(output) + + return str(output) + "@" + \ + str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')) + + """ + async def DeviceWampDisconnect(self): + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + import threading, time + + def delayDisconnection(): + time.sleep(5) + SESSION.disconnect() + + threading.Thread(target=delayDisconnection).start() + + return "Device disconnection in 5 seconds..." + """ diff --git a/iotronic_lightningrod/modules/plugin_manager.py b/iotronic_lightningrod/modules/plugin_manager.py index e4b230f..2c4f02f 100644 --- a/iotronic_lightningrod/modules/plugin_manager.py +++ b/iotronic_lightningrod/modules/plugin_manager.py @@ -14,10 +14,10 @@ # under the License. from __future__ import absolute_import -__author__ = "Nicola Peditto " + proxy_type.upper() + " module imported!") + + proxy = proxy_module.ProxyManager() + + proxy_meth_list = inspect.getmembers( + proxy, + predicate=inspect.ismethod + ) + + RPC_proxies[proxy_type] = proxy_meth_list + + board.proxy = proxy + + self._proxyWampRegister(proxy_meth_list, board) + + else: + LOG.warning("Proxy '" + proxy_type + "' not supported!") + + def finalize(self): + + proxy_status = json.loads(self.board.proxy._proxyInfo()) + LOG.info("--> Proxy " + self.board.proxy.type.upper() + + " status:\n Active: " + str(proxy_status['status']) + + "\n Info: " + str(proxy_status['log'])) + + LOG.info("Webservice exposed on device:") + active_webservice_list = self.board.proxy._webserviceList() + if len(active_webservice_list) != 0: + for ws in active_webservice_list: + LOG.info("-> " + ws) + else: + LOG.info("-> NO WebService!") + + LOG.info("WebService Manager initialized!") + + def restore(self): + LOG.info("WebService Manager restored.") + + def _proxyWampRegister(self, proxy_meth_list, board): + + LOG.info(" - " + str(board.proxy.type).upper() + + " proxy registering RPCs:") + + for meth in proxy_meth_list: + if (meth[0] != "__init__") & (meth[0] != "finalize") \ + & (meth[0] != "restore"): + # LOG.info(" - " + str(meth[0])) + rpc_addr = u'iotronic.' + board.uuid + '.' + meth[0] + # LOG.debug(" --> " + str(rpc_addr)) + if not meth[0].startswith('_'): + SESSION.register(meth[1], rpc_addr) + LOG.info(" --> " + str(meth[0])) + + async def ExposeWebservice(self, service_name, local_port): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + response = self.board.proxy._exposeWebservice(service_name, local_port) + + response = json.loads(response) + + if(response['result'] == "SUCCESS"): + message = "Webservice '" + service_name + "' successfully exposed!" + LOG.info("--> " + str(message)) + w_msg = WM.WampSuccess(response) + else: + message = "Error exposing webservice '" + service_name + "'" + LOG.warning("--> " + str(response['message'])) + w_msg = WM.WampWarning(response) + + return w_msg.serialize() + + async def UnexposeWebservice(self, service_name): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + response = self.board.proxy._disableWebservice(service_name) + + response = json.loads(response) + + if (response['result'] == "SUCCESS"): + LOG.info("--> " + str(response['message'])) + w_msg = WM.WampSuccess(response) + else: + LOG.warning("--> " + str(response['message'])) + w_msg = WM.WampWarning(response) + + return w_msg.serialize() + + async def BoardDnsCertsSetup(self, board_dns, owner_email): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self.board.proxy._proxyBoardDnsSetup(board_dns, owner_email) + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() diff --git a/iotronic_lightningrod/plugins/Plugin.py b/iotronic_lightningrod/plugins/Plugin.py index 55787f8..28a3073 100644 --- a/iotronic_lightningrod/plugins/Plugin.py +++ b/iotronic_lightningrod/plugins/Plugin.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -__author__ = "Nicola Peditto " + nginxMsg['log']) + else: + nginxMsg['log'] = "NGINX is not running" + nginxMsg['status'] = False + # LOG.warning("--> " + nginxMsg['log']) + + except Exception as err: + nginxMsg['log'] = "Error check NGINX status: " + str(err) + nginxMsg['status'] = True + # LOG.error("--> " + nginxMsg['log']) + + return json.dumps(nginxMsg) + + def _proxyReload(self): + + nginxMsg = {} + + try: + + stat = subprocess.call('service nginx reload', shell=True) + + if stat != 0: + raise NginxError(str(stat)) + + else: + nginxMsg['log'] = "NGINX successfully reloaded" + nginxMsg['code'] = stat + LOG.info("--> " + nginxMsg['log']) + + except NginxError: + nginxMsg['log'] = "NGINX reloading error" + nginxMsg['code'] = stat + LOG.warning("--> " + nginxMsg['log']) + + except Exception as err: + nginxMsg['log'] = "NGINX Generic error: " + str(err) + nginxMsg['code'] = stat + LOG.warning("--> " + nginxMsg['log']) + + nginxMsg = json.dumps(nginxMsg) + return nginxMsg + + def _proxyRestart(self): + + nginxMsg = {} + + try: + stat = os.system('systemctl restart nginx') + + if stat != 0: + raise NginxError(str(stat)) + + else: + nginxMsg['log'] = "NGINX successfully restart" + nginxMsg['code'] = stat + LOG.info("--> " + nginxMsg['log']) + + except NginxError: + nginxMsg['log'] = "NGINX restarting error" + nginxMsg['code'] = stat + LOG.warning("--> " + nginxMsg['log']) + + except Exception as err: + nginxMsg['log'] = "NGINX generic error: " + str(err) + nginxMsg['code'] = stat + LOG.warning("--> " + nginxMsg['log']) + + return json.dumps(nginxMsg) + + def _proxyBoardDnsSetup(self, board_dns, owner_email): + + nginxMsg = {} + + try: + + py_dist_pack = site.getsitepackages()[0] + + iotronic_nginx_path = "/etc/nginx/conf.d/iotronic" + iotronic_nginx_default = "/etc/nginx/conf.d/iotronic/default" + + if not os.path.exists(iotronic_nginx_path): + os.makedirs(iotronic_nginx_path) + + nginx_default = '''proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade";''' + + with open(iotronic_nginx_default, "w") as text_file: + text_file.write("%s" % nginx_default) + + iotronic_nginx_avl_path = "/etc/nginx/sites-available/iotronic" + + string = '''server {{ + listen 80; + + server_name {0}; + include conf.d/iotronic/*; + }}'''.format(board_dns) + + with open(iotronic_nginx_avl_path, "w") as text_file: + text_file.write("%s" % string) + + os.system( + 'ln -s ' + '/etc/nginx/sites-available/iotronic ' + '/etc/nginx/sites-enabled/' + ) + + time.sleep(3) + self._proxyReload() + time.sleep(3) + + command = '/usr/bin/certbot -n ' \ + '--redirect --authenticator webroot ' \ + '--installer nginx -w /var/www/html/ ' \ + '--domain ' + board_dns + ' --agree-tos ' \ + '--email ' + owner_email + + LOG.debug(command) + call(command, shell=True) + + except Exception as err: + nginxMsg['log'] = "NGINX DNS setup error: " + str(err) + nginxMsg['code'] = "" + LOG.warning("--> " + nginxMsg['log']) + + return json.dumps(nginxMsg) + + def _exposeWebservice(self, service_name, local_port): + + nginxMsg = {} + + try: + + nginx_path = "/etc/nginx/conf.d/iotronic" + + if not os.path.exists(nginx_path): + os.makedirs(nginx_path) + + fp = nginx_path + "/" + service_name + + string = '''location /{0}/ {{ + proxy_pass http://localhost:{1}/; + include conf.d/iotronic/default; + }} + + location /{0} {{ + rewrite ^ $scheme://$http_host/{0}/ redirect; + }} + '''.format(service_name, local_port) + + with open(fp, "w") as ws_nginx_conf: + ws_nginx_conf.write("%s" % string) + + time.sleep(3) + + nginxMsg['message'] = "Webservice '" + service_name + \ + "' configuration injected in NGINX." + nginxMsg['result'] = "SUCCESS" + LOG.info("--> " + nginxMsg['message']) + + self._proxyReload() + + time.sleep(3) + + except Exception as e: + nginxMsg['message'] = "Error exposing Webservice '" + \ + service_name + \ + "' configuration in NGINX: {}".format(e) + nginxMsg['result'] = "ERROR" + LOG.warning("--> " + nginxMsg['message']) + + return json.dumps(nginxMsg) + + def _disableWebservice(self, service_name): + + nginxMsg = {} + + try: + + nginx_path = "/etc/nginx/conf.d/iotronic" + service_path = nginx_path + "/" + service_name + + if os.path.exists(service_path): + + os.remove(service_path) + + time.sleep(3) + + nginxMsg['message'] = "webservice '" \ + + service_name + "' disabled." + nginxMsg['result'] = "SUCCESS" + # LOG.info("--> " + nginxMsg['message']) + + self._proxyReload() + + time.sleep(3) + + else: + nginxMsg['message'] = "webservice file " \ + + service_path + " does not exist" + nginxMsg['result'] = "ERROR" + # LOG.info("--> " + nginxMsg['message']) + + except Exception as e: + nginxMsg['message'] = "Error disabling Webservice '" + \ + service_name + "': {}".format(e) + nginxMsg['result'] = "ERROR" + # LOG.warning("--> " + nginxMsg['message']) + + return json.dumps(nginxMsg) + + def _webserviceList(self): + + nginx_path = "/etc/nginx/conf.d/iotronic" + + if os.path.exists(nginx_path): + service_list = [f for f in os.listdir(nginx_path) + if os.path.isfile(os.path.join(nginx_path, f))] + + service_list.remove('default') + else: + service_list = [] + + return service_list + + async def NginxInfo(self): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self._proxyInfo() + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() + + async def NginxStatus(self): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self._proxyStatus() + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() + + async def NginxReload(self): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self._proxyReload() + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() + + async def NginxRestart(self): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self._proxyRestart() + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() + + async def NginxIotronicConf(self): + + rpc_name = utils.getFuncName() + LOG.info("RPC " + rpc_name + " CALLED") + + message = self._proxyIotronicConf() + w_msg = WM.WampSuccess(message) + + return w_msg.serialize() diff --git a/iotronic_lightningrod/wampmessage.py b/iotronic_lightningrod/wampmessage.py index 12b58db..6714b68 100644 --- a/iotronic_lightningrod/wampmessage.py +++ b/iotronic_lightningrod/wampmessage.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -__author__ = "Nicola Peditto " + exit + fi + # BACKUP + echo "Backing up Iotronic configuration" + now_date=`date '+%Y%m%d%H%M%S'` + device=`cat /etc/iotronic/settings.json | grep name | awk '{print $2}' | tr -d \" | tr -d ,` + bkp_filename="bkp_"$device"_"$now_date".tar.gz" + echo "-> backup filename: " $bkp_filename + tar zcvf $bkp_filename /var/lib/iotronic /etc/iotronic /etc/letsencrypt /etc/nginx > /dev/null + +elif [ "$1" = "restore" ]; then + if [ "$#" -ne 2 ]; then + echo "You have to specify: 'restore' " + exit + fi + # RESTORE + echo "Restoring Iotronic configuration" + tar -xvzf $2 -C / + + service nginx restart + + sleep 3 + + echo -e "\nCompleted!" + +else + echo "You have to specify:" + echo " - for backup: 'backup'" + echo " - for restore: 'restore' " + exit +fi diff --git a/scripts/lr_configure b/scripts/lr_configure index 53b072d..6e5334a 100755 --- a/scripts/lr_configure +++ b/scripts/lr_configure @@ -22,8 +22,8 @@ if len(sys.argv) < 3: print('Arguments required: " ', str(sys.argv)) else: - os.system('sed -i "s||' - + sys.argv[1] + '|g" /etc/iotronic/settings.json') - os.system('sed -i "s|ws://:/|' - + sys.argv[2] + '|g" /etc/iotronic/settings.json') + os.system('sed -i "s|\\"code\\":.*|\\"code\\": \\"' + + sys.argv[1] + '\\"|g" /etc/iotronic/settings.json') + os.system('sed -i "s|\\"url\\":.*|\\"url\\": \\"' + + sys.argv[2] + '\\",|g" /etc/iotronic/settings.json') os.system('sed -i "s||s4t|g" /etc/iotronic/settings.json') diff --git a/scripts/lr_install b/scripts/lr_install index dadd12b..b1f6950 100755 --- a/scripts/lr_install +++ b/scripts/lr_install @@ -18,10 +18,18 @@ import os import site +if os.path.exists('/iotronic_lightningrod/'): + print("Cleaning install folders...") + os.system('rm -rf ' + site.getsitepackages()[0] + + '/iotronic_lightningrod/etc') + os.system('rm -rf ' + site.getsitepackages()[0] + + '/iotronic_lightningrod/templates') + print("Moving installation folders...") + os.system('mv -f /iotronic_lightningrod/* ' + + site.getsitepackages()[0] + '/iotronic_lightningrod/') py_dist_pack = site.getsitepackages()[0] -print(py_dist_pack) - +print("Python packages folder: " + py_dist_pack) print('Iotronic environment creation:') if not os.path.exists('/etc/iotronic/'): diff --git a/setup.cfg b/setup.cfg index 94d15d0..f13caac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ summary = Implementation of the Lightning-rod, the Stack4Things board-side probe description-file = README.rst author = Nicola Peditto, Fabio Verboso -author-email = unime.mdslab@gmail.com +author-email = n.peditto@gmail.com home-page = http://stack4things.unime.it/ classifier = Environment :: OpenStack @@ -21,7 +21,7 @@ classifier = [files] packages = iotronic_lightningrod -data_files = +data-files = bin/ = scripts/* /iotronic_lightningrod/templates = templates/* /iotronic_lightningrod/etc/iotronic = etc/iotronic/iotronic.conf @@ -65,8 +65,9 @@ s4t.modules = plugin = iotronic_lightningrod.modules.plugin_manager:PluginManager device = iotronic_lightningrod.modules.device_manager:DeviceManager service = iotronic_lightningrod.modules.service_manager:ServiceManager - network = iotronic_lightningrod.modules.network_manager:NetworkManager -# vfs = iotronic_lightningrod.modules.vfs_manager:VfsManager + # network = iotronic_lightningrod.modules.network_manager:NetworkManager + webservice = iotronic_lightningrod.modules.webservice_manager:WebServiceManager + # vfs = iotronic_lightningrod.modules.vfs_manager:VfsManager [options] build_scripts =