From 1eeb1de33a995b15ff77bbc82234d0ca838ed8f0 Mon Sep 17 00:00:00 2001 From: Tong Li Date: Tue, 29 Mar 2016 23:36:34 -0400 Subject: [PATCH] Changed vagrant file so that installation can be easier Change-Id: I986c293d9f0a63cbc63c52ce68b1d821b681f48c --- .gitignore | 4 +- ChangeLog | 23 +- tools/colorizer.py | 330 ------------------ tools/install_venv.py | 73 ---- tools/install_venv_common.py | 172 --------- tools/vagrant/rebootvm.sh | 7 + tools/vagrant/stvm.sh | 3 + tools/vagrant/ubuntu/Vagrantfile | 92 ++--- tools/vagrant/ubuntu/onvm/conf/ids.conf.yml | 7 + tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml | 49 +++ tools/vagrant/ubuntu/onvm/scripts/ini-config | 330 ++++++++++++++++++ .../ubuntu/onvm/scripts/install-agent.sh | 12 + .../ubuntu/onvm/scripts/install-devstack.sh | 12 + .../ubuntu/onvm/scripts/install-elastic.sh | 20 ++ .../ubuntu/onvm/scripts/install-java.sh | 16 + .../ubuntu/onvm/scripts/install-kafka.sh | 21 ++ .../ubuntu/onvm/scripts/install-kiloeyes.sh | 19 + .../ubuntu/ubuntu-virtualbox.yml.sample | 6 - tools/with_venv.sh | 7 - 19 files changed, 556 insertions(+), 647 deletions(-) mode change 100644 => 100755 ChangeLog delete mode 100755 tools/colorizer.py delete mode 100755 tools/install_venv.py delete mode 100755 tools/install_venv_common.py create mode 100755 tools/vagrant/rebootvm.sh create mode 100755 tools/vagrant/stvm.sh create mode 100644 tools/vagrant/ubuntu/onvm/conf/ids.conf.yml create mode 100644 tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml create mode 100644 tools/vagrant/ubuntu/onvm/scripts/ini-config create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-agent.sh create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-devstack.sh create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-elastic.sh create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-java.sh create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-kafka.sh create mode 100644 tools/vagrant/ubuntu/onvm/scripts/install-kiloeyes.sh delete mode 100644 tools/vagrant/ubuntu/ubuntu-virtualbox.yml.sample delete mode 100755 tools/with_venv.sh diff --git a/.gitignore b/.gitignore index 1ef448c..493f70d 100755 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .Python .testrepository/ .venv/ +.vagrant/ env/ bin/ build/ @@ -22,10 +23,11 @@ sdist/ var/ covhtml/ cover/ +tools/vagrant/ubuntu/leapbin *.egg-info/ .installed.cfg *.egg - +.DS_Store # Installer logs pip-log.txt pip-delete-this-directory.txt diff --git a/ChangeLog b/ChangeLog old mode 100644 new mode 100755 index af98ffe..8d62175 --- a/ChangeLog +++ b/ChangeLog @@ -1,18 +1,7 @@ -CHANGES -======= +kiloeyes (1.0) -* ES now returns timestamp as milliseconds vs seconds -* enable bulk message post on persister -* Added more instructions -* bulk insert can not be done implicitly -* fix the partitions data type error -* Updated the installation instructions -* added more instruction on how to create an all-in-one kiloeyes -* unit test passed for py27 -* Add Vagrant sample file to ease development environment bootstrap -* Make the server more flexible with configuration files -* remove old openstack incubator project reference -* remove old oslo.config and use new oslo_config -* Make minor modifications in the README -* seeding the project -* Added .gitreview + * Initial project setup + Choose framework of wsgiref, pasteDeploy, falcon. + The server will be wsgiref server like any other OpenStack server + Use PasteDeploy to allow WSGI pipelines + Use Falcon framework to implement ReSTful API services diff --git a/tools/colorizer.py b/tools/colorizer.py deleted file mode 100755 index ad4f898..0000000 --- a/tools/colorizer.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2013, Nebula, Inc. -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# Colorizer Code is borrowed from Twisted: -# Copyright (c) 2001-2010 Twisted Matrix Laboratories. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Display a subunit stream through a colorized unittest test runner.""" - -import heapq -import sys -import unittest - -import subunit -import testtools - - -class _AnsiColorizer(object): - """A colorizer is an object that loosely wraps around a stream. - - That allows callers to write text to the stream in a particular color. - Colorizer classes must implement C{supported()} and C{write(text, color)}. - """ - _colors = dict(black=30, red=31, green=32, yellow=33, - blue=34, magenta=35, cyan=36, white=37) - - def __init__(self, stream): - self.stream = stream - - @staticmethod - def supported(stream=sys.stdout): - """Method that checks if the current terminal supports coloring. - - Returns True or False. - """ - if not stream.isatty(): - return False # auto color only on TTYs - try: - import curses - except ImportError: - return False - else: - try: - try: - return curses.tigetnum("colors") > 2 - except curses.error: - curses.setupterm() - return curses.tigetnum("colors") > 2 - except Exception: - # guess false in case of error - return False - - def write(self, text, color): - """Write the given text to the stream in the given color. - - @param text: Text to be written to the stream. - - @param color: A string label for a color. e.g. 'red', 'white'. - - """ - color = self._colors[color] - self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text)) - - -class _Win32Colorizer(object): - """See _AnsiColorizer docstring.""" - def __init__(self, stream): - import win32console - red, green, blue, bold = (win32console.FOREGROUND_RED, - win32console.FOREGROUND_GREEN, - win32console.FOREGROUND_BLUE, - win32console.FOREGROUND_INTENSITY) - self.stream = stream - self.screenBuffer = win32console.GetStdHandle( - win32console.STD_OUT_HANDLE) - self._colors = { - 'normal': red | green | blue, - 'red': red | bold, - 'green': green | bold, - 'blue': blue | bold, - 'yellow': red | green | bold, - 'magenta': red | blue | bold, - 'cyan': green | blue | bold, - 'white': red | green | blue | bold - } - - @staticmethod - def supported(stream=sys.stdout): - try: - import win32console - screenBuffer = win32console.GetStdHandle( - win32console.STD_OUT_HANDLE) - except ImportError: - return False - import pywintypes - try: - screenBuffer.SetConsoleTextAttribute( - win32console.FOREGROUND_RED | - win32console.FOREGROUND_GREEN | - win32console.FOREGROUND_BLUE) - except pywintypes.error: - return False - else: - return True - - def write(self, text, color): - color = self._colors[color] - self.screenBuffer.SetConsoleTextAttribute(color) - self.stream.write(text) - self.screenBuffer.SetConsoleTextAttribute(self._colors['normal']) - - -class _NullColorizer(object): - """See _AnsiColorizer docstring.""" - def __init__(self, stream): - self.stream = stream - - @staticmethod - def supported(stream=sys.stdout): - return True - - def write(self, text, color): - self.stream.write(text) - - -def get_elapsed_time_color(elapsed_time): - if elapsed_time > 1.0: - return 'red' - elif elapsed_time > 0.25: - return 'yellow' - else: - return 'green' - - -class SubunitTestResult(testtools.TestResult): - def __init__(self, stream, descriptions, verbosity): - super(SubunitTestResult, self).__init__() - self.stream = stream - self.showAll = verbosity > 1 - self.num_slow_tests = 10 - self.slow_tests = [] # this is a fixed-sized heap - self.colorizer = None - # NOTE(vish): reset stdout for the terminal check - stdout = sys.stdout - sys.stdout = sys.__stdout__ - for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]: - if colorizer.supported(): - self.colorizer = colorizer(self.stream) - break - sys.stdout = stdout - self.start_time = None - self.last_time = {} - self.results = {} - self.last_written = None - - def _writeElapsedTime(self, elapsed): - color = get_elapsed_time_color(elapsed) - self.colorizer.write(" %.2f" % elapsed, color) - - def _addResult(self, test, *args): - try: - name = test.id() - except AttributeError: - name = 'Unknown.unknown' - test_class, test_name = name.rsplit('.', 1) - - elapsed = (self._now() - self.start_time).total_seconds() - item = (elapsed, test_class, test_name) - if len(self.slow_tests) >= self.num_slow_tests: - heapq.heappushpop(self.slow_tests, item) - else: - heapq.heappush(self.slow_tests, item) - - self.results.setdefault(test_class, []) - self.results[test_class].append((test_name, elapsed) + args) - self.last_time[test_class] = self._now() - self.writeTests() - - def _writeResult(self, test_name, elapsed, long_result, color, - short_result, success): - if self.showAll: - self.stream.write(' %s' % str(test_name).ljust(66)) - self.colorizer.write(long_result, color) - if success: - self._writeElapsedTime(elapsed) - self.stream.writeln() - else: - self.colorizer.write(short_result, color) - - def addSuccess(self, test): - super(SubunitTestResult, self).addSuccess(test) - self._addResult(test, 'OK', 'green', '.', True) - - def addFailure(self, test, err): - if test.id() == 'process-returncode': - return - super(SubunitTestResult, self).addFailure(test, err) - self._addResult(test, 'FAIL', 'red', 'F', False) - - def addError(self, test, err): - super(SubunitTestResult, self).addFailure(test, err) - self._addResult(test, 'ERROR', 'red', 'E', False) - - def addSkip(self, test, reason=None, details=None): - super(SubunitTestResult, self).addSkip(test, reason, details) - self._addResult(test, 'SKIP', 'blue', 'S', True) - - def startTest(self, test): - self.start_time = self._now() - super(SubunitTestResult, self).startTest(test) - - def writeTestCase(self, cls): - if not self.results.get(cls): - return - if cls != self.last_written: - self.colorizer.write(cls, 'white') - self.stream.writeln() - for result in self.results[cls]: - self._writeResult(*result) - del self.results[cls] - self.stream.flush() - self.last_written = cls - - def writeTests(self): - time = self.last_time.get(self.last_written, self._now()) - if not self.last_written or (self._now() - time).total_seconds() > 2.0: - diff = 3.0 - while diff > 2.0: - classes = self.results.keys() - oldest = min(classes, key=lambda x: self.last_time[x]) - diff = (self._now() - self.last_time[oldest]).total_seconds() - self.writeTestCase(oldest) - else: - self.writeTestCase(self.last_written) - - def done(self): - self.stopTestRun() - - def stopTestRun(self): - for cls in list(self.results.iterkeys()): - self.writeTestCase(cls) - self.stream.writeln() - self.writeSlowTests() - - def writeSlowTests(self): - # Pare out 'fast' tests - slow_tests = [item for item in self.slow_tests - if get_elapsed_time_color(item[0]) != 'green'] - if slow_tests: - slow_total_time = sum(item[0] for item in slow_tests) - slow = ("Slowest %i tests took %.2f secs:" - % (len(slow_tests), slow_total_time)) - self.colorizer.write(slow, 'yellow') - self.stream.writeln() - last_cls = None - # sort by name - for elapsed, cls, name in sorted(slow_tests, - key=lambda x: x[1] + x[2]): - if cls != last_cls: - self.colorizer.write(cls, 'white') - self.stream.writeln() - last_cls = cls - self.stream.write(' %s' % str(name).ljust(68)) - self._writeElapsedTime(elapsed) - self.stream.writeln() - - def printErrors(self): - if self.showAll: - self.stream.writeln() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - - def printErrorList(self, flavor, errors): - for test, err in errors: - self.colorizer.write("=" * 70, 'red') - self.stream.writeln() - self.colorizer.write(flavor, 'red') - self.stream.writeln(": %s" % test.id()) - self.colorizer.write("-" * 70, 'red') - self.stream.writeln() - self.stream.writeln("%s" % err) - - -test = subunit.ProtocolTestCase(sys.stdin, passthrough=None) - -if sys.version_info[0:2] <= (2, 6): - runner = unittest.TextTestRunner(verbosity=2) -else: - runner = unittest.TextTestRunner( - verbosity=2, resultclass=SubunitTestResult) - -if runner.run(test).wasSuccessful(): - exit_code = 0 -else: - exit_code = 1 -sys.exit(exit_code) diff --git a/tools/install_venv.py b/tools/install_venv.py deleted file mode 100755 index 166c03d..0000000 --- a/tools/install_venv.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2010 OpenStack Foundation -# Copyright 2013 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Installation script for Glance's development virtualenv -""" - -from __future__ import print_function - -import os -import sys - -import install_venv_common as install_venv # noqa - - -def print_help(): - help = """ - Glance development environment setup is complete. - - Glance development uses virtualenv to track and manage Python dependencies - while in development and testing. - - To activate the Glance virtualenv for the extent of your current shell session - you can run: - - $ source .venv/bin/activate - - Or, if you prefer, you can run commands in the virtualenv on a case by case - basis by running: - - $ tools/with_venv.sh - - Also, make test will automatically use the virtualenv. - """ - print(help) - - -def main(argv): - root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - venv = os.path.join(root, '.venv') - pip_requires = os.path.join(root, 'requirements.txt') - test_requires = os.path.join(root, 'test-requirements.txt') - py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) - project = 'Kiloeyes' - install = install_venv.InstallVenv(root, venv, pip_requires, test_requires, - py_version, project) - options = install.parse_args(argv) - install.check_python_version() - install.check_dependencies() - install.create_virtualenv(no_site_packages=options.no_site_packages) - install.install_dependencies() - install.run_command([os.path.join(venv, 'bin/python'), - 'setup.py', 'develop']) - print_help() - -if __name__ == '__main__': - main(sys.argv) diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py deleted file mode 100755 index e279159..0000000 --- a/tools/install_venv_common.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# Copyright 2013 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Provides methods needed by installation script for OpenStack development -virtual environments. - -Since this script is used to bootstrap a virtualenv from the system's Python -environment, it should be kept strictly compatible with Python 2.6. - -Synced in from openstack-common -""" - -from __future__ import print_function - -import optparse -import os -import subprocess -import sys - - -class InstallVenv(object): - - def __init__(self, root, venv, requirements, - test_requirements, py_version, - project): - self.root = root - self.venv = venv - self.requirements = requirements - self.test_requirements = test_requirements - self.py_version = py_version - self.project = project - - def die(self, message, *args): - print(message % args, file=sys.stderr) - sys.exit(1) - - def check_python_version(self): - if sys.version_info < (2, 6): - self.die("Need Python Version >= 2.6") - - def run_command_with_code(self, cmd, redirect_output=True, - check_exit_code=True): - """Runs a command in an out-of-process shell. - - Returns the output of that command. Working directory is self.root. - """ - if redirect_output: - stdout = subprocess.PIPE - else: - stdout = None - - proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) - output = proc.communicate()[0] - if check_exit_code and proc.returncode != 0: - self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) - return (output, proc.returncode) - - def run_command(self, cmd, redirect_output=True, check_exit_code=True): - return self.run_command_with_code(cmd, redirect_output, - check_exit_code)[0] - - def get_distro(self): - if (os.path.exists('/etc/fedora-release') or - os.path.exists('/etc/redhat-release')): - return Fedora( - self.root, self.venv, self.requirements, - self.test_requirements, self.py_version, self.project) - else: - return Distro( - self.root, self.venv, self.requirements, - self.test_requirements, self.py_version, self.project) - - def check_dependencies(self): - self.get_distro().install_virtualenv() - - def create_virtualenv(self, no_site_packages=True): - """Creates the virtual environment and installs PIP. - - Creates the virtual environment and installs PIP only into the - virtual environment. - """ - if not os.path.isdir(self.venv): - print('Creating venv...', end=' ') - if no_site_packages: - self.run_command(['virtualenv', '-q', '--no-site-packages', - self.venv]) - else: - self.run_command(['virtualenv', '-q', self.venv]) - print('done.') - else: - print("venv already exists...") - pass - - def pip_install(self, *args): - self.run_command(['tools/with_venv.sh', - 'pip', 'install', '--upgrade'] + list(args), - redirect_output=False) - - def install_dependencies(self): - print('Installing dependencies with pip (this can take a while)...') - - # First things first, make sure our venv has the latest pip and - # setuptools and pbr - self.pip_install('pip>=1.4') - self.pip_install('setuptools') - self.pip_install('pbr') - - self.pip_install('-r', self.requirements, '-r', self.test_requirements) - - def parse_args(self, argv): - """Parses command-line arguments.""" - parser = optparse.OptionParser() - parser.add_option('-n', '--no-site-packages', - action='store_true', - help="Do not inherit packages from global Python " - "install.") - return parser.parse_args(argv[1:])[0] - - -class Distro(InstallVenv): - - def check_cmd(self, cmd): - return bool(self.run_command(['which', cmd], - check_exit_code=False).strip()) - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if self.check_cmd('easy_install'): - print('Installing virtualenv via easy_install...', end=' ') - if self.run_command(['easy_install', 'virtualenv']): - print('Succeeded') - return - else: - print('Failed') - - self.die('ERROR: virtualenv not found.\n\n%s development' - ' requires virtualenv, please install it using your' - ' favorite package management tool' % self.project) - - -class Fedora(Distro): - """This covers all Fedora-based distributions. - - Includes: Fedora, RHEL, CentOS, Scientific Linux - """ - - def check_pkg(self, pkg): - return self.run_command_with_code(['rpm', '-q', pkg], - check_exit_code=False)[1] == 0 - - def install_virtualenv(self): - if self.check_cmd('virtualenv'): - return - - if not self.check_pkg('python-virtualenv'): - self.die("Please install 'python-virtualenv'.") - - super(Fedora, self).install_virtualenv() diff --git a/tools/vagrant/rebootvm.sh b/tools/vagrant/rebootvm.sh new file mode 100755 index 0000000..20966c1 --- /dev/null +++ b/tools/vagrant/rebootvm.sh @@ -0,0 +1,7 @@ +VBoxManage snapshot h2-compute01 restore "Snapshot 3" +VBoxManage snapshot h2-nova restore "Snapshot 3" +VBoxManage snapshot h2-controller restore "Snapshot 3" + +vboxmanage startvm h2-compute01 --type headless +vboxmanage startvm h2-nova --type headless +vboxmanage startvm h2-controller --type headless diff --git a/tools/vagrant/stvm.sh b/tools/vagrant/stvm.sh new file mode 100755 index 0000000..8c4892a --- /dev/null +++ b/tools/vagrant/stvm.sh @@ -0,0 +1,3 @@ +vboxmanage controlvm h2-compute01 acpipowerbutton +vboxmanage controlvm h2-nova acpipowerbutton +vboxmanage controlvm h2-controller acpipowerbutton diff --git a/tools/vagrant/ubuntu/Vagrantfile b/tools/vagrant/ubuntu/Vagrantfile index 7d0c859..b72ee60 100644 --- a/tools/vagrant/ubuntu/Vagrantfile +++ b/tools/vagrant/ubuntu/Vagrantfile @@ -2,49 +2,59 @@ # vi: set ft=ruby : require 'yaml' -config_file=File.expand_path(File.join(File.dirname(__FILE__), 'ubuntu-virtualbox.yml')) -settings=YAML.load_file(config_file) +nodes = YAML.load_file("onvm/conf/nodes.conf.yml") +ids = YAML.load_file("onvm/conf/ids.conf.yml") -VM_MEM = settings['vm_memory'] -KAFKA_URL = settings['kafka_url'] -# ElasticSearch URL -ES_URL = settings['elasticsearch_url'] +Vagrant.configure("2") do |config| + config.vm.box = "tknerr/managed-server-dummy" + config.ssh.username = ids['username'] + config.ssh.password = ids['password'] -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. + config.vm.synced_folder ".", "/vagrant", disabled: true + config.vm.synced_folder "onvm", "/onvm", disabled: false, create: true - # Every Vagrant development environment requires a box. You can search for - # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "ubuntu/trusty64" - config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # # Customize the amount of memory on the VM: - vb.memory = VM_MEM - config.vm.network "private_network", ip: "192.168.0.10" - end - - config.vm.provision "shell", inline: <<-SHELL - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | sudo tee /etc/apt/sources.list.d/webupd8team-java.list - echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | sudo tee -a /etc/apt/sources.list.d/webupd8team-java.list - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 - sudo apt-get update - sudo apt-get install -y git build-essential python-pip python-dev vim - sudo apt-get install -y virtualenvwrapper - cd /vagrant && sudo git clone https://github.com/openstack/kiloeyes.git - echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections - echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections - sudo apt-get install -y oracle-java8-installer - cd /vagrant && wget #{KAFKA_URL} -O - | tar -zxvf - -C /vagrant - wget #{ES_URL} -O /tmp/es.deb && sudo dpkg -i /tmp/es.deb - echo source /usr/share/virtualenvwrapper/virtualenvwrapper_lazy.sh | tee -a ~/.profile - SHELL + sync_cfg = nodes['synchfolders'] + + lnodes = nodes['ctlnodes'] + if lnodes + lnodes.each do |key| + config.vm.define "#{key}" do |node| + nodekey = nodes['logical2physical'][key] + + if sync_cfg[key] + node.vm.synced_folder sync_cfg[key]['source'], + sync_cfg[key]['target'], disabled: false, create: true + end + + node.vm.provider :managed do |managed| + managed.server = nodes[nodekey]['eth0'] + end + + node.vm.provision "#{key}-install", type: "shell" do |s| + s.path = "onvm/scripts/install-" + key + ".sh" + s.args = ids['sys_password'] + ' ' + nodes[nodekey]['eth0'] + end + end + end + end + + # agent node setup + lnodes = nodes['agentes'] + if lnodes + lnodes.each do |key| + config.vm.define "#{key}" do |node| + node.vm.provider :managed do |managed| + managed.server = nodes[key]['eth0'] + end + + node.vm.provision "#{key}-install", type: "shell" do |s| + s.path = "onvm/scripts/install-agent.sh" + s.args = ids['sys_password'] + " " + nodes[key]['eth0'] + end + + end + end + end + end diff --git a/tools/vagrant/ubuntu/onvm/conf/ids.conf.yml b/tools/vagrant/ubuntu/onvm/conf/ids.conf.yml new file mode 100644 index 0000000..8ab55a6 --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/conf/ids.conf.yml @@ -0,0 +1,7 @@ +--- +username: "root" + +password: "time4fun" + +sys_password: "secret" + diff --git a/tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml b/tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml new file mode 100644 index 0000000..a233a4f --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml @@ -0,0 +1,49 @@ +--- +repo: + host_name: repo.leap.dev + eth0: 192.168.1.88 + eth1: 192.168.1.88 + +controller: + host_name: controller.leap.dev + eth0: 192.168.1.90 + +devstack: + host_name: devstack.leap.dev + eth0: 192.168.1.93 + +agent01: + host_name: agent01.leap.dev + eth0: 192.168.1.88 + +logical2physical: + kiloeyes: controller + elastic: controller + kafka: controller + devstack: devstack + + +# Define how many logical nodes and the sequence of the installation +ctlnodes: + - devstack + - elastic + - kafka + - kiloeyes + +agentes: + - agent01 + +uselocalrepo: yes + +aptopt: --force-yes + + +# The nodes should be a list of logical name +# The folder should be a local directory start from the project root +synchfolders: + elastic: + source: leapbin + target: /leapbin + kiloeyes: + source: ./../../../../kiloeyes + target: /opt/kiloeyes diff --git a/tools/vagrant/ubuntu/onvm/scripts/ini-config b/tools/vagrant/ubuntu/onvm/scripts/ini-config new file mode 100644 index 0000000..dce2d05 --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/scripts/ini-config @@ -0,0 +1,330 @@ +#!/bin/bash +# +# **inc/ini-config** - Configuration/INI functions +# +# Support for manipulating INI-style configuration files +# +# These functions have no external dependencies and no side-effects + +# Save trace setting +INC_CONF_TRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Config Functions +# ================ + +# Append a new option in an ini file without replacing the old value +# iniadd [-sudo] config-file section option value1 value2 value3 ... +function iniadd { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="-sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + shift 3 + + local values="$(iniget_multiline $file $section $option) $@" + iniset_multiline $sudo $file $section $option $values + $xtrace +} + +# Comment an option in an INI file +# inicomment [-sudo] config-file section option +function inicomment { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + + $sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ s|^\($option[ \t]*=.*$\)|#\1|" "$file" + $xtrace +} + +# Get an option from an INI file +# iniget config-file section option +function iniget { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local file=$1 + local section=$2 + local option=$3 + local line + + line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file") + echo ${line#*=} + $xtrace +} + +# Get a multiple line option from an INI file +# iniget_multiline config-file section option +function iniget_multiline { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local file=$1 + local section=$2 + local option=$3 + local values + + values=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { s/^$option[ \t]*=[ \t]*//gp; }" "$file") + echo ${values} + $xtrace +} + +# Determinate is the given option present in the INI file +# ini_has_option config-file section option +function ini_has_option { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local file=$1 + local section=$2 + local option=$3 + local line + + line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file") + $xtrace + [ -n "$line" ] +} + +# Add another config line for a multi-line option. +# It's normally called after iniset of the same option and assumes +# that the section already exists. +# +# Note that iniset_multiline requires all the 'lines' to be supplied +# in the argument list. Doing that will cause incorrect configuration +# if spaces are used in the config values. +# +# iniadd_literal [-sudo] config-file section option value +function iniadd_literal { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + local value=$4 + + if [[ -z $section || -z $option ]]; then + $xtrace + return + fi + + # Add it + $sudo sed -i -e "/^\[$section\]/ a\\ +$option = $value +" "$file" + + $xtrace +} + +# Remove an option from an INI file +# inidelete [-sudo] config-file section option +function inidelete { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + + if [[ -z $section || -z $option ]]; then + $xtrace + return + fi + + # Remove old values + $sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ d; }" "$file" + + $xtrace +} + +# Set an option in an INI file +# iniset [-sudo] config-file section option value +# - if the file does not exist, it is created +function iniset { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + local value=$4 + + if [[ -z $section || -z $option ]]; then + $xtrace + return + fi + + if ! grep -q "^\[$section\]" "$file" 2>/dev/null; then + # Add section at the end + echo -e "\n[$section]" | $sudo tee --append "$file" > /dev/null + fi + if ! ini_has_option "$file" "$section" "$option"; then + # Add it + $sudo sed -i -e "/^\[$section\]/ a\\ +$option = $value +" "$file" + else + local sep + sep=$(echo -ne "\x01") + # Replace it + $sudo sed -i -e '/^\['${section}'\]/,/^\[.*\]/ s'${sep}'^\('${option}'[ \t]*=[ \t]*\).*$'${sep}'\1'"${value}"${sep} "$file" + fi + $xtrace +} + +# Set an option in a conf file +# confset [-sudo] config-file option value +# - if the file does not exist, it is created +function confset { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local option=$2 + local value=$3 + + if ! grep -q "^$option" "$file" 2>/dev/null; then + echo -e "\n$option = $value\n" | $sudo tee --append "$file" > /dev/null + else + $sudo sed -i -e 's/^'$option'.*$/'$option' = '$value'/g' $file + fi + + $xtrace +} + +# Set a multiple line option in an INI file +# iniset_multiline [-sudo] config-file section option value1 value2 value3 ... +function iniset_multiline { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + + shift 3 + local values + for v in $@; do + # The later sed command inserts each new value in the line next to + # the section identifier, which causes the values to be inserted in + # the reverse order. Do a reverse here to keep the original order. + values="$v ${values}" + done + if ! grep -q "^\[$section\]" "$file"; then + # Add section at the end + echo -e "\n[$section]" | $sudo tee --append "$file" > /dev/null + else + # Remove old values + $sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ d; }" "$file" + fi + # Add new ones + for v in $values; do + $sudo sed -i -e "/^\[$section\]/ a\\ +$option = $v +" "$file" + done + $xtrace +} + +# Uncomment an option in an INI file +# iniuncomment config-file section option +function iniuncomment { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local section=$2 + local option=$3 + $sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ s|[^ \t]*#[ \t]*\($option[ \t]*=.*$\)|\1|" "$file" + $xtrace +} + +## Remove all comments in a file +function iniremcomment { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local sudo="" + if [ $1 == "-sudo" ]; then + sudo="sudo " + shift + fi + local file=$1 + local tpfile="$(mktemp)" + $sudo grep -v "^\#" $1 | sed '/^$/N;/^\n$/D' > $tpfile + $sudo cp $tpfile $1 + $sudo rm -rf $tmpfile + $xtrace +} + +function parse_yaml { + local prefix=$2 + local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + sed -ne "s|^\($s\):|\1|" \ + -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | + awk -F$fs '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i> /etc/elasticsearch/elasticsearch.yml + update-rc.d elasticsearch defaults + service elasticsearch restart + echo 'Elastic install is now complete!' +else + echo 'Elasticsearch binary was not found!' + echo 'Download elasticsearch 2.2.x.deb and place it in tools/vagrant/ubuntu/leapbin directory.' +fi diff --git a/tools/vagrant/ubuntu/onvm/scripts/install-java.sh b/tools/vagrant/ubuntu/onvm/scripts/install-java.sh new file mode 100644 index 0000000..41b7747 --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/scripts/install-java.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# $1 sys_password +# $2 public ip eth0 + +java_installed=$(which java) +if [ -z $java_installed ] && [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then + mkdir -p /opt/jdk + tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk + mv /opt/jdk/* /opt/jdk/jdk1.8 + update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100 + update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100 + echo 'Java install is now complete!' +else + echo 'Java has been either installed or not binary package found!' + echo 'Download java and place it in tools/vagrant/ubuntu/leapbin directory.' +fi diff --git a/tools/vagrant/ubuntu/onvm/scripts/install-kafka.sh b/tools/vagrant/ubuntu/onvm/scripts/install-kafka.sh new file mode 100644 index 0000000..894041c --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/scripts/install-kafka.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# $1 sys_password +# $2 public ip eth0 + +source /onvm/scripts/ini-config +eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_') + +if [ -f /leapbin/kafka_*-0.9.0.0.tgz ]; then + tar -zxf /leapbin/kafka_*-0.9.0.0.tgz -C /opt + mv /opt/kafka_*-0.9.0.0 /opt/kafka_0.9.0.0 + + /opt/kafka_0.9.0.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_0.9.0.0/config/zookeeper.properties + sleep 2 + + /opt/kafka_0.9.0.0/bin/kafka-server-start.sh -daemon /opt/kafka_0.9.0.0/config/server.properties + + echo 'Kafka install is now complete!' +else + echo 'Kafka binary was not found!' +fi + diff --git a/tools/vagrant/ubuntu/onvm/scripts/install-kiloeyes.sh b/tools/vagrant/ubuntu/onvm/scripts/install-kiloeyes.sh new file mode 100644 index 0000000..b584682 --- /dev/null +++ b/tools/vagrant/ubuntu/onvm/scripts/install-kiloeyes.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# $1 sys_password +# $2 public ip eth0 + +source /onvm/scripts/ini-config +eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_') + +wget https://bootstrap.pypa.io/ez_setup.py -O - | python +apt-get update +apt-get -qqy install git python-dev python-pip python-pastedeploy + +cd /opt/kiloeyes +pip install -r requirements.txt +python setup.py install + +# Config the kiloeyes + +echo 'Kiloeyes install is now complete!' + diff --git a/tools/vagrant/ubuntu/ubuntu-virtualbox.yml.sample b/tools/vagrant/ubuntu/ubuntu-virtualbox.yml.sample deleted file mode 100644 index 5f5c11f..0000000 --- a/tools/vagrant/ubuntu/ubuntu-virtualbox.yml.sample +++ /dev/null @@ -1,6 +0,0 @@ ---- - -vm_memory: 8192 -kafka_url: http://apache.stu.edu.tw/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz -elasticsearch_url: https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.1.0/elasticsearch-2.1.0.deb - diff --git a/tools/with_venv.sh b/tools/with_venv.sh deleted file mode 100755 index 7303990..0000000 --- a/tools/with_venv.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -TOOLS_PATH=${TOOLS_PATH:-$(dirname $0)} -VENV_PATH=${VENV_PATH:-${TOOLS_PATH}} -VENV_DIR=${VENV_NAME:-/../.venv} -TOOLS=${TOOLS_PATH} -VENV=${VENV:-${VENV_PATH}/${VENV_DIR}} -source ${VENV}/bin/activate && "$@"