Restructure the vagrant scripts and fix a dependency error

The vagrant scripts buried so deep. Move it up a bit.
The python-kafka client 1.0 release breaks a lot of things plus
it peeks the CPU as soon as it starts. Have to make sure we use
the python-kafka version less than 1.0.0

Change-Id: Ib271dde51dcb0c5705dffbd277ba70f25cd4403b
This commit is contained in:
Tong Li 2016-03-30 14:45:33 -04:00
parent 3df93aa053
commit 771e4b5a28
18 changed files with 109 additions and 42 deletions

2
.gitignore vendored
View File

@ -23,7 +23,7 @@ sdist/
var/
covhtml/
cover/
tools/vagrant/ubuntu/leapbin
vagrant/leapbin
*.egg-info/
.installed.cfg
*.egg

View File

@ -159,10 +159,12 @@ class KafkaConnection(object):
if not self._consumer:
self._init_consumer()
for msg in self._consumer:
if msg.message:
LOG.debug(msg.message.value)
yield msg
if self._consumer:
for msg in self._consumer:
if msg.message:
LOG.debug(msg.message.value)
yield msg
time.sleep(self.wait_time)
except common.OffsetOutOfRangeError:
self._consumer.seek(0, 0)
LOG.error('Seems consumer has been down for a long time.')

View File

@ -6,7 +6,7 @@
falcon==0.1.9
gunicorn==19.1.0
iso8601>=0.1.9
kafka-python>=0.8.1
kafka-python<1.0.0,>=0.9.5
oslo.config>=1.11.0
oslo.i18n>=1.5.0
oslo.log>=1.0.0

View File

@ -1,16 +0,0 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
java_installed=$(which java)
if [ -z $java_installed ] && [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then
mkdir -p /opt/jdk
tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk
mv /opt/jdk/* /opt/jdk/jdk1.8
update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100
update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100
echo 'Java install is now complete!'
else
echo 'Java has been either installed or not binary package found!'
echo 'Download java and place it in tools/vagrant/ubuntu/leapbin directory.'
fi

View File

@ -1,19 +0,0 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
wget https://bootstrap.pypa.io/ez_setup.py -O - | python
apt-get update
apt-get -qqy install git python-dev python-pip python-pastedeploy
cd /opt/kiloeyes
pip install -r requirements.txt
python setup.py install
# Config the kiloeyes
echo 'Kiloeyes install is now complete!'

View File

@ -45,5 +45,5 @@ synchfolders:
source: leapbin
target: /leapbin
kiloeyes:
source: ./../../../../kiloeyes
source: ./../../kiloeyes
target: /opt/kiloeyes

View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
java_installed=$(which java)
if [ -z $java_installed ]; then
if [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then
mkdir -p /opt/jdk
tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk
mv /opt/jdk/* /opt/jdk/jdk1.8
update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100
update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100
echo 'Java install is now complete!'
else
echo 'Download java and place it in /vagrant/leapbin directory.'
fi
else
echo 'Java has been installed!'
fi

View File

@ -5,10 +5,17 @@
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
# Java is required, install java first
source /onvm/scripts/install-java.sh
if [ -f /leapbin/kafka_*-0.9.0.0.tgz ]; then
tar -zxf /leapbin/kafka_*-0.9.0.0.tgz -C /opt
mv /opt/kafka_*-0.9.0.0 /opt/kafka_0.9.0.0
echo '# Changes made by kiloeyes installation...' >> /opt/kafka_0.9.0.0/config/server.properties
echo 'auto.create.topics.enable=true' >> /opt/kafka_0.9.0.0/config/server.properties
echo "host.name=$2" >> /opt/kafka_0.9.0.0/config/server.properties
/opt/kafka_0.9.0.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_0.9.0.0/config/zookeeper.properties
sleep 2

View File

@ -0,0 +1,74 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
wget https://bootstrap.pypa.io/ez_setup.py -O - | python
apt-get update
apt-get -qqy install git python-dev python-pip python-pastedeploy
cd /opt/kiloeyes
pip install -r requirements.txt
python setup.py install
echo 'Finding IP addresses...'
eval node_ip=\$leap_${leap_logical2physical_kafka}_eth0; node_ip=`echo $node_ip`
kafka_ip=$node_ip
eval node_ip=\$leap_${leap_logical2physical_elastic}_eth0; node_ip=`echo $node_ip`
elastic_ip=$node_ip
k_log_dir='/var/log/kiloeyes'
k_pid_dir='/var/run/kiloeyes'
mkdir -p $k_log_dir $k_pid_dir
# Config the kiloeyes
echo 'Config /etc/kiloeyes/kiloeyes.conf file...'
iniset /etc/kiloeyes/kiloeyes.conf DEFAULT log_dir $k_log_dir
iniset /etc/kiloeyes/kiloeyes.conf kafka_opts uri $kafka_ip:9092
iniset /etc/kiloeyes/kiloeyes.conf es_conn uri http://$elastic_ip:9200
echo 'Config /etc/kiloeyes/metrics-persister.conf...'
iniset /etc/kiloeyes/metrics-persister.conf DEFAULT log_dir $k_log_dir
iniset /etc/kiloeyes/metrics-persister.conf kafka_opts uri $kafka_ip:9092
iniset /etc/kiloeyes/metrics-persister.conf es_conn uri http://$elastic_ip:9200
echo 'Config /etc/kiloeyes/alarms-persister.conf...'
iniset /etc/kiloeyes/alarms-persister.conf DEFAULT log_dir $k_log_dir
iniset /etc/kiloeyes/alarms-persister.conf kafka_opts uri $kafka_ip:9092
iniset /etc/kiloeyes/alarms-persister.conf es_conn uri http://$elastic_ip:9200
echo 'Config /etc/kiloeyes/kiloeyes-notification-engine.conf...'
iniset /etc/kiloeyes/kiloeyes-notification-engine.conf DEFAULT log_dir $k_log_dir
iniset /etc/kiloeyes/kiloeyes-notification-engine.conf kafka_opts uri $kafka_ip:9092
iniset /etc/kiloeyes/kiloeyes-notification-engine.conf es_conn uri http://$elastic_ip:9200
echo 'Config /etc/kiloeyes/kiloeyes-threshold-engine.conf...'
iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf DEFAULT log_dir $k_log_dir
iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf kafka_opts uri $kafka_ip:9092
iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf es_conn uri http://$elastic_ip:9200
echo 'Start all kiloeyes services...'
gunicorn -k eventlet --worker-connections=20 --backlog=10 --paste /etc/kiloeyes/kiloeyes.ini -D
start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
--pidfile $k_pid_dir/metrics-persister.pid --make-pidfile --background \
-- --config-file /etc/kiloeyes/metrics-persister.conf >> /dev/null 2>&1
start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
--pidfile $k_pid_dir/alarms-persister.pid --make-pidfile --background \
-- --config-file /etc/kiloeyes/alarms-persister.conf >> /dev/null 2>&1
start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
--pidfile $k_pid_dir/kiloeyes-notification-engine.pid --make-pidfile --background \
-- --config-file /etc/kiloeyes/kiloeyes-notification-engine.conf >> /dev/null 2>&1
start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
--pidfile $k_pid_dir/kiloeyes-threshold-engine.pid --make-pidfile --background \
-- --config-file /etc/kiloeyes/kiloeyes-threshold-engine.conf >> /dev/null 2>&1
echo 'Kiloeyes install is now complete!'