diff --git a/devstack/README.md b/devstack/README.md index f91d4b9..2f6a66a 100644 --- a/devstack/README.md +++ b/devstack/README.md @@ -1,6 +1,6 @@ # Monasca-transform DevStack Plugin -The Monasca-transform DevStack plugin is tested only on Ubuntu 14.04 (Trusty). +The Monasca-transform DevStack plugin is tested only on Ubuntu 16.04 (Xenial). A short cut to running monasca-transform in devstack is implemented with vagrant. @@ -28,6 +28,13 @@ and deploys using that. Changes made by the user need to be committed in order to be used in the devstack instance. It is important therefore that changes should not be pushed from the vm as the unevaluated commit would be pushed. +N.B. If you are running with virtualbox you may find that the `./stack.sh` fails with the filesystem becoming read only. There is a work around: + + 1. vagrant up --no-provision && vagrant halt + 2. open virtualbox gui + 3. open target vm settings and change storage controller from SCSI to SATA + 4. vagrant up + ### Using the upstream committed state of monasca-transform This should operate the same as for any other devstack plugin. However, to use @@ -42,7 +49,7 @@ above it is sufficient to do: and change the line - enable_plugin monasca-transform /home/vagrant/monasca-transform + enable_plugin monasca-transform /home/ubuntu/monasca-transform to @@ -63,7 +70,7 @@ The devstack vm vagrant up process generates a private key which can be used for passwordless ssh to the host as follows: cd tools/vagrant - ssh -i .vagrant/machines/default/virtualbox/private_key vagrant@192.168.15.6 + ssh -i .vagrant/machines/default/virtualbox/private_key ubuntu@192.168.15.6 ### Running tox on devstack diff --git a/devstack/files/spark/spark-master.conf b/devstack/files/spark/spark-master.conf deleted file mode 100644 index 410e44f..0000000 --- a/devstack/files/spark/spark-master.conf +++ /dev/null @@ -1,12 +0,0 @@ -description "Spark Master" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 32768 32768 - -expect daemon - -exec /etc/spark/init/start-spark-master.sh \ No newline at end of file diff --git a/devstack/files/spark/spark-master.service b/devstack/files/spark/spark-master.service new file mode 100644 index 0000000..c299c85 --- /dev/null +++ b/devstack/files/spark/spark-master.service @@ -0,0 +1,12 @@ +[Unit] +Description=Spark Master +After=zookeeper.service + +[Service] +User=spark +Group=spark +ExecStart=/etc/spark/init/start-spark-master.sh +Restart=on-failure + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/devstack/files/spark/spark-worker.conf b/devstack/files/spark/spark-worker.conf deleted file mode 100644 index 6996167..0000000 --- a/devstack/files/spark/spark-worker.conf +++ /dev/null @@ -1,12 +0,0 @@ -description "Spark Worker" - -start on runlevel [2345] -stop on runlevel [!2345] - -respawn - -limit nofile 32768 32768 - -expect daemon - -exec /etc/spark/init/start-spark-worker.sh \ No newline at end of file diff --git a/devstack/files/spark/spark-worker.service b/devstack/files/spark/spark-worker.service new file mode 100644 index 0000000..cc238c8 --- /dev/null +++ b/devstack/files/spark/spark-worker.service @@ -0,0 +1,9 @@ +[Unit] +Description=Spark Worker +After=zookeeper.service + +[Service] +User=spark +Group=spark +ExecStart=/etc/spark/init/start-spark-worker.sh +Restart=on-failure diff --git a/devstack/plugin.sh b/devstack/plugin.sh index cdbbb17..8f48fc4 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -232,21 +232,21 @@ function delete_spark_start_scripts { function copy_spark_upstart_definitions { - SPARK_UPSTART_DEFINITIONS=("spark-master.conf" "spark-worker.conf") - for SPARK_UPSTART_DEFINITION in "${SPARK_UPSTART_DEFINITIONS[@]}" + SPARK_SERVICE_DEFINITIONS=("spark-master.service" "spark-worker.service") + for SPARK_SERVICE_DEFINITION in "${SPARK_SERVICE_DEFINITIONS[@]}" do - sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/spark/"${SPARK_UPSTART_DEFINITION}" /etc/init/. - sudo chmod 644 /etc/init/"${SPARK_UPSTART_DEFINITION}" + sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/spark/"${SPARK_SERVICE_DEFINITION}" /etc/systemd/system/. + sudo chmod 644 /etc/systemd/system/"${SPARK_SERVICE_DEFINITION}" done } function delete_spark_upstart_definitions { - SPARK_UPSTART_DEFINITIONS=("spark-master.conf" "spark-worker.conf") - for SPARK_UPSTART_DEFINITION in "${SPARK_UPSTART_DEFINITIONS[@]}" + SPARK_SERVICE_DEFINITIONS=("spark-master.service" "spark-worker.service") + for SPARK_SERVICE_DEFINITION in "${SPARK_SERVICE_DEFINITIONS[@]}" do - sudo rm /etc/init/${SPARK_UPSTART_DEFINITION} || true + sudo rm /etc/systemd/system/${SPARK_SERVICE_DEFINITION} || true done } @@ -265,7 +265,7 @@ function install_monasca_transform { copy_monasca_transform_files create_monasca_transform_venv - sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/monasca-transform/monasca_transform_init.conf /etc/init/monasca-transform.conf + sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/monasca-transform/monasca-transform.service /etc/systemd/system/. sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/monasca-transform/start-monasca-transform.sh /etc/monasca/transform/init/. sudo chmod +x /etc/monasca/transform/init/start-monasca-transform.sh sudo cp -f "${MONASCA_TRANSFORM_BASE}"/monasca-transform/devstack/files/monasca-transform/service_runner.py /etc/monasca/transform/init/. diff --git a/tools/vagrant/Vagrantfile b/tools/vagrant/Vagrantfile index 4d7e705..db4d3d3 100644 --- a/tools/vagrant/Vagrantfile +++ b/tools/vagrant/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure(2) do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "ubuntu/trusty64" + config.vm.box = "ubuntu/xenial64" config.vm.hostname = "pg-tips" # Disable automatic box update checking. If you disable this, then @@ -80,7 +80,7 @@ Vagrant.configure(2) do |config| # pg-tips will now have devstack cloned so push our local.conf file into place config.vm.provision "file", source: "local.conf", \ - destination: "/home/vagrant/devstack/local.conf" + destination: "/home/ubuntu/devstack/local.conf" if !ENV['LOCAL_REPO'] || ENV['LOCAL_REPO'].empty? puts "Using default repo" diff --git a/tools/vagrant/local.conf b/tools/vagrant/local.conf index 67919a6..0d7a15b 100644 --- a/tools/vagrant/local.conf +++ b/tools/vagrant/local.conf @@ -8,6 +8,8 @@ ADMIN_PASSWORD=secretadmin SERVICE_PASSWORD=secretservice SERVICE_TOKEN=111222333444 +HOST_IP=10.0.2.15 + LOGFILE=$DEST/logs/stack.sh.log LOGDIR=$DEST/logs LOG_COLOR=False @@ -29,15 +31,15 @@ MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-influxdb} # MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-vertica} # This line will enable all of Monasca. -enable_plugin monasca-api /home/vagrant/monasca-api +enable_plugin monasca-api /home/ubuntu/monasca-api # the following may be disabled to leave monasca transform still functional #disable_service monasca-persister -#disable_service monasca-thresh +disable_service monasca-thresh #disable_service monasca-notification -#disable_service horizon +disable_service horizon #disable_service tempest #disable_service cinder -enable_plugin monasca-transform /home/vagrant/monasca-transform +enable_plugin monasca-transform /home/ubuntu/monasca-transform # the following must be disabled as the test does not work at this point # see Bug #1636508 disable_service monasca-smoke-test diff --git a/tools/vagrant/refresh_monasca_transform.sh b/tools/vagrant/refresh_monasca_transform.sh index beff33a..795a875 100755 --- a/tools/vagrant/refresh_monasca_transform.sh +++ b/tools/vagrant/refresh_monasca_transform.sh @@ -6,9 +6,9 @@ else echo Yikes, no - this is not pg-tips! exit 1 fi -if [ -d "/home/vagrant/devstack" ] ; then +if [ -d "/home/ubuntu/devstack" ] ; then -. /home/vagrant/devstack/.stackenv +. /home/ubuntu/devstack/.stackenv fi @@ -22,21 +22,21 @@ else echo "monasca-transform service not running" fi -sudo rm -rf /home/vagrant/monasca-transform-source /home/vagrant/monasca-transform +sudo rm -rf /home/ubuntu/monasca-transform-source /home/ubuntu/monasca-transform sudo ./setup_local_repos.sh # update the database with configuration -sudo cp /home/vagrant/monasca-transform/scripts/ddl/pre_transform_specs.sql /opt/monasca/transform/lib/pre_transform_specs.sql -sudo cp /home/vagrant/monasca-transform/scripts/ddl/transform_specs.sql /opt/monasca/transform/lib/transform_specs.sql +sudo cp /home/ubuntu/monasca-transform/scripts/ddl/pre_transform_specs.sql /opt/monasca/transform/lib/pre_transform_specs.sql +sudo cp /home/ubuntu/monasca-transform/scripts/ddl/transform_specs.sql /opt/monasca/transform/lib/transform_specs.sql sudo mysql -h "127.0.0.1" -um-transform -ppassword < /opt/monasca/transform/lib/pre_transform_specs.sql sudo mysql -h "127.0.0.1" -um-transform -ppassword < /opt/monasca/transform/lib/transform_specs.sql # update the zip file used for spark submit -sudo cp /home/vagrant/monasca-transform/scripts/monasca-transform.zip /opt/monasca/transform/lib/. +sudo cp /home/ubuntu/monasca-transform/scripts/monasca-transform.zip /opt/monasca/transform/lib/. # update the configuration file -sudo cp /home/vagrant/monasca-transform/devstack/files/monasca-transform/monasca-transform.conf /etc/. +sudo cp /home/ubuntu/monasca-transform/devstack/files/monasca-transform/monasca-transform.conf /etc/. if [ -n "$SERVICE_HOST" ]; then sudo sudo sed -i "s/brokers=192\.168\.15\.6:9092/brokers=${SERVICE_HOST}:9092/g" /etc/monasca-transform.conf fi @@ -47,7 +47,7 @@ sudo rm -rf /opt/monasca/transform/venv # refresh the monasca-transform code to /opt/stack sudo rm -rf /opt/stack/monasca-transform pushd /opt/stack -sudo git clone /home/vagrant/monasca-transform +sudo git clone /home/ubuntu/monasca-transform sudo chown -R monasca-transform:monasca-transform /opt/stack/monasca-transform sudo su - monasca-transform -c " virtualenv /opt/monasca/transform/venv ; diff --git a/tools/vagrant/setup_local_repos.sh b/tools/vagrant/setup_local_repos.sh index 1e64b3c..a7ec0ca 100755 --- a/tools/vagrant/setup_local_repos.sh +++ b/tools/vagrant/setup_local_repos.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -rsync -a --exclude='tools/vagrant/.vagrant' /monasca-transform-source /home/vagrant/ -mv /home/vagrant/monasca-transform-source /home/vagrant/monasca-transform -pushd /home/vagrant/monasca-transform +rsync -a --exclude='tools/vagrant/.vagrant' /monasca-transform-source /home/ubuntu/ +mv /home/ubuntu/monasca-transform-source /home/ubuntu/monasca-transform +pushd /home/ubuntu/monasca-transform # prepare the codebase # # generate the sql scripts to populate the database @@ -20,11 +20,11 @@ if [ ${CURRENT_BRANCH} != 'master' ] then echo Maintaining current branch ${CURRENT_BRANCH} # set the branch to what we're using in local.conf - sed -i "s/enable_plugin monasca-transform \/home\/vagrant\/monasca-transform//g" /home/vagrant/devstack/local.conf - sed -i "s/# END DEVSTACK LOCAL.CONF CONTENTS//g" /home/vagrant/devstack/local.conf + sed -i "s/enable_plugin monasca-transform \/home\/ubuntu\/monasca-transform//g" /home/ubuntu/devstack/local.conf + sed -i "s/# END DEVSTACK LOCAL.CONF CONTENTS//g" /home/ubuntu/devstack/local.conf - printf "enable_plugin monasca-transform /home/vagrant/monasca-transform ${CURRENT_BRANCH}\n" >> /home/vagrant/devstack/local.conf - printf "# END DEVSTACK LOCAL.CONF CONTENTS" >> /home/vagrant/devstack/local.conf + printf "enable_plugin monasca-transform /home/ubuntu/monasca-transform ${CURRENT_BRANCH}\n" >> /home/ubuntu/devstack/local.conf + printf "# END DEVSTACK LOCAL.CONF CONTENTS" >> /home/ubuntu/devstack/local.conf fi diff --git a/tools/vagrant/setup_maven_proxy.sh b/tools/vagrant/setup_maven_proxy.sh index a095251..2bb009c 100644 --- a/tools/vagrant/setup_maven_proxy.sh +++ b/tools/vagrant/setup_maven_proxy.sh @@ -12,5 +12,5 @@ else fi echo Now we need to move the settings.xml into place... -cp /home/vagrant/settings.xml /root/.m2/. +cp /home/ubuntu/settings.xml /root/.m2/. chown root:root /root/.m2/settings.xml diff --git a/tox.ini b/tox.ini index cab436a..9b0a9e0 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ setenv = PYTHONUNBUFFERED=1 VIRTUAL_ENV={envdir} DISCOVER_DIRECTORY=tests - PYSPARK_HOME=/home/vagrant/pyspark_venv/bin/python + PYSPARK_HOME=/home/ubuntu/pyspark_venv/bin/python SPARK_HOME=/opt/spark/current deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt