fix for taskflow, py34, and image-build

updated taskflow related code to use newest version of taskflow

it seems like filtering logic in monitor_service does not work in py34,
changing to more verbose code that seem to work in both py27 and py34

rabbitmq-image build is broken and causing verification gates to fail,
applying fixes to get build to work and also updating to remove some
dependencies on tripleo-image-elements

Change-Id: Ic5fac635fa9f1ebdc6cb7498370c701de8172766
This commit is contained in:
Min Pae 2016-06-02 21:02:05 -07:00
parent 235d5da845
commit 8ccb914277
15 changed files with 74 additions and 95 deletions

3
.gitignore vendored
View File

@ -74,3 +74,6 @@ target/
# Testr coverage
.coverage.*
# pyenv
.python-version

View File

@ -1,4 +1 @@
iptables
package-installs
sysctl

View File

@ -1 +0,0 @@
deb http://www.rabbitmq.com/debian/ testing main

View File

@ -1,27 +0,0 @@
start on runlevel [2345]
stop on runlevel [016]
respawn
# The default post-start of 1 second sleep delays respawning enough to
# not hit the default of 10 times in 5 seconds. Make it 2 times in 5s.
respawn limit 2 5
# Process will exit from SIGTERM due to post-stop pkill, prevent this
# causing a respawn
normal exit 0 TERM
env RUN_DIR=/var/run/rabbitmq
env PID_FILE=$RUN_DIR/pid
env OS_SVC_ENABLE_CONTROL=1
export OS_SVC_ENABLE_CONTROL
pre-start script
[ -d "$RUN_DIR" ] || install -d -D -m 0755 -o rabbitmq -g rabbitmq $RUN_DIR
end script
exec /usr/sbin/rabbitmq-server > /var/log/rabbitmq/startup_log \
2> /var/log/rabbitmq/startup_err
post-start exec /usr/sbin/rabbitmqctl wait $PID_FILE >/dev/null 2>&1
pre-stop exec /usr/sbin/rabbitmqctl stop $PID_FILE >/dev/null 2>&1
# Get the Erlang nameserver too.
post-stop exec /usr/bin/pkill -u rabbitmq >/dev/null 2>&1

View File

@ -1,7 +0,0 @@
HOME=/var/lib/rabbitmq
LOG_BASE=/var/log/rabbitmq
MNESIA_BASE=/var/lib/rabbitmq/mnesia
PID_FILE=/var/run/rabbitmq/pid
# We need to customize log file names so that they do not have @s
RABBITMQ_LOGS=$LOG_BASE/$(hostname -s).log
RABBITMQ_SASL_LOGS=$LOG_BASE/$(hostname -s)-sasl.log

View File

@ -0,0 +1,8 @@
ntp:
phase: install.d
python-pip:
phase: install.d
rabbitmq-server:
phase: install.d

View File

@ -4,8 +4,6 @@ set -eux
FILES="$(dirname $0)/../files"
install-packages -u rabbitmq-server
if [ "$DISTRO_NAME" = "ubuntu" ] || [ "$DISTRO_NAME" = "debian" ]; then
# Prevent rabbitmq-server from starting automatically
update-rc.d -f rabbitmq-server disable
@ -19,12 +17,27 @@ if [ "$DIB_INIT_SYSTEM" = "systemd" ]; then
sed -i 's/\[Service\]/\[Service\]\nRestart=on-failure/g' /lib/systemd/system/rabbitmq-server.service
fi
FILE=/etc/rabbitmq/rabbitmq-env.conf
install -g root -o root -m 0755 "${FILES}${FILE}" "${FILE}"
# Enable ulimits in pam if needed
PAM_FILE=/etc/pam.d/su
sed -i '/# session.*pam_limits\.so/s/# //' ${PAM_FILE}
# Reserve the cluster port (61000) from the ephemeral port range.
sysctl-append-value net.ipv4.ip_local_reserved_ports 61000
EXISTING_RESERVED_PORTS=$(grep -r net.ipv4.ip_local_reserved_ports /etc/sysctl.conf /etc/sysctl.d 2> /dev/null | cut -d'=' -f2)
RESERVED_PORTS=61000
if ! [ -z $EXISTING_RESERVED_PORTS ]; then
# create one port reservation list
for port in $EXISTING_RESERVED_PORTS; do
RESERVED_PORTS=$RESERVED_PORTS,$port
done
# find files with port reservation settings
RESERVATION_FILE_LIST=$(grep -r net.ipv4.ip_local_reserved_ports /etc/sysctl.conf /etc/sysctl.d 2> /dev/null | cut -d':' -f1 | sort | uniq)
# comment out existing port reservation lines
for file in $RESERVATION_FILE_LIST; do
sed -i -e 's/\(^net.ipv4.ip_local_reserved_ports=.*\)/#\1/' $file
done
# add port reservation
echo "net.ipv4.ip_local_reserved_ports=${RESERVED_PORTS}"
fi

View File

@ -2,15 +2,6 @@
set -eux
FILES="$(dirname $0)/../files"
FILE=/etc/apt/sources.list.d/rabbitmq.list
install -g root -o root -m 0755 "${FILES}${FILE}" "${FILE}"
pushd /tmp
wget https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
apt-key add rabbitmq-signing-key-public.asc
popd
echo 'deb http://www.rabbitmq.com/debian/ testing main' > /etc/apt/sources.list.d/rabbitmq.list
wget -O- https://www.rabbitmq.com/rabbitmq-release-signing-key.asc |
sudo apt-key add -

View File

@ -19,7 +19,7 @@ COMMON_ELEMENTS=${COMMON_ELEMENTS:-"vm ubuntu"}
# Common Settings for all msgaas images builds
SIZE="2"
ELEMENTS="ntp hosts cue-rabbitmq-base ifmetric"
ELEMENTS="cue-rabbitmq-base ifmetric"
ELEMENTS_PATH="$CUE_HOME/contrib/image-elements"
# QEMU Image options
@ -28,22 +28,16 @@ QEMU_IMG_OPTIONS='compat=0.10'
# Install some required apt packages if needed
if ! [ -e /usr/sbin/debootstrap -a -e /usr/bin/qemu-img ]; then
sudo apt-get update
sudo apt-get install --yes debootstrap qemu-utils
sudo apt-get install --yes debootstrap qemu-utils git python-virtualenv uuid-runtime curl wget parted kpartx
fi
if [ ! -d $BUILD_DIR/diskimage-builder ]; then
echo "---> Cloning diskimage-builder"
git clone https://git.openstack.org/openstack/diskimage-builder $BUILD_DIR/diskimage-builder
fi
if [ ! -d $BUILD_DIR/tripleo-image-elements ]; then
echo "---> Cloning tripleo-image-elements"
git clone https://git.openstack.org/openstack/tripleo-image-elements $BUILD_DIR/tripleo-image-elements
fi
# Setup the elements path
export ELEMENTS_PATH="$ELEMENTS_PATH:$BUILD_DIR/tripleo-image-elements/elements:$BUILD_DIR/diskimage-builder/elements"
export ELEMENTS_PATH="$ELEMENTS_PATH:$BUILD_DIR/diskimage-builder/elements"
# Prepare the build directory
if [ ! -d $BUILD_DIR/dist ]; then

View File

@ -73,17 +73,17 @@ class MonitorService(service.Service):
taskflow_client_instance = taskflow_client.get_client_instance()
job_list = taskflow_client_instance.joblist()
job_list = filter(lambda job:
'cluster_status_check' in job.details['store'],
job_list)
job_list = map(lambda job:
job.details['store']['cluster_id'],
job_list)
clusters = filter(lambda cluster:
cluster[0] not in job_list,
clusters)
cluster_ids = []
for job in job_list:
if 'cluster_status_check' in job.details['store']:
cluster_ids.append(job.details['store']['cluster_id'])
filtered_clusters = []
for cluster in clusters:
if cluster[0] not in cluster_ids:
filtered_clusters.append(cluster)
for cluster in filtered_clusters:
job_args = {
'cluster_status_check': '',
'cluster_id': cluster[0],

View File

@ -21,7 +21,7 @@ from six.moves import urllib_parse
import taskflow.engines as engines
import taskflow.jobs.backends as job_backends
import taskflow.persistence.backends as persistence_backends
import taskflow.persistence.logbook as logbook
import taskflow.persistence.models as persistence_models
def _make_conf(backend_uri):
@ -237,10 +237,13 @@ class Client(object):
tx_uuid = uuidutils.generate_uuid()
job_name = "%s[%s]" % (flow_factory.__name__, tx_uuid)
book = logbook.LogBook(job_name, uuid=tx_uuid)
book = persistence_models.LogBook(job_name, uuid=tx_uuid)
if flow_factory is not None:
flow_detail = logbook.FlowDetail(job_name, str(uuid.uuid4()))
flow_detail = persistence_models.FlowDetail(
job_name,
str(uuid.uuid4())
)
book.add(flow_detail)
job_details = {'store': job_args or {}}
@ -290,4 +293,4 @@ class Client(object):
job = j
self.jobboard.claim(job, self._client_name)
self.jobboard.consume(job, self._client_name)
self.jobboard.consume(job, self._client_name)

View File

@ -19,8 +19,8 @@ import time
from oslo_config import cfg
from oslo_log import log as logging
from taskflow.conductors import backends as conductors
from taskflow.conductors.backends import impl_executor
from taskflow.conductors import single_threaded
import cue.taskflow.client as tf_client
import cue.version as version
@ -151,7 +151,8 @@ class ConductorService(object):
persistence=persistence,
)
self._conductor = single_threaded.SingleThreadedConductor(
self._conductor = conductors.fetch(
kind="blocking",
name=self._host,
jobboard=jobboard,
persistence=persistence,

View File

@ -30,9 +30,9 @@ CONF = cfg.CONF
class MonitorFunctionalTests(base.FunctionalTestCase):
cue_monitor_service = None
test_uuid_1 = None
test_uuid_2 = None
test_uuid_3 = None
active_cluster_id = None
error_cluster_id = None
down_cluster_id = None
def setUp(self):
super(MonitorFunctionalTests, self).setUp()
@ -44,19 +44,19 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
CONF.set_override("zk_port", "", group="taskflow",
enforce_type=True)
self.test_uuid_1 = uuid.uuid4()
self.test_uuid_2 = uuid.uuid4()
self.test_uuid_3 = uuid.uuid4()
self.active_cluster_id = uuid.uuid4()
self.error_cluster_id = uuid.uuid4()
self.down_cluster_id = uuid.uuid4()
# Add some test clusters
set_up_test_clusters(
self.context, models.Status.ACTIVE, self.test_uuid_1, 3
self.context, models.Status.ACTIVE, self.active_cluster_id, 3
)
set_up_test_clusters(
self.context, models.Status.ERROR, self.test_uuid_2, 3
self.context, models.Status.ERROR, self.error_cluster_id, 3
)
set_up_test_clusters(
self.context, models.Status.DOWN, self.test_uuid_3, 1
self.context, models.Status.DOWN, self.down_cluster_id, 1
)
self.cue_monitor_service = cue_monitor_service.MonitorService()
@ -81,12 +81,14 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
def test_check(self):
tf_instance = tf_client.get_client_instance()
start_job_list_length = len(tf_instance.joblist())
start_job_list = tf_instance.joblist()
start_job_list_length = len(start_job_list)
# Test while job board is empty
self.cue_monitor_service.check()
end_job_list_length = len(tf_instance.joblist())
end_job_list = sorted(tf_instance.joblist())
end_job_list_length = len(end_job_list)
self.assertEqual(2, end_job_list_length - start_job_list_length,
"Job list should only have two "
@ -96,7 +98,10 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
self.cue_monitor_service.check()
# No new jobs should have been added.
self.assertEqual(0, len(tf_instance.joblist()) - end_job_list_length)
new_end_job_list = sorted(tf_instance.joblist())
self.assertEqual(end_job_list, new_end_job_list,
"Job list should not have changed")
def test_get_cluster_id_node_ids(self):
clusters = cue_monitor_service.get_cluster_id_node_ids()
@ -106,11 +111,11 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
" and DOWN clusters. Found: " + str(len(clusters)))
for cluster in clusters:
if cluster[0] == str(self.test_uuid_1):
if cluster[0] == str(self.active_cluster_id):
self.assertEqual(3, len(cluster[1]),
"Expected to find three nodes in this "
"cluster. Found: " + str(cluster[1]))
elif cluster[0] == str(self.test_uuid_3):
elif cluster[0] == str(self.down_cluster_id):
self.assertEqual(1, len(cluster[1]),
"Expected to find one node in this "
"cluster. Found: " + str(cluster[1]))