fix for taskflow, py34, and image-build
updated taskflow related code to use newest version of taskflow it seems like filtering logic in monitor_service does not work in py34, changing to more verbose code that seem to work in both py27 and py34 rabbitmq-image build is broken and causing verification gates to fail, applying fixes to get build to work and also updating to remove some dependencies on tripleo-image-elements Change-Id: Ic5fac635fa9f1ebdc6cb7498370c701de8172766
This commit is contained in:
parent
235d5da845
commit
8ccb914277
|
@ -74,3 +74,6 @@ target/
|
|||
|
||||
# Testr coverage
|
||||
.coverage.*
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
|
|
@ -1,4 +1 @@
|
|||
iptables
|
||||
package-installs
|
||||
sysctl
|
||||
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
deb http://www.rabbitmq.com/debian/ testing main
|
|
@ -1,27 +0,0 @@
|
|||
start on runlevel [2345]
|
||||
stop on runlevel [016]
|
||||
respawn
|
||||
# The default post-start of 1 second sleep delays respawning enough to
|
||||
# not hit the default of 10 times in 5 seconds. Make it 2 times in 5s.
|
||||
respawn limit 2 5
|
||||
|
||||
# Process will exit from SIGTERM due to post-stop pkill, prevent this
|
||||
# causing a respawn
|
||||
normal exit 0 TERM
|
||||
|
||||
env RUN_DIR=/var/run/rabbitmq
|
||||
env PID_FILE=$RUN_DIR/pid
|
||||
env OS_SVC_ENABLE_CONTROL=1
|
||||
export OS_SVC_ENABLE_CONTROL
|
||||
|
||||
pre-start script
|
||||
[ -d "$RUN_DIR" ] || install -d -D -m 0755 -o rabbitmq -g rabbitmq $RUN_DIR
|
||||
end script
|
||||
exec /usr/sbin/rabbitmq-server > /var/log/rabbitmq/startup_log \
|
||||
2> /var/log/rabbitmq/startup_err
|
||||
|
||||
post-start exec /usr/sbin/rabbitmqctl wait $PID_FILE >/dev/null 2>&1
|
||||
pre-stop exec /usr/sbin/rabbitmqctl stop $PID_FILE >/dev/null 2>&1
|
||||
|
||||
# Get the Erlang nameserver too.
|
||||
post-stop exec /usr/bin/pkill -u rabbitmq >/dev/null 2>&1
|
|
@ -1,7 +0,0 @@
|
|||
HOME=/var/lib/rabbitmq
|
||||
LOG_BASE=/var/log/rabbitmq
|
||||
MNESIA_BASE=/var/lib/rabbitmq/mnesia
|
||||
PID_FILE=/var/run/rabbitmq/pid
|
||||
# We need to customize log file names so that they do not have @s
|
||||
RABBITMQ_LOGS=$LOG_BASE/$(hostname -s).log
|
||||
RABBITMQ_SASL_LOGS=$LOG_BASE/$(hostname -s)-sasl.log
|
|
@ -1 +0,0 @@
|
|||
rabbitmq-server
|
|
@ -0,0 +1,8 @@
|
|||
ntp:
|
||||
phase: install.d
|
||||
|
||||
python-pip:
|
||||
phase: install.d
|
||||
|
||||
rabbitmq-server:
|
||||
phase: install.d
|
|
@ -4,8 +4,6 @@ set -eux
|
|||
|
||||
FILES="$(dirname $0)/../files"
|
||||
|
||||
install-packages -u rabbitmq-server
|
||||
|
||||
if [ "$DISTRO_NAME" = "ubuntu" ] || [ "$DISTRO_NAME" = "debian" ]; then
|
||||
# Prevent rabbitmq-server from starting automatically
|
||||
update-rc.d -f rabbitmq-server disable
|
||||
|
@ -19,12 +17,27 @@ if [ "$DIB_INIT_SYSTEM" = "systemd" ]; then
|
|||
sed -i 's/\[Service\]/\[Service\]\nRestart=on-failure/g' /lib/systemd/system/rabbitmq-server.service
|
||||
fi
|
||||
|
||||
FILE=/etc/rabbitmq/rabbitmq-env.conf
|
||||
install -g root -o root -m 0755 "${FILES}${FILE}" "${FILE}"
|
||||
|
||||
# Enable ulimits in pam if needed
|
||||
PAM_FILE=/etc/pam.d/su
|
||||
sed -i '/# session.*pam_limits\.so/s/# //' ${PAM_FILE}
|
||||
|
||||
# Reserve the cluster port (61000) from the ephemeral port range.
|
||||
sysctl-append-value net.ipv4.ip_local_reserved_ports 61000
|
||||
EXISTING_RESERVED_PORTS=$(grep -r net.ipv4.ip_local_reserved_ports /etc/sysctl.conf /etc/sysctl.d 2> /dev/null | cut -d'=' -f2)
|
||||
RESERVED_PORTS=61000
|
||||
if ! [ -z $EXISTING_RESERVED_PORTS ]; then
|
||||
# create one port reservation list
|
||||
for port in $EXISTING_RESERVED_PORTS; do
|
||||
RESERVED_PORTS=$RESERVED_PORTS,$port
|
||||
done
|
||||
|
||||
# find files with port reservation settings
|
||||
RESERVATION_FILE_LIST=$(grep -r net.ipv4.ip_local_reserved_ports /etc/sysctl.conf /etc/sysctl.d 2> /dev/null | cut -d':' -f1 | sort | uniq)
|
||||
|
||||
# comment out existing port reservation lines
|
||||
for file in $RESERVATION_FILE_LIST; do
|
||||
sed -i -e 's/\(^net.ipv4.ip_local_reserved_ports=.*\)/#\1/' $file
|
||||
done
|
||||
|
||||
# add port reservation
|
||||
echo "net.ipv4.ip_local_reserved_ports=${RESERVED_PORTS}"
|
||||
fi
|
|
@ -2,15 +2,6 @@
|
|||
|
||||
set -eux
|
||||
|
||||
FILES="$(dirname $0)/../files"
|
||||
|
||||
FILE=/etc/apt/sources.list.d/rabbitmq.list
|
||||
install -g root -o root -m 0755 "${FILES}${FILE}" "${FILE}"
|
||||
|
||||
pushd /tmp
|
||||
|
||||
wget https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
|
||||
|
||||
apt-key add rabbitmq-signing-key-public.asc
|
||||
|
||||
popd
|
||||
echo 'deb http://www.rabbitmq.com/debian/ testing main' > /etc/apt/sources.list.d/rabbitmq.list
|
||||
wget -O- https://www.rabbitmq.com/rabbitmq-release-signing-key.asc |
|
||||
sudo apt-key add -
|
||||
|
|
|
@ -19,7 +19,7 @@ COMMON_ELEMENTS=${COMMON_ELEMENTS:-"vm ubuntu"}
|
|||
|
||||
# Common Settings for all msgaas images builds
|
||||
SIZE="2"
|
||||
ELEMENTS="ntp hosts cue-rabbitmq-base ifmetric"
|
||||
ELEMENTS="cue-rabbitmq-base ifmetric"
|
||||
ELEMENTS_PATH="$CUE_HOME/contrib/image-elements"
|
||||
|
||||
# QEMU Image options
|
||||
|
@ -28,22 +28,16 @@ QEMU_IMG_OPTIONS='compat=0.10'
|
|||
# Install some required apt packages if needed
|
||||
if ! [ -e /usr/sbin/debootstrap -a -e /usr/bin/qemu-img ]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes debootstrap qemu-utils
|
||||
sudo apt-get install --yes debootstrap qemu-utils git python-virtualenv uuid-runtime curl wget parted kpartx
|
||||
fi
|
||||
|
||||
|
||||
if [ ! -d $BUILD_DIR/diskimage-builder ]; then
|
||||
echo "---> Cloning diskimage-builder"
|
||||
git clone https://git.openstack.org/openstack/diskimage-builder $BUILD_DIR/diskimage-builder
|
||||
fi
|
||||
|
||||
if [ ! -d $BUILD_DIR/tripleo-image-elements ]; then
|
||||
echo "---> Cloning tripleo-image-elements"
|
||||
git clone https://git.openstack.org/openstack/tripleo-image-elements $BUILD_DIR/tripleo-image-elements
|
||||
fi
|
||||
|
||||
# Setup the elements path
|
||||
export ELEMENTS_PATH="$ELEMENTS_PATH:$BUILD_DIR/tripleo-image-elements/elements:$BUILD_DIR/diskimage-builder/elements"
|
||||
export ELEMENTS_PATH="$ELEMENTS_PATH:$BUILD_DIR/diskimage-builder/elements"
|
||||
|
||||
# Prepare the build directory
|
||||
if [ ! -d $BUILD_DIR/dist ]; then
|
||||
|
|
|
@ -73,17 +73,17 @@ class MonitorService(service.Service):
|
|||
taskflow_client_instance = taskflow_client.get_client_instance()
|
||||
job_list = taskflow_client_instance.joblist()
|
||||
|
||||
job_list = filter(lambda job:
|
||||
'cluster_status_check' in job.details['store'],
|
||||
job_list)
|
||||
job_list = map(lambda job:
|
||||
job.details['store']['cluster_id'],
|
||||
job_list)
|
||||
clusters = filter(lambda cluster:
|
||||
cluster[0] not in job_list,
|
||||
clusters)
|
||||
cluster_ids = []
|
||||
for job in job_list:
|
||||
if 'cluster_status_check' in job.details['store']:
|
||||
cluster_ids.append(job.details['store']['cluster_id'])
|
||||
|
||||
filtered_clusters = []
|
||||
for cluster in clusters:
|
||||
if cluster[0] not in cluster_ids:
|
||||
filtered_clusters.append(cluster)
|
||||
|
||||
for cluster in filtered_clusters:
|
||||
job_args = {
|
||||
'cluster_status_check': '',
|
||||
'cluster_id': cluster[0],
|
||||
|
|
|
@ -21,7 +21,7 @@ from six.moves import urllib_parse
|
|||
import taskflow.engines as engines
|
||||
import taskflow.jobs.backends as job_backends
|
||||
import taskflow.persistence.backends as persistence_backends
|
||||
import taskflow.persistence.logbook as logbook
|
||||
import taskflow.persistence.models as persistence_models
|
||||
|
||||
|
||||
def _make_conf(backend_uri):
|
||||
|
@ -237,10 +237,13 @@ class Client(object):
|
|||
tx_uuid = uuidutils.generate_uuid()
|
||||
|
||||
job_name = "%s[%s]" % (flow_factory.__name__, tx_uuid)
|
||||
book = logbook.LogBook(job_name, uuid=tx_uuid)
|
||||
book = persistence_models.LogBook(job_name, uuid=tx_uuid)
|
||||
|
||||
if flow_factory is not None:
|
||||
flow_detail = logbook.FlowDetail(job_name, str(uuid.uuid4()))
|
||||
flow_detail = persistence_models.FlowDetail(
|
||||
job_name,
|
||||
str(uuid.uuid4())
|
||||
)
|
||||
book.add(flow_detail)
|
||||
|
||||
job_details = {'store': job_args or {}}
|
||||
|
@ -290,4 +293,4 @@ class Client(object):
|
|||
job = j
|
||||
|
||||
self.jobboard.claim(job, self._client_name)
|
||||
self.jobboard.consume(job, self._client_name)
|
||||
self.jobboard.consume(job, self._client_name)
|
||||
|
|
|
@ -19,8 +19,8 @@ import time
|
|||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from taskflow.conductors import backends as conductors
|
||||
from taskflow.conductors.backends import impl_executor
|
||||
from taskflow.conductors import single_threaded
|
||||
|
||||
import cue.taskflow.client as tf_client
|
||||
import cue.version as version
|
||||
|
@ -151,7 +151,8 @@ class ConductorService(object):
|
|||
persistence=persistence,
|
||||
)
|
||||
|
||||
self._conductor = single_threaded.SingleThreadedConductor(
|
||||
self._conductor = conductors.fetch(
|
||||
kind="blocking",
|
||||
name=self._host,
|
||||
jobboard=jobboard,
|
||||
persistence=persistence,
|
||||
|
|
|
@ -30,9 +30,9 @@ CONF = cfg.CONF
|
|||
class MonitorFunctionalTests(base.FunctionalTestCase):
|
||||
|
||||
cue_monitor_service = None
|
||||
test_uuid_1 = None
|
||||
test_uuid_2 = None
|
||||
test_uuid_3 = None
|
||||
active_cluster_id = None
|
||||
error_cluster_id = None
|
||||
down_cluster_id = None
|
||||
|
||||
def setUp(self):
|
||||
super(MonitorFunctionalTests, self).setUp()
|
||||
|
@ -44,19 +44,19 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
|
|||
CONF.set_override("zk_port", "", group="taskflow",
|
||||
enforce_type=True)
|
||||
|
||||
self.test_uuid_1 = uuid.uuid4()
|
||||
self.test_uuid_2 = uuid.uuid4()
|
||||
self.test_uuid_3 = uuid.uuid4()
|
||||
self.active_cluster_id = uuid.uuid4()
|
||||
self.error_cluster_id = uuid.uuid4()
|
||||
self.down_cluster_id = uuid.uuid4()
|
||||
|
||||
# Add some test clusters
|
||||
set_up_test_clusters(
|
||||
self.context, models.Status.ACTIVE, self.test_uuid_1, 3
|
||||
self.context, models.Status.ACTIVE, self.active_cluster_id, 3
|
||||
)
|
||||
set_up_test_clusters(
|
||||
self.context, models.Status.ERROR, self.test_uuid_2, 3
|
||||
self.context, models.Status.ERROR, self.error_cluster_id, 3
|
||||
)
|
||||
set_up_test_clusters(
|
||||
self.context, models.Status.DOWN, self.test_uuid_3, 1
|
||||
self.context, models.Status.DOWN, self.down_cluster_id, 1
|
||||
)
|
||||
|
||||
self.cue_monitor_service = cue_monitor_service.MonitorService()
|
||||
|
@ -81,12 +81,14 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
|
|||
|
||||
def test_check(self):
|
||||
tf_instance = tf_client.get_client_instance()
|
||||
start_job_list_length = len(tf_instance.joblist())
|
||||
start_job_list = tf_instance.joblist()
|
||||
start_job_list_length = len(start_job_list)
|
||||
|
||||
# Test while job board is empty
|
||||
self.cue_monitor_service.check()
|
||||
|
||||
end_job_list_length = len(tf_instance.joblist())
|
||||
end_job_list = sorted(tf_instance.joblist())
|
||||
end_job_list_length = len(end_job_list)
|
||||
|
||||
self.assertEqual(2, end_job_list_length - start_job_list_length,
|
||||
"Job list should only have two "
|
||||
|
@ -96,7 +98,10 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
|
|||
self.cue_monitor_service.check()
|
||||
|
||||
# No new jobs should have been added.
|
||||
self.assertEqual(0, len(tf_instance.joblist()) - end_job_list_length)
|
||||
new_end_job_list = sorted(tf_instance.joblist())
|
||||
|
||||
self.assertEqual(end_job_list, new_end_job_list,
|
||||
"Job list should not have changed")
|
||||
|
||||
def test_get_cluster_id_node_ids(self):
|
||||
clusters = cue_monitor_service.get_cluster_id_node_ids()
|
||||
|
@ -106,11 +111,11 @@ class MonitorFunctionalTests(base.FunctionalTestCase):
|
|||
" and DOWN clusters. Found: " + str(len(clusters)))
|
||||
|
||||
for cluster in clusters:
|
||||
if cluster[0] == str(self.test_uuid_1):
|
||||
if cluster[0] == str(self.active_cluster_id):
|
||||
self.assertEqual(3, len(cluster[1]),
|
||||
"Expected to find three nodes in this "
|
||||
"cluster. Found: " + str(cluster[1]))
|
||||
elif cluster[0] == str(self.test_uuid_3):
|
||||
elif cluster[0] == str(self.down_cluster_id):
|
||||
self.assertEqual(1, len(cluster[1]),
|
||||
"Expected to find one node in this "
|
||||
"cluster. Found: " + str(cluster[1]))
|
||||
|
|
Loading…
Reference in New Issue