Use a smaller base job for the perfload run

In this job we install placement by hand, based on the
instructions in
https://docs.openstack.org/placement/latest/contributor/quick-dev.html
and run the placeload command against it. This avoids a lot of node
set up time.

* mysql is installed, placement is installed, uwsgi is installed
* the database is synced
* the service started, via uwsgi, which run with 5 processs each
  with 25 threads, otherwise writing the resource providers is
  very slow and causes errors in placeload. It's an 8 core vm.
* placeload is called

A post.yaml is added to get the generated logs back to zuul.

Change-Id: I93875e3ce1f77fdb237e339b7b3e38abe3dad8f7
This commit is contained in:
Chris Dent 2018-11-21 13:59:45 +00:00
parent e6545dc2b2
commit 3ae8653338
3 changed files with 54 additions and 37 deletions

View File

@ -22,23 +22,13 @@
- openstack-tox-functional-py35
- openstack-tox-functional-py36
# TODO(cdent): Using devstack (even minimal) here is more than we need. We
# ought to be able to take a blank node, install placement on it, use
# environment variables for configuration, run placeload, and collect the logs.
# However, that requires more knowledge of zuul and ansible than I currently
# have and we can iterate.
- job:
name: placement-perfload
parent: devstack-minimal
parent: base
description: |
Run a tiny devstack with just placement and keystone and log placement
performance.
A simple node on which to run placement with the barest of configs and
make performance related tests against it.
required-projects:
- git.openstack.org/openstack/keystone
- git.openstack.org/openstack/placement
run: playbooks/perfload.yaml
vars:
devstack_services:
mysql: true
placement-api: true
key: true
post-run: playbooks/post.yaml

View File

@ -1,9 +1,3 @@
- hosts: all
# Do the default devstack orchestration and then follow with perfload
strategy: linear
roles:
- orchestrate-devstack
- hosts: all
tasks:
# Borrowed from devstack tasks but we want it early.
@ -13,7 +7,39 @@
path: "{{ ansible_user_dir }}/logs"
state: directory
owner: "{{ ansible_user }}"
- name: start placement
args:
chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/placement"
shell:
executable: /bin/bash
cmd: |
set -x
# TODO(cdent): Presumably ansible can do this, perhaps with 'package'.
# create database
sudo debconf-set-selections <<MYSQL_PRESEED
mysql-server mysql-server/root_password password secret
mysql-server mysql-server/root_password_again password secret
mysql-server mysql-server/start_on_boot boolean true
MYSQL_PRESEED
sudo apt-get install -y mysql-server mysql-client libmysqlclient-dev
sudo mysql -uroot -psecret -e "DROP DATABASE IF EXISTS placement;"
sudo mysql -uroot -psecret -e "CREATE DATABASE placement CHARACTER SET utf8;"
sudo mysql -uroot -psecret -e "GRANT ALL PRIVILEGES ON placement.* TO 'root'@'%' identified by 'secret';"
python -m virtualenv -p python3 .placement
. .placement/bin/activate
pip install . PyMySQL uwsgi
# set empty config, we shouldn't need a file, see https://review.openstack.org/#/c/619049
sudo mkdir /etc/placement
sudo touch /etc/placement/placement.conf
export OS_PLACEMENT_DATABASE__CONNECTION=mysql+pymysql://root:secret@127.0.0.1/placement?charset=utf8
export OS_DEFAULT__DEBUG=True
export OS_API__AUTH_STRATEGY=noauth2
# sync tables
placement-manage db sync
uwsgi --http :8000 --wsgi-file .placement/bin/placement-api --daemonize {{ ansible_user_dir }}/logs/placement-api.log --processes 5 --threads 25
- name: placement performance
args:
chdir: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/placement"
shell:
executable: /bin/bash
# TODO(cdent): Change this task to a role?
@ -52,19 +78,18 @@
AGGREGATE="14a5c8a3-5a99-4e8f-88be-00d85fcb1c17"
TRAIT="HW_CPU_X86_AVX2"
PLACEMENT_QUERY="resources=VCPU:1,DISK_GB:10,MEMORY_MB:256&member_of=${AGGREGATE}&required=${TRAIT}"
RETURN_CODE=0
BASE=${BASE:-/opt/stack}
source ${BASE}/devstack/functions
source ${BASE}/devstack/lib/placement
# Putting the log here ought to mean it is automatically gathered by zuul
LOG=placement-perf.txt
LOG_DEST={{ ansible_user_dir }}/logs
COUNT=1000
trap "sudo cp -p $LOG $LOG_DEST" EXIT
function check_placement {
local placement_url
local rp_count
local code
code=0
python -m virtualenv -p python3 .placeload
. .placeload/bin/activate
@ -72,16 +97,14 @@
# install placeload
pip install 'placeload==0.3.0'
# Turn off keystone auth
iniset -sudo $PLACEMENT_CONF api auth_strategy noauth2
restart_service devstack@placement-api
# get placement endpoint
placement_url=$(get_endpoint_url placement public)
placement_url="http://localhost:8000"
# load with placeload
(
echo "$EXPLANATION"
# preheat the aggregates to avoid https://bugs.launchpad.net/nova/+bug/1804453
placeload $placement_url 10
placeload $placement_url $COUNT
) | tee -a $LOG
rp_count=$(curl -H 'x-auth-token: admin' $placement_url/resource_providers |json_pp|grep -c '"name"')
@ -96,16 +119,11 @@
(
echo "Unable to create expected number of resource providers. Expected: ${COUNT}, Got: $rp_count"
echo "See job-output.txt.gz and logs/screen-placement-api.txt.gz for additional detail."
RETURN_CODE=1
) | tee -a $LOG
code=1
fi
deactivate
sudo cp -p $LOG $LOG_DEST
exit $code
}
# Be admin
set +x
source $BASE/devstack/openrc admin
set -x
check_placement
exit $RETURN_CODE

9
playbooks/post.yaml Normal file
View File

@ -0,0 +1,9 @@
- hosts: all
tasks:
- name: Copy logs back to the executor
synchronize:
src: "{{ ansible_user_dir }}/logs"
dest: "{{ zuul.executor.log_root }}/"
mode: pull
rsync_opts:
- "--quiet"