From 20e04092c8ce311017b2d490c4041cfb9b3a9395 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 3 May 2018 10:51:30 +1000 Subject: [PATCH] Fix gate issues This is a combination of 3 commits. This is the 1st commit message: Work around uwsgi breakage uwsgi broke installation from source with their latest release [0]. Since we want to move away from source installation anyway, make grenade based jobs non-voting for the moment so that we can backport [1] properly. [0] https://bugs.launchpad.net/bugs/1883468 [1] https://review.opendev.org/577955 This is the commit message #2: Use packaged uwsgi on Fedora and Ubuntu Building uwsgi from source was a workaround that was introduced a long time ago, it doesn't seem like it is needed anymore and will actually fail for Ubuntu 20.04. Also it doesn't match what will happen for most real-world installations, so let's try to get back to using distro packages. We'll still use the source install for RHEL/Centos, it remains to be tested whether we can get back to using distro versions there, too. This is the commit message #3: Use uwsgi binary from path All these uwsgi invocations assume that the uwsgi binary is in the same directory as their project binaries are installed into (probably /usr/bin). That may not be correct -- for example if using a packaged uwsgi on Fedora the binary will live in /usr/sbin/uwsgi (not /usr/bin where the project files from pip are). Switch invocations to just find it in the path. While all this was happening, Zuul updated to Ansible 2.9 as well, so we require a fix for Ansible syntax in 736006 as well. Depends-On: https://review.opendev.org/735535 Depends-On: https://review.opendev.org/736006 Related-Bug: 1883468 Change-Id: I82f539bfa533349293dd5a8ce309c9cc0ffb0393 (cherry picked from commit 84737ebd96327310ec5f8f7c312aeae12cbeb234) (cherry picked from commit 2d903568ed4158aa668bbda6986307a8780c71a4) (cherry picked from commit 312517d5101206b33d3c395d27ec93d385b7fd24) --- .zuul.yaml | 28 ++++++++++++++------ lib/apache | 66 ++++++++++++++++++++++++++++++++-------------- lib/cinder | 2 +- lib/glance | 2 +- lib/keystone | 2 +- lib/neutron | 2 +- lib/neutron-legacy | 2 +- lib/nova | 4 +-- lib/placement | 2 +- 9 files changed, 74 insertions(+), 36 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index ff3f205d32..0112bdc203 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -619,14 +619,16 @@ check: jobs: - devstack - - devstack-xenial + - devstack-xenial: + voting: false - devstack-ipv6 - devstack-platform-centos-7 - devstack-platform-opensuse-15 - devstack-platform-fedora-29 - devstack-platform-xenial - devstack-multinode - - devstack-multinode-xenial + - devstack-multinode-xenial: + voting: false - devstack-unit-tests - openstack-tox-bashate - ironic-tempest-ipa-wholedisk-bios-agent_ipmitool-tinyipa: @@ -640,7 +642,13 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + - grenade: + voting: false + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ - neutron-grenade-multinode: + voting: false irrelevant-files: - ^.*\.rst$ - ^doc/.*$ @@ -664,16 +672,16 @@ gate: jobs: - devstack - - devstack-xenial + #- devstack-xenial - devstack-ipv6 - devstack-multinode - - devstack-multinode-xenial + #- devstack-multinode-xenial - devstack-unit-tests - openstack-tox-bashate - - neutron-grenade-multinode: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + # - neutron-grenade-multinode: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - neutron-tempest-linuxbridge: irrelevant-files: - ^.*\.rst$ @@ -682,6 +690,10 @@ irrelevant-files: - ^.*\.rst$ - ^doc/.*$ + # - grenade: + # irrelevant-files: + # - ^.*\.rst$ + # - ^doc/.*$ - openstacksdk-functional-devstack: irrelevant-files: - ^.*\.rst$ diff --git a/lib/apache b/lib/apache index 84cec73234..a31188bebb 100644 --- a/lib/apache +++ b/lib/apache @@ -82,26 +82,52 @@ function install_apache_uwsgi { apxs="apxs" fi - # Ubuntu xenial is back level on uwsgi so the proxy doesn't - # actually work. Hence we have to build from source for now. + # This varies based on packaged/installed. If we've + # pip_installed, then the pip setup will only build a "python" + # module that will be either python2 or python3 depending on what + # it was built with. # - # Centos 7 actually has the module in epel, but there was a big - # push to disable epel by default. As such, compile from source - # there as well. + # For package installs, the distro ships both plugins and you need + # to select the right one ... it will not be autodetected. + if python3_enabled; then + UWSGI_PYTHON_PLUGIN=python3 + else + UWSGI_PYTHON_PLUGIN=python + fi - local dir - dir=$(mktemp -d) - pushd $dir - pip_install uwsgi - pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt - local uwsgi - uwsgi=$(ls uwsgi*) - tar xvf $uwsgi - cd uwsgi*/apache2 - sudo $apxs -i -c mod_proxy_uwsgi.c - popd - # delete the temp directory - sudo rm -rf $dir + if is_ubuntu; then + install_package uwsgi \ + uwsgi-plugin-python \ + uwsgi-plugin-python3 \ + libapache2-mod-proxy-uwsgi + elif [[ $os_VENDOR == "Fedora" ]]; then + # Note httpd comes with mod_proxy_uwsgi and it is loaded by + # default; the mod_proxy_uwsgi package actually conflicts now. + # See: + # https://bugzilla.redhat.com/show_bug.cgi?id=1574335 + # + # Thus there is nothing else to do after this install + install_package uwsgi \ + uwsgi-plugin-python3 + else + # Centos actually has the module in epel, but there was a big + # push to disable epel by default. As such, compile from source + # there. + local dir + dir=$(mktemp -d) + pushd $dir + pip_install uwsgi + pip download uwsgi -c $REQUIREMENTS_DIR/upper-constraints.txt + local uwsgi + uwsgi=$(ls uwsgi*) + tar xvf $uwsgi + cd uwsgi*/apache2 + sudo $apxs -i -c mod_proxy_uwsgi.c + popd + # delete the temp directory + sudo rm -rf $dir + UWSGI_PYTHON_PLUGIN=python + fi if is_ubuntu || is_suse ; then # we've got to enable proxy and proxy_uwsgi for this to work @@ -265,7 +291,7 @@ function write_uwsgi_config { # configured after graceful shutdown iniset "$file" uwsgi worker-reload-mercy $WORKER_TIMEOUT iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM @@ -318,7 +344,7 @@ function write_local_uwsgi_http_config { iniset "$file" uwsgi die-on-term true iniset "$file" uwsgi exit-on-reload false iniset "$file" uwsgi enable-threads true - iniset "$file" uwsgi plugins python + iniset "$file" uwsgi plugins http,${UWSGI_PYTHON_PLUGIN} # uwsgi recommends this to prevent thundering herd on accept. iniset "$file" uwsgi thunder-lock true # Set hook to trigger graceful shutdown on SIGTERM diff --git a/lib/cinder b/lib/cinder index fd960535d9..c2e55f9173 100644 --- a/lib/cinder +++ b/lib/cinder @@ -492,7 +492,7 @@ function start_cinder { start_tls_proxy cinder '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT fi else - run_process "c-api" "$CINDER_BIN_DIR/uwsgi --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" + run_process "c-api" "$(which uwsgi) --procname-prefix cinder-api --ini $CINDER_UWSGI_CONF" cinder_url=$service_protocol://$SERVICE_HOST/volume/v3 fi fi diff --git a/lib/glance b/lib/glance index 54d3276433..56166a0013 100644 --- a/lib/glance +++ b/lib/glance @@ -325,7 +325,7 @@ function start_glance { run_process g-reg "$GLANCE_BIN_DIR/glance-registry --config-file=$GLANCE_CONF_DIR/glance-registry.conf" if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process g-api "$GLANCE_BIN_DIR/uwsgi --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" + run_process g-api "$(which uwsgi) --procname-prefix glance-api --ini $GLANCE_UWSGI_CONF" else run_process g-api "$GLANCE_BIN_DIR/glance-api --config-dir=$GLANCE_CONF_DIR" fi diff --git a/lib/keystone b/lib/keystone index 9ceb829264..1fdd941a19 100644 --- a/lib/keystone +++ b/lib/keystone @@ -523,7 +523,7 @@ function start_keystone { enable_apache_site keystone restart_apache_server else # uwsgi - run_process keystone "$KEYSTONE_BIN_DIR/uwsgi --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" + run_process keystone "$(which uwsgi) --procname-prefix keystone --ini $KEYSTONE_PUBLIC_UWSGI_CONF" "" fi echo "Waiting for keystone to start..." diff --git a/lib/neutron b/lib/neutron index 888b5e864e..16c20563bd 100644 --- a/lib/neutron +++ b/lib/neutron @@ -466,7 +466,7 @@ function start_neutron_api { done if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$service_protocol://$NEUTRON_SERVICE_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $opts" diff --git a/lib/neutron-legacy b/lib/neutron-legacy index dbd6e2c06b..49014808d8 100644 --- a/lib/neutron-legacy +++ b/lib/neutron-legacy @@ -483,7 +483,7 @@ function start_neutron_service_and_check { # Start the Neutron service if [ "$NEUTRON_DEPLOY_MOD_WSGI" == "True" ]; then enable_service neutron-api - run_process neutron-api "$NEUTRON_BIN_DIR/uwsgi --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" + run_process neutron-api "$(which uwsgi) --procname-prefix neutron-api --ini $NEUTRON_UWSGI_CONF" neutron_url=$Q_PROTOCOL://$Q_HOST/networking/ enable_service neutron-rpc-server run_process neutron-rpc-server "$NEUTRON_BIN_DIR/neutron-rpc-server $cfg_file_options" diff --git a/lib/nova b/lib/nova index c41f881fa1..5b3246a2c2 100644 --- a/lib/nova +++ b/lib/nova @@ -850,7 +850,7 @@ function start_nova_api { start_tls_proxy nova '*' $NOVA_SERVICE_PORT $NOVA_SERVICE_HOST $NOVA_SERVICE_PORT_INT fi else - run_process "n-api" "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" + run_process "n-api" "$(which uwsgi) --procname-prefix nova-api --ini $NOVA_UWSGI_CONF" nova_url=$service_protocol://$SERVICE_HOST/compute/v2.1/ fi @@ -941,7 +941,7 @@ function start_nova_rest { if [ "$NOVA_USE_MOD_WSGI" == "False" ]; then run_process n-api-meta "$NOVA_BIN_DIR/nova-api-metadata --config-file $compute_cell_conf" else - run_process n-api-meta "$NOVA_BIN_DIR/uwsgi --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" + run_process n-api-meta "$(which uwsgi) --procname-prefix nova-api-meta --ini $NOVA_METADATA_UWSGI_CONF" fi export PATH=$old_path diff --git a/lib/placement b/lib/placement index 785b0ddfca..2a449bfa90 100644 --- a/lib/placement +++ b/lib/placement @@ -144,7 +144,7 @@ function install_placement { # start_placement_api() - Start the API processes ahead of other things function start_placement_api { if [[ "$WSGI_MODE" == "uwsgi" ]]; then - run_process "placement-api" "$PLACEMENT_BIN_DIR/uwsgi --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" + run_process "placement-api" "$(which uwsgi) --procname-prefix placement --ini $PLACEMENT_UWSGI_CONF" else enable_apache_site placement-api restart_apache_server