Enable to run rootwrap/privsep related functional tests

This change defines tools/deploy_rootwrap.sh. It builds neutron-rootwrap
config from neutron, neutron-fwaas and functests[1] filters. It's an
enabler to run functional tests which requires rootwrap/privsep
features.

An alternative was to build neutron-rootwrap config from neutron-fwaas
and functests filters[1] only. But it implies to add many filters from
neutron in functests filters in order to run FWaaS functional tests on
the long term.

Most of the logic in gate_hook.sh[2] has been removed because it's
now done by neutron tools/configure_for_func_testing.sh through
configure_for_fwaas_func_testing.sh[3].

post_test_hook.sh changes now neutron permissions in order to allow
stack user to run pip install neutron in /opt/stack/new/neutron.

This change defines the modules [5][6] to validate that we can run
privileged privsep process in functional tests. These modules will be
removed when "real" functional tests will be added.

[1] neutron_fwaas/tests/contrib/functional-testing.filters
[2] neutron_fwaas/tests/contrib/gate_hook.sh
[3] tools/configure_for_fwaas_func_testing.sh
[4] neutron_fwaas/tests/contrib/post_test_hook.sh
[5] neutron_fwaas.privileged.tests.functional.dummy
[6] neutron_fwaas.tests.functional.privileged.test_dummy

Change-Id: Ie077092c03efca3856c27c581bba5c5b84db3a2a

neutron_fwaas/privileged/__init__.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2017 Thales Services SAS
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_privsep import capabilities as c
+from oslo_privsep import priv_context
+
+# It is expected that most (if not all) neutron-fwaas operations can be
+# executed with these privileges.
+default = priv_context.PrivContext(
+    __name__,
+    cfg_section='privsep',
+    pypath=__name__ + '.default',
+    # TODO(gus): CAP_SYS_ADMIN is required (only?) for manipulating
+    # network namespaces.  SYS_ADMIN is a lot of scary powers, so
+    # consider breaking this out into a separate minimal context.
+    capabilities=[c.CAP_SYS_ADMIN, c.CAP_NET_ADMIN],
+)

neutron_fwaas/privileged/tests/functional/dummy.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2017 Thales Services SAS
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_utils import uuidutils
+from pyroute2 import netns as pynetns
+
+from neutron_fwaas import privileged
+
+
+# TODO(cby): move this method in neutron.tests.functional.privileged associated
+# to a new privsep context.
+@privileged.default.entrypoint
+def dummy():
+    """This method aim is to validate that we can use privsep in functests."""
+    namespace = 'dummy-%s' % uuidutils.generate_uuid()
+    pynetns.create(namespace)
+    pynetns.remove(namespace)

neutron_fwaas/tests/contrib/functional-testing.filters

@@ -0,0 +1,7 @@
+# neutron-rootwrap command filters to support functional testing.  It
+# is NOT intended to be used outside of a test environment.
+#
+# This file should be owned by (and only-writeable by) the root user
+
+[Filters]
+#none currently

neutron_fwaas/tests/contrib/gate_hook.sh

@@ -2,84 +2,23 @@
 
 set -ex
 
-# Below variables are set to execute this script
-IS_GATE=${IS_GATE:-False}
-INSTALL_MYSQL_ONLY=${INSTALL_MYSQL_ONLY:-False}
+VENV=${1:-"dsvm-functional"}
 
-CONTRIB_DIR="$BASE/new/neutron-fwaas/neutron_fwaas/tests/contrib"
+GATE_DEST=$BASE/new
+FWAAS_PATH=$GATE_DEST/neutron-fwaas
+DEVSTACK_PATH=$GATE_DEST/devstack
 
-$BASE/new/devstack-gate/devstack-vm-gate.sh
 
-# Add a rootwrap filter to support test-only
-# configuration (e.g. a KillFilter for processes that
-# use the python installed in a tox env).
-FUNC_FILTER=$CONTRIB_DIR/filters.template
-sed -e "s+\$BASE_PATH+$BASE/new/neutron-fwaas/.tox/dsvm-functional+" \
-    $FUNC_FILTER | sudo tee /etc/neutron/rootwrap.d/functional.filters > /dev/null
+case $VENV in
+    "dsvm-functional"|"dsvm-fullstack")
+    # The following need to be set before sourcing
+    # configure_for_fwaas_func_testing.
+    GATE_STACK_USER=stack
+    PROJECT_NAME=neutron-fwaas
+    IS_GATE=True
 
-# Use devstack functions to install mysql and psql servers
-TOP_DIR=$BASE/new/devstack
-source $TOP_DIR/functions
-source $TOP_DIR/inc/meta-config
-source $TOP_DIR/stackrc
-source $TOP_DIR/lib/database
+    source $FWAAS_PATH/tools/configure_for_fwaas_func_testing.sh
 
-# Install_databases [install_pg]
-# Tweak the script accordingly if we need psql in future
-function _install_databases {
-    local install_pg=${1:-True}
-
-    echo_summary "Installing databases"
-
-    # Avoid attempting to configure the db if it appears to already
-    # have run.  The setup as currently defined is not idempotent.
-    if mysql openstack_citest > /dev/null 2>&1 < /dev/null; then
-        echo_summary "DB config appears to be complete, skipping."
-        return 0
-    fi
-
-    enable_service mysql
-    initialize_database_backends
-    install_database
-    configure_database_mysql
-
-    if [[ "$install_pg" == "True" ]]; then
-        enable_service postgresql
-        initialize_database_backends
-        install_database
-        configure_database_postgresql
-    fi
-
-    # Set up the 'openstack_citest' user and database in each backend
-    tmp_dir=$(mktemp -d)
-    trap "rm -rf $tmp_dir" EXIT
-
-    cat << EOF > $tmp_dir/mysql.sql
-CREATE DATABASE openstack_citest;
-CREATE USER 'openstack_citest'@'localhost' IDENTIFIED BY 'openstack_citest';
-CREATE USER 'openstack_citest' IDENTIFIED BY 'openstack_citest';
-GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest'@'localhost';
-GRANT ALL PRIVILEGES ON *.* TO 'openstack_citest';
-FLUSH PRIVILEGES;
-EOF
-    /usr/bin/mysql -u root < $tmp_dir/mysql.sql
-
-    if [[ "$install_pg" == "True" ]]; then
-        cat << EOF > $tmp_dir/postgresql.sql
-CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest';
-CREATE DATABASE openstack_citest WITH OWNER openstack_citest;
-EOF
-
-        # User/group postgres needs to be given access to tmp_dir
-        setfacl -m g:postgres:rwx $tmp_dir
-        sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql
-    fi
-}
-
-if [[ "$IS_GATE" != "True" ]]; then
-    if [[ "$INSTALL_MYSQL_ONLY" == "True" ]]; then
-        _install_databases nopg
-    else
-        _install_databases
-    fi
-fi
+    configure_host_for_func_testing
+    ;;
+esac

neutron_fwaas/tests/contrib/post_test_hook.sh

@@ -2,7 +2,8 @@
 
 set -xe
 
-NEUTRON_DIR="$BASE/new/neutron-fwaas"
+FWAAS_DIR="$BASE/new/neutron-fwaas"
+NEUTRON_DIR="$BASE/new/neutron"
 TEMPEST_DIR="$BASE/new/tempest"
 SCRIPTS_DIR="/usr/os-testr-env/bin"
 
@@ -29,7 +30,8 @@ owner=stack
 prep_func="dsvm_functional_prep_func"
 
 # Set owner permissions according to job's requirements.
-cd $NEUTRON_DIR
+cd $FWAAS_DIR
+sudo chown -R $owner:stack $FWAAS_DIR
 sudo chown -R $owner:stack $NEUTRON_DIR
 # Prep the environment according to job's requirements.
 $prep_func

neutron_fwaas/tests/functional/privileged/test_dummy.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2017 Thales Services SAS
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from neutron.tests.functional import base
+
+from neutron_fwaas.privileged.tests.functional import dummy
+
+
+class DummyTest(base.BaseSudoTestCase):
+
+    def test_dummy(self):
+        dummy.dummy()

tools/configure_for_fwaas_func_testing.sh

@@ -0,0 +1,9 @@
+set -e
+
+
+IS_GATE=${IS_GATE:-False}
+USE_CONSTRAINT_ENV=${USE_CONSTRAINT_ENV:-False}
+PROJECT_NAME=${PROJECT_NAME:-neutron-fwaas}
+REPO_BASE=${GATE_DEST:-$(cd $(dirname "$BASH_SOURCE")/../.. && pwd)}
+
+source $REPO_BASE/neutron/tools/configure_for_func_testing.sh

tools/deploy_rootwrap.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+if [ "$#" -ne 3 ]; then
+  >&2 echo "Usage: $0 /path/to/neutron_fwaas /path/to/target/etc /path/to/target/bin
+Deploy Neutron FWaaS's rootwrap configuration.
+
+Warning: Any existing rootwrap files at the specified etc path will be
+removed by this script.
+
+Optional: set OS_SUDO_TESTING=1 to deploy the filters required by
+Neutron's functional testing suite."
+  exit 1
+fi
+
+OS_SUDO_TESTING=${OS_SUDO_TESTING:-0}
+
+neutron_path=${OS_NEUTRON_PATH}
+fwaas_path=$1
+target_etc_path=$2
+target_bin_path=$3
+
+src_conf_path=${neutron_path}/etc
+src_conf=${src_conf_path}/rootwrap.conf
+src_rootwrap_path=${src_conf_path}/neutron/rootwrap.d
+
+fwaas_src_conf_path=${fwaas_path}/etc
+fwaas_src_rootwrap_path=${fwaas_src_conf_path}/neutron/rootwrap.d
+
+dst_conf_path=${target_etc_path}/neutron
+dst_conf=${dst_conf_path}/rootwrap.conf
+dst_rootwrap_path=${dst_conf_path}/rootwrap.d
+
+if [[ -d "$dst_rootwrap_path" ]]; then
+    rm -rf ${dst_rootwrap_path}
+fi
+mkdir -p -m 755 ${dst_rootwrap_path}
+
+cp -p ${src_rootwrap_path}/* ${fwaas_src_rootwrap_path}/* ${dst_rootwrap_path}/
+cp -p ${src_conf} ${dst_conf}
+sed -i "s:^filters_path=.*$:filters_path=${dst_rootwrap_path}:" ${dst_conf}
+sed -i "s:^\(exec_dirs=.*\)$:\1,${target_bin_path}:" ${dst_conf}
+
+if [[ "$OS_SUDO_TESTING" = "1" ]]; then
+    sed -i 's/use_syslog=False/use_syslog=True/g' ${dst_conf}
+    sed -i 's/syslog_log_level=ERROR/syslog_log_level=DEBUG/g' ${dst_conf}
+    cp -p ${fwaas_path}/neutron_fwaas/tests/contrib/functional-testing.filters \
+        ${dst_rootwrap_path}/
+fi

tox.ini

@@ -49,11 +49,13 @@ setenv =
   OS_ROOTWRAP_CMD=sudo {envdir}/bin/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
   OS_ROOTWRAP_DAEMON_CMD=sudo {envdir}/bin/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
   OS_FAIL_ON_MISSING_DEPS=1
+  OS_NEUTRON_PATH={env:OS_NEUTRON_PATH:/opt/stack/new/neutron}
 whitelist_externals =
   sh
   cp
   sudo
 commands =
+  {toxinidir}/tools/deploy_rootwrap.sh {toxinidir} {envdir}/etc {envdir}/bin
   python setup.py testr --slowest --testr-args='{posargs}'
 
 [testenv:releasenotes]