Add http proxy test scenario

This patch adds a test scenario which deploys a squid proxy on
the AIO host and uses deployment_environment_variables to direct
all http and https traffic through that proxy.

In order to make the test reasonably realistic eth0 is removed
from all containers which means they have no default route
and must retrieve all external dependencies via the proxy.

As eth0 is removed we can no longer use it to NAT the traffic to
the neutron public ip address range during tempest tests. Instead
we add static routes to those address ranges to eth1 in the
containers and expect the host to forward via br-mgmt.

Depends-On: https://review.openstack.org/625312
Depends-On: https://review.openstack.org/625670

Change-Id: I554cfd149374f0863ac058f26862b5439c5d53d8

etc/openstack_deploy/openstack_user_config.yml.aio.j2

@@ -45,6 +45,16 @@ global_overrides:
           - all_containers
           - hosts
         is_container_address: true
+        # define static routes to the neutron public IP ranges via br-mgmt
+        # this is AIO specific and relies on the host forwarding to reach instance
+        # floating ips using the br-mgmt interface as a gateway
+        static_routes:
+          # neutron public addresses, LXC
+          - cidr: 172.29.248.0/22
+            gateway: 172.29.236.100
+          # neutron public networks, nspawn
+          - cidr: 172.29.240.0/22
+            gateway: 172.29.236.100
     - network:
         container_bridge: "br-vxlan"
         container_type: "veth"

tests/roles/bootstrap-host/files/squid.conf

@@ -0,0 +1,20 @@
+acl SSL_ports port 443
+acl CONNECT method CONNECT
+acl lan src 172.29.236.0/22
+
+http_access deny CONNECT !SSL_ports
+http_access allow localhost manager
+http_access allow lan
+http_access deny manager
+http_access allow localhost
+http_access deny all
+
+http_port 3128
+
+coredump_dir /var/spool/squid
+
+refresh_pattern ^ftp:           1440    20%     10080
+refresh_pattern ^gopher:        1440    0%      1440
+refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
+refresh_pattern (Release|Packages(.gz)*)$      0       20%     2880
+refresh_pattern .               0       20%     4320

tests/roles/bootstrap-host/handlers/main.yml

@@ -0,0 +1,19 @@
+---
+# Copyright 2018, BBC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Restart squid
+  service:
+    name: squid
+    state: restarted

tests/roles/bootstrap-host/tasks/main.yml

@@ -134,6 +134,13 @@
   tags:
     - prepare-ssh-keys
 
+# Prepare local squid proxy
+- include: prepare_squid.yml
+  when:
+    - "bootstrap_host_scenario is search('proxy')"
+  tags:
+    - prepare-squid
+
 # Put the OpenStack-Ansible configuration for an All-In-One on the host
 - include: prepare_aio_config.yml
   when:

tests/roles/bootstrap-host/tasks/prepare_squid.yml

@@ -0,0 +1,28 @@
+---
+# Copyright 2018, BBC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install squid packages
+  package:
+    name: squid
+    state: present
+    update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
+  notify: Restart squid
+  tags:
+    - install-packages
+
+- name: Install squid config
+  copy:
+    src: "squid.conf"
+    dest: "/etc/squid/squid.conf"

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -206,3 +206,17 @@ openstack_user_kernel_options:
 neutron_lbaas_octavia: True
 octavia_management_net_subnet_cidr: "{{ (bootstrap_host_container_tech == 'nspawn') | ternary('172.29.240.0/22', '172.29.252.0/22') }}"
 {% endif %}
+
+{% if bootstrap_host_scenario is search('proxy') %}
+# For testing with the 'proxy' scenario configure deployment environment
+# to point to the local squid
+# Playbooks will set a runtime proxy to the AIO host squid
+deployment_environment_variables:
+  http_proxy: http://172.29.236.100:3128/
+  https_proxy: http://172.29.236.100:3128/
+  no_proxy: "localhost,127.0.0.1,172.29.236.100,{{ bootstrap_host_public_address | default(ansible_default_ipv4.address) }}"
+
+# Remove eth0 from all container so there is no default route and everything
+# must go via the http proxy
+lxc_container_networks: {}
+{% endif %}

tests/roles/bootstrap-host/vars/main.yml

@@ -16,7 +16,7 @@
 bootstrap_host_services: >-
   {%- set scenario_list = (bootstrap_host_scenario.split('_') | reject('equalto', '')) | list %}
   {%- set service_list = ['keystone'] %}
-  {%- set service_list_extra = scenario_list | difference(['aio', 'distro', 'lxc', 'nspawn', 'metal', 'source', 'translations']) %}
+  {%- set service_list_extra = scenario_list | difference(['aio', 'distro', 'lxc', 'nspawn', 'metal', 'source', 'translations', 'proxy']) %}
   {%- if 'metal' not in scenario_list %}
   {%-   set _ = service_list.append('haproxy') %}
   {%- endif %}

zuul.d/jobs.yaml

@@ -94,6 +94,14 @@
       action: deploy
       scenario: aio_ceph
 
+- job:
+    name: openstack-ansible-deploy-aio_proxy-ubuntu-bionic
+    parent: openstack-ansible-deploy-aio
+    nodeset: ubuntu-bionic
+    vars:
+      action: deploy
+      scenario: aio_proxy
+
 - job:
     name: openstack-ansible-deploy-aio_distro_ceph-ubuntu-bionic
     parent: openstack-ansible-deploy-aio

zuul.d/project-templates.yaml

@@ -114,6 +114,15 @@
         - openstack-ansible-deploy-aio_ceph-ubuntu-bionic
         - openstack-ansible-upgrade-aio_ceph-ubuntu-bionic
 
+- project-template:
+    name: openstack-ansible-deploy-proxy-jobs
+    check:
+      jobs:
+        - openstack-ansible-deploy-aio_proxy-ubuntu-bionic
+    gate:
+      jobs:
+        - openstack-ansible-deploy-aio_proxy-ubuntu-bionic
+
 - project-template:
     name: openstack-ansible-deploy-distro_ceph-jobs
     check:

zuul.d/project.yaml

@@ -20,5 +20,6 @@
       - openstack-ansible-deploy-aio_distro_lxc-jobs
       - openstack-ansible-deploy-aio_metal-jobs
       - openstack-ansible-deploy-ceph-jobs
+      - openstack-ansible-deploy-proxy-jobs
       - openstack-ansible-deploy-distro_ceph-jobs
       - publish-openstack-docs-pti