From d162040d76a2048de0ecc7deb8999c16e0f97d06 Mon Sep 17 00:00:00 2001
From: German Eichberger <German.eichberger@rackspace.com>
Date: Wed, 21 Mar 2018 15:20:55 -0700
Subject: [PATCH] Downloads amphora images from artifact storage

This will download a test amphora image per default form the
Open Stack artifact storage to speed up tests. Operators can
configure their won artifact storage to simplify deploys.

(cherry picked from commit 95eee6bc11e97105cb0356d7475bee699d404bda0)

Change-Id: I408d4128d35aab889dbe89fd9497d83a7830129b
---
 defaults/main.yml                             |  17 ++-
 doc/source/configure-octavia.rst              |  14 +++
 .../image_download-754d13e7df9b7891.yaml      |  16 +++
 tasks/main.yml                                |   4 +
 tasks/octavia_amp_image.yml                   | 105 ++++++++++++++++++
 tasks/octavia_post_install.yml                |  48 --------
 tests/os_octavia-overrides.yml                |   1 +
 tests/test-configure-octavia.yml              |  30 -----
 tests/test-octavia.yml                        |  11 --
 9 files changed, 153 insertions(+), 93 deletions(-)
 create mode 100644 releasenotes/notes/image_download-754d13e7df9b7891.yaml
 create mode 100644 tasks/octavia_amp_image.yml

diff --git a/defaults/main.yml b/defaults/main.yml
index 859ac577..37ffe621 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -238,12 +238,21 @@ octavia_glance_image_tag: octavia-amphora-image
 octavia_amp_image_owner_id:
 # add here the glance image id if tagging is not used (not recommended for prod)
 octavia_amp_image_id:
-# add here the file name of the image if it should be uplaoded automatically
-# Note: This does not support image tags so don't use for prod
+# download the image from an artefact server
+# Note: The default is the Octavia test image so don't use that in prod
+octavia_download_artefact: True
+# The host to download images to if enabled
+# Options are ['deployment-host', 'target-host']
+octavia_image_downloader: "deployment-host"
+# The URL to downlaod from
+octavia_artefact_url: http://tarballs.openstack.org/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-xenial.qcow2
+# the directory to store the downloaded file to
+octavia_amp_image_path: "~/"
+# add here the file name of the image if it should be uploaded automatically
 octavia_amp_image_file_name:
 # enable uploading image to glance automatically
-# Note: This does not support image tags so don't use for prod
-octavia_amp_image_upload_enabled: False
+octavia_amp_image_upload_enabled: "{{ octavia_download_artefact }}"
+
 # Name of the Octavia security group
 octavia_security_group_name: octavia_sec_grp
 # Restrict access to only authorized hosts
diff --git a/doc/source/configure-octavia.rst b/doc/source/configure-octavia.rst
index 3cef99b3..4447bf63 100644
--- a/doc/source/configure-octavia.rst
+++ b/doc/source/configure-octavia.rst
@@ -76,6 +76,16 @@ ips which overlap with ips assigned to hosts or containers (see the
 Building Octavia images
 ~~~~~~~~~~~~~~~~~~~~~~~
 
+.. note::
+    The default behavior is to download a test image from the OpenStack artifact
+    storage the Octavia team provides daily. Because this image doesn't apply
+    operating system security patches in a timely manner it is unsuited
+    for production use.
+
+    Some Operating System vendors might provide official amphora builds or an
+    organization might maintain their own artifact storage - for those cases the
+    automatic download can be leveraged, too.
+
 Images using the ``diskimage-builder`` must be built outside of a container.
 For this process, use one of the physical hosts within the environment.
 
@@ -115,6 +125,10 @@ For this process, use one of the physical hosts within the environment.
       glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
          --container-format bare --tags octavia-amphora-image </var/lib/octavia/amphora-x64-haproxy.qcow2
 
+   .. note::
+        Alternatively you can specify the new image in the appropriate settings and rerun the
+        ansible with an appropriate tag.
+
 You can find more information abpout the diskimage script and the process at
 https://github.com/openstack/octavia/tree/master/diskimage-create
 
diff --git a/releasenotes/notes/image_download-754d13e7df9b7891.yaml b/releasenotes/notes/image_download-754d13e7df9b7891.yaml
new file mode 100644
index 00000000..c034e504
--- /dev/null
+++ b/releasenotes/notes/image_download-754d13e7df9b7891.yaml
@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    This consolidates the amphora image tasks in a common file and adds a way
+    to download an amphora image from an artefact storage over http(s). With
+    the Octavia team providing test images the tests were modified to not
+    build images any longer but download them.
+security:
+  - |
+    It is commonly considered bad practice to downlaod random images from the
+    Internet expecially the test images the Octavia team provides which could
+    potentially include unpatched operating system packages - so for any
+    production deploy adjust the download url to an artifact storage your
+    organization controls. The system also does not authenticate the image
+    (e.g. with an md5) so should only be used on networks your organization
+    controls.
diff --git a/tasks/main.yml b/tasks/main.yml
index 2b73c0fc..1beb30c3 100644
--- a/tasks/main.yml
+++ b/tasks/main.yml
@@ -87,6 +87,10 @@
     - octavia-install
     - octavia-config
 
+- include: octavia_amp_image.yml
+  tags:
+    - octavia-config
+
 
 - name: Flush handlers
   meta: flush_handlers
diff --git a/tasks/octavia_amp_image.yml b/tasks/octavia_amp_image.yml
new file mode 100644
index 00000000..3cd451ad
--- /dev/null
+++ b/tasks/octavia_amp_image.yml
@@ -0,0 +1,105 @@
+---
+  # Copyright 2018, Rackspace US, Inc.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+
+- name: Download image from artefact server
+  get_url:
+    url: "{{ octavia_artefact_url }}"
+    dest: "{{ octavia_amp_image_path }}"
+  retries: 10
+  delay: 10
+  register: octavia_download_result
+  when:
+    - octavia_download_artefact|bool
+  delegate_to: "{{ (octavia_image_downloader == 'deployment-host') | ternary('localhost', inventory_hostname) }}"
+
+- name: Set the filename fact
+  set_fact:
+    octavia_amp_image_file_name: "{{ octavia_download_result.dest }}"
+  when:
+    - octavia_image_downloader == "deployment-host"
+    - octavia_download_artefact|bool
+
+- name: Copy download images from deployment-host to target-host(s)
+  copy:
+    src: "{{ octavia_amp_image_file_name }}"
+    dest: "~/{{ octavia_amp_image_file_name|basename }}"
+  when:
+    - octavia_amp_image_upload_enabled
+    - octavia_image_downloader == "deployment-host"
+  register: octavia_amp_image_copy_result
+  until: octavia_amp_image_copy_result | success
+  retries: 6
+  delay: 5
+
+- name: Set if we need to upload an image
+  set_fact:
+    octavia_amp_image_needs_upload: "{{ (octavia_image_downloader != 'deployment-host')|ternary(octavia_download_result|changed, octavia_amp_image_copy_result|changed) }}"
+    octavia_dst_amp_image_path: "{{ ((octavia_image_downloader == 'deployment-host') and (octavia_download_artefact|bool))|ternary(octavia_amp_image_copy_result.dest, octavia_download_result.dest) }}"
+  when:
+    - octavia_amp_image_upload_enabled
+
+- name: Get curremt image id
+  os_image_facts:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    image: amphora-x64-haproxy
+  when:
+     - octavia_amp_image_needs_upload|default(False)
+
+# use shell since os_image doesn't support tags
+- name: Upload new image to glance
+  shell: |
+    . {{ ansible_env.HOME }}/openrc
+    openstack image create --file  {{ octavia_dst_amp_image_path }} --disk-format qcow2 \
+    --tag {{ octavia_glance_image_tag }}  --private --project service amphora-x64-haproxy
+  when:
+    - octavia_amp_image_needs_upload|default(False)
+  run_once: True
+  tags:
+    - skip_ansible_lint
+
+- name: Delete old image from glance
+  os_image:
+    auth:
+      auth_url: "{{ keystone_service_adminurl }}"
+      username: "{{ octavia_service_user_name }}"
+      password: "{{ octavia_service_password }}"
+      project_name: "{{ octavia_service_project_name }}"
+      user_domain_name: "{{ octavia_service_user_domain_id }}"
+      project_domain_name: "{{ octavia_service_project_domain_id }}"
+    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
+    region_name: "{{ octavia_service_region }}"
+    validate_certs: "{{ keystone_service_adminuri_insecure }}"
+    auth_type: "{{ octavia_keystone_auth_plugin }}"
+    id: "{{ openstack.id }}"
+    state: absent
+  when:
+   - openstack is defined # result from os_image_facts
+
+
+
+
+
+
+
diff --git a/tasks/octavia_post_install.yml b/tasks/octavia_post_install.yml
index 3e92c6af..a8190601 100644
--- a/tasks/octavia_post_install.yml
+++ b/tasks/octavia_post_install.yml
@@ -56,54 +56,6 @@
     - src: "{{ octavia_ca_private_key }}"
       dest: "/etc/octavia/certs/ca_key.pem"
 
-- name: Copy image
-  copy:
-    src: "{{ octavia_amp_image_file_name}}"
-    dest: "{{ octavia_amp_image_file_name}}"
-  when: octavia_amp_image_upload_enabled
-
-- name: Upload image to glance
-  os_image:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    name: amphora-x64-haproxy
-    container_format: bare
-    disk_format: qcow2
-    state: present
-    filename: "{{ octavia_amp_image_file_name}}"
-    is_public:
-  when: octavia_amp_image_upload_enabled
-
-- name: Get image uuid
-  os_image_facts:
-    auth:
-      auth_url: "{{ keystone_service_adminurl }}"
-      username: "{{ octavia_service_user_name }}"
-      password: "{{ octavia_service_password }}"
-      project_name: "{{ octavia_service_project_name }}"
-      user_domain_name: "{{ octavia_service_user_domain_id }}"
-      project_domain_name: "{{ octavia_service_project_domain_id }}"
-    endpoint_type: "{{ octavia_ansible_endpoint_type }}"
-    region_name: "{{ octavia_service_region }}"
-    validate_certs: "{{ keystone_service_adminuri_insecure }}"
-    auth_type: "{{ octavia_keystone_auth_plugin }}"
-    image: amphora-x64-haproxy
-  when: octavia_amp_image_upload_enabled
-
-- name: Set image UUID fact
-  set_fact:
-     octavia_amp_image_id: "{{ openstack_image.id }}"
-  when: octavia_amp_image_upload_enabled
-
 - name: Copy user provided HAProxy templates
   copy:
     src: "{{ item.src }}"
diff --git a/tests/os_octavia-overrides.yml b/tests/os_octavia-overrides.yml
index 9d27b60e..334b1f26 100644
--- a/tests/os_octavia-overrides.yml
+++ b/tests/os_octavia-overrides.yml
@@ -15,6 +15,7 @@
 ## Octavia rewuires currently nested virtualization (VT-X) which only one jenkins
 ## gate cloud provides reliabley. To cut down on gate errors this will
 ## disbale the tests which spin up an amphora
+## As of 4/6/18 zuul runs out of memory testing with this enabled
 test_octavia_amphora: False
 
 test_octavia_api_group: "{{ ((groups['octavia_api'] is defined) and (groups['octavia_api'] | length > 0)) | ternary('octavia_api', 'all_containers') }}"
diff --git a/tests/test-configure-octavia.yml b/tests/test-configure-octavia.yml
index 1fc1e79b..889e7667 100644
--- a/tests/test-configure-octavia.yml
+++ b/tests/test-configure-octavia.yml
@@ -28,42 +28,12 @@
       retries: 5
       delay: 2
       with_items:
-        - qemu
-        - uuid-runtime
-        - curl
-        - kpartx
         - git
-    - name: Install pip requirements
-      pip:
-        name: "{{ item }}"
-        state: "{{ octavia_pip_package_state }}"
-      register: install_packages
-      until: install_packages|success
-      retries: 5
-      delay: 2
-      with_items:
-        - argparse
-        - "Babel>=1.3"
-        - dib-utils
-        - PyYAML
     - name: Clone Octavia
       git:
         repo: "https://git.openstack.org/openstack/octavia"
         dest: "{{ octavia_system_home_folder }}/octavia"
         version: "{{ octavia_git_install_branch }}"
-    - name: Clone Diskimage-Builder
-      git:
-        repo: "https://git.openstack.org/openstack/diskimage-builder"
-        dest: "{{ octavia_system_home_folder }}/diskimage-builder"
-        version: "2.9.0"
-    - name: Create amphora image
-      shell: "./diskimage-create.sh -o {{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      args:
-        chdir: "{{ octavia_system_home_folder }}/octavia/diskimage-create"
-        creates: "{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Change permission
       file:
         path: "{{ octavia_system_home_folder }}/octavia/bin/create_certificates.sh"
diff --git a/tests/test-octavia.yml b/tests/test-octavia.yml
index 7d0cd36d..5f69e6b4 100644
--- a/tests/test-octavia.yml
+++ b/tests/test-octavia.yml
@@ -73,17 +73,6 @@
         network_name: "public"
         name: "public-subnet"
         cidr: "10.1.3.0/24"
-    - name: Upload image to glance
-      shell: >-
-          glance image-create --name amphora-x64-haproxy --visibility private --disk-format qcow2 \
-            --container-format bare --tags octavia-amphora-image <{{ octavia_system_home_folder }}/amphora-x64-haproxy.qcow2 \
-            && touch {{ octavia_system_home_folder }}/image
-      args:
-        creates: "{{ octavia_system_home_folder }}/image"
-      environment: "{{ env }}"
-      when: test_octavia_amphora | bool
-      tags:
-        - skip_ansible_lint
     - name: Create ssh-key
       shell: >
           cat /dev/zero | ssh-keygen -q -N ""