Implement shippable venvs

The change builds venvs in a single repo container and then
ships them to to all targets. The built venvs will be within
the repo servers and will allow for faster deployments,
upgrades, and more consistent deployments for the life cycle
of the deployment.

This will create a versioned tarball that will allow for
greater visablility into the build process as well as giving
deployers/developers the ability to compair a release in
place.

Change-Id: Ieef0b89ebc009d1453c99e19e53a36eb2d70edae
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

defaults/main.yml

@@ -30,3 +30,13 @@ repo_build_pip_default_index: "https://pypi.python.org/simple"
 repo_build_pip_extra_index: "https://pypi.python.org/simple"
 
 repo_build_timeout: 120
+
+repo_build_venv_force_rebuild: false
+repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
+repo_build_venv_dir: "/var/www/repo/venvs"
+repo_build_venv_pip_install_options: >
+  --timeout 120
+  --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --no-index
+  --verbose
+  --log /var/log/repo/repo_venv_builder.log

tasks/main.yml

@@ -13,8 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Wheel building
 - include: repo_clone_git.yml
 - include: repo_set_facts.yml
 - include: repo_pre_build.yml
 - include: repo_build.yml
 - include: repo_post_build.yml
+
+# Venv building
+- include: repo_venv.yml
+  tags:
+    - repo-build-venvs

tasks/repo_venv.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: repo_venv_pre_build.yml
+- include: repo_venv_build.yml
+- include: repo_venv_post_build.yml

tasks/repo_venv_build.yml

@@ -0,0 +1,81 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get venv command path
+  command: which virtualenv
+  register: virtualenv_path
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Set virtualenv command path
+  set_fact:
+    virtualenv_bin: "{{ virtualenv_path.stdout }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Check for created venvs
+  command: >
+    ls -1 "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/"
+  register: created_venvs
+  tags:
+    - repo-create-venv
+
+- name: Set existing venv fact
+  set_fact:
+    existing_venvs: "{{ created_venvs.stdout_lines }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Create role based venv
+  pip:
+    name: "{{ item.value | join(' ') }}"
+    state: present
+    virtualenv: "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}"
+    virtualenv_site_packages: "no"
+    virtualenv_command: "{{ virtualenv_bin }} --always-copy"
+    extra_args: "{{ repo_build_venv_pip_install_options }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+    - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs"
+  tags:
+    - repo-create-venv
+
+- name: Create venv archive
+  shell: |
+    # This is to clean up pyc files which makes the archived venv smaller
+    #  TODO(cloudnull) This should use the find module when we move to Ansible 2.0
+    find "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" -name '*.pyc' -delete
+    # Create archive
+    tar -czf "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" \
+        -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" .
+  args:
+    chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz"
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+  tags:
+    - skip_ansible_lint
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+    - repo-create-venv

tasks/repo_venv_post_build.yml

@@ -0,0 +1,23 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Cleanup venv directory
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-cleanup-venv-location

tasks/repo_venv_pre_build.yml

@@ -0,0 +1,49 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Make sure old venv build directories are clean
+  file:
+    path: "{{ item }}"
+    state: "absent"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Destroy base venvs to rebuild them
+  file:
+    path: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    state: "absent"
+  when: repo_build_venv_force_rebuild | bool
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Create venv directory
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    owner: "{{ repo_build_service_user_name }}"
+    mode: "2755"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}/venvs"
+    - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive