Merge "Implement shippable venvs"

defaults/main.yml

@@ -30,3 +30,13 @@ repo_build_pip_default_index: "https://pypi.python.org/simple"
 repo_build_pip_extra_index: "https://pypi.python.org/simple"
 
 repo_build_timeout: 120
+
+repo_build_venv_force_rebuild: false
+repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
+repo_build_venv_dir: "/var/www/repo/venvs"
+repo_build_venv_pip_install_options: >
+  --timeout 120
+  --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
+  --no-index
+  --verbose
+  --log /var/log/repo/repo_venv_builder.log

tasks/main.yml

@@ -13,8 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Wheel building
 - include: repo_clone_git.yml
 - include: repo_set_facts.yml
 - include: repo_pre_build.yml
 - include: repo_build.yml
 - include: repo_post_build.yml
+
+# Venv building
+- include: repo_venv.yml
+  tags:
+    - repo-build-venvs

tasks/repo_venv.yml

@@ -0,0 +1,18 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- include: repo_venv_pre_build.yml
+- include: repo_venv_build.yml
+- include: repo_venv_post_build.yml

tasks/repo_venv_build.yml

@@ -0,0 +1,81 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get venv command path
+  command: which virtualenv
+  register: virtualenv_path
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Set virtualenv command path
+  set_fact:
+    virtualenv_bin: "{{ virtualenv_path.stdout }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Check for created venvs
+  command: >
+    ls -1 "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/"
+  register: created_venvs
+  tags:
+    - repo-create-venv
+
+- name: Set existing venv fact
+  set_fact:
+    existing_venvs: "{{ created_venvs.stdout_lines }}"
+  tags:
+    - repo-command-bin
+    - repo-create-venv
+
+- name: Create role based venv
+  pip:
+    name: "{{ item.value | join(' ') }}"
+    state: present
+    virtualenv: "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}"
+    virtualenv_site_packages: "no"
+    virtualenv_command: "{{ virtualenv_bin }} --always-copy"
+    extra_args: "{{ repo_build_venv_pip_install_options }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+    - "'{{ item.key | replace('os_', '') }}.tgz' not in existing_venvs"
+  tags:
+    - repo-create-venv
+
+- name: Create venv archive
+  shell: |
+    # This is to clean up pyc files which makes the archived venv smaller
+    #  TODO(cloudnull) This should use the find module when we move to Ansible 2.0
+    find "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" -name '*.pyc' -delete
+    # Create archive
+    tar -czf "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" \
+        -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" .
+  args:
+    chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}.tgz"
+  with_dict: local_packages.results.0.item.role_packages
+  when:
+    - '"os_" in item.key'
+  tags:
+    - skip_ansible_lint
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+    - repo-create-venv

tasks/repo_venv_post_build.yml

@@ -0,0 +1,23 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Cleanup venv directory
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-cleanup-venv-location

tasks/repo_venv_pre_build.yml

@@ -0,0 +1,49 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Make sure old venv build directories are clean
+  file:
+    path: "{{ item }}"
+    state: "absent"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Destroy base venvs to rebuild them
+  file:
+    path: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+    state: "absent"
+  when: repo_build_venv_force_rebuild | bool
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive
+
+- name: Create venv directory
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    owner: "{{ repo_build_service_user_name }}"
+    mode: "2755"
+  with_items:
+    - "{{ repo_build_venv_build_dir }}/venvs"
+    - "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
+  tags:
+    - repo-create-venv-location
+    - repo-venv-compress-archive
+    - repo-create-venv-archive