Combine pip_install and pip_lockdown roles

Since lockdown is really just a form of specific pip
configuration and since the pip_install role already does
some pip configuration, it seems logical and desirable for
these functions to take place in the same role.

This change should pave the way for a simplification of
role dependencies and the removal of explicit pip_lockdown
role usage with various playbooks that will already have a
dependency on pip_install.

Change-Id: Ia0fc276c2b501f16d4acf73bbbcad6f80804628e

.gitignore

@@ -57,3 +57,6 @@ ChangeLog
 
 # Files created by releasenotes build
 releasenotes/build
+
+# Vagrant testing artifacts
+.vagrant

Vagrantfile

@@ -0,0 +1,9 @@
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo su -
+    cd /vagrant
+    apt-get update
+    ./run_tests.sh
+  SHELL
+end

defaults/main.yml

@@ -28,3 +28,22 @@ pip_packages: []
 # Validate Certificates when downloading pip. May be set to "no" when proxy server
 # is intercepting the certificates.
 pip_validate_certs: "yes"
+
+pip_lock_to_internal_repo: False
+
+# Options for pip global
+pip_enable_pre_releases: true
+pip_timeout: 120
+
+# Options for pip install
+pip_upgrade: true
+
+# Drop link files to lock down pip
+# Example:
+#   pip_links:
+#     - name: "openstack_release"
+#       link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/"
+pip_links: []
+
+## Tunable overrides
+pip_global_conf_overrides: {}

files/pip-link-build.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>
+
+"""
+This script will build a pip.conf file dynamically based on a simple
+configuration layout.  The purpose of this script is to allow automation to
+deploy parts of the main `pip.conf` file incrementally creating links and
+sections as needed.
+
+Structure:
+    $HOME/.pip/
+    $HOME/.pip/base
+    $HOME/.pip/links.d
+
+creates:
+    $HOME/.pip/pip.conf
+
+
+* The script reads all configuration files from the base directory and then
+  applies the sections to the main config file at "$HOME/.pip/pip.conf"
+* Within the [install] section will be generated with the value `find-links`
+  built from the link files found in "$HOME/.pip/links.d".
+"""
+
+import ConfigParser
+import os
+import urlparse
+
+
+def config_files(config_dir_path, extension='.link'):
+    """Discover all link files.
+
+    :param config_dir_path: ``str`` Path to link directory
+    :param extension: ``str`` Extension for files
+    :return: ``list``
+    """
+    link_files = list()
+    for root_path, _, pip_files in os.walk(config_dir_path):
+        for f in pip_files:
+            if f.endswith(extension):
+                link_files.append(os.path.join(root_path, f))
+    else:
+        return link_files
+
+
+def pip_links(links_files):
+    """Read all link files.
+
+    :param links_files: ``list`` List of files to read containing links
+    :return: `list``
+    """
+    links = list()
+    for link in links_files:
+        with open(link, 'rb') as f:
+            links.extend(f.readlines())
+    else:
+        return links
+
+
+def load_config(config_file):
+    """Load config from a file.
+
+    :param config_file: ``str``  path to config file
+    :return: ``object``
+    """
+    config = ConfigParser.ConfigParser()
+    if config_file is None:
+        return config
+
+    try:
+        with open(config_file) as f:
+            config.readfp(f)
+    except IOError:
+        return config
+    else:
+        return config
+
+
+def set_links(links):
+    """Set all links and ensure there are no blank lines.
+
+    :param links: ``list`` List of all raw links
+    :return: ``str``
+    """
+    pip_find_links = list()
+    for link in links:
+        if link != '\n' or not link:
+            pip_find_links.append(link.rstrip('\n'))
+
+    links = [i for i in list(set(pip_find_links))]
+    return '\n%s' % '\n'.join(links)
+
+
+def build_main_config(add_conf, main_config):
+    """Build configuration from all found conf files.
+
+    :param add_conf: ``object`` ConfigParser object
+    :param main_config: ``object`` ConfigParser object
+    """
+    for section in add_conf.sections():
+        try:
+            main_config.add_section(section)
+        except ConfigParser.DuplicateSectionError:
+            pass
+
+        for k, v in add_conf.items(section):
+            main_config.set(section, k, v)
+
+
+def build_install_section(main_dir_path, main_config):
+    """Build the install section with links.
+
+    :param main_dir_path: ``str`` Directory path
+    :param main_config: ``object`` ConfigParser object
+    """
+    links = list()
+    trusted_host = list()
+    links_dir = os.path.join(main_dir_path, 'links.d')
+    if os.path.isdir(links_dir):
+        _link = config_files(config_dir_path=links_dir, extension='.link')
+        _links = pip_links(_link)
+        links.extend(_links)
+        for _link in _links:
+            # Make sure that just the hostname/ip is used.
+            trusted_host.append(urlparse.urlparse(_link).netloc.split(':')[0])
+        else:
+            main_config.set('global', 'trusted-host', set_links(trusted_host))
+
+    # Add install section if not already found
+    try:
+        main_config.add_section('install')
+    except ConfigParser.DuplicateSectionError:
+        pass
+
+    # Get all items from the install section
+    try:
+        install_items = main_config.items('install')
+    except ConfigParser.NoSectionError:
+        install_items = None
+
+    link_strings = set_links(links)
+    if install_items:
+        for item in install_items:
+            if item[0] != 'find-links':
+                main_config.set('install', *item)
+
+    main_config.set('install', 'find-links', link_strings)
+
+
+def main(user_home=None):
+    """Run the main application."""
+    if not user_home:
+        user_home = '~/.pip/pip.conf'
+    main_file_path = os.path.expanduser(user_home)
+    main_config = load_config(config_file=None)
+
+    main_dir_path = os.path.dirname(main_file_path)
+    base_dir_path = os.path.join(main_dir_path, 'base')
+    if os.path.isdir(base_dir_path):
+        _confs = config_files(base_dir_path, extension='.conf')
+        for _conf in _confs:
+            _config = load_config(config_file=_conf)
+            build_main_config(_config, main_config)
+
+    build_install_section(main_dir_path, main_config)
+
+    # Write out the config file
+    with open(main_file_path, 'wb') as f:
+        main_config.write(f)
+
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) >= 1:
+        main(os.path.join(sys.argv[1], '.pip/pip.conf'))
+    else:
+        main()

releasenotes/notes/combine_pip_roles-ba524dbaa601e1a1.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - The pip_install role can now configure pip to be locked down to the
+    repository built by OpenStack-Ansible. To enable the lockdown
+    configuration, deployers may set ``pip_lock_to_internal_repo`` to
+    ``true`` in ``/etc/openstack_deploy/user_variables.yml``.

tasks/configure.yml

@@ -0,0 +1,49 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create pip config directory
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    group: "{{ ansible_user_id }}"
+    owner: "{{ ansible_user_id }}"
+  with_items:
+    - "/opt"
+    - "{{ ansible_env.HOME }}/.cache"
+    - "{{ ansible_env.HOME }}/.cache/pip"
+    - "{{ ansible_env.HOME }}/.pip"
+    - "{{ ansible_env.HOME }}/.pip/base"
+  tags:
+    - pip-directories
+
+- name: Drop pip file(s)
+  copy:
+    src: "selfcheck.json"
+    dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json"
+    owner: "{{ ansible_user_id }}"
+    group: "{{ ansible_user_id }}"
+    mode: "0644"
+  tags:
+    - pip-files
+
+- name: Drop pip global config(s)
+  config_template:
+    src: "global.conf.j2"
+    dest: "{{ ansible_env.HOME }}/.pip/base/global.conf"
+    owner: "{{ ansible_user_id }}"
+    group: "{{ ansible_user_id }}"
+    mode: "0644"
+    config_overrides: "{{ pip_global_conf_overrides }}"
+    config_type: "ini"

tasks/install.yml

@@ -0,0 +1,58 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Get Modern PIP
+  get_url:
+    url: "{{ pip_upstream_url }}"
+    dest: "/opt/get-pip.py"
+    force: "yes"
+    validate_certs: "{{ pip_validate_certs }}"
+  register: get_pip
+  until: get_pip | success
+  ignore_errors: True
+  retries: 5
+  delay: 2
+  tags:
+    - pip-install-script
+
+- name: Get Modern PIP using fallback URL
+  get_url:
+    url: "{{ pip_fallback_url }}"
+    dest: "/opt/get-pip.py"
+    force: "yes"
+    validate_certs: "{{ pip_validate_certs }}"
+  when: get_pip | failed
+  register: get_pip_fallback
+  until: get_pip_fallback | success
+  retries: 5
+  delay: 2
+  tags:
+    - pip-install-script
+
+- name: Install PIP
+  shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
+  ignore_errors: true
+  register: pip_install
+  until: pip_install | success
+  retries: 3
+  delay: 2
+
+- name: Install PIP (fall back mode)
+  shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
+  when: pip_install.rc != 0
+  register: pip_install_fall_back
+  until: pip_install_fall_back | success
+  retries: 3
+  delay: 2

tasks/lockdown.yml

@@ -0,0 +1,52 @@
+---
+# Copyright 2014, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create pip links directory
+  file:
+    path: "{{ item }}"
+    state: "directory"
+    group: "{{ ansible_user_id }}"
+    owner: "{{ ansible_user_id }}"
+  with_items:
+    - "{{ ansible_env.HOME }}/.pip/links.d"
+  tags:
+    - lock-directories-pip
+
+- name: Drop pip lockdown file(s)
+  copy:
+    src: "pip-link-build.py"
+    dest: "{{ ansible_env.HOME }}/.pip/pip-link-build.py"
+    owner: "{{ ansible_user_id }}"
+    group: "{{ ansible_user_id }}"
+    mode: "0755"
+  tags:
+    - lock-pip-files
+
+- name: Drop pip link file(s)
+  template:
+    src: "link_file.j2"
+    dest: "{{ ansible_env.HOME }}/.pip/links.d/{{ item.name }}.link"
+    owner: "{{ ansible_user_id }}"
+    group: "{{ ansible_user_id }}"
+    mode: "{{ item.mode|default('0644') }}"
+  with_items: pip_links
+  tags:
+    - lock-pip-files
+
+- name: Execute pip config builder
+  command: "{{ ansible_env.HOME }}/.pip/pip-link-build.py {{ ansible_env.HOME }}"
+  changed_when: false
+  tags:
+    - lock-down-pip-conf

tasks/main.yml

@@ -13,77 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Create pip config directory
-  file:
-    path: "{{ item }}"
-    state: "directory"
-    group: "{{ ansible_user_id }}"
-    owner: "{{ ansible_user_id }}"
-  with_items:
-    - "/opt"
-    - "{{ ansible_env.HOME }}/.cache"
-    - "{{ ansible_env.HOME }}/.cache/pip"
+- include: configure.yml
   tags:
-    - pip-directories
+    - pip-configuration
 
-- name: Drop pip file(s)
-  copy:
-    src: "{{ item.src }}"
-    dest: "{{ item.dest }}"
-    owner: "{{ ansible_user_id }}"
-    group: "{{ ansible_user_id }}"
-    mode: "{{ item.mode|default('0644') }}"
-  with_items:
-    - { src: "selfcheck.json", dest: "{{ ansible_env.HOME }}/.cache/pip/selfcheck.json" }
-  tags:
-    - pip-files
+- include: lockdown.yml
+  when:
+    - pip_lock_to_internal_repo | bool
 
-- name: Get Modern PIP
-  get_url:
-    url: "{{ pip_upstream_url }}"
-    dest: "/opt/get-pip.py"
-    force: "yes"
-    validate_certs: "{{ pip_validate_certs }}"
-  register: get_pip
-  until: get_pip | success
-  ignore_errors: True
-  retries: 5
-  delay: 2
-  tags:
-    - pip-install-script
-    - pip-install
-
-- name: Get Modern PIP using fallback URL
-  get_url:
-    url: "{{ pip_fallback_url }}"
-    dest: "/opt/get-pip.py"
-    force: "yes"
-    validate_certs: "{{ pip_validate_certs }}"
-  when: get_pip | failed
-  register: get_pip_fallback
-  until: get_pip_fallback | success
-  retries: 5
-  delay: 2
-  tags:
-    - pip-install-script
-    - pip-install
-
-- name: Install PIP
-  shell: "python /opt/get-pip.py {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
-  ignore_errors: true
-  register: pip_install
-  until: pip_install | success
-  retries: 3
-  delay: 2
-  tags:
-    - pip-install
-
-- name: Install PIP (fall back mode)
-  shell: "python /opt/get-pip.py --isolated {{ pip_get_pip_options }} {{ pip_packages | map('quote') | join (' ') }}"
-  when: pip_install.rc != 0
-  register: pip_install_fall_back
-  until: pip_install_fall_back | success
-  retries: 3
-  delay: 2
+- include: install.yml
   tags:
     - pip-install

templates/global.conf.j2

@@ -0,0 +1,11 @@
+# {{ ansible_managed }}
+
+[global]
+{% if pip_lock_to_internal_repo %}
+no-index = true
+{% endif %}
+pre = {{ pip_enable_pre_releases }}
+timeout = {{ pip_timeout }}
+
+[install]
+upgrade = {{ pip_upgrade }}

templates/link_file.j2

@@ -0,0 +1 @@
+{{ item.link }}