From de1b45553e856ac618562476095e4023c8f627be Mon Sep 17 00:00:00 2001 From: Major Hayden Date: Mon, 14 Aug 2017 10:08:29 -0500 Subject: [PATCH] Download LXC image with async via aria2 This patch adds an async task to download the LXC image using aria2 with retries and read timeouts. Closes-Bug: 1709329 Change-Id: Ib9ec6195dcb7e0e4b18b8526f030e6738f9953e8 Signed-off-by: Kevin Carter --- defaults/main.yml | 28 +++---- ...xc_cache_environment-f14701a7f8f4b8ca.yaml | 5 -- ...c_image_cache_server-f14701a7f8f4b8ca.yaml | 14 ++++ tasks/lxc_cache.yml | 8 -- tasks/lxc_cache_create.yml | 31 +++++--- tasks/lxc_cache_preparation.yml | 26 ++----- tasks/lxc_cache_preparation_systemd_new.yml | 13 +++- tasks/lxc_cache_preparation_systemd_old.yml | 26 +++---- tasks/lxc_cache_prestage.yml | 78 +++++++++++++++++++ tasks/main.yml | 4 + templates/systemd-proxy-unit.conf.j2 | 2 +- tests/test.yml | 2 +- vars/redhat-7.yml | 1 + vars/suse-42.yml | 1 + vars/ubuntu-16.04.yml | 1 + 15 files changed, 165 insertions(+), 75 deletions(-) delete mode 100644 releasenotes/notes/lxc_cache_environment-f14701a7f8f4b8ca.yaml create mode 100644 releasenotes/notes/lxc_image_cache_server-f14701a7f8f4b8ca.yaml create mode 100644 tasks/lxc_cache_prestage.yml diff --git a/defaults/main.yml b/defaults/main.yml index ba4127d6..19ff6f21 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Environment to use when building LXC caches -# You can set http_proxy environment variables here for example -# if you would like to proxy all image downloads -lxc_cache_environment: {} +# Validate certificates when downloading LXC templates +lxc_hosts_validate_certs: yes # Set the package install state for distribution and pip packages # Options are 'present' and 'latest' @@ -126,21 +124,25 @@ lxc_cache_prep_dns: lxc_cache_prep_pre_commands: '## pre command skipped ##' lxc_cache_prep_post_commands: '## post command skipped ##' +# Set the servers to download LXC images from +# NOTE(mhayden): The main images.linuxcontainers.org site will redirect +# requests to (us|uk).images.linuxcontainers.org upon the first request. We +# add the mirrors here to get around some HTTP 400 errors and allow aria2 to +# download from both mirrors at the same time. +lxc_image_cache_server_mirrors: + - https://us.images.linuxcontainers.org + - https://uk.images.linuxcontainers.org + # The DNS name of the LXD server to source the base container cache from -lxc_image_cache_server: images.linuxcontainers.org - -# The protocol to use to fetch from lxc_image_cache_server -lxc_image_cache_server_proto: http - -# Path to images on server -lxc_image_cache_server_path: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/images/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}/" +# NOTE(cloudnull): This var should be removed in R. +lxc_image_cache_server: "{{ lxc_image_cache_server_mirrors[0].strip('http(?s)://') }}" # Local path to cached image lxc_image_cache_path: "/var/lib/machines/{{ lxc_container_base_name }}" # Mode to pull image. This is used to pull the image from a remote source. -# Valid options are [pull-tar, pull-raw] -lxc_image_cache_pull_mode: pull-tar +# Valid options are [import-tar, import-raw] +lxc_image_cache_pull_mode: import-tar # Set this option to true to pull a new cached image. lxc_image_cache_refresh: false diff --git a/releasenotes/notes/lxc_cache_environment-f14701a7f8f4b8ca.yaml b/releasenotes/notes/lxc_cache_environment-f14701a7f8f4b8ca.yaml deleted file mode 100644 index f331d00b..00000000 --- a/releasenotes/notes/lxc_cache_environment-f14701a7f8f4b8ca.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -features: - - The variable ``lxc_cache_environment`` has been added. This dictionary - can be overridden by deployers to set HTTP proxy environment variables that - will be applied to all lxc container download tasks. diff --git a/releasenotes/notes/lxc_image_cache_server-f14701a7f8f4b8ca.yaml b/releasenotes/notes/lxc_image_cache_server-f14701a7f8f4b8ca.yaml new file mode 100644 index 00000000..8299cb83 --- /dev/null +++ b/releasenotes/notes/lxc_image_cache_server-f14701a7f8f4b8ca.yaml @@ -0,0 +1,14 @@ +--- +features: + - The variable ``lxc_image_cache_server_mirrors`` has been added to + the "lxc_hosts" role. This is a list type variable and gives + deployers the ability to specify multiple lxc-image mirrors at the + same time. + +deprecations: + - The variable ``lxc_image_cache_server`` has been deprecated in the + "lxc_hosts" role. By default this value will pull the first item + out of ``lxc_image_cache_server_mirrors`` list which is only done + for compatibility (legacy) purposes. The default string type + variable, ``lxc_image_cache_server``, will be removed from the + "lxc_hosts" role in the in "R" release. diff --git a/tasks/lxc_cache.yml b/tasks/lxc_cache.yml index e0d260f0..b0dab0c1 100644 --- a/tasks/lxc_cache.yml +++ b/tasks/lxc_cache.yml @@ -13,14 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Set LXC cache path fact - set_fact: - cache_path_fact: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}" - cache_index_item: "{{ lxc_cache_map.distro }};{{ lxc_cache_map.release }};{{ lxc_cache_map.arch }};{{ lxc_cache_default_variant }}" - cache_time: "{{ ansible_date_time.epoch }}" - tags: - - always - - name : Check cached image status command: "machinectl image-status {{ lxc_container_base_name }}" register: cache_check diff --git a/tasks/lxc_cache_create.yml b/tasks/lxc_cache_create.yml index 64680c1e..0d20fd13 100644 --- a/tasks/lxc_cache_create.yml +++ b/tasks/lxc_cache_create.yml @@ -37,13 +37,23 @@ tags: - skip_ansible_lint -- name: Update LXC image meta data - get_url: - url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/{{ item.split(';')[-1] }}/meta.tar.xz" - dest: "/tmp/meta.tar.xz" - with_items: "{{ lxc_images }}" - retries: 5 - delay: 2 +- name: Fetch LXC image-metadata + shell: > + aria2c + --max-connection-per-server=4 + --continue + --allow-overwrite=true + --dir=/tmp + --out=meta.tar.xz + --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }} + {% for server in lxc_image_cache_server_mirrors %}{{ server }}/{{ item.split(';')[-1] }}/meta.tar.xz {% endfor %} + args: + warn: no + with_items: + - "{{ lxc_images }}" + tags: + - skip_ansible_lint + - always - name: Place container metadata unarchive: @@ -51,6 +61,11 @@ dest: "{{ cache_path_fact }}" remote_src: True +- name: Remove metadata archive + file: + path: "/tmp/meta.tar.xz" + state: "absent" + - name: Set cache expiry shell: "date -d @{{ (cache_time | int) + 31536000 }} > {{ cache_path_fact }}/expiry" tags: @@ -72,7 +87,6 @@ retries: 3 delay: 10 until: cache_download|success - environment: "{{ lxc_cache_environment }}" when: - lxc_container_backing_store is defined - lxc_container_backing_store == 'overlayfs' @@ -88,7 +102,6 @@ retries: 3 delay: 10 until: cache_download|success - environment: "{{ lxc_cache_environment }}" when: - lxc_container_backing_store is defined - lxc_container_backing_store == 'lvm' diff --git a/tasks/lxc_cache_preparation.yml b/tasks/lxc_cache_preparation.yml index 41782be2..35b66bf7 100644 --- a/tasks/lxc_cache_preparation.yml +++ b/tasks/lxc_cache_preparation.yml @@ -18,25 +18,6 @@ changed_when: false register: systemd_version -- name: Retrieve Image Index - uri: - url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/meta/1.0/index-system" - return_content: yes - register: image_index - retries: 5 - delay: 2 - -- name: Set image index fact - set_fact: - lxc_images: > - {%- set images = [] %} - {%- for image in image_index.content.splitlines() %} - {%- if image | match("^" + cache_index_item) %} - {%- set _ = images.append(image) %} - {%- endif %} - {%- endfor %} - {{- images -}} - - block: - name: Create machined proxy override unit directories file: @@ -59,10 +40,15 @@ with_items: - systemd-machined.service.d - systemd-importd.service.d - when: lxc_cache_environment.keys() | length > 0 + when: (deployment_environment_variables | default({})).keys() | length > 0 - include: "lxc_cache_preparation_systemd_{{ (systemd_version.stdout_lines[0].split()[-1] | int > 219) | ternary('new', 'old') }}.yml" +- name: Remove rootfs archive + file: + path: "/tmp/rootfs.tar.xz" + state: "absent" + - name: Generate apt keys from LXC host for the container cache shell: apt-key exportall > /root/repo.keys changed_when: False diff --git a/tasks/lxc_cache_preparation_systemd_new.yml b/tasks/lxc_cache_preparation_systemd_new.yml index 03522554..df3f8bac 100644 --- a/tasks/lxc_cache_preparation_systemd_new.yml +++ b/tasks/lxc_cache_preparation_systemd_new.yml @@ -25,18 +25,27 @@ # NOTE(cloudnull): When modern SystemD is running everywhere this can be # collapsed back into the base preparation task file. -- name : Set volume size +- name: Set volume size shell: machinectl set-limit {{ lxc_host_machine_volume_size }}G changed_when: false args: executable: /bin/bash +- name: Ensure image has been pre-staged + async_status: + jid: "{{ item.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + with_items: + - "{{ prestage_image.results }}" + - name: Retrieve base image command: >- machinectl --verify=no {{ lxc_image_cache_pull_mode }} - {{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz + /tmp/rootfs.tar.xz {{ lxc_container_base_name }} register: pull_image until: pull_image | success diff --git a/tasks/lxc_cache_preparation_systemd_old.yml b/tasks/lxc_cache_preparation_systemd_old.yml index f989e822..3377c2f0 100644 --- a/tasks/lxc_cache_preparation_systemd_old.yml +++ b/tasks/lxc_cache_preparation_systemd_old.yml @@ -115,7 +115,7 @@ # Because of this post and it's related bug(s) this is adding the container # volumes the old way. The new way would simply be calling `machinectl`. # * https://www.mail-archive.com/systemd-devel@lists.freedesktop.org/msg28255.html -- name : Remove old image cache +- name: Remove old image cache command: "btrfs subvolume delete /var/lib/machines/{{ lxc_container_base_name }}" register: cache_refresh_del changed_when: cache_refresh_del.rc == 0 @@ -123,7 +123,7 @@ when: - lxc_image_cache_refresh | bool -- name : Add image cache +- name: Add image cache command: "btrfs subvolume create /var/lib/machines/{{ lxc_container_base_name }}" register: cache_refresh_add changed_when: cache_refresh_add.rc == 0 @@ -131,23 +131,17 @@ when: - lxc_image_cache_refresh | bool -- name: Retrieve base image - get_url: - url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz" - dest: "/tmp/rootfs.tar.xz" - register: pull_image - until: pull_image | success - retries: 5 - delay: 2 - with_items: "{{ lxc_images }}" +- name: Ensure image has been pre-staged + async_status: + jid: "{{ item.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + with_items: + - "{{ prestage_image.results }}" - name: Place container rootfs unarchive: src: "/tmp/rootfs.tar.xz" dest: "/var/lib/machines/{{ lxc_container_base_name }}" remote_src: True - -- name: Remove rootfs archive - file: - path: "/tmp/rootfs.tar.xz" - state: "absent" diff --git a/tasks/lxc_cache_prestage.yml b/tasks/lxc_cache_prestage.yml new file mode 100644 index 00000000..0390e6c5 --- /dev/null +++ b/tasks/lxc_cache_prestage.yml @@ -0,0 +1,78 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set LXC cache path fact + set_fact: + cache_path_fact: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}" + cache_index_item: "{{ lxc_cache_map.distro }};{{ lxc_cache_map.release }};{{ lxc_cache_map.arch }};{{ lxc_cache_default_variant }}" + cache_time: "{{ ansible_date_time.epoch }}" + tags: + - always + +- name: Fetch LXC image-index + shell: > + aria2c + --max-connection-per-server=4 + --continue + --allow-overwrite=true + --dir=/tmp + --out=index-system + --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }} + {% for server in lxc_image_cache_server_mirrors %}{{ server }}/meta/1.0/index-system {% endfor %} + args: + warn: no + tags: + - skip_ansible_lint + - always + +- name: Retrieve the expiry object + slurp: + src: "/tmp/index-system" + changed_when: false + register: image_index + tags: + - always + +- name: Set image index fact + set_fact: + lxc_images: > + {%- set images = [] %} + {%- for image in (image_index.content | b64decode).splitlines() %} + {%- if image | match("^" + cache_index_item) %} + {%- set _ = images.append(image) %} + {%- endif %} + {%- endfor %} + {{- images -}} + +- name: Pre-stage the LXC image on the system + shell: > + aria2c + --max-connection-per-server=4 + --continue + --allow-overwrite=true + --dir=/tmp + --out=rootfs.tar.xz + --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }} + {% for server in lxc_image_cache_server_mirrors %}{{ server }}{{ item.split(';')[-1] }}rootfs.tar.xz {% endfor %} + args: + warn: no + register: prestage_image + async: 300 + poll: 0 + with_items: + - "{{ lxc_images }}" + tags: + - skip_ansible_lint + - always diff --git a/tasks/main.yml b/tasks/main.yml index b119f86b..f2d11576 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -50,6 +50,10 @@ tags: - lxc_hosts-install +- include: lxc_cache_prestage.yml + tags: + - lxc_hosts-config + - include: lxc_post_install.yml tags: - lxc_hosts-config diff --git a/templates/systemd-proxy-unit.conf.j2 b/templates/systemd-proxy-unit.conf.j2 index e3f76cf4..cd243e49 100644 --- a/templates/systemd-proxy-unit.conf.j2 +++ b/templates/systemd-proxy-unit.conf.j2 @@ -1,6 +1,6 @@ # {{ ansible_managed }} [Service] -{% for key, value in lxc_cache_environment.items() %} +{% for key, value in (deployment_environment_variables | default({})).items() %} Environment={{ key }}={{ value }} {% endfor %} diff --git a/tests/test.yml b/tests/test.yml index a261a4d9..f888e907 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -67,7 +67,7 @@ - skip_ansible_lint - name: Set a fact to override lxc_image_cache_server value when in nodepool set_fact: - lxc_image_cache_server: "{{ lxc_reverse_proxy.stdout }}" + lxc_image_cache_server_mirrors: ["http://{{ lxc_reverse_proxy.stdout.strip('/') }}"] when: - nodepool.stat.exists | bool diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index 7ef5eee1..606f9d2b 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -18,6 +18,7 @@ systemd_utils_prefix: "/lib/systemd" # Required rpm packages. lxc_hosts_distro_packages: + - aria2 - bridge-utils - btrfs-progs - dbus diff --git a/vars/suse-42.yml b/vars/suse-42.yml index d8b795b9..88f9096a 100644 --- a/vars/suse-42.yml +++ b/vars/suse-42.yml @@ -22,6 +22,7 @@ lxc_hosts_distro_packages: - apparmor-parser - apparmor-profiles - apparmor-utils + - aria2 - bridge-utils - btrfsprogs - dnsmasq diff --git a/vars/ubuntu-16.04.yml b/vars/ubuntu-16.04.yml index e3383ff3..26cd36bd 100644 --- a/vars/ubuntu-16.04.yml +++ b/vars/ubuntu-16.04.yml @@ -20,6 +20,7 @@ cache_timeout: 600 lxc_hosts_distro_packages: - apparmor - apparmor-utils + - aria2 - bridge-utils - btrfs-tools - cgmanager