From 3ecfa1968d0c0eb5f35fbe85e846e840b672be90 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 23 Jan 2019 09:27:07 -0800 Subject: [PATCH] Initial pass at global opendev base job set This adds needed roles, playbooks and secrets for our global base jobs. Change-Id: I466bc1b8b33ea806f0ec39aa9aca32b91e28e7f1 --- LICENSE | 202 ++++++++++++++++++ playbooks/base-test/post-logs.yaml | 19 ++ playbooks/base-test/post.yaml | 11 + playbooks/base-test/pre.yaml | 45 ++++ playbooks/base/post-logs.yaml | 21 ++ playbooks/base/post.yaml | 11 + playbooks/base/pre.yaml | 40 ++++ roles/configure-unbound/README.rst | 48 +++++ roles/configure-unbound/defaults/main.yaml | 24 +++ roles/configure-unbound/handlers/main.yaml | 5 + roles/configure-unbound/tasks/main.yaml | 84 ++++++++ .../templates/forwarding.conf.j2 | 6 + roles/configure-unbound/templates/ttl.conf.j2 | 5 + roles/configure-unbound/vars/Debian.yaml | 1 + roles/configure-unbound/vars/default.yaml | 1 + roles/mirror-info/README.rst | 7 + roles/mirror-info/tasks/main.yaml | 17 ++ roles/mirror-info/templates/mirror_info.sh.j2 | 74 +++++++ zuul.yaml | 144 +++++++++++++ 19 files changed, 765 insertions(+) create mode 100644 LICENSE create mode 100644 playbooks/base-test/post-logs.yaml create mode 100644 playbooks/base-test/post.yaml create mode 100644 playbooks/base-test/pre.yaml create mode 100644 playbooks/base/post-logs.yaml create mode 100644 playbooks/base/post.yaml create mode 100644 playbooks/base/pre.yaml create mode 100644 roles/configure-unbound/README.rst create mode 100644 roles/configure-unbound/defaults/main.yaml create mode 100644 roles/configure-unbound/handlers/main.yaml create mode 100644 roles/configure-unbound/tasks/main.yaml create mode 100644 roles/configure-unbound/templates/forwarding.conf.j2 create mode 100644 roles/configure-unbound/templates/ttl.conf.j2 create mode 100644 roles/configure-unbound/vars/Debian.yaml create mode 100644 roles/configure-unbound/vars/default.yaml create mode 100644 roles/mirror-info/README.rst create mode 100644 roles/mirror-info/tasks/main.yaml create mode 100644 roles/mirror-info/templates/mirror_info.sh.j2 create mode 100644 zuul.yaml diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..75b5248 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/playbooks/base-test/post-logs.yaml b/playbooks/base-test/post-logs.yaml new file mode 100644 index 0000000..ad044d7 --- /dev/null +++ b/playbooks/base-test/post-logs.yaml @@ -0,0 +1,19 @@ +- hosts: localhost + roles: + - role: add-fileserver + fileserver: "{{ site_logs }}" + - role: ara-report + +- hosts: "{{ site_logs.fqdn }}" + gather_facts: False + roles: + - role: test-upload-logs + zuul_log_url: "http://logs.openstack.org" + +- hosts: localhost + # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail + # successful jobs. + ignore_errors: yes + roles: + - submit-logstash-jobs + - submit-subunit-jobs diff --git a/playbooks/base-test/post.yaml b/playbooks/base-test/post.yaml new file mode 100644 index 0000000..c4a139d --- /dev/null +++ b/playbooks/base-test/post.yaml @@ -0,0 +1,11 @@ +- hosts: all + roles: + - fetch-output + - merge-output-to-logs + +- hosts: all + # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail + # successful jobs. + ignore_errors: yes + roles: + - remove-build-sshkey diff --git a/playbooks/base-test/pre.yaml b/playbooks/base-test/pre.yaml new file mode 100644 index 0000000..c14dd19 --- /dev/null +++ b/playbooks/base-test/pre.yaml @@ -0,0 +1,45 @@ +- hosts: localhost + roles: + - role: emit-job-header + zuul_log_url: "http://logs.openstack.org" + +- hosts: all + pre_tasks: + # NOTE(pabelanger): Until we hit the validate-host role, we have a minimal + # set of ansible variables collected by zuul-executor. This doesn't include + # network variables (ansible_default_ipv4 / ansible_default_ipv6) so gather + # these variables as they are important to the configure-unbound role. + - name: Gather network facts + setup: + gather_subset: 'network' + + roles: + - add-build-sshkey + - start-zuul-console + - ensure-output-dirs + - log-inventory + +- hosts: all + roles: + # NOTE(pabelanger): We run this role in its own play to ensure unbound is + # restarted before proceeding with any other role. This is because we use + # notify / handler to restart the unbound service. With ansible notify + # actions are triggered at the end of each block of tasks in a play. + - configure-unbound + +- hosts: all + roles: + - validate-host + - use-cached-repos + - test-mirror-workspace-git-repos + - mirror-info + - role: configure-mirrors + set_apt_mirrors_trusted: True + - role: fetch-zuul-cloner + destination: "/usr/zuul-env/bin/zuul-cloner" + repo_src_dir: "/home/zuul/src/git.openstack.org" + + pre_tasks: + - name: Check that regional mirror is online + uri: + url: "http://{{ zuul_site_mirror_fqdn }}" diff --git a/playbooks/base/post-logs.yaml b/playbooks/base/post-logs.yaml new file mode 100644 index 0000000..b959b58 --- /dev/null +++ b/playbooks/base/post-logs.yaml @@ -0,0 +1,21 @@ +- hosts: localhost + roles: + - role: add-fileserver + fileserver: "{{ site_logs }}" + - role: ara-report + +- hosts: "{{ site_logs.fqdn }}" + gather_facts: False + roles: + - role: upload-logs + zuul_log_url: "http://logs.openstack.org" + +- hosts: localhost + # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail + # successful jobs. + ignore_errors: yes + roles: + # TODO do we want to assume subunit on opendev or should this move + # into tenant config + - submit-logstash-jobs + - submit-subunit-jobs diff --git a/playbooks/base/post.yaml b/playbooks/base/post.yaml new file mode 100644 index 0000000..c4a139d --- /dev/null +++ b/playbooks/base/post.yaml @@ -0,0 +1,11 @@ +- hosts: all + roles: + - fetch-output + - merge-output-to-logs + +- hosts: all + # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail + # successful jobs. + ignore_errors: yes + roles: + - remove-build-sshkey diff --git a/playbooks/base/pre.yaml b/playbooks/base/pre.yaml new file mode 100644 index 0000000..3ec1a34 --- /dev/null +++ b/playbooks/base/pre.yaml @@ -0,0 +1,40 @@ +- hosts: localhost + roles: + - role: emit-job-header + zuul_log_url: "http://logs.openstack.org" + +- hosts: all + pre_tasks: + # NOTE(pabelanger): Until we hit the validate-host role, we have a minimal + # set of ansible variables collected by zuul-executor. This doesn't include + # network variables (ansible_default_ipv4 / ansible_default_ipv6) so gather + # these variables as they are important to the configure-unbound role. + - name: Gather network facts + setup: + gather_subset: 'network' + + roles: + - add-build-sshkey + - start-zuul-console + - ensure-output-dirs + - log-inventory + +- hosts: all + roles: + # NOTE(pabelanger): We run this role in its own play to ensure unbound is + # restarted before proceeding with any other role. This is because we use + # notify / handler to restart the unbound service. With ansible notify + # actions are triggered at the end of each block of tasks in a play. + - configure-unbound + +- hosts: all + roles: + - validate-host + - use-cached-repos + - mirror-workspace-git-repos + - mirror-info + - role: configure-mirrors + set_apt_mirrors_trusted: True + - role: fetch-zuul-cloner + destination: "/usr/zuul-env/bin/zuul-cloner" + repo_src_dir: "/home/zuul/src/git.openstack.org" diff --git a/roles/configure-unbound/README.rst b/roles/configure-unbound/README.rst new file mode 100644 index 0000000..9e80061 --- /dev/null +++ b/roles/configure-unbound/README.rst @@ -0,0 +1,48 @@ +An ansible role to dynamically configure DNS forwarders for the +``unbound`` caching service. IPv6 will be preferred when there is a +usable IPv6 default route, otherwise IPv4. + +.. note:: This is not a standalone unbound configuration role. Base + setup is done during image builds in + ``project-config:nodepool/elements/nodepool-base/finalise.d/89-unbound``; + here we just do dynamic configuration of forwarders based on + the interfaces available on the actual host. + +**Role Variables** + +.. zuul:rolevar:: unbound_primary_nameserver_v4 + :default: 208.67.222.222 (OpenDNS) + + The primary IPv4 nameserver for fowarding requests + +.. zuul:rolevar:: unbound_secondary_nameserver_v4 + :default: 8.8.8.8 (Google) + + The secondary IPv4 nameserver for fowarding requests + +.. zuul:rolevar:: unbound_primary_nameserver_v6 + :default: 2620:0:ccc::2 (OpenDNS) + + The primary IPv6 nameserver for fowarding requests + +.. zuul:rolevar:: unbound_secondary_nameserver_v6 + :default: 2001:4860:4860::8888 (Google) + + The seconary IPv6 nameserver for fowarding requests + +.. zuul:rolevar:: unbound_cache_max_ttl + :default: 86400 + + Maximum TTL in seconds to keep successful queries cached for. + + This TTL will have precedence if the DNS record TTL is higher. + For example, a TTL of 90000 would be reduced to 86400. + +.. zuul:rolevar:: unbound_cache_min_ttl + :default: 0 + + Minimum TTL in seconds to keep queries cached for. + Note that this is effective for both successful and failed queries. + + This TTL will have precedence if the DNS record TTL is lower. + For example, a TTL of 60 would be raised to 900. diff --git a/roles/configure-unbound/defaults/main.yaml b/roles/configure-unbound/defaults/main.yaml new file mode 100644 index 0000000..e67192d --- /dev/null +++ b/roles/configure-unbound/defaults/main.yaml @@ -0,0 +1,24 @@ +# OpenDNS +unbound_primary_nameserver_v6: "2620:0:ccc::2" +unbound_primary_nameserver_v4: "208.67.222.222" + +# Google +unbound_secondary_nameserver_v6: "2001:4860:4860::8888" +unbound_secondary_nameserver_v4: "8.8.8.8" + +# Time to live maximum for RRsets and messages in the cache. +# Default is 86400 seconds (1 day). If the maximum kicks in, +# responses to clients still get decrementing TTLs based on the +# original (larger) values. When the internal TTL expires, the +# cache item has expired. Can be set lower to force the resolver +# to query for data often, and not trust (very large) TTL values. +unbound_cache_max_ttl: 86400 + +# Time to live minimum for RRsets and messages in the cache. +# Default is 0. If the minimum kicks in, the data is cached for +# longer than the domain owner intended, and thus less queries are +# made to look up the data. Zero makes sure the data in the cache +# is as the domain owner intended, higher values, especially more +# than an hour or so, can lead to trouble as the data in the cache +# does not match up with the actual data any more. +unbound_cache_min_ttl: 0 diff --git a/roles/configure-unbound/handlers/main.yaml b/roles/configure-unbound/handlers/main.yaml new file mode 100644 index 0000000..7199e29 --- /dev/null +++ b/roles/configure-unbound/handlers/main.yaml @@ -0,0 +1,5 @@ +- name: Restart unbound + become: yes + service: + name: unbound + state: restarted diff --git a/roles/configure-unbound/tasks/main.yaml b/roles/configure-unbound/tasks/main.yaml new file mode 100644 index 0000000..6b66745 --- /dev/null +++ b/roles/configure-unbound/tasks/main.yaml @@ -0,0 +1,84 @@ +# This role assumes that Unbound is already installed, fail early if it isn't. +- name: Check that Unbound is installed + stat: + path: /etc/unbound + register: unbound_config + +- name: Ensure that Unbound is installed + assert: + that: + - unbound_config.stat.exists + +# ansible_default_ipv6 can either be undefined (no ipv6) or blank (no +# routable address). We only want to use ipv6 if it's available & +# routable; combine these checks into this fact. +- name: Check for IPv6 + when: + - hostvars[inventory_hostname]['ansible_default_ipv6'] is defined + - hostvars[inventory_hostname]['ansible_default_ipv6']['address'] is defined + set_fact: + unbound_use_ipv6: True + +# Use *only* ipv6 resolvers if ipv6 is present and routable. This +# avoids traversing potential NAT when using ipv4 which can be +# unreliable. +- name: Set IPv6 nameservers + when: + - unbound_use_ipv6 is defined + set_fact: + unbound_primary_nameserver: '{{ unbound_primary_nameserver_v6 }}' + unbound_secondary_nameserver: '{{ unbound_secondary_nameserver_v6 }}' + +# Fallback to default ipv4 if there is no ipv6 available as this +# causes timeouts and failovers that are unnecesary. +- name: Set IPv4 nameservers + when: + - unbound_use_ipv6 is not defined + set_fact: + unbound_primary_nameserver: '{{ unbound_primary_nameserver_v4 }}' + unbound_secondary_nameserver: '{{ unbound_secondary_nameserver_v4 }}' + +- name: Include OS-specific variables + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution }}.yaml" + - "{{ ansible_os_family }}.yaml" + - "default.yaml" + +- name: Ensure Unbound conf.d directory exists + become: yes + file: + path: "{{ unbound_confd }}" + state: directory + +# TODO: Move this to /etc/unbound/conf.d ? +- name: Configure unbound forwarding + become: yes + template: + dest: /etc/unbound/forwarding.conf + owner: root + group: root + mode: 0644 + src: forwarding.conf.j2 + register: forwarding_config + notify: + - Restart unbound + +- name: Configure unbound TTL + become: yes + template: + dest: "{{ unbound_confd }}/ttl.conf" + owner: root + group: root + mode: 0644 + src: ttl.conf.j2 + register: ttl_config + notify: + - Restart unbound + +- name: Start unbound + become: yes + service: + name: unbound + state: started + enabled: yes diff --git a/roles/configure-unbound/templates/forwarding.conf.j2 b/roles/configure-unbound/templates/forwarding.conf.j2 new file mode 100644 index 0000000..3b52571 --- /dev/null +++ b/roles/configure-unbound/templates/forwarding.conf.j2 @@ -0,0 +1,6 @@ +# {{ ansible_managed }} + +forward-zone: + name: "." + forward-addr: {{ unbound_primary_nameserver }} + forward-addr: {{ unbound_secondary_nameserver }} diff --git a/roles/configure-unbound/templates/ttl.conf.j2 b/roles/configure-unbound/templates/ttl.conf.j2 new file mode 100644 index 0000000..34b5881 --- /dev/null +++ b/roles/configure-unbound/templates/ttl.conf.j2 @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +server: + cache-min-ttl: {{ unbound_cache_min_ttl }} + cache-max-ttl: {{ unbound_cache_max_ttl }} diff --git a/roles/configure-unbound/vars/Debian.yaml b/roles/configure-unbound/vars/Debian.yaml new file mode 100644 index 0000000..ccb6146 --- /dev/null +++ b/roles/configure-unbound/vars/Debian.yaml @@ -0,0 +1 @@ +unbound_confd: /etc/unbound/unbound.conf.d diff --git a/roles/configure-unbound/vars/default.yaml b/roles/configure-unbound/vars/default.yaml new file mode 100644 index 0000000..48bfc75 --- /dev/null +++ b/roles/configure-unbound/vars/default.yaml @@ -0,0 +1 @@ +unbound_confd: /etc/unbound/conf.d diff --git a/roles/mirror-info/README.rst b/roles/mirror-info/README.rst new file mode 100644 index 0000000..1b892de --- /dev/null +++ b/roles/mirror-info/README.rst @@ -0,0 +1,7 @@ +An ansible role to configure the ``/etc/ci/mirror_info.sh`` script + +**Role Variables** + +.. zuul:rolevar:: mirror_fqdn + + The base host for mirror servers. diff --git a/roles/mirror-info/tasks/main.yaml b/roles/mirror-info/tasks/main.yaml new file mode 100644 index 0000000..241b148 --- /dev/null +++ b/roles/mirror-info/tasks/main.yaml @@ -0,0 +1,17 @@ +- name: Create /etc/ci + become: yes + file: + path: /etc/ci + state: directory + owner: root + group: root + mode: 0755 + +- name: Install ci_mirror script + become: yes + template: + dest: '/etc/ci/mirror_info.sh' + owner: root + group: root + mode: 0644 + src: mirror_info.sh.j2 diff --git a/roles/mirror-info/templates/mirror_info.sh.j2 b/roles/mirror-info/templates/mirror_info.sh.j2 new file mode 100644 index 0000000..906ec19 --- /dev/null +++ b/roles/mirror-info/templates/mirror_info.sh.j2 @@ -0,0 +1,74 @@ +#!/bin/bash -xe + +# {{ ansible_managed }} + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +export NODEPOOL_MIRROR_HOST={{ mirror_fqdn }} + +# This script generates a descriptor slug to use with AFS, composed of the +# operating system, its version, and the processor architecture. + +# Pull in the os release. +# ID is 'fedora', 'centos', 'ubuntu' +# VERSION_ID is '23', '7', '14.04' +# Nothing else is useful and/or reliable across distros +. /etc/os-release + +################################################################################ +# Generate an OS Release Name +OS_TYPE=$ID + +################################################################################ +# Generate a version string. +OS_VERSION=$VERSION_ID +if [ "$OS_TYPE" != "ubuntu" ]; then + OS_VERSION=$(echo $OS_VERSION | cut -d'.' -f1) +fi + +################################################################################ +# Get the processor architecture. +# x86_64, i386, armv7l, armv6l +OS_ARCH=$(uname -m) + +################################################################################ +# Build the name +AFS_SLUG="$OS_TYPE-$OS_VERSION-$OS_ARCH" +AFS_SLUG=$(echo "$AFS_SLUG" | tr '[:upper:]' '[:lower:]') + +export AFS_SLUG +export NODEPOOL_DEBIAN_MIRROR=${NODEPOOL_DEBIAN_MIRROR:-http://$NODEPOOL_MIRROR_HOST/debian} +export NODEPOOL_PYPI_MIRROR=${NODEPOOL_PYPI_MIRROR:-http://$NODEPOOL_MIRROR_HOST/pypi/simple} +export NODEPOOL_WHEEL_MIRROR=${NODEPOOL_WHEEL_MIRROR:-http://$NODEPOOL_MIRROR_HOST/wheel/$AFS_SLUG} +export NODEPOOL_UBUNTU_MIRROR=${NODEPOOL_UBUNTU_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu} +export NODEPOOL_CENTOS_MIRROR=${NODEPOOL_CENTOS_MIRROR:-http://$NODEPOOL_MIRROR_HOST/centos} +export NODEPOOL_DEBIAN_OPENSTACK_MIRROR=${NODEPOOL_DEBIAN_OPENSTACK_MIRROR:-http://$NODEPOOL_MIRROR_HOST/debian-openstack} +export NODEPOOL_EPEL_MIRROR=${NODEPOOL_EPEL_MIRROR:-http://$NODEPOOL_MIRROR_HOST/epel} +export NODEPOOL_FEDORA_MIRROR=${NODEPOOL_FEDORA_MIRROR:-http://$NODEPOOL_MIRROR_HOST/fedora} +export NODEPOOL_OPENSUSE_MIRROR=${NODEPOOL_OPENSUSE_MIRROR:-http://$NODEPOOL_MIRROR_HOST/opensuse} +export NODEPOOL_CEPH_MIRROR=${NODEPOOL_CEPH_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ceph-deb-hammer} +export NODEPOOL_UCA_MIRROR=${NODEPOOL_UCA_MIRROR:-http://$NODEPOOL_MIRROR_HOST/ubuntu-cloud-archive} +# Reverse proxy servers +export NODEPOOL_BUILDLOGS_CENTOS_PROXY=${NODEPOOL_BUILDLOGS_CENTOS_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/buildlogs.centos} +export NODEPOOL_CBS_CENTOS_PROXY=${NODEPOOL_CBS_CENTOS_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/cbs.centos} +export NODEPOOL_DOCKER_REGISTRY_PROXY=${NODEPOOL_DOCKER_REGISTRY_PROXY:-http://$NODEPOOL_MIRROR_HOST:8081/registry-1.docker/} +export NODEPOOL_RDO_PROXY=${NODEPOOL_RDO_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/rdo} +export NODEPOOL_RUGYGEMS_PROXY=${NODEPOOL_RUBYGEMS_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/rubygems/} +export NODEPOOL_NPM_REGISTRY_PROXY=${NODEPOOL_NPM_REGISTRY_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/registry.npmjs} +export NODEPOOL_TARBALLS_PROXY=${NODEPOOL_TARBALLS_PROXY:-http://$NODEPOOL_MIRROR_HOST:8080/tarballs} + +# NOTE(mnaser): The following three proxies are consumed by OpenStack Ansible. They do not contain `http://` because we +# can only override hosts, not the entire base URL. +export NODEPOOL_LXC_IMAGE_PROXY=${NODEPOOL_LXC_IMAGE_PROXY:-$NODEPOOL_MIRROR_HOST:8080/images.linuxcontainers} +export NODEPOOL_PERCONA_PROXY=${NODEPOOL_PERCONA_PROXY:-$NODEPOOL_MIRROR_HOST:8080/percona} +export NODEPOOL_MARIADB_PROXY=${NODEPOOL_MARIADB_PROXY:-$NODEPOOL_MIRROR_HOST:8080/MariaDB} diff --git a/zuul.yaml b/zuul.yaml new file mode 100644 index 0000000..9104d2b --- /dev/null +++ b/zuul.yaml @@ -0,0 +1,144 @@ +# Shared zuul config common to all opendev tenants. +# Contains definitions of trusted jobs + + +# Changes to this job require a special procedure, because they can +# not be tested before landing, and if they are faulty, they will +# break all jobs, meaning subsequent corrections will not be able to +# land. To make a change: +# +# 1) Ensure that base-test and its playbooks are identical to base. +# 2) Make the change to base-test and/or its playbooks. +# 3) Merge the change from step 2. No jobs normally use base-test, so +# this is safe. +# 4) Propose a change to a job to reparent it to base-test. Choose a +# job which will exercise whatever you are changing. The +# "unittests" job in zuul-jobs is a good choice. Use [DNM] in the +# commit subject so that people know not to merge the change. Set +# it to "Work in progress" so people don't review it. +# 5) Once test results arrive for the change in step 2, make a change +# which copies the job and/or playbooks of base-test to base. In +# the commit message, link to (without using Depends-On:) the +# change from step 4 so reviewers can see the test results. +# 6) Once the change in step 5 merges, abandon the change from step 4. + +- secret: + name: site_logs + data: + fqdn: logs.openstack.org + path: /srv/static/logs + ssh_known_hosts: | + logs.openstack.org,23.253.108.137,2001:4800:7817:104:be76:4eff:fe05:dbee ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDDcvLuGLagUAZfc0BThLus8ufSPCrIhDtG0BdXvhblJjvIbkuELD3dRWRZVSYZAdzGZRY3t6vTAcguTrkbQg5ngXfdfF+OKPkaH8DiZwAX/1g/iRXhInkZTGBVqHo9pLAMeNNwviSy2JjpTqdD6fLEkHwW+uw4E2YZhYivctTSbOepMkzAtFV0w5cpyBzjAT/Hax2x5un6es8R0Iw3AAnUmtapn5e5NCrg2rPNpd0nve84wUavvbC2DeGDOZQdnIahwo60Sder5ZE/x6cG39bkSDdgFQArAzrNrH6BHmNGjfFPpnGmfc7P8gQwDPtMf02HvKapqATXpIxdbSGimWLL + ssh_username: jenkins + ssh_private_key: !encrypted/pkcs1-oaep + - t9SCvfU4po36HYV0yCxivgaDF+L6BQVUGramqW3dgARxP+Mdl51h1+K/8EdNke0wzfDWX + tdVL6Vsh4D5/evfLuBgeILjXT/pzozfhDksjz78TiWBnFQyiC3FHwVB6tZ9903fIiltw5 + aXg9AB3iYxSE/XQUKU3ThCt7zDJ0FoTrASVlKWaGeeMUiLBSaXaNrRTEFWyUJn7OU3nrj + 646ac7QJnkZ5j/kQbKDdWF73tCrL69fOoHHZtc0QbnizbBRjdVyECktVy3jvYfIAEdsKW + Apg1HCQBJETe64PQR1OKv18sC6MdfSVP//8mpOAMdVeJzfNqkk83V1IBWHWTQgIAAyt/4 + wB0aXUjX2rwMkInJfO6g2b+tMUajqEntib6IRKKXMb7/kS7ZcXwDkMj6bxBmnKgMLSx89 + +fhBnYLoaNv9keBlDLtGc62glO3B9TxcxNzOFuBp0mLPR28v6DXBn0uXJwzdqXf1WAUsQ + m6BKVE34J99vuzHFDn7J0ov/biZtJLAsD6q0enBm0nJQPuXfrW0c/jcUO4D+SjStBo/t+ + ZLMzzJvoygXTBFkiDX+6icIzLJMbpS8rBrGj+NbE+k1Lzni9Gq9Wo2xgDnGPwWDD97eup + H3cCIfhcFCP9m9YINLxxsJzpK8+Xss7LNqN8NbEbLPAbDH7b+rqIjoBPEAfVPM= + - C/Oz2r1fTYChvAbFpOdCF7+ZmEzSDYphP7fY/ENTOlvhq98QS3fGxRqj+oNEEppnM1oS1 + Cc/bR3kzSqgMK629H0qVVqJhR0ffNT6ip6CIP2BkAaqT/6yUY5tp0BjZyC+O7tV6QtWkq + gj6k/cJcgT7JKMLSN4zjdO1A9qeLpjc9y98lArIeYXFvJHpXC9J8Vj8Fd+ODhH/YUUEkQ + nqCXcBTd2k1RFEWvCVRN7tKkiuAa4HPPmj+In9TKw3j2grn3LMmkUrQn5G7bWyuzQGp2u + 2pVwvYNSEKxJiMMA0pTNLDMKaA5kvCQsQdt61FVN3AYZyCEbXq/6Is+JKoiZjBeyfUurB + btEoPNpjVmPQysCrvakSfbMi+Pn3jrZToxRNC30r1LWdHfKo0ovVRN0CEfce3suRu7uP8 + BXH7Ow4sYKF5FLjzwzCO6VuoDg+SrfjbBwnzoySIsB3CXXieMUj+0ytfG1FBmKg2IiLQ7 + Eaz+G4gCMe+1dMG87cKmizz7vC21ZFyeF3C2jBmXMMRvFgLCphHZOPfUOcy2yCPPFYmsg + 2DBxx2VrvcPljTW6woVbb4Kxrd7+2TRbT9mzWDQDDdKGveIqUnEURGacJ+WRc8ZlBpFwN + cmwbJal3VSo0sB/X25ZNnF7Y7JHrXI6a3s/ck2ppid+2h1sk1oE6br/DRjYCN4= + - k8yssVEnQr58u8krETfjnByeO6UmQL7+JfXSYHI79z9n3Fp3nIRrFoH177d47iHtcYxyP + 8IsQD2HMIGuRhyKZk5ruYwod/yeXZBwBcs7YSsof0U5gJ4gh6gw+bLQamKEaI4smq+xQA + UxxoHDw5m96+VUBeLdnXDFkq0qXiWOMmrCnVGgnDeuPZfyDbu8ILZi6c4WUFwj5o0oqRZ + pWEls8IfULjBEDMfbWhMrUh7zKurUwDXycmTAv4PriUdMdoMacqz/brxZZKC07+mzFiMj + iJvwV6STxATXy78+wWrM7MReoGownI0M0DKh07w/DEG000NTQnRz42DbwGbQQb8ugj4ee + 1sB3+pz3udnwffREtht2uf2C48dHFqMOKeGNV3MJv8Z93H6rpgdpuySZwXC3iL2ga8m4I + U8ypFoCXXR5rHRqAL8xmuUVoavYC4XLPN1QvKueZnQW5XntZxXH/lSe9OnEo6SVya4v8p + CEQ6+XIWQCKIFPXxFM+KCoh7c8FASmJ7Tw1WLw+DNdSKL8kewk0Z2FvkR6bTzzcKT3RCf + /xM/+N674GhkYRFCMsQxrT9e6cfB2FRbBrxR1GJQQrS9KHPGn7dgKNN4/0snbtypekhjl + 7oDENP6sbflXAo3Zeuq/XlvW0uobBqdI6bbkdMISAd779hVT5eQWvftwozrjHI= + - VjHYrglFpBi8Apnb64NYiblBANVDC0tXgAOzC7/NhcZ9Vc4rI7oRPfc48hrxjFlC+Uvtg + yI9cwu9y4FDDgGQ6qLovzP/Dvcwoga0YOZ7RYxdsT7N0/okRlWPRyj2h/7nlhrIxwK8bN + xRi7t/JniQkMrWiDckgw0YflLboMYQg8ShtCy1bZL1m0ISuBbodeswOLTiKFk2IG3R58h + Xylmgi2iM1md5ZeM9PhyLd8DrhuuJiKvhIiszdQNJN5Gg2CymYBveMfglE9r/10qgOM21 + 3UC37hSArn7WTu9Rwbo9bdNVePNik/x2O3fgMGND6ySX9vG8npPjOaomTGpds/z7DUn6F + 0B4RWDoYDD57BHviUSYDDEbfpNS6dk/K4RpArjpS7ZZcUIok5sXSV18zSI8Gaa32SKU59 + MdHuBtGW6p6kUTnuMSNCVsKGNOvjHsfnWFomUddEwhNFJW+tangCSkNaTQq/Yaf394lw8 + nOsautk56uoiZPhSzdBpR9s8z0z1z0eGzdeBWyV+IFF/UJCftDiOSu0zA28RgDIwIg690 + jVFWkZZRprDU6/5zgZPTLHOfz00IoMbGBKWSfvuOhF5l6VpSC3JVvcRd6/bivUq/1XkzP + uMv41vSFc4Kac1KmgAi96zglyRkzQgYVtLVNYyKbuLhVfx4U34mal/05sU3/MI= + +- job: + name: base + parent: null + abstract: true + description: | + The base job for OpenDev's installation of Zuul. + + All jobs ultimately inherit from this. It runs a pre-playbook + which copies all of the job's prepared git repos on to all of + the nodes in the nodeset. It runs a post-playbook which copies + all of the files in the logs/ subdirectory of the executor + work directory to the logserver. + + It also sets default timeout and nodeset values (which may be + overidden). + + Responds to these variables: + + .. zuul:jobvar:: base_serial + :default: Omitted + + This sets the serial keyword in the pre and post playbooks + which can be an integer or percentage. + + See ansible documentation for more information: + http://docs.ansible.com/ansible/latest/playbooks_delegation.html + + pre-run: playbooks/base/pre.yaml + post-run: + - playbooks/base/post.yaml + - playbooks/base/post-logs.yaml + roles: + - zuul: openstack-infra/zuul-jobs + vars: + ara_report_type: database + ara_report_path: ara-report + timeout: 1800 + post-timeout: 1800 + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + secrets: + - site_logs + +# See the procedure described above "base" before making changes to +# this job. +- job: + name: base-test + parent: null + description: | + A job to test changes to the base job without disturbing the + main job in production. Not for general use. + pre-run: playbooks/base-test/pre.yaml + post-run: + - playbooks/base-test/post.yaml + - playbooks/base-test/post-logs.yaml + roles: + - zuul: openstack-infra/zuul-jobs + timeout: 1800 + post-timeout: 1800 + vars: + ara_report_type: database + ara_report_path: ara-report + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + secrets: + - site_logs