From fc3eb127fc706bc3c6aa91424f966247bb29a7fc Mon Sep 17 00:00:00 2001 From: Tee Ngo Date: Mon, 27 May 2019 16:59:26 -0400 Subject: [PATCH] Set value for individual registry service parameter This commit implements a near/mid-term solution to help accelerate the switch in automation to using Ansible bootstrap. The long term solution is to allow dynamic addition of new registries for custom applications. This will require change to sysinv for a complete solution. Closes-Bug: 1830319 Change-Id: I9b92fd0c5ddc226cb47e6586f8dfbdff44eed96b Signed-off-by: Tee Ngo --- .../playbooks/bootstrap/host_vars/default.yml | 30 ++++++--- .../tasks/bringup_helm.yml | 7 +-- .../tasks/bringup_kubemaster.yml | 63 +++---------------- .../bringup-essential-services/tasks/main.yml | 13 +++- .../files/populate_initial_config.py | 15 ++++- .../roles/prepare-env/tasks/main.yml | 26 +++++--- .../roles/validate-config/tasks/main.yml | 33 +++++----- 7 files changed, 95 insertions(+), 92 deletions(-) diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml index 3157255ac5..8c62f823b6 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/host_vars/default.yml @@ -92,7 +92,23 @@ dynamic_address_allocation: True # DOCKER REGISTRIES # ================= # -# The default list of registries can be extended with new entries. +# The docker_registries is a map of known registry keys and their +# source values. Each key is a fully scoped registry name and the +# same name is used as the default value. For instance, +# k8s.gcr.io registry which hosts Kubernetes related images, has +# the default registry value of k8s.gcr.io +# +# To overwrite a particular registry, use the original registry +# value as the key followed by a custom IP address or domain for +# the value. +# +# The "unified" is a special registry key. Defining and giving +# it a value implies all images are to be retrieved from this +# single source. Hence, registry values of all other registry keys +# if specified will be ignored. +# +# The docker registries map can be extended with new custom keys in +# the near future. # # The valid formats for a registry value are: # - domain (e.g. example.domain) @@ -102,17 +118,15 @@ dynamic_address_allocation: True # - IPv6 address (e.g. FD01::0100) # - IPv6 address with port (e.g. [FD01::0100]:5000 # -# If the registry list contains only a single item, the specified registry is -# considered a unified registry and will replace all default registries. -# # Parameter is_secure_registry is only relevant when a unified registry is # used. docker_registries: -# - k8s.gcr.io -# - gcr.io -# - quay.io -# - docker.io + k8s.gcr.io: + gcr.io: + quay.io: + docker.io: + # unified: example.domain #is_secure_registry: True diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml index f02f096ee0..24deb4fe43 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_helm.yml @@ -94,11 +94,10 @@ regexp: "var.pxeboot_ip_network =.*$" replace: 'var.pxeboot_ip_network = "{{ pxeboot_subnet }}"' -- name: Update tiller image tag if using unified registry +- name: Update Tiller and Armada image tags set_fact: - tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ docker_registries[0] }}') }}" - armada_img: "{{ armada_img | regex_replace('quay.io', '{{ docker_registries[0] }}') }}" - when: use_unified_registry + tiller_img: "{{ tiller_img | regex_replace('gcr.io', '{{ gcr_registry }}') }}" + armada_img: "{{ armada_img | regex_replace('quay.io', '{{ quay_registry }}') }}" - name: Pull Tiller and Armada images docker_image: diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml index 6494fc661e..048b76408a 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/bringup_kubemaster.yml @@ -59,7 +59,7 @@ when: not is_secure_registry environment: - DOCKER_REGISTRY_IP: "{{ docker_registries[0] }}" + DOCKER_REGISTRY_IP: "{{ docker_registry }}" when: use_unified_registry - name: Update kernel parameters for iptables @@ -136,14 +136,7 @@ ETCD_PORT: 2379 POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}" SERVICE_NETWORK_CIDR: "{{ cluster_service_subnet }}" - K8S_REGISTRY: "{{ default_k8s_registry }}" - -- name: Update image repo in admin yaml if unified registry is used - replace: - path: /etc/kubernetes/kubeadm.yaml - regexp: "imageRepository: .*$" - replace: 'imageRepository: "{{ docker_registries[0] }}"' - when: use_unified_registry + K8S_REGISTRY: "{{ k8s_registry }}" - name: Initializing Kubernetes master command: kubeadm init --config=/etc/kubernetes/kubeadm.yaml @@ -167,21 +160,11 @@ remote_src: yes - name: Update Multus config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update Multus yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/multus.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate Multus Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/multus.yaml" @@ -204,15 +187,7 @@ environment: CLUSTER_IP: "{{ cluster_floating_address }}" POD_NETWORK_CIDR: "{{ cluster_pod_subnet }}" - QUAY_REGISTRY: "{{ default_quay_registry }}" - -- name: Update Calico yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_quay_registry }}|'$QUAY_REGISTRY'|g' /etc/kubernetes/calico.yaml" - args: - warn: false - environment: - QUAY_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + QUAY_REGISTRY: "{{ quay_registry }}" - name: Activate Calico Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/calico.yaml" @@ -224,21 +199,11 @@ remote_src: yes - name: Update SRIOV Networking config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update SRIOV Networking yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriov-cni.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate SRIOV Networking command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriov-cni.yaml" @@ -250,21 +215,11 @@ remote_src: yes - name: Update SRIOV device plugin config file - command: "{{ item }}" - args: - warn: false - with_items: - - "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" - environment: - DOCKER_REGISTRY: "{{ default_docker_registry }}" - -- name: Update SRIOV device plugin yaml file with new registry info if unified registry is used - command: "sed -i -e 's|{{ default_docker_registry }}|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" + command: "sed -i -e 's|<%= @docker_registry %>|'$DOCKER_REGISTRY'|g' /etc/kubernetes/sriovdp-daemonset.yaml" args: warn: false environment: - DOCKER_REGISTRY: "{{ docker_registries[0] }}" - when: use_unified_registry + DOCKER_REGISTRY: "{{ docker_registry }}" - name: Activate SRIOV device plugin command: "kubectl --kubeconfig=/etc/kubernetes/admin.conf apply -f /etc/kubernetes/sriovdp-daemonset.yaml" diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml index a3dcdfabeb..de31518ad7 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/bringup-essential-services/tasks/main.yml @@ -96,7 +96,6 @@ - "prev_external_oam_floating_address: {{ external_oam_floating_address }}" - "prev_management_multicast_subnet: {{ management_multicast_subnet }}" - "prev_dns_servers: {{ dns_servers | join(',') }}" - - "prev_docker_registries: {{ docker_registries | join(',') }}" - "prev_docker_http_proxy: {{ docker_http_proxy }}" - "prev_docker_https_proxy: {{ docker_https_proxy }}" - "prev_docker_no_proxy: {{ docker_no_proxy | join(',') }}" @@ -121,4 +120,16 @@ - "prev_external_oam_node_0_address: {{ external_oam_node_0_address }}" - "prev_external_oam_node_1_address: {{ external_oam_node_1_address }}" + # Need to save the dictionary this way for proper comparison during replay + - name: Save previous docker registries header + lineinfile: + path: "{{ last_bootstrap_config_file }}" + line: "prev_docker_registries:" + + - name: Save previous docker registries content + lineinfile: + path: "{{ last_bootstrap_config_file }}" + line: " {{ item.key }}: {{ item.value }}" + with_dict: "{{ docker_registries }}" + when: save_config diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py index 3f55c094fa..44d36b71d9 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/persist-config/files/populate_initial_config.py @@ -450,11 +450,17 @@ def populate_docker_config(client): 'BOOTSTRAP_CONFIG', 'USE_DEFAULT_REGISTRIES') if not use_default_registries: - registries = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRIES') secure_registry = CONF.getboolean('BOOTSTRAP_CONFIG', 'IS_SECURE_REGISTRY') parameters = {} - parameters['registries'] = registries + + # TODO(tngo): The following 4 service parameters will be removed when + # we switch to the long term solution using a single "registries" + # service parameter that is extensible. + parameters['k8s'] = CONF.get('BOOTSTRAP_CONFIG', 'K8S_REGISTRY') + parameters['gcr'] = CONF.get('BOOTSTRAP_CONFIG', 'GCR_REGISTRY') + parameters['quay'] = CONF.get('BOOTSTRAP_CONFIG', 'QUAY_REGISTRY') + parameters['docker'] = CONF.get('BOOTSTRAP_CONFIG', 'DOCKER_REGISTRY') if not secure_registry: parameters['insecure_registry'] = "True" @@ -469,7 +475,10 @@ def populate_docker_config(client): if RECONFIGURE_SERVICE: parameters = client.sysinv.service_parameter.list() for parameter in parameters: - if (parameter.name == 'registries' or + if (parameter.name == 'k8s' or + parameter.name == 'gcr' or + parameter.name == 'quay' or + parameter.name == 'docker' or parameter.name == 'insecure_registry'): client.sysinv.service_parameter.delete( parameter.uuid) diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml index c30fd3a600..dc270cfa8a 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/prepare-env/tasks/main.yml @@ -135,14 +135,17 @@ external_oam_node_0_address: "{{ external_oam_node_0_address | default('derived') }}" external_oam_node_1_address: "{{ external_oam_node_1_address | default('derived') }}" - - name: Set docker registries to default values if not specified + - name: Set default registries dictionary set_fact: - docker_registries: - - k8s.gcr.io - - gcr.io - - quay.io - - docker.io - when: docker_registries is none + default_docker_registries: + k8s.gcr.io: k8s.gcr.io + gcr.io: gcr.io + quay.io: quay.io + docker.io: docker.io + + - name: Merge user and default registries dictionaries + set_fact: + docker_registries: "{{ default_docker_registries | combine(docker_registries) }}" - name: Initialize some flags to be used in subsequent roles/tasks set_fact: @@ -186,6 +189,13 @@ when: (docker_http_proxy is defined and docker_http_proxy is not none) or (docker_https_proxy is defined and docker_https_proxy is not none) + - name: Set default values for platform registries + set_fact: + default_k8s_registry: k8s.gcr.io + default_gcr_registry: gcr.io + default_quay_registry: quay.io + default_docker_registry: docker.io + - name: Set default values for docker proxies if not defined set_fact: docker_http_proxy: "{{ docker_http_proxy | default('undef') }}" @@ -277,7 +287,7 @@ - name: Turn on docker reconfiguration flag if docker config is changed set_fact: docker_config_update: true - when: (prev_docker_registries.split(',') | sort != docker_registries | sort) or + when: (prev_docker_registries != docker_registries) or ((use_docker_proxy) and (prev_docker_http_proxy != docker_http_proxy or prev_docker_https_proxy != docker_https_proxy or diff --git a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml index c118e97c5d..e469eb50b8 100644 --- a/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml +++ b/playbookconfig/playbookconfig/playbooks/bootstrap/roles/validate-config/tasks/main.yml @@ -257,11 +257,10 @@ - block: - set_fact: use_default_registries: true - # Define these just in case we need them later - default_k8s_registry: k8s.gcr.io - default_gcr_registry: gcr.io - default_quay_registry: quay.io - default_docker_registry: docker.io + k8s_registry: "{{ docker_registries[default_k8s_registry] if docker_registries[default_k8s_registry] is not none else default_k8s_registry }}" + gcr_registry: "{{ docker_registries[default_gcr_registry] if docker_registries[default_gcr_registry] is not none else default_gcr_registry }}" + quay_registry: "{{ docker_registries[default_quay_registry] if docker_registries[default_quay_registry] is not none else default_quay_registry }}" + docker_registry: "{{ docker_registries[default_docker_registry] if docker_registries[default_docker_registry] is not none else default_docker_registry }}" default_no_proxy: - localhost - 127.0.0.1 @@ -314,23 +313,26 @@ - name: Turn on use_unified_registry flag set_fact: use_unified_registry: true - unified_registry: "{{ docker_registries }}" + k8s_registry: "{{ docker_registries['unified'] }}" + gcr_registry: "{{ docker_registries['unified'] }}" + quay_registry: "{{ docker_registries['unified'] }}" + docker_registry: "{{ docker_registries['unified'] }}" - when: docker_registries|length == 1 + when: docker_registries['unified'] is defined and docker_registries['unified'] is not none - name: Update use_default_registries flag set_fact: use_default_registries: false when: use_unified_registry or docker_registries|length != 4 or - default_k8s_registry not in docker_registries or - default_gcr_registry not in docker_registries or - default_quay_registry not in docker_registries or - default_docker_registry not in docker_registries + k8s_registry != default_k8s_registry or + gcr_registry != default_gcr_registry or + quay_registry != default_quay_registry or + docker_registry != default_docker_registry - block: - - include: validate_address.yml input_address={{ item }} - with_items: "{{ docker_registries }}" + - include: validate_address.yml input_address={{ item.value }} + with_dict: "{{ docker_registries }}" when: not use_default_registries @@ -417,7 +419,10 @@ - "DOCKER_HTTP_PROXY={{ docker_http_proxy }}" - "DOCKER_HTTPS_PROXY={{ docker_https_proxy }}" - "DOCKER_NO_PROXY={{ docker_no_proxy_combined | join(',') }}" - - "DOCKER_REGISTRIES={{ docker_registries | join(',') }}" + - "K8S_REGISTRY={{ k8s_registry }}" + - "GCR_REGISTRY={{ gcr_registry }}" + - "QUAY_REGISTRY={{ quay_registry }}" + - "DOCKER_REGISTRY={{ docker_registry }}" - "USE_DEFAULT_REGISTRIES={{ use_default_registries }}" - "IS_SECURE_REGISTRY={{ is_secure_registry | default(True) }}" - "RECONFIGURE_ENDPOINTS={{ reconfigure_endpoints }}"