Merge "Add support for node-level conf overrides"

This commit is contained in:
Zuul 2018-01-09 15:38:29 +00:00 committed by Gerrit Code Review
commit a78e638897
4 changed files with 436 additions and 25 deletions

View File

@ -475,3 +475,110 @@ up:
Finally, you should now be able to access horizon at http:// using
admin/password
Node and label specific configurations
--------------------------------------
There are situations where we need to define configuration differently for
different nodes in the environment. For example, we may require that some nodes
have a different vcpu_pin_set or other hardware specific deltas in nova.conf.
To do this, we can specify overrides in the values fed to the chart. Ex:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "0-31"
cpu_allocation_ratio: 3.0
overrides:
nova_compute:
labels:
- label:
key: compute-type
values:
- "dpdk"
- "sriov"
conf:
nova:
DEFAULT:
vcpu_pin_set: "0-15"
- label:
key: another-label
values:
- "another-value"
conf:
nova:
DEFAULT:
vcpu_pin_set: "16-31"
hosts:
- name: host1.fqdn
conf:
nova:
DEFAULT:
vcpu_pin_set: "8-15"
- name: host2.fqdn
conf:
nova:
DEFAULT:
vcpu_pin_set: "16-23"
Note that only one set of overrides is applied per node, such that:
1. Host overrides supercede label overrides
2. The farther down the list the label appears, the greater precedence it has.
e.g., "another-label" overrides will apply to a node containing both labels.
Also note that other non-overridden values are inherited by hosts and labels with overrides.
The following shows a set of example hosts and the values fed into the configmap for each:
1. ``host1.fqdn`` with labels ``compute-type: dpdk, sriov`` and ``another-label: another-value``:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "8-15"
cpu_allocation_ratio: 3.0
2. ``host2.fqdn`` with labels ``compute-type: dpdk, sriov`` and ``another-label: another-value``:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "16-23"
cpu_allocation_ratio: 3.0
3. ``host3.fqdn`` with labels ``compute-type: dpdk, sriov`` and ``another-label: another-value``:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "16-31"
cpu_allocation_ratio: 3.0
4. ``host4.fqdn`` with labels ``compute-type: dpdk, sriov``:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "0-15"
cpu_allocation_ratio: 3.0
5. ``host5.fqdn`` with no labels:
.. code-block:: yaml
conf:
nova:
DEFAULT:
vcpu_pin_set: "0-31"
cpu_allocation_ratio: 3.0

View File

@ -0,0 +1,286 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.utils.daemonset_overrides" }}
{{- $daemonset := index . 0 }}
{{- $daemonset_yaml := index . 1 }}
{{- $configmap_include := index . 2 }}
{{- $configmap_name := index . 3 }}
{{- $context := index . 4 }}
{{- $_ := unset $context ".Files" }}
{{- $_ := set $context.Values "__daemonset_yaml" $daemonset_yaml }}
{{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }}
{{- $_ := set $context.Values "__daemonset_list" list }}
{{- $_ := set $context.Values "__default" dict }}
{{- if hasKey $context.Values.conf "overrides" }}
{{- range $key, $val := $context.Values.conf.overrides }}
{{- if eq $key $daemonset_root_name }}
{{- range $type, $type_data := . }}
{{- if eq $type "hosts" }}
{{- range $host_data := . }}
{{/* dictionary that will contain all info needed to generate this
iteration of the daemonset */}}
{{- $current_dict := dict }}
{{/* set daemonset name */}}
{{- $_ := set $current_dict "name" $host_data.name }}
{{/* apply overrides */}}
{{- $override_conf_copy := $host_data.conf }}
{{- $root_conf_copy := omit $context.Values.conf "overrides" }}
{{- $merged_dict := merge $override_conf_copy $root_conf_copy }}
{{- $root_conf_copy2 := dict "conf" $merged_dict }}
{{- $context_values := omit $context.Values "conf" }}
{{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
{{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
{{- $_ := set $current_dict "nodeData" $root_conf_copy4 }}
{{/* Schedule to this host explicitly. */}}
{{- $nodeSelector_dict := dict }}
{{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }}
{{- $_ := set $nodeSelector_dict "operator" "In" }}
{{- $values_list := list $host_data.name }}
{{- $_ := set $nodeSelector_dict "values" $values_list }}
{{- $list_aggregate := list $nodeSelector_dict }}
{{- $_ := set $current_dict "matchExpressions" $list_aggregate }}
{{/* store completed daemonset entry/info into global list */}}
{{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }}
{{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
{{- end }}
{{- end }}
{{- if eq $type "labels" }}
{{- $_ := set $context.Values "__label_list" . }}
{{- range $label_data := . }}
{{/* dictionary that will contain all info needed to generate this
iteration of the daemonset. */}}
{{- $_ := set $context.Values "__current_label" dict }}
{{/* set daemonset name */}}
{{- $_ := set $context.Values.__current_label "name" $label_data.label.key }}
{{/* apply overrides */}}
{{- $override_conf_copy := $label_data.conf }}
{{- $root_conf_copy := omit $context.Values.conf "overrides" }}
{{- $merged_dict := merge $override_conf_copy $root_conf_copy }}
{{- $root_conf_copy2 := dict "conf" $merged_dict }}
{{- $context_values := omit $context.Values "conf" }}
{{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
{{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
{{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }}
{{/* Schedule to the provided label value(s) */}}
{{- $label_dict := omit $label_data.label "NULL" }}
{{- $_ := set $label_dict "operator" "In" }}
{{- $list_aggregate := list $label_dict }}
{{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
{{/* Do not schedule to other specified labels, with higher
precedence as the list position increases. Last defined label
is highest priority. */}}
{{- $other_labels := without $context.Values.__label_list $label_data }}
{{- range $label_data2 := $other_labels }}
{{- $label_dict := omit $label_data2.label "NULL" }}
{{- $_ := set $label_dict "operator" "NotIn" }}
{{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }}
{{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
{{- end }}
{{- $_ := set $context.Values "__label_list" $other_labels }}
{{/* Do not schedule to any other specified hosts */}}
{{- range $type, $type_data := $val }}
{{- if eq $type "hosts" }}
{{- range $host_data := . }}
{{- $label_dict := dict }}
{{- $_ := set $label_dict "key" "kubernetes.io/hostname" }}
{{- $_ := set $label_dict "operator" "NotIn" }}
{{- $values_list := list $host_data.name }}
{{- $_ := set $label_dict "values" $values_list }}
{{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }}
{{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }}
{{- end }}
{{- end }}
{{- end }}
{{/* store completed daemonset entry/info into global list */}}
{{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }}
{{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
{{- $_ := unset $context.Values "__current_label" }}
{{- end }}
{{- end }}
{{- end }}
{{/* scheduler exceptions for the default daemonset */}}
{{- $_ := set $context.Values.__default "matchExpressions" list }}
{{- range $type, $type_data := . }}
{{/* Do not schedule to other specified labels */}}
{{- if eq $type "labels" }}
{{- range $label_data := . }}
{{- $default_dict := omit $label_data.label "NULL" }}
{{- $_ := set $default_dict "operator" "NotIn" }}
{{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }}
{{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }}
{{- end }}
{{- end }}
{{/* Do not schedule to other specified hosts */}}
{{- if eq $type "hosts" }}
{{- range $host_data := . }}
{{- $default_dict := dict }}
{{- $_ := set $default_dict "key" "kubernetes.io/hostname" }}
{{- $_ := set $default_dict "operator" "NotIn" }}
{{- $values_list := list $host_data.name }}
{{- $_ := set $default_dict "values" $values_list }}
{{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }}
{{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{/* generate the default daemonset */}}
{{/* set name */}}
{{- $_ := set $context.Values.__default "name" "default" }}
{{/* no overrides apply, so copy as-is */}}
{{- $root_conf_copy1 := omit $context.Values.conf "overrides" }}
{{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }}
{{- $context_values := omit $context.Values "conf" }}
{{- $root_conf_copy3 := merge $context_values $root_conf_copy2 }}
{{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }}
{{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }}
{{/* add to global list */}}
{{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }}
{{- $_ := set $context.Values "__daemonset_list" $list_aggregate }}
{{- $_ := set $context.Values "__last_configmap_name" $configmap_name }}
{{- range $current_dict := $context.Values.__daemonset_list }}
{{- $context_novalues := omit $context "Values" }}
{{- $merged_dict := merge $current_dict.nodeData $context_novalues }}
{{- $_ := set $current_dict "nodeData" $merged_dict }}
{{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}}
{{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }}
{{/* labels may contain underscores which would be invalid here, so we replace them with dashes
there may be other valid label names which would make for an invalid DNS-1123 name
but these will be easier to handle in future with sprig regex* functions
(not availabile in helm 2.5.1) */}}
{{- $name_format2 := $name_format1 | replace "_" "-" }}
{{/* To account for the case where the same label is defined multiple times in overrides
(but with different label values), we add a sha of the scheduling data to ensure
name uniqueness */}}
{{- $_ := set $current_dict "dns_1123_name" dict }}
{{- if hasKey $current_dict "matchExpressions" }}
{{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" ($current_dict.matchExpressions | quote | sha256sum | trunc 8))) }}
{{- else }}
{{- $_ := set $current_dict "dns_1123_name" $name_format2 }}
{{- end }}
{{/* set daemonset metadata name */}}
{{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}
{{/* set container name
assume not more than one container is defined */}}
{{- $container := first $context.Values.__daemonset_yaml.spec.template.spec.containers }}
{{- $_ := set $container "name" $current_dict.dns_1123_name }}
{{- $cont_list := list $container }}
{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $cont_list }}
{{/* cross-reference configmap name to container volume definitions */}}
{{- $_ := set $context.Values "__volume_list" list }}
{{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }}
{{- $_ := set $context.Values "__volume" $current_volume }}
{{- if hasKey $context.Values.__volume "configMap" }}
{{- if eq $context.Values.__volume.configMap.name $context.Values.__last_configmap_name }}
{{- $_ := set $context.Values.__volume.configMap "name" $current_dict.dns_1123_name }}
{{- $_ := set $context.Values.__volume "name" $current_dict.dns_1123_name }}
{{- end }}
{{- end }}
{{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }}
{{- $_ := set $context.Values "__volume_list" $updated_list }}
{{- end }}
{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }}
{{/* set names for container volume mounts */}}
{{- $_ := set $context.Values "__volume_mount_list" list }}
{{- range $vol_mount := $container.volumeMounts }}
{{- $_ := set $context.Values "__volume_mount" $vol_mount }}
{{- if eq $vol_mount.name $context.Values.__last_configmap_name }}
{{- $_ := set $context.Values.__volume_mount "name" $current_dict.dns_1123_name }}
{{- end }}
{{- $updated_list := append $context.Values.__volume_mount_list $context.Values.__volume_mount }}
{{- $_ := set $context.Values "__volume_mount_list" $updated_list }}
{{- end }}
{{- $_ := set $container "volumeMounts" $context.Values.__volume_mount_list }}
{{- $container_list := list $container }}
{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $container_list }}
{{/* populate scheduling restrictions */}}
{{- if hasKey $current_dict "matchExpressions" }}
{{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }}
{{- $match_exprs := dict }}
{{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }}
{{- $appended_match_expr := list $match_exprs }}
{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }}
{{- end }}
{{/* input value hash for current set of values overrides */}}
{{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }}
{{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }}
{{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }}
{{- $values_hash := $cmap | quote | sha256sum }}
{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }}
{{/* generate configmap */}}
---
{{ $cmap }}
{{/* generate daemonset yaml */}}
---
{{ $context.Values.__daemonset_yaml | toYaml }}
{{- $_ := set $context.Values "__last_configmap_name" $current_dict.dns_1123_name }}
{{- end }}
{{- end }}

View File

@ -14,8 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
{{- define "nova.configmap.etc" }}
{{- $configMapName := index . 0 }}
{{- $envAll := index . 1 }}
{{- with $envAll }}
{{- if empty .Values.conf.nova.keystone_authtoken.auth_uri -}}
{{- tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | set .Values.conf.nova.keystone_authtoken "auth_uri" | quote | trunc 0 -}}
@ -148,7 +150,7 @@ limitations under the License.
apiVersion: v1
kind: ConfigMap
metadata:
name: nova-etc
name: {{ $configMapName }}
data:
rally_tests.yaml: |+
{{ toYaml .Values.conf.rally_tests.tests | indent 4 }}
@ -173,3 +175,7 @@ data:
ssh-config.sh: |+
{{- tuple .Values.conf.ssh "etc/_ssh-config.tpl" . | include "helm-toolkit.utils.configmap_templater" }}
{{- end }}
{{- end }}
{{- if .Values.manifests.configmap_etc }}
{{- list "nova-etc" . | include "nova.configmap.etc" }}
{{- end }}

View File

@ -14,29 +14,30 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.daemonset_compute }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.compute }}
{{- define "nova.compute.daemonset" }}
{{- $daemonset := index . 0 }}
{{- $configMapName := index . 1 }}
{{- $serviceAccountName := index . 2 }}
{{- $dependencies := index . 3 }}
{{- $envAll := index . 4 }}
{{- with $envAll }}
{{- $mounts_nova_compute := .Values.pod.mounts.nova_compute.nova_compute }}
{{- $mounts_nova_compute_init := .Values.pod.mounts.nova_compute.init_container }}
{{- $serviceAccountName := "nova-compute" }}
{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nova-compute
spec:
{{ tuple $envAll "compute" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "nova" "compute" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
@ -132,42 +133,42 @@ spec:
mountPath: /tmp/nova-compute.sh
subPath: nova-compute.sh
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/nova.conf
subPath: nova.conf
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/api-paste.ini
subPath: api-paste.ini
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/policy.yaml
subPath: policy.yaml
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
# NOTE (Portdirect): We mount here to override Kollas
# custom sudoers file when using Kolla images, this
# location will also work fine for other images.
mountPath: /etc/sudoers.d/kolla_nova_sudoers
subPath: nova_sudoers
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/rootwrap.conf
subPath: rootwrap.conf
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/rootwrap.d/api-metadata.filters
subPath: api-metadata.filters
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/rootwrap.d/compute.filters
subPath: compute.filters
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /etc/nova/rootwrap.d/network.filters
subPath: network.filters
readOnly: true
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /root/.ssh/config
subPath: ssh-config
readOnly: true
@ -220,13 +221,13 @@ spec:
mountPath: /var/lib/nova
- name: varliblibvirt
mountPath: /var/lib/libvirt
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /root/.ssh/id_rsa
subPath: ssh-key-private
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /root/.ssh/id_rsa.pub
subPath: ssh-key-public
- name: nova-etc
- name: {{ $configMapName }}
mountPath: /root/.ssh/authorized_keys
subPath: ssh-key-public
- name: nova-bin
@ -238,9 +239,9 @@ spec:
configMap:
name: nova-bin
defaultMode: 0555
- name: nova-etc
- name: {{ $configMapName }}
configMap:
name: nova-etc
name: {{ $configMapName }}
defaultMode: 0444
{{- if .Values.ceph.enabled }}
- name: etcceph
@ -275,3 +276,14 @@ spec:
path: /etc/machine-id
{{ if $mounts_nova_compute.volumes }}{{ toYaml $mounts_nova_compute.volumes | indent 8 }}{{ end }}
{{- end }}
{{- end }}
{{- if .Values.manifests.daemonset_compute }}
{{- $daemonset := "compute" }}
{{- $configMapName := "nova-etc" }}
{{- $serviceAccountName := "nova-compute" }}
{{- $dependencies := .Values.dependencies.compute }}
{{ tuple . $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName $dependencies . | include "nova.compute.daemonset" | toString | fromYaml }}
{{- $configmap_yaml := "nova.configmap.etc" }}
{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "helm-toolkit.utils.daemonset_overrides" }}
{{- end }}