Merge "Support soft-anti-affinity policy for nodes"

This commit is contained in:
Zuul 2018-01-29 17:02:17 +00:00 committed by Gerrit Code Review
commit a43ab8f125
26 changed files with 219 additions and 3 deletions

View File

@ -32,6 +32,12 @@ cluster_def_opts = [
'magnum_vm_ubuntu_mesos'],
help=_('Enabled cluster definition entry points.'),
deprecated_group='bay'),
cfg.StrOpt('nodes_affinity_policy',
default='soft-anti-affinity',
help=_('Affinity policy for server group of cluster nodes.'
'Possible values include "affinity", "anti-affinity",'
'"soft-affinity" and "soft-anti-affinity".')
),
]

View File

@ -87,6 +87,9 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
if container_infra_prefix:
extra_params['container_infra_prefix'] = container_infra_prefix
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
return super(K8sFedoraTemplateDefinition,
self).get_params(context, cluster_template, cluster,
extra_params=extra_params,

View File

@ -102,6 +102,8 @@ class SwarmFedoraTemplateDefinition(template_def.BaseTemplateDefinition):
'swarm_strategy']
extra_params['auth_url'] = context.auth_url
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
# set docker_volume_type
# use the configuration default if None provided

View File

@ -105,6 +105,8 @@ class SwarmModeTemplateDefinition(template_def.BaseTemplateDefinition):
label_list = ['rexray_preempt']
extra_params['auth_url'] = context.auth_url
extra_params['nodes_affinity_policy'] = \
CONF.cluster.nodes_affinity_policy
for label in label_list:
extra_params[label] = cluster_template.labels.get(label)

View File

@ -294,6 +294,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -400,6 +408,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -465,6 +484,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -520,6 +540,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

View File

@ -229,6 +229,9 @@ parameters:
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
@ -511,6 +514,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: OS::Neutron::Port

View File

@ -160,6 +160,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
minion_wait_handle:
@ -338,6 +342,7 @@ resources:
user_data: {get_resource: kube_minion_init}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: OS::Neutron::Port

View File

@ -356,6 +356,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -465,6 +473,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -537,6 +556,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -606,6 +626,7 @@ resources:
dns_service_ip: {get_param: dns_service_ip}
dns_cluster_domain: {get_param: dns_cluster_domain}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

View File

@ -263,6 +263,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
master_wait_handle:
@ -521,6 +525,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: OS::Neutron::Port

View File

@ -231,6 +231,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
minion_wait_handle:
@ -433,6 +437,7 @@ resources:
user_data: {get_resource: kube_minion_init}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: OS::Neutron::Port

View File

@ -347,6 +347,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
api_lb:
@ -443,6 +451,17 @@ resources:
public_ip: {get_attr: [api_address_lb_switch, public_ip]}
private_ip: {get_attr: [api_address_lb_switch, private_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# kubernetes masters. This is a resource group that will create
@ -509,6 +528,7 @@ resources:
wc_curl_cli: {get_attr: [master_wait_handle, curl_cli]}
etcd_lb_vip: {get_attr: [etcd_lb, address]}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -552,6 +572,7 @@ resources:
flannel_network_cidr: {get_param: flannel_network_cidr}
external_network: {get_param: external_network}
kube_software_configs: {get_attr: [kubeminion_software_configs, kube_minion_init]}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#

View File

@ -242,6 +242,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -491,6 +495,7 @@ resources:
user_data: {get_resource: kube_master_init}
networks:
- port: {get_resource: kube_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_master_eth0:
type: "OS::Neutron::Port"

View File

@ -48,6 +48,10 @@ parameters:
description : >
ID of the multipart mime.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -69,6 +73,7 @@ resources:
user_data: {get_param: kube_software_configs}
networks:
- port: {get_resource: kube_minion_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
kube_minion_eth0:
type: "OS::Neutron::Port"

View File

@ -216,6 +216,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -369,6 +377,17 @@ resources:
master_public_ip: {get_attr: [mesos_masters, resource.0.mesos_master_external_ip]}
master_private_ip: {get_attr: [mesos_masters, resource.0.mesos_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# Mesos masters. This is a resource group that will create
@ -397,6 +416,7 @@ resources:
secgroup_mesos_id: {get_resource: secgroup_master}
api_pool_id: {get_attr: [api_lb, pool_id]}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#
@ -426,6 +446,7 @@ resources:
external_network: {get_param: external_network}
secgroup_slave_all_open_id: {get_resource: secgroup_slave_all_open}
mesos_slave_software_configs: {get_attr: [mesos_slave_software_configs, mesos_init]}
nodes_server_group_id: {get_resource: nodes_server_group}
######################################################################
#

View File

@ -48,6 +48,10 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
add_ext_ca_certs:
@ -85,6 +89,7 @@ resources:
user_data: {get_resource: mesos_master_init}
networks:
- port: {get_resource: mesos_master_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
mesos_master_eth0:
type: OS::Neutron::Port

View File

@ -43,6 +43,10 @@ parameters:
type: string
description: ID of the multipart mime.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
######################################################################
@ -64,6 +68,7 @@ resources:
user_data: {get_param: mesos_slave_software_configs}
networks:
- port: {get_resource: mesos_slave_eth0}
scheduler_hints: { group: { get_param: nodes_server_group_id }}
mesos_slave_eth0:
type: OS::Neutron::Port

View File

@ -255,6 +255,13 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
@ -338,6 +345,17 @@ resources:
master_public_ip: {get_attr: [swarm_masters, resource.0.swarm_master_external_ip]}
master_private_ip: {get_attr: [swarm_masters, resource.0.swarm_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
etcd_address_lb_switch:
type: Magnum::ApiGatewaySwitcher
properties:
@ -401,6 +419,7 @@ resources:
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_nodes:
type: "OS::Heat::ResourceGroup"
@ -453,6 +472,7 @@ resources:
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

View File

@ -178,6 +178,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
master_wait_handle:
@ -444,6 +448,7 @@ resources:
networks:
- port:
get_resource: swarm_master_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_master_eth0:
type: "OS::Neutron::Port"

View File

@ -177,6 +177,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
node_wait_handle:
@ -400,6 +404,7 @@ resources:
networks:
- port:
get_resource: swarm_node_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_node_eth0:
type: "OS::Neutron::Port"

View File

@ -186,6 +186,14 @@ parameters:
hidden: true
description: The OpenStack CA certificate to install on the node.
nodes_affinity_policy:
type: string
description: >
affinity policy for nodes server group
constraints:
- allowed_values: ["affinity", "anti-affinity", "soft-affinity",
"soft-anti-affinity"]
resources:
######################################################################
@ -260,6 +268,17 @@ resources:
master_public_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_external_ip]}
master_private_ip: {get_attr: [swarm_primary_master, resource.0.swarm_master_ip]}
######################################################################
#
# resources that expose the server group for all nodes include master
# and minions.
#
nodes_server_group:
type: OS::Nova::ServerGroup
properties:
policies: [{get_param: nodes_affinity_policy}]
######################################################################
#
# Swarm manager is responsible for the entire cluster and manages the
@ -309,6 +328,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_secondary_masters:
type: "OS::Heat::ResourceGroup"
@ -352,6 +372,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
swarm_nodes:
type: "OS::Heat::ResourceGroup"
@ -395,6 +416,7 @@ resources:
rexray_preempt: {get_param: rexray_preempt}
verify_ca: {get_param: verify_ca}
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
outputs:

View File

@ -140,6 +140,9 @@ parameters:
openstack_ca:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
@ -322,6 +325,7 @@ resources:
networks:
- port:
get_resource: swarm_master_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_master_eth0:
type: "OS::Neutron::Port"

View File

@ -133,6 +133,10 @@ parameters:
type: string
description: The OpenStack CA certificate to install on the node.
nodes_server_group_id:
type: string
description: ID of the server group for kubernetes cluster nodes.
resources:
node_wait_handle:
@ -293,6 +297,7 @@ resources:
networks:
- port:
get_resource: swarm_node_eth0
scheduler_hints: { group: { get_param: nodes_server_group_id }}
swarm_node_eth0:
type: "OS::Neutron::Port"

View File

@ -229,6 +229,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -325,6 +326,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
@ -408,6 +410,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'username': 'fake_user',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -731,6 +734,7 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity"
}
self.assertEqual(expected, definition)
self.assertEqual(

View File

@ -164,6 +164,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -242,6 +243,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -314,6 +316,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'verify_ca': True,
'node_flavor': 'flavor_id',
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -386,6 +389,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -459,6 +463,7 @@ class TestClusterConductorWithSwarm(base.TestCase):
'docker_volume_type': 'lvmdriver-1',
'verify_ca': True,
'openstack_ca': '',
'nodes_affinity_policy': 'soft-anti-affinity'
}
self.assertEqual(expected, definition)
self.assertEqual(

View File

@ -296,7 +296,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'magnum_url': mock_osc.magnum_url.return_value,
'region_name': mock_osc.cinder_region_name.return_value,
'kube_tag': kube_tag,
'container_infra_prefix': container_infra_prefix}}
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -381,7 +382,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'loadbalancing_protocol': 'HTTP',
'kubernetes_port': 8080,
'kube_tag': kube_tag,
'container_infra_prefix': container_infra_prefix}}
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -789,7 +791,8 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v3',
'rexray_preempt': rexray_preempt,
'swarm_strategy': swarm_strategy,
'docker_volume_type': docker_volume_type}}
'docker_volume_type': docker_volume_type,
'nodes_affinity_policy': 'soft-anti-affinity'}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,

View File

@ -0,0 +1,7 @@
---
issues:
- |
Enhancement to support anfinity policy for cluster nodes. Before this patch,
There is no way to gurantee all nodes of a cluster created on different
compute hosts to get high availbility.