From 297eb5e88f8f586fc2ead52ae6a89edb189dc67a Mon Sep 17 00:00:00 2001 From: Lakshmi Prasanna Goutham Pratapa Date: Wed, 25 Apr 2018 14:19:07 +0530 Subject: [PATCH] Add resource Constraints to Nova containers. This commit will constrain the dimensions of service `Nova` and sub-containers deployed along with it. A user can give the dimension values in `/etc/kolla/globals.yml` the data-types just like stated in this commit. Reference-Docs: https://docs.docker.com/config/containers/resource_constraints/ Added Test-cases for the same. Partially-Implements: blueprint resource-constraints Change-Id: I6458d8fb7b26a6e7c3a9fd0d674d9cf129b0bf5d --- ansible/group_vars/all.yml | 5 ++ ansible/library/kolla_docker.py | 15 +++- ansible/roles/nova/defaults/main.yml | 25 ++++++ ansible/roles/nova/handlers/main.yml | 12 +++ doc/source/reference/index.rst | 1 + doc/source/reference/resource-constraints.rst | 84 +++++++++++++++++++ etc/kolla/globals.yml | 22 +++++ tests/test_kolla_docker.py | 29 ++++++- 8 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 doc/source/reference/resource-constraints.rst diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index bc4302e29a..49af58ea41 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -108,6 +108,11 @@ docker_common_options: restart_policy: "{{ docker_restart_policy }}" restart_retries: "{{ docker_restart_policy_retry }}" +#################### +# Dimensions options +#################### +# Dimension options for Docker Containers +default_container_dimensions: {} #################### # keepalived options diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py index f736dd7bf6..4d4fe73145 100644 --- a/ansible/library/kolla_docker.py +++ b/ansible/library/kolla_docker.py @@ -552,6 +552,18 @@ class DockerWorker(object): 'volumes_from': self.params.get('volumes_from') } + if self.params.get('dimensions'): + supported = {'cpu_period', 'cpu_quota', 'cpu_shares', + 'cpuset_cpus', 'cpuset_mems', 'mem_limit', + 'mem_reservation', 'memswap_limit', + 'kernel_memory', 'blkio_weight'} + unsupported = set(self.params.get('dimensions')) - supported + if unsupported: + self.module.exit_json(failed=True, + msg=repr("Unsupported dimensions"), + unsupported_dimensions=unsupported) + options.update(self.params.get('dimensions')) + if self.params.get('restart_policy') in ['on-failure', 'always', 'unless-stopped']: @@ -772,7 +784,8 @@ def generate_module(): tls_key=dict(required=False, type='str'), tls_cacert=dict(required=False, type='str'), volumes=dict(required=False, type='list'), - volumes_from=dict(required=False, type='list') + volumes_from=dict(required=False, type='list'), + dimensions=dict(required=False, type='dict', default=dict()) ) required_if = [ ['action', 'pull_image', ['image']], diff --git a/ansible/roles/nova/defaults/main.yml b/ansible/roles/nova/defaults/main.yml index 8a22cbffdf..92a8625238 100644 --- a/ansible/roles/nova/defaults/main.yml +++ b/ansible/roles/nova/defaults/main.yml @@ -21,6 +21,7 @@ nova_services: - "nova_compute:/var/lib/nova/" - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}" - "nova_libvirt_qemu:/etc/libvirt/qemu" + dimensions: "{{ nova_libvirt_dimensions }}" nova-ssh: container_name: "nova_ssh" group: "compute" @@ -32,6 +33,7 @@ nova_services: - "kolla_logs:/var/log/kolla" - "nova_compute:/var/lib/nova" - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}" + dimensions: "{{ nova_ssh_dimensions }}" placement-api: container_name: "placement_api" group: "placement-api" @@ -41,6 +43,7 @@ nova_services: - "{{ node_config_directory }}/placement-api/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ placement_api_dimensions }}" nova-api: container_name: "nova_api" group: "nova-api" @@ -52,6 +55,7 @@ nova_services: - "/etc/localtime:/etc/localtime:ro" - "/lib/modules:/lib/modules:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_api_dimensions }}" nova-consoleauth: container_name: "nova_consoleauth" group: "nova-consoleauth" @@ -61,6 +65,7 @@ nova_services: - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_consoleauth_dimensions }}" nova-novncproxy: container_name: "nova_novncproxy" group: "nova-novncproxy" @@ -70,6 +75,7 @@ nova_services: - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_novncproxy_dimensions }}" nova-scheduler: container_name: "nova_scheduler" group: "nova-scheduler" @@ -79,6 +85,7 @@ nova_services: - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_scheduler_dimensions }}" nova-spicehtml5proxy: container_name: "nova_spicehtml5proxy" group: "nova-spicehtml5proxy" @@ -88,6 +95,7 @@ nova_services: - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_spicehtml5proxy_dimensions }}" nova-serialproxy: container_name: "nova_serialproxy" group: "nova-serialproxy" @@ -97,6 +105,7 @@ nova_services: - "{{ node_config_directory }}/nova-serialproxy/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_serialproxy_dimensions }}" nova-conductor: container_name: "nova_conductor" group: "nova-conductor" @@ -106,6 +115,7 @@ nova_services: - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_conductor_dimensions }}" nova-compute: container_name: "nova_compute" group: "compute" @@ -126,6 +136,7 @@ nova_services: - "libvirtd:/var/lib/libvirt" - "nova_compute:/var/lib/nova/" - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}" + dimensions: "{{ nova_compute_dimensions }}" nova-compute-ironic: container_name: "nova_compute_ironic" group: "nova-compute-ironic" @@ -135,6 +146,7 @@ nova_services: - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla/" + dimensions: "{{ nova_compute_ironic_dimensions }}" #################### # Ceph @@ -232,6 +244,19 @@ placement_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d placement_api_tag: "{{ nova_tag }}" placement_api_image_full: "{{ placement_api_image }}:{{ placement_api_tag }}" +nova_libvirt_dimensions: "{{ default_container_dimensions }}" +nova_ssh_dimensions: "{{ default_container_dimensions }}" +placement_api_dimensions: "{{ default_container_dimensions }}" +nova_api_dimensions: "{{ default_container_dimensions }}" +nova_consoleauth_dimensions: "{{ default_container_dimensions }}" +nova_novncproxy_dimensions: "{{ default_container_dimensions }}" +nova_scheduler_dimensions: "{{ default_container_dimensions }}" +nova_spicehtml5proxy_dimensions: "{{ default_container_dimensions }}" +nova_serialproxy_dimensions: "{{ default_container_dimensions }}" +nova_conductor_dimensions: "{{ default_container_dimensions }}" +nova_compute_dimensions: "{{ default_container_dimensions }}" +nova_compute_ironic_dimensions: "{{ default_container_dimensions }}" + #################### # OpenStack #################### diff --git a/ansible/roles/nova/handlers/main.yml b/ansible/roles/nova/handlers/main.yml index 3e037554da..079c253d72 100644 --- a/ansible/roles/nova/handlers/main.yml +++ b/ansible/roles/nova/handlers/main.yml @@ -14,6 +14,7 @@ pid_mode: "{{ service.pid_mode | default('') }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -37,6 +38,7 @@ pid_mode: "{{ service.pid_mode | default('') }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" register: restart_nova_libvirt # NOTE(Jeffrey4l): retry 5 to remove nova_libvirt container because when # guests running, nova_libvirt will raise error even though it is removed. @@ -65,6 +67,7 @@ name: "{{ service.container_name }}" image: "{{ service.image }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -91,6 +94,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -116,6 +120,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -141,6 +146,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -167,6 +173,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -192,6 +199,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -217,6 +225,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -242,6 +251,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -269,6 +279,7 @@ privileged: "{{ service.privileged | default(False) }}" ipc_mode: "{{ service.ipc_mode | default(omit) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] @@ -295,6 +306,7 @@ image: "{{ service.image }}" privileged: "{{ service.privileged | default(False) }}" volumes: "{{ service.volumes|reject('equalto', '')|list }}" + dimensions: "{{ service.dimensions }}" when: - kolla_action != "config" - inventory_hostname in groups[service.group] diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index b9d041d6df..7427439c8f 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -27,3 +27,4 @@ Projects Deployment References tacker-guide xenserver-guide horizon-guide + resource-constraints diff --git a/doc/source/reference/resource-constraints.rst b/doc/source/reference/resource-constraints.rst new file mode 100644 index 0000000000..875db5bd83 --- /dev/null +++ b/doc/source/reference/resource-constraints.rst @@ -0,0 +1,84 @@ +.. _resource-constraints: + +===================================== +Resource Constraints in Kolla Ansible +===================================== + +Overview +~~~~~~~~ + +Since the Rocky release it is possible to restrict +the resource usage of deployed containers. +The following components support this feature: + +* Nova + +In Kolla Ansible, +container resources to be constrained are referred to as dimensions. + +The `Docker documentation `__ +provides information on container resource constraints. +The resources currently supported by Kolla Ansible are: + +.. code-block:: console + + cpu_period + cpu_quota + cpu_shares + cpuset_cpus + cpuset_mems + mem_limit + mem_reservation + memswap_limit + kernel_memory + blkio_weight + +.. end + +Pre-deployment Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Dimensions are defined as a mapping from a Docker resource name +(e.g. ``cpu_period``) to a string constraint value. + +The variable ``default_container_dimensions`` sets the default dimensions +for all supported containers, and by default these are unconstrained. + +Each supported container has an associated variable, +``_dimensions``, that can be used to set the resources +for the container. For example, dimensions for the ``nova_libvirt`` +container are set via the variable ``nova_libvirt_dimensions``. + +For example, +to constrain the number of CPUs that may be used by all supported containers, +add the following to the dimensions options section in +``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + default_container_dimensions: + cpuset_cpus: "1" + +.. end + +For example, to constrain the number of CPUs that may be used by +the ``nova_libvirt`` container, add the following to the dimensions +options section in ``/etc/kolla/globals.yml``: + +.. code-block:: yaml + + nova_libvirt_dimensions: + cpuset_cpus: "2" + +.. end + +Deployment +~~~~~~~~~~ + +To deploy resource constrained containers, run the deployment as usual: + +.. code-block:: console + + $ kolla-ansible deploy -i /path/to/inventory + +.. end diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 2e9200a2b4..571eca4b0f 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -113,6 +113,28 @@ kolla_internal_vip_address: "10.10.10.254" # Arbitrary unique number from 0..255 #keepalived_virtual_router_id: "51" +################### +# Dimension options +################### +# This is to provide an extra option to deploy containers with Resource constraints. +# We call it dimensions here. +# The dimensions for each container are defined by a mapping, where each dimension value should be a +# string. +# Reference_Docs +# https://docs.docker.com/config/containers/resource_constraints/ +# eg: +# _dimensions: +# blkio_weight: +# cpu_period: +# cpu_quota: +# cpu_shares: +# cpuset_cpus: +# cpuset_mems: +# mem_limit: +# mem_reservation: +# memswap_limit: +# kernel_memory: + ############# # TLS options diff --git a/tests/test_kolla_docker.py b/tests/test_kolla_docker.py index f0ea08fdb7..7029856eb0 100644 --- a/tests/test_kolla_docker.py +++ b/tests/test_kolla_docker.py @@ -83,7 +83,8 @@ class ModuleArgsTest(base.BaseTestCase): tls_key=dict(required=False, type='str'), tls_cacert=dict(required=False, type='str'), volumes=dict(required=False, type='list'), - volumes_from=dict(required=False, type='list') + volumes_from=dict(required=False, type='list'), + dimensions=dict(required=False, type='dict', default=dict()) ) required_if = [ ['action', 'pull_image', ['image']], @@ -191,14 +192,38 @@ class TestContainer(base.BaseTestCase): super(TestContainer, self).setUp() self.fake_data = copy.deepcopy(FAKE_DATA) - def test_create_container(self): + def test_create_container_without_dimensions(self): self.dw = get_DockerWorker(self.fake_data['params']) self.dw.dc.create_host_config = mock.MagicMock( return_value=self.fake_data['params']['host_config']) self.dw.create_container() self.assertTrue(self.dw.changed) + + def test_create_container_with_dimensions(self): + self.fake_data['params']['dimensions'] = {'blkio_weight': 10} + self.dw = get_DockerWorker(self.fake_data['params']) + self.dw.dc.create_host_config = mock.MagicMock( + return_value=self.fake_data['params']['host_config']) + self.dw.create_container() + self.assertTrue(self.dw.changed) + self.fake_data['params'].pop('dimensions') + self.fake_data['params']['host_config']['blkio_weight'] = '10' self.dw.dc.create_container.assert_called_once_with( **self.fake_data['params']) + self.dw.dc.create_host_config.assert_called_with( + cap_add=None, network_mode='host', ipc_mode=None, + pid_mode=None, volumes_from=None, blkio_weight=10, + security_opt=None, privileged=None) + + def test_create_container_wrong_dimensions(self): + self.fake_data['params']['dimensions'] = {'random': 10} + self.dw = get_DockerWorker(self.fake_data['params']) + self.dw.dc.create_host_config = mock.MagicMock( + return_value=self.fake_data['params']['host_config']) + self.dw.create_container() + self.dw.module.exit_json.assert_called_once_with( + failed=True, msg=repr("Unsupported dimensions"), + unsupported_dimensions=set(['random'])) def test_start_container_without_pull(self): self.fake_data['params'].update({'auth_username': 'fake_user',