From eb3d3b75bb8bf300fb52390610a6a53ddbfaa48e Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Wed, 12 Feb 2020 10:21:38 -0500 Subject: [PATCH] Cleanup containers in the same loop as they are created Split delete_missing_and_updated() into 2 methods: * delete_missing(), that will remove all containers installed on the host but missing from the given config. This runs outside of the loop, once. * delete_updated(), that will remove a container installed on the host (if present), that is part of the config, if config_data changed or didn't exist. It runs within the create loop, so the downtime between a container removal and creation should be shorter than before. * make delete_missing(), delete_updated() and rename_containers() returning True, if any container has been touched by either. Use that flag in order to keep the container_names contents always actual. * in order to make that cached container_names working and saving off extra podman ps/inspect calls, rework it to return a list instead of an iterator. There is no huge lists of containers, iterators buy us nothing here, while podman CLI calls are the more expensive thing and we optimize the latter instead. Co-Authored-By: Bogdan Dobrelya Change-Id: I3c6d0670e11d035287d12f4207489a13e0891943 Closes-Bug: #1862954 --- paunch/builder/base.py | 73 ++++++---- paunch/runner.py | 7 +- paunch/tests/test_builder_base.py | 213 ++++++++++++------------------ 3 files changed, 142 insertions(+), 151 deletions(-) diff --git a/paunch/builder/base.py b/paunch/builder/base.py index eb3a2a0..dae67b4 100644 --- a/paunch/builder/base.py +++ b/paunch/builder/base.py @@ -12,6 +12,7 @@ # import distutils.spawn +import itertools import json import os import re @@ -50,16 +51,30 @@ class BaseBuilder(object): if pull_returncode != 0: return stdout, stderr, pull_returncode - self.delete_missing_and_updated() - deploy_status_code = 0 key_fltr = lambda k: self.config[k].get('start_order', 0) failed_containers = [] container_names = self.runner.container_names(self.config_id) + + # Cleanup containers missing from the config. + # Also applying new containers configs is an opportunity for + # renames to their preferred names. + changed = self.delete_missing(container_names) + changed |= self.runner.rename_containers() + if changed: + # If anything has been changed, refresh the container_names + container_names = self.runner.container_names(self.config_id) desired_names = set([cn[-1] for cn in container_names]) for container in sorted(self.config, key=key_fltr): + # Before creating the container, figure out if it needs to be + # removed because of its configuration has changed. + # If anything has been deleted, refresh the container_names/desired + if self.delete_updated(container, container_names): + container_names = self.runner.container_names(self.config_id) + desired_names = set([cn[-1] for cn in container_names]) + self.log.debug("Running container: %s" % container) cconfig = self.config[container] action = cconfig.get('action', 'run') @@ -172,44 +187,52 @@ class BaseBuilder(object): return stdout, stderr, deploy_status_code - def delete_missing_and_updated(self): - container_names = self.runner.container_names(self.config_id) + def delete_missing(self, container_names): + deleted = False for cn in container_names: container = cn[0] - # if the desired name is not in the config, delete it if cn[-1] not in self.config: if self.cleanup: self.log.debug("Deleting container (removed): " "%s" % container) self.runner.remove_container(container) + deleted = True else: self.log.debug("Skipping container (cleanup disabled): " "%s" % container) - continue + return deleted - ex_data_str = self.runner.inspect( - container, '{{index .Config.Labels "config_data"}}') - if not ex_data_str: - self.log.debug("Deleting container (no config_data): %s" - % container) + def delete_updated(self, container, container_names): + # If a container is not deployed, there is nothing to delete + if (container not in + list(itertools.chain.from_iterable(container_names))): + return False + + ex_data_str = self.runner.inspect( + container, '{{index .Config.Labels "config_data"}}') + if not ex_data_str: + if self.cleanup: + self.log.debug("Deleting container (no_config_data): " + "%s" % container) self.runner.remove_container(container) - continue + return True + else: + self.log.debug("Skipping container (cleanup disabled): " + "%s" % container) - try: - ex_data = yaml.safe_load(str(ex_data_str)) - except Exception: - ex_data = None + try: + ex_data = yaml.safe_load(str(ex_data_str)) + except Exception: + ex_data = None - new_data = self.config.get(cn[-1]) - if new_data != ex_data: - self.log.debug("Deleting container (changed config_data): %s" - % container) - self.runner.remove_container(container) - - # deleting containers is an opportunity for renames to their - # preferred name - self.runner.rename_containers() + new_data = self.config[container] + if new_data != ex_data: + self.log.debug("Deleting container (changed config_data): %s" + % container) + self.runner.remove_container(container) + return True + return False def label_arguments(self, cmd, container): if self.labels: diff --git a/paunch/runner.py b/paunch/runner.py index 99a4a3a..078a633 100644 --- a/paunch/runner.py +++ b/paunch/runner.py @@ -272,9 +272,11 @@ class BaseRunner(object): if returncode != 0: return results += cmd_stdout.split("\n") + result = [] for line in results: if line: - yield line.split() + result.append(line.split()) + return result def remove_containers(self, conf_id): for container in self.containers_in_config(conf_id): @@ -301,6 +303,7 @@ class BaseRunner(object): def rename_containers(self): current_containers = [] need_renaming = {} + renamed = False for entry in self.container_names(): current_containers.append(entry[0]) @@ -321,7 +324,9 @@ class BaseRunner(object): else: self.log.info('Renaming "%s" to "%s"' % (current, desired)) self.rename_container(current, desired) + renamed = True current_containers.append(desired) + return renamed def validate_volume_source(self, volume): """Validate that the provided volume diff --git a/paunch/tests/test_builder_base.py b/paunch/tests/test_builder_base.py index 40ee804..51df837 100644 --- a/paunch/tests/test_builder_base.py +++ b/paunch/tests/test_builder_base.py @@ -27,7 +27,9 @@ from paunch.tests import base class TestBaseBuilder(base.TestCase): @mock.patch("psutil.Process.cpu_affinity", return_value=[0, 1, 2, 3]) - def test_apply(self, mock_cpu): + @mock.patch("paunch.builder.base.BaseBuilder.delete_updated", + return_value=False) + def test_apply(self, mock_delete_updated, mock_cpu): orig_call = tenacity.wait.wait_random_exponential.__call__ orig_argspec = inspect.getargspec(orig_call) config = { @@ -61,17 +63,25 @@ class TestBaseBuilder(base.TestCase): ('', '', 1), # inspect for missing image centos:7 ('Pulled centos:7', 'ouch', 1), # pull centos:6 fails ('Pulled centos:7', '', 0), # pull centos:6 succeeds - ('', '', 0), # ps for delete_missing_and_updated container_names - ('', '', 0), # ps2 for delete_missing_and_updated container_names - ('', '', 0), # ps for after delete_missing_and_updated renames - ('', '', 0), # ps2 for after delete_missing_and_updated renames - ('', '', 0), # ps to only create containers which don't exist - ('', '', 0), # ps2 to only create containers which don't exist + # container_names for delete_missing (twice by managed_by) + ('', '', 0), + ('''five five +six six +two two +three-12345678 three''', '', 0), + ('', '', 0), # stop five + ('', '', 0), # rm five + ('', '', 0), # stop six + ('', '', 0), # rm six + # container_names for rename_containers + ('three-12345678 three', '', 0), + ('', '', 0), # rename three + # desired/container_names to be refreshed after delete/rename + ('three three', '', 0), # renamed three already exists ('Created one-12345678', '', 0), ('Created two-12345678', '', 0), - ('Created three-12345678', '', 0), ('Created four-12345678', '', 0), - ('a\nb\nc', '', 0) + ('a\nb\nc', '', 0) # exec four ] r.discover_container_name = lambda n, c: '%s-12345678' % n r.unique_container_name = lambda n: '%s-12345678' % n @@ -89,7 +99,6 @@ class TestBaseBuilder(base.TestCase): 'Pulled centos:7', 'Created one-12345678', 'Created two-12345678', - 'Created three-12345678', 'Created four-12345678', 'a\nb\nc' ], stdout) @@ -114,7 +123,7 @@ class TestBaseBuilder(base.TestCase): mock.call( ['docker', 'pull', 'centos:7'], mock.ANY ), - # ps for delete_missing_and_updated container_names + # container_names for delete_missing mock.call( ['docker', 'ps', '-a', '--filter', 'label=managed_by=tester', @@ -122,7 +131,6 @@ class TestBaseBuilder(base.TestCase): '--format', '{{.Names}} {{.Label "container_name"}}'], mock.ANY ), - # ps2 for delete_missing_and_updated container_names mock.call( ['docker', 'ps', '-a', '--filter', 'label=managed_by=paunch', @@ -130,21 +138,22 @@ class TestBaseBuilder(base.TestCase): '--format', '{{.Names}} {{.Label "container_name"}}'], mock.ANY ), - # ps for after delete_missing_and_updated renames + # rm containers missing in config + mock.call(['docker', 'stop', 'five'], mock.ANY), + mock.call(['docker', 'rm', 'five'], mock.ANY), + mock.call(['docker', 'stop', 'six'], mock.ANY), + mock.call(['docker', 'rm', 'six'], mock.ANY), + # container_names for rename mock.call( ['docker', 'ps', '-a', '--filter', 'label=managed_by=tester', '--format', '{{.Names}} {{.Label "container_name"}}'], mock.ANY ), - # ps2 for after delete_missing_and_updated renames - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=paunch', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), - # ps to only create containers which don't exist + # rename three from an ephemeral to the static name + mock.call(['docker', 'rename', 'three-12345678', 'three'], + mock.ANY), + # container_names to be refreshed after delete/rename mock.call( ['docker', 'ps', '-a', '--filter', 'label=managed_by=tester', @@ -152,14 +161,6 @@ class TestBaseBuilder(base.TestCase): '--format', '{{.Names}} {{.Label "container_name"}}'], mock.ANY ), - # ps2 to only create containers which don't exist - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=paunch', - '--filter', 'label=config_id=foo', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), # run one mock.call( ['docker', 'run', '--name', 'one-12345678', @@ -180,16 +181,6 @@ class TestBaseBuilder(base.TestCase): '--detach=true', '--cpuset-cpus=0,1,2,3', 'centos:7'], mock.ANY ), - # run three - mock.call( - ['docker', 'run', '--name', 'three-12345678', - '--label', 'config_id=foo', - '--label', 'container_name=three', - '--label', 'managed_by=tester', - '--label', 'config_data=%s' % json.dumps(config['three']), - '--detach=true', '--cpuset-cpus=0,1,2,3', - 'centos:6'], mock.ANY - ), # run four mock.call( ['docker', 'run', '--name', 'four-12345678', @@ -208,19 +199,22 @@ class TestBaseBuilder(base.TestCase): ]) @mock.patch("psutil.Process.cpu_affinity", return_value=[0, 1, 2, 3]) - def test_apply_idempotency(self, mock_cpu): + @mock.patch("paunch.runner.BaseRunner.container_names") + @mock.patch("paunch.runner.BaseRunner.discover_container_name", + return_value='one') + def test_apply_idempotency(self, mock_dname, mock_cnames, mock_cpu): config = { - # not running yet + # running with the same config and given an ephemeral name 'one': { 'start_order': 0, 'image': 'centos:7', }, - # running, but with a different config + # not running yet 'two': { 'start_order': 1, 'image': 'centos:7', }, - # running with the same config + # running, but with a different config 'three': { 'start_order': 2, 'image': 'centos:7', @@ -230,49 +224,46 @@ class TestBaseBuilder(base.TestCase): 'start_order': 10, 'image': 'centos:7', }, - 'four_ls': { + 'one_ls': { 'action': 'exec', 'start_order': 20, - 'command': ['four', 'ls', '-l', '/'] + 'command': ['one', 'ls', '-l', '/'] } + # five is running but is not managed by us } - + # represents the state before and after renaming/removing things + mock_cnames.side_effect = ( + # delete_missing + [['five', 'five'], ['one-12345678', 'one'], ['three', 'three']], + # rename_containers + [['one-12345678', 'one']], + # refresh container_names/desired after del/rename + [['one', 'one'], ['three', 'three']], + # refresh container_names/desired after delete_updated + [['one', 'one']] + ) r = runner.DockerRunner(managed_by='tester', cont_cmd='docker') exe = mock.Mock() exe.side_effect = [ # inspect for image centos:7 ('exists', '', 0), - # ps for delete_missing_and_updated container_names - ('''five five -six six -two-12345678 two -three-12345678 three''', '', 0), # stop five ('', '', 0), # rm five ('', '', 0), - # stop six - ('', '', 0), - # rm six - ('', '', 0), - # inspect two - ('{"start_order": 1, "image": "centos:6"}', '', 0), - # stop two, changed config data - ('', '', 0), - # rm two, changed config data - ('', '', 0), - # inspect three - ('{"start_order": 2, "image": "centos:7"}', '', 0), - # ps for after delete_missing_and_updated renames - ('', '', 0), - # ps2 for after delete_missing_and_updated renames - ('', '', 0), - # ps to only create containers which don't exist - ('three-12345678 three', '', 0), - ('Created one-12345678', '', 0), + ('', '', 0), # ps for rename one + # inspect one + ('{"start_order": 0, "image": "centos:7"}', '', 0), ('Created two-12345678', '', 0), + # inspect three + ('{"start_order": 42, "image": "centos:7"}', '', 0), + # stop three, changed config data + ('', '', 0), + # rm three, changed config data + ('', '', 0), + ('Created three-12345678', '', 0), ('Created four-12345678', '', 0), - ('a\nb\nc', '', 0) + ('a\nb\nc', '', 0) # exec one ] r.discover_container_name = lambda n, c: '%s-12345678' % n r.unique_container_name = lambda n: '%s-12345678' % n @@ -282,8 +273,8 @@ three-12345678 three''', '', 0), stdout, stderr, deploy_status_code = builder.apply() self.assertEqual(0, deploy_status_code) self.assertEqual([ - 'Created one-12345678', 'Created two-12345678', + 'Created three-12345678', 'Created four-12345678', 'a\nb\nc' ], stdout) @@ -295,61 +286,17 @@ three-12345678 three''', '', 0), ['docker', 'inspect', '--type', 'image', '--format', 'exists', 'centos:7'], mock.ANY, False ), - # ps for delete_missing_and_updated container_names - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=tester', - '--filter', 'label=config_id=foo', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), # rm containers not in config mock.call(['docker', 'stop', 'five'], mock.ANY), mock.call(['docker', 'rm', 'five'], mock.ANY), - mock.call(['docker', 'stop', 'six'], mock.ANY), - mock.call(['docker', 'rm', 'six'], mock.ANY), - # rm two, changed config + # rename one from an ephemeral to the static name + mock.call(['docker', 'rename', 'one-12345678', 'one'], + mock.ANY), + # check the renamed one, config hasn't changed mock.call(['docker', 'inspect', '--type', 'container', '--format', '{{index .Config.Labels "config_data"}}', - 'two-12345678'], mock.ANY, False), - mock.call(['docker', 'stop', 'two-12345678'], mock.ANY), - mock.call(['docker', 'rm', 'two-12345678'], mock.ANY), - # check three, config hasn't changed - mock.call(['docker', 'inspect', '--type', 'container', - '--format', '{{index .Config.Labels "config_data"}}', - 'three-12345678'], mock.ANY, False), - # ps for after delete_missing_and_updated renames - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=tester', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), - # ps2 for after delete_missing_and_updated renames - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=paunch', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), - # ps to only create containers which don't exist - mock.call( - ['docker', 'ps', '-a', - '--filter', 'label=managed_by=tester', - '--filter', 'label=config_id=foo', - '--format', '{{.Names}} {{.Label "container_name"}}'], - mock.ANY - ), - # run one - mock.call( - ['docker', 'run', '--name', 'one-12345678', - '--label', 'config_id=foo', - '--label', 'container_name=one', - '--label', 'managed_by=tester', - '--label', 'config_data=%s' % json.dumps(config['one']), - '--detach=true', '--cpuset-cpus=0,1,2,3', - 'centos:7'], mock.ANY - ), + 'one'], mock.ANY, False), + # don't run one, its already running # run two mock.call( ['docker', 'run', '--name', 'two-12345678', @@ -360,7 +307,22 @@ three-12345678 three''', '', 0), '--detach=true', '--cpuset-cpus=0,1,2,3', 'centos:7'], mock.ANY ), - # don't run three, its already running + # rm three, changed config + mock.call(['docker', 'inspect', '--type', 'container', + '--format', '{{index .Config.Labels "config_data"}}', + 'three'], mock.ANY, False), + mock.call(['docker', 'stop', 'three'], mock.ANY), + mock.call(['docker', 'rm', 'three'], mock.ANY), + # run three + mock.call( + ['docker', 'run', '--name', 'three-12345678', + '--label', 'config_id=foo', + '--label', 'container_name=three', + '--label', 'managed_by=tester', + '--label', 'config_data=%s' % json.dumps(config['three']), + '--detach=true', '--cpuset-cpus=0,1,2,3', + 'centos:7'], mock.ANY + ), # run four mock.call( ['docker', 'run', '--name', 'four-12345678', @@ -371,9 +333,10 @@ three-12345678 three''', '', 0), '--detach=true', '--cpuset-cpus=0,1,2,3', 'centos:7'], mock.ANY ), - # execute within four + # FIXME(bogdando): shall exec ls in the renamed one! + # Why discover_container_name is never called to get it as c_name? mock.call( - ['docker', 'exec', 'four-12345678', 'ls', '-l', + ['docker', 'exec', 'one-12345678', 'ls', '-l', '/'], mock.ANY ), ])