diff --git a/paunch/runner.py b/paunch/runner.py
index b896ef1..722fddd 100644
--- a/paunch/runner.py
+++ b/paunch/runner.py
@@ -284,7 +284,8 @@ class BaseRunner(object):
     def remove_container(self, container):
         if self.cont_cmd == 'podman':
             systemd.service_delete(container=container, log=self.log)
-        cmd = [self.cont_cmd, 'rm', '-f', container]
+        self.execute([self.cont_cmd, 'stop', container], self.log)
+        cmd = [self.cont_cmd, 'rm', container]
         cmd_stdout, cmd_stderr, returncode = self.execute(cmd, self.log)
         if returncode != 0:
             self.log.error('Error removing container: %s' % container)
diff --git a/paunch/tests/test_builder_base.py b/paunch/tests/test_builder_base.py
index 897f565..4b3e38f 100644
--- a/paunch/tests/test_builder_base.py
+++ b/paunch/tests/test_builder_base.py
@@ -246,12 +246,18 @@ class TestBaseBuilder(base.TestCase):
 six six
 two-12345678 two
 three-12345678 three''', '', 0),
+            # stop five
+            ('', '', 0),
             # rm five
             ('', '', 0),
+            # stop six
+            ('', '', 0),
             # rm six
             ('', '', 0),
             # inspect two
             ('{"start_order": 1, "image": "centos:6"}', '', 0),
+            # stop two, changed config data
+            ('', '', 0),
             # rm two, changed config data
             ('', '', 0),
             # inspect three
@@ -297,13 +303,16 @@ three-12345678 three''', '', 0),
                 mock.ANY
             ),
             # rm containers not in config
-            mock.call(['docker', 'rm', '-f', 'five'], mock.ANY),
-            mock.call(['docker', 'rm', '-f', 'six'], mock.ANY),
+            mock.call(['docker', 'stop', 'five'], mock.ANY),
+            mock.call(['docker', 'rm', 'five'], mock.ANY),
+            mock.call(['docker', 'stop', 'six'], mock.ANY),
+            mock.call(['docker', 'rm', 'six'], mock.ANY),
             # rm two, changed config
             mock.call(['docker', 'inspect', '--type', 'container',
                        '--format', '{{index .Config.Labels "config_data"}}',
                        'two-12345678'], mock.ANY, False),
-            mock.call(['docker', 'rm', '-f', 'two-12345678'], mock.ANY),
+            mock.call(['docker', 'stop', 'two-12345678'], mock.ANY),
+            mock.call(['docker', 'rm', 'two-12345678'], mock.ANY),
             # check three, config hasn't changed
             mock.call(['docker', 'inspect', '--type', 'container',
                        '--format', '{{index .Config.Labels "config_data"}}',
diff --git a/paunch/tests/test_runner.py b/paunch/tests/test_runner.py
index e0862d4..2fc5af8 100644
--- a/paunch/tests/test_runner.py
+++ b/paunch/tests/test_runner.py
@@ -269,7 +269,7 @@ class TestBaseRunner(base.TestCase):
 
         self.runner.remove_container('one')
         self.assert_execute(
-            popen, ['docker', 'rm', '-f', 'one']
+            popen, ['docker', 'rm', 'one']
         )
 
     @mock.patch('subprocess.Popen')