diff --git a/kuryr_kubernetes/tests/unit/test_watcher.py b/kuryr_kubernetes/tests/unit/test_watcher.py
index 97edbb4fd..b65ff03e4 100644
--- a/kuryr_kubernetes/tests/unit/test_watcher.py
+++ b/kuryr_kubernetes/tests/unit/test_watcher.py
@@ -332,3 +332,14 @@ class TestWatcher(test_base.TestCase):
 
         m_handler.assert_has_calls([mock.call(e) for e in events])
         m_sys_exit.assert_called_once_with(1)
+
+    def test_watch_restart(self):
+        tg = mock.Mock()
+        w = watcher.Watcher(lambda e: None, tg)
+        w.add('/test')
+        w.start()
+        tg.add_thread.assert_called_once_with(mock.ANY, '/test')
+        w.stop()
+        tg.add_thread = mock.Mock()  # Reset mock.
+        w.start()
+        tg.add_thread.assert_called_once_with(mock.ANY, '/test')
diff --git a/kuryr_kubernetes/watcher.py b/kuryr_kubernetes/watcher.py
index 36c2af8c5..5be3ddafe 100644
--- a/kuryr_kubernetes/watcher.py
+++ b/kuryr_kubernetes/watcher.py
@@ -144,14 +144,17 @@ class Watcher(health.HealthHandler):
 
     def _stop_watch(self, path):
         if self._idle.get(path):
-            if self._thread_group:
+            if self._thread_group and path in self._watching:
                 self._watching[path].stop()
+                # NOTE(dulek): Thread gets killed immediately, so we need to
+                # take care of this ourselves.
+                self._watching.pop(path, None)
+                self._idle.pop(path, None)
 
     def _graceful_watch_exit(self, path):
         try:
-            self.remove(path)
-            self._watching.pop(path)
-            self._idle.pop(path)
+            self._watching.pop(path, None)
+            self._idle.pop(path, None)
             LOG.info("Stopped watching '%s'", path)
         except KeyError:
             LOG.error("Failed to exit watch gracefully")