diff --git a/connection/ssh.py b/connection/ssh.py
index 1931e60..547d431 100644
--- a/connection/ssh.py
+++ b/connection/ssh.py
@@ -61,7 +61,9 @@ class Connection(SSH.Connection):
 
     def set_host_overrides(self, host, hostvars=None):
         if self._container_check() or self._chroot_check():
-            self.host = self._play_context.remote_addr = hostvars.get('physical_host_addr')
+            physical_host_addr = hostvars.get('physical_host_addr')
+            if physical_host_addr:
+                self.host = self._play_context.remote_addr = physical_host_addr
 
     def exec_command(self, cmd, in_data=None, sudoable=True):
         """run a command on the remote host."""