Catch OSError in processutils

Some gate failures showed that subprocess.Popen.communicate() may raise
an OSError where the proper response is to just call it again.  This
patch adds that handling.

Change-Id: Ibca16d2141ca3aacacc73062757c97a3220bd02d
Closes-bug: #1271331

openstack/common/processutils.py

@@ -17,6 +17,7 @@
 System-level utilities and helper functions.
 """
 
+import errno
 import logging as stdlib_logging
 import os
 import random
@@ -25,6 +26,7 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+import six
 
 from openstack.common.gettextutils import _
 from openstack.common import log as logging
@@ -167,10 +169,19 @@ def execute(*cmd, **kwargs):
                                    preexec_fn=preexec_fn,
                                    shell=shell)
             result = None
-            if process_input is not None:
-                result = obj.communicate(process_input)
-            else:
-                result = obj.communicate()
+            for _i in six.moves.range(20):
+                # NOTE(russellb) 20 is an arbitrary number of retries to
+                # prevent any chance of looping forever here.
+                try:
+                    if process_input is not None:
+                        result = obj.communicate(process_input)
+                    else:
+                        result = obj.communicate()
+                except OSError as e:
+                    if e.errno in (errno.EAGAIN, errno.EINTR):
+                        continue
+                    raise
+                break
             obj.stdin.close()  # pylint: disable=E1101
             _returncode = obj.returncode  # pylint: disable=E1101
             LOG.log(loglevel, _('Result was %s') % _returncode)

tests/unit/test_processutils.py

@@ -15,6 +15,7 @@
 
 from __future__ import print_function
 
+import errno
 import os
 import tempfile
 
@@ -179,6 +180,24 @@ grep foo
             os.unlink(tmpfilename)
             os.unlink(tmpfilename2)
 
+    def test_retry_on_communicate_error(self):
+        self.called = False
+
+        def fake_communicate(*args, **kwargs):
+            if self.called:
+                return ('', '')
+            self.called = True
+            e = OSError('foo')
+            e.errno = errno.EAGAIN
+            raise e
+
+        self.useFixture(fixtures.MonkeyPatch(
+            'subprocess.Popen.communicate', fake_communicate))
+
+        processutils.execute('/usr/bin/env', 'true', check_exit_code=False)
+
+        self.assertTrue(self.called)
+
 
 def fake_execute(*cmd, **kwargs):
     return 'stdout', 'stderr'