Fix bug #18420 with deadlocks communicating with subprocess on MS-Windows.

 src/w32.c (fcntl): Support O_NONBLOCK fcntl on the write side of
 pipes.
 (sys_write): When a write to a non-blocking pipe returns ENOSPC,
 set errno to EAGAIN instead, to allow the caller to retry the
 write after some waiting.  Fixes deadlocks when Emacs exchanges a
 lot of data through the pipe.

2 files changed, 55 insertions, 8 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 4f851edb0fd..c32b4c44988 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,12 @@
+2014-09-14  Eli Zaretskii  <eliz@gnu.org>
+
+	* w32.c (fcntl): Support O_NONBLOCK fcntl on the write side of
+	pipes.
+	(sys_write): When a write to a non-blocking pipe returns ENOSPC,
+	set errno to EAGAIN instead, to allow the caller to retry the
+	write after some waiting.  Fixes deadlocks when Emacs exchanges a
+	lot of data through the pipe.  (Bug#18420)
+
 2014-09-13  Eli Zaretskii  <eliz@gnu.org>
 
 	* sound.c (Fplay_sound_internal): Encode the sound file name in
diff --git a/src/w32.c b/src/w32.c
index fee1be22739..aba0b5a81f9 100644
--- a/src/w32.c
+++ b/src/w32.c
@@ -7690,15 +7690,15 @@ fcntl (int s, int cmd, int options)
   if (cmd == F_DUPFD_CLOEXEC)
     return sys_dup (s);
 
-  if (winsock_lib == NULL)
-    {
-      errno = ENETDOWN;
-      return -1;
-    }
-
   check_errno ();
   if (fd_info[s].flags & FILE_SOCKET)
     {
+      if (winsock_lib == NULL)
+	{
+	  errno = ENETDOWN;
+	  return -1;
+	}
+
       if (cmd == F_SETFL && options == O_NONBLOCK)
 	{
 	  unsigned long nblock = 1;
@@ -7715,13 +7715,36 @@ fcntl (int s, int cmd, int options)
 	  return SOCKET_ERROR;
 	}
     }
+  else if ((fd_info[s].flags & (FILE_PIPE | FILE_WRITE))
+	   == (FILE_PIPE | FILE_WRITE))
+    {
+      /* Force our writes to pipes be non-blocking.  */
+      if (cmd == F_SETFL && options == O_NONBLOCK)
+	{
+	  HANDLE h = (HANDLE)_get_osfhandle (s);
+	  DWORD pipe_mode = PIPE_NOWAIT;
+
+	  if (!SetNamedPipeHandleState (h, &pipe_mode, NULL, NULL))
+	    {
+	      DebPrint (("SetNamedPipeHandleState: %lu\n", GetLastError ()));
+	      return SOCKET_ERROR;
+	    }
+	  fd_info[s].flags |= FILE_NDELAY;
+	  return 0;
+	}
+      else
+	{
+	  errno = EINVAL;
+	  return SOCKET_ERROR;
+	}
+    }
   errno = ENOTSOCK;
   return SOCKET_ERROR;
 }
 
 
 /* Shadow main io functions: we need to handle pipes and sockets more
-   intelligently, and implement non-blocking mode as well. */
+   intelligently.  */
 
 int
 sys_close (int fd)
@@ -8206,7 +8229,6 @@ sys_read (int fd, char * buffer, unsigned int count)
 /* From w32xfns.c */
 extern HANDLE interrupt_handle;
 
-/* For now, don't bother with a non-blocking mode */
 int
 sys_write (int fd, const void * buffer, unsigned int count)
 {
@@ -8341,6 +8363,22 @@ sys_write (int fd, const void * buffer, unsigned int count)
 	  nchars += n;
 	  if (n < 0)
 	    {
+	      /* When there's no buffer space in a pipe that is in the
+		 non-blocking mode, _write returns ENOSPC.  We return
+		 EAGAIN instead, which should trigger the logic in
+		 send_process that enters waiting loop and calls
+		 wait_reading_process_output to allow process input to
+		 be accepted during the wait.  Those calls to
+		 wait_reading_process_output allow sys_select to
+		 notice when process input becomes available, thus
+		 avoiding deadlock whereby each side of the pipe is
+		 blocked on write, waiting for the other party to read
+		 its end of the pipe.  */
+	      if (errno == ENOSPC
+		  && fd < MAXDESC
+		  && ((fd_info[fd].flags & (FILE_PIPE | FILE_NDELAY))
+		      == (FILE_PIPE | FILE_NDELAY)))
+		errno = EAGAIN;
 	      nchars = n;
 	      break;
 	    }