Fix the handling of out-of-band (OOB) data in a socket.

* fhandler.h (class fhandler_socket_inet): Add variable bool oobinline.
* fhandler_socket_inet.cc (fhandler_socket_inet::fhandler_socket_inet):
  Initialize variable oobinline.
(fhandler_socket_inet::recv_internal): Make the handling of OOB data
  as consistent with POSIX as possible. Add simulation of inline mode
  for OOB data as a workaround for broken winsock behavior.
(fhandler_socket_inet::setsockopt): Ditto.
(fhandler_socket_inet::getsockopt): Ditto.
(fhandler_socket_wsock::ioctl): Fix return value of SIOCATMARK command.
  The return value of SIOCATMARK of winsock is almost opposite to
  expectation.
* fhandler_socket_local.cc (fhandler_socket_local::recv_internal):
  Remove the handling of OOB data from AF_LOCAL domain socket. Operation
  related to OOB data will result in an error like Linux does.
(fhandler_socket_local::sendto): Ditto.
(fhandler_socket_local::sendmsg): Ditto.

This fixes the issue reported in following post.
https://cygwin.com/ml/cygwin/2018-06/msg00143.html

winsup/cygwin/fhandler.h

@@ -684,6 +684,8 @@ class fhandler_socket_wsock: public fhandler_socket
 
 class fhandler_socket_inet: public fhandler_socket_wsock
 {
+ private:
+  bool oobinline; /* True if option SO_OOBINLINE is set */
  protected:
   int af_local_connect () { return 0; }
 

winsup/cygwin/fhandler_socket_inet.cc

@@ -685,7 +685,8 @@ fhandler_socket_wsock::set_socket_handle (SOCKET sock, int af, int type,
 }
 
 fhandler_socket_inet::fhandler_socket_inet () :
-  fhandler_socket_wsock ()
+  fhandler_socket_wsock (),
+  oobinline (false)
 {
 }
 
@@ -1044,10 +1045,11 @@ fhandler_socket_inet::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 {
   ssize_t res = 0;
   DWORD ret = 0, wret;
-  int evt_mask = FD_READ | ((wsamsg->dwFlags & MSG_OOB) ? FD_OOB : 0);
+  int evt_mask = (wsamsg->dwFlags & MSG_OOB) ? FD_OOB : FD_READ;
   LPWSABUF &wsabuf = wsamsg->lpBuffers;
   ULONG &wsacnt = wsamsg->dwBufferCount;
   static NO_COPY LPFN_WSARECVMSG WSARecvMsg;
+  bool read_oob = false;
 
   /* CV 2014-10-26: Do not check for the connect_state at this point.  In
      certain scenarios there's no way to check the connect state reliably.
@@ -1086,12 +1088,64 @@ fhandler_socket_inet::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 	waitall = false;
     }
 
+  /* recv() returns EINVAL if MSG_OOB flag is set in inline mode. */
+  if (oobinline && (wsamsg->dwFlags & MSG_OOB))
+    {
+      set_errno (EINVAL);
+      return SOCKET_ERROR;
+    }
+
+  /* Check whether OOB data is ready or not */
+  if (get_socket_type () == SOCK_STREAM)
+    if ((wsamsg->dwFlags & MSG_OOB) || oobinline)
+      {
+	u_long atmark = 0;
+#ifdef __x86_64__
+	/* SIOCATMARK = _IOR('s',7,u_long) */
+	int err = ::ioctlsocket (get_socket (), _IOR('s',7,u_long), &atmark);
+#else
+	int err = ::ioctlsocket (get_socket (), SIOCATMARK, &atmark);
+#endif
+	if (err)
+	  {
+	    set_winsock_errno ();
+	    return SOCKET_ERROR;
+	  }
+	/* If there is no OOB data, recv() with MSG_OOB returns EINVAL.
+	   Note: The return value of SIOCATMARK in non-inline mode of
+	   winsock is FALSE if OOB data exists, TRUE otherwise. */
+	if (atmark && (wsamsg->dwFlags & MSG_OOB))
+	  {
+	    /* No OOB data */
+	    set_errno (EINVAL);
+	    return SOCKET_ERROR;
+	  }
+	/* Inline mode for out-of-band (OOB) data of winsock is
+	   completely broken. That is, SIOCATMARK always returns
+	   TRUE in inline mode. Due to this problem, application
+	   cannot determine OOB data at all. Therefore the behavior
+	   of a socket with SO_OOBINLINE set is simulated using
+	   a socket with SO_OOBINLINE not set. In this fake inline
+	   mode, the order of the OOB and non-OOB data is not
+	   preserved. OOB data is read before non-OOB data sent
+	   prior to the OOB data.  However, this most likely is
+	   not a problem in most cases. */
+	/* If there is OOB data, read OOB data using MSG_OOB in
+	   fake inline mode. */
+	if (!atmark && oobinline)
+	  {
+	    read_oob = true;
+	    evt_mask = FD_OOB;
+	  }
+      }
+
   /* Note: Don't call WSARecvFrom(MSG_PEEK) without actually having data
      waiting in the buffers, otherwise the event handling gets messed up
      for some reason. */
   while (!(res = wait_for_events (evt_mask | FD_CLOSE, wait_flags))
 	 || saw_shutdown_read ())
     {
+      DWORD dwFlags = wsamsg->dwFlags | (read_oob ? MSG_OOB : 0);
       if (use_recvmsg)
 	res = WSARecvMsg (get_socket (), wsamsg, &wret, NULL, NULL);
       /* This is working around a really weird problem in WinSock.
@@ -1113,11 +1167,11 @@ fhandler_socket_inet::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 	 namelen is a valid pointer while name is NULL.  Both parameters are
 	 ignored for TCP sockets, so this only occurs when using UDP socket. */
       else if (!wsamsg->name || get_socket_type () == SOCK_STREAM)
-	res = WSARecv (get_socket (), wsabuf, wsacnt, &wret, &wsamsg->dwFlags,
+	res = WSARecv (get_socket (), wsabuf, wsacnt, &wret, &dwFlags,
 		       NULL, NULL);
       else
 	res = WSARecvFrom (get_socket (), wsabuf, wsacnt, &wret,
-			   &wsamsg->dwFlags, wsamsg->name, &wsamsg->namelen,
+			   &dwFlags, wsamsg->name, &wsamsg->namelen,
 			   NULL, NULL);
       if (!res)
 	{
@@ -1561,6 +1615,23 @@ fhandler_socket_inet::setsockopt (int level, int optname, const void *optval,
 	    set_errno (EDOM);
 	  return ret;
 
+	case SO_OOBINLINE:
+	  /* Inline mode for out-of-band (OOB) data of winsock is
+	     completely broken. That is, SIOCATMARK always returns
+	     TRUE in inline mode. Due to this problem, application
+	     cannot determine OOB data at all. Therefore the behavior
+	     of a socket with SO_OOBINLINE set is simulated using
+	     a socket with SO_OOBINLINE not set. In this fake inline
+	     mode, the order of the OOB and non-OOB data is not
+	     preserved. OOB data is read before non-OOB data sent
+	     prior to the OOB data.  However, this most likely is
+	     not a problem in most cases. */
+	  /* Here, instead of actually setting inline mode, simply
+	     set the variable oobinline. */
+	  oobinline = *(int *) optval ? true : false;
+	  ignore = true;
+	  break;
+
 	default:
 	  break;
 	}
@@ -1713,6 +1784,10 @@ fhandler_socket_inet::getsockopt (int level, int optname, const void *optval,
 	    return 0;
 	  }
 
+	case SO_OOBINLINE:
+	  *(int *) optval = oobinline ? 1 : 0;
+	  return 0;
+
 	default:
 	  break;
 	}
@@ -1850,6 +1925,17 @@ fhandler_socket_wsock::ioctl (unsigned int cmd, void *p)
 	}
       else
 	res = ::ioctlsocket (get_socket (), cmd, (u_long *) p);
+      /* In winsock, the return value of SIOCATMARK is FALSE if
+	 OOB data exists, TRUE otherwise. This is almost opposite
+	 to expectation. */
+#ifdef __x86_64__
+      /* SIOCATMARK = _IOR('s',7,u_long) */
+      if (cmd == _IOR('s',7,u_long) && !res)
+	*(u_long *)p = !*(u_long *)p;
+#else
+      if (cmd == SIOCATMARK && !res)
+	*(u_long *)p = !*(u_long *)p;
+#endif
       break;
     default:
       res = fhandler_socket::ioctl (cmd, p);

winsup/cygwin/fhandler_socket_local.cc

@@ -1065,7 +1065,7 @@ fhandler_socket_local::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 {
   ssize_t res = 0;
   DWORD ret = 0, wret;
-  int evt_mask = FD_READ | ((wsamsg->dwFlags & MSG_OOB) ? FD_OOB : 0);
+  int evt_mask = FD_READ;
   LPWSABUF &wsabuf = wsamsg->lpBuffers;
   ULONG &wsacnt = wsamsg->dwBufferCount;
   static NO_COPY LPFN_WSARECVMSG WSARecvMsg;
@@ -1080,7 +1080,15 @@ fhandler_socket_local::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 
   DWORD wait_flags = wsamsg->dwFlags;
   bool waitall = !!(wait_flags & MSG_WAITALL);
-  wsamsg->dwFlags &= (MSG_OOB | MSG_PEEK | MSG_DONTROUTE);
+
+  /* Out-of-band data not supported by AF_LOCAL */
+  if (wsamsg->dwFlags & MSG_OOB)
+    {
+      set_errno (EOPNOTSUPP);
+      return SOCKET_ERROR;
+    }
+
+  wsamsg->dwFlags &= (MSG_PEEK | MSG_DONTROUTE);
   if (use_recvmsg)
     {
       if (!WSARecvMsg
@@ -1104,7 +1112,7 @@ fhandler_socket_local::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 	  set_winsock_errno ();
 	  return SOCKET_ERROR;
 	}
-      if (is_nonblocking () || (wsamsg->dwFlags & (MSG_OOB | MSG_PEEK)))
+      if (is_nonblocking () || (wsamsg->dwFlags & MSG_PEEK))
 	waitall = false;
     }
 
@@ -1114,6 +1122,7 @@ fhandler_socket_local::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
   while (!(res = wait_for_events (evt_mask | FD_CLOSE, wait_flags))
 	 || saw_shutdown_read ())
     {
+      DWORD dwFlags = wsamsg->dwFlags;
       if (use_recvmsg)
 	res = WSARecvMsg (get_socket (), wsamsg, &wret, NULL, NULL);
       /* This is working around a really weird problem in WinSock.
@@ -1135,11 +1144,11 @@ fhandler_socket_local::recv_internal (LPWSAMSG wsamsg, bool use_recvmsg)
 	 namelen is a valid pointer while name is NULL.  Both parameters are
 	 ignored for TCP sockets, so this only occurs when using UDP socket. */
       else if (!wsamsg->name || get_socket_type () == SOCK_STREAM)
-	res = WSARecv (get_socket (), wsabuf, wsacnt, &wret, &wsamsg->dwFlags,
+	res = WSARecv (get_socket (), wsabuf, wsacnt, &wret, &dwFlags,
 		       NULL, NULL);
       else
 	res = WSARecvFrom (get_socket (), wsabuf, wsacnt, &wret,
-			   &wsamsg->dwFlags, wsamsg->name, &wsamsg->namelen,
+			   &dwFlags, wsamsg->name, &wsamsg->namelen,
 			   NULL, NULL);
       if (!res)
 	{
@@ -1236,6 +1245,13 @@ fhandler_socket_local::sendto (const void *in_ptr, size_t len, int flags,
   char *ptr = (char *) in_ptr;
   struct sockaddr_storage sst;
 
+  /* Out-of-band data not supported by AF_LOCAL */
+  if (flags & MSG_OOB)
+    {
+      set_errno (EOPNOTSUPP);
+      return SOCKET_ERROR;
+    }
+
   if (to && get_inet_addr_local (to, tolen, &sst, &tolen) == SOCKET_ERROR)
     return SOCKET_ERROR;
 
@@ -1274,6 +1290,13 @@ fhandler_socket_local::sendmsg (const struct msghdr *msg, int flags)
   struct sockaddr_storage sst;
   int len = 0;
 
+  /* Out-of-band data not supported by AF_LOCAL */
+  if (flags & MSG_OOB)
+    {
+      set_errno (EOPNOTSUPP);
+      return SOCKET_ERROR;
+    }
+
   if (msg->msg_name
       && get_inet_addr_local ((struct sockaddr *) msg->msg_name,
 			      msg->msg_namelen, &sst, &len) == SOCKET_ERROR)