Cygwin: drop internal O_NOSYMLINK and O_DIROPEN flags

Both flags are outdated and collide with official flags in
sys/_default_fcntl.h, which may result in weird misbehaviour
of file functions.

O_NOSYMLINK is not used anyway.

O_DIROPEN is used in fhandler_virtual and derived classes.
The collision with O_NOFOLLOW results in spurious EISDIR
errors when, e. g., reading files in the registry.
fhandler_base::open_fs uses O_DIROPEN in the call to
fhandler_base::open, but it's not used in this context
further down the road.

Drop both flags and create an alternative "diropen" bool
flag in fhandler_virtual.

Signed-off-by: Corinna Vinschen <corinna@vinschen.de>

winsup/cygwin/fhandler.h

@@ -14,10 +14,6 @@ details. */
 #include <cygwin/_ucred.h>
 #include <sys/un.h>
 
-/* fcntl flags used only internaly. */
-#define O_NOSYMLINK	0x080000
-#define O_DIROPEN	0x100000
-
 /* newlib used to define O_NDELAY differently from O_NONBLOCK.  Now it
    properly defines both to be the same.  Unfortunately, we have to
    behave properly the old version, too, to accommodate older executables. */
@@ -2640,6 +2636,7 @@ class fhandler_virtual : public fhandler_base
   off_t filesize;
   off_t position;
   int fileid; // unique within each class
+  bool diropen;
  public:
 
   fhandler_virtual ();

winsup/cygwin/fhandler_disk_file.cc

@@ -1469,7 +1469,7 @@ fhandler_base::open_fs (int flags, mode_t mode)
 
   bool new_file = !exists ();
 
-  int res = fhandler_base::open (flags | O_DIROPEN, mode);
+  int res = fhandler_base::open (flags, mode);
   if (res)
     {
       /* The file info in pc is wrong at this point for newly created files.

winsup/cygwin/fhandler_proc.cc

@@ -315,7 +315,7 @@ fhandler_proc::open (int flags, mode_t mode)
 	}
       else
 	{
-	  flags |= O_DIROPEN;
+	  diropen = true;
 	  goto success;
 	}
     }
@@ -342,7 +342,7 @@ fhandler_proc::open (int flags, mode_t mode)
 	      }
 	    else
 	      {
-		flags |= O_DIROPEN;
+		diropen = true;
 		goto success;
 	      }
 	  }

winsup/cygwin/fhandler_process.cc

@@ -272,7 +272,7 @@ fhandler_process::open (int flags, mode_t mode)
 	}
       else
 	{
-	  flags |= O_DIROPEN;
+	  diropen = true;
 	  goto success;
 	}
     }
@@ -287,7 +287,7 @@ fhandler_process::open (int flags, mode_t mode)
     }
   if (entry->fhandler == FH_PROCESSFD)
     {
-      flags |= O_DIROPEN;
+      diropen = true;
       goto success;
     }
   if (flags & O_WRONLY)

winsup/cygwin/fhandler_procnet.cc

@@ -141,7 +141,7 @@ fhandler_procnet::open (int flags, mode_t mode)
 	}
       else
 	{
-	  flags |= O_DIROPEN;
+	  diropen = true;
 	  goto success;
 	}
     }

winsup/cygwin/fhandler_procsysvipc.cc

@@ -161,7 +161,7 @@ fhandler_procsysvipc::open (int flags, mode_t mode)
 	}
       else
 	{
-	  flags |= O_DIROPEN;
+	  diropen = true;
 	  goto success;
 	}
     }

winsup/cygwin/fhandler_registry.cc

@@ -784,7 +784,7 @@ fhandler_registry::open (int flags, mode_t mode)
 	}
       else
 	{
-	  flags |= O_DIROPEN;
+	  diropen = true;
 	  /* Marking as nohandle allows to call dup. */
 	  nohandle (true);
 	  goto success;
@@ -824,7 +824,7 @@ fhandler_registry::open (int flags, mode_t mode)
 		   handles. */
 		if (get_handle () >= HKEY_CLASSES_ROOT)
 		  nohandle (true);
-		flags |= O_DIROPEN;
+		diropen = true;
 		goto success;
 	      }
 	  }
@@ -872,13 +872,13 @@ fhandler_registry::open (int flags, mode_t mode)
 	    }
 	}
       else
-	flags |= O_DIROPEN;
+	diropen = true;
 
       set_handle (handle);
       set_close_on_exec (!!(flags & O_CLOEXEC));
       value_name = cwcsdup (dec_file);
 
-      if (!(flags & O_DIROPEN) && !fill_filebuf ())
+      if (!diropen && !fill_filebuf ())
 	{
 	  RegCloseKey (handle);
 	  res = 0;

winsup/cygwin/fhandler_virtual.cc

@@ -182,7 +182,7 @@ fhandler_virtual::read (void *ptr, size_t& len)
 {
   if (len == 0)
     return;
-  if (openflags & O_DIROPEN)
+  if (diropen)
     {
       set_errno (EISDIR);
       len = (size_t) -1;

winsup/cygwin/release/3.2.0

@@ -30,3 +30,6 @@ Bug Fixes
 
 - Fix SEGV in modfl call.
   Addresses: https://cygwin.com/pipermail/cygwin/2020-August/246056.html
+
+- Fix a collision of offical and internally used file flags.
+  Addresses: https://cygwin.com/pipermail/cygwin/2020-September/246174.html