From e5fcb021cc9dcb1f19d45030457be86b4a226e65 Mon Sep 17 00:00:00 2001
From: Ken Brown <kbrown@cornell.edu>
Date: Sat, 18 Sep 2021 08:13:55 -0400
Subject: [PATCH] Cygwin: allow open_setup to fail

Convert fhandler_base::open_setup to a (virtual) method that returns a
bool result.  For the moment, it and its overrides always return true.
---
 winsup/cygwin/fhandler.cc         |  3 ++-
 winsup/cygwin/fhandler.h          | 10 +++++-----
 winsup/cygwin/fhandler_console.cc |  4 ++--
 winsup/cygwin/fhandler_pipe.cc    |  9 +++++++--
 winsup/cygwin/fhandler_tty.cc     |  8 ++++----
 5 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/winsup/cygwin/fhandler.cc b/winsup/cygwin/fhandler.cc
index 9dfe70be3..1af469e0c 100644
--- a/winsup/cygwin/fhandler.cc
+++ b/winsup/cygwin/fhandler.cc
@@ -789,9 +789,10 @@ fhandler_base::fd_reopen (int, mode_t)
   return NULL;
 }
 
-void
+bool
 fhandler_base::open_setup (int)
 {
+  return true;
 }
 
 /* states:
diff --git a/winsup/cygwin/fhandler.h b/winsup/cygwin/fhandler.h
index 61113e698..3471e95b9 100644
--- a/winsup/cygwin/fhandler.h
+++ b/winsup/cygwin/fhandler.h
@@ -355,7 +355,7 @@ class fhandler_base
   int open_null (int flags);
   virtual int open (int, mode_t);
   virtual fhandler_base *fd_reopen (int, mode_t);
-  virtual void open_setup (int flags);
+  virtual bool open_setup (int flags);
   void set_unique_id (int64_t u) { unique_id = u; }
   void set_unique_id () { NtAllocateLocallyUniqueId ((PLUID) &unique_id); }
 
@@ -1206,7 +1206,7 @@ public:
   select_record *select_except (select_stuff *);
   char *get_proc_fd_name (char *buf);
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   void fixup_after_fork (HANDLE);
   int dup (fhandler_base *child, int);
   void set_close_on_exec (bool val);
@@ -2132,7 +2132,7 @@ private:
   bool use_archetype () const {return true;}
 
   int open (int flags, mode_t mode);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   int dup (fhandler_base *, int);
 
   void __reg3 read (void *ptr, size_t& len);
@@ -2300,7 +2300,7 @@ class fhandler_pty_slave: public fhandler_pty_common
   HANDLE& get_handle_nat () { return io_handle_nat; }
 
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   ssize_t __stdcall write (const void *ptr, size_t len);
   void __reg3 read (void *ptr, size_t& len);
   int init (HANDLE, DWORD, mode_t);
@@ -2399,7 +2399,7 @@ public:
   void doecho (const void *str, DWORD len);
   int accept_input ();
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   ssize_t __stdcall write (const void *ptr, size_t len);
   void __reg3 read (void *ptr, size_t& len);
   int close ();
diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index e00f2cdbc..ee862b17d 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -1366,13 +1366,13 @@ fhandler_console::open (int flags, mode_t)
   return 1;
 }
 
-void
+bool
 fhandler_console::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   if (myself->set_ctty (this, flags) && !myself->cygstarted)
     init_console_handler (true);
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 int
diff --git a/winsup/cygwin/fhandler_pipe.cc b/winsup/cygwin/fhandler_pipe.cc
index 73ace3ac5..590ecf667 100644
--- a/winsup/cygwin/fhandler_pipe.cc
+++ b/winsup/cygwin/fhandler_pipe.cc
@@ -191,10 +191,11 @@ out:
   return 0;
 }
 
-void
+bool
 fhandler_pipe::open_setup (int flags)
 {
-  fhandler_base::open_setup (flags);
+  if (!fhandler_base::open_setup (flags))
+    goto err;
   if (get_dev () == FH_PIPER && !read_mtx)
     {
       SECURITY_ATTRIBUTES *sa = sec_none_cloexec (flags);
@@ -211,6 +212,10 @@ fhandler_pipe::open_setup (int flags)
     }
   if (get_dev () == FH_PIPEW && !query_hdl)
     set_pipe_non_blocking (is_nonblocking ());
+  return true;
+
+err:
+  return false;
 }
 
 off_t
diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 1ea9a47ac..05fe5348a 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -964,13 +964,13 @@ err_no_msg:
   return 0;
 }
 
-void
+bool
 fhandler_pty_slave::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   myself->set_ctty (this, flags);
   report_tty_counts (this, "opened", "");
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 void
@@ -1947,14 +1947,14 @@ fhandler_pty_master::open (int flags, mode_t)
   return 1;
 }
 
-void
+bool
 fhandler_pty_master::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   char buf[sizeof ("opened pty master for ptyNNNNNNNNNNN")];
   __small_sprintf (buf, "opened pty master for pty%d", get_minor ());
   report_tty_counts (this, buf, "");
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 off_t