* dtable.cc (fh_oom): New static fhandler storage.

(fh_calloc): New static function.  Add a comment to explain why this
	is needed.
	(cnew): Call fh_calloc as placement argument.
	(build_fh_name): Check return code from cnew against address of
	fh_oom to test for out of memory condition.
	(fh_alloc): Ditto.
	(build_fh_pc): Avoid a crash due to useing a NULL fhandler.
	* pipe.cc (fhandler_pipe::create): Check if build_fh_dev returned a
	valid pointer before using it.

winsup/cygwin/ChangeLog

@@ -1,3 +1,16 @@
+2011-06-30  Corinna Vinschen  <corinna@vinschen.de>
+
+	* dtable.cc (fh_oom): New static fhandler storage.
+	(fh_calloc): New static function.  Add a comment to explain why this
+	is needed.
+	(cnew): Call fh_calloc as placement argument.
+	(build_fh_name): Check return code from cnew against address of
+	fh_oom to test for out of memory condition.
+	(fh_alloc): Ditto.
+	(build_fh_pc): Avoid a crash due to useing a NULL fhandler.
+	* pipe.cc (fhandler_pipe::create): Check if build_fh_dev returned a
+	valid pointer before using it.
+
 2011-06-28  Corinna Vinschen  <corinna@vinschen.de>
 
 	* fhandler_process.cc (heap_info::fill_if_match): Rename info to

winsup/cygwin/dtable.cc

@@ -393,7 +393,17 @@ dtable::init_std_file_from_handle (int fd, HANDLE handle)
     }
 }
 
-#define cnew(name) new ((void *) ccalloc (HEAP_FHANDLER, 1, sizeof (name))) name
+/* This is a workaround for the fact that the placement new operator
+   always calls the constructor, even if the placement pointer is NULL. */
+static fhandler_union fh_oom;
+static void *
+fh_calloc (size_t size)
+{
+  void *ret = ccalloc (HEAP_FHANDLER, 1, size);
+  return ret ?: (void *) &fh_oom;
+}
+
+#define cnew(name) new (fh_calloc (sizeof (name))) name
 
 fhandler_base *
 build_fh_name (const char *name, unsigned opt, suffix_info *si)
@@ -402,7 +412,7 @@ build_fh_name (const char *name, unsigned opt, suffix_info *si)
   if (pc.error)
     {
       fhandler_base *fh = cnew (fhandler_nodevice) ();
-      if (fh)
+      if (fh != (fhandler_base *) &fh_oom)
 	fh->set_error (pc.error);
       set_errno (fh ? pc.error : EMFILE);
       return fh;
@@ -549,6 +559,8 @@ fh_alloc (device dev)
 
   if (fh == fh_unset)
     fh = cnew (fhandler_nodevice) ();
+  if (fh == (fhandler_base *) &fh_oom)
+    fh = NULL;
   return fh;
 }
 
@@ -558,7 +570,10 @@ build_fh_pc (path_conv& pc, bool set_name)
   fhandler_base *fh = fh_alloc (pc.dev);
 
   if (!fh)
-    set_errno (EMFILE);
+    {
+      set_errno (EMFILE);
+      goto out;
+    }
   else if (fh->dev () == FH_ERROR)
     goto out;
   else if (fh->dev () != FH_NADA)

winsup/cygwin/pipe.cc

@@ -303,19 +303,23 @@ fhandler_pipe::create (fhandler_pipe *fhs[2], unsigned psize, int mode)
 {
   HANDLE r, w;
   SECURITY_ATTRIBUTES *sa = sec_none_cloexec (mode);
-  int res;
+  int res = -1;
 
   int ret = create_selectable (sa, r, w, psize);
   if (ret)
+    __seterrno_from_win_error (ret);
+  else if ((fhs[0] = (fhandler_pipe *) build_fh_dev (*piper_dev)) == NULL)
     {
-      __seterrno_from_win_error (ret);
+      CloseHandle (r);
-      res = -1;
+      CloseHandle (w);
+    }
+  else if ((fhs[1] = (fhandler_pipe *) build_fh_dev (*pipew_dev)) == NULL)
+    {
+      delete fhs[0];
+      CloseHandle (w);
     }
   else
     {
-      fhs[0] = (fhandler_pipe *) build_fh_dev (*piper_dev);
-      fhs[1] = (fhandler_pipe *) build_fh_dev (*pipew_dev);
-
       mode |= mode & O_TEXT ?: O_BINARY;
       fhs[0]->init (r, FILE_CREATE_PIPE_INSTANCE | GENERIC_READ, mode);
       fhs[1]->init (w, FILE_CREATE_PIPE_INSTANCE | GENERIC_WRITE, mode);