Cygwin: open_shared: try harder allocating a shared region

For fixed regions (cygwin/user/myself/shared console), try fixed
address first.  Fallback to non-fixed region.  Don't even try fixed
address if the Cygwin DLL gets dynamically loaded.

For non-fixed regions, try to allocate in a loop within the area
from SHARED_REGIONS_ADDRESS_LOW to SHARED_REGIONS_ADDRESS_HIGH.

Fixes: 60675f1a7e ("Cygwin: decouple shared mem regions from Cygwin DLL")
Signed-off-by: Corinna Vinschen <corinna@vinschen.de>

winsup/cygwin/mm/shared.cc

@@ -122,6 +122,7 @@ static uintptr_t region_address[] =
   SHARED_CONSOLE_REGION_ADDRESS,	/* SH_SHARED_CONSOLE */
   0
 };
+static NO_COPY uintptr_t next_address = SHARED_REGIONS_ADDRESS_LOW;
 
 void *
 open_shared (const WCHAR *name, int n, HANDLE& shared_h, DWORD size,
@@ -138,14 +139,9 @@ open_shared (const WCHAR *name, int n, HANDLE& shared_h, DWORD size,
 {
   WCHAR map_buf[MAX_PATH];
   WCHAR *mapname = NULL;
-  void *shared;
+  void *shared = NULL;
   void *addr;
 
-  if (m == SH_JUSTCREATE || m == SH_JUSTOPEN)
-    addr = NULL;
-  else
-    addr = (void *) region_address[m];
-
   created = false;
   if (!shared_h)
     {
@@ -170,14 +166,54 @@ open_shared (const WCHAR *name, int n, HANDLE& shared_h, DWORD size,
 	return NULL;
     }
 
-  shared = MapViewOfFileEx (shared_h, FILE_MAP_READ | FILE_MAP_WRITE,
+  if (m < SH_TOTAL_SIZE && !dynamically_loaded)
-			    0, 0, 0, addr);
+    {
+      /* Fixed regions.  Don't do that if Cygwin gets dynamically loaded.
+	 The process loading the DLL might be configured with High-Entropy
+	 ASLR.  Chances for collisions are pretty high.
+
+	 Note that we don't actually *need* fixed addresses.  The only
+	 advantage is reproducibility to help /proc/<PID>/maps along. */
+      addr = (void *) region_address[m];
+      shared = MapViewOfFileEx (shared_h, FILE_MAP_READ | FILE_MAP_WRITE,
+				0, 0, 0, addr);
+    }
+  /* Also catch the unlikely case that a fixed region can't be mapped at the
+     fixed address. */
+  if (!shared)
+    {
+      /* Locate shared regions in the area between SHARED_REGIONS_ADDRESS_LOW
+	 and SHARED_REGIONS_ADDRESS_HIGH, retrying until we have a slot.
+	 Don't use MapViewOfFile3 (loader deadlock during fork. */
+      bool loop = false;
+
+      addr = (void *) next_address;
+      do
+	{
+	  shared = MapViewOfFileEx (shared_h, FILE_MAP_READ | FILE_MAP_WRITE,
+				    0, 0, 0, addr);
+	  if (!shared)
+	    {
+	      next_address += wincap.allocation_granularity ();
+	      if (next_address >= SHARED_REGIONS_ADDRESS_HIGH)
+		{
+		  if (loop)
+		    break;
+		  next_address = SHARED_REGIONS_ADDRESS_LOW;
+		  loop = true;
+		}
+	      addr = (void *) next_address;
+	    }
+	}
+      while (!shared);
+    }
 
   if (!shared)
-    api_fatal ("MapViewOfFileEx '%W'(%p), %E.  Terminating.", mapname, shared_h);
+    api_fatal ("MapViewOfFileEx '%W'(%p, size %u, m %d, created %d), %E.  "
+	       "Terminating.", mapname, shared_h, size, m, created);
 
-  debug_printf ("name %W, n %d, shared %p (wanted %p), h %p, m %d",
+  debug_printf ("name %W, shared %p (wanted %p), h %p, m %d, created %d",
-		mapname, n, shared, addr, shared_h, m);
+		mapname, shared, addr, shared_h, m, created);
 
   return shared;
 }

winsup/cygwin/release/3.4.4

@@ -16,3 +16,9 @@ Bug Fixes
 
 - Fix vmstat(1) printing an error message on single core CPUs.
   Addresses: https://cygwin.com/pipermail/cygwin/2023-January/252857.html
+
+- Fix potential process termination during process initialization.
+  Most easily reproducible is the case of non-Cygwin processes running
+  with high-entropy VA enabled and loading the Cygwin DLL dynamically.
+  Addresses: https://cygwin.com/pipermail/cygwin/2023-January/252765.html
+	     https://cygwin.com/pipermail/cygwin/2023-January/252865.html