Cygwin: get/setrlimit: move RLIMIT_AS handling into static functions

To keep getrlimit/setrlimit clean, move the RLIMIT_AS code into
local static functions __set_rlimit_as and __get_rlimit_as.

Also, make adding process to the job the last step, to be able to
close and release the job resources if anything failed.  Add matching
comment.

Signed-off-by: Corinna Vinschen <corinna@vinschen.de>

winsup/cygwin/resource.cc

@@ -174,22 +174,9 @@ job_shared_name (PWCHAR buf, LONG num)
   return buf;
 }
 
-extern "C" int
+static void
-getrlimit (int resource, struct rlimit *rlp)
+__get_rlimit_as (struct rlimit *rlp)
 {
-  __try
-    {
-      rlp->rlim_cur = RLIM_INFINITY;
-      rlp->rlim_max = RLIM_INFINITY;
-
-      switch (resource)
-	{
-	case RLIMIT_CPU:
-	case RLIMIT_FSIZE:
-	case RLIMIT_DATA:
-	  break;
-	case RLIMIT_AS:
-	  {
   UNICODE_STRING uname;
   WCHAR jobname[32];
   OBJECT_ATTRIBUTES attr;
@@ -210,13 +197,80 @@ getrlimit (int resource, struct rlimit *rlp)
   status = NtQueryInformationJobObject (job,
 			      JobObjectExtendedLimitInformation,
 			      &jobinfo, sizeof jobinfo, NULL);
-	    if (!NT_SUCCESS (status))
+  if (NT_SUCCESS (status)
-	      break;
+      && (jobinfo.BasicLimitInformation.LimitFlags
-	    if (jobinfo.BasicLimitInformation.LimitFlags & JOB_OBJECT_LIMIT_PROCESS_MEMORY)
+	  & JOB_OBJECT_LIMIT_PROCESS_MEMORY))
     rlp->rlim_cur = rlp->rlim_max = jobinfo.ProcessMemoryLimit;
   if (job)
     NtClose (job);
+}
+
+static int
+__set_rlimit_as (unsigned long new_as_limit)
+{
+  LONG new_as_id = 0;
+  UNICODE_STRING uname;
+  WCHAR jobname[32];
+  OBJECT_ATTRIBUTES attr;
+  NTSTATUS status = STATUS_SUCCESS;
+  HANDLE job = NULL;
+  JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobinfo = { 0 };
+
+  /* If we already have a limit, we must not change it because that
+     would potentially influence already running child processes.
+     Just try to create another, nested job.  On systems prior to
+     Windows 8 / Server 2012 this will fail, but that's ok. */
+  while (new_as_id == 0)
+    new_as_id = InterlockedIncrement (&job_serial_number);
+  RtlInitUnicodeString (&uname,
+			job_shared_name (jobname, new_as_id));
+  InitializeObjectAttributes (&attr, &uname, 0,
+			      get_session_parent_dir (), NULL);
+  status = NtCreateJobObject (&job, JOB_OBJECT_ALL_ACCESS, &attr);
+  if (!NT_SUCCESS (status))
+    {
+      __seterrno_from_nt_status (status);
+      return -1;
     }
+  jobinfo.BasicLimitInformation.LimitFlags
+    = JOB_OBJECT_LIMIT_PROCESS_MEMORY;
+  /* Per Linux man page, round down to system pagesize. */
+  jobinfo.ProcessMemoryLimit
+    = rounddown (new_as_limit, wincap.allocation_granularity ());
+  status = NtSetInformationJobObject (job,
+				JobObjectExtendedLimitInformation,
+				&jobinfo, sizeof jobinfo);
+  /* If creating the job and setting up the job limits succeeded,
+     try to add the process to the job.  This must be the last step,
+     otherwise we couldn't remove the job if anything failed. */
+  if (NT_SUCCESS (status))
+    status = NtAssignProcessToJobObject (job, NtCurrentProcess ());
+  NtClose (job);
+  if (!NT_SUCCESS (status))
+    {
+      __seterrno_from_nt_status (status);
+      return -1;
+    }
+  cygheap->rlim_as_id = new_as_id;
+  return 0;
+}
+
+extern "C" int
+getrlimit (int resource, struct rlimit *rlp)
+{
+  __try
+    {
+      rlp->rlim_cur = RLIM_INFINITY;
+      rlp->rlim_max = RLIM_INFINITY;
+
+      switch (resource)
+	{
+	case RLIMIT_CPU:
+	case RLIMIT_FSIZE:
+	case RLIMIT_DATA:
+	  break;
+	case RLIMIT_AS:
+	  __get_rlimit_as (rlp);
 	  break;
 	case RLIMIT_STACK:
 	  __get_rlimit_stack (rlp);
@@ -270,51 +324,8 @@ setrlimit (int resource, const struct rlimit *rlp)
       switch (resource)
 	{
 	case RLIMIT_AS:
-	  {
+	  if (rlp->rlim_cur != RLIM_INFINITY)
-	    LONG new_as_id = 0;
+	    return __set_rlimit_as (rlp->rlim_cur);
-	    UNICODE_STRING uname;
-	    WCHAR jobname[32];
-	    OBJECT_ATTRIBUTES attr;
-	    NTSTATUS status = STATUS_SUCCESS;
-	    HANDLE job = NULL;
-	    JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobinfo = { 0 };
-
-	    /* If we already have a limit, we must not change it because that
-	       would potentially influence already running child processes.
-	       Just try to create another, nested job.  On systems prior to
-	       Windows 8 / Server 2012 this will fail, but that's ok. */
-	    while (new_as_id == 0)
-	      new_as_id = InterlockedIncrement (&job_serial_number);
-	    RtlInitUnicodeString (&uname,
-				  job_shared_name (jobname, new_as_id));
-	    InitializeObjectAttributes (&attr, &uname, 0,
-					get_session_parent_dir (), NULL);
-	    status = NtCreateJobObject (&job, JOB_OBJECT_ALL_ACCESS, &attr);
-	    if (!NT_SUCCESS (status))
-	      {
-		__seterrno_from_nt_status (status);
-		__leave;
-	      }
-	    status = NtAssignProcessToJobObject (job, NtCurrentProcess ());
-	    if (NT_SUCCESS (status))
-	      {
-		jobinfo.BasicLimitInformation.LimitFlags
-		  = JOB_OBJECT_LIMIT_PROCESS_MEMORY;
-		/* Per Linux man page, round down to system pagesize. */
-		jobinfo.ProcessMemoryLimit
-		  = rounddown (rlp->rlim_cur, wincap.allocation_granularity ());
-		status = NtSetInformationJobObject (job,
-					      JobObjectExtendedLimitInformation,
-					      &jobinfo, sizeof jobinfo);
-	      }
-	    NtClose (job);
-	    if (!NT_SUCCESS (status))
-	      {
-		__seterrno_from_nt_status (status);
-		__leave;
-	      }
-	    cygheap->rlim_as_id = new_as_id;
-	  }
 	  break;
 	case RLIMIT_CORE:
 	  cygheap->rlim_core = rlp->rlim_cur;