libm/math: set errno to ERANGE at gamma poles

For POSIX, gamma(i) (i non-positive integer) should set errno to
ERANGE instead of EDOM.

Signed-off-by: Keith Packard <keithp@keithp.com>

newlib/libm/math/w_gamma.c

@@ -155,16 +155,10 @@ in terms of the base return values, although the <[signgam]> global for
         y = __ieee754_gamma_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finite(y)&&finite(x)) {
-	    if(floor(x)==x&&x<=0.0) {
+	    /* gamma(finite) overflow */
-		/* gamma(-integer) or gamma(0) */
+	    errno = ERANGE;
-		errno = EDOM;
+        }
-	    } else {
+	return y;
-		/* gamma(finite) overflow */
-		errno = ERANGE;
-	    }
-	    return HUGE_VAL;
-        } else
-            return y;
 #endif
 }             
 

newlib/libm/math/w_lgamma.c

@@ -38,15 +38,10 @@
         y = __ieee754_lgamma_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finite(y)&&finite(x)) {
-	    if(floor(x)==x&&x<=0.0)
+	    /* lgamma(finite) overflow */
-	      /* lgamma(-integer) */
+	    errno = ERANGE;
-	      errno = EDOM;
+	}
-	    else
+	return y;
-	      /* lgamma(finite) overflow */
-	      errno = ERANGE;
-	    return HUGE_VAL;
-	} else
-            return y;
 #endif
 }             
 

newlib/libm/math/wf_gamma.c

@@ -32,16 +32,10 @@
         y = __ieee754_gammaf_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finitef(y)&&finitef(x)) {
-	    if(floorf(x)==x&&x<=0.0f) {
+	    /* gammaf(finite) overflow */
-		/* gammaf(-integer) or gammaf(0) */
+	    errno = ERANGE;
-		errno = EDOM;
+        }
-            } else {
+	return y;
-		/* gammaf(finite) overflow */
-		errno = ERANGE;
-            }
-	    return HUGE_VALF;
-        } else
-            return y;
 #endif
 }             
 

newlib/libm/math/wf_lgamma.c

@@ -32,16 +32,10 @@
         y = __ieee754_lgammaf_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finitef(y)&&finitef(x)) {
-	    if(floorf(x)==x&&x<=0.0f) {
+	    /* lgammaf(finite) overflow */
-		/* lgammaf(-integer) */
+	    errno = ERANGE;
-		errno = EDOM;
+        }
-	    } else {
+	return y;
-		/* lgammaf(finite) overflow */
-		errno = ERANGE;
-	    }
-            return HUGE_VALF;
-        } else
-            return y;
 #endif
 }