libm/math: set errno to ERANGE at gamma poles

For POSIX, gamma(i) (i non-positive integer) should set errno to
ERANGE instead of EDOM.

Signed-off-by: Keith Packard <keithp@keithp.com>

newlib/libm/math/w_gamma.c

@@ -155,15 +155,9 @@ in terms of the base return values, although the <[signgam]> global for
         y = __ieee754_gamma_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finite(y)&&finite(x)) {
-	    if(floor(x)==x&&x<=0.0) {
-		/* gamma(-integer) or gamma(0) */
-		errno = EDOM;
-	    } else {
 	    /* gamma(finite) overflow */
 	    errno = ERANGE;
         }
-	    return HUGE_VAL;
-        } else
 	return y;
 #endif
 }             

newlib/libm/math/w_lgamma.c

@@ -38,14 +38,9 @@
         y = __ieee754_lgamma_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finite(y)&&finite(x)) {
-	    if(floor(x)==x&&x<=0.0)
-	      /* lgamma(-integer) */
-	      errno = EDOM;
-	    else
 	    /* lgamma(finite) overflow */
 	    errno = ERANGE;
-	    return HUGE_VAL;
+	}
-	} else
 	return y;
 #endif
 }             

newlib/libm/math/wf_gamma.c

@@ -32,15 +32,9 @@
         y = __ieee754_gammaf_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finitef(y)&&finitef(x)) {
-	    if(floorf(x)==x&&x<=0.0f) {
-		/* gammaf(-integer) or gammaf(0) */
-		errno = EDOM;
-            } else {
 	    /* gammaf(finite) overflow */
 	    errno = ERANGE;
         }
-	    return HUGE_VALF;
-        } else
 	return y;
 #endif
 }             

newlib/libm/math/wf_lgamma.c

@@ -32,15 +32,9 @@
         y = __ieee754_lgammaf_r(x,&(_REENT_SIGNGAM(_REENT)));
         if(_LIB_VERSION == _IEEE_) return y;
         if(!finitef(y)&&finitef(x)) {
-	    if(floorf(x)==x&&x<=0.0f) {
-		/* lgammaf(-integer) */
-		errno = EDOM;
-	    } else {
 	    /* lgammaf(finite) overflow */
 	    errno = ERANGE;
         }
-            return HUGE_VALF;
-        } else
 	return y;
 #endif
 }