summaryrefslogtreecommitdiff
path: root/lib/libm/src
diff options
context:
space:
mode:
authoreadler <eadler@NetBSD.org>2018-03-10 09:44:47 +0000
committereadler <eadler@NetBSD.org>2018-03-10 09:44:47 +0000
commitbb8d31f415a9cfc59beda6fb21e02dbae38b9af1 (patch)
treee4b729d2e8dc63e10e4cb31adf802bd0d4858ab4 /lib/libm/src
parentc2fc4f5d1e80e95d0660d4f5d0ef19c6d5b50722 (diff)
Fix signed overflow in atan2
As a component of atan2(y, x), the case of x == 1.0 is farmed out to atan(y). The current implementation of this comparison is vulnerable to signed integer underflow (that is, undefined behavior), and it's performed in a somewhat more complicated way than it need be. Change it to not be quite so cute, rather directly comparing the high/low bits of x to the specific IEEE-754 bit pattern that encodes 1.0. ok martin@ pgoyette@ maya@ obtained from FreeBSD
Diffstat (limited to 'lib/libm/src')
-rw-r--r--lib/libm/src/e_atan2.c4
1 files changed, 2 insertions, 2 deletions
diff --git a/lib/libm/src/e_atan2.c b/lib/libm/src/e_atan2.c
index 08d8f06da3a..c2a91547692 100644
--- a/lib/libm/src/e_atan2.c
+++ b/lib/libm/src/e_atan2.c
@@ -12,7 +12,7 @@
#include <sys/cdefs.h>
#if defined(LIBM_SCCS) && !defined(lint)
-__RCSID("$NetBSD: e_atan2.c,v 1.12 2002/05/26 22:01:48 wiz Exp $");
+__RCSID("$NetBSD: e_atan2.c,v 1.13 2018/03/10 09:44:47 eadler Exp $");
#endif
/* __ieee754_atan2(y,x)
@@ -67,7 +67,7 @@ __ieee754_atan2(double y, double x)
if(((ix|((lx|-lx)>>31))>0x7ff00000)||
((iy|((ly|-ly)>>31))>0x7ff00000)) /* x or y is NaN */
return x+y;
- if(((hx-0x3ff00000)|lx)==0) return atan(y); /* x=1.0 */
+ if(hx==0x3ff00000&&lx==0) return atan(y); /* x=1.0 */
m = ((hy>>31)&1)|((hx>>30)&2); /* 2*sign(x)+sign(y) */
/* when y = 0 */