[pypy-commit] pypy propogate-nans: we can really only store up to 52 bits of mantissa in a nan, bit 53 is special

[mattip]

Author: mattip <matti.picus at gmail.com>
Branch: propogate-nans
Changeset: r80587:5632cebfe86b
Date: 2015-11-07 21:06 +0200
http://bitbucket.org/pypy/pypy/changeset/5632cebfe86b/

Log:	we can really only store up to 52 bits of mantissa in a nan, bit 53
	is special

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -1857,7 +1857,7 @@
         #     convert_to_float64 -> convert_to_float16 -> uint16
         #  even for float16 various float16 nans
         from numpy import array, arange
-        all_f16 = arange(0xfe00, 0xfe08, dtype='uint16')
+        all_f16 = arange(0xfe00, 0xffff, dtype='uint16')
         all_f16.dtype = 'float16'
         all_f32 = array(all_f16, dtype='float32')
         b = array(all_f32, dtype='float16')
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -64,10 +64,10 @@
         if mant == 0:
             result = rfloat.INFINITY
         else:
-            # preserve mant value but pad w/zeros
+            # preserve at most 52 bits of mant value, but pad w/zeros
             exp = r_ulonglong(0x7ff) << 52
             sign = r_ulonglong(sign) << 63
-            mant = r_ulonglong(mant) << (53 - MANT_DIG)
+            mant = r_ulonglong(mant) << (52 - MANT_DIG) 
             uint = exp | mant | sign
             result =  longlong2float(cast(LONGLONG, uint))
             return result
@@ -150,9 +150,10 @@
     elif rfloat.isnan(x):
         asint = cast(ULONGLONG, float2longlong(x))
         mant = asint & ((r_ulonglong(1) << 52) - 1)
-        sign = asint < 0
+        sign = asint >> 63
         # shift off lower bits, perhaps losing data
-        mant = mant >> (53 - MANT_DIG)
+        if MANT_DIG <= 52:
+            mant = mant >> (52 - MANT_DIG)
         if mant == 0:
             mant = r_ulonglong(1) << (MANT_DIG-2)
         exp = MAX_EXP - MIN_EXP + 2