[Numpy-svn] r8316 - in trunk/numpy/core: . tests

[numpy-svn at scipy.org]

Author: ptvirtan
Date: 2010-04-04 15:20:48 -0500 (Sun, 04 Apr 2010)
New Revision: 8316

Modified:
   trunk/numpy/core/_internal.py
   trunk/numpy/core/tests/test_multiarray.py
Log:
ENH: core: improve PEP 3118 parser's alignment handling

Now, padding will be more forcefully inserted for native-aligned items.

Modified: trunk/numpy/core/_internal.py
===================================================================
--- trunk/numpy/core/_internal.py	2010-04-04 20:20:40 UTC (rev 8315)
+++ trunk/numpy/core/_internal.py	2010-04-04 20:20:48 UTC (rev 8316)
@@ -403,9 +403,23 @@
 
     fields = {}
     offset = 0
-    findex = 0
     explicit_name = False
+    this_explicit_name = False
+    common_alignment = 1
+    is_padding = False
+    last_offset = 0
 
+    dummy_name_index = [0]
+    def next_dummy_name():
+        dummy_name_index[0] += 1
+    def get_dummy_name():
+        while True:
+            name = 'f%d' % dummy_name_index[0]
+            if name not in fields:
+                return name
+            next_dummy_name()
+
+    # Parse spec
     while spec:
         value = None
 
@@ -448,11 +462,9 @@
         is_padding = False
 
         if spec[:2] == 'T{':
-            value, spec = _dtype_from_pep3118(spec[2:], byteorder=byteorder,
-                                              is_subdtype=True)
-            if itemsize != 1:
-                # Not supported
-                raise ValueError("Non item-size 1 structures not supported")
+            value, spec, align = _dtype_from_pep3118(spec[2:],
+                                                     byteorder=byteorder,
+                                                     is_subdtype=True)
         elif spec[0] in type_map_chars:
             if spec[0] == 'Z':
                 j = 2
@@ -467,6 +479,7 @@
                 itemsize = 1
             numpy_byteorder = {'@': '=', '^': '='}.get(byteorder, byteorder)
             value = dtype(numpy_byteorder + dtypechar)
+            align = value.alignment
         else:
             raise ValueError("Unknown PEP 3118 data type specifier %r" % spec)
 
@@ -476,8 +489,8 @@
         #      that the start of the array is *also* aligned.
         extra_offset = 0
         if byteorder == '@':
-            start_padding = (-offset) % value.alignment
-            intra_padding = (-value.itemsize) % value.alignment
+            start_padding = (-offset) % align
+            intra_padding = (-value.itemsize) % align
 
             offset += start_padding
 
@@ -488,6 +501,10 @@
                 else:
                     extra_offset += intra_padding
 
+            # Update common alignment
+            common_alignment = (align*common_alignment
+                                / _gcd(align, common_alignment))
+
         # Convert itemsize to sub-array
         if itemsize != 1:
             value = dtype((value, (itemsize,)))
@@ -505,20 +522,37 @@
             explicit_name = True
             this_explicit_name = True
         else:
-            name = 'f%d' % findex
-            findex += 1
+            name = get_dummy_name()
 
         if not is_padding or this_explicit_name:
+            if name in fields:
+                raise RuntimeError("Duplicate field name '%s' in PEP3118 format"
+                                   % name)
             fields[name] = (value, offset)
+            if not this_explicit_name:
+                next_dummy_name()
+
+        last_offset = offset
         offset += value.itemsize
         offset += extra_offset
 
+    if is_padding and not this_explicit_name:
+        # Trailing padding must be made explicit
+        name = get_dummy_name()
+        fields[name] = ('V%d' % (offset - last_offset), last_offset)
+
     if len(fields.keys()) == 1 and not explicit_name and fields['f0'][1] == 0:
         ret = fields['f0'][0]
     else:
         ret = dtype(fields)
 
     if is_subdtype:
-        return ret, spec
+        return ret, spec, common_alignment
     else:
         return ret
+
+def _gcd(a, b):
+    """Calculate the greatest common divisor of a and b"""
+    while b:
+        a, b = b, a%b
+    return a

Modified: trunk/numpy/core/tests/test_multiarray.py
===================================================================
--- trunk/numpy/core/tests/test_multiarray.py	2010-04-04 20:20:40 UTC (rev 8315)
+++ trunk/numpy/core/tests/test_multiarray.py	2010-04-04 20:20:48 UTC (rev 8316)
@@ -1456,6 +1456,32 @@
     if sys.version_info[:2] == (2, 6):
         from numpy.core.multiarray import memorysimpleview as memoryview
 
+    from numpy.core._internal import _dtype_from_pep3118
+
+    class TestPEP3118Dtype(object):
+        def _check(self, spec, wanted):
+            assert_equal(_dtype_from_pep3118(spec), np.dtype(wanted),
+                         err_msg="spec %r != dtype %r" % (spec, wanted))
+
+        def test_native_padding(self):
+            align = np.dtype('i').alignment
+            for j in xrange(8):
+                if j == 0:
+                    s = 'bi'
+                else:
+                    s = 'b%dxi' % j
+                self._check('@'+s, {'f0': ('i1', 0),
+                                    'f1': ('i', align*(1 + j//align))})
+                self._check('='+s, {'f0': ('i1', 0),
+                                    'f1': ('i', 1+j)})
+
+        def test_native_padding_2(self):
+            self._check('x3T{xi}', {'f0': (({'f0': ('i', 4)}, (3,)), 4)})
+            self._check('=x3T{xi}', {'f0': (({'f0': ('i', 1)}, (3,)), 1)})
+
+        def test_trailing_padding(self):
+            self._check('ix', [('f0', 'i'), ('f1', 'V1')])
+
     class TestNewBufferProtocol(object):
         def _check_roundtrip(self, obj):
             obj = np.asarray(obj)