From jython-checkins at python.org  Fri Nov 15 00:01:35 2013
From: jython-checkins at python.org (jeff.allen)
Date: Fri, 15 Nov 2013 00:01:35 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_memoryview=3A_correct_doubl?=
	=?utf-8?q?e_acquisition_of_export?=
Message-ID: <3dLJ8v0ZWTz7Ljj@mail.python.org>

http://hg.python.org/jython/rev/360eb0a402fe
changeset:   7148:360eb0a402fe
parent:      7146:4af2f4241912
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Fri Nov 01 22:58:33 2013 +0000
summary:
  memoryview: correct double acquisition of export
Added test to test_bytes to show that a released memoryview
no longer locks its underlying object (Jython only, in v2.7).

files:
  Lib/test/test_bytes.py                |   5 +++
  src/org/python/core/PyMemoryView.java |  20 +++++++-------
  2 files changed, 15 insertions(+), 10 deletions(-)


diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -886,6 +886,11 @@
         self.assertRaises(BufferError, delslice)
         self.assertEqual(b, orig)
 
+        if test.test_support.is_jython:
+            # Show that releasing v releases the bytearray for size change
+            v.release()
+            b.pop()
+
     def test_empty_bytearray(self):
         # Issue #7561: operations on empty bytearrays could crash in many
         # situations, due to a fragile implementation of the
diff --git a/src/org/python/core/PyMemoryView.java b/src/org/python/core/PyMemoryView.java
--- a/src/org/python/core/PyMemoryView.java
+++ b/src/org/python/core/PyMemoryView.java
@@ -38,14 +38,19 @@
     private boolean hashCacheValid = false;
 
     /**
-     * Construct a PyMemoryView from a PyBuffer interface. The buffer so obtained will be writable
-     * if the underlying object permits it. The <code>memoryview</code> takes a new lease on the
-     * <code>PyBuffer</code>.
+     * Construct a <code>PyMemoryView</code> from an object bearing the {@link BufferProtocol}
+     * interface. If this object is already an exported buffer, the <code>memoryview</code> takes a
+     * new lease on it. The buffer so obtained will be writable if the underlying object permits it.
      *
      * @param pybuf buffer exported by some underlying object
      */
-    public PyMemoryView(PyBuffer pybuf) {
+    public PyMemoryView(BufferProtocol pybuf) {
         super(TYPE);
+        /*
+         * Ask for the full set of facilities (strides, indirect, etc.) from the object in case they
+         * are necessary for navigation, but only ask for read access. If the object is writable,
+         * the PyBuffer will be writable.
+         */
         backing = pybuf.getBuffer(PyBUF.FULL_RO);
     }
 
@@ -63,12 +68,7 @@
         PyObject obj = ap.getPyObject(0);
 
         if (obj instanceof BufferProtocol) {
-            /*
-             * Ask for the full set of facilities (strides, indirect, etc.) from the object in case
-             * they are necessary for navigation, but only ask for read access. If the object is
-             * writable, the PyBuffer will be writable.
-             */
-            return new PyMemoryView(((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO));
+            return new PyMemoryView((BufferProtocol)obj);
         } else {
             throw Py.TypeError("cannot make memory view because object does not have "
                     + "the buffer interface");

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 15 00:01:36 2013
From: jython-checkins at python.org (jeff.allen)
Date: Fri, 15 Nov 2013 00:01:36 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Buffer_API_added_to_PyArray?=
	=?utf-8?q?_=28for_bytes_only=29=2E?=
Message-ID: <3dLJ8w4wDKz7Ljq@mail.python.org>

http://hg.python.org/jython/rev/7bd4080ed5a0
changeset:   7149:7bd4080ed5a0
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sun Nov 03 21:40:57 2013 +0000
summary:
  Buffer API added to PyArray (for bytes only).
Many checks added to prevent resizing with exports; test of this added to
test_array.py. Change to TextIOBase and PyFile so that special handling of
arrays has precedence over other buffer API objects. We need this until the API
supports non-byte arrays too.

files:
  Lib/test/test_array.py                 |   73 ++-
  src/org/python/core/PyArray.java       |  324 ++++++++++--
  src/org/python/core/PyFile.java        |   34 +-
  src/org/python/core/io/TextIOBase.java |   26 +-
  4 files changed, 369 insertions(+), 88 deletions(-)


diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -799,10 +799,11 @@
         gc.collect()
 
     def test_buffer(self):
-        a = array.array(self.typecode, self.example)
-        with test_support.check_py3k_warnings():
-            b = buffer(a)
-        self.assertEqual(b[0], a.tostring()[0])
+        if self.typecode == 'b':    # FIXME: Jython supports only the byte buffer
+            a = array.array(self.typecode, self.example)
+            with test_support.check_py3k_warnings():
+                b = buffer(a)
+            self.assertEqual(b[0], a.tostring()[0])
 
     def test_weakref(self):
         s = array.array(self.typecode, self.example)
@@ -827,6 +828,67 @@
         # SF bug #1486663 -- this used to erroneously raise a TypeError
         ArraySubclassWithKwargs('b', newarg=1)
 
+    @unittest.skipIf(not test_support.is_jython, "array supports buffer interface in Jython")
+    def test_resize_forbidden(self):
+        # Test that array resizing is forbidden with buffer exports (Jython addition).
+        # Test adapted from corresponding one in test_bytes.
+        # We can't resize an array when there are buffer exports, even
+        # if it wouldn't reallocate the underlying array.
+        # Furthermore, no destructive changes to the buffer may be applied
+        # before raising the error.
+        if self.typecode == 'b':    # Jython supports only the byte buffer
+            a = array.array(self.typecode, self.example)
+            def resize(n):
+                "n = -1 -> Smaller, 0 -> the same, or 1 -> larger."
+                a[1:-1] = array.array(self.typecode, self.example[1-n:-1])
+
+            v = memoryview(a)
+            orig = a[:]
+
+            self.assertRaises(BufferError, resize, -1)
+            self.assertEqual(a, orig)
+            #self.assertRaises(BufferError, resize, 0)
+            #self.assertEqual(a, orig)
+            self.assertRaises(BufferError, resize, 1)
+            self.assertEqual(a, orig)
+
+            # Other operations implying resize
+            self.assertRaises(BufferError, a.pop, 0)
+            self.assertEqual(a, orig)
+            self.assertRaises(BufferError, a.remove, a[1])
+            self.assertEqual(a, orig)
+            self.assertRaises(BufferError, a.append, self.outside)
+            self.assertEqual(a, orig)
+            self.assertRaises(BufferError, a.insert, 1, self.outside)
+            self.assertEqual(a, orig)
+            self.assertRaises(BufferError, a.extend, self.example)
+            self.assertEqual(a, orig)
+
+            def iadd(x):
+                x += array.array(self.typecode, self.biggerexample)
+            self.assertRaises(BufferError, iadd, a)
+            self.assertEqual(a, orig)
+
+            def imul(x):
+                x *= 3
+            self.assertRaises(BufferError, imul, a)
+            self.assertEqual(a, orig)
+
+            def delitem():
+                del a[1]
+            self.assertRaises(BufferError, delitem)
+            self.assertEqual(a, orig)
+
+            # deleting a non-contiguous slice
+            def delslice():
+                del a[1:-1:2]
+            self.assertRaises(BufferError, delslice)
+            self.assertEqual(a, orig)
+
+            # Show that releasing v releases the array for size change
+            v.release()
+            a.pop()
+
 
 class StringTest(BaseTest):
 
@@ -1134,9 +1196,6 @@
         # CPython specific; returns a memory address
         del BaseTest.test_buffer_info
 
-        # No buffers in Jython
-        del BaseTest.test_buffer
-
     test_support.run_unittest(*tests)
 
     # verify reference counting
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -9,8 +9,11 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.ref.WeakReference;
 import java.lang.reflect.Array;
 
+import org.python.core.buffer.BaseBuffer;
+import org.python.core.buffer.SimpleWritableBuffer;
 import org.python.core.util.ByteSwapper;
 import org.python.core.util.StringUtil;
 import org.python.expose.ExposedGet;
@@ -27,7 +30,7 @@
  * See also the jarray module.
  */
 @ExposedType(name = "array.array", base = PyObject.class)
-public class PyArray extends PySequence implements Cloneable {
+public class PyArray extends PySequence implements Cloneable, BufferProtocol {
 
     public static final PyType TYPE = PyType.fromClass(PyArray.class);
 
@@ -257,9 +260,13 @@
 
     @ExposedMethod(type = MethodType.BINARY)
     final PyObject array___imul__(PyObject o) {
+
         if (!o.isIndex()) {
             return null;
         }
+
+        resizeCheck();  // Prohibited if exporting a buffer
+
         if (delegate.getSize() > 0) {
             int count = o.asIndex(Py.OverflowError);
             if (count <= 0) {
@@ -308,6 +315,7 @@
 
     @ExposedMethod(type = MethodType.BINARY)
     final PyObject array___iadd__(PyObject other) {
+
         if (!(other instanceof PyArray)) {
             return null;
         }
@@ -317,6 +325,9 @@
             throw Py.TypeError("can only append arrays of the same type, expected '" + this.type
                     + ", found " + otherArr.type);
         }
+
+        resizeCheck();  // Prohibited if exporting a buffer
+
         delegate.appendArray(otherArr.delegate.copyArray());
         return this;
     }
@@ -430,7 +441,8 @@
 
     @ExposedMethod
     public final void array_append(PyObject value) {
-        append(value);
+        resizeCheck();  // Prohibited if exporting a buffer
+        appendUnchecked(value);
     }
 
     private static int getCodePoint(PyObject obj) {
@@ -464,21 +476,31 @@
      *
      * @param value item to be appended to the array
      */
+    public void append(PyObject value) {
+        resizeCheck();  // Prohibited if exporting a buffer
+        appendUnchecked(value);
+    }
 
-    public void append(PyObject value) {
-        // Currently, this is asymmetric with extend, which
+    /**
+     * Common helper method used internally to append a new value x to the end of the array:
+     * {@link #resizeCheck()} is not called, so the client must do so in advance.
+     *
+     * @param value item to be appended to the array
+     */
+    private final void appendUnchecked(PyObject value) {
+        // Currently, append is asymmetric with extend, which
         // *will* do conversions like append(5.0) to an int array.
-        // Also, cpython 2.2 will do the append coersion. However,
-        // it is deprecated in cpython 2.3, so maybe we are just
+        // Also, CPython 2.2 will do the append coercion. However,
+        // it is deprecated in CPython 2.3, so maybe we are just
         // ahead of our time ;-)
 
         int afterLast = delegate.getSize();
+
         if ("u".equals(typecode)) {
             int codepoint = getCodePoint(value);
             delegate.makeInsertSpace(afterLast);
             Array.setInt(data, afterLast, codepoint);
         } else {
-
             delegate.makeInsertSpace(afterLast);
             try {
                 set(afterLast, value);
@@ -660,8 +682,7 @@
      */
     @Override
     protected void del(int i) {
-        // Now the AbstractArray can support this:
-        // throw Py.TypeError("can't remove from array");
+        resizeCheck();  // Prohibited if exporting a buffer
         delegate.remove(i);
     }
 
@@ -673,6 +694,7 @@
      */
     @Override
     protected void delRange(int start, int stop) {
+        resizeCheck();  // Prohibited if exporting a buffer
         delegate.remove(start, stop);
     }
 
@@ -700,8 +722,8 @@
      *
      * @param iterable object of type PyString, PyArray or any object that can be iterated over.
      */
+    private void extendInternal(PyObject iterable) {
 
-    private void extendInternal(PyObject iterable) {
         if (iterable instanceof PyUnicode) {
             if ("u".equals(typecode)) {
                 extendUnicodeIter(iterable);
@@ -710,14 +732,18 @@
             } else {
                 throw Py.TypeError("an integer is required");
             }
+
         } else if (iterable instanceof PyString) {
             fromstring(((PyString)iterable).toString());
+
         } else if (iterable instanceof PyArray) {
             PyArray source = (PyArray)iterable;
             if (!source.typecode.equals(typecode)) {
                 throw Py.TypeError("can only extend with array of same kind");
             }
+            resizeCheck();  // Prohibited if exporting a buffer
             delegate.appendArray(source.delegate.copyArray());
+
         } else {
             extendInternalIter(iterable);
         }
@@ -729,40 +755,66 @@
      * @param iterable any object that can be iterated over.
      */
     private void extendInternalIter(PyObject iterable) {
-        // iterable object without a length property - cannot presize the
-        // array, so append each item
-        if (iterable.__findattr__("__len__") == null) {
-            for (PyObject item : iterable.asIterable()) {
-                append(item);
-            }
-        } else {
-            // create room
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
+        if (iterable.__findattr__("__len__") != null) {
+            // Make room according to source length
             int last = delegate.getSize();
             delegate.ensureCapacity(last + iterable.__len__());
             for (PyObject item : iterable.asIterable()) {
                 set(last++, item);
                 delegate.size++;
             }
-        }
-    }
 
-    private void extendUnicodeIter(PyObject iterable) {
-        for (PyObject item : iterable.asIterable()) {
-            PyUnicode uitem;
-            try {
-                uitem = (PyUnicode)item;
-            } catch (ClassCastException e) {
-                throw Py.TypeError("Type not compatible with array type");
-            }
-            for (int codepoint : uitem.toCodePoints()) {
-                int afterLast = delegate.getSize();
-                delegate.makeInsertSpace(afterLast);
-                Array.setInt(data, afterLast, codepoint);
+        } else {
+            // iterable has no length property: cannot size the array so append each item.
+            for (PyObject item : iterable.asIterable()) {
+                appendUnchecked(item); // we already did a resizeCheck
             }
         }
     }
 
+    /**
+     * Helper used only when the array elements are Unicode characters (<code>typecode=='u'</code>).
+     * (Characters are stored as integer point codes.) The parameter must be an iterable yielding
+     * <code>PyUnicode</code>s. Often this will be an instance of {@link PyUnicode}, which is an
+     * iterable yielding single-character <code>PyUnicode</code>s. But it is also acceptable to this
+     * method for the argument to yield arbitrary <code>PyUnicode</code>s, which will be
+     * concatenated in the array.
+     *
+     * @param iterable of <code>PyUnicode</code>s
+     */
+    private void extendUnicodeIter(PyObject iterable) {
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
+        try {
+
+            // Append all the code points of all the strings in the iterable
+            for (PyObject item : iterable.asIterable()) {
+                PyUnicode uitem = (PyUnicode)item;
+                // Append all the code points of this item
+                for (int codepoint : uitem.toCodePoints()) {
+                    int afterLast = delegate.getSize();
+                    delegate.makeInsertSpace(afterLast);
+                    Array.setInt(data, afterLast, codepoint);
+                }
+            }
+
+        } catch (ClassCastException e) {
+            // One of the PyUnicodes wasn't
+            throw Py.TypeError("Type not compatible with array type");
+        }
+    }
+
     private void extendArray(int[] items) {
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         int last = delegate.getSize();
         delegate.ensureCapacity(last + items.length);
         for (int item : items) {
@@ -787,21 +839,32 @@
      * @param count number of array elements to read
      */
     public void fromfile(PyObject f, int count) {
-        // check for arg1 as file object
-        if (!(f instanceof PyFile)) {
-            throw Py.TypeError("arg1 must be open file");
+        /*
+         * Prohibit when exporting a buffer. Different from CPython, BufferError takes precedence in
+         * Jython over EOFError: if there's nowhere to write the data, we don't read it.
+         */
+        resizeCheck();
+
+        /*
+         * Now get the required number of bytes from the file. Guard against non-file or closed.
+         */
+        if (f instanceof PyFile) {
+            PyFile file = (PyFile)f;
+            if (!file.getClosed()) {
+                // Load required amount or whatever is available into a bytes object
+                int readbytes = count * getStorageSize();
+                String buffer = file.read(readbytes).toString();
+                fromstring(buffer);
+                // check for underflow
+                if (buffer.length() < readbytes) {
+                    int readcount = buffer.length() / getStorageSize();
+                    throw Py.EOFError("not enough items in file. " + Integer.toString(count)
+                            + " requested, " + Integer.toString(readcount) + " actually read");
+                }
+            }
+            return;
         }
-        PyFile file = (PyFile)f;
-        int readbytes = count * getStorageSize();
-        String buffer = file.read(readbytes).toString();
-        // load whatever was collected into the array
-        fromstring(buffer);
-        // check for underflow
-        if (buffer.length() < readbytes) {
-            int readcount = buffer.length() / getStorageSize();
-            throw Py.EOFError("not enough items in file. " + Integer.toString(count)
-                    + " requested, " + Integer.toString(readcount) + " actually read");
-        }
+        throw Py.TypeError("arg1 must be open file");
     }
 
     @ExposedMethod
@@ -810,7 +873,7 @@
     }
 
     /**
-     * Append items from the list. This is equivalent to "for x in list: a.append(x)"except that if
+     * Append items from the list. This is equivalent to "for x in list: a.append(x)" except that if
      * there is a type error, the array is unchanged.
      *
      * @param obj input list object that will be appended to the array
@@ -819,6 +882,10 @@
         if (!(obj instanceof PyList)) {
             throw Py.TypeError("arg must be list");
         }
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         // store the current size of the internal array
         int size = delegate.getSize();
         try {
@@ -862,11 +929,8 @@
         // Current number of items present
         int origsize = delegate.getSize();
 
-        // Reserve capacity for 'count' items
-        delegate.setSize(origsize + count);
-
         // Read into the array, after the current contents, up to new size (or EOF thrown)
-        int n = fromStream(is, origsize, delegate.getSize(), true);
+        int n = fromStream(is, origsize, origsize + count, true);
         return n - origsize;
     }
 
@@ -888,11 +952,14 @@
      * Helper for reading primitive values from a stream into a slice of the array. Data is read
      * until the array slice is filled or the stream runs out. The purpose of the method is to
      * concentrate in one place the manipulation of bytes into the several primitive element types
-     * on behalf of {@link #fillFromStream(InputStream)} etc.. Since different read methods respond
-     * differently to it, the caller must specify whether the exhaustion of the stream (EOF) should
-     * be treated as an error or not. If the stream does not contain a whole number of items
-     * (possible if the item size is not one byte), the behaviour in respect of the final partial
-     * item and stream position is not defined.
+     * on behalf of {@link #fillFromStream(InputStream)} etc.. The storage is resized if the slice
+     * being written ends beyond the current end of the array, i.e. it is increased to the value of
+     * <code>limit</code>.
+     * <p>
+     * Since different read methods respond differently to it, the caller must specify whether the
+     * exhaustion of the stream (EOF) should be treated as an error or not. If the stream does not
+     * contain a whole number of items (possible if the item size is not one byte), the behaviour in
+     * respect of the final partial item and stream position is not defined.
      *
      * @param dis data stream source for the values
      * @param index first element index to read
@@ -905,7 +972,14 @@
     private int fromStream(InputStream is, int index, int limit, boolean eofIsError)
             throws IOException, EOFException {
 
-        // We need a wrapper capable of encoding the data
+        // Ensure the array is dimensioned to fit the data expected
+        if (limit > delegate.getSize()) {
+            // Prohibited operation if exporting a buffer
+            resizeCheck();
+            delegate.setSize(limit);
+        }
+
+        // We need a wrapper capable of decoding the data from the representation defined by Java.
         DataInputStream dis = new DataInputStream(is);
 
         try {
@@ -1010,11 +1084,18 @@
      */
     @ExposedMethod
     final void array_fromstring(String input) {
+
+        // Check validity wrt array itemsize
         int itemsize = getStorageSize();
         int strlen = input.length();
         if ((strlen % itemsize) != 0) {
             throw Py.ValueError("string length not a multiple of item size");
         }
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
+        // Provide argument as stream of bytes for fromstream method
         ByteArrayInputStream bis = new ByteArrayInputStream(StringUtil.toBytes(input));
         int origsize = delegate.getSize();
         try {
@@ -1279,6 +1360,7 @@
      * @param value value to be inserted into array
      */
     public void insert(int index, PyObject value) {
+        resizeCheck();  // Prohibited operation if exporting a buffer
         index = boundToSequence(index);
         if ("u".equals(typecode)) {
             int codepoint = getCodePoint(value);
@@ -1325,6 +1407,10 @@
         if (index == -1) {
             throw Py.IndexError("pop index out of range");
         }
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         PyObject ret = Py.java2py(Array.get(data, index));
         delegate.remove(index);
         return ret;
@@ -1343,6 +1429,8 @@
     public void remove(PyObject value) {
         int index = indexInternal(value);
         if (index != -1) {
+            // Prohibited operation if exporting a buffer
+            resizeCheck();
             delegate.remove(index);
             return;
         }
@@ -1359,7 +1447,6 @@
     protected PyObject repeat(int count) {
         Object arraycopy = delegate.copyArray();
         PyArray ret = new PyArray(type, 0);
-        // XXX:
         ret.typecode = typecode;
         for (int i = 0; i < count; i++) {
             ret.delegate.appendArray(arraycopy);
@@ -1500,9 +1587,11 @@
      */
     @Override
     protected void setslice(int start, int stop, int step, PyObject value) {
+
         if (stop < start) {
             stop = start;
         }
+
         if (type == Character.TYPE && value instanceof PyString) {
             char[] chars = null;
             // if (value instanceof PyString) {
@@ -1510,8 +1599,14 @@
                 throw Py.ValueError("invalid bounds for setting from string");
             }
             chars = value.toString().toCharArray();
+            if (start + chars.length != stop) {
+                // This is a size-changing operation: check for buffer exports
+                resizeCheck();
+            }
             delegate.replaceSubArray(start, stop, chars, 0, chars.length);
+
         } else {
+
             if (value instanceof PyString && type == Byte.TYPE) {
                 byte[] chars = ((PyString)value).toBytes();
                 if (chars.length == stop - start && step == 1) {
@@ -1519,12 +1614,14 @@
                 } else {
                     throw Py.ValueError("invalid bounds for setting from string");
                 }
+
             } else if (value instanceof PyArray) {
                 PyArray array = (PyArray)value;
                 if (!array.typecode.equals(typecode)) {
                     throw Py.TypeError("bad argument type for built-in operation|" + array.typecode
                             + "|" + typecode);
                 }
+
                 if (step == 1) {
                     Object arrayDelegate;
                     if (array == this) {
@@ -1532,19 +1629,25 @@
                     } else {
                         arrayDelegate = array.delegate.getArray();
                     }
+                    int len = array.delegate.getSize();
+                    if (start + len != stop) {
+                        // This is a size-changing operation: check for buffer exports
+                        resizeCheck();
+                    }
                     try {
-                        delegate.replaceSubArray(start, stop, arrayDelegate, 0,
-                                array.delegate.getSize());
+                        delegate.replaceSubArray(start, stop, arrayDelegate, 0, len);
                     } catch (IllegalArgumentException e) {
                         throw Py.TypeError("Slice typecode '" + array.typecode
                                 + "' is not compatible with this array (typecode '" + this.typecode
                                 + "')");
                     }
+
                 } else if (step > 1) {
                     int len = array.__len__();
                     for (int i = 0, j = 0; i < len; i++, j += step) {
                         Array.set(data, j + start, Array.get(array.data, i));
                     }
+
                 } else if (step < 0) {
                     if (array == this) {
                         array = (PyArray)array.clone();
@@ -1554,6 +1657,7 @@
                         Array.set(data, j, Array.get(array.data, i));
                     }
                 }
+
             } else {
                 throw Py.TypeError(String.format("can only assign array (not \"%.200s\") to array "
                         + "slice", value.getType().fastGetName()));
@@ -1805,4 +1909,104 @@
             return Array.newInstance(baseType, size);
         }
     }
+
+    /*
+     * ============================================================================================
+     * Support for the Buffer API
+     * ============================================================================================
+     *
+     * The buffer API allows other classes to access the storage directly.
+     *
+     * This is a close duplicate of the same mechanism in PyByteArray. There is perhaps scope for a
+     * shared helper class to implement this logic. For type code 'b', the workings are almost
+     * identical. The fully-fledged buffer interface for PyArray is richer, more like the Python 3
+     * memoryview, as it must cope with items of size other than one byte. This goes beyond the
+     * capabilities of the Jython BufferProtocol at this stage of its development.
+     */
+
+    /**
+     * Hold weakly a reference to a PyBuffer export not yet released, used to prevent untimely
+     * resizing.
+     */
+    private WeakReference<BaseBuffer> export;
+
+    /**
+     * {@inheritDoc}
+     * <p>
+     * The {@link PyBuffer} returned from this method is a one-dimensional array of single byte
+     * items that allows modification of the object state. The existence of this export <b>prohibits
+     * resizing</b> the byte array. This prohibition is not only on the consumer of the view but
+     * extends to any other operations, such as any kind or insertion or deletion.
+     */
+    @Override
+    public synchronized PyBuffer getBuffer(int flags) {
+
+        // If we have already exported a buffer it may still be available for re-use
+        BaseBuffer pybuf = getExistingBuffer(flags);
+
+        if (pybuf == null) {
+            // No existing export we can re-use: create a new one
+            if ("b".equals(typecode)) {
+                // This is byte data, so we are within the state of the art
+                byte[] storage = (byte[])data;
+                int size = delegate.getSize();
+                pybuf = new SimpleWritableBuffer(flags, storage, 0, size);
+                // Hold a reference for possible re-use
+                export = new WeakReference<BaseBuffer>(pybuf);
+
+            } else {
+                // For the time being ...
+                throw Py.NotImplementedError("only array('b') can export a buffer");
+            }
+        }
+
+        return pybuf;
+    }
+
+    /**
+     * Try to re-use an existing exported buffer, or return <code>null</code> if we can't.
+     *
+     * @throws PyException (BufferError) if the the flags are incompatible with the buffer
+     */
+    private BaseBuffer getExistingBuffer(int flags) throws PyException {
+        BaseBuffer pybuf = null;
+        if (export != null) {
+            // A buffer was exported at some time.
+            pybuf = export.get();
+            if (pybuf != null) {
+                /*
+                 * We do not test for pybuf.isReleased() as, if any operation had taken place that
+                 * invalidated the buffer, resizeCheck() would have set export=null. The exported
+                 * buffer (navigation, buf member, etc.) remains valid through any operation that
+                 * does not need a resizeCheck.
+                 */
+                pybuf = pybuf.getBufferAgain(flags);
+            }
+        }
+        return pybuf;
+    }
+
+    /**
+     * Test to see if the array may be resized and raise a BufferError if not. This must be called
+     * by the implementation of any operation that changes the number of elements in the array.
+     *
+     * @throws PyException (BufferError) if there are buffer exports preventing a resize
+     */
+    private void resizeCheck() throws PyException {
+        if (export != null) {
+            // A buffer was exported at some time and we have not explicitly discarded it.
+            PyBuffer pybuf = export.get();
+            if (pybuf != null && !pybuf.isReleased()) {
+                // A consumer still has the exported buffer
+                throw Py.BufferError("cannot resize an array that is exporting buffers");
+            } else {
+                /*
+                 * Either the reference has expired or all consumers have released it. Either way,
+                 * the weak reference is useless now.
+                 */
+                export = null;
+            }
+        }
+    }
+
 }
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -377,15 +377,30 @@
     }
 
     /**
-     * Return a String for writing to the underlying file from obj.
+     * Return a String for writing to the underlying file from obj. This is a helper for {@link file_write}
+     * and {@link file_writelines}.
+     *
+     * @param obj to write
+     * @param message for TypeError if raised (or null for default message)
+     * @return bytes representing tha value (as a String in the Jython convention)
      */
     private String asWritable(PyObject obj, String message) {
 
         if (obj instanceof PyUnicode) {
+            // By convention, use platform default encoding to bytes
             return ((PyUnicode)obj).encode();
 
         } else if (obj instanceof PyString) {
-            return ((PyString) obj).getString();
+            // Take a short cut
+            return ((PyString)obj).getString();
+
+        } else if (obj instanceof PyArray) {
+            if (binary) {
+                // PyArray has the buffer interface but it only works for bytes at present
+                return ((PyArray)obj).tostring();
+            } else {
+                // Fall through to TypeError
+            }
 
         } else if (obj instanceof BufferProtocol) {
             // Try to get a simple byte-oriented buffer
@@ -394,21 +409,19 @@
                 buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
                 return StringUtil.fromBytes(buf);
             } catch (Exception e) {
-                // Wrong kind of buffer: generic error message will do
+                // Wrong kind of buffer: generic/supplied error message will do
             } finally {
                 // If we got a buffer, we should release it
                 if (buf != null) {
                     buf.release();
                 }
             }
+        }
 
-        } else if (binary && obj instanceof PyArray) {
-            return ((PyArray)obj).tostring();
-        }
         if (message == null) {
-            message = String.format("argument 1 must be string or %sbuffer, not %.200s",
-                                    binary ? "" : "read-only character ",
-                                    obj.getType().fastGetName());
+            // Messages differ for text or binary streams (CPython) but we always add the type
+            String.format("%s buffer, not %.200s", (binary ? "must be string or"
+                    : "expected a character"), obj.getType().fastGetName());
         }
         throw Py.TypeError(message);
     }
@@ -581,7 +594,7 @@
         }
     }
 
-  
+
     /**
      * XXX update docs - A mechanism to make sure PyFiles are closed on exit. On creation Closer adds itself
      * to a list of Closers that will be run by PyFileCloser on JVM shutdown. When a
@@ -615,6 +628,7 @@
         }
 
         /** For closing as part of a shutdown process */
+        @Override
         public Void call() {
             file.close();
             sys = null;
diff --git a/src/org/python/core/io/TextIOBase.java b/src/org/python/core/io/TextIOBase.java
--- a/src/org/python/core/io/TextIOBase.java
+++ b/src/org/python/core/io/TextIOBase.java
@@ -101,14 +101,25 @@
      * @return the amount of data read as an int
      */
     public int readinto(PyObject buf) {
+
         // This is an inefficient version of readinto: but readinto is
         // not recommended for use in Python 2.x anyway
-        if (buf instanceof BufferProtocol) {
+
+        if (buf instanceof PyArray) {
+            // PyArray has the buffer interface but it only works for bytes at present
+            PyArray array = (PyArray)buf;
+            String read = read(array.__len__());
+            for (int i = 0; i < read.length(); i++) {
+                array.set(i, new PyString(read.charAt(i)));
+            }
+            return read.length();
+
+        } else if (buf instanceof BufferProtocol) {
             PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.SIMPLE);
             if (view.isReadonly()) {
                 // More helpful than falling through to CPython message
-                throw Py.TypeError("cannot read into read-only "
-                        + buf.getType().fastGetName());
+                throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
+
             } else {
                 try {
                     // Inefficiently, we have to go via a String
@@ -122,15 +133,8 @@
                     // We should release the buffer explicitly
                     view.release();
                 }
+
             }
-
-        } else if (buf instanceof PyArray) {
-            PyArray array = (PyArray)buf;
-            String read = read(array.__len__());
-            for (int i = 0; i < read.length(); i++) {
-                array.set(i, new PyString(read.charAt(i)));
-            }
-            return read.length();
         }
 
         // No valid alternative worked

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 15 00:01:38 2013
From: jython-checkins at python.org (jeff.allen)
Date: Fri, 15 Nov 2013 00:01:38 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_PyArray=2Efromstring_accept?=
	=?utf-8?q?s_buffer_API_objects?=
Message-ID: <3dLJ8y0kZNz7Ljq@mail.python.org>

http://hg.python.org/jython/rev/db2d93777787
changeset:   7150:db2d93777787
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Mon Nov 11 22:21:12 2013 +0000
summary:
  PyArray.fromstring accepts buffer API objects
Also test in test_array to accept sliced/strided buffers.

files:
  Lib/test/test_array.py           |   35 ++++-
  src/org/python/core/PyArray.java |  143 ++++++++++++++++--
  2 files changed, 159 insertions(+), 19 deletions(-)


diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -246,6 +246,39 @@
         self.assertEqual(a, b)
         if a.itemsize>1 and self.typecode not in ('b', 'B'):
             self.assertRaises(ValueError, b.fromstring, "x")
+        # Test from byte string available via buffer API (Jython addition)
+        if test_support.is_jython:
+            for buftype in (buffer, memoryview, bytearray):
+                b = array.array(self.typecode)
+                b.fromstring(buftype(a.tostring()))
+                self.assertEqual(a, b)
+
+    @unittest.skipUnless(test_support.is_jython, "Jython supports memoryview slices")
+    def test_tofromstring_sliced(self):
+        a = array.array(self.typecode, self.example)
+        r = bytearray(a.tostring())
+        R = len(r)
+        D = 3*R
+
+        def checkSlice(x, y, z=None):
+            # Scatter the bytes of a.tostring() into d
+            d = bytearray(D)
+            d[x:y:z] = r
+            # Now gather through a memoryview slice
+            with memoryview(d) as m:
+                # Requires proper use of strides when z not None and not 1
+                b = array.array(self.typecode)
+                b.fromstring(m[x:y:z])
+                self.assertEqual(a, b)
+
+        # The slices all have R elements and the whole range D = 3*R
+        checkSlice(None, R)
+        checkSlice(2, 2+R)
+        checkSlice(D-R, None)
+        checkSlice(None, None, 3)
+        checkSlice(None, None, -3)
+        checkSlice(None, D-R-1, -1)
+        checkSlice(R-1, None, -1)
 
     def test_filewrite(self):
         a = array.array(self.typecode, 2*self.example)
@@ -828,7 +861,7 @@
         # SF bug #1486663 -- this used to erroneously raise a TypeError
         ArraySubclassWithKwargs('b', newarg=1)
 
-    @unittest.skipIf(not test_support.is_jython, "array supports buffer interface in Jython")
+    @unittest.skipUnless(test_support.is_jython, "array supports buffer interface in Jython")
     def test_resize_forbidden(self):
         # Test that array resizing is forbidden with buffer exports (Jython addition).
         # Test adapted from corresponding one in test_bytes.
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -77,19 +77,26 @@
     @ExposedNew
     static final PyObject array_new(PyNewWrapper new_, boolean init, PyType subtype,
             PyObject[] args, String[] keywords) {
+
         if (new_.for_type != subtype && keywords.length > 0) {
+            /*
+             * We're constructing as a base for a derived type (via PyDerived) and there are
+             * keywords. The effective args locally should not include the keywords.
+             */
             int argc = args.length - keywords.length;
             PyObject[] justArgs = new PyObject[argc];
             System.arraycopy(args, 0, justArgs, 0, argc);
             args = justArgs;
         }
+
+        // Build the argument parser for this call
         ArgParser ap =
                 new ArgParser("array", args, Py.NoKeywords,
                         new String[] {"typecode", "initializer"}, 1);
         ap.noKeywords();
+
+        // Retrieve the mandatory type code that determines the element type
         PyObject obj = ap.getPyObject(0);
-        PyObject initial = ap.getPyObject(1, null);
-
         Class<?> type;
         String typecode;
         if (obj instanceof PyString && !(obj instanceof PyUnicode)) {
@@ -106,32 +113,48 @@
                     + obj.getType().fastGetName());
         }
 
+        /*
+         * Create a 'blank canvas' of the appropriate concrete class.
+         */
         PyArray self;
         if (new_.for_type == subtype) {
             self = new PyArray(subtype);
         } else {
             self = new PyArrayDerived(subtype);
         }
+
         // Initialize the typecode (and validate type) before creating the backing Array
         class2char(type);
         self.setup(type, Array.newInstance(type, 0));
         self.typecode = typecode;
+
+        /*
+         * The initialiser may be omitted, or may validly be one of several types in the broad
+         * categories of a byte string (which is treated as a machine representation of the data) or
+         * an iterable yielding values assignable to the elements. There is special treatment for
+         * type 'u' Unicode.
+         */
+        PyObject initial = ap.getPyObject(1, null);
         if (initial == null) {
-            return self;
-        }
-        if (initial instanceof PyList) {
+            // Fall through
+
+        } else if (initial instanceof PyList) {
             self.fromlist(initial);
+
         } else if (initial instanceof PyString && !(initial instanceof PyUnicode)) {
             self.fromstring(initial.toString());
+
         } else if ("u".equals(typecode)) {
             if (initial instanceof PyUnicode) {
                 self.extendArray(((PyUnicode)initial).toCodePoints());
             } else {
                 self.extendUnicodeIter(initial);
             }
+
         } else {
             self.extendInternal(initial);
         }
+
         return self;
     }
 
@@ -707,7 +730,7 @@
      * Append items from <code>iterable</code> to the end of the array. If iterable is another
      * array, it must have exactly the same type code; if not, TypeError will be raised. If iterable
      * is not an array, it must be iterable and its elements must be the right type to be appended
-     * to the array. Changed in version 2.4: Formerly, the argument could only be another array.
+     * to the array.
      *
      * @param iterable iterable object used to extend the array
      */
@@ -733,8 +756,9 @@
                 throw Py.TypeError("an integer is required");
             }
 
-        } else if (iterable instanceof PyString) {
-            fromstring(((PyString)iterable).toString());
+// } else if (iterable instanceof PyString) {
+// // XXX CPython treats a str/bytes as an iterable, not as previously here:
+// fromstring(((PyString)iterable).toString());
 
         } else if (iterable instanceof PyArray) {
             PyArray source = (PyArray)iterable;
@@ -1067,12 +1091,14 @@
     }
 
     /**
-     * Appends items from the string, interpreting the string as an array of machine values (as if
-     * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+     * Appends items from the object, which is a byte string of some kind (PyString or object with
+     * the buffer interface providing bytes) The string of bytes is interpreted as an array of
+     * machine values (as if it had been read from a file using the {@link #fromfile(PyObject, int)
+     * fromfile()} method).
      *
      * @param input string of bytes containing array data
      */
-    public void fromstring(String input) {
+    public void fromstring(PyObject input) {
         array_fromstring(input);
     }
 
@@ -1082,27 +1108,108 @@
      *
      * @param input string of bytes containing array data
      */
+    public void fromstring(String input) {
+        frombytesInternal(StringUtil.toBytes(input));
+    }
+
+    /**
+     * Appends items from the string, interpreting the string as an array of machine values (as if
+     * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+     *
+     * @param input string of bytes containing array data
+     */
     @ExposedMethod
-    final void array_fromstring(String input) {
+    final void array_fromstring(PyObject input) {
+
+        if (input instanceof BufferProtocol) {
+
+            if (input instanceof PyUnicode) {
+                // Unicode is treated as specifying a byte string via the default encoding.
+                String s = ((PyUnicode)input).encode();
+                frombytesInternal(StringUtil.toBytes(s));
+
+            } else {
+                // Access the bytes
+                PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO);
+                try {
+                    // Provide argument as stream of bytes for fromstream method
+                    if (pybuf.getNdim() == 1) {
+                        if (pybuf.getStrides()[0] == 1) {
+                            // Data are contiguous in a byte[]
+                            PyBuffer.Pointer b = pybuf.getBuf();
+                            frombytesInternal(b.storage, b.offset, pybuf.getLen());
+                        } else {
+                            // As frombytesInternal only knows contiguous bytes, make a copy.
+                            byte[] copy = new byte[pybuf.getLen()];
+                            pybuf.copyTo(copy, 0);
+                            frombytesInternal(copy);
+                        }
+                    } else {
+                        // Currently don't support n-dimensional sources
+                        throw Py.ValueError("multi-dimensional buffer not supported");
+                    }
+                } finally {
+                    pybuf.release();
+                }
+            }
+
+        } else {
+            String fmt = "must be string or read-only buffer, not %s";
+            throw Py.TypeError(String.format(fmt, input.getType().fastGetName()));
+        }
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code>
+     *
+     * @param input string of bytes encoding the array data
+     */
+    private final void fromstringInternal(String input) {
+        frombytesInternal(StringUtil.toBytes(input));
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+     * <code>.frombytes()</code> (Python 3.2+ name).
+     *
+     * @param bytes array containing the new array data in machine encoding
+     */
+    private final void frombytesInternal(byte[] bytes) {
+        frombytesInternal(bytes, 0, bytes.length);
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+     * <code>.frombytes()</code> (Python 3.2+ name).
+     *
+     * @param bytes array containing the new array data in machine encoding
+     * @param offset of the first byte to read
+     * @param count of bytes to read
+     */
+    private final void frombytesInternal(byte[] bytes, int offset, int count) {
+
+        // Access the bytes
+        int origsize = delegate.getSize();
 
         // Check validity wrt array itemsize
         int itemsize = getStorageSize();
-        int strlen = input.length();
-        if ((strlen % itemsize) != 0) {
+        if ((count % itemsize) != 0) {
             throw Py.ValueError("string length not a multiple of item size");
         }
 
-        // Prohibited operation if exporting a buffer
+        // Prohibited operation if we are exporting a buffer
         resizeCheck();
 
-        // Provide argument as stream of bytes for fromstream method
-        ByteArrayInputStream bis = new ByteArrayInputStream(StringUtil.toBytes(input));
-        int origsize = delegate.getSize();
         try {
+
+            // Provide argument as stream of bytes for fromstream method
+            ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset, count);
             fromStream(bis);
+
         } catch (EOFException e) {
             // stubbed catch for fromStream throws
             throw Py.EOFError("not enough items in string");
+
         } catch (IOException e) {
             // discard anything successfully loaded
             delegate.setSize(origsize);

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 15 00:01:39 2013
From: jython-checkins at python.org (jeff.allen)
Date: Fri, 15 Nov 2013 00:01:39 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_read-only_buffer_API_to?=
 =?utf-8?q?_array=2Earray_for_all_type_codes=2E?=
Message-ID: <3dLJ8z3jdgz7LmW@mail.python.org>

http://hg.python.org/jython/rev/0d5f6b1e0a04
changeset:   7151:0d5f6b1e0a04
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Thu Nov 14 21:29:18 2013 +0000
summary:
  Add read-only buffer API to array.array for all type codes.
Fixes a regression in test_re where arrays of arbitrary type are required
to be arguments to PatternObject.match(), which accepts buffer
API objects.

files:
  Lib/test/test_array.py                        |  101 ++++-----
  src/org/python/core/PyArray.java              |   11 +-
  src/org/python/core/PyFile.java               |   25 +-
  src/org/python/modules/sre/PatternObject.java |   22 +-
  4 files changed, 70 insertions(+), 89 deletions(-)


diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -832,11 +832,10 @@
         gc.collect()
 
     def test_buffer(self):
-        if self.typecode == 'b':    # FIXME: Jython supports only the byte buffer
-            a = array.array(self.typecode, self.example)
-            with test_support.check_py3k_warnings():
-                b = buffer(a)
-            self.assertEqual(b[0], a.tostring()[0])
+        a = array.array(self.typecode, self.example)
+        with test_support.check_py3k_warnings():
+            b = buffer(a)
+        self.assertEqual(b[0], a.tostring()[0])
 
     def test_weakref(self):
         s = array.array(self.typecode, self.example)
@@ -869,58 +868,57 @@
         # if it wouldn't reallocate the underlying array.
         # Furthermore, no destructive changes to the buffer may be applied
         # before raising the error.
-        if self.typecode == 'b':    # Jython supports only the byte buffer
-            a = array.array(self.typecode, self.example)
-            def resize(n):
-                "n = -1 -> Smaller, 0 -> the same, or 1 -> larger."
-                a[1:-1] = array.array(self.typecode, self.example[1-n:-1])
+        a = array.array(self.typecode, self.example)
+        def resize(n):
+            "n = -1 -> Smaller, 0 -> the same, or 1 -> larger."
+            a[1:-1] = array.array(self.typecode, self.example[1-n:-1])
 
-            v = memoryview(a)
-            orig = a[:]
+        v = memoryview(a)
+        orig = a[:]
 
-            self.assertRaises(BufferError, resize, -1)
-            self.assertEqual(a, orig)
-            #self.assertRaises(BufferError, resize, 0)
-            #self.assertEqual(a, orig)
-            self.assertRaises(BufferError, resize, 1)
-            self.assertEqual(a, orig)
+        self.assertRaises(BufferError, resize, -1)
+        self.assertEqual(a, orig)
+        #self.assertRaises(BufferError, resize, 0)
+        #self.assertEqual(a, orig)
+        self.assertRaises(BufferError, resize, 1)
+        self.assertEqual(a, orig)
 
-            # Other operations implying resize
-            self.assertRaises(BufferError, a.pop, 0)
-            self.assertEqual(a, orig)
-            self.assertRaises(BufferError, a.remove, a[1])
-            self.assertEqual(a, orig)
-            self.assertRaises(BufferError, a.append, self.outside)
-            self.assertEqual(a, orig)
-            self.assertRaises(BufferError, a.insert, 1, self.outside)
-            self.assertEqual(a, orig)
-            self.assertRaises(BufferError, a.extend, self.example)
-            self.assertEqual(a, orig)
+        # Other operations implying resize
+        self.assertRaises(BufferError, a.pop, 0)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.remove, a[1])
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.append, self.outside)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.insert, 1, self.outside)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.extend, self.example)
+        self.assertEqual(a, orig)
 
-            def iadd(x):
-                x += array.array(self.typecode, self.biggerexample)
-            self.assertRaises(BufferError, iadd, a)
-            self.assertEqual(a, orig)
+        def iadd(x):
+            x += array.array(self.typecode, self.biggerexample)
+        self.assertRaises(BufferError, iadd, a)
+        self.assertEqual(a, orig)
 
-            def imul(x):
-                x *= 3
-            self.assertRaises(BufferError, imul, a)
-            self.assertEqual(a, orig)
+        def imul(x):
+            x *= 3
+        self.assertRaises(BufferError, imul, a)
+        self.assertEqual(a, orig)
 
-            def delitem():
-                del a[1]
-            self.assertRaises(BufferError, delitem)
-            self.assertEqual(a, orig)
+        def delitem():
+            del a[1]
+        self.assertRaises(BufferError, delitem)
+        self.assertEqual(a, orig)
 
-            # deleting a non-contiguous slice
-            def delslice():
-                del a[1:-1:2]
-            self.assertRaises(BufferError, delslice)
-            self.assertEqual(a, orig)
+        # deleting a non-contiguous slice
+        def delslice():
+            del a[1:-1:2]
+        self.assertRaises(BufferError, delslice)
+        self.assertEqual(a, orig)
 
-            # Show that releasing v releases the array for size change
-            v.release()
-            a.pop()
+        # Show that releasing v releases the array for size change
+        v.release()
+        a.pop()
 
 
 class StringTest(BaseTest):
@@ -1033,9 +1031,8 @@
         self.assertEqual(a, array.array(self.typecode, [1,2,3,4,5,6,7,8,9]))
         # test issue7788
         a = array.array(self.typecode, range(10))
-        # FIXME #1860: not working on Jython yet.
-        if not test_support.is_jython:
-            del a[9::1<<333]
+        del a[9::1<<333]
+        self.assertEqual(a, array.array(self.typecode, range(9)))
 
     def test_assignment(self):
         a = array.array(self.typecode, range(10))
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -13,6 +13,7 @@
 import java.lang.reflect.Array;
 
 import org.python.core.buffer.BaseBuffer;
+import org.python.core.buffer.SimpleStringBuffer;
 import org.python.core.buffer.SimpleWritableBuffer;
 import org.python.core.util.ByteSwapper;
 import org.python.core.util.StringUtil;
@@ -2058,13 +2059,15 @@
                 byte[] storage = (byte[])data;
                 int size = delegate.getSize();
                 pybuf = new SimpleWritableBuffer(flags, storage, 0, size);
-                // Hold a reference for possible re-use
-                export = new WeakReference<BaseBuffer>(pybuf);
-
+            } else if ((flags & PyBUF.WRITABLE) == 0) {
+                // As the client only intends to read, fake the answer with a String
+                pybuf = new SimpleStringBuffer(flags, tostring());
             } else {
                 // For the time being ...
-                throw Py.NotImplementedError("only array('b') can export a buffer");
+                throw Py.NotImplementedError("only array('b') can export a writable buffer");
             }
+            // Hold a reference for possible re-use
+            export = new WeakReference<BaseBuffer>(pybuf);
         }
 
         return pybuf;
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -394,27 +394,18 @@
             // Take a short cut
             return ((PyString)obj).getString();
 
-        } else if (obj instanceof PyArray) {
-            if (binary) {
-                // PyArray has the buffer interface but it only works for bytes at present
-                return ((PyArray)obj).tostring();
-            } else {
-                // Fall through to TypeError
-            }
+        } else if (obj instanceof PyArray && !binary) {
+            // Fall through to TypeError. (If binary, BufferProtocol takes care of PyArray.)
 
         } else if (obj instanceof BufferProtocol) {
-            // Try to get a simple byte-oriented buffer
-            PyBuffer buf = null;
+            // Try to get a byte-oriented buffer
+            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
             try {
-                buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
-                return StringUtil.fromBytes(buf);
-            } catch (Exception e) {
-                // Wrong kind of buffer: generic/supplied error message will do
+                // ... and treat those bytes as a String
+                return buf.toString();
             } finally {
-                // If we got a buffer, we should release it
-                if (buf != null) {
-                    buf.release();
-                }
+                // We should release the buffer
+                buf.release();
             }
         }
 
diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java
--- a/src/org/python/modules/sre/PatternObject.java
+++ b/src/org/python/modules/sre/PatternObject.java
@@ -372,28 +372,18 @@
             return (PyString)obj;
 
         } else if (obj instanceof BufferProtocol) {
-            // Try to get a simple byte-oriented buffer
-            PyBuffer buf = null;
+            // Try to get a byte-oriented buffer
+            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
             try {
-                buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
                 // ... and treat those bytes as a PyString
-                String s = StringUtil.fromBytes(buf);
-                return new PyString(s);
-            } catch (Exception e) {
-                // Wrong kind of buffer: generic error message will do
+                return new PyString(buf.toString());
             } finally {
-                // If we got a buffer, we should release it
-                if (buf != null) {
-                    buf.release();
-                }
+                // We should release the buffer
+                buf.release();
             }
-
-        } else if (obj instanceof PyArray) {
-            // PyArray can do something similar
-            return new PyString(obj.toString());
         }
 
-        // None of those things worked
+        // Neither of those things worked
         throw Py.TypeError("expected string or buffer, but got " + obj.getType());
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 15 00:01:41 2013
From: jython-checkins at python.org (jeff.allen)
Date: Fri, 15 Nov 2013 00:01:41 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_buffer_work_on_PyArray?=
Message-ID: <3dLJ911wq0z7Lll@mail.python.org>

http://hg.python.org/jython/rev/5f306c9c85e3
changeset:   7152:5f306c9c85e3
parent:      7147:07ca5124f859
parent:      7151:0d5f6b1e0a04
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Thu Nov 14 21:38:49 2013 +0000
summary:
  Merge buffer work on PyArray

files:
  Lib/test/test_array.py                        |  101 ++-
  Lib/test/test_bytes.py                        |    5 +
  src/org/python/core/PyArray.java              |  466 ++++++++-
  src/org/python/core/PyFile.java               |   43 +-
  src/org/python/core/PyMemoryView.java         |   20 +-
  src/org/python/core/io/TextIOBase.java        |   26 +-
  src/org/python/modules/sre/PatternObject.java |   22 +-
  7 files changed, 545 insertions(+), 138 deletions(-)


diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -246,6 +246,39 @@
         self.assertEqual(a, b)
         if a.itemsize>1 and self.typecode not in ('b', 'B'):
             self.assertRaises(ValueError, b.fromstring, "x")
+        # Test from byte string available via buffer API (Jython addition)
+        if test_support.is_jython:
+            for buftype in (buffer, memoryview, bytearray):
+                b = array.array(self.typecode)
+                b.fromstring(buftype(a.tostring()))
+                self.assertEqual(a, b)
+
+    @unittest.skipUnless(test_support.is_jython, "Jython supports memoryview slices")
+    def test_tofromstring_sliced(self):
+        a = array.array(self.typecode, self.example)
+        r = bytearray(a.tostring())
+        R = len(r)
+        D = 3*R
+
+        def checkSlice(x, y, z=None):
+            # Scatter the bytes of a.tostring() into d
+            d = bytearray(D)
+            d[x:y:z] = r
+            # Now gather through a memoryview slice
+            with memoryview(d) as m:
+                # Requires proper use of strides when z not None and not 1
+                b = array.array(self.typecode)
+                b.fromstring(m[x:y:z])
+                self.assertEqual(a, b)
+
+        # The slices all have R elements and the whole range D = 3*R
+        checkSlice(None, R)
+        checkSlice(2, 2+R)
+        checkSlice(D-R, None)
+        checkSlice(None, None, 3)
+        checkSlice(None, None, -3)
+        checkSlice(None, D-R-1, -1)
+        checkSlice(R-1, None, -1)
 
     def test_filewrite(self):
         a = array.array(self.typecode, 2*self.example)
@@ -827,6 +860,66 @@
         # SF bug #1486663 -- this used to erroneously raise a TypeError
         ArraySubclassWithKwargs('b', newarg=1)
 
+    @unittest.skipUnless(test_support.is_jython, "array supports buffer interface in Jython")
+    def test_resize_forbidden(self):
+        # Test that array resizing is forbidden with buffer exports (Jython addition).
+        # Test adapted from corresponding one in test_bytes.
+        # We can't resize an array when there are buffer exports, even
+        # if it wouldn't reallocate the underlying array.
+        # Furthermore, no destructive changes to the buffer may be applied
+        # before raising the error.
+        a = array.array(self.typecode, self.example)
+        def resize(n):
+            "n = -1 -> Smaller, 0 -> the same, or 1 -> larger."
+            a[1:-1] = array.array(self.typecode, self.example[1-n:-1])
+
+        v = memoryview(a)
+        orig = a[:]
+
+        self.assertRaises(BufferError, resize, -1)
+        self.assertEqual(a, orig)
+        #self.assertRaises(BufferError, resize, 0)
+        #self.assertEqual(a, orig)
+        self.assertRaises(BufferError, resize, 1)
+        self.assertEqual(a, orig)
+
+        # Other operations implying resize
+        self.assertRaises(BufferError, a.pop, 0)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.remove, a[1])
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.append, self.outside)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.insert, 1, self.outside)
+        self.assertEqual(a, orig)
+        self.assertRaises(BufferError, a.extend, self.example)
+        self.assertEqual(a, orig)
+
+        def iadd(x):
+            x += array.array(self.typecode, self.biggerexample)
+        self.assertRaises(BufferError, iadd, a)
+        self.assertEqual(a, orig)
+
+        def imul(x):
+            x *= 3
+        self.assertRaises(BufferError, imul, a)
+        self.assertEqual(a, orig)
+
+        def delitem():
+            del a[1]
+        self.assertRaises(BufferError, delitem)
+        self.assertEqual(a, orig)
+
+        # deleting a non-contiguous slice
+        def delslice():
+            del a[1:-1:2]
+        self.assertRaises(BufferError, delslice)
+        self.assertEqual(a, orig)
+
+        # Show that releasing v releases the array for size change
+        v.release()
+        a.pop()
+
 
 class StringTest(BaseTest):
 
@@ -938,9 +1031,8 @@
         self.assertEqual(a, array.array(self.typecode, [1,2,3,4,5,6,7,8,9]))
         # test issue7788
         a = array.array(self.typecode, range(10))
-        # FIXME #1860: not working on Jython yet.
-        if not test_support.is_jython:
-            del a[9::1<<333]
+        del a[9::1<<333]
+        self.assertEqual(a, array.array(self.typecode, range(9)))
 
     def test_assignment(self):
         a = array.array(self.typecode, range(10))
@@ -1134,9 +1226,6 @@
         # CPython specific; returns a memory address
         del BaseTest.test_buffer_info
 
-        # No buffers in Jython
-        del BaseTest.test_buffer
-
     test_support.run_unittest(*tests)
 
     # verify reference counting
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -886,6 +886,11 @@
         self.assertRaises(BufferError, delslice)
         self.assertEqual(b, orig)
 
+        if test.test_support.is_jython:
+            # Show that releasing v releases the bytearray for size change
+            v.release()
+            b.pop()
+
     def test_empty_bytearray(self):
         # Issue #7561: operations on empty bytearrays could crash in many
         # situations, due to a fragile implementation of the
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -9,8 +9,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.lang.ref.WeakReference;
 import java.lang.reflect.Array;
 
+import org.python.core.buffer.BaseBuffer;
+import org.python.core.buffer.SimpleStringBuffer;
+import org.python.core.buffer.SimpleWritableBuffer;
 import org.python.core.util.ByteSwapper;
 import org.python.core.util.StringUtil;
 import org.python.expose.ExposedGet;
@@ -27,7 +31,7 @@
  * See also the jarray module.
  */
 @ExposedType(name = "array.array", base = PyObject.class)
-public class PyArray extends PySequence implements Cloneable {
+public class PyArray extends PySequence implements Cloneable, BufferProtocol {
 
     public static final PyType TYPE = PyType.fromClass(PyArray.class);
 
@@ -74,19 +78,26 @@
     @ExposedNew
     static final PyObject array_new(PyNewWrapper new_, boolean init, PyType subtype,
             PyObject[] args, String[] keywords) {
+
         if (new_.for_type != subtype && keywords.length > 0) {
+            /*
+             * We're constructing as a base for a derived type (via PyDerived) and there are
+             * keywords. The effective args locally should not include the keywords.
+             */
             int argc = args.length - keywords.length;
             PyObject[] justArgs = new PyObject[argc];
             System.arraycopy(args, 0, justArgs, 0, argc);
             args = justArgs;
         }
+
+        // Build the argument parser for this call
         ArgParser ap =
                 new ArgParser("array", args, Py.NoKeywords,
                         new String[] {"typecode", "initializer"}, 1);
         ap.noKeywords();
+
+        // Retrieve the mandatory type code that determines the element type
         PyObject obj = ap.getPyObject(0);
-        PyObject initial = ap.getPyObject(1, null);
-
         Class<?> type;
         String typecode;
         if (obj instanceof PyString && !(obj instanceof PyUnicode)) {
@@ -103,32 +114,48 @@
                     + obj.getType().fastGetName());
         }
 
+        /*
+         * Create a 'blank canvas' of the appropriate concrete class.
+         */
         PyArray self;
         if (new_.for_type == subtype) {
             self = new PyArray(subtype);
         } else {
             self = new PyArrayDerived(subtype);
         }
+
         // Initialize the typecode (and validate type) before creating the backing Array
         class2char(type);
         self.setup(type, Array.newInstance(type, 0));
         self.typecode = typecode;
+
+        /*
+         * The initialiser may be omitted, or may validly be one of several types in the broad
+         * categories of a byte string (which is treated as a machine representation of the data) or
+         * an iterable yielding values assignable to the elements. There is special treatment for
+         * type 'u' Unicode.
+         */
+        PyObject initial = ap.getPyObject(1, null);
         if (initial == null) {
-            return self;
-        }
-        if (initial instanceof PyList) {
+            // Fall through
+
+        } else if (initial instanceof PyList) {
             self.fromlist(initial);
+
         } else if (initial instanceof PyString && !(initial instanceof PyUnicode)) {
             self.fromstring(initial.toString());
+
         } else if ("u".equals(typecode)) {
             if (initial instanceof PyUnicode) {
                 self.extendArray(((PyUnicode)initial).toCodePoints());
             } else {
                 self.extendUnicodeIter(initial);
             }
+
         } else {
             self.extendInternal(initial);
         }
+
         return self;
     }
 
@@ -257,9 +284,13 @@
 
     @ExposedMethod(type = MethodType.BINARY)
     final PyObject array___imul__(PyObject o) {
+
         if (!o.isIndex()) {
             return null;
         }
+
+        resizeCheck();  // Prohibited if exporting a buffer
+
         if (delegate.getSize() > 0) {
             int count = o.asIndex(Py.OverflowError);
             if (count <= 0) {
@@ -308,6 +339,7 @@
 
     @ExposedMethod(type = MethodType.BINARY)
     final PyObject array___iadd__(PyObject other) {
+
         if (!(other instanceof PyArray)) {
             return null;
         }
@@ -317,6 +349,9 @@
             throw Py.TypeError("can only append arrays of the same type, expected '" + this.type
                     + ", found " + otherArr.type);
         }
+
+        resizeCheck();  // Prohibited if exporting a buffer
+
         delegate.appendArray(otherArr.delegate.copyArray());
         return this;
     }
@@ -430,7 +465,8 @@
 
     @ExposedMethod
     public final void array_append(PyObject value) {
-        append(value);
+        resizeCheck();  // Prohibited if exporting a buffer
+        appendUnchecked(value);
     }
 
     private static int getCodePoint(PyObject obj) {
@@ -464,21 +500,31 @@
      *
      * @param value item to be appended to the array
      */
+    public void append(PyObject value) {
+        resizeCheck();  // Prohibited if exporting a buffer
+        appendUnchecked(value);
+    }
 
-    public void append(PyObject value) {
-        // Currently, this is asymmetric with extend, which
+    /**
+     * Common helper method used internally to append a new value x to the end of the array:
+     * {@link #resizeCheck()} is not called, so the client must do so in advance.
+     *
+     * @param value item to be appended to the array
+     */
+    private final void appendUnchecked(PyObject value) {
+        // Currently, append is asymmetric with extend, which
         // *will* do conversions like append(5.0) to an int array.
-        // Also, cpython 2.2 will do the append coersion. However,
-        // it is deprecated in cpython 2.3, so maybe we are just
+        // Also, CPython 2.2 will do the append coercion. However,
+        // it is deprecated in CPython 2.3, so maybe we are just
         // ahead of our time ;-)
 
         int afterLast = delegate.getSize();
+
         if ("u".equals(typecode)) {
             int codepoint = getCodePoint(value);
             delegate.makeInsertSpace(afterLast);
             Array.setInt(data, afterLast, codepoint);
         } else {
-
             delegate.makeInsertSpace(afterLast);
             try {
                 set(afterLast, value);
@@ -660,8 +706,7 @@
      */
     @Override
     protected void del(int i) {
-        // Now the AbstractArray can support this:
-        // throw Py.TypeError("can't remove from array");
+        resizeCheck();  // Prohibited if exporting a buffer
         delegate.remove(i);
     }
 
@@ -673,6 +718,7 @@
      */
     @Override
     protected void delRange(int start, int stop) {
+        resizeCheck();  // Prohibited if exporting a buffer
         delegate.remove(start, stop);
     }
 
@@ -685,7 +731,7 @@
      * Append items from <code>iterable</code> to the end of the array. If iterable is another
      * array, it must have exactly the same type code; if not, TypeError will be raised. If iterable
      * is not an array, it must be iterable and its elements must be the right type to be appended
-     * to the array. Changed in version 2.4: Formerly, the argument could only be another array.
+     * to the array.
      *
      * @param iterable iterable object used to extend the array
      */
@@ -700,8 +746,8 @@
      *
      * @param iterable object of type PyString, PyArray or any object that can be iterated over.
      */
+    private void extendInternal(PyObject iterable) {
 
-    private void extendInternal(PyObject iterable) {
         if (iterable instanceof PyUnicode) {
             if ("u".equals(typecode)) {
                 extendUnicodeIter(iterable);
@@ -710,14 +756,19 @@
             } else {
                 throw Py.TypeError("an integer is required");
             }
-        } else if (iterable instanceof PyString) {
-            fromstring(((PyString)iterable).toString());
+
+// } else if (iterable instanceof PyString) {
+// // XXX CPython treats a str/bytes as an iterable, not as previously here:
+// fromstring(((PyString)iterable).toString());
+
         } else if (iterable instanceof PyArray) {
             PyArray source = (PyArray)iterable;
             if (!source.typecode.equals(typecode)) {
                 throw Py.TypeError("can only extend with array of same kind");
             }
+            resizeCheck();  // Prohibited if exporting a buffer
             delegate.appendArray(source.delegate.copyArray());
+
         } else {
             extendInternalIter(iterable);
         }
@@ -729,40 +780,66 @@
      * @param iterable any object that can be iterated over.
      */
     private void extendInternalIter(PyObject iterable) {
-        // iterable object without a length property - cannot presize the
-        // array, so append each item
-        if (iterable.__findattr__("__len__") == null) {
-            for (PyObject item : iterable.asIterable()) {
-                append(item);
-            }
-        } else {
-            // create room
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
+        if (iterable.__findattr__("__len__") != null) {
+            // Make room according to source length
             int last = delegate.getSize();
             delegate.ensureCapacity(last + iterable.__len__());
             for (PyObject item : iterable.asIterable()) {
                 set(last++, item);
                 delegate.size++;
             }
-        }
-    }
 
-    private void extendUnicodeIter(PyObject iterable) {
-        for (PyObject item : iterable.asIterable()) {
-            PyUnicode uitem;
-            try {
-                uitem = (PyUnicode)item;
-            } catch (ClassCastException e) {
-                throw Py.TypeError("Type not compatible with array type");
-            }
-            for (int codepoint : uitem.toCodePoints()) {
-                int afterLast = delegate.getSize();
-                delegate.makeInsertSpace(afterLast);
-                Array.setInt(data, afterLast, codepoint);
+        } else {
+            // iterable has no length property: cannot size the array so append each item.
+            for (PyObject item : iterable.asIterable()) {
+                appendUnchecked(item); // we already did a resizeCheck
             }
         }
     }
 
+    /**
+     * Helper used only when the array elements are Unicode characters (<code>typecode=='u'</code>).
+     * (Characters are stored as integer point codes.) The parameter must be an iterable yielding
+     * <code>PyUnicode</code>s. Often this will be an instance of {@link PyUnicode}, which is an
+     * iterable yielding single-character <code>PyUnicode</code>s. But it is also acceptable to this
+     * method for the argument to yield arbitrary <code>PyUnicode</code>s, which will be
+     * concatenated in the array.
+     *
+     * @param iterable of <code>PyUnicode</code>s
+     */
+    private void extendUnicodeIter(PyObject iterable) {
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
+        try {
+
+            // Append all the code points of all the strings in the iterable
+            for (PyObject item : iterable.asIterable()) {
+                PyUnicode uitem = (PyUnicode)item;
+                // Append all the code points of this item
+                for (int codepoint : uitem.toCodePoints()) {
+                    int afterLast = delegate.getSize();
+                    delegate.makeInsertSpace(afterLast);
+                    Array.setInt(data, afterLast, codepoint);
+                }
+            }
+
+        } catch (ClassCastException e) {
+            // One of the PyUnicodes wasn't
+            throw Py.TypeError("Type not compatible with array type");
+        }
+    }
+
     private void extendArray(int[] items) {
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         int last = delegate.getSize();
         delegate.ensureCapacity(last + items.length);
         for (int item : items) {
@@ -787,21 +864,32 @@
      * @param count number of array elements to read
      */
     public void fromfile(PyObject f, int count) {
-        // check for arg1 as file object
-        if (!(f instanceof PyFile)) {
-            throw Py.TypeError("arg1 must be open file");
+        /*
+         * Prohibit when exporting a buffer. Different from CPython, BufferError takes precedence in
+         * Jython over EOFError: if there's nowhere to write the data, we don't read it.
+         */
+        resizeCheck();
+
+        /*
+         * Now get the required number of bytes from the file. Guard against non-file or closed.
+         */
+        if (f instanceof PyFile) {
+            PyFile file = (PyFile)f;
+            if (!file.getClosed()) {
+                // Load required amount or whatever is available into a bytes object
+                int readbytes = count * getStorageSize();
+                String buffer = file.read(readbytes).toString();
+                fromstring(buffer);
+                // check for underflow
+                if (buffer.length() < readbytes) {
+                    int readcount = buffer.length() / getStorageSize();
+                    throw Py.EOFError("not enough items in file. " + Integer.toString(count)
+                            + " requested, " + Integer.toString(readcount) + " actually read");
+                }
+            }
+            return;
         }
-        PyFile file = (PyFile)f;
-        int readbytes = count * getStorageSize();
-        String buffer = file.read(readbytes).toString();
-        // load whatever was collected into the array
-        fromstring(buffer);
-        // check for underflow
-        if (buffer.length() < readbytes) {
-            int readcount = buffer.length() / getStorageSize();
-            throw Py.EOFError("not enough items in file. " + Integer.toString(count)
-                    + " requested, " + Integer.toString(readcount) + " actually read");
-        }
+        throw Py.TypeError("arg1 must be open file");
     }
 
     @ExposedMethod
@@ -810,7 +898,7 @@
     }
 
     /**
-     * Append items from the list. This is equivalent to "for x in list: a.append(x)"except that if
+     * Append items from the list. This is equivalent to "for x in list: a.append(x)" except that if
      * there is a type error, the array is unchanged.
      *
      * @param obj input list object that will be appended to the array
@@ -819,6 +907,10 @@
         if (!(obj instanceof PyList)) {
             throw Py.TypeError("arg must be list");
         }
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         // store the current size of the internal array
         int size = delegate.getSize();
         try {
@@ -862,11 +954,8 @@
         // Current number of items present
         int origsize = delegate.getSize();
 
-        // Reserve capacity for 'count' items
-        delegate.setSize(origsize + count);
-
         // Read into the array, after the current contents, up to new size (or EOF thrown)
-        int n = fromStream(is, origsize, delegate.getSize(), true);
+        int n = fromStream(is, origsize, origsize + count, true);
         return n - origsize;
     }
 
@@ -888,11 +977,14 @@
      * Helper for reading primitive values from a stream into a slice of the array. Data is read
      * until the array slice is filled or the stream runs out. The purpose of the method is to
      * concentrate in one place the manipulation of bytes into the several primitive element types
-     * on behalf of {@link #fillFromStream(InputStream)} etc.. Since different read methods respond
-     * differently to it, the caller must specify whether the exhaustion of the stream (EOF) should
-     * be treated as an error or not. If the stream does not contain a whole number of items
-     * (possible if the item size is not one byte), the behaviour in respect of the final partial
-     * item and stream position is not defined.
+     * on behalf of {@link #fillFromStream(InputStream)} etc.. The storage is resized if the slice
+     * being written ends beyond the current end of the array, i.e. it is increased to the value of
+     * <code>limit</code>.
+     * <p>
+     * Since different read methods respond differently to it, the caller must specify whether the
+     * exhaustion of the stream (EOF) should be treated as an error or not. If the stream does not
+     * contain a whole number of items (possible if the item size is not one byte), the behaviour in
+     * respect of the final partial item and stream position is not defined.
      *
      * @param dis data stream source for the values
      * @param index first element index to read
@@ -905,7 +997,14 @@
     private int fromStream(InputStream is, int index, int limit, boolean eofIsError)
             throws IOException, EOFException {
 
-        // We need a wrapper capable of encoding the data
+        // Ensure the array is dimensioned to fit the data expected
+        if (limit > delegate.getSize()) {
+            // Prohibited operation if exporting a buffer
+            resizeCheck();
+            delegate.setSize(limit);
+        }
+
+        // We need a wrapper capable of decoding the data from the representation defined by Java.
         DataInputStream dis = new DataInputStream(is);
 
         try {
@@ -993,12 +1092,14 @@
     }
 
     /**
-     * Appends items from the string, interpreting the string as an array of machine values (as if
-     * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+     * Appends items from the object, which is a byte string of some kind (PyString or object with
+     * the buffer interface providing bytes) The string of bytes is interpreted as an array of
+     * machine values (as if it had been read from a file using the {@link #fromfile(PyObject, int)
+     * fromfile()} method).
      *
      * @param input string of bytes containing array data
      */
-    public void fromstring(String input) {
+    public void fromstring(PyObject input) {
         array_fromstring(input);
     }
 
@@ -1008,20 +1109,108 @@
      *
      * @param input string of bytes containing array data
      */
+    public void fromstring(String input) {
+        frombytesInternal(StringUtil.toBytes(input));
+    }
+
+    /**
+     * Appends items from the string, interpreting the string as an array of machine values (as if
+     * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+     *
+     * @param input string of bytes containing array data
+     */
     @ExposedMethod
-    final void array_fromstring(String input) {
+    final void array_fromstring(PyObject input) {
+
+        if (input instanceof BufferProtocol) {
+
+            if (input instanceof PyUnicode) {
+                // Unicode is treated as specifying a byte string via the default encoding.
+                String s = ((PyUnicode)input).encode();
+                frombytesInternal(StringUtil.toBytes(s));
+
+            } else {
+                // Access the bytes
+                PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO);
+                try {
+                    // Provide argument as stream of bytes for fromstream method
+                    if (pybuf.getNdim() == 1) {
+                        if (pybuf.getStrides()[0] == 1) {
+                            // Data are contiguous in a byte[]
+                            PyBuffer.Pointer b = pybuf.getBuf();
+                            frombytesInternal(b.storage, b.offset, pybuf.getLen());
+                        } else {
+                            // As frombytesInternal only knows contiguous bytes, make a copy.
+                            byte[] copy = new byte[pybuf.getLen()];
+                            pybuf.copyTo(copy, 0);
+                            frombytesInternal(copy);
+                        }
+                    } else {
+                        // Currently don't support n-dimensional sources
+                        throw Py.ValueError("multi-dimensional buffer not supported");
+                    }
+                } finally {
+                    pybuf.release();
+                }
+            }
+
+        } else {
+            String fmt = "must be string or read-only buffer, not %s";
+            throw Py.TypeError(String.format(fmt, input.getType().fastGetName()));
+        }
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code>
+     *
+     * @param input string of bytes encoding the array data
+     */
+    private final void fromstringInternal(String input) {
+        frombytesInternal(StringUtil.toBytes(input));
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+     * <code>.frombytes()</code> (Python 3.2+ name).
+     *
+     * @param bytes array containing the new array data in machine encoding
+     */
+    private final void frombytesInternal(byte[] bytes) {
+        frombytesInternal(bytes, 0, bytes.length);
+    }
+
+    /**
+     * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+     * <code>.frombytes()</code> (Python 3.2+ name).
+     *
+     * @param bytes array containing the new array data in machine encoding
+     * @param offset of the first byte to read
+     * @param count of bytes to read
+     */
+    private final void frombytesInternal(byte[] bytes, int offset, int count) {
+
+        // Access the bytes
+        int origsize = delegate.getSize();
+
+        // Check validity wrt array itemsize
         int itemsize = getStorageSize();
-        int strlen = input.length();
-        if ((strlen % itemsize) != 0) {
+        if ((count % itemsize) != 0) {
             throw Py.ValueError("string length not a multiple of item size");
         }
-        ByteArrayInputStream bis = new ByteArrayInputStream(StringUtil.toBytes(input));
-        int origsize = delegate.getSize();
+
+        // Prohibited operation if we are exporting a buffer
+        resizeCheck();
+
         try {
+
+            // Provide argument as stream of bytes for fromstream method
+            ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset, count);
             fromStream(bis);
+
         } catch (EOFException e) {
             // stubbed catch for fromStream throws
             throw Py.EOFError("not enough items in string");
+
         } catch (IOException e) {
             // discard anything successfully loaded
             delegate.setSize(origsize);
@@ -1279,6 +1468,7 @@
      * @param value value to be inserted into array
      */
     public void insert(int index, PyObject value) {
+        resizeCheck();  // Prohibited operation if exporting a buffer
         index = boundToSequence(index);
         if ("u".equals(typecode)) {
             int codepoint = getCodePoint(value);
@@ -1325,6 +1515,10 @@
         if (index == -1) {
             throw Py.IndexError("pop index out of range");
         }
+
+        // Prohibited operation if exporting a buffer
+        resizeCheck();
+
         PyObject ret = Py.java2py(Array.get(data, index));
         delegate.remove(index);
         return ret;
@@ -1343,6 +1537,8 @@
     public void remove(PyObject value) {
         int index = indexInternal(value);
         if (index != -1) {
+            // Prohibited operation if exporting a buffer
+            resizeCheck();
             delegate.remove(index);
             return;
         }
@@ -1359,7 +1555,6 @@
     protected PyObject repeat(int count) {
         Object arraycopy = delegate.copyArray();
         PyArray ret = new PyArray(type, 0);
-        // XXX:
         ret.typecode = typecode;
         for (int i = 0; i < count; i++) {
             ret.delegate.appendArray(arraycopy);
@@ -1500,9 +1695,11 @@
      */
     @Override
     protected void setslice(int start, int stop, int step, PyObject value) {
+
         if (stop < start) {
             stop = start;
         }
+
         if (type == Character.TYPE && value instanceof PyString) {
             char[] chars = null;
             // if (value instanceof PyString) {
@@ -1510,8 +1707,14 @@
                 throw Py.ValueError("invalid bounds for setting from string");
             }
             chars = value.toString().toCharArray();
+            if (start + chars.length != stop) {
+                // This is a size-changing operation: check for buffer exports
+                resizeCheck();
+            }
             delegate.replaceSubArray(start, stop, chars, 0, chars.length);
+
         } else {
+
             if (value instanceof PyString && type == Byte.TYPE) {
                 byte[] chars = ((PyString)value).toBytes();
                 if (chars.length == stop - start && step == 1) {
@@ -1519,12 +1722,14 @@
                 } else {
                     throw Py.ValueError("invalid bounds for setting from string");
                 }
+
             } else if (value instanceof PyArray) {
                 PyArray array = (PyArray)value;
                 if (!array.typecode.equals(typecode)) {
                     throw Py.TypeError("bad argument type for built-in operation|" + array.typecode
                             + "|" + typecode);
                 }
+
                 if (step == 1) {
                     Object arrayDelegate;
                     if (array == this) {
@@ -1532,19 +1737,25 @@
                     } else {
                         arrayDelegate = array.delegate.getArray();
                     }
+                    int len = array.delegate.getSize();
+                    if (start + len != stop) {
+                        // This is a size-changing operation: check for buffer exports
+                        resizeCheck();
+                    }
                     try {
-                        delegate.replaceSubArray(start, stop, arrayDelegate, 0,
-                                array.delegate.getSize());
+                        delegate.replaceSubArray(start, stop, arrayDelegate, 0, len);
                     } catch (IllegalArgumentException e) {
                         throw Py.TypeError("Slice typecode '" + array.typecode
                                 + "' is not compatible with this array (typecode '" + this.typecode
                                 + "')");
                     }
+
                 } else if (step > 1) {
                     int len = array.__len__();
                     for (int i = 0, j = 0; i < len; i++, j += step) {
                         Array.set(data, j + start, Array.get(array.data, i));
                     }
+
                 } else if (step < 0) {
                     if (array == this) {
                         array = (PyArray)array.clone();
@@ -1554,6 +1765,7 @@
                         Array.set(data, j, Array.get(array.data, i));
                     }
                 }
+
             } else {
                 throw Py.TypeError(String.format("can only assign array (not \"%.200s\") to array "
                         + "slice", value.getType().fastGetName()));
@@ -1805,4 +2017,106 @@
             return Array.newInstance(baseType, size);
         }
     }
+
+    /*
+     * ============================================================================================
+     * Support for the Buffer API
+     * ============================================================================================
+     *
+     * The buffer API allows other classes to access the storage directly.
+     *
+     * This is a close duplicate of the same mechanism in PyByteArray. There is perhaps scope for a
+     * shared helper class to implement this logic. For type code 'b', the workings are almost
+     * identical. The fully-fledged buffer interface for PyArray is richer, more like the Python 3
+     * memoryview, as it must cope with items of size other than one byte. This goes beyond the
+     * capabilities of the Jython BufferProtocol at this stage of its development.
+     */
+
+    /**
+     * Hold weakly a reference to a PyBuffer export not yet released, used to prevent untimely
+     * resizing.
+     */
+    private WeakReference<BaseBuffer> export;
+
+    /**
+     * {@inheritDoc}
+     * <p>
+     * The {@link PyBuffer} returned from this method is a one-dimensional array of single byte
+     * items that allows modification of the object state. The existence of this export <b>prohibits
+     * resizing</b> the byte array. This prohibition is not only on the consumer of the view but
+     * extends to any other operations, such as any kind or insertion or deletion.
+     */
+    @Override
+    public synchronized PyBuffer getBuffer(int flags) {
+
+        // If we have already exported a buffer it may still be available for re-use
+        BaseBuffer pybuf = getExistingBuffer(flags);
+
+        if (pybuf == null) {
+            // No existing export we can re-use: create a new one
+            if ("b".equals(typecode)) {
+                // This is byte data, so we are within the state of the art
+                byte[] storage = (byte[])data;
+                int size = delegate.getSize();
+                pybuf = new SimpleWritableBuffer(flags, storage, 0, size);
+            } else if ((flags & PyBUF.WRITABLE) == 0) {
+                // As the client only intends to read, fake the answer with a String
+                pybuf = new SimpleStringBuffer(flags, tostring());
+            } else {
+                // For the time being ...
+                throw Py.NotImplementedError("only array('b') can export a writable buffer");
+            }
+            // Hold a reference for possible re-use
+            export = new WeakReference<BaseBuffer>(pybuf);
+        }
+
+        return pybuf;
+    }
+
+    /**
+     * Try to re-use an existing exported buffer, or return <code>null</code> if we can't.
+     *
+     * @throws PyException (BufferError) if the the flags are incompatible with the buffer
+     */
+    private BaseBuffer getExistingBuffer(int flags) throws PyException {
+        BaseBuffer pybuf = null;
+        if (export != null) {
+            // A buffer was exported at some time.
+            pybuf = export.get();
+            if (pybuf != null) {
+                /*
+                 * We do not test for pybuf.isReleased() as, if any operation had taken place that
+                 * invalidated the buffer, resizeCheck() would have set export=null. The exported
+                 * buffer (navigation, buf member, etc.) remains valid through any operation that
+                 * does not need a resizeCheck.
+                 */
+                pybuf = pybuf.getBufferAgain(flags);
+            }
+        }
+        return pybuf;
+    }
+
+    /**
+     * Test to see if the array may be resized and raise a BufferError if not. This must be called
+     * by the implementation of any operation that changes the number of elements in the array.
+     *
+     * @throws PyException (BufferError) if there are buffer exports preventing a resize
+     */
+    private void resizeCheck() throws PyException {
+        if (export != null) {
+            // A buffer was exported at some time and we have not explicitly discarded it.
+            PyBuffer pybuf = export.get();
+            if (pybuf != null && !pybuf.isReleased()) {
+                // A consumer still has the exported buffer
+                throw Py.BufferError("cannot resize an array that is exporting buffers");
+            } else {
+                /*
+                 * Either the reference has expired or all consumers have released it. Either way,
+                 * the weak reference is useless now.
+                 */
+                export = null;
+            }
+        }
+    }
+
 }
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -377,38 +377,42 @@
     }
 
     /**
-     * Return a String for writing to the underlying file from obj.
+     * Return a String for writing to the underlying file from obj. This is a helper for {@link file_write}
+     * and {@link file_writelines}.
+     *
+     * @param obj to write
+     * @param message for TypeError if raised (or null for default message)
+     * @return bytes representing tha value (as a String in the Jython convention)
      */
     private String asWritable(PyObject obj, String message) {
 
         if (obj instanceof PyUnicode) {
+            // By convention, use platform default encoding to bytes
             return ((PyUnicode)obj).encode();
 
         } else if (obj instanceof PyString) {
-            return ((PyString) obj).getString();
+            // Take a short cut
+            return ((PyString)obj).getString();
+
+        } else if (obj instanceof PyArray && !binary) {
+            // Fall through to TypeError. (If binary, BufferProtocol takes care of PyArray.)
 
         } else if (obj instanceof BufferProtocol) {
-            // Try to get a simple byte-oriented buffer
-            PyBuffer buf = null;
+            // Try to get a byte-oriented buffer
+            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
             try {
-                buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
-                return StringUtil.fromBytes(buf);
-            } catch (Exception e) {
-                // Wrong kind of buffer: generic error message will do
+                // ... and treat those bytes as a String
+                return buf.toString();
             } finally {
-                // If we got a buffer, we should release it
-                if (buf != null) {
-                    buf.release();
-                }
+                // We should release the buffer
+                buf.release();
             }
+        }
 
-        } else if (binary && obj instanceof PyArray) {
-            return ((PyArray)obj).tostring();
-        }
         if (message == null) {
-            message = String.format("argument 1 must be string or %sbuffer, not %.200s",
-                                    binary ? "" : "read-only character ",
-                                    obj.getType().fastGetName());
+            // Messages differ for text or binary streams (CPython) but we always add the type
+            String.format("%s buffer, not %.200s", (binary ? "must be string or"
+                    : "expected a character"), obj.getType().fastGetName());
         }
         throw Py.TypeError(message);
     }
@@ -581,7 +585,7 @@
         }
     }
 
-  
+
     /**
      * XXX update docs - A mechanism to make sure PyFiles are closed on exit. On creation Closer adds itself
      * to a list of Closers that will be run by PyFileCloser on JVM shutdown. When a
@@ -615,6 +619,7 @@
         }
 
         /** For closing as part of a shutdown process */
+        @Override
         public Void call() {
             file.close();
             sys = null;
diff --git a/src/org/python/core/PyMemoryView.java b/src/org/python/core/PyMemoryView.java
--- a/src/org/python/core/PyMemoryView.java
+++ b/src/org/python/core/PyMemoryView.java
@@ -38,14 +38,19 @@
     private boolean hashCacheValid = false;
 
     /**
-     * Construct a PyMemoryView from a PyBuffer interface. The buffer so obtained will be writable
-     * if the underlying object permits it. The <code>memoryview</code> takes a new lease on the
-     * <code>PyBuffer</code>.
+     * Construct a <code>PyMemoryView</code> from an object bearing the {@link BufferProtocol}
+     * interface. If this object is already an exported buffer, the <code>memoryview</code> takes a
+     * new lease on it. The buffer so obtained will be writable if the underlying object permits it.
      *
      * @param pybuf buffer exported by some underlying object
      */
-    public PyMemoryView(PyBuffer pybuf) {
+    public PyMemoryView(BufferProtocol pybuf) {
         super(TYPE);
+        /*
+         * Ask for the full set of facilities (strides, indirect, etc.) from the object in case they
+         * are necessary for navigation, but only ask for read access. If the object is writable,
+         * the PyBuffer will be writable.
+         */
         backing = pybuf.getBuffer(PyBUF.FULL_RO);
     }
 
@@ -63,12 +68,7 @@
         PyObject obj = ap.getPyObject(0);
 
         if (obj instanceof BufferProtocol) {
-            /*
-             * Ask for the full set of facilities (strides, indirect, etc.) from the object in case
-             * they are necessary for navigation, but only ask for read access. If the object is
-             * writable, the PyBuffer will be writable.
-             */
-            return new PyMemoryView(((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO));
+            return new PyMemoryView((BufferProtocol)obj);
         } else {
             throw Py.TypeError("cannot make memory view because object does not have "
                     + "the buffer interface");
diff --git a/src/org/python/core/io/TextIOBase.java b/src/org/python/core/io/TextIOBase.java
--- a/src/org/python/core/io/TextIOBase.java
+++ b/src/org/python/core/io/TextIOBase.java
@@ -101,14 +101,25 @@
      * @return the amount of data read as an int
      */
     public int readinto(PyObject buf) {
+
         // This is an inefficient version of readinto: but readinto is
         // not recommended for use in Python 2.x anyway
-        if (buf instanceof BufferProtocol) {
+
+        if (buf instanceof PyArray) {
+            // PyArray has the buffer interface but it only works for bytes at present
+            PyArray array = (PyArray)buf;
+            String read = read(array.__len__());
+            for (int i = 0; i < read.length(); i++) {
+                array.set(i, new PyString(read.charAt(i)));
+            }
+            return read.length();
+
+        } else if (buf instanceof BufferProtocol) {
             PyBuffer view = ((BufferProtocol)buf).getBuffer(PyBUF.SIMPLE);
             if (view.isReadonly()) {
                 // More helpful than falling through to CPython message
-                throw Py.TypeError("cannot read into read-only "
-                        + buf.getType().fastGetName());
+                throw Py.TypeError("cannot read into read-only " + buf.getType().fastGetName());
+
             } else {
                 try {
                     // Inefficiently, we have to go via a String
@@ -122,15 +133,8 @@
                     // We should release the buffer explicitly
                     view.release();
                 }
+
             }
-
-        } else if (buf instanceof PyArray) {
-            PyArray array = (PyArray)buf;
-            String read = read(array.__len__());
-            for (int i = 0; i < read.length(); i++) {
-                array.set(i, new PyString(read.charAt(i)));
-            }
-            return read.length();
         }
 
         // No valid alternative worked
diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java
--- a/src/org/python/modules/sre/PatternObject.java
+++ b/src/org/python/modules/sre/PatternObject.java
@@ -372,28 +372,18 @@
             return (PyString)obj;
 
         } else if (obj instanceof BufferProtocol) {
-            // Try to get a simple byte-oriented buffer
-            PyBuffer buf = null;
+            // Try to get a byte-oriented buffer
+            PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO);
             try {
-                buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE);
                 // ... and treat those bytes as a PyString
-                String s = StringUtil.fromBytes(buf);
-                return new PyString(s);
-            } catch (Exception e) {
-                // Wrong kind of buffer: generic error message will do
+                return new PyString(buf.toString());
             } finally {
-                // If we got a buffer, we should release it
-                if (buf != null) {
-                    buf.release();
-                }
+                // We should release the buffer
+                buf.release();
             }
-
-        } else if (obj instanceof PyArray) {
-            // PyArray can do something similar
-            return new PyString(obj.toString());
         }
 
-        // None of those things worked
+        // Neither of those things worked
         throw Py.TypeError("expected string or buffer, but got " + obj.getType());
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 16 01:22:12 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 16 Nov 2013 01:22:12 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_for_bug_1860=2E?=
Message-ID: <3dLxvS2Mplz7Ljc@mail.python.org>

http://hg.python.org/jython/rev/ffa4255db50d
changeset:   7153:ffa4255db50d
user:        Chris Simpson <cjsimpson at gmail.com>
date:        Sun Sep 29 11:33:27 2013 -0400
summary:
  Fix for bug 1860.
Changed loop logic in PyArray to correctly handle negative step size when setting a slice.
Added Java unittest covering this case.
Removed skip from test_extended_set_del_slice in test_array.py since this will now pass.

files:
  Lib/test/test_array.py                      |   1 -
  src/org/python/core/PyArray.java            |   3 +-
  tests/java/org/python/core/PyArrayTest.java |  53 ++++++++++
  3 files changed, 55 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -686,7 +686,6 @@
         self.assertRaises(TypeError, a.__setitem__, slice(0, 0), b)
         self.assertRaises(TypeError, a.__setitem__, slice(0, 1), b)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME #1860: Not working on Jython")
     def test_extended_set_del_slice(self):
         indices = (0, None, 1, 3, 19, 100, -1, -2, -31, -100)
         for start in indices:
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -1760,8 +1760,9 @@
                     if (array == this) {
                         array = (PyArray)array.clone();
                     }
+                    
                     int len = array.__len__();
-                    for (int i = 0, j = delegate.getSize() - 1; i < len; i++, j += step) {
+                    for (int i = 0, j = start; i < len; i++, j+=step) {
                         Array.set(data, j, Array.get(array.data, i));
                     }
                 }
diff --git a/tests/java/org/python/core/PyArrayTest.java b/tests/java/org/python/core/PyArrayTest.java
new file mode 100644
--- /dev/null
+++ b/tests/java/org/python/core/PyArrayTest.java
@@ -0,0 +1,53 @@
+package org.python.core;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for PyArray.
+ */
+public class PyArrayTest extends TestCase {
+
+    public void testSetSliceNegativeStep() {
+    	PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	
+    	//Replacing single element
+    	PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
+    	arrayToModify.setslice(0, 0, -1, arrayOneElement);
+    	assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
+    	
+    	//Replacing multiple elements
+    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	PyArray arrayThreeElements = new PyArray(PyString.class, new String[] {"x", "y", "z"});
+    	arrayToModify.setslice(2, 0, -1, arrayThreeElements);
+    	assertEquals(new PyArray(PyString.class, new String[] {"z", "y", "x", "d"}), arrayToModify);
+
+    	//Replacing multiple elements - step size = (-2)
+    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	PyArray arrayTwoElements = new PyArray(PyString.class, new String[] {"x", "y"});
+    	arrayToModify.setslice(3, 0, -2, arrayTwoElements);
+    	assertEquals(new PyArray(PyString.class, new String[] {"a", "y", "c", "x"}), arrayToModify);
+    	
+    }
+    
+    public void testSetSlicePositiveStep() {
+    	PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	
+    	//Replacing single element
+    	PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
+    	arrayToModify.setslice(0, 1, 1, arrayOneElement);
+    	assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
+    	
+    	//Replacing multiple elements
+    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	PyArray arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
+    	arrayToModify.setslice(1, 3, 1, arrayMultipleElements);
+    	assertEquals(new PyArray(PyString.class, new String[] {"a", "x", "y", "d"}), arrayToModify);
+    	
+    	//Replace multiple elements - step = 2
+    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+    	arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
+    	arrayToModify.setslice(0, 3, 2, arrayMultipleElements);
+    	assertEquals(new PyArray(PyString.class, new String[] {"x", "b", "y", "d"}), arrayToModify);
+    	
+    }
+}

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 16 01:22:13 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 16 Nov 2013 01:22:13 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Formatting=2C_NEWS_and_ackn?=
	=?utf-8?q?owledgement=2E?=
Message-ID: <3dLxvT5JhFz7LmN@mail.python.org>

http://hg.python.org/jython/rev/7b44ea13a028
changeset:   7154:7b44ea13a028
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Nov 16 00:09:39 2013 +0000
summary:
  Formatting, NEWS and acknowledgement.
Mostly related to #1860.

files:
  ACKNOWLEDGMENTS                             |   5 +-
  NEWS                                        |   4 +
  src/org/python/core/PyArray.java            |   4 +-
  tests/java/org/python/core/PyArrayTest.java |  76 +++++-----
  4 files changed, 47 insertions(+), 42 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -1,5 +1,5 @@
 ACKNOWLEDGMENTS
-Copyright ? Corporation for National Research Initiatives
+Copyright (c) Corporation for National Research Initiatives
 
     Jim Hugunin invented JPython and was the primary developer while
     he was at CNRI.  In February 1999, Jim left CNRI for sunny
@@ -75,7 +75,7 @@
     Geoffrey French
     Tobias Ivarsson
     Lino Mastrodomenico
-    S?bastien Boisg?rault
+    S?bastien Boisg?rault
     Jim Baker
     Charlie Groves
     Otmar Humbel
@@ -104,6 +104,7 @@
     Jezreel Ng
     Santoso Wijaya
     Brandon Pedersen
+    Chris Simpson
 
 Local Variables:
 mode: indented-text
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,10 @@
     - [ 1926 ] Adjust MutableSet.pop test so we do not need to skip it
     - [ 2020 ] str.translate should delete characters in the second arg when table is None
     - [ 1753 ] zlib doesn't call end() on compress and decompress
+    - [ 1860 ] test failures in test_array.py
+    - [ 2046 ] sys.stdin.readline() hangs when used interactively (JLine, Windows)
+    - [ 2060 ] Thread ident missing
+    - [ 2082 ] Unexpected (Pdb) prompt during regression tests
 
 Jython 2.7b1
   Bugs Fixed
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -1760,9 +1760,9 @@
                     if (array == this) {
                         array = (PyArray)array.clone();
                     }
-                    
+
                     int len = array.__len__();
-                    for (int i = 0, j = start; i < len; i++, j+=step) {
+                    for (int i = 0, j = start; i < len; i++, j += step) {
                         Array.set(data, j, Array.get(array.data, i));
                     }
                 }
diff --git a/tests/java/org/python/core/PyArrayTest.java b/tests/java/org/python/core/PyArrayTest.java
--- a/tests/java/org/python/core/PyArrayTest.java
+++ b/tests/java/org/python/core/PyArrayTest.java
@@ -8,46 +8,46 @@
 public class PyArrayTest extends TestCase {
 
     public void testSetSliceNegativeStep() {
-    	PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	
-    	//Replacing single element
-    	PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
-    	arrayToModify.setslice(0, 0, -1, arrayOneElement);
-    	assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
-    	
-    	//Replacing multiple elements
-    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	PyArray arrayThreeElements = new PyArray(PyString.class, new String[] {"x", "y", "z"});
-    	arrayToModify.setslice(2, 0, -1, arrayThreeElements);
-    	assertEquals(new PyArray(PyString.class, new String[] {"z", "y", "x", "d"}), arrayToModify);
+        PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
 
-    	//Replacing multiple elements - step size = (-2)
-    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	PyArray arrayTwoElements = new PyArray(PyString.class, new String[] {"x", "y"});
-    	arrayToModify.setslice(3, 0, -2, arrayTwoElements);
-    	assertEquals(new PyArray(PyString.class, new String[] {"a", "y", "c", "x"}), arrayToModify);
-    	
+        // Replacing single element
+        PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
+        arrayToModify.setslice(0, 0, -1, arrayOneElement);
+        assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
+
+        // Replacing multiple elements
+        arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+        PyArray arrayThreeElements = new PyArray(PyString.class, new String[] {"x", "y", "z"});
+        arrayToModify.setslice(2, 0, -1, arrayThreeElements);
+        assertEquals(new PyArray(PyString.class, new String[] {"z", "y", "x", "d"}), arrayToModify);
+
+        // Replacing multiple elements - step size = (-2)
+        arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+        PyArray arrayTwoElements = new PyArray(PyString.class, new String[] {"x", "y"});
+        arrayToModify.setslice(3, 0, -2, arrayTwoElements);
+        assertEquals(new PyArray(PyString.class, new String[] {"a", "y", "c", "x"}), arrayToModify);
+
     }
-    
+
     public void testSetSlicePositiveStep() {
-    	PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	
-    	//Replacing single element
-    	PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
-    	arrayToModify.setslice(0, 1, 1, arrayOneElement);
-    	assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
-    	
-    	//Replacing multiple elements
-    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	PyArray arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
-    	arrayToModify.setslice(1, 3, 1, arrayMultipleElements);
-    	assertEquals(new PyArray(PyString.class, new String[] {"a", "x", "y", "d"}), arrayToModify);
-    	
-    	//Replace multiple elements - step = 2
-    	arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
-    	arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
-    	arrayToModify.setslice(0, 3, 2, arrayMultipleElements);
-    	assertEquals(new PyArray(PyString.class, new String[] {"x", "b", "y", "d"}), arrayToModify);
-    	
+        PyArray arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+
+        // Replacing single element
+        PyArray arrayOneElement = new PyArray(PyString.class, new String[] {"z"});
+        arrayToModify.setslice(0, 1, 1, arrayOneElement);
+        assertEquals(new PyArray(PyString.class, new String[] {"z", "b", "c", "d"}), arrayToModify);
+
+        // Replacing multiple elements
+        arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+        PyArray arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
+        arrayToModify.setslice(1, 3, 1, arrayMultipleElements);
+        assertEquals(new PyArray(PyString.class, new String[] {"a", "x", "y", "d"}), arrayToModify);
+
+        // Replace multiple elements - step = 2
+        arrayToModify = new PyArray(PyString.class, new String[] {"a", "b", "c", "d"});
+        arrayMultipleElements = new PyArray(PyString.class, new String[] {"x", "y"});
+        arrayToModify.setslice(0, 3, 2, arrayMultipleElements);
+        assertEquals(new PyArray(PyString.class, new String[] {"x", "b", "y", "d"}), arrayToModify);
+
     }
 }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Wed Nov 20 21:07:41 2013
From: jython-checkins at python.org (jeff.allen)
Date: Wed, 20 Nov 2013 21:07:41 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Close_and_delete_files_ever?=
	=?utf-8?q?y_time_in_test=5Ffile2k=2E?=
Message-ID: <3dPw1T0Skjz7Lm1@mail.python.org>

http://hg.python.org/jython/rev/c84270fca342
changeset:   7155:c84270fca342
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Wed Nov 20 08:00:26 2013 +0000
summary:
  Close and delete files every time in test_file2k.
Each test now closes and deletes its test file, not relying on deterministic
garbage collection. On Windows, open files cannot be deleted, and so sloppiness
in one test used to create a cascade of "unlink" errors in subsequent tests.

files:
  Lib/test/test_file2k.py |  190 +++++++++++++++------------
  1 files changed, 108 insertions(+), 82 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -1,7 +1,7 @@
 import sys
 import os
+import errno
 import unittest
-import itertools
 import time
 from array import array
 from weakref import proxy
@@ -190,14 +190,29 @@
 
 class OtherFileTests(unittest.TestCase):
 
+    def setUp(self):
+        # (Jython addition) track open file so we can clean up
+        self.f = None
+        self.filename = TESTFN
+
+    def tearDown(self):
+        # (Jython addition) clean up to prevent errors cascading
+        if self.f:
+            self.f.close()
+        try:
+            os.remove(self.filename)
+        except EnvironmentError as ee:
+            if ee.errno != errno.ENOENT:
+                raise ee
+
     def testOpenDir(self):
         this_dir = os.path.dirname(__file__) or os.curdir
         for mode in (None, "w"):
             try:
                 if mode:
-                    f = open(this_dir, mode)
+                    self.f = open(this_dir, mode)
                 else:
-                    f = open(this_dir)
+                    self.f = open(this_dir)
             except IOError as e:
                 self.assertEqual(e.filename, this_dir)
             else:
@@ -207,7 +222,7 @@
         # check invalid mode strings
         for mode in ("", "aU", "wU+"):
             try:
-                f = open(TESTFN, mode)
+                self.f = f = open(TESTFN, mode)
             except ValueError:
                 pass
             else:
@@ -218,7 +233,7 @@
         # Issue3965: avoid a crash on Windows when filename is unicode
         for name in (TESTFN, unicode(TESTFN), unicode(TESTFN + '\t')):
             try:
-                f = open(name, "rr")
+                self.f = f = open(name, "rr")
             except (IOError, ValueError):
                 pass
             else:
@@ -236,7 +251,7 @@
 
     def testUnicodeOpen(self):
         # verify repr works for unicode too
-        f = open(unicode(TESTFN), "w")
+        self.f = f = open(unicode(TESTFN), "w")
         self.assertTrue(repr(f).startswith("<open file u'" + TESTFN))
         f.close()
         os.unlink(TESTFN)
@@ -245,7 +260,7 @@
         # verify that we get a sensible error message for bad mode argument
         bad_mode = "qwerty"
         try:
-            f = open(TESTFN, bad_mode)
+            self.f = f = open(TESTFN, bad_mode)
         except ValueError, msg:
             if msg.args[0] != 0:
                 s = str(msg)
@@ -262,11 +277,11 @@
         # misbehaviour especially with repeated close() calls
         for s in (-1, 0, 1, 512):
             try:
-                f = open(TESTFN, 'w', s)
+                self.f = f = open(TESTFN, 'w', s)
                 f.write(str(s))
                 f.close()
                 f.close()
-                f = open(TESTFN, 'r', s)
+                self.f = f = open(TESTFN, 'r', s)
                 d = int(f.read())
                 f.close()
                 f.close()
@@ -275,16 +290,15 @@
             self.assertEqual(d, s)
 
     def testTruncateOnWindows(self):
-        os.unlink(TESTFN)
 
         def bug801631():
             # SF bug <http://www.python.org/sf/801631>
             # "file.truncate fault on windows"
-            f = open(TESTFN, 'wb')
+            self.f = f = open(TESTFN, 'wb')
             f.write('12345678901')   # 11 bytes
             f.close()
 
-            f = open(TESTFN,'rb+')
+            self.f = f = open(TESTFN,'rb+')
             data = f.read(5)
             if data != '12345':
                 self.fail("Read on file opened for update failed %r" % data)
@@ -339,7 +353,7 @@
             bag.close()
             # Test for appropriate errors mixing read* and iteration
             for methodname, args in methods:
-                f = open(TESTFN)
+                self.f = f = open(TESTFN)
                 if f.next() != filler:
                     self.fail, "Broken testfile"
                 meth = getattr(f, methodname)
@@ -359,7 +373,7 @@
             # ("h", "a", "m", "\n"), so 4096 lines of that should get us
             # exactly on the buffer boundary for any power-of-2 buffersize
             # between 4 and 16384 (inclusive).
-            f = open(TESTFN)
+            self.f = f = open(TESTFN)
             for i in range(nchunks):
                 f.next()
             testline = testlines.pop(0)
@@ -401,7 +415,7 @@
                 self.fail("readlines() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
             # Reading after iteration hit EOF shouldn't hurt either
-            f = open(TESTFN)
+            self.f = f = open(TESTFN)
             try:
                 for line in f:
                     pass
@@ -441,10 +455,14 @@
     # (including close()) concurrently without crashing the Python interpreter.
     # See #815646, #595601
 
+    # Modified for Jython so that each worker thread holds *and closes* its own
+    # file object, since we cannot rely on immediate garbage collection closing
+    # files. (Open file objects prevent deletion of TESTFN on Windows at least.)
+
     def setUp(self):
         self._threads = test_support.threading_setup()
-        self.f = None
         self.filename = TESTFN
+        self.exc_info = None
         with open(self.filename, "w") as f:
             f.write("\n".join("0123456789"))
         self._count_lock = threading.Lock()
@@ -453,35 +471,32 @@
         self.use_buffering = False
 
     def tearDown(self):
-        if self.f:
-            try:
-                self.f.close()
-            except (EnvironmentError, ValueError):
-                pass
         try:
             os.remove(self.filename)
-        except EnvironmentError:
-            pass
+        except EnvironmentError as ee:
+            # (Jython addition) detect failure common on Windows, on missing
+            # close, that creates spurious errors in subsequent tests.
+            if ee.errno != errno.ENOENT:
+                raise ee
         test_support.threading_cleanup(*self._threads)
 
     def _create_file(self):
         if self.use_buffering:
-            self.f = open(self.filename, "w+", buffering=1024*16)
+            return open(self.filename, "w+", buffering=1024*16)
         else:
-            self.f = open(self.filename, "w+")
+            return open(self.filename, "w+")
 
-    def _close_file(self):
+    def _close_file(self, f):
         with self._count_lock:
             self.close_count += 1
-        self.f.close()
+        f.close()
         with self._count_lock:
             self.close_success_count += 1
 
-    def _close_and_reopen_file(self):
-        self._close_file()
-        # if close raises an exception thats fine, self.f remains valid so
-        # we don't need to reopen.
-        self._create_file()
+    # Close one file object and return a new one
+    def _close_and_reopen_file(self, f):
+        self._close_file(f)
+        return self._create_file()
 
     def _run_workers(self, func, nb_workers, duration=0.2):
         with self._count_lock:
@@ -508,104 +523,116 @@
                 t.join()
 
     def _test_close_open_io(self, io_func, nb_workers=5):
+
         def worker():
-            self._create_file()
-            funcs = itertools.cycle((
-                lambda: io_func(),
-                lambda: self._close_and_reopen_file(),
-            ))
-            for f in funcs:
-                if not self.do_continue:
-                    break
-                try:
-                    f()
-                except (IOError, ValueError):
-                    pass
+            # Each worker has its own currently open file object
+            myfile = None
+            try:
+                myfile = self._create_file()
+                while self.do_continue:
+                    io_func(myfile)
+                    myfile = self._close_and_reopen_file(myfile)
+            except Exception as e:
+                # Stop the test (other threads) and remember why
+                self.do_continue = False
+                self.exc_info = sys.exc_info()
+            # Finally close the last file object
+            if myfile:
+                self._close_file(myfile)
+
         self._run_workers(worker, nb_workers)
+        if self.exc_info:
+            # Some worker saved an exception: re-raise it now
+            raise self.exc_info[0], self.exc_info[1], self.exc_info[2]
+
         if test_support.verbose:
             # Useful verbose statistics when tuning this test to take
             # less time to run but still ensuring that its still useful.
             #
             # the percent of close calls that raised an error
-            percent = 100. - 100.*self.close_success_count/self.close_count
+            percent = 100.
+            if self.close_count > 0:
+                percent -= 100.*self.close_success_count/self.close_count
             print self.close_count, ('%.4f ' % percent),
 
+    # Each test function defines an operation on the worker's file object
+
     def test_close_open(self):
-        def io_func():
+        def io_func(f):
             pass
         self._test_close_open_io(io_func)
 
     def test_close_open_flush(self):
-        def io_func():
-            self.f.flush()
+        def io_func(f):
+            f.flush()
         self._test_close_open_io(io_func)
 
     def test_close_open_iter(self):
-        def io_func():
-            list(iter(self.f))
+        def io_func(f):
+            list(iter(f))
         self._test_close_open_io(io_func)
 
     def test_close_open_isatty(self):
-        def io_func():
-            self.f.isatty()
+        def io_func(f):
+            f.isatty()
         self._test_close_open_io(io_func)
 
     def test_close_open_print(self):
-        def io_func():
-            print >> self.f, ''
+        def io_func(f):
+            print >> f, ''
         self._test_close_open_io(io_func)
 
     @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def test_close_open_print_buffered(self):
         self.use_buffering = True
-        def io_func():
-            print >> self.f, ''
+        def io_func(f):
+            print >> f, ''
         self._test_close_open_io(io_func)
 
     def test_close_open_read(self):
-        def io_func():
-            self.f.read(0)
+        def io_func(f):
+            f.read(0)
         self._test_close_open_io(io_func)
 
     def test_close_open_readinto(self):
-        def io_func():
+        def io_func(f):
             a = array('c', 'xxxxx')
-            self.f.readinto(a)
+            f.readinto(a)
         self._test_close_open_io(io_func)
 
     def test_close_open_readline(self):
-        def io_func():
-            self.f.readline()
+        def io_func(f):
+            f.readline()
         self._test_close_open_io(io_func)
 
     def test_close_open_readlines(self):
-        def io_func():
-            self.f.readlines()
+        def io_func(f):
+            f.readlines()
         self._test_close_open_io(io_func)
 
     def test_close_open_seek(self):
-        def io_func():
-            self.f.seek(0, 0)
+        def io_func(f):
+            f.seek(0, 0)
         self._test_close_open_io(io_func)
 
     def test_close_open_tell(self):
-        def io_func():
-            self.f.tell()
+        def io_func(f):
+            f.tell()
         self._test_close_open_io(io_func)
 
     def test_close_open_truncate(self):
-        def io_func():
-            self.f.truncate()
+        def io_func(f):
+            f.truncate()
         self._test_close_open_io(io_func)
 
     def test_close_open_write(self):
-        def io_func():
-            self.f.write('')
+        def io_func(f):
+            f.write('')
         self._test_close_open_io(io_func)
 
     def test_close_open_writelines(self):
-        def io_func():
-            self.f.writelines('')
+        def io_func(f):
+            f.writelines('')
         self._test_close_open_io(io_func)
 
 
@@ -683,14 +710,13 @@
 
 
 def test_main():
-    # Historically, these tests have been sloppy about removing TESTFN.
-    # So get rid of it no matter what.
-    try:
-        run_unittest(AutoFileTests, OtherFileTests, FileSubclassTests,
-            FileThreadingTests, StdoutTests)
-    finally:
-        if os.path.exists(TESTFN):
-            os.unlink(TESTFN)
+    run_unittest(
+             AutoFileTests, 
+             OtherFileTests, 
+             FileSubclassTests,
+             FileThreadingTests, 
+             StdoutTests
+         )
 
 if __name__ == '__main__':
     test_main()

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:45 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:45 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_wrong_name_expected_for?=
 =?utf-8?q?_file=28=29_keyword_argument=2E?=
Message-ID: <3dWzCx67NDz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/18c94ce2ce25
changeset:   7156:18c94ce2ce25
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Wed Nov 20 21:16:21 2013 +0000
summary:
  Fix wrong name expected for file() keyword argument.
Should be 'buffering' not 'bufsize': this must go back some time!

files:
  Lib/test/test_file2k.py                       |  1 -
  src/org/python/core/PyFile.java               |  2 +-
  src/org/python/core/__builtin__.java          |  2 +-
  src/org/python/modules/posix/PosixModule.java |  2 +-
  4 files changed, 3 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -582,7 +582,6 @@
             print >> f, ''
         self._test_close_open_io(io_func)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def test_close_open_print_buffered(self):
         self.use_buffering = True
         def io_func(f):
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -139,7 +139,7 @@
     @ExposedNew
     @ExposedMethod(doc = BuiltinDocs.file___init___doc)
     final void file___init__(PyObject[] args, String[] kwds) {
-        ArgParser ap = new ArgParser("file", args, kwds, new String[] {"name", "mode", "bufsize"},
+        ArgParser ap = new ArgParser("file", args, kwds, new String[] {"name", "mode", "buffering"},
                                      1);
         PyObject name = ap.getPyObject(0);
         if (!(name instanceof PyString)) {
diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java
--- a/src/org/python/core/__builtin__.java
+++ b/src/org/python/core/__builtin__.java
@@ -1720,7 +1720,7 @@
 
     @Override
     public PyObject __call__(PyObject args[], String kwds[]) {
-        ArgParser ap = new ArgParser("file", args, kwds, new String[] {"name", "mode", "bufsize"},
+        ArgParser ap = new ArgParser("file", args, kwds, new String[] {"name", "mode", "buffering"},
                                      1);
         PyObject obj = ap.getPyObject(0);
         if (obj.getJavaProxy() != null) {
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -738,7 +738,7 @@
             if (!file.canWrite()) {
                 throw Py.OSError(Errno.EPERM, path);
             }
-            throw Py.OSError("unlink(): an unknown error occurred" + absolutePath);
+            throw Py.OSError("unlink(): an unknown error occurred: " + absolutePath);
         }
     }
 

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:47 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:47 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_repr=28file=29_to_use_r?=
 =?utf-8?q?epr=28filename=29_with_escapes_and_quotes=2E?=
Message-ID: <3dWzCz0lRJz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/884533bda7d7
changeset:   7157:884533bda7d7
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Wed Nov 20 22:21:53 2013 +0000
summary:
  Fix repr(file) to use repr(filename) with escapes and quotes.
Removes a skip in test_file2k, and aligns Jython behaviour with CPython 2.7.5,
amending test_repr to require %r not '%s' filename format.

files:
  Lib/test/test_file2k.py         |   8 +++++---
  Lib/test/test_repr.py           |   4 ++--
  src/org/python/core/PyFile.java |  10 +++++++---
  3 files changed, 14 insertions(+), 8 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -87,13 +87,15 @@
         self.assertRaises(TypeError, self.f.writelines,
                           [NonString(), NonString()])
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def testRepr(self):
         # verify repr works
         self.assertTrue(repr(self.f).startswith("<open file '" + TESTFN))
         # see issue #14161
-        # Windows doesn't like \r\n\t" in the file name, but ' is ok
-        fname = 'xx\rxx\nxx\'xx"xx' if sys.platform != "win32" else "xx'xx"
+        if sys.platform == "win32" or (test_support.is_jython and os._name == "nt"):
+            # Windows doesn't like \r\n\t" in the file name, but ' is ok
+            fname = "xx'xx"
+        else:
+            fname = 'xx\rxx\nxx\'xx"xx'
         with open(fname, 'w') as f:
             self.addCleanup(os.remove, fname)
             self.assertTrue(repr(f).startswith(
diff --git a/Lib/test/test_repr.py b/Lib/test/test_repr.py
--- a/Lib/test/test_repr.py
+++ b/Lib/test/test_repr.py
@@ -130,10 +130,10 @@
     def test_file(self):
         fp = open(unittest.__file__)
         self.assertTrue(repr(fp).startswith(
-            "<open file '%s', mode 'r' at 0x" % unittest.__file__))
+            "<open file %r, mode 'r' at 0x" % unittest.__file__))
         fp.close()
         self.assertTrue(repr(fp).startswith(
-            "<closed file '%s', mode 'r' at 0x" % unittest.__file__))
+            "<closed file %r, mode 'r' at 0x" % unittest.__file__))
 
     def test_lambda(self):
         self.assertTrue(repr(lambda x: x).startswith(
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -521,11 +521,15 @@
     final String file_toString() {
         String state = file.closed() ? "closed" : "open";
         String id = Py.idstr(this);
+        String escapedName;
         if (name instanceof PyUnicode) {
-            String escapedName = PyString.encode_UnicodeEscape(name.toString(), false);
-            return String.format("<%s file u'%s', mode '%s' at %s>", state, escapedName, mode, id);
+            // unicode: always uses the format u'%s', and the escaped value thus:
+            escapedName = "u'"+PyString.encode_UnicodeEscape(name.toString(), false)+"'";
+        } else {
+            // anything else: uses repr(), which for str (common case) is smartly quoted
+            escapedName = name.__repr__().getString();
         }
-        return String.format("<%s file '%s', mode '%s' at %s>", state, name, mode, id);
+        return String.format("<%s file %s, mode '%s' at %s>", state, escapedName, mode, id);
     }
 
     @Override

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:48 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:48 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Re-work_PyFile=2EparseMode_?=
 =?utf-8?q?to_accept_=27Ub=27=2C_curing_test=5Ffile2k_failure=2E?=
Message-ID: <3dWzD03S7hz7Lkv@mail.python.org>

http://hg.python.org/jython/rev/4c2a46a387ec
changeset:   7158:4c2a46a387ec
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Nov 23 08:37:33 2013 +0000
summary:
  Re-work PyFile.parseMode to accept 'Ub', curing test_file2k failure.
Re-used the logic from _io.OpenMode rather than emulate the string-mangling from
CPython. Removes a skip.

files:
  Lib/test/test_file2k.py         |    4 +-
  src/org/python/core/PyFile.java |  108 +++++++++++++++----
  2 files changed, 86 insertions(+), 26 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -155,8 +155,10 @@
                 yield str(i)
         self.assertRaises(ValueError, self.f.writelines, nasty())
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def testIssue5677(self):
+        # We don't use the already-open file.
+        self.f.close()
+
         # Remark: Do not perform more than one test per open file,
         # since that does NOT catch the readline error on Windows.
         data = 'xxx'
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -22,7 +22,6 @@
 import org.python.core.io.TextIOBase;
 import org.python.core.io.TextIOWrapper;
 import org.python.core.io.UniversalIOWrapper;
-import org.python.core.util.StringUtil;
 import org.python.expose.ExposedDelete;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
@@ -199,41 +198,100 @@
     }
 
     /**
-     * Parse and validate the python file mode, returning a cleaned
-     * file mode suitable for FileIO.
+     * Parse and validate the python file mode, returning a cleaned file mode suitable for FileIO.
      *
      * @param mode a python file mode String
      * @return a RandomAccessFile mode String
      */
     private String parseMode(String mode) {
-        if (mode.length() == 0) {
-            throw Py.ValueError("empty mode string");
+
+        String message = null;
+        boolean duplicate = false, invalid = false;
+        int n = mode.length();
+
+        // Convert the letters to booleans, noticing duplicates
+        for (int i = 0; i < n; i++) {
+            char c = mode.charAt(i);
+
+            switch (c) {
+                case 'r':
+                    duplicate = reading;
+                    reading = true;
+                    break;
+                case 'w':
+                    duplicate = writing;
+                    writing = true;
+                    break;
+                case 'a':
+                    duplicate = appending;
+                    appending = true;
+                    break;
+                case '+':
+                    duplicate = updating;
+                    updating = true;
+                    break;
+                case 'b':
+                    duplicate = binary;
+                    binary = true;
+                    break;
+                case 'U':
+                    duplicate = universal;
+                    universal = true;
+                    break;
+                default:
+                    invalid = true;
+            }
+
+            // duplicate is set iff c was encountered previously */
+            if (duplicate) {
+                invalid = true;
+                break;
+            }
         }
 
-        String origMode = mode;
-        if (mode.contains("U")) {
-            universal = true;
-            mode = mode.replace("U", "");
-            if (mode.length() == 0) {
-                mode = "r";
-            } else if ("wa+".indexOf(mode.charAt(0)) > -1) {
-                throw Py.ValueError("universal newline mode can only be used with modes starting "
-                                    + "with 'r'");
+        // Implications
+        reading |= universal;
+        binary |= universal;
+
+        // Standard tests and the mode for FileIO
+        StringBuilder fileioMode = new StringBuilder();
+        if (!invalid) {
+            if (universal && (writing || appending)) {
+                // Not quite true, consider 'Ub', but it's what CPython says:
+                message = "universal newline mode can only be used with modes starting with 'r'";
+            } else {
+                // Build the FileIO mode string
+                if (reading) {
+                    fileioMode.append('r');
+                }
+                if (writing) {
+                    fileioMode.append('w');
+                }
+                if (appending) {
+                    fileioMode.append('a');
+                }
+                if (fileioMode.length() != 1) {
+                    // We should only have added one of the above
+                    message = "mode string must begin with one of 'r', 'w', 'a' or 'U', not '" //
+                            + mode + "'";
+                }
+                if (updating) {
+                    fileioMode.append('+');
+                }
             }
-        }
-        if ("rwa".indexOf(mode.charAt(0)) == -1) {
-            throw Py.ValueError("mode string must begin with one of 'r', 'w', 'a' or 'U', not '"
-                                + origMode + "'");
+            invalid |= (message != null);
         }
 
-        binary = mode.contains("b");
-        reading = mode.contains("r");
-        writing = mode.contains("w");
-        appending = mode.contains("a");
-        updating = mode.contains("+");
+        // Finally, if invalid, report this as an error
+        if (invalid) {
+            if (message == null) {
+                // Duplicates discovered or invalid symbols
+                message = String.format("invalid mode: '%.20s'", mode);
+            }
+            throw Py.ValueError(message);
+        }
 
-        return (reading ? "r" : "") + (writing ? "w" : "") + (appending ? "a" : "")
-                + (updating ? "+" : "");
+        return fileioMode.toString();
     }
 
     @ExposedMethod(defaults = {"-1"}, doc = BuiltinDocs.file_read_doc)

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:49 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:49 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Resolve_test=5Ffile2k_weak_?=
	=?utf-8?q?reference_test_failure=2E?=
Message-ID: <3dWzD155Tmz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/512780fedc0a
changeset:   7159:512780fedc0a
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Nov 23 09:21:12 2013 +0000
summary:
  Resolve test_file2k weak reference test failure.
This change set borrows the technique, used elsewhere in the suite, of
giving Java GC a chance to operate before testing reference expiration.

files:
  Lib/test/test_file2k.py |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -25,7 +25,6 @@
             self.f.close()
         os.remove(TESTFN)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def testWeakRefs(self):
         # verify weak references
         p = proxy(self.f)
@@ -33,6 +32,9 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        if test_support.is_jython: # GC is not immediate: borrow a trick
+            from test_weakref import extra_collect
+            extra_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:50 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:50 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_print_built-in_now_detects_?=
 =?utf-8?q?missing_stdout_as_RuntimeError=2E?=
Message-ID: <3dWzD26gkTz7Lks@mail.python.org>

http://hg.python.org/jython/rev/9e2bc96f3dbb
changeset:   7160:9e2bc96f3dbb
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Nov 23 17:37:35 2013 +0000
summary:
  print built-in now detects missing stdout as RuntimeError.
Fixes test failure in test_file2k (test_del_stdout_before_print) and removes the skip.

files:
  Lib/test/test_file2k.py                |   1 -
  src/org/python/core/StdoutWrapper.java |  10 +++++++---
  2 files changed, 7 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -659,7 +659,6 @@
         finally:
             sys.stdout = save_stdout
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def test_del_stdout_before_print(self):
         # Issue 4597: 'print' with no argument wasn't reporting when
         # sys.stdout was deleted.
diff --git a/src/org/python/core/StdoutWrapper.java b/src/org/python/core/StdoutWrapper.java
--- a/src/org/python/core/StdoutWrapper.java
+++ b/src/org/python/core/StdoutWrapper.java
@@ -25,10 +25,12 @@
         PyObject out = getObject(ss);
         if (out == null) {
             throw Py.AttributeError("missing sys." + this.name);
-        }
-        if (out.getJavaProxy() != null) {
+
+        } else if (out instanceof PyAttributeDeleted) {
+            throw Py.RuntimeError("lost sys." + this.name);
+
+        } else if (out.getJavaProxy() != null) {
             PyFile f = null;
-
             Object tojava = out.__tojava__(OutputStream.class);
             if (tojava != null && tojava != Py.NoConversion) {
                 f = new PyFile((OutputStream)tojava);
@@ -192,6 +194,7 @@
                 file.softspace = false;
             }
             file.flush();
+
         } else if (out instanceof PyFileWriter) {
             PyFileWriter file = (PyFileWriter)out;
             if (file.softspace) {
@@ -215,6 +218,7 @@
                 file.softspace = false;
             }
             file.flush();
+
         } else {
             PyObject ss = out.__findattr__("softspace");
             if (ss != null && ss.__nonzero__()) {

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:52 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:52 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_In_file=2Ewritelines=28=29?=
 =?utf-8?q?=2C_notice_if_an_iterable_argument_closes_the_file=2E?=
Message-ID: <3dWzD41DYYz7Lky@mail.python.org>

http://hg.python.org/jython/rev/3994804235da
changeset:   7161:3994804235da
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Mon Nov 25 22:11:29 2013 +0000
summary:
  In file.writelines(), notice if an iterable argument closes the file.
Resolves test_file2k failure in testNastyWritelinesGenerator.

files:
  Lib/test/test_file2k.py         |  1 -
  src/org/python/core/PyFile.java |  1 +
  2 files changed, 1 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -148,7 +148,6 @@
     def testReadWhenWriting(self):
         self.assertRaises(IOError, self.f.read)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def testNastyWritelinesGenerator(self):
         def nasty():
             for i in range(5):
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -425,6 +425,7 @@
         checkClosed();
         PyObject iter = Py.iter(lines, "writelines() requires an iterable argument");
         for (PyObject item = null; (item = iter.__iternext__()) != null;) {
+            checkClosed(); // ... in case a nasty iterable closed this file
             softspace = false;
             file.write(asWritable(item, "writelines() argument must be a sequence of strings"));
         }

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:53 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:53 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_test_that_Jython_file?=
 =?utf-8?q?=2Enext=28=29_mixes_with_file=2Eread*_operations=2E?=
Message-ID: <3dWzD52s0bz7Lkb@mail.python.org>

http://hg.python.org/jython/rev/8ee902d3eabc
changeset:   7162:8ee902d3eabc
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Mon Nov 25 23:02:03 2013 +0000
summary:
  Add test that Jython file.next() mixes with file.read* operations.
Remove FIXME from skip of CPython-specific test and add a Jython-specific test,
to show that iteartion and normal reading mix correctly.

files:
  Lib/test/test_file2k.py |  42 ++++++++++++++++++++++++++++-
  1 files changed, 41 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -324,7 +324,7 @@
         finally:
             os.unlink(TESTFN)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
+    @unittest.skipIf(test_support.is_jython, "Specific to CPython")
     def testIteration(self):
         # Test the complex interaction when mixing file-iteration and the
         # various read* methods. Ostensibly, the mixture could just be tested
@@ -436,6 +436,46 @@
         finally:
             os.unlink(TESTFN)
 
+    @unittest.skipUnless(test_support.is_jython, "Applicable to Jython")
+    def testIterationMixes(self):
+        # And now for something completely different. An implementation where
+        # various read* methods mix happily with iteration over the lines of
+        # a file using next().
+
+        sheep = [
+            "It's my belief that these sheep\n",
+            "are labouring under the\n",
+            "mis-apprehension that they're birds.\n",
+            "Now witness their attempts\n",
+            "to fly from tree to tree.\n",
+            "Notice that they do not so much fly\n",
+            "as plummet.\n"
+        ]
+
+        # Prepare the testfile
+        self.f = f = open(TESTFN, "w")
+        f.writelines(sheep)
+        f.close()
+
+        # Test for appropriate results mixing read* and iteration
+        self.f = f = open(TESTFN)
+        self.assertEqual(f.next(), sheep[0])
+        self.assertEqual(f.readline(), sheep[1])
+        self.assertEqual(f.next(), sheep[2])
+        self.assertEqual(f.read(5), sheep[3][:5])
+
+        r = array('c', "1234567")
+        f.readinto(r)
+        self.assertEqual(r, array('c', sheep[3][5:12]))
+
+        self.assertEqual(f.next(), sheep[3][12:])
+        r = f.readlines()
+        self.assertEqual(r, sheep[4:])
+        self.assertRaises(StopIteration, f.next)
+
+        f.close()
+
+
 class FileSubclassTests(unittest.TestCase):
 
     @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")

-- 
Repository URL: http://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 30 17:52:54 2013
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 30 Nov 2013 17:52:54 +0100 (CET)
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Sub-classes_of_file_are_mad?=
 =?utf-8?q?e_able_to_override_close=28=29_method=2E?=
Message-ID: <3dWzD64YZJz7LjQ@mail.python.org>

http://hg.python.org/jython/rev/1a8d17fa62ae
changeset:   7163:1a8d17fa62ae
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Tue Nov 26 22:31:53 2013 +0000
summary:
  Sub-classes of file are made able to override close() method.
Adds a close() method to file.derived and provides a regenerated
PyFileDerived class. testExit in test_file2k now passes.

files:
  Lib/test/test_file2k.py                |   7 +--
  src/org/python/core/PyFile.java        |   2 +-
  src/org/python/core/PyFileDerived.java |  24 ++++++++++++++
  src/templates/file.derived             |  13 +++++++
  4 files changed, 41 insertions(+), 5 deletions(-)


diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py
--- a/Lib/test/test_file2k.py
+++ b/Lib/test/test_file2k.py
@@ -478,7 +478,6 @@
 
 class FileSubclassTests(unittest.TestCase):
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: Not working on Jython")
     def testExit(self):
         # test that exiting with context calls subclass' close
         class C(file):
@@ -754,10 +753,10 @@
 
 def test_main():
     run_unittest(
-             AutoFileTests, 
-             OtherFileTests, 
+             AutoFileTests,
+             OtherFileTests,
              FileSubclassTests,
-             FileThreadingTests, 
+             FileThreadingTests,
              StdoutTests
          )
 
diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java
--- a/src/org/python/core/PyFile.java
+++ b/src/org/python/core/PyFile.java
@@ -526,7 +526,7 @@
 
     @ExposedMethod(doc = BuiltinDocs.file___exit___doc)
     final void file___exit__(PyObject type, PyObject value, PyObject traceback) {
-        file_close();
+        close();
     }
 
     public void __exit__(PyObject type, PyObject value, PyObject traceback) {
diff --git a/src/org/python/core/PyFileDerived.java b/src/org/python/core/PyFileDerived.java
--- a/src/org/python/core/PyFileDerived.java
+++ b/src/org/python/core/PyFileDerived.java
@@ -560,6 +560,18 @@
         return super.__ne__(other);
     }
 
+    public PyObject __format__(PyObject other) {
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("__format__");
+        if (impl!=null) {
+            PyObject res=impl.__get__(this,self_type).__call__(other);
+            if (res==Py.NotImplemented)
+                return null;
+            return res;
+        }
+        return super.__format__(other);
+    }
+
     public PyObject __iadd__(PyObject other) {
         PyType self_type=getType();
         PyObject impl=self_type.lookup("__iadd__");
@@ -1109,6 +1121,18 @@
         return super.__coerce_ex__(o);
     }
 
+    // Hand-crafted in file.derived
+
+    public void close() {
+        PyType self_type=getType();
+        PyObject impl=self_type.lookup("close");
+        if (impl!=null) {
+            impl.__get__(this,self_type).__call__();
+        } else {
+            super.close();
+        }
+    }
+
     public String toString() {
         PyType self_type=getType();
         PyObject impl=self_type.lookup("__repr__");
diff --git a/src/templates/file.derived b/src/templates/file.derived
--- a/src/templates/file.derived
+++ b/src/templates/file.derived
@@ -2,3 +2,16 @@
 want_dict: true
 ctr:
 incl: object
+
+rest:
+    // Hand-crafted in file.derived
+
+    public void close() {
+        PyType self_type = getType();
+        PyObject impl = self_type.lookup("close");
+        if (impl != null) {
+            impl.__get__(this, self_type).__call__();
+        } else {
+            super.close();
+        }
+    }

-- 
Repository URL: http://hg.python.org/jython