[Jython-checkins] jython (merge default -> default): Merge recent bytearray and buffer API work.
jeff.allen
jython-checkins at python.org
Fri Sep 7 23:40:22 CEST 2012
http://hg.python.org/jython/rev/09b64583841d
changeset: 6860:09b64583841d
parent: 6855:5e12ad3013ff
parent: 6859:357c8c1127c3
user: Jeff Allen <ja...py at farowl.co.uk>
date: Fri Sep 07 22:29:01 2012 +0100
summary:
Merge recent bytearray and buffer API work.
files:
.classpath | 2 +-
NEWS | 4 +
src/org/python/core/BaseBytes.java | 1288 +++------
src/org/python/core/BufferProtocol.java | 13 +-
src/org/python/core/PyBUF.java | 147 +-
src/org/python/core/PyBuffer.java | 165 +-
src/org/python/core/PyByteArray.java | 104 +-
src/org/python/core/PyMemoryView.java | 288 +-
src/org/python/core/PyString.java | 48 +-
src/org/python/core/buffer/BaseBuffer.java | 770 ++++-
src/org/python/core/buffer/SimpleBuffer.java | 218 +-
src/org/python/core/buffer/SimpleReadonlyBuffer.java | 128 -
src/org/python/core/buffer/SimpleStringBuffer.java | 116 +-
src/org/python/core/buffer/SimpleBuffer.java | 167 +-
src/org/python/core/buffer/Strided1DBuffer.java | 250 +
src/org/python/core/buffer/Strided1DWritableBuffer.java | 159 +
tests/java/org/python/core/BaseBytesTest.java | 2 +-
tests/java/org/python/core/PyBufferTest.java | 297 +-
18 files changed, 2660 insertions(+), 1506 deletions(-)
diff --git a/.classpath b/.classpath
--- a/.classpath
+++ b/.classpath
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
- <classpathentry kind="lib" path="build/exposed"/>
+ <classpathentry kind="lib" path="build/exposed" sourcepath="src"/>
<classpathentry excluding="com/ziclix/python/sql/handler/InformixDataHandler.java|com/ziclix/python/sql/handler/OracleDataHandler.java" kind="src" path="src"/>
<classpathentry kind="src" path="tests/modjy/java"/>
<classpathentry kind="src" path="build/gensrc"/>
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,7 @@
Bugs Fixed
- [ 1309 ] Server sockets do not support client options and propagate them to 'accept'ed client sockets.
- [ 1951 ] Bytecode Interpreter stack optimization for larger arguments
+ - [ 1894 ] bytearray does not support '+' or .join()
- [ 1921 ] compiler module broken in Jython 2.7
- [ 1920 ] Backport CO_FUTURE_PRINT_FUNCTION to Lib/compiler/pycodegen.py
- [ 1914 ] Float formatting broken in many non-English locales in Jython 2.7
@@ -16,6 +17,9 @@
- [ 1913 ] Support short -W options
- [ 1897 ] 2.7.0ax only has partial ssl support
- array_class in jarray module returns the "Array of a type" class
+ New Features
+ - bytearray complete
+ - a buffer API
Jython 2.7a2
- [ 1892 ] site-packages is not in sys.path
diff --git a/src/org/python/core/BaseBytes.java b/src/org/python/core/BaseBytes.java
--- a/src/org/python/core/BaseBytes.java
+++ b/src/org/python/core/BaseBytes.java
@@ -1,6 +1,5 @@
package org.python.core;
-import java.nio.charset.Charset;
import java.util.AbstractList;
import java.util.Arrays;
import java.util.Collection;
@@ -9,8 +8,6 @@
import java.util.List;
import java.util.ListIterator;
-import org.python.core.buffer.SimpleReadonlyBuffer;
-
/**
* Base class for Jython bytearray (and bytes in due course) that provides most of the Java API,
* including Java List behaviour. Attempts to modify the contents through this API will throw a
@@ -151,9 +148,8 @@
* ============================================================================================
*
* Methods here help subclasses set the initial state. They are designed with bytearray in mind,
- * but note that from Python 3, bytes() has the same set of calls and behaviours. In
- * Peterson's "sort of backport" to Python 2.x, bytes is effectively an alias for str and it
- * shows.
+ * but note that from Python 3, bytes() has the same set of calls and behaviours. In Peterson's
+ * "sort of backport" to Python 2.x, bytes is effectively an alias for str and it shows.
*/
/**
@@ -330,24 +326,25 @@
*/
protected void init(BufferProtocol value) throws PyException {
// Get the buffer view
- PyBuffer view = value.getBuffer(PyBUF.SIMPLE);
+ PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
// Create storage for the bytes and have the view drop them in
newStorage(view.getLen());
view.copyTo(storage, offset);
}
- /**
- * Helper for the Java API constructor from a {@link #View}. View is (perhaps) a stop-gap until
- * the Jython implementation of PEP 3118 (buffer API) is embedded.
- *
- * @param value a byte-oriented view
- */
- void init(View value) {
- int n = value.size();
- newStorage(n);
- value.copyTo(storage, offset);
- }
-
+// /**
+// * Helper for the Java API constructor from a {@link #PyBuffer}. View is (perhaps) a stop-gap
+// until
+// * the Jython implementation of PEP 3118 (buffer API) is embedded.
+// *
+// * @param value a byte-oriented view
+// */
+// void init(PyBuffer value) {
+// int n = value.getLen();
+// newStorage(n);
+// value.copyTo(storage, offset);
+// }
+//
/**
* Helper for <code>__new__</code> and <code>__init__</code> and the Java API constructor from
* bytearray or bytes in subclasses.
@@ -624,462 +621,53 @@
}
}
- /*
- * ============================================================================================
- * Wrapper class to make other objects into byte arrays
- * ============================================================================================
- *
- * In much of the bytearray and bytes API, the "other sequence" argument will accept any type
- * that supports the buffer protocol, that is, the object can supply a memoryview through which
- * the value is treated as a byte array. We have not implemented memoryview objects yet, and it
- * is not clear what the Java API should be. As a temporary expedient, we define here a
- * byte-oriented view on the key built-in types.
- */
-
- interface View {
-
- /**
- * Return the indexed byte as a byte
- *
- * @param index
- * @return byte at index
- */
- public byte byteAt(int index);
-
- /**
- * Return the indexed byte as an unsigned integer
- *
- * @param index
- * @return value of the byte at index
- */
- public int intAt(int index);
-
- /**
- * Number of bytes in the view: valid indexes are from <code>0</code> to
- * <code>size()-1</code>.
- *
- * @return the size
- */
- public int size();
-
- /**
- * Return a new view that is a simple slice of this one defined by <code>[start:end]</code>.
- * <code>Py.None</code> or <code>null</code> are acceptable for start and end, and have
- * Python slice semantics. Negative values for start or end are treated as "from the end",
- * in the usual manner of Python slices.
- *
- * @param start first element to include
- * @param end first element after slice, not to include
- * @return byte-oriented view
- */
- public View slice(PyObject start, PyObject end);
-
- /**
- * Copy the bytes of this view to the specified position in a destination array. All the
- * bytes of the View are copied.
- *
- * @param dest destination array
- * @param destPos index in the destination at which this.byteAt(0) is written
- * @throws ArrayIndexOutOfBoundsException if the destination is too small
- */
- public void copyTo(byte[] dest, int destPos) throws ArrayIndexOutOfBoundsException;
-
- /**
- * Test whether this View has the given prefix, that is, that the first bytes of this View
- * match all the bytes of the given prefix. By implication, the test returns false if there
- * are too few bytes in this view.
- *
- * @param prefix pattern to match
- * @return true if and only if this view has the given prefix
- */
- public boolean startswith(View prefix);
-
- /**
- * Test whether the slice <code>[offset:]</code> of this View has the given prefix, that is,
- * that the bytes of this View from index <code>offset</code> match all the bytes of the
- * give prefix. By implication, the test returns false if the offset puts the start or end
- * of the prefix outside this view (when <code>offset<0</code> or
- * <code>offset+prefix.size()>size()</code>). Python slice semantics are <em>not</em>
- * applied to <code>offset</code>.
- *
- * @param prefix pattern to match
- * @param offset at which to start the comparison in this view
- * @return true if and only if the slice [offset:<code>]</code> this view has the given
- * prefix
- */
- public boolean startswith(View prefix, int offset);
-
- /**
- * The standard memoryview out of bounds message (does not refer to the underlying type).
- */
- public static final String OUT_OF_BOUNDS = "index out of bounds";
-
- }
-
/**
- * Some common apparatus for views including the implementation of slice semantics.
- */
- static abstract class ViewBase implements View {
-
- /**
- * Provides an implementation of {@link View#slice(PyObject, PyObject)} that implements
- * Python contiguous slice semantics so that sub-classes only receive simplified requests
- * involving properly-bounded integer arguments via {@link #sliceImpl(int, int)}, a call to
- * {@link #byteAt(int)}, if the slice has length 1, or in the extreme case of a zero length
- * slice, no call at all.
- */
- public View slice(PyObject ostart, PyObject oend) {
- PySlice s = new PySlice(ostart, oend, null);
- int[] index = s.indicesEx(size()); // [ start, end, 1, end-start ]
- int len = index[3];
- // Provide efficient substitute when length is zero or one
- if (len < 1) {
- return new ViewOfNothing();
- } else if (len == 1) {
- return new ViewOfByte(byteAt(index[0]));
- } else { // General case: delegate to sub-class
- return sliceImpl(index[0], index[1]);
- }
- }
-
- /**
- * Implementation-specific part of returning a slice of the current view. This is called by
- * the default implementations of {@link #slice(int, int)} and
- * {@link #slice(PyObject, PyObject)} once the <code>start</code> and <code>end</code>
- * arguments have been reduced to simple integer indexes. It is guaranteed that
- * <code>start>=0</code> and <code>size()>=end>=start+2</code> when the method is called.
- * View objects for slices of length zero and one are dealt with internally by the
- * {@link #slice(PyObject, PyObject)} method, see {@link ViewOfNothing} and
- * {@link ViewOfByte}. Implementors are encouraged to do something more efficient than
- * piling on another wrapper.
- *
- * @param start first element to include
- * @param end first element after slice, not to include
- * @return byte-oriented view
- */
- protected abstract View sliceImpl(int start, int end);
-
- /**
- * Copy the bytes of this view to the specified position in a destination array. All the
- * bytes of the View are copied. The Base implementation simply loops over byteAt().
- */
- public void copyTo(byte[] dest, int destPos) throws ArrayIndexOutOfBoundsException {
- int n = this.size(), p = destPos;
- for (int i = 0; i < n; i++) {
- dest[p++] = byteAt(i);
- }
- }
-
- /**
- * Test whether this View has the given prefix, that is, that the first bytes of this View
- * match all the bytes of the given prefix. This class provides an implementation of
- * {@link View#startswith(View)} that simply returns <code>startswith(prefix,0)</code>
- */
- @Override
- public boolean startswith(View prefix) {
- return startswith(prefix, 0);
- }
-
- /**
- * Test whether this View has the given prefix, that is, that the first bytes of this View
- * match all the bytes of the given prefix. This class provides an implementation of
- * {@link View#startswith(View,int)} that loops over
- * <code>byteAt(i+offset)==prefix.byteAt(i)</code>
- */
- @Override
- public boolean startswith(View prefix, int offset) {
- int j = offset; // index in this
- if (j < 0) {
- // // Start of prefix is outside this view
- return false;
- } else {
- int len = prefix.size();
- if (j + len > this.size()) {
- // End of prefix is outside this view
- return false;
- } else {
- // Last resort: we have actually to look at the bytes!
- for (int i = 0; i < len; i++) {
- if (byteAt(j++) != prefix.byteAt(i)) {
- return false;
- }
- }
- return true; // They must all have matched
- }
- }
- }
-
- }
-
- /**
- * Return a wrapper providing a byte-oriented view for whatever object is passed, or return
- * <code>null</code> if we don't know how.
+ * Return a buffer exported by the argument, or return <code>null</code> if it does not bear the
+ * buffer API. The caller is responsible for calling {@link PyBuffer#release()} on the buffer,
+ * if the return value is not <code>null</code>.
*
* @param b object to wrap
* @return byte-oriented view or null
*/
- protected static View getView(PyObject b) {
+ protected static PyBuffer getView(PyObject b) {
+
if (b == null) {
return null;
- } else if (b instanceof BaseBytes) {
- BaseBytes bb = (BaseBytes)b;
- int len = bb.size;
- // Provide efficient substitute when length is zero or one
- if (len < 1) {
- return new ViewOfNothing();
- } else if (len == 1) {
- return new ViewOfByte(bb.byteAt(0));
- } else { // General case
- return new ViewOfBytes(bb);
- }
- } else if (b.getType() == PyString.TYPE) {
- String bs = b.asString();
- int len = bs.length();
- // Provide efficient substitute when length is zero
- if (len < 1) {
- return new ViewOfNothing();
- } else if (len == 1) {
- return new ViewOfByte(byteCheck(bs.charAt(0)));
- } else { // General case
- return new ViewOfString(bs);
- }
- }
- return null;
- }
-
- /**
- * Return a wrapper providing a byte-oriented view for a slice of whatever object is passed, or
- * return <code>null</code> if we don't know how.
- *
- * @param b object to wrap
- * @param start index of first byte to include
- * @param end index of first byte after slice
- * @return byte-oriented view or null
- */
- protected static View getView(PyObject b, PyObject start, PyObject end) {
- View whole = getView(b);
- if (whole != null) {
- return whole.slice(start, end);
+
+ } else if (b instanceof PyUnicode) {
+ /*
+ * PyUnicode has the BufferProtocol interface as it extends PyString. (It would bring
+ * you 0xff&charAt(i) in practice.) However, in CPython the unicode string does not have
+ * the buffer API.
+ */
+ return null;
+
+ } else if (b instanceof BufferProtocol) {
+ return ((BufferProtocol)b).getBuffer(PyBUF.FULL_RO);
+
} else {
return null;
}
}
/**
- * Return a wrapper providing a byte-oriented view for whatever object is passed, or raise an
- * exception if we don't know how.
+ * Return a buffer exported by the argument or raise an exception if it does not bear the buffer
+ * API. The caller is responsible for calling {@link PyBuffer#release()} on the buffer. The
+ * return value is never <code>null</code>.
*
* @param b object to wrap
* @return byte-oriented view
*/
- protected static View getViewOrError(PyObject b) {
- View res = getView(b);
- if (res == null) {
- String fmt = "cannot access type %s as bytes";
+ protected static PyBuffer getViewOrError(PyObject b) {
+ PyBuffer buffer = getView(b);
+ if (buffer != null) {
+ return buffer;
+ } else {
+ String fmt = "Type %s doesn't support the buffer API";
throw Py.TypeError(String.format(fmt, b.getType().fastGetName()));
- // A more honest response here would have been:
- // . String fmt = "type %s doesn't support the buffer API"; // CPython
- // . throw Py.NotImplementedError(String.format(fmt, b.getType().fastGetName()));
- // since our inability to handle certain types is lack of a buffer API generally.
}
- return res;
}
- /**
- * Return a wrapper providing a byte-oriented view for a slice of whatever object is passed, or
- * raise an exception if we don't know how.
- *
- * @param b object to wrap
- * @param start index of first byte to include
- * @param end index of first byte after slice
- * @return byte-oriented view or null
- */
- protected static View getViewOrError(PyObject b, PyObject start, PyObject end) {
- View whole = getViewOrError(b);
- return whole.slice(start, end);
- }
-
- /**
- * Wrapper providing a byte-oriented view for String (or PyString).
- */
- protected static class ViewOfString extends ViewBase {
-
- private String str;
-
- /**
- * Create a byte-oriented view of a String.
- *
- * @param str
- */
- public ViewOfString(String str) {
- this.str = str;
- }
-
- public byte byteAt(int index) {
- return byteCheck(str.charAt(index));
- }
-
- public int intAt(int index) {
- return str.charAt(index);
- }
-
- public int size() {
- return str.length();
- }
-
- public View sliceImpl(int start, int end) {
- return new ViewOfString(str.substring(start, end));
- }
-
- }
-
- /**
- * Wrapper providing a byte-oriented view for byte arrays descended from BaseBytes. Not that
- * this view is not safe against concurrent modification by this or another thread: if the byte
- * array type is mutable, and the contents change, the contents of the view are likely to be
- * invalid.
- */
- protected static class ViewOfBytes extends ViewBase {
-
- private byte[] storage;
- private int offset;
- private int size;
-
- /**
- * Create a byte-oriented view of a byte array descended from BaseBytes.
- *
- * @param obj
- */
- public ViewOfBytes(BaseBytes obj) {
- this.storage = obj.storage;
- this.offset = obj.offset;
- this.size = obj.size;
- }
-
- /**
- * Create a byte-oriented view of a slice of a byte array explicitly. If the size<=0, a zero-length
- * slice results.
- *
- * @param storage storage array
- * @param offset
- * @param size
- */
- ViewOfBytes(byte[] storage, int offset, int size) {
- if (size > 0) {
- this.storage = storage;
- this.offset = offset;
- this.size = size;
- } else {
- this.storage = emptyStorage;
- this.offset = 0;
- this.size = 0;
- }
- }
-
- public byte byteAt(int index) {
- return storage[offset + index];
- }
-
- public int intAt(int index) {
- return 0xff & storage[offset + index];
- }
-
- public int size() {
- return size;
- }
-
- public View sliceImpl(int start, int end) {
- return new ViewOfBytes(storage, offset + start, end - start);
- }
-
- /**
- * Copy the bytes of this view to the specified position in a destination array. All the
- * bytes of the View are copied. The view is of a byte array, so er can provide a more
- * efficient implementation than the default.
- */
- @Override
- public void copyTo(byte[] dest, int destPos) throws ArrayIndexOutOfBoundsException {
- System.arraycopy(storage, offset, dest, destPos, size);
- }
-
- }
-
- /**
- * Wrapper providing a byte-oriented view of just one byte. It looks silly, but it helps our
- * efficiency and code re-use.
- */
- protected static class ViewOfByte extends ViewBase {
-
- private byte storage;
-
- /**
- * Create a byte-oriented view of a byte array descended from BaseBytes.
- *
- * @param obj
- */
- public ViewOfByte(byte obj) {
- this.storage = obj;
- }
-
- public byte byteAt(int index) {
- return storage;
- }
-
- public int intAt(int index) {
- return 0xff & storage;
- }
-
- public int size() {
- return 1;
- }
-
- public View sliceImpl(int start, int end) {
- return new ViewOfByte(storage);
- }
-
- /**
- * Copy the byte the specified position in a destination array.
- */
- @Override
- public void copyTo(byte[] dest, int destPos) throws ArrayIndexOutOfBoundsException {
- dest[destPos] = storage;
- }
-
- }
-
- /**
- * Wrapper providing a byte-oriented view of an empty byte array or string. It looks even
- * sillier than wrapping one byte, but again helps our regularity and code re-use.
- */
- protected static class ViewOfNothing extends ViewBase {
-
- public byte byteAt(int index) {
- throw Py.IndexError(OUT_OF_BOUNDS);
- }
-
- public int intAt(int index) {
- throw Py.IndexError(OUT_OF_BOUNDS);
- }
-
- public int size() {
- return 0;
- }
-
- public View sliceImpl(int start, int end) {
- return new ViewOfNothing();
- }
-
- /**
- * Copy zero bytes the specified position, i.e. do nothing, even if dest[destPos] is out of
- * bounds.
- */
- @Override
- public void copyTo(byte[] dest, int destPos) {}
-
- }
-
- protected static final ViewOfNothing viewOfNothing = new ViewOfNothing();
-
/*
* ============================================================================================
* API for org.python.core.PySequence
@@ -1175,68 +763,24 @@
}
/**
- * Comparison function between two byte arrays returning 1, 0, or -1 as a>b, a==b, or a<b
- * respectively. The comparison is by value, using Python unsigned byte conventions, and
+ * Comparison function between a byte array and a buffer of bytes exported by some other object,
+ * such as a String, presented as a <code>PyBuffer</code>, returning 1, 0 or -1 as a>b, a==b, or
+ * a<b respectively. The comparison is by value, using Python unsigned byte conventions,
* left-to-right (low to high index). Zero bytes are significant, even at the end of the array:
- * <code>[1,2,3]<[1,2,3,0]</code>, for example and <code>[]</code> is less than every other
- * value, even <code>[0]</code>.
- *
- * @param a left-hand array in the comparison
- * @param b right-hand array in the comparison
- * @return 1, 0 or -1 as a>b, a==b, or a<b respectively
- */
- private static int compare(BaseBytes a, BaseBytes b) {
-
- // Compare elements one by one in these ranges:
- int ap = a.offset;
- int aEnd = ap + a.size;
- int bp = b.offset;
- int bEnd = bp + b.size;
-
- while (ap < aEnd) {
- if (bp >= bEnd) {
- // a is longer than b
- return 1;
- } else {
- // Compare the corresponding bytes (as unsigned ints)
- int aVal = 0xff & a.storage[ap++];
- int bVal = 0xff & b.storage[bp++];
- int diff = aVal - bVal;
- if (diff != 0) {
- return (diff < 0) ? -1 : 1;
- }
- }
- }
-
- // All the bytes matched and we reached the end of a
- if (bp < bEnd) {
- // But we didn't reach the end of b
- return -1;
- } else {
- // And the end of b at the same time, so they're equal
- return 0;
- }
-
- }
-
- /**
- * Comparison function between a byte array and a byte-oriented View of some other object, such
- * as a String, returning 1, 0 or -1 as a>b, a==b, or a<b respectively. The comparison is by
- * value, using Python unsigned byte conventions, left-to-right (low to high index). Zero bytes
- * are significant, even at the end of the array: <code>[65,66,67]<"ABC\u0000"</code>, for
- * example and <code>[]</code> is less than every non-empty b, while <code>[]==""</code>.
+ * <code>[65,66,67]<"ABC\u0000"</code>, for example and <code>[]</code> is less than every
+ * non-empty b, while <code>[]==""</code>.
*
* @param a left-hand array in the comparison
* @param b right-hand wrapped object in the comparison
* @return 1, 0 or -1 as a>b, a==b, or a<b respectively
*/
- private static int compare(BaseBytes a, View b) {
+ private static int compare(BaseBytes a, PyBuffer b) {
// Compare elements one by one in these ranges:
int ap = a.offset;
int aEnd = ap + a.size;
int bp = 0;
- int bEnd = b.size();
+ int bEnd = b.getLen();
while (ap < aEnd) {
if (bp >= bEnd) {
@@ -1265,8 +809,8 @@
}
/**
- * Comparison function between byte array types and any other object. The set of 6
- * "rich comparison" operators are based on this.
+ * Comparison function between byte array types and any other object. The six "rich comparison"
+ * operators are based on this.
*
* @param b
* @return 1, 0 or -1 as this>b, this==b, or this<b respectively, or -2 if the comparison is
@@ -1283,16 +827,20 @@
} else {
// Try to get a byte-oriented view
- View bv = getView(b);
+ PyBuffer bv = getView(b);
if (bv == null) {
- // Signifies a type mis-match. See PyObject _cmp_unsafe() and related code.
+ // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
return -2;
} else {
- // Object supported by our interim memory view
- return compare(this, bv);
-
+ try {
+ // Compare this with other object viewed as a buffer
+ return compare(this, bv);
+ } finally {
+ // Must alsways let go of the buffer
+ bv.release();
+ }
}
}
}
@@ -1314,20 +862,25 @@
} else {
// Try to get a byte-oriented view
- View bv = getView(b);
+ PyBuffer bv = getView(b);
if (bv == null) {
- // Signifies a type mis-match. See PyObject _cmp_unsafe() and related code.
+ // Signifies a type mis-match. See PyObject._cmp_unsafe() and related code.
return -2;
- } else if (bv.size() != size) {
- // Different size: can't be equal, and we don't care which is bigger
- return 1;
-
} else {
- // Object supported by our interim memory view
- return compare(this, bv);
-
+ try {
+ if (bv.getLen() != size) {
+ // Different size: can't be equal, and we don't care which is bigger
+ return 1;
+ } else {
+ // Compare this with other object viewed as a buffer
+ return compare(this, bv);
+ }
+ } finally {
+ // Must alsways let go of the buffer
+ bv.release();
+ }
}
}
}
@@ -1455,10 +1008,14 @@
return index(b) >= 0;
} else {
// Caller is treating this as a byte-string and looking for substring 'target'
- View targetView = getViewOrError(target);
- Finder finder = new Finder(targetView);
- finder.setText(this);
- return finder.nextIndex() >= 0;
+ PyBuffer targetView = getViewOrError(target);
+ try {
+ Finder finder = new Finder(targetView);
+ finder.setText(this);
+ return finder.nextIndex() >= 0;
+ } finally {
+ targetView.release();
+ }
}
}
@@ -1471,55 +1028,102 @@
*
* @param target prefix or suffix sequence to find (of a type viewable as a byte sequence) or a
* tuple of those.
- * @param start of slice to search.
- * @param end of slice to search.
+ * @param ostart of slice to search.
+ * @param oend of slice to search.
* @param endswith true if we are doing endswith, false if startswith.
* @return true if and only if this bytearray ends with (one of) <code>target</code>.
*/
protected final synchronized boolean basebytes_starts_or_endswith(PyObject target,
- PyObject start,
- PyObject end,
- boolean endswith) {
+ PyObject ostart, PyObject oend, boolean endswith) {
/*
- * This cheap trick saves us from maintaining two almost identical methods and mirrors
- * CPython's _bytearray_tailmatch().
- *
- * Start with a view of the slice we are searching.
+ * This cheap 'endswith' trick saves us from maintaining two almost identical methods and
+ * mirrors CPython's _bytearray_tailmatch().
*/
- View v = new ViewOfBytes(this).slice(start, end);
- int len = v.size();
- int offset = 0;
+ int[] index = indicesEx(ostart, oend); // [ start, end, 1, end-start ]
if (target instanceof PyTuple) {
// target is a tuple of suffixes/prefixes and only one need match
- for (PyObject s : ((PyTuple)target).getList()) {
- // Error if not something we can treat as a view of bytes
- View vt = getViewOrError(s);
- if (endswith) {
- offset = len - vt.size();
- }
- if (v.startswith(vt, offset)) {
+ for (PyObject t : ((PyTuple)target).getList()) {
+ if (match(t, index[0], index[3], endswith)) {
return true;
}
}
return false; // None of them matched
} else {
- // Error if target is not something we can treat as a view of bytes
- View vt = getViewOrError(target);
- if (endswith) {
- offset = len - vt.size();
+ return match(target, index[0], index[3], endswith);
+ }
+ }
+
+ /**
+ * Test whether the slice <code>[pos:pos+n]</code> of this byte array matches the given target
+ * object (accessed as a {@link PyBuffer}) at one end or the orher. That is, if
+ * <code>endswith==false</code> test whether the bytes from index <code>pos</code> match all the
+ * bytes of the target; if <code>endswith==false</code> test whether the bytes up to index
+ * <code>pos+n-1</code> match all the bytes of the target. By implication, the test returns
+ * false if the target is bigger than n. The caller guarantees that the slice
+ * <code>[pos:pos+n]</code> is within the byte array.
+ *
+ * @param target pattern to match
+ * @param pos at which to start the comparison
+ * @return true if and only if the slice [offset:<code>]</code> matches the given target
+ */
+ private boolean match(PyObject target, int pos, int n, boolean endswith) {
+
+ // Error if not something we can treat as a view of bytes
+ PyBuffer vt = getViewOrError(target);
+
+ try {
+ int j = 0, len = vt.getLen();
+
+ if (!endswith) {
+ // Match is at the start of the range [pos:pos+n]
+ if (len > n) {
+ return false;
+ }
+ } else {
+ // Match is at the end of the range [pos:pos+n]
+ j = n - len;
+ if (j < 0) {
+ return false;
+ }
}
- return v.startswith(vt, offset);
+
+ // Last resort: we have actually to look at the bytes!
+ j += offset + pos;
+ for (int i = 0; i < len; i++) {
+ if (storage[j++] != vt.byteAt(i)) {
+ return false;
+ }
+ }
+ return true; // They must all have matched
+
+ } finally {
+ // Let go of the buffer we acquired
+ vt.release();
}
}
/**
+ * Helper to convert [ostart:oend] to integers with slice semantics relative to this byte array.
+ * The retruned array of ints contains [ start, end, 1, end-start ].
+ *
+ * @param ostart of slice to define.
+ * @param oend of slice to define.
+ * @return [ start, end, 1, end-start ]
+ */
+ private int[] indicesEx(PyObject ostart, PyObject oend) {
+ // Convert [ostart:oend] to integers with slice semantics relative to this byte array
+ PySlice s = new PySlice(ostart, oend, null);
+ return s.indicesEx(size); // [ start, end, 1, end-start ]
+ }
+
+ /**
* Present the bytes of a byte array, with no decoding, as a Java String. The bytes are treated
* as unsigned character codes, and copied to the to the characters of a String with no change
* in ordinal value. This could also be described as 'latin-1' or 'ISO-8859-1' decoding of the
* byte array to a String, since this character encoding is numerically equal to Unicode.
- *
+ *
* @return the byte array as a String
*/
@Override
@@ -1641,10 +1245,10 @@
* Python API for find and replace operations
* ============================================================================================
*
- * A large part of the CPython bytearray.c is devoted to replace( old, new [, count ] ).
- * The special section here reproduces that in Java, but whereas CPython makes heavy use
- * of the buffer API and C memcpy(), we use View.copyTo. The logic is much the same, however,
- * even down to variable names.
+ * A large part of the CPython bytearray.c is devoted to replace( old, new [, count ] ). The
+ * special section here reproduces that in Java, but whereas CPython makes heavy use of the
+ * buffer API and C memcpy(), we use PyBuffer.copyTo. The logic is much the same, however, even
+ * down to variable names.
*/
/**
@@ -1665,23 +1269,23 @@
}
/**
- * This class implements the Boyer-Moore-Horspool Algorithm for findind a pattern in text,
- * applied to byte arrays. The BMH algorithm uses a table of bad-character skips derived from
- * the pattern. The bad-character skips table tells us how far from the end of the pattern is a
- * byte that might match the text byte currently aligned with the end of the pattern. For
- * example, suppose the pattern (of length 6) is at position 4:
+ * This class implements the Boyer-Moore-Horspool Algorithm for find a pattern in text, applied
+ * to byte arrays. The BMH algorithm uses a table of bad-character skips derived from the
+ * pattern. The bad-character skips table tells us how far from the end of the pattern is a byte
+ * that might match the text byte currently aligned with the end of the pattern. For example,
+ * suppose the pattern ("panama") is at position 6:
*
* <pre>
* 1 2 3
* 0123456789012345678901234567890
* Text: a man, a map, a panama canal
- * Pattern: panama
+ * Pattern: panama
* </pre>
*
- * This puts the 'm' of 'map' against the last byte 'a' of the pattern. Rather than testing the
- * pattern, we will look up 'm' in the skip table. There is an 'm' just one step from the end of
- * the pattern, so we will move the pattern one place to the right before trying to match it.
- * This allows us to move in large strides throughthe text.
+ * This puts the 'p' of 'map' against the last byte 'a' of the pattern. Rather than testing the
+ * pattern, we will look up 'p' in the skip table. There is an 'p' just 5 steps from the end of
+ * the pattern, so we will move the pattern 5 places to the right before trying to match it.
+ * This allows us to move in large strides through the text.
*/
protected static class Finder {
@@ -1691,7 +1295,7 @@
*
* @param pattern A vew that presents the pattern as an array of bytes
*/
- public Finder(View pattern) {
+ public Finder(PyBuffer pattern) {
this.pattern = pattern;
}
@@ -1717,7 +1321,7 @@
*/
protected int[] calculateSkipTable() {
int[] skipTable = new int[MASK + 1];
- int m = pattern.size();
+ int m = pattern.getLen();
// Default skip is the pattern length: for bytes not in the pattern.
Arrays.fill(skipTable, m);
// For each byte in the pattern, make an entry for how far it is from the end.
@@ -1761,30 +1365,31 @@
this.text = text;
this.left = start;
- right = start + size - pattern.size() + 1; // Last pattern position + 1
+ right = start + size - pattern.getLen() + 1; // Last pattern position + 1
/*
* We defer computing the table from construction to this point mostly because
* calculateSkipTable() may be overridden, and we want to use the right one.
*/
- if (pattern.size() > 1 && skipTable == null) {
+ if (pattern.getLen() > 1 && skipTable == null) {
skipTable = calculateSkipTable();
}
}
- protected final View pattern;
+ protected final PyBuffer pattern;
protected byte[] text = emptyStorage; // in case we forget to setText()
protected int left = 0; // Leftmost pattern position to use
protected int right = 0; // Rightmost pattern position + 1
/**
- * Return the index in the text array where the preceding pattern match ends (one beyond the last
- * character matched), which may also be one beyond the effective end ofthe text.
- * Between a call to setText() and the first call to
- * <code>nextIndex()</code> return the start position.
+ * Return the index in the text array where the preceding pattern match ends (one beyond the
+ * last character matched), which may also be one beyond the effective end ofthe text.
+ * Between a call to setText() and the first call to <code>nextIndex()</code> return the
+ * start position.
* <p>
* The following idiom may be used:
+ *
* <pre>
* f.setText(text);
* int p = f.nextIndex();
@@ -1806,7 +1411,7 @@
* @return matching index or -1 if no (further) occurrences found
*/
public int nextIndex() {
- int m = pattern.size();
+ int m = pattern.getLen();
if (skipTable != null) { // ... which it will not be if m>1 and setText() was called
/*
@@ -1927,7 +1532,7 @@
*
* @param pattern A vew that presents the pattern as an array of bytes
*/
- public ReverseFinder(View pattern) {
+ public ReverseFinder(PyBuffer pattern) {
super(pattern);
}
@@ -1952,7 +1557,7 @@
*/
protected int[] calculateSkipTable() {
int[] skipTable = new int[MASK + 1];
- int m = pattern.size();
+ int m = pattern.getLen();
// Default skip is the pattern length: for bytes not in the pattern.
Arrays.fill(skipTable, m);
// For each byte in the pattern, make an entry for how far it is from the start.
@@ -1968,7 +1573,7 @@
* @return the new effective end of the text
*/
public int currIndex() {
- return right+pattern.size()-1;
+ return right + pattern.getLen() - 1;
}
/**
@@ -1980,7 +1585,7 @@
*/
public int nextIndex() {
- int m = pattern.size();
+ int m = pattern.getLen();
if (skipTable != null) { // ... which it will not be if m>1 and setText() was called
/*
@@ -2058,8 +1663,8 @@
*
* @param bytes to be in the set.
*/
- public ByteSet(View bytes) {
- int n = bytes.size();
+ public ByteSet(PyBuffer bytes) {
+ int n = bytes.getLen();
for (int i = 0; i < n; i++) {
int c = bytes.intAt(i);
long mask = 1L << c; // Only uses low 6 bits of c (JLS)
@@ -2081,7 +1686,7 @@
}
/**
- * Test to see if the byte (expressed an an integer) is in the set.
+ * Test to see if the byte (expressed as an integer) is in the set.
*
* @param b integer value of the byte
* @return true iff b is in the set
@@ -2096,15 +1701,15 @@
}
/**
- * Convenience routine producing a ValueError for "empty separator" if the View is of an object with zero length,
- * and returning the length otherwise.
+ * Convenience routine producing a ValueError for "empty separator" if the PyBuffer is of an
+ * object with zero length, and returning the length otherwise.
*
* @param separator view to test
* @return the length of the separator
- * @throws PyException if the View is zero length
+ * @throws PyException if the PyBuffer is zero length
*/
- protected final static int checkForEmptySeparator(View separator) throws PyException {
- int n = separator.size();
+ protected final static int checkForEmptySeparator(PyBuffer separator) throws PyException {
+ int n = separator.getLen();
if (n == 0) {
throw Py.ValueError("empty separator");
}
@@ -2200,14 +1805,18 @@
* @return count of occurrences of sub within this byte array
*/
final int basebytes_count(PyObject sub, PyObject ostart, PyObject oend) {
- Finder finder = new Finder(getViewOrError(sub));
-
- // Convert [start:end] to integers
- PySlice s = new PySlice(ostart, oend, null);
- int[] index = s.indicesEx(size()); // [ start, end, 1, end-start ]
-
- // Make this slice the thing we count within.
- return finder.count(storage, offset + index[0], index[3]);
+ PyBuffer vsub = getViewOrError(sub);
+ try {
+ Finder finder = new Finder(vsub);
+
+ // Convert [ostart:oend] to integers
+ int[] index = indicesEx(ostart, oend); // [ start, end, 1, end-start ]
+
+ // Make this slice the thing we count within.
+ return finder.count(storage, offset + index[0], index[3]);
+ } finally {
+ vsub.release();
+ }
}
/**
@@ -2224,8 +1833,13 @@
* @return index of start of occurrence of sub within this byte array
*/
final int basebytes_find(PyObject sub, PyObject ostart, PyObject oend) {
- Finder finder = new Finder(getViewOrError(sub));
- return find(finder, ostart, oend);
+ PyBuffer vsub = getViewOrError(sub);
+ try {
+ Finder finder = new Finder(vsub);
+ return find(finder, ostart, oend);
+ } finally {
+ vsub.release();
+ }
}
/**
@@ -2269,9 +1883,9 @@
value = (value << 4) + hexDigit(c);
r[p++] = (byte)value;
} catch (IllegalArgumentException e) {
- throw Py.ValueError(String.format(fmt, i-1));
+ throw Py.ValueError(String.format(fmt, i - 1));
} catch (IndexOutOfBoundsException e) {
- throw Py.ValueError(String.format(fmt, i-2));
+ throw Py.ValueError(String.format(fmt, i - 2));
}
}
}
@@ -2310,53 +1924,62 @@
*/
final synchronized PyByteArray basebytes_join(Iterable<? extends PyObject> iter) {
- List<View> iterList = new LinkedList<View>();
+ List<PyBuffer> iterList = new LinkedList<PyBuffer>();
long mysize = this.size;
long totalSize = 0;
boolean first = true;
- for (PyObject o : iter) {
- // Scan the iterable into a list, checking type and accumulating size
- View v = getView(o);
- if (v == null) {
- // Unsuitable object to be in this join
- String fmt = "can only join an iterable of bytes (item %d has type '%.80s')";
- throw Py.TypeError(String.format(fmt, iterList.size(), o.getType().fastGetName()));
+ try {
+ for (PyObject o : iter) {
+ // Scan the iterable into a list, checking type and accumulating size
+ PyBuffer v = getView(o);
+ if (v == null) {
+ // Unsuitable object to be in this join
+ String fmt = "can only join an iterable of bytes (item %d has type '%.80s')";
+ throw Py.TypeError(String.format(fmt, iterList.size(), o.getType()
+ .fastGetName()));
+ }
+ iterList.add(v);
+ totalSize += v.getLen();
+
+ // Each element after the first is preceded by a copy of this
+ if (!first) {
+ totalSize += mysize;
+ } else {
+ first = false;
+ }
+
+ if (totalSize > Integer.MAX_VALUE) {
+ throw Py.OverflowError("join() result would be too long");
+ }
}
- iterList.add(v);
- totalSize += v.size();
-
- // Each element after the first is preceded by a copy of this
- if (!first) {
- totalSize += mysize;
- } else {
- first = false;
+
+ // Load the Views from the iterator into a new PyByteArray
+ PyByteArray result = new PyByteArray((int)totalSize);
+ int p = result.offset; // Copy-to pointer
+ first = true;
+
+ for (PyBuffer v : iterList) {
+ // Each element after the first is preceded by a copy of this
+ if (!first) {
+ System.arraycopy(storage, offset, result.storage, p, size);
+ p += size;
+ } else {
+ first = false;
+ }
+ // Then the element from the iterable
+ v.copyTo(result.storage, p);
+ p += v.getLen();
}
- if (totalSize > Integer.MAX_VALUE) {
- throw Py.OverflowError("join() result would be too long");
+ return result;
+
+ } finally {
+ // All the buffers we acquired have to be realeased
+ for (PyBuffer v : iterList) {
+ v.release();
}
}
-
- // Load the Views from the iterator into a new PyByteArray
- PyByteArray result = new PyByteArray((int)totalSize);
- int p = result.offset; // Copy-to pointer
- first = true;
-
- for (View v : iterList) {
- // Each element after the first is preceded by a copy of this
- if (!first) {
- System.arraycopy(storage, offset, result.storage, p, size);
- p += size;
- } else {
- first = false;
- }
- // Then the element from the iterable
- v.copyTo(result.storage, p);
- p += v.size();
- }
-
- return result;
}
/**
@@ -2383,20 +2006,26 @@
*/
final synchronized PyTuple basebytes_partition(PyObject sep) {
- // Create a Finder for the separtor and set it on this byte array
- View separator = getViewOrError(sep);
- int n = checkForEmptySeparator(separator);
- Finder finder = new Finder(separator);
- finder.setText(this);
-
- // We only uuse it once, to find the first occurrence
- int p = finder.nextIndex() - offset;
- if (p >= 0) {
- // Found at p, so we'll be returning ([0:p], [p:p+n], [p+n:])
- return partition(p, p + n);
- } else {
- // Not found: choose values leading to ([0:size], '', '')
- return partition(size, size);
+ // View the separator as a byte array (or error if we can't)
+ PyBuffer separator = getViewOrError(sep);
+
+ try {
+ // Create a Finder for the separator and set it on this byte array
+ int n = checkForEmptySeparator(separator);
+ Finder finder = new Finder(separator);
+ finder.setText(this);
+
+ // We only use it once, to find the first occurrence
+ int p = finder.nextIndex() - offset;
+ if (p >= 0) {
+ // Found at p, so we'll be returning ([0:p], [p:p+n], [p+n:])
+ return partition(p, p + n);
+ } else {
+ // Not found: choose values leading to ([0:size], '', '')
+ return partition(size, size);
+ }
+ } finally {
+ separator.release();
}
}
@@ -2415,7 +2044,7 @@
return new PyTuple(head, sep, tail);
}
- /**
+ /**
* Ready-to-expose implementation of Python <code>rfind( sub [, start [, end ]] )</code>. Return
* the highest index in the byte array where byte sequence <code>sub</code> is found, such that
* <code>sub</code> is contained in the slice <code>[start:end]</code>. Arguments
@@ -2429,14 +2058,20 @@
* @return index of start of occurrence of sub within this byte array
*/
final int basebytes_rfind(PyObject sub, PyObject ostart, PyObject oend) {
- Finder finder = new ReverseFinder(getViewOrError(sub));
- return find(finder, ostart, oend);
+ PyBuffer vsub = getViewOrError(sub);
+ try {
+ Finder finder = new ReverseFinder(vsub);
+ return find(finder, ostart, oend);
+ } finally {
+ vsub.release();
+ }
}
/**
* Common code for Python <code>find( sub [, start [, end ]] )</code> and
* <code>rfind( sub [, start [, end ]] )</code>. Return the lowest or highest index in the byte
- * array where byte sequence used to construct <code>finder</code> is found.
+ * array where byte sequence used to construct <code>finder</code> is found. The particular type
+ * (plain <code>Finder</code> or <code>ReverseFinder</code>) determines the direction.
*
* @param finder for the bytes to find, sometime forwards, sometime backwards
* @param ostart of slice to search
@@ -2445,9 +2080,8 @@
*/
private final int find(Finder finder, PyObject ostart, PyObject oend) {
- // Convert [start:end] to integers
- PySlice s = new PySlice(ostart, oend, null);
- int[] index = s.indicesEx(size()); // [ start, end, 1, end-start ]
+ // Convert [ostart:oend] to integers
+ int[] index = indicesEx(ostart, oend); // [ start, end, 1, end-start ]
// Make this slice the thing we search. Note finder works with Java index in storage.
finder.setText(storage, offset + index[0], index[3]);
@@ -2471,70 +2105,81 @@
*/
final synchronized PyByteArray basebytes_replace(PyObject oldB, PyObject newB, int maxcount) {
- View from = getViewOrError(oldB);
- View to = getViewOrError(newB);
-
- /*
- * The logic of the first section is copied exactly from CPython in order to get the same
- * behaviour. The "headline" description of replace is simple enough but the corner cases
- * can be surprising:
- */
- // >>> bytearray(b'hello').replace(b'',b'-')
- // bytearray(b'-h-e-l-l-o-')
- // >>> bytearray(b'hello').replace(b'',b'-',3)
- // bytearray(b'-h-e-llo')
- // >>> bytearray(b'hello').replace(b'',b'-',1)
- // bytearray(b'-hello')
- // >>> bytearray().replace(b'',b'-')
- // bytearray(b'-')
- // >>> bytearray().replace(b'',b'-',1) # ?
- // bytearray(b'')
-
- if (maxcount < 0) {
- maxcount = Integer.MAX_VALUE;
-
- } else if (maxcount == 0 || size == 0) {
- // nothing to do; return the original bytes
- return new PyByteArray(this);
- }
-
- int from_len = from.size();
- int to_len = to.size();
-
- if (maxcount == 0 || (from_len == 0 && to_len == 0)) {
- // nothing to do; return the original bytes
- return new PyByteArray(this);
-
- } else if (from_len == 0) {
- // insert the 'to' bytes everywhere.
- // >>> "Python".replace("", ".")
- // '.P.y.t.h.o.n.'
- return replace_interleave(to, maxcount);
-
- } else if (size == 0) {
- // Special case for "".replace("", "A") == "A"
- return new PyByteArray(to);
-
- } else if (to_len == 0) {
- // Delete occurrences of the 'from' bytes
- return replace_delete_substring(from, maxcount);
-
- } else if (from_len == to_len) {
- // The result is the same size as this byte array, whatever the number of replacements.
- return replace_substring_in_place(from, to, maxcount);
-
- } else {
- // Otherwise use the generic algorithm
- return replace_substring(from, to, maxcount);
+ // View the to and from as byte arrays (or error if we can't)
+ PyBuffer to = getViewOrError(newB), from = null;
+ try {
+ from = getViewOrError(oldB);
+ /*
+ * The logic of the first section is copied exactly from CPython in order to get the
+ * same behaviour. The "headline" description of replace is simple enough but the corner
+ * cases can be surprising:
+ */
+ // >>> bytearray(b'hello').replace(b'',b'-')
+ // bytearray(b'-h-e-l-l-o-')
+ // >>> bytearray(b'hello').replace(b'',b'-',3)
+ // bytearray(b'-h-e-llo')
+ // >>> bytearray(b'hello').replace(b'',b'-',1)
+ // bytearray(b'-hello')
+ // >>> bytearray().replace(b'',b'-')
+ // bytearray(b'-')
+ // >>> bytearray().replace(b'',b'-',1) # ?
+ // bytearray(b'')
+
+ if (maxcount < 0) {
+ maxcount = Integer.MAX_VALUE;
+
+ } else if (maxcount == 0 || size == 0) {
+ // nothing to do; return the original bytes
+ return new PyByteArray(this);
+ }
+
+ int from_len = from.getLen();
+ int to_len = to.getLen();
+
+ if (maxcount == 0 || (from_len == 0 && to_len == 0)) {
+ // nothing to do; return the original bytes
+ return new PyByteArray(this);
+
+ } else if (from_len == 0) {
+ // insert the 'to' bytes everywhere.
+ // >>> "Python".replace("", ".")
+ // '.P.y.t.h.o.n.'
+ return replace_interleave(to, maxcount);
+
+ } else if (size == 0) {
+ // Special case for "".replace("", "A") == "A"
+ return new PyByteArray(to);
+
+ } else if (to_len == 0) {
+ // Delete occurrences of the 'from' bytes
+ return replace_delete_substring(from, maxcount);
+
+ } else if (from_len == to_len) {
+ // Result is same size as this byte array, whatever the number of replacements.
+ return replace_substring_in_place(from, to, maxcount);
+
+ } else {
+ // Otherwise use the generic algorithm
+ return replace_substring(from, to, maxcount);
+ }
+
+ } finally {
+ /*
+ * Release the buffers we acquired: there must be a to buffer and there might be a from
+ * buffer.
+ */
+ to.release();
+ if (from != null) {
+ from.release();
+ }
}
}
/*
* Algorithms for different cases of string replacement. CPython also has specialisations for
- * when 'from' or 'to' or both are single bytes. In Java we think this is unnecessary because
- * such speed gain as might be available that way is obtained by using the efficient one-byte
- * View object. Because Java cannot access memory bytes directly, unlike C, there is not so much
- * to be gained.
+ * when 'from' or 'to' or both are single bytes. This may also be worth doing in Java when the
+ * 'to' is a single byte. (The 'from' is turned into a Finder object which already makes a
+ * special case of single bytes.)
*/
/**
@@ -2546,7 +2191,7 @@
* @param maxcount maximum number of replacements to make
* @return the result as a new PyByteArray
*/
- private PyByteArray replace_substring(View from, View to, int maxcount) {
+ private PyByteArray replace_substring(PyBuffer from, PyBuffer to, int maxcount) {
// size>=1, len(from)>=1, len(to)>=1, maxcount>=1
// Initialise a Finder for the 'from' pattern
@@ -2558,8 +2203,8 @@
return new PyByteArray(this);
}
- int from_len = from.size();
- int to_len = to.size();
+ int from_len = from.getLen();
+ int to_len = to.getLen();
// Calculate length of result and check for too big
long result_len = size + count * (to_len - from_len);
@@ -2613,12 +2258,12 @@
/**
* Handle the interleaving case b'hello'.replace(b'', b'..') = b'..h..e..l..l..o..' At the call
- * site we are guaranteed: size>=1, to.size()>=1, maxcount>=1
+ * site we are guaranteed: size>=1, to.getLen()>=1, maxcount>=1
*
* @param to the replacement bytes as a byte-oriented view
* @param maxcount upper limit on number of insertions
*/
- private PyByteArray replace_interleave(View to, int maxcount) {
+ private PyByteArray replace_interleave(PyBuffer to, int maxcount) {
// Insert one at the beginning and one after every byte, or as many as allowed
int count = size + 1;
@@ -2626,7 +2271,7 @@
count = maxcount;
}
- int to_len = to.size();
+ int to_len = to.getLen();
// Calculate length of result and check for too big
long result_len = ((long)count) * to_len + size;
@@ -2671,7 +2316,7 @@
* @param maxcount maximum number of deletions to make
* @return the result as a new PyByteArray
*/
- private PyByteArray replace_delete_substring(View from, int maxcount) {
+ private PyByteArray replace_delete_substring(PyBuffer from, int maxcount) {
// len(self)>=1, len(from)>=1, to="", maxcount>=1
// Initialise a Finder for the 'from' pattern
@@ -2683,7 +2328,7 @@
return new PyByteArray(this);
}
- int from_len = from.size();
+ int from_len = from.getLen();
long result_len = size - (count * from_len);
assert (result_len >= 0);
@@ -2742,7 +2387,7 @@
* @param maxcount maximum number of replacements to make
* @return the result as a new PyByteArray
*/
- private PyByteArray replace_substring_in_place(View from, View to, int maxcount) {
+ private PyByteArray replace_substring_in_place(PyBuffer from, PyBuffer to, int maxcount) {
// len(self)>=1, len(from)==len(to)>=1, maxcount>=1
// Initialise a Finder for the 'from' pattern
@@ -2801,20 +2446,25 @@
*/
final synchronized PyTuple basebytes_rpartition(PyObject sep) {
- // Create a Finder for the separtor and set it on this byte array
- View separator = getViewOrError(sep);
- int n = checkForEmptySeparator(separator);
- Finder finder = new ReverseFinder(separator);
- finder.setText(this);
-
- // We only use it once, to find the first (from the right) occurrence
- int p = finder.nextIndex() - offset;
- if (p >= 0) {
- // Found at p, so we'll be returning ([0:p], [p:p+n], [p+n:])
- return partition(p, p + n);
- } else {
- // Not found: choose values leading to ('', '', [0:size])
- return partition(0, 0);
+ // View the separator as a byte array (or error if we can't)
+ PyBuffer separator = getViewOrError(sep);
+ try {
+ // Create a Finder for the separtor and set it on this byte array
+ int n = checkForEmptySeparator(separator);
+ Finder finder = new ReverseFinder(separator);
+ finder.setText(this);
+
+ // We only use it once, to find the first (from the right) occurrence
+ int p = finder.nextIndex() - offset;
+ if (p >= 0) {
+ // Found at p, so we'll be returning ([0:p], [p:p+n], [p+n:])
+ return partition(p, p + n);
+ } else {
+ // Not found: choose values leading to ('', '', [0:size])
+ return partition(0, 0);
+ }
+ } finally {
+ separator.release();
}
}
@@ -2822,7 +2472,6 @@
* Implementation of Python <code>rsplit()</code>, that returns a list of the words in the byte
* array, using whitespace as the delimiter. See {@link #rsplit(PyObject, int)}.
*
- * @param maxsplit maximum number of splits
* @return PyList of byte arrays that result from the split
*/
public PyList rsplit() {
@@ -2835,7 +2484,6 @@
* of the separator.
*
* @param sep bytes, or object viewable as bytes, defining the separator
- * @param maxsplit maximum number of splits
* @return PyList of byte arrays that result from the split
*/
public PyList rsplit(PyObject sep) {
@@ -2897,41 +2545,46 @@
final synchronized PyList basebytes_rsplit_explicit(PyObject sep, int maxsplit) {
// The separator may be presented as anything viewable as bytes
- View separator = getViewOrError(sep);
- int n = checkForEmptySeparator(separator);
-
- PyList result = new PyList();
-
- // Use the Finder class to search in the storage of this byte array
- Finder finder = new ReverseFinder(separator);
- finder.setText(this);
-
- int q = offset + size; // q points to "honorary separator"
- int p;
-
- // At this point storage[q-1] is the last byte of the rightmost unsplit word, or
- // q=offset if there aren't any. While we have some splits left to do ...
- while (q > offset && maxsplit-- != 0) {
- // Delimit the word whose last byte is storage[q-1]
- int r = q;
- // Skip p backwards over the word and the separator
- q = finder.nextIndex();
- if (q < 0) {
- p = offset;
- } else {
- p = q + n;
+ PyBuffer separator = getViewOrError(sep);
+
+ try {
+ int n = checkForEmptySeparator(separator);
+
+ PyList result = new PyList();
+
+ // Use the Finder class to search in the storage of this byte array
+ Finder finder = new ReverseFinder(separator);
+ finder.setText(this);
+
+ int q = offset + size; // q points to "honorary separator"
+ int p;
+
+ // At this point storage[q-1] is the last byte of the rightmost unsplit word, or
+ // q=offset if there aren't any. While we have some splits left to do ...
+ while (q > offset && maxsplit-- != 0) {
+ // Delimit the word whose last byte is storage[q-1]
+ int r = q;
+ // Skip p backwards over the word and the separator
+ q = finder.nextIndex();
+ if (q < 0) {
+ p = offset;
+ } else {
+ p = q + n;
+ }
+ // storage[p] is the first byte of the word.
+ BaseBytes word = getslice(p - offset, r - offset);
+ result.add(0, word);
}
- // storage[p] is the first byte of the word.
- BaseBytes word = getslice(p - offset, r - offset);
- result.add(0, word);
+
+ // Prepend the remaining unsplit text if any
+ if (q >= offset) {
+ BaseBytes word = getslice(0, q - offset);
+ result.add(0, word);
+ }
+ return result;
+ } finally {
+ separator.release();
}
-
- // Prepend the remaining unsplit text if any
- if (q >= offset) {
- BaseBytes word = getslice(0, q - offset);
- result.add(0, word);
- }
- return result;
}
/**
@@ -3056,7 +2709,7 @@
* @return PyList of byte arrays that result from the split
*/
final PyList basebytes_split(PyObject sep, int maxsplit) {
- if (sep == null || sep==Py.None) {
+ if (sep == null || sep == Py.None) {
return basebytes_split_whitespace(maxsplit);
} else {
return basebytes_split_explicit(sep, maxsplit);
@@ -3076,32 +2729,36 @@
final synchronized PyList basebytes_split_explicit(PyObject sep, int maxsplit) {
// The separator may be presented as anything viewable as bytes
- View separator = getViewOrError(sep);
- checkForEmptySeparator(separator);
-
- PyList result = new PyList();
-
- // Use the Finder class to search in the storage of this byte array
- Finder finder = new Finder(separator);
- finder.setText(this);
-
- // Look for the first separator
- int p = finder.currIndex(); // = offset
- int q = finder.nextIndex(); // First separator (or <0 if not found)
-
- // Note: bytearray().split(' ') == [bytearray(b'')]
-
- // While we found a separator, and we have some splits left (if maxsplit started>=0)
- while (q >= 0 && maxsplit-- != 0) {
- // Note the Finder works in terms of indexes into this.storage
- result.append(getslice(p - offset, q - offset));
- p = finder.currIndex(); // Start of unsplit text
- q = finder.nextIndex(); // Next separator (or <0 if not found)
+ PyBuffer separator = getViewOrError(sep);
+ try {
+ checkForEmptySeparator(separator);
+
+ PyList result = new PyList();
+
+ // Use the Finder class to search in the storage of this byte array
+ Finder finder = new Finder(separator);
+ finder.setText(this);
+
+ // Look for the first separator
+ int p = finder.currIndex(); // = offset
+ int q = finder.nextIndex(); // First separator (or <0 if not found)
+
+ // Note: bytearray().split(' ') == [bytearray(b'')]
+
+ // While we found a separator, and we have some splits left (if maxsplit started>=0)
+ while (q >= 0 && maxsplit-- != 0) {
+ // Note the Finder works in terms of indexes into this.storage
+ result.append(getslice(p - offset, q - offset));
+ p = finder.currIndex(); // Start of unsplit text
+ q = finder.nextIndex(); // Next separator (or <0 if not found)
+ }
+
+ // Append the remaining unsplit text
+ result.append(getslice(p - offset, size));
+ return result;
+ } finally {
+ separator.release();
}
-
- // Append the remaining unsplit text
- result.append(getslice(p - offset, size));
- return result;
}
/**
@@ -3149,7 +2806,7 @@
}
// Append the remaining unsplit text if any
- if (p<limit) {
+ if (p < limit) {
result.append(getslice(p - offset, size));
}
return result;
@@ -3346,7 +3003,7 @@
final BaseBytes basebytes_expandtabs(int tabsize) {
// We could only work out the true size by doing the work twice,
// so make a guess and let the Builder re-size if it's not enough.
- int estimatedSize = size + size/8;
+ int estimatedSize = size + size / 8;
Builder builder = getBuilder(estimatedSize);
int carriagePosition = 0;
@@ -3577,9 +3234,9 @@
* pont codes and is consistent with Java's {@link Character#isUpperCase(char)} and
* {@link Character#isLowerCase(char)}.
*
- * @return true if the string is a titlecased string and there is at least one cased byte, for example
- * uppercase characters may only follow uncased bytes and lowercase characters only
- * cased ones. Return false otherwise.
+ * @return true if the string is a titlecased string and there is at least one cased byte, for
+ * example uppercase characters may only follow uncased bytes and lowercase characters
+ * only cased ones. Return false otherwise.
*/
public boolean istitle() {
return basebytes_istitle();
@@ -3588,9 +3245,9 @@
/**
* Ready-to-expose implementation of Python <code>istitle()</code>.
*
- * @return true if the string is a titlecased string and there is at least one cased byte, for example
- * uppercase characters may only follow uncased bytes and lowercase characters only
- * cased ones. Return false otherwise.
+ * @return true if the string is a titlecased string and there is at least one cased byte, for
+ * example uppercase characters may only follow uncased bytes and lowercase characters
+ * only cased ones. Return false otherwise.
*/
final boolean basebytes_istitle() {
@@ -3941,9 +3598,8 @@
* @param c curren (maybe unprintable) character
*/
private static final void appendHexEscape(StringBuilder buf, int c) {
- buf.append("\\x")
- .append(Character.forDigit((c & 0xf0) >> 4, 16))
- .append(Character.forDigit(c & 0xf, 16));
+ buf.append("\\x").append(Character.forDigit((c & 0xf0) >> 4, 16))
+ .append(Character.forDigit(c & 0xf, 16));
}
/**
@@ -4009,7 +3665,7 @@
* ============================================================================================
*/
- /**
+ /**
* Access to the bytearray (or bytes) as a {@link java.util.List}. The List interface supplied
* by BaseBytes delegates to this object.
*/
@@ -4395,17 +4051,17 @@
void append(BaseBytes b, int start, int end) {
int n = end - start;
makeRoomFor(n);
- System.arraycopy(b.storage, b.offset+start, storage, size, n);
+ System.arraycopy(b.storage, b.offset + start, storage, size, n);
size += n;
}
/**
- * Append the contents of the given {@link View}.
+ * Append the contents of the given {@link PyBuffer}.
*
* @param b
*/
- void append(View v) {
- int n = v.size();
+ void append(PyBuffer v) {
+ int n = v.getLen();
makeRoomFor(n);
v.copyTo(storage, size);
size += n;
diff --git a/src/org/python/core/BufferProtocol.java b/src/org/python/core/BufferProtocol.java
--- a/src/org/python/core/BufferProtocol.java
+++ b/src/org/python/core/BufferProtocol.java
@@ -6,12 +6,15 @@
public interface BufferProtocol {
/**
- * Method by which the consumer requests the buffer from the exporter. The consumer
- * provides information on its intended method of navigation and the optional
- * features the buffer object must provide.
+ * Method by which the consumer requests the buffer from the exporter. The consumer provides
+ * information on its intended method of navigation and the features the buffer object is asked
+ * (or assumed) to provide. Each consumer requesting a buffer in this way, when it has finished
+ * using it, should make a corresponding call to {@link PyBuffer#release()} on the buffer it
+ * obtained, since some objects alter their behaviour while buffers are exported.
*
- * @param flags specification of options and the navigational capabilities of the consumer
+ * @param flags specifying features demanded and the navigational capabilities of the consumer
* @return exported buffer
+ * @throws PyException (BufferError) when expectations do not correspond with the buffer
*/
- PyBuffer getBuffer(int flags);
+ PyBuffer getBuffer(int flags) throws PyException;
}
diff --git a/src/org/python/core/PyBUF.java b/src/org/python/core/PyBUF.java
--- a/src/org/python/core/PyBUF.java
+++ b/src/org/python/core/PyBUF.java
@@ -3,7 +3,7 @@
/**
* This interface provides a base for the key interface of the buffer API, {@link PyBuffer},
* including symbolic constants used by the consumer of a <code>PyBuffer</code> to specify its
- * requirements. The Jython buffer API emulates the CPython buffer API closely.
+ * requirements and assumptions. The Jython buffer API emulates the CPython buffer API.
* <ul>
* <li>There are two reasons for separating parts of <code>PyBuffer</code> into this interface: The
* constants defined in CPython have the names <code>PyBUF_SIMPLE</code>,
@@ -11,12 +11,13 @@
* {@link PyBUF#SIMPLE}, {@link PyBUF#WRITABLE}, etc. so source code looks similar.</li>
* <li>It is not so easy in Java as it is in C to treat a <code>byte</code> array as storing
* anything other than <code>byte</code>, and we prepare for the possibility of buffers with a
- * series of different primitive types by defining here, those methods that would be in common
+ * series of different primitive types by defining here those methods that would be in common
* between <code>(Byte)Buffer</code> and an assumed future <code>FloatBuffer</code> or
* <code>TypedBuffer<T></code>. (Compare <code>java.nio.Buffer</code>.)</li>
* </ul>
* Except for other interfaces, it is unlikely any classes would implement <code>PyBUF</code>
- * directly.
+ * directly. Users of the Jython buffer API can mostly overlook the distinction and just use
+ * <code>PyBuffer</code>.
*/
public interface PyBUF {
@@ -29,7 +30,8 @@
/**
* The number of dimensions to the buffer. This number is the length of the <code>shape</code>
- * array.
+ * array. The actual storage may be a linear array, but this is the number of dimensions in the
+ * interpretation that the exporting object gives the data.
*
* @return number of dimensions
*/
@@ -41,7 +43,8 @@
* is the amount of buffer content addressed by one index or set of indices. In the simplest
* case an item is a single unit (byte), and there is one dimension. In complex cases, the array
* is multi-dimensional, and the item at each location is multi-unit (multi-byte). The consumer
- * must not modify this array.
+ * must not modify this array. A valid <code>shape</code> array is always returned (difference
+ * from CPython).
*
* @return the dimensions of the buffer as an array
*/
@@ -56,40 +59,35 @@
/**
* The total number of units (bytes) stored, which will be the product of the elements of the
- * shape, and the item size.
+ * <code>shape</code> array, and the item size.
*
* @return the total number of units stored.
*/
int getLen();
/**
- * A buffer is (usually) coupled to the internal state of an exporting Python object, and that
- * object may have to restrict its behaviour while the buffer exists. The consumer must
- * therefore say when it has finished.
- */
- void release();
-
- /**
- * The "strides" array gives the distance in the storage array between adjacent items (in each
- * dimension). If the rawest parts of the buffer API, the consumer of the buffer is able to
- * navigate the exported storage. The "strides" array is part of the support for interpreting
- * the buffer as an n-dimensional array of items. In the one-dimensional case, the "strides"
- * array is In more dimensions, it provides the coefficients of the "addressing polynomial".
- * (More on this in the CPython documentation.) The consumer must not modify this array.
+ * The <code>strides</code> array gives the distance in the storage array between adjacent items
+ * (in each dimension). In the rawest parts of the buffer API, the consumer of the buffer is
+ * able to navigate the exported storage. The "strides" array is part of the support for
+ * interpreting the buffer as an n-dimensional array of items. It provides the coefficients of
+ * the "addressing polynomial". (More on this in the CPython documentation.) The consumer must
+ * not modify this array. A valid <code>strides</code> array is always returned (difference from
+ * CPython).
*
* @return the distance in the storage array between adjacent items (in each dimension)
*/
int[] getStrides();
/**
- * The "suboffsets" array is a further part of the support for interpreting the buffer as an
- * n-dimensional array of items, where the array potentially uses indirect addressing (like a
- * real Java array of arrays, in fact). This is only applicable when there are more than 1
- * dimension and works in conjunction with the <code>strides</code> array. (More on this in the
- * CPython documentation.) When used, <code>suboffsets[k]</code> is an integer index, bit a byte
- * offset as in CPython. The consumer must not modify this array.
+ * The <code>suboffsets</code> array is a further part of the support for interpreting the
+ * buffer as an n-dimensional array of items, where the array potentially uses indirect
+ * addressing (like a real Java array of arrays, in fact). This is only applicable when there
+ * are more than 1 dimension and works in conjunction with the <code>strides</code> array. (More
+ * on this in the CPython documentation.) When used, <code>suboffsets[k]</code> is an integer
+ * index, bit a byte offset as in CPython. The consumer must not modify this array. When not
+ * needed for navigation <code>null</code> is returned (as in CPython).
*
- * @return
+ * @return suboffsets array or null in not necessary for navigation
*/
int[] getSuboffsets();
@@ -102,10 +100,10 @@
*/
boolean isContiguous(char order);
- /* Constants taken from CPython object.h in v3.3.0a */
+ /* Constants taken from CPython object.h in v3.3 */
/**
- * The maximum allowed number of dimensions (NumPy restriction?).
+ * The maximum allowed number of dimensions (CPython restriction).
*/
static final int MAX_NDIM = 64;
/**
@@ -123,53 +121,58 @@
static final int SIMPLE = 0;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it requires {@link PyBuffer#getFormat()} to return the type of the unit (rather
- * than return <code>null</code>).
+ * specify that it requires {@link PyBuffer#getFormat()} to return a <code>String</code>
+ * indicating the type of the unit. This exists for compatibility with CPython, as Jython as the
+ * format is always provided by <code>getFormat()</code>.
*/
- // I don't understand why we need this, or why format MUST be null of this is not set.
static final int FORMAT = 0x0004;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it it is prepared to navigate the buffer as multi-dimensional.
- * <code>getBuffer</code> will raise an exception if consumer does not specify the flag but the
- * exporter's buffer cannot be navigated without taking into account its multiple dimensions.
+ * specify that it is prepared to navigate the buffer as multi-dimensional using the
+ * <code>shape</code> array. <code>getBuffer</code> will raise an exception if consumer does not
+ * specify the flag but the exporter's buffer cannot be navigated without taking into account
+ * its multiple dimensions.
*/
- static final int ND = 0x0008 | SIMPLE; // Differs from CPython by or'ing in SIMPLE
+ static final int ND = 0x0008;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it it expects to use the "strides" array. <code>getBuffer</code> will raise an
- * exception if consumer does not specify the flag but the exporter's buffer cannot be navigated
- * without using the "strides" array.
+ * specify that it expects to use the <code>strides</code> array. <code>getBuffer</code> will
+ * raise an exception if consumer does not specify the flag but the exporter's buffer cannot be
+ * navigated without using the <code>strides</code> array.
*/
static final int STRIDES = 0x0010 | ND;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it will assume C-order organisation of the units. <code>getBuffer</code> will raise an
- * exception if the exporter's buffer is not C-ordered. <code>C_CONTIGUOUS</code> implies
- * <code>STRIDES</code>.
+ * specify that it will assume C-order organisation of the units. <code>getBuffer</code> will
+ * raise an exception if the exporter's buffer is not C-ordered. <code>C_CONTIGUOUS</code>
+ * implies <code>STRIDES</code>.
*/
+ // It is possible this should have been (0x20|ND) expressing the idea that C-order addressing
+ // will be assumed *instead of* using a strides array.
static final int C_CONTIGUOUS = 0x0020 | STRIDES;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it will assume Fortran-order organisation of the units. <code>getBuffer</code> will raise an
- * exception if the exporter's buffer is not Fortran-ordered. <code>F_CONTIGUOUS</code> implies
- * <code>STRIDES</code>.
+ * specify that it will assume Fortran-order organisation of the units. <code>getBuffer</code>
+ * will raise an exception if the exporter's buffer is not Fortran-ordered.
+ * <code>F_CONTIGUOUS</code> implies <code>STRIDES</code>.
*/
static final int F_CONTIGUOUS = 0x0040 | STRIDES;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it
+ * specify that it will assume a contiguous organisation of the units, but will enquire which
+ * organisation it actually is.
*
* getBuffer will raise an exception if the exporter's buffer is not contiguous.
* <code>ANY_CONTIGUOUS</code> implies <code>STRIDES</code>.
*/
+ // Further CPython strangeness since it uses the strides array to answer the enquiry.
static final int ANY_CONTIGUOUS = 0x0080 | STRIDES;
/**
* A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it understands the "suboffsets" array. <code>getBuffer</code> will raise an
- * exception if consumer does not specify the flag but the exporter's buffer cannot be navigated
- * without understanding the "suboffsets" array. <code>INDIRECT</code> implies
- * <code>STRIDES</code>.
+ * specify that it understands the <code>suboffsets</code> array. <code>getBuffer</code> will
+ * raise an exception if consumer does not specify the flag but the exporter's buffer cannot be
+ * navigated without understanding the <code>suboffsets</code> array. <code>INDIRECT</code>
+ * implies <code>STRIDES</code>.
*/
static final int INDIRECT = 0x0100 | STRIDES;
/**
@@ -197,42 +200,50 @@
*/
static final int RECORDS_RO = STRIDES | FORMAT;
/**
- * Equivalent to <code>(INDIRECT | WRITABLE | FORMAT)</code>
+ * Equivalent to <code>(INDIRECT | WRITABLE | FORMAT)</code>. Also use this as the request flags
+ * if you plan only to use the fully-encapsulated API (<code>byteAt</code>, <code>storeAt</code>
+ * , <code>copyTo</code>, <code>copyFrom</code>, etc.), without ever calling
+ * {@link PyBuffer#getBuf()}.
*/
static final int FULL = INDIRECT | WRITABLE | FORMAT;
/**
- * Equivalent to <code>(INDIRECT | FORMAT)</code>
+ * Equivalent to <code>(INDIRECT | FORMAT)</code> Also use this as the request flags if you plan
+ * only to use the fully-encapsulated API (<code>byteAt</code>, <code>copyTo</code>, etc.),
+ * without ever calling {@link PyBuffer#getBuf()}.
*/
static final int FULL_RO = INDIRECT | FORMAT;
/* Constants for readability, not standard for CPython */
/**
- * Field mask, use as in <code>if ((capabilityFlags&ORGANISATION) == STRIDES) ...</code>.
+ * Field mask, use as in <code>if ((flags&NAVIGATION) == STRIDES) ...</code>. The importance of
+ * the subset of flags defined by this mask is not so much in their "navigational" character as
+ * in the way they are treated in a buffer request.
+ * <p>
+ * The <code>NAVIGATION</code> set are used to specify which navigation arrays the consumer will
+ * use, and therefore the consumer must ask for all those necessary to use the buffer
+ * successfully (which is a function of the buffer's actual type). Asking for extra ones is not
+ * an error, since all are supplied (in Jython): asking for too few is an error.
+ * <p>
+ * Flags outside the <code>NAVIGATION</code> set, work the other way round. Asking for one the
+ * buffer cannot match is an error: not asking for a feature the buffer does not have is an
+ * error.
*/
- static final int ORGANISATION = SIMPLE | ND | STRIDES | INDIRECT;
+ static final int NAVIGATION = SIMPLE | ND | STRIDES | INDIRECT;
/**
- * Field mask, use as in if <code>((capabilityFlags&ORGANIZATION) == STRIDES) ...</code>.
- *
- * @see #ORGANISATION
- */
- static final int ORGANIZATION = ORGANISATION;
- /**
- * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it will assume C-order organisation of the units, irrespective of whether
- * the strides array is to be provided. <code>getBuffer</code> will raise an
- * exception if the exporter's buffer is not C-ordered. <code>C_CONTIGUOUS = IS_C_CONTIGUOUS | STRIDES</code>.
+ * A constant used by the exporter in processing {@link BufferProtocol#getBuffer(int)} to check
+ * for assumed C-order organisation of the units.
+ * <code>C_CONTIGUOUS = IS_C_CONTIGUOUS | STRIDES</code>.
*/
static final int IS_C_CONTIGUOUS = C_CONTIGUOUS & ~STRIDES;
/**
- * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
- * specify that it will assume Fortran-order organisation of the units, irrespective of whether
- * the strides array is to be provided. <code>getBuffer</code> will raise an
- * exception if the exporter's buffer is not Fortran-ordered. <code>F_CONTIGUOUS = IS_F_CONTIGUOUS | STRIDES</code>.
+ * A constant used by the exporter in processing {@link BufferProtocol#getBuffer(int)} to check
+ * for assumed C-order Fortran-order organisation of the units.
+ * <code>F_CONTIGUOUS = IS_F_CONTIGUOUS | STRIDES</code>.
*/
static final int IS_F_CONTIGUOUS = F_CONTIGUOUS & ~STRIDES;
/**
- * Field mask, use as in <code>if (capabilityFlags&CONTIGUITY== ... ) ...</code>.
+ * Field mask, use as in <code>if ((flags&CONTIGUITY)== ... ) ...</code>.
*/
static final int CONTIGUITY = (C_CONTIGUOUS | F_CONTIGUOUS | ANY_CONTIGUOUS) & ~STRIDES;
}
\ No newline at end of file
diff --git a/src/org/python/core/PyBuffer.java b/src/org/python/core/PyBuffer.java
--- a/src/org/python/core/PyBuffer.java
+++ b/src/org/python/core/PyBuffer.java
@@ -5,7 +5,7 @@
* the counterpart of the CPython <code>Py_buffer</code> struct. Several concrete types implement
* this interface in order to provide tailored support for different storage organisations.
*/
-public interface PyBuffer extends PyBUF {
+public interface PyBuffer extends PyBUF, BufferProtocol {
/*
* The different behaviours required as the actual structure of the buffer changes (from one
@@ -14,9 +14,10 @@
* array must be used, or the array is C or F contiguous, since they know the answer to these
* questions already, and can just get on with the request in their own way.
*
- * The issue of consumer requests is different: the strides array will be present if the
- * consumer asked for it, yet the methods of the buffer implementation do not have to use it
- * (and won't).
+ * The issue of consumer requests via getBuffer(int) is greatly simplified relative to CPython
+ * by the choice always to supply a full description of the buffer organisation, whether the
+ * consumer asked for it in the flags or not. Of course, implementations don't actually have to
+ * create (for example) a strides array until getStrides() is called.
*/
// Informational methods inherited from PyBUF
@@ -28,9 +29,9 @@
/**
* Return the byte indexed from a one-dimensional buffer with item size one. This is part of the
- * fully-encapsulated API: the exporter takes care of navigating the structure of the buffer.
- * Results are undefined where the number of dimensions is not one or if
- * <code>itemsize>1</code>.
+ * fully-encapsulated API: the buffer implementation exported takes care of navigating the
+ * structure of the buffer. Results are undefined where the number of dimensions is not one or
+ * if <code>itemsize>1</code>.
*
* @param index to retrieve from
* @return the item at index, which is a byte
@@ -50,9 +51,9 @@
/**
* Store the given byte at the indexed location in of a one-dimensional buffer with item size
- * one. This is part of the fully-encapsulated API: the exporter takes care of navigating the
- * structure of the buffer. Results are undefined where the number of dimensions is not one or
- * if <code>itemsize>1</code>.
+ * one. This is part of the fully-encapsulated API: the buffer implementation exported takes
+ * care of navigating the structure of the buffer. Results are undefined where the number of
+ * dimensions is not one or if <code>itemsize>1</code>.
*
* @param value to store
* @param index to location
@@ -63,9 +64,9 @@
//
/**
* Return the byte indexed from an N-dimensional buffer with item size one. This is part of the
- * fully-encapsulated API: the exporter takes care of navigating the structure of the buffer.
- * The indices must be correct in length and value for the array shape. Results are undefined
- * where <code>itemsize>1</code>.
+ * fully-encapsulated API: the buffer implementation exported takes care of navigating the
+ * structure of the buffer. The indices must be correct in number and range for the array shape.
+ * Results are undefined where <code>itemsize>1</code>.
*
* @param indices specifying location to retrieve from
* @return the item at location, which is a byte
@@ -74,11 +75,11 @@
/**
* Return the unsigned byte value indexed from an N-dimensional buffer with item size one. This
- * is part of the fully-encapsulated API: the exporter takes care of navigating the structure of
- * the buffer. The indices must be correct in length and value for the array shape. Results are
- * undefined where <code>itemsize>1</code>.
+ * is part of the fully-encapsulated API: the buffer implementation exported takes care of
+ * navigating the structure of the buffer. The indices must be correct in number and range for
+ * the array shape. Results are undefined where <code>itemsize>1</code>.
*
- * @param index to retrieve from
+ * @param indices specifying location to retrieve from
* @return the item at location, treated as an unsigned byte, <code>=0xff & byteAt(index)</code>
*/
int intAt(int... indices) throws IndexOutOfBoundsException;
@@ -86,7 +87,7 @@
/**
* Store the given byte at the indexed location in of an N-dimensional buffer with item size
* one. This is part of the fully-encapsulated API: the exporter takes care of navigating the
- * structure of the buffer. The indices must be correct in length and value for the array shape.
+ * structure of the buffer. The indices must be correct in number and range for the array shape.
* Results are undefined where <code>itemsize>1</code>.
*
* @param value to store
@@ -98,49 +99,138 @@
//
/**
* Copy the contents of the buffer to the destination byte array. The number of bytes will be
- * that returned by {@link #getLen()}, and the order is the natural ordering according to the
- * contiguity type.
+ * that returned by {@link #getLen()}, and the order is the storage order in the exporter.
+ * (Note: Correct ordering for multidimensional arrays, including those with indirection needs
+ * further study.)
*
* @param dest destination byte array
* @param destPos index in the destination array of the byte [0]
* @throws IndexOutOfBoundsException if the destination cannot hold it
*/
- void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException;
+ void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException, PyException;
/**
* Copy a simple slice of the buffer to the destination byte array, defined by a starting index
* and length in the source buffer. This may validly be done only for a one-dimensional buffer,
- * as the meaning of the starting index is otherwise not defined.
+ * as the meaning of the starting index is otherwise not defined. The length (like the source
+ * index) is in source buffer <b>items</b>: <code>length*itemsize</code> bytes will be occupied
+ * in the destination.
*
* @param srcIndex starting index in the source buffer
* @param dest destination byte array
- * @param destPos index in the destination array of the byte [0,...]
- * @param length number of bytes to copy
+ * @param destPos index in the destination array of the item [0,...]
+ * @param length number of items to copy
* @throws IndexOutOfBoundsException if access out of bounds in source or destination
*/
void copyTo(int srcIndex, byte[] dest, int destPos, int length) // mimic arraycopy args
- throws IndexOutOfBoundsException;
+ throws IndexOutOfBoundsException, PyException;
/**
* Copy bytes from a slice of a (Java) byte array into the buffer. This may validly be done only
- * for a one-dimensional buffer, as the meaning of the starting index is otherwise not defined.
+ * for a one-dimensional buffer, as the meaning of the starting index is not otherwise defined.
+ * The length (like the destination index) is in buffer <b>items</b>:
+ * <code>length*itemsize</code> bytes will be read from the source.
*
* @param src source byte array
* @param srcPos location in source of first byte to copy
* @param destIndex starting index in the destination (i.e. <code>this</code>)
* @param length number of bytes to copy in
* @throws IndexOutOfBoundsException if access out of bounds in source or destination
- * @throws PyException (BufferError) if read-only buffer
+ * @throws PyException (TypeError) if read-only buffer
*/
void copyFrom(byte[] src, int srcPos, int destIndex, int length) // mimic arraycopy args
throws IndexOutOfBoundsException, PyException;
+ /**
+ * Copy the whole of another PyBuffer into this buffer. This may validly be done only for
+ * buffers that are consistent in their dimensions. When it is necessary to copy partial
+ * buffers, this may be achieved using a buffer slice on the source or destination.
+ *
+ * @param src source buffer
+ * @throws IndexOutOfBoundsException if access out of bounds in source or destination
+ * @throws PyException (TypeError) if read-only buffer
+ */
+ void copyFrom(PyBuffer src) throws IndexOutOfBoundsException, PyException;
+
// Bulk access in n-dimensions may be added here if desired semantics can be settled
//
- // Buffer management inherited from PyBUF
+ // Buffer management
//
- // void release();
+ /**
+ * {@inheritDoc}
+ * <p>
+ * When a <code>PyBuffer</code> is the target, the same checks are carried out on the consumer
+ * flags, and a return will normally be a reference to that buffer. A Jython
+ * <code>PyBuffer</code> keeps count of these re-exports in order to match them with the number
+ * of calls to {@link #release()}. When the last matching release() arrives it is considered
+ * "final", and release actions may then take place on the exporting object. After the final
+ * release of a buffer, a call to <code>getBuffer</code> should raise an exception.
+ */
+ @Override
+ PyBuffer getBuffer(int flags) throws PyException;
+
+ /**
+ * A buffer is (usually) a view onto to the internal state of an exporting object, and that
+ * object may have to restrict its behaviour while the buffer exists. The consumer must
+ * therefore say when it has finished with the buffer if the exporting object is to be released
+ * from this constraint. Each consumer that obtains a reference to a buffer by means of a call
+ * to {@link BufferProtocol#getBuffer(int)} or {@link PyBuffer#getBuffer(int)} should make a
+ * matching call to {@link #release()}. The consumer may be sharing the <code>PyBuffer</code>
+ * with other consumers and the buffer uses the pairing of <code>getBuffer</code> and
+ * <code>release</code> to manage the lock on behalf of the exporter. It is an error to make
+ * more than one call to <code>release</code> for a single call to <code>getBuffer</code>.
+ */
+ void release();
+
+ /**
+ * True only if the buffer has been released with (the required number of calls to)
+ * {@link #release()} or some equivalent operation. The consumer may be sharing the reference
+ * with other consumers and the buffer only achieves the released state when all consumers who
+ * called <code>getBuffer</code> have called <code>release</code>.
+ */
+ boolean isReleased();
+
+ /**
+ * Equivalent to {@link #getBufferSlice(int, int, int, int)} with stride 1.
+ *
+ * @param flags specifying features demanded and the navigational capabilities of the consumer
+ * @param start index in the current buffer
+ * @param length number of items in the required slice
+ * @return a buffer representing the slice
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length);
+
+ /**
+ * Get a <code>PyBuffer</code> that represents a slice of the current one described in terms of
+ * a start index, number of items to include in the slice, and the stride in the current buffer.
+ * A consumer that obtains a <code>PyBuffer</code> with <code>getBufferSlice</code> must release
+ * it with {@link PyBuffer#release} just as if it had been obtained with
+ * {@link PyBuffer#getBuffer(int)}
+ * <p>
+ * Suppose that <i>x(i)</i> denotes the <i>i</i>th element of the current buffer, that is, the
+ * byte retrieved by <code>this.byteAt(i)</code> or the unit indicated by
+ * <code>this.getPointer(i)</code>. A request for a slice where <code>start</code> <i>= s</i>,
+ * <code>length</code> <i>= N</i> and <code>stride</code> <i>= m</i>, results in a buffer
+ * <i>y</i> such that <i>y(k) = x(s+km)</i> where <i>k=0..(N-1)</i>. In Python terms, this is
+ * the slice <i>x[s : s+(N-1)m+1 : m]</i> (if m>0) or the slice <i>x[s : s+(N-1)m-1 : m]</i>
+ * (if m<0). Implementations should check that this range is entirely within the current
+ * buffer.
+ * <p>
+ * In a simple buffer backed by a contiguous byte array, the result is a strided PyBuffer on the
+ * same storage but where the offset is adjusted by <i>s</i> and the stride is as supplied. If
+ * the current buffer is already strided and/or has an item size larger than single bytes, the
+ * new offset, size and stride will be translated from the arguments given, through this
+ * buffer's stride and item size. The consumer always expresses <code>start</code> and
+ * <code>strides</code> in terms of the abstract view of this buffer.
+ *
+ * @param flags specifying features demanded and the navigational capabilities of the consumer
+ * @param start index in the current buffer
+ * @param length number of items in the required slice
+ * @param stride index-distance in the current buffer between consecutive items in the slice
+ * @return a buffer representing the slice
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride);
// Direct access to actual storage
//
@@ -150,7 +240,6 @@
* where <code>obj</code> has type <code>BufferProtocol</code>:
*
* <pre>
- *
* PyBuffer a = obj.getBuffer();
* int itemsize = a.getItemsize();
* BufferPointer b = a.getBuf();
@@ -163,7 +252,8 @@
* <p>
* If the buffer is multidimensional or non-contiguous, <code>b.storage[b.offset]</code> is
* still the (first byte of) the item at index [0] or [0,...,0]. However, it is necessary to
- * navigate <code>b</code> using the shape, strides and sub-offsets provided by the API.
+ * navigate <code>b</code> using the <code>shape</code>, <code>strides</code> and maybe
+ * <code>suboffsets</code> provided by the API.
*
* @return structure defining the byte[] slice that is the shared data
*/
@@ -196,12 +286,13 @@
/**
* Return a structure describing the slice of a byte array that holds a single item from the
- * data being exported to the consumer, in the case that array may be multi-dimensional. For an
+ * data being exported to the consumer, in the case that array may be multi-dimensional. For a
* 3-dimensional contiguous buffer, assuming the following client code where <code>obj</code>
* has type <code>BufferProtocol</code>:
*
* <pre>
- * int i, j, k = ... ;
+ * int i, j, k;
+ * // ... calculation that assigns i, j, k
* PyBuffer a = obj.getBuffer();
* int itemsize = a.getItemsize();
* BufferPointer b = a.getPointer(i,j,k);
@@ -233,10 +324,12 @@
// Interpretation of bytes as items
/**
* A format string in the language of Python structs describing how the bytes of each item
- * should be interpreted (or null if {@link PyBUF#FORMAT} was not part of the consumer's flags).
+ * should be interpreted. Irrespective of the {@link PyBUF#FORMAT} bit in the consumer's call to
+ * <code>getBuffer</code>, a valid <code>format</code> string is always returned (difference
+ * from CPython).
* <p>
- * This is provided for compatibility with CPython. Jython only implements "B" so far, and it is
- * debatable whether anything fancier than "<n>B" can be supported in Java.
+ * Jython only implements "B" so far, and it is debatable whether anything fancier than
+ * "<n>B" can be supported in Java.
*
* @return the format string
*/
diff --git a/src/org/python/core/PyByteArray.java b/src/org/python/core/PyByteArray.java
--- a/src/org/python/core/PyByteArray.java
+++ b/src/org/python/core/PyByteArray.java
@@ -1,8 +1,10 @@
package org.python.core;
+import java.lang.ref.WeakReference;
import java.util.Arrays;
-import org.python.core.buffer.SimpleBuffer;
+import org.python.core.buffer.BaseBuffer;
+import org.python.core.buffer.SimpleWritableBuffer;
import org.python.expose.ExposedClassMethod;
import org.python.expose.ExposedMethod;
import org.python.expose.ExposedNew;
@@ -76,11 +78,11 @@
/**
* Create a new array filled exactly by a copy of the contents of the source, which is a
- * byte-oriented view.
+ * byte-oriented {@link PyBuffer}.
*
* @param value source of the bytes (and size)
*/
- PyByteArray(View value) {
+ PyByteArray(PyBuffer value) {
super(TYPE);
init(value);
}
@@ -206,46 +208,82 @@
*/
/**
+ * Hold weakly a reference to a PyBuffer export not yet released, used to prevent untimely
+ * resizing.
+ */
+ private WeakReference<BaseBuffer> export;
+
+ /**
* {@inheritDoc}
* <p>
* The {@link PyBuffer} returned from this method is a one-dimensional array of single byte
- * items, that allows modification of the object state but <b>prohibits resizing</b> the byte array.
- * This prohibition is not only on the consumer of the view but extends to any other operations,
- * such as any kind or insertion or deletion.
+ * items that allows modification of the object state. The existence of this export <b>prohibits
+ * resizing</b> the byte array. This prohibition is not only on the consumer of the view but
+ * extends to any other operations, such as any kind or insertion or deletion.
*/
@Override
public synchronized PyBuffer getBuffer(int flags) {
- exportCount++;
- return new SimpleBuffer(this, new BufferPointer(storage, offset, size), flags) {
- @Override
- public void releaseAction() {
- // synchronise on the same object as getBuffer()
- synchronized (obj) {
- exportCount--;
- }
- }
- };
+ // If we have already exported a buffer it may still be available for re-use
+ BaseBuffer pybuf = getExistingBuffer(flags);
+
+ if (pybuf == null) {
+ // No existing export we can re-use: create a new one
+ pybuf = new SimpleWritableBuffer(flags, storage, offset, size);
+ // Hold a reference for possible re-use
+ export = new WeakReference<BaseBuffer>(pybuf);
+ }
+
+ return pybuf;
}
/**
- * Test to see if the byte array may be resized and raise a BufferError if not.
+ * Try to re-use an existing exported buffer, or return <code>null</code> if we can't.
+ *
+ * @throws PyException (BufferError) if the the flags are incompatible with the buffer
+ */
+ private BaseBuffer getExistingBuffer(int flags) throws PyException {
+ BaseBuffer pybuf = null;
+ if (export != null) {
+ // A buffer was exported at some time.
+ pybuf = export.get();
+ if (pybuf != null) {
+ /*
+ * We do not test for pybuf.isReleased() as, if any operation had taken place that
+ * invalidated the buffer, resizeCheck() would have set export=null. The exported
+ * buffer (navigation, buf member, etc.) remains valid through any operation that
+ * does not need a resizeCheck.
+ */
+ pybuf = pybuf.getBufferAgain(flags);
+ }
+ }
+ return pybuf;
+ }
+
+ /**
+ * Test to see if the byte array may be resized and raise a BufferError if not. This must be
+ * called by the implementation of any append or insert that changes the number of bytes in the
+ * array.
*
* @throws PyException (BufferError) if there are buffer exports preventing a resize
*/
protected void resizeCheck() throws PyException {
- // XXX Quite likely this is not called in all the places it should be
- if (exportCount!=0) {
- throw Py.BufferError("Existing exports of data: object cannot be re-sized");
+ if (export != null) {
+ // A buffer was exported at some time and we have not explicitly discarded it.
+ PyBuffer pybuf = export.get();
+ if (pybuf != null && !pybuf.isReleased()) {
+ // A consumer still has the exported buffer
+ throw Py.BufferError("Existing exports of data: object cannot be re-sized");
+ } else {
+ /*
+ * Either the reference has expired or all consumers have released it. Either way,
+ * the weak reference is useless now.
+ */
+ export = null;
+ }
}
}
- /**
- * Count of PyBuffer exports not yet released, used to prevent untimely resizing.
- */
- private int exportCount;
-
-
/* ============================================================================================
* API for org.python.core.PySequence
* ============================================================================================
@@ -390,10 +428,8 @@
* The actual behaviour depends on the nature (type) of value. It may be any kind of
* PyObject (but not other kinds of Java Object). The function is implementing assignment to
* a slice. PEP 3137 declares that the value may be "any type that implements the PEP 3118
- * buffer interface, which isn't implemented yet in Jython.
- */
- // XXX correct this when the buffer interface is available in Jython
- /*
+ * buffer interface".
+ *
* The following is essentially equivalent to b[start:stop[:step]]=bytearray(value) except
* we avoid constructing a copy of value if we can easily do so. The logic is the same as
* BaseBytes.init(PyObject), without support for value==null.
@@ -503,8 +539,8 @@
* @throws PyException(SliceSizeError) if the value size is inconsistent with an extended slice
*/
private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException {
- PyBuffer view = value.getBuffer(PyBUF.SIMPLE);
+ PyBuffer view = value.getBuffer(PyBUF.FULL_RO);
int len = view.getLen();
@@ -841,7 +877,7 @@
PyByteArray sum = null;
- // XXX re-write using View
+ // XXX re-write using buffer API
if (o instanceof BaseBytes) {
@@ -1977,10 +2013,10 @@
final PyByteArray bytearray_translate(PyObject table, PyObject deletechars) {
// Normalise the translation table to a View
- View tab = null;
+ PyBuffer tab = null;
if (table != null && table != Py.None) {
tab = getViewOrError(table);
- if (tab.size() != 256) {
+ if (tab.getLen() != 256) {
throw Py.ValueError("translation table must be 256 bytes long");
}
}
diff --git a/src/org/python/core/PyMemoryView.java b/src/org/python/core/PyMemoryView.java
--- a/src/org/python/core/PyMemoryView.java
+++ b/src/org/python/core/PyMemoryView.java
@@ -10,35 +10,35 @@
* missing.
*/
@ExposedType(name = "memoryview", base = PyObject.class, isBaseType = false)
-public class PyMemoryView extends PyObject {
-
- // XXX This should probably extend PySequence to get the slice behaviour
+public class PyMemoryView extends PySequence implements BufferProtocol {
public static final PyType TYPE = PyType.fromClass(PyMemoryView.class);
+ /** The buffer exported by the object of which this is a view. */
+ private PyBuffer backing;
/**
- * The buffer exported by the object. We do not a present implement the buffer sharing strategy
- * used by CPython <code>memoryview</code>.
+ * A memoryview in the released state forbids most Python API actions. If the underlying
+ * PyBuffer is shared, the memoryview may be released while the underlying PyBuffer is not
+ * "finally" released.
*/
- private PyBuffer backing;
+ private boolean released;
/** Cache the result of getting shape here. */
- private PyTuple shape;
+ private PyObject shape;
/** Cache the result of getting strides here. */
- private PyTuple strides;
+ private PyObject strides;
+ /** Cache the result of getting suboffsets here. */
+ private PyObject suboffsets;
/**
- * Construct a PyMemoryView from an object that bears the necessary BufferProtocol interface.
- * The buffer so obtained will be writable if the underlying object permits it.
- *
- * @param obj object that will export the buffer
+ * Construct a PyMemoryView from a PyBuffer interface. The buffer so obtained will be writable
+ * if the underlying object permits it. The <code>memoryview</code> takes a new lease on the
+ * <code>PyBuffer</code>.
+ *
+ * @param pybuf buffer exported by some underlying object
*/
- public PyMemoryView(BufferProtocol obj) {
- /*
- * Ask for the full set of facilities (strides, indirect, etc.) from the object in case they
- * are necessary for navigation, but only ask for read access. If the object is writable,
- * the PyBuffer will be writable.
- */
- backing = obj.getBuffer(PyBUF.FULL_RO);
+ public PyMemoryView(PyBuffer pybuf) {
+ super(TYPE);
+ backing = pybuf.getBuffer(PyBUF.FULL_RO);
}
@ExposedNew
@@ -46,7 +46,12 @@
PyObject[] args, String[] keywords) {
PyObject obj = args[0];
if (obj instanceof BufferProtocol) {
- return new PyMemoryView((BufferProtocol)obj);
+ /*
+ * Ask for the full set of facilities (strides, indirect, etc.) from the object in case
+ * they are necessary for navigation, but only ask for read access. If the object is
+ * writable, the PyBuffer will be writable.
+ */
+ return new PyMemoryView(((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO));
} else {
throw Py.TypeError("cannot make memory view because object does not have "
+ "the buffer interface");
@@ -64,7 +69,7 @@
}
@ExposedGet(doc = shape_doc)
- public PyTuple shape() {
+ public PyObject shape() {
if (shape == null) {
shape = tupleOf(backing.getShape());
}
@@ -73,34 +78,51 @@
@ExposedGet(doc = ndim_doc)
public int ndim() {
- return backing.getShape().length;
+ return backing.getNdim();
}
@ExposedGet(doc = strides_doc)
- public PyTuple strides() {
+ public PyObject strides() {
if (strides == null) {
strides = tupleOf(backing.getStrides());
}
return strides;
}
+ @ExposedGet(doc = suboffsets_doc)
+ public PyObject suboffsets() {
+ if (suboffsets == null) {
+ suboffsets = tupleOf(backing.getSuboffsets());
+ }
+ return suboffsets;
+ }
+
@ExposedGet(doc = readonly_doc)
public boolean readonly() {
return backing.isReadonly();
}
/**
- * Make an integer array into a PyTuple of PyInteger values.
- *
- * @param x the array
- * @return the PyTuple
+ * Make an integer array into a PyTuple of PyLong values or None if the argument is null.
+ *
+ * @param x the array (or null)
+ * @return the PyTuple (or Py.None)
*/
- private PyTuple tupleOf(int[] x) {
- PyInteger[] pyx = new PyInteger[x.length];
- for (int k = 0; k < x.length; k++) {
- pyx[k] = new PyInteger(x[k]);
+ private PyObject tupleOf(int[] x) {
+ if (x != null) {
+ PyLong[] pyx = new PyLong[x.length];
+ for (int k = 0; k < x.length; k++) {
+ pyx[k] = new PyLong(x[k]);
+ }
+ return new PyTuple(pyx, false);
+ } else {
+ return Py.None;
}
- return new PyTuple(pyx, false);
+ }
+
+ @Override
+ public int __len__() {
+ return backing.getLen();
}
/*
@@ -134,7 +156,209 @@
+ "A tuple of integers the length of ndim giving the size in bytes to access\n"
+ "each element for each dimension of the array.\n";
+ private final static String suboffsets_doc = "suboffsets\n"
+ + "A tuple of integers the length of ndim, or None, used to access\n"
+ + "each element for each dimension of an indirect array.\n";
+
private final static String readonly_doc = "readonly\n"
+ "A bool indicating whether the memory is read only.\n";
+ /*
+ * ============================================================================================
+ * Support for the Buffer API
+ * ============================================================================================
+ *
+ * The buffer API allows other classes to access the storage directly.
+ */
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The {@link PyBuffer} returned from this method is just the one on which the
+ * <code>memoryview</code> was first constructed. The Jython buffer API is such that sharing
+ * directly is safe (as long as the get-release discipline is observed).
+ */
+ @Override
+ public synchronized PyBuffer getBuffer(int flags) {
+ /*
+ * The PyBuffer itself does all the export counting, and since the behaviour of memoryview
+ * need not change, it really is a simple as:
+ */
+ return backing.getBuffer(flags);
+ }
+
+ /**
+ * Request a release of the underlying buffer exposed by the <code>memoryview</code> object.
+ * Many objects take special actions when a view is held on them (for example, a
+ * <code>bytearray</code> would temporarily forbid resizing); therefore, calling
+ * <code>release()</code> is handy to remove these restrictions (and free any dangling
+ * resources) as soon as possible.
+ * <p>
+ * After this method has been called, any further operation on the view raises a
+ * <code>ValueError</code> (except <code>release()</code> itself which can be called multiple
+ * times with the same effect as just one call).
+ * <p>
+ * This becomes an exposed method only in Python 3.2, but the Jython implementation of
+ * <code>memoryview</code> follows the Python 3.3 design internally, which is the version that
+ * resolved some long-standing design issues.
+ */
+ public synchronized void release() {
+ /*
+ * It is not an error to call this release method while this <code>memoryview</code> has
+ * buffer exports (e.g. another <code>memoryview</code> was created on it), but it will not
+ * release the underlying object until the last consumer releases the buffer.
+ */
+ if (!released) {
+ // Release the buffer (which is not necessarily final)
+ backing.release();
+ // Remember we've been released
+ released = true;
+ }
+ }
+
+ /*
+ * ============================================================================================
+ * API for org.python.core.PySequence
+ * ============================================================================================
+ */
+ /**
+ * Gets the indexed element of the memoryview as an integer. This is an extension point
+ * called by PySequence in its implementation of {@link #__getitem__}. It is guaranteed by
+ * PySequence that the index is within the bounds of the memoryview.
+ *
+ * @param index index of the element to get.
+ */
+ @Override
+ protected PyInteger pyget(int index) {
+ return new PyInteger(backing.intAt(index));
+ }
+
+ /**
+ * Returns a slice of elements from this sequence as a PyMemoryView.
+ *
+ * @param start the position of the first element.
+ * @param stop one more than the position of the last element.
+ * @param step the step size.
+ * @return a PyMemoryView corresponding the the given range of elements.
+ */
+ @Override
+ protected synchronized PyMemoryView getslice(int start, int stop, int step) {
+ int n = sliceLength(start, stop, step);
+ PyBuffer view = backing.getBufferSlice(PyBUF.FULL_RO, start, n, step);
+ PyMemoryView ret = new PyMemoryView(view);
+ view.release(); // We've finished (new PyMemoryView holds a lease)
+ return ret;
+ }
+
+ /**
+ * memoryview*int is not implemented in Python, so this should never be called. We still have to override
+ * it to satisfy PySequence.
+ *
+ * @param count the number of times to repeat this.
+ * @return never
+ * @throws PyException(NotImlemented) always
+ */
+ @Override
+ protected synchronized PyMemoryView repeat(int count) throws PyException {
+ throw Py.NotImplementedError("memoryview.repeat()");
+ }
+
+ /**
+ * Sets the indexed element of the memoryview to the given value. This is an extension point
+ * called by PySequence in its implementation of {@link #__setitem__} It is guaranteed by
+ * PySequence that the index is within the bounds of the memoryview. Any other clients calling
+ * <tt>pyset(int)</tt> must make the same guarantee.
+ *
+ * @param index index of the element to set.
+ * @param value the value to set this element to.
+ * @throws PyException(AttributeError) if value cannot be converted to an integer
+ * @throws PyException(ValueError) if value<0 or value>255
+ */
+ public synchronized void pyset(int index, PyObject value) throws PyException {
+ backing.storeAt(BaseBytes.byteCheck(value), index);
+ }
+
+ /**
+ * Sets the given range of elements according to Python slice assignment semantics. If the step
+ * size is one, it is a simple slice and the operation is equivalent to replacing that slice,
+ * with the value, accessing the value via the buffer protocol.
+ *
+ * <pre>
+ * a = bytearray(b'abcdefghijklmnopqrst')
+ * m = memoryview(a)
+ * m[2:7] = "ABCDE"
+ * </pre>
+ *
+ * Results in <code>a=bytearray(b'abABCDEhijklmnopqrst')</code>.
+ * <p>
+ * If the step size is one, but stop-start does not match the length of the right-hand-side a
+ * ValueError is thrown.
+ * <p>
+ * If the step size is not one, and start!=stop, the slice defines a certain number of elements
+ * to be replaced. This function is not available in Python 2.7 (but it is in Python 3.3).
+ * <p>
+ *
+ * <pre>
+ * a = bytearray(b'abcdefghijklmnopqrst')
+ * a[2:12:2] = iter( [65, 66, 67, long(68), "E"] )
+ * </pre>
+ *
+ * Results in <code>a=bytearray(b'abAdBfChDjElmnopqrst')</code> in Python 3.3.
+ *
+ * @param start the position of the first element.
+ * @param stop one more than the position of the last element.
+ * @param step the step size.
+ * @param value an object consistent with the slice assignment
+ */
+ @Override
+ protected synchronized void setslice(int start, int stop, int step, PyObject value) {
+
+ if (step == 1 && stop < start) {
+ // Because "b[5:2] = v" means insert v just before 5 not 2.
+ // ... although "b[5:2:-1] = v means b[5]=v[0], b[4]=v[1], b[3]=v[2]
+ stop = start;
+ }
+
+ if (!(value instanceof BufferProtocol)) {
+ String fmt = "'%s' does not support the buffer interface";
+ throw Py.TypeError(String.format(fmt, value.getType().getName()));
+ }
+
+ // We'll try to get two new buffers: and finally release them.
+ PyBuffer valueBuf = null, backingSlice = null;
+
+ try {
+ // Get a buffer API on the value being assigned
+ valueBuf = ((BufferProtocol)value).getBuffer(PyBUF.FULL_RO);
+
+ // How many destination items? Has to match size of value.
+ int n = sliceLength(start, stop, step);
+ if (n != valueBuf.getLen()) {
+ // CPython 2.7 message
+ throw Py.ValueError("cannot modify size of memoryview object");
+ }
+
+ /*
+ * In the next section, we get a sliced view of the backing and write the value to it.
+ * The approach to errors is unusual for compatibility with CPython. We pretend we will
+ * not need a WRITABLE buffer in order to avoid throwing a BufferError. This does not
+ * stop the returned object being writable, simply avoids the check. If in fact it is
+ * read-only, then trying to write raises TypeError.
+ */
+
+ backingSlice = backing.getBufferSlice(PyBUF.FULL_RO, start, n, step);
+ backing.copyFrom(valueBuf);
+
+ } finally {
+
+ // Release the buffers we obtained (if we did)
+ if (backingSlice != null) {
+ backingSlice.release();
+ }
+ if (valueBuf != null) {
+ valueBuf.release();
+ }
+ }
+ }
+
}
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -1,11 +1,14 @@
/// Copyright (c) Corporation for National Research Initiatives
package org.python.core;
+import java.lang.ref.Reference;
+import java.lang.ref.SoftReference;
import java.math.BigInteger;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import org.python.core.StringFormatter.DecimalFormatTemplate;
+import org.python.core.buffer.BaseBuffer;
import org.python.core.buffer.SimpleStringBuffer;
import org.python.core.stringlib.FieldNameIterator;
import org.python.core.stringlib.InternalFormatSpec;
@@ -27,6 +30,8 @@
public static final PyType TYPE = PyType.fromClass(PyString.class);
protected String string; // cannot make final because of Python intern support
protected transient boolean interned=false;
+ /** Supports the buffer API, see {@link #getBuffer(int)}. */
+ private Reference<BaseBuffer> export;
public String getString() {
return string;
@@ -96,18 +101,47 @@
}
/**
- * Create a read-only buffer view of the contents of the string, treating it as a sequence of
+ * Return a read-only buffer view of the contents of the string, treating it as a sequence of
* unsigned bytes. The caller specifies its requirements and navigational capabilities in the
- * <code>flags</code> argument (see the constants in class {@link PyBUF} for an explanation).
+ * <code>flags</code> argument (see the constants in interface {@link PyBUF} for an
+ * explanation). The method may return the same PyBuffer object to more than one consumer.
*
* @param flags consumer requirements
* @return the requested buffer
*/
- public PyBuffer getBuffer(int flags) {
- /*
- * Return a buffer, but specialised to defer construction of the buf object.
- */
- return new SimpleStringBuffer(this, getString(), flags);
+ public synchronized PyBuffer getBuffer(int flags) {
+ // If we have already exported a buffer it may still be available for re-use
+ BaseBuffer pybuf = getExistingBuffer(flags);
+ if (pybuf == null) {
+ /*
+ * No existing export we can re-use. Return a buffer, but specialised to defer
+ * construction of the buf object, and cache a soft reference to it.
+ */
+ pybuf = new SimpleStringBuffer(flags, getString());
+ export = new SoftReference<BaseBuffer>(pybuf);
+ }
+ return pybuf;
+ }
+
+ /**
+ * Helper for {@link #getBuffer(int)} that tries to re-use an existing exported buffer, or
+ * returns null if can't.
+ */
+ private BaseBuffer getExistingBuffer(int flags) {
+ BaseBuffer pybuf = null;
+ if (export != null) {
+ // A buffer was exported at some time.
+ pybuf = export.get();
+ if (pybuf != null) {
+ /*
+ * And this buffer still exists. Even in the case where the buffer has been released
+ * by all its consumers, it remains safe to re-acquire it because the target String
+ * has not changed.
+ */
+ pybuf = pybuf.getBufferAgain(flags);
+ }
+ }
+ return pybuf;
}
public String substring(int start, int end) {
diff --git a/src/org/python/core/buffer/BaseBuffer.java b/src/org/python/core/buffer/BaseBuffer.java
--- a/src/org/python/core/buffer/BaseBuffer.java
+++ b/src/org/python/core/buffer/BaseBuffer.java
@@ -8,44 +8,50 @@
import org.python.core.PyException;
/**
- * Base implementation of the Buffer API for implementations to extend. The default implementation
- * provides some mechanisms for checking the consumer's capabilities against those stated as
- * necessary by the exporter. Default implementations of methods are provided for the standard array
- * organisations. The implementors of simple buffers will find it more efficient to override methods
- * to which performance might be sensitive with a calculation specific to their actual type.
+ * Base implementation of the Buffer API providing variables and accessors for the navigational
+ * arrays (without actually creating the arrays), methods for expressing and checking the buffer
+ * request flags, methods and mechanism for get-release counting, boilerplate error checks and their
+ * associated exceptions, and default implementations of some methods for access to the buffer
+ * content. The design aim is to ensure unglamorous common code need only be implemented once.
* <p>
- * The default implementation raises a read-only exception for those methods that store data in the
- * buffer, and {@link #isReadonly()} returns <code>true</code>. Writable types must override this
- * implementation. Default implementations of other methods are generally oriented towards
- * contiguous N-dimensional arrays.
+ * Where provided, the buffer access methods are appropriate to 1-dimensional arrays where the units
+ * are single bytes, stored contiguously. Sub-classes that deal with N-dimensional arrays,
+ * discontiguous storage and items that are not single bytes must override the default
+ * implementations.
* <p>
- * At the time of writing, only the SIMPLE organisation (one-dimensional, of item size one) is used
- * in the Jython core.
+ * This base implementation is writable only if {@link PyBUF#WRITABLE} is in the feature flags
+ * passed to the constructor. Otherwise, all methods for write access raise a
+ * <code>BufferError</code> read-only exception and {@link #isReadonly()} returns <code>true</code>.
+ * Sub-classes can follow the same pattern, setting {@link PyBUF#WRITABLE} in the constructor and,
+ * if they have to override the operations that write (<code>storeAt</code> and
+ * <code>copyFrom</code>). The recommended pattern is:
+ *
+ * <pre>
+ * if (isReadonly()) {
+ * throw notWritable();
+ * }
+ * // ... implementation of the write operation
+ * </pre>
+ *
+ * The implementors of simple buffers will find it efficient to override the generic access methods
+ * to which performance might be sensitive, with a calculation specific to their actual type.
+ * <p>
+ * At the time of writing, only one-dimensional buffers of item size one are used in the Jython
+ * core.
*/
public abstract class BaseBuffer implements PyBuffer {
/**
- * The object from which this buffer export must be released (see {@link PyBuffer#release()}).
- * This is normally the original exporter of this buffer and the owner of the underlying
- * storage. Exceptions to this occur when some other object is managing release (this is the
- * case when a <code>memoryview</code> has provided the buffer), and when disposal can safely be
- * left to the Java garbage collector (local temporaries and perhaps exports from immutable
- * objects).
- */
- protected BufferProtocol obj;
- /**
* The dimensions of the array represented by the buffer. The length of the <code>shape</code>
* array is the number of dimensions. The <code>shape</code> array should always be created and
- * filled (difference from CPython).
+ * filled (difference from CPython). This value is returned by {@link #getShape()}.
*/
protected int[] shape;
/**
* Step sizes in the underlying buffer essential to correct translation of an index (or indices)
- * into an index into the storage. This reference will be <code>null</code> if not needed for
- * the storage organisation, and not requested by the consumer in <code>flags</code>. If it is
- * either necessary for the buffer navigation, or requested by the consumer in flags, the
- * <code>strides</code> array must be correctly filled to at least the length of the
- * <code>shape</code> array.
+ * into an index into the storage. The <code>strides</code> array should always be created and
+ * correctly filled to at least the length of the <code>shape</code> array (difference from
+ * CPython). This value is returned by {@link #getStrides()}.
*/
protected int[] strides;
/**
@@ -54,131 +60,147 @@
*/
protected BufferPointer buf;
/**
- * Bit pattern using the constants defined in {@link PyBUF} that records the actual capabilities
- * this buffer offers. See {@link #assignCapabilityFlags(int, int, int, int)}.
+ * Count the number of times {@link #release()} must be called before actual release actions
+ * need to take place. Equivalently, this is the number of calls to
+ * {@link BufferProtocol#getBuffer(int)} that have returned this object: one for the call on the
+ * original exporting object that constructed <code>this</code>, and one for each subsequent
+ * call to {@link PyBuffer#getBuffer(int)} that returned <code>this</code>.
*/
- protected int capabilityFlags;
+ protected int exports = 1;
/**
- * The result of the operation is to set the {@link #capabilityFlags} according to the
- * capabilities this instance should support. This method is normally called in the constructor
- * of each particular sub-class of <code>BaseBuffer</code>, passing in a <code>flags</code>
- * argument that originated in the consumer's call to {@link BufferProtocol#getBuffer(int)}.
+ * Bit pattern using the constants defined in {@link PyBUF} that records the actual features
+ * this buffer offers. When checking consumer flags against the features of the buffer, it is an
+ * error if the consumer requests a capability the buffer does not offer, and it is an error if
+ * the consumer does not specify that it will use a navigation array the buffer requires.
* <p>
- * The consumer supplies as a set of <code>flags</code>, using constants from {@link PyBUF}, the
- * capabilities that it expects from the buffer. These include a statement of which navigational
- * arrays it will use ( <code>shape</code>, <code>strides</code>, and <code>suboffsets</code>),
- * whether it wants the <code>format</code> string set so it describes the item type or left
- * null, and whether it expects the buffer to be writable. The consumer flags are taken by this
- * method both as a statement of needs to be met by the buffer, and as a statement of
- * capabilities in the consumer to navigate different buffers.
- * <p>
- * In its call to this method, the exporter specifies the capabilities it requires the consumer
- * to have (and indicate by asking for them in <code>flags</code>) in order to navigate the
- * buffer successfully. For example, if the buffer is a strided array, the consumer must specify
- * that it expects the <code>strides</code> array. Otherwise the method concludes the consumer
- * is not capable of the navigation required. Capabilities specified in the
- * <code>requiredFlags</code> must appear in the consumer's <code>flags</code> request. If any
- * don't, a Python <code>BufferError</code> will be raised. If there is no error these flags
- * will be set in <code>capabilityFlags</code> as required of the buffer.
- * <p>
- * The exporter specifies some capabilities it <i>allows</i> the consumer to request, such as
- * the <code>format</code> string. Depending on the type of exporter, the navigational arrays (
- * <code>shape</code>, <code>strides</code>, and <code>suboffsets</code>) may also be allowed
- * rather than required. Capabilities specified in the <code>allowedFlags</code>, if they also
- * appear in the consumer's <code>flags</code>, will be set in <code>capabilityFlags</code>.
- * <p>
- * The exporter specifies some capabilities that will be supplied whether requested or not. For
- * example (and it might be the only one) this is used only to express that an unstrided,
- * one-dimensional array is <code>C_CONTIGUOUS</code>, <code>F_CONTIGUOUS</code>, and
- * <code>ANY_CONTIGUOUS</code>, all at once. Capabilities specified in the
- * <code>impliedFlags</code>, will be set in <code>capabilityFlags</code> whether in the
- * consumer's <code>flags</code> or not.
- * <p>
- * Capabilities specified in the consumer's <code>flags</code> request, if they do not appear in
- * the exporter's <code>requiredFlags</code> <code>allowedFlags</code> or
- * <code>impliedFlags</code>, will cause a Python <code>BufferError</code>.
- * <p>
- * Note that this method cannot actually set the <code>shape</code>, <code>strides</code> and
- * <code>suboffsets</code> properties: the implementation of the specific buffer type must do
- * that based on the <code>capabilityFlags</code>. This forms a partial counterpart to CPython
- * <code>PyBuffer_FillInfo()</code> but it is not specific to the simple type of buffer, and
- * covers the flag processing of all buffer types. This is complex (in CPython) and the Jython
- * approach attempts to be compatible yet comprehensible.
+ * In order to support efficient checking with {@link #checkRequestFlags(int)} we store a
+ * mutilated version of the apparent <code>featureFlags</code> in which the non-navigational
+ * flags are inverted. The syndrome <code>S</code> of the error is computed as follows. Let
+ * <code>N=1</code> where we are dealing with a navigation flag, let <code>F</code> be a buffer
+ * feature flag, and let <code>X</code> be the consumer request flags.
+ *
+ * <pre>
+ * A = F N X'
+ * B = F'N'X
+ * S = A + B = F N X' + F'N'X
+ * </pre>
+ *
+ * In the above, <code>A=0</code> only if all the navigation flags set in <code>F</code> are
+ * also set in <code>X</code>, and <code>B=0</code> only if all the non-navigation flags clear
+ * in <code>F</code> are also clear in <code>X</code>. <code>S=0</code> only if both these
+ * conditions are true and furthermore the positions of the <code>1</code>s in the syndrome
+ * <code>S</code> tell us which bits in <code>X</code> are at fault. Now if we define:
+ * <code>G = N F + N'F'</code> then the syndrome is:
+ *
+ * <pre>
+ * S = G (N X' + N'X)
+ * </pre>
+ *
+ * Which permits the check in one XOR and one AND operation instead of four ANDs and an OR. The
+ * down-side is that we have to provide methods for setting and getting the actual flags in
+ * terms a client might expect them to be expressed. We can recover the original <code>F</code>
+ * since:
+ *
+ * <pre>
+ * N G + N'G' = F
+ * </pre>
*/
- protected void assignCapabilityFlags(int flags, int requiredFlags, int allowedFlags,
- int impliedFlags) {
+ private int gFeatureFlags = ~NAVIGATION; // featureFlags = 0
- // Ensure what may be requested includes what must be and what comes unasked
- allowedFlags = allowedFlags | requiredFlags | impliedFlags;
-
- // Look for request flags (other than buffer organisation) outside what is allowed
- int syndrome = flags & ~(allowedFlags | ORGANISATION);
-
- if (syndrome != 0) {
- // Some flag was set that is neither required nor allowed
- if ((syndrome & WRITABLE) != 0) {
- throw notWritable();
- } else if ((syndrome & C_CONTIGUOUS) != 0) {
- throw bufferIsNot("C-contiguous");
- } else if ((syndrome & F_CONTIGUOUS) != 0) {
- throw bufferIsNot("Fortran-contiguous");
- } else if ((syndrome & ANY_CONTIGUOUS) != 0) {
- throw bufferIsNot("contiguous");
- } else {
- // Catch-all error (never in practice?)
- throw bufferIsNot("capable of matching request");
- }
-
- } else if ((flags & requiredFlags) != requiredFlags) {
- // This buffer needs more capability to navigate than the consumer has requested
- if ((flags & ND) != ND) {
- throw bufferRequires("shape");
- } else if ((flags & STRIDES) != STRIDES) {
- throw bufferRequires("strides");
- } else if ((flags & INDIRECT) != INDIRECT) {
- throw bufferRequires("suboffsets");
- } else {
- // Catch-all error
- throw bufferRequires("feature consumer lacks");
- }
-
- } else {
- // These flags control returns from (default) getShape etc..
- capabilityFlags = (flags & allowedFlags) | impliedFlags;
- // Note that shape and strides are still to be initialised
- }
-
- /*
- * Caller must responds to the requested/required capabilities with shape and strides arrays
- * suited to the actual type of buffer.
- */
+ /**
+ * Construct an instance of BaseBuffer in support of a sub-class, specifying the 'feature
+ * flags', or at least a starting set to be adjusted later. These are the features of the buffer
+ * exported, not the flags that form the consumer's request. The buffer will be read-only unless
+ * {@link PyBUF#WRITABLE} is set in the feature flags. {@link PyBUF#FORMAT} is implicitly added
+ * to the feature flags. The navigation arrays are all null, awaiting action by the sub-class
+ * constructor. To complete initialisation, the sub-class normally must assign: {@link #buf},
+ * {@link #shape}, and {@link #strides}, and call {@link #checkRequestFlags(int)} passing the
+ * consumer's request flags.
+ *
+ * <pre>
+ * this.buf = buf; // Wraps exported data
+ * this.shape = shape; // Array of dimensions of exported data (in units)
+ * this.strides = strides; // Byte offset between successive items in each dimension
+ * checkRequestFlags(flags); // Check request is compatible with type
+ * </pre>
+ *
+ * @param featureFlags bit pattern that specifies the actual features allowed/required
+ */
+ protected BaseBuffer(int featureFlags) {
+ setFeatureFlags(featureFlags | FORMAT);
}
/**
- * Provide an instance of BaseBuffer or a sub-class meeting the consumer's expectations as
- * expressed in the flags argument. Compare CPython:
+ * Get the features of this buffer expressed using the constants defined in {@link PyBUF}. A
+ * client request may be tested against the consumer's request flags with
+ * {@link #checkRequestFlags(int)}.
*
- * <pre>
- * int PyBuffer_FillInfo(Py_buffer *view, PyObject *exporter,
- * void *buf, Py_ssize_t len,
- * int readonly, int flags)
- * </pre>
+ * @return capabilities of and navigation required by the exporter/buffer
+ */
+ protected final int getFeatureFlags() {
+ return NAVIGATION ^ (~gFeatureFlags);
+ }
+
+ /**
+ * Set the features of this buffer expressed using the constants defined in {@link PyBUF},
+ * replacing any previous set. Set individual flags or add to those already set by using
+ * {@link #addFeatureFlags(int)}.
*
- * @param exporter the exporting object
- * @param buf descriptor for the exported buffer itself
+ * @param flags new value for the feature flags
*/
- protected BaseBuffer(BufferProtocol exporter, BufferPointer buf) {
- // Exporting object (is allowed to be null)
- this.obj = exporter;
- // Exported data (not normally allowed to be null)
- this.buf = buf;
+ protected final void setFeatureFlags(int flags) {
+ gFeatureFlags = (~NAVIGATION) ^ flags;
+ }
+
+ /**
+ * Add to the features of this buffer expressed using the constants defined in {@link PyBUF},
+ * setting individual flags specified while leaving those already set. Equivalent to
+ * <code>setFeatureFlags(flags | getFeatureFlags())</code>.
+ *
+ * @param flags to set within the feature flags
+ */
+ protected final void addFeatureFlags(int flags) {
+ setFeatureFlags(flags | getFeatureFlags());
+ }
+
+ /**
+ * General purpose method to check the consumer request flags (typically the argument to
+ * {@link BufferProtocol#getBuffer(int)}) against the feature flags (see
+ * {@link #getFeatureFlags()}) that characterise the features of the buffer, and to raise an
+ * exception (Python <code>BufferError</code>) with an appropriate message in the case of a
+ * mismatch. The flags are defined in the interface {@link PyBUF} and are used in two ways.
+ * <p>
+ * In a subset of the flags, the consumer specifies assumptions it makes about the index order
+ * (contiguity) of the buffer, and whether it is writable. When the buffer implementation calls
+ * this check method, it has already specified in {@link #setFeatureFlags(int)} what
+ * capabilities this type (or instance) buffer actually has. It is an error, for the consumer to
+ * specify in its request a feature that the buffer does not offer.
+ * <p>
+ * In a subset of the flags, the consumer specifies the set of navigational arrays (
+ * <code>shape</code>, <code>strides</code>, and <code>suboffsets</code>) it intends to use in
+ * navigating the buffer. When the buffer implementation calls this check method, it has already
+ * specified in {@link #setFeatureFlags(int)} what navigation is necessary for the consumer to
+ * make sense of the buffer. It is an error for the consumer <i>not to specify</i> the flag
+ * corresponding to an array that the buffer deems necessary.
+ *
+ * @param flags capabilities of and navigation assumed by the consumer
+ * @throws PyException (BufferError) when expectations do not correspond with the buffer
+ */
+ protected void checkRequestFlags(int flags) throws PyException {
+ /*
+ * It is an error if any of the navigation flags is 0 when it should be 1, or if any of the
+ * non-navigation flags is 1 when it should be 0.
+ */
+ int syndrome = gFeatureFlags & (flags ^ NAVIGATION);
+ if (syndrome != 0) {
+ throw bufferErrorFromSyndrome(syndrome);
+ }
}
@Override
public boolean isReadonly() {
- // Default position is read only: mutable buffers must override
- return true;
+ return (gFeatureFlags & WRITABLE) == 0;
}
@Override
@@ -194,12 +216,14 @@
@Override
public int getLen() {
- // Correct if contiguous. Override if strided or indirect with itemsize*product(shape).
- return buf.size;
+ // Correct if one-dimensional. Override if N-dimensional with itemsize*product(shape).
+ return shape[0];
}
- // Let the sub-class implement:
- // @Override public byte byteAt(int index) throws IndexOutOfBoundsException {}
+ @Override
+ public byte byteAt(int index) throws IndexOutOfBoundsException {
+ return buf.storage[calcIndex(index)];
+ }
@Override
public int intAt(int index) throws IndexOutOfBoundsException {
@@ -208,11 +232,28 @@
@Override
public void storeAt(byte value, int index) throws IndexOutOfBoundsException, PyException {
- throw notWritable();
+ if (isReadonly()) {
+ throw notWritable();
+ }
+ buf.storage[calcIndex(index)] = value;
}
- // Let the sub-class implement:
- // @Override public byte byteAt(int... indices) throws IndexOutOfBoundsException {}
+ /**
+ * Convert an item index (for a one-dimensional buffer) to an absolute byte index in the actual
+ * storage being shared by the exporter. See {@link #calcIndex(int...)} for discussion.
+ *
+ * @param index from consumer
+ * @return index in actual storage
+ */
+ protected int calcIndex(int index) throws IndexOutOfBoundsException {
+ // Treat as one-dimensional
+ return buf.offset + index * getStrides()[0];
+ }
+
+ @Override
+ public byte byteAt(int... indices) throws IndexOutOfBoundsException {
+ return buf.storage[calcIndex(indices)];
+ }
@Override
public int intAt(int... indices) throws IndexOutOfBoundsException {
@@ -221,50 +262,277 @@
@Override
public void storeAt(byte value, int... indices) throws IndexOutOfBoundsException, PyException {
- throw notWritable();
+ if (isReadonly()) {
+ throw notWritable();
+ }
+ buf.storage[calcIndex(indices)] = value;
}
- @Override
- public void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException {
- // Correct for contiguous arrays (if destination expects same F or C contiguity)
- copyTo(0, dest, destPos, getLen());
- }
-
- // Let the sub-class implement:
- // @Override public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
- // throws IndexOutOfBoundsException {}
-
- @Override
- public void copyFrom(byte[] src, int srcPos, int destIndex, int length)
- throws IndexOutOfBoundsException, PyException {
- throw notWritable();
+ /**
+ * Convert a multi-dimensional item index (if we are not using indirection) to an absolute byte
+ * index in the actual storage array being shared by the exporter. The purpose of this method is
+ * to allow a sub-class to define, in one place, an indexing calculation that maps the index as
+ * provided by the consumer into an index in the storage as seen by the buffer.
+ * <p>
+ * In the usual case where the storage is referenced via the <code>BufferPointer</code> member
+ * {@link #buf}, the buffer implementation may use <code>buf.storage[calcIndex(i)]</code> to
+ * reference the (first byte of) the item x[i]. This is what the default implementation of
+ * accessors in <code>BaseBuffer</code> will do. In the simplest cases, this is fairly
+ * inefficient, and an implementation will override the accessors to in-line the calculation.
+ * The default implementation here is suited to N-dimensional arrays.
+ *
+ * @param indices of the item from the consumer
+ * @return index relative to item x[0,...,0] in actual storage
+ */
+ protected int calcIndex(int... indices) throws IndexOutOfBoundsException {
+ final int N = checkDimension(indices);
+ // In general: buf.offset + sum(k=0,N-1) indices[k]*strides[k]
+ int index = buf.offset;
+ if (N > 0) {
+ int[] strides = getStrides();
+ for (int k = 0; k < N; k++) {
+ index += indices[k] * strides[k];
+ }
+ }
+ return index;
}
/**
* {@inheritDoc}
* <p>
- * The implementation here calls {@link #releaseAction()}, which the implementer of a specific
- * buffer type should override with the necessary actions to release the buffer from the
- * exporter. It is not an error to call this method more than once (difference from CPython), or
- * on a temporary buffer that needs no release action. If not released explicitly, it will be
- * called during object finalisation (before garbage collection) of the buffer object.
+ * The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
+ * case of arbitrary item size and stride.
*/
@Override
- public final void release() {
- if (obj != null) {
+ public void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException {
+ // Note shape[0] is the number of items in the array
+ copyTo(0, dest, destPos, shape[0]);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
+ * case of arbitrary item size and stride.
+ */
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ throws IndexOutOfBoundsException {
+
+ // Data is here in the buffers
+ int s = calcIndex(srcIndex);
+ int d = destPos;
+
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int itemsize = getItemsize();
+ int stride = getStrides()[0];
+ int skip = stride - itemsize;
+
+ // Strategy depends on whether items are laid end-to-end contiguously or there are gaps
+ if (skip == 0) {
+ // stride == itemsize: straight copy of contiguous bytes
+ System.arraycopy(buf.storage, s, dest, d, length * itemsize);
+
+ } else if (itemsize == 1) {
+ // Discontiguous copy: single byte items
+ int limit = s + length * stride;
+ for (; s < limit; s += stride) {
+ dest[d++] = buf.storage[s];
+ }
+
+ } else {
+ // Discontiguous copy: each time, copy itemsize bytes then skip
+ int limit = s + length * stride;
+ for (; s < limit; s += skip) {
+ int t = s + itemsize;
+ while (s < t) {
+ dest[d++] = buf.storage[s++];
+ }
+ }
+ }
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
+ * case of arbitrary item size and stride.
+ */
+ @Override
+ public void copyFrom(byte[] src, int srcPos, int destIndex, int length)
+ throws IndexOutOfBoundsException, PyException {
+
+ // Block operation if read-only
+ if (isReadonly()) {
+ throw notWritable();
+ }
+
+ // Data is here in the buffers
+ int s = srcPos;
+ int d = calcIndex(destIndex);
+
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int itemsize = getItemsize();
+ int stride = getStrides()[0];
+ int skip = stride - itemsize;
+
+ // Strategy depends on whether items are laid end-to-end or there are gaps
+ if (skip == 0) {
+ // Straight copy of contiguous bytes
+ System.arraycopy(src, srcPos, buf.storage, d, length * itemsize);
+
+ } else if (itemsize == 1) {
+ // Discontiguous copy: single byte items
+ int limit = d + length * stride;
+ for (; d != limit; d += stride) {
+ buf.storage[d] = src[s++];
+ }
+
+ } else {
+ // Discontiguous copy: each time, copy itemsize bytes then skip
+ int limit = d + length * stride;
+ for (; d != limit; d += skip) {
+ int t = d + itemsize;
+ while (d < t) {
+ buf.storage[d++] = src[s++];
+ }
+ }
+ }
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
+ * case.
+ */
+ @Override
+ public void copyFrom(PyBuffer src) throws IndexOutOfBoundsException, PyException {
+
+ // Block operation if read-only and same length
+ if (isReadonly()) {
+ throw notWritable();
+ } else if (src.getLen() != buf.size || src.getItemsize() != getItemsize()) {
+ throw differentStructure();
+ }
+
+ // Data is here in the buffers
+ int s = 0;
+ int d = calcIndex(0);
+
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int itemsize = getItemsize();
+ int stride = getStrides()[0];
+
+ // Strategy depends on whether items are laid end-to-end or there are gaps
+ if (stride == itemsize) {
+ // Straight copy to contiguous bytes
+ src.copyTo(buf.storage, d);
+
+ } else if (itemsize == 1) {
+ // Discontiguous copy: single byte items
+ int limit = d + src.getLen() * stride;
+ for (; d != limit; d += stride) {
+ buf.storage[d] = src.byteAt(s++);
+ }
+
+ } else {
+ // Discontiguous copy: each time, copy itemsize bytes then skip
+ int limit = d + src.getShape()[0] * stride;
+ for (; d != limit; d += stride) {
+ BufferPointer srcItem = src.getPointer(s++);
+ System.arraycopy(srcItem.storage, srcItem.offset, buf.storage, d, itemsize);
+ }
+ }
+
+ }
+
+ @Override
+ public synchronized PyBuffer getBuffer(int flags) {
+ if (exports > 0) {
+ // Always safe to re-export if the current count is not zero
+ return getBufferAgain(flags);
+ } else {
+ // exports==0 so refuse
+ throw bufferReleased("getBuffer");
+ }
+ }
+
+ /**
+ * Allow an exporter to re-use a BaseBytes even if it has been "finally" released. Many
+ * sub-classes of <code>BaseBytes</code> can be re-used even after a final release by consumers,
+ * simply by incrementing the <code>exports</code> count again: the navigation arrays and the
+ * buffer view of the exporter's state all remain valid. We do not let consumers do this through
+ * the {@link PyBuffer} interface: from their perspective, calling {@link PyBuffer#release()}
+ * should mean the end of their access, although we can't stop them holding a reference to the
+ * PyBuffer. Only the exporting object, which is handles the implementation type is trusted to
+ * know when re-use is safe.
+ * <p>
+ * An exporter will use this method as part of its implementation of
+ * {@link BufferProtocol#getBuffer(int)}. On return from that, the buffer <i>and the exporting
+ * object</i> must then be in effectively the same state as if the buffer had just been
+ * constructed by that method. Exporters that destroy related resources on final release of
+ * their buffer (by overriding {@link #releaseAction()}), or permit themselves structural change
+ * invalidating the buffer, must either reconstruct the missing resources or avoid
+ * <code>getBufferAgain</code>.
+ */
+ public synchronized BaseBuffer getBufferAgain(int flags) {
+ // If only the request flags are correct for this type, we can re-use this buffer
+ checkRequestFlags(flags);
+ // Count another consumer of this
+ exports += 1;
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * When the final matching release occurs (that is the number of <code>release</code> calls
+ * equals the number of <code>getBuffer</code> calls), the implementation here calls
+ * {@link #releaseAction()}, which the implementer of a specific buffer type should override if
+ * it needs specific actions to take place.
+ */
+ @Override
+ public void release() {
+ if (--exports == 0) {
+ // This is a final release.
releaseAction();
+ } else if (exports < 0) {
+ // Buffer already had 0 exports. (Put this right, in passing.)
+ exports = 0;
+ throw bufferReleased("release");
}
- obj = null;
}
@Override
+ public boolean isReleased() {
+ return exports <= 0;
+ }
+
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int length) {
+ return getBufferSlice(flags, start, length, 1);
+ }
+
+ // Let the sub-class implement
+ // @Override public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {}
+
+ @Override
public BufferPointer getBuf() {
return buf;
}
- // Let the sub-class implement:
- // @Override public BufferPointer getPointer(int index) { return null; }
- // @Override public BufferPointer getPointer(int... indices) { return null; }
+ @Override
+ public BufferPointer getPointer(int index) {
+ return new BufferPointer(buf.storage, calcIndex(index), getItemsize());
+ }
+
+ @Override
+ public BufferPointer getPointer(int... indices) {
+ return new BufferPointer(buf.storage, calcIndex(indices), getItemsize());
+ }
@Override
public int[] getStrides() {
@@ -273,18 +541,20 @@
@Override
public int[] getSuboffsets() {
+ // No actual 'suboffsets' member until a sub-class needs it
return null;
}
@Override
public boolean isContiguous(char order) {
+ // Correct for one-dimensional buffers
return true;
}
@Override
public String getFormat() {
// Avoid having to have an actual 'format' member
- return ((capabilityFlags & FORMAT) == 0) ? null : "B";
+ return "B";
}
@Override
@@ -294,52 +564,132 @@
}
/**
- * Ensure buffer, if not released sooner, is released from the exporter during object
- * finalisation (before garbage collection) of the buffer object.
- */
- @Override
- protected void finalize() throws Throwable {
- release();
- super.finalize();
- }
-
- /**
- * This method will be called when the consumer calls {@link #release()} (to be precise, only on
- * the first call). The default implementation does nothing. Override this method to add release
- * behaviour specific to exporter. A common convention is to do this within the definition of
- * {@link BufferProtocol#getBuffer(int)} within the exporting class, where a nested class is
- * finally defined.
+ * This method will be called when the number of calls to {@link #release()} on this buffer is
+ * equal to the number of calls to {@link PyBuffer#getBuffer(int)} and to
+ * {@link BufferProtocol#getBuffer(int)} that returned this buffer. The default implementation
+ * does nothing. Override this method to add release behaviour specific to an exporter. A common
+ * convention is to do this within the definition of {@link BufferProtocol#getBuffer(int)}
+ * within the exporting class, where a nested class is ultimately defined.
*/
protected void releaseAction() {}
/**
+ * Some <code>PyBuffer</code>s, those created by slicing a <code>PyBuffer</code> are related to
+ * a root <code>PyBuffer</code>. During creation of such a slice, we need to supply a value for
+ * this root. If the present object is not itself a slice, this is root is the object itself; if
+ * the buffer is already a slice, it is the root it was given at creation time. Often this is
+ * the only difference between a slice-view and a directly-exported buffer. Override this method
+ * in slices to return the root buffer of the slice.
+ *
+ * @return this buffer (or the root buffer if this is a sliced view)
+ */
+ protected PyBuffer getRoot() {
+ return this;
+ }
+
+ /**
* Check the number of indices (but not their values), raising a Python BufferError if this does
- * not match the number of dimensions.
+ * not match the number of dimensions. This is a helper for N-dimensional arrays.
*
* @param indices into the buffer (to test)
* @return number of dimensions
* @throws PyException (BufferError) if wrong number of indices
*/
- final int checkDimension(int[] indices) throws PyException {
- int ndim = shape.length;
- if (indices.length != ndim) {
- if (indices.length < ndim) {
- throw Py.BufferError("too few indices supplied");
- } else {
- throw Py.BufferError("too many indices supplied");
- }
+ int checkDimension(int[] indices) throws PyException {
+ int n = indices.length;
+ checkDimension(n);
+ return n;
+ }
+
+ /**
+ * Check that the number offered is in fact the number of dimensions in this buffer, raising a
+ * Python BufferError if this does not match the number of dimensions. This is a helper for
+ * N-dimensional arrays.
+ *
+ * @param n number of dimensions being assumed by caller
+ * @throws PyException (BufferError) if wrong number of indices
+ */
+ void checkDimension(int n) throws PyException {
+ int ndim = getNdim();
+ if (n != ndim) {
+ String fmt = "buffer with %d dimension%s accessed as having %d dimension%s";
+ String msg = String.format(fmt, ndim, ndim == 1 ? "" : "s", n, n, n == 1 ? "" : "s");
+ throw Py.BufferError(msg);
}
- return ndim;
+ }
+
+ /**
+ * Check that the argument is within the buffer <code>buf</code>. An exception is raised if
+ * <code>i<buf.offset</code> or <code>i>buf.offset+buf.size-1</code>
+ *
+ * @param i index to check
+ * @throws IndexOutOfBoundsException if <code>i<buf.offset</code> or
+ * <code>i>buf.offset+buf.size-1</code>.
+ */
+ protected void checkInBuf(int i) throws IndexOutOfBoundsException {
+ int a = buf.offset;
+ int b = a + buf.size - 1;
+ // Check: b >= i >= a. Cheat.
+ if (((i - a) | (b - i)) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+
+ /**
+ * Check that the both arguments are within the buffer <code>buf</code>. An exception is raised
+ * if <code>i<buf.offset</code>, <code>j<buf.offset</code>,
+ * <code>i>buf.offset+buf.size-1</code>, or <code>j>buf.offset+buf.size-1</code>
+ *
+ * @param i index to check
+ * @param j index to check
+ * @throws IndexOutOfBoundsException if <code>i<buf.offset</code> or
+ * <code>i>buf.offset+buf.size-1</code>
+ */
+ protected void checkInBuf(int i, int j) throws IndexOutOfBoundsException {
+ int a = buf.offset;
+ int b = a + buf.size - 1;
+ // Check: b >= i >= a and b >= j >= a. Cheat.
+ if (((i - a) | (j - a) | (b - i) | (b - j)) < 0) {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+
+ /**
+ * General purpose method to construct an exception to throw according to the syndrome.
+ *
+ * @param syndrome of the mis-match between buffer and requested features
+ * @return PyException (BufferError) specifying the mis-match
+ */
+ private static PyException bufferErrorFromSyndrome(int syndrome) {
+
+ if ((syndrome & ND) != 0) {
+ return bufferRequires("shape");
+ } else if ((syndrome & STRIDES) != 0) {
+ return bufferRequires("strides");
+ } else if ((syndrome & INDIRECT) != 0) {
+ return bufferRequires("suboffsets");
+ } else if ((syndrome & WRITABLE) != 0) {
+ return bufferIsNot("writable");
+ } else if ((syndrome & C_CONTIGUOUS) != 0) {
+ return bufferIsNot("C-contiguous");
+ } else if ((syndrome & F_CONTIGUOUS) != 0) {
+ return bufferIsNot("Fortran-contiguous");
+ } else if ((syndrome & ANY_CONTIGUOUS) != 0) {
+ return bufferIsNot("contiguous");
+ } else {
+ // Catch-all error (never in practice if this method is complete)
+ return bufferIsNot("capable of matching request");
+ }
}
/**
* Convenience method to create (for the caller to throw) a
- * <code>BufferError("underlying buffer is not writable")</code>.
+ * <code>TypeError("cannot modify read-only memory")</code>.
*
* @return the error as a PyException
*/
- protected PyException notWritable() {
- return bufferIsNot("writable");
+ protected static PyException notWritable() {
+ return Py.TypeError("cannot modify read-only memory");
}
/**
@@ -349,19 +699,41 @@
* @param property
* @return the error as a PyException
*/
- protected PyException bufferIsNot(String property) {
+ protected static PyException bufferIsNot(String property) {
return Py.BufferError("underlying buffer is not " + property);
}
/**
* Convenience method to create (for the caller to throw) a
+ * <code>ValueError("buffer ... different structures")</code>.
+ *
+ * @return the error as a PyException
+ */
+ protected static PyException differentStructure() {
+ return Py.ValueError("buffer assignment: lvalue and rvalue have different structures");
+ }
+
+ /**
+ * Convenience method to create (for the caller to throw) a
* <code>BufferError("underlying buffer requires {feature}")</code>.
*
* @param feature
* @return the error as a PyException
*/
- protected PyException bufferRequires(String feature) {
+ protected static PyException bufferRequires(String feature) {
return Py.BufferError("underlying buffer requires " + feature);
}
+ /**
+ * Convenience method to create (for the caller to throw) a
+ * <code>BufferError("{operation} operation forbidden on released buffer object")</code>.
+ *
+ * @param operation name of operation or null
+ * @return the error as a PyException
+ */
+ protected static PyException bufferReleased(String operation) {
+ String op = (operation == null) ? "" : operation + " ";
+ return Py.BufferError(op + "operation forbidden on released buffer object");
+ }
+
}
diff --git a/src/org/python/core/buffer/SimpleBuffer.java b/src/org/python/core/buffer/SimpleBuffer.java
--- a/src/org/python/core/buffer/SimpleBuffer.java
+++ b/src/org/python/core/buffer/SimpleBuffer.java
@@ -1,66 +1,222 @@
package org.python.core.buffer;
import org.python.core.BufferPointer;
-import org.python.core.BufferProtocol;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
/**
- * Buffer API over a writable one-dimensional array of one-byte items.
+ * Buffer API over a read-only one-dimensional array of one-byte items.
*/
-public class SimpleBuffer extends SimpleReadonlyBuffer {
+public class SimpleBuffer extends BaseBuffer {
/**
- * <code>SimpleBuffer</code> allows consumer requests that are the same as
- * <code>SimpleReadonlyBuffer</code>, with the addition of WRITABLE.
+ * The strides array for this type is always a single element array with a 1 in it.
*/
- protected static final int ALLOWED_FLAGS = WRITABLE | SimpleReadonlyBuffer.ALLOWED_FLAGS;
+ protected static final int[] SIMPLE_STRIDES = {1};
/**
- * Provide an instance of <code>SimpleBuffer</code> in a default, semi-constructed state. The
- * sub-class constructor takes responsibility for completing construction with a call to
- * {@link #assignCapabilityFlags(int, int, int, int)}.
+ * Provide an instance of <code>SimpleBuffer</code> with navigation variables partly
+ * initialised, for sub-class use. One-dimensional arrays without slicing are C- and
+ * F-contiguous. To complete initialisation, the sub-class normally must assign: {@link #buf}
+ * and {@link #shape}[0], and call {@link #checkRequestFlags(int)} passing the consumer's
+ * request flags.
*
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
+ * <pre>
+ * this.buf = buf; // Wraps exported data
+ * this.shape[0] = n; // Number of units in exported data
+ * checkRequestFlags(flags); // Check request is compatible with type
+ * </pre>
*/
- protected SimpleBuffer(BufferProtocol exporter, BufferPointer buf) {
- super(exporter, buf);
+ protected SimpleBuffer() {
+ super(CONTIGUITY | SIMPLE);
+ // Initialise navigation
+ shape = new int[1];
+ strides = SIMPLE_STRIDES;
+ // suboffsets is always null for this type.
}
/**
- * Provide an instance of SimpleBuffer meeting the consumer's expectations as expressed in the
- * flags argument.
+ * Provide an instance of <code>SimpleBuffer</code>, on a slice of a byte array, meeting the
+ * consumer's expectations as expressed in the <code>flags</code> argument, which is checked
+ * against the capabilities of the buffer type.
*
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
* @param flags consumer requirements
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @param offset where the data starts in that array (item[0])
+ * @param size the number of bytes occupied
+ * @throws PyException (BufferError) when expectations do not correspond with the type
*/
- public SimpleBuffer(BufferProtocol exporter, BufferPointer buf, int flags) {
- super(exporter, buf);
- assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
- fillInfo();
+ public SimpleBuffer(int flags, byte[] storage, int offset, int size) throws PyException {
+ this();
+ // Wrap the exported data on a BufferPointer object
+ this.buf = new BufferPointer(storage, offset, size);
+ this.shape[0] = size; // Number of units in exported data
+ checkRequestFlags(flags); // Check request is compatible with type
+ }
+
+ /**
+ * Provide an instance of <code>SimpleBuffer</code>, on the entirety of a byte array, meeting
+ * the consumer's expectations as expressed in the <code>flags</code> argument, which is checked
+ * against the capabilities of the buffer type.
+ *
+ * @param flags consumer requirements
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SimpleBuffer(int flags, byte[] storage) throws PyException {
+ this(flags, storage, 0, storage.length);
}
@Override
public boolean isReadonly() {
- return false;
+ return true;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public byte byteAt(int index) throws IndexOutOfBoundsException {
+ // Implement directly: a bit quicker than the default
+ return buf.storage[buf.offset + index];
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public int intAt(int index) throws IndexOutOfBoundsException {
+ // Implement directly: a bit quicker than the default
+ return 0xff & buf.storage[buf.offset + index];
}
@Override
- public void storeAt(byte value, int index) {
- buf.storage[buf.offset + index] = value;
+ protected int calcIndex(int index) throws IndexOutOfBoundsException {
+ return buf.offset + index;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public byte byteAt(int... indices) throws IndexOutOfBoundsException {
+ checkDimension(indices.length);
+ return byteAt(indices[0]);
}
@Override
- public void storeAt(byte value, int... indices) {
- if (indices.length != 1) {
- checkDimension(indices);
- }
- storeAt(value, indices[0]);
+ protected int calcIndex(int... indices) throws IndexOutOfBoundsException {
+ // BaseBuffer implementation can be simplified since if indices.length!=1 we error.
+ checkDimension(indices.length); // throws if != 1
+ return calcIndex(indices[0]);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ throws IndexOutOfBoundsException {
+ System.arraycopy(buf.storage, buf.offset + srcIndex, dest, destPos, length);
}
@Override
- public void copyFrom(byte[] src, int srcPos, int destIndex, int length) {
- System.arraycopy(src, srcPos, buf.storage, buf.offset + destIndex, length);
+ public PyBuffer getBufferSlice(int flags, int start, int length) {
+ // Translate relative to underlying buffer
+ int compIndex0 = buf.offset + start;
+ // Check the arguments define a slice within this buffer
+ checkInBuf(compIndex0, compIndex0 + length - 1);
+ // Create the slice from the sub-range of the buffer
+ return new SimpleView(getRoot(), flags, buf.storage, compIndex0, length);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation for slicing contiguous bytes in one
+ * dimension. In that case, <i>x(i) = u(r+i)</i> for <i>i = 0..L-1</i> where u is the underlying
+ * buffer, and <i>r</i> and <i>L</i> are the start and length with which <i>x</i> was created
+ * from <i>u</i>. Thus <i>y(k) = u(r+s+km)</i>, that is, the composite offset is <i>r+s</i> and
+ * the stride is <i>m</i>.
+ */
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
+
+ if (stride == 1) {
+ // Unstrided slice of simple buffer is itself simple
+ return getBufferSlice(flags, start, length);
+
+ } else {
+ // Translate relative to underlying buffer
+ int compIndex0 = buf.offset + start;
+ // Check the slice sits within the present buffer (first and last indexes)
+ checkInBuf(compIndex0, compIndex0 + (length - 1) * stride);
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new Strided1DBuffer.SlicedView(getRoot(), flags, buf.storage, compIndex0,
+ length, stride);
+ }
+ }
+
+ @Override
+ public BufferPointer getPointer(int index) {
+ return new BufferPointer(buf.storage, buf.offset + index, 1);
+ }
+
+ @Override
+ public BufferPointer getPointer(int... indices) {
+ checkDimension(indices.length);
+ return getPointer(indices[0]);
+ }
+
+ /**
+ * A <code>SimpleBuffer.SimpleView</code> represents a contiguous subsequence of another
+ * <code>SimpleBuffer</code>.
+ */
+ static class SimpleView extends SimpleBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a SimpleBuffer.
+ *
+ * @param root buffer which will be acquired and must be released ultimately
+ * @param flags the request flags of the consumer that requested the slice
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @param offset where the data starts in that array (item[0])
+ * @param size the number of bytes occupied
+ */
+ public SimpleView(PyBuffer root, int flags, byte[] storage, int offset, int size) {
+ // Create a new SimpleBuffer on the buffer passed in (part of the root)
+ super(flags, storage, offset, size);
+ // Get a lease on the root PyBuffer
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void release() {
+ // We have to release both this and the root
+ super.release();
+ root.release();
+ }
+
}
}
diff --git a/src/org/python/core/buffer/SimpleReadonlyBuffer.java b/src/org/python/core/buffer/SimpleReadonlyBuffer.java
deleted file mode 100644
--- a/src/org/python/core/buffer/SimpleReadonlyBuffer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-package org.python.core.buffer;
-
-import org.python.core.BufferPointer;
-import org.python.core.BufferProtocol;
-
-/**
- * Buffer API over a one-dimensional array of one-byte items providing read-only API. A writable
- * simple buffer will extend this implementation.
- */
-public class SimpleReadonlyBuffer extends BaseBuffer {
-
- /**
- * Using the PyBUF constants, express capabilities the consumer must request if it is to
- * navigate the storage successfully. (None.)
- */
- public static final int REQUIRED_FLAGS = 0;
- /**
- * Using the PyBUF constants, express capabilities the consumer may request so it can navigate
- * the storage in its chosen way. The buffer instance has to implement these mechanisms if and
- * only if they are requested. (FORMAT | ND | STRIDES | INDIRECT)
- */
- public static final int ALLOWED_FLAGS = FORMAT | ND | STRIDES | INDIRECT;
- /**
- * Using the PyBUF constants, express capabilities the consumer doesn't need to request because
- * they will be there anyway. (One-dimensional arrays (including those sliced with step size
- * one) are C- and F-contiguous.)
- */
- public static final int IMPLIED_FLAGS = CONTIGUITY;
- /**
- * The strides array for this type is always a single element array with a 1 in it.
- */
- protected static final int[] SIMPLE_STRIDES = {1};
-
- /**
- * Partial counterpart to CPython <code>PyBuffer_FillInfo()</code> specific to the simple type
- * of buffer and called from the constructor. The base constructor will already have been
- * called, filling {@link #buf} and {@link #obj}. And the method
- * {@link #assignCapabilityFlags(int, int, int, int)} has set {@link #capabilityFlags}.
- */
- protected void fillInfo() {
- /*
- * We will already have called: assignCapabilityFlags(flags, requiredFlags, allowedFlags,
- * impliedFlags); So capabilityFlags holds the requests for shape, strides, writable, etc..
- */
- // Difference from CPython: never null, even when the consumer doesn't request it
- shape = new int[1];
- shape[0] = getLen();
-
- // Following CPython: provide strides only when the consumer requests it
- if ((capabilityFlags & STRIDES) == STRIDES) {
- strides = SIMPLE_STRIDES;
- }
-
- // Even when the consumer requests suboffsets, the exporter is allowed to supply null.
- // In theory, the exporter could require that it be requested and still supply null.
- }
-
- /**
- * Provide an instance of <code>SimpleReadonlyBuffer</code> in a default, semi-constructed
- * state. The sub-class constructor takes responsibility for completing construction including a
- * call to {@link #assignCapabilityFlags(int, int, int, int)}.
- *
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
- */
- protected SimpleReadonlyBuffer(BufferProtocol exporter, BufferPointer buf) {
- super(exporter, buf);
- }
-
- /**
- * Provide an instance of SimpleReadonlyBuffer meeting the consumer's expectations as expressed
- * in the flags argument.
- *
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
- * @param flags consumer requirements
- */
- public SimpleReadonlyBuffer(BufferProtocol exporter, BufferPointer buf, int flags) {
- super(exporter, buf);
- assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
- fillInfo();
- }
-
- @Override
- public int getNdim() {
- return 1;
- }
-
- @Override
- public byte byteAt(int index) throws IndexOutOfBoundsException {
- // offset is not necessarily zero
- return buf.storage[buf.offset + index];
- }
-
- @Override
- public int intAt(int index) throws IndexOutOfBoundsException {
- // Implement directly: a bit quicker than the default
- return 0xff & buf.storage[buf.offset + index];
- }
-
- @Override
- public byte byteAt(int... indices) throws IndexOutOfBoundsException {
- if (indices.length != 1) {
- checkDimension(indices);
- }
- return byteAt(indices[0]);
- }
-
- @Override
- public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
- throws IndexOutOfBoundsException {
- System.arraycopy(buf.storage, buf.offset + srcIndex, dest, destPos, length);
- }
-
- @Override
- public BufferPointer getPointer(int index) {
- return new BufferPointer(buf.storage, buf.offset + index, 1);
- }
-
- @Override
- public BufferPointer getPointer(int... indices) {
- if (indices.length != 1) {
- checkDimension(indices);
- }
- return getPointer(indices[0]);
- }
-
-}
diff --git a/src/org/python/core/buffer/SimpleStringBuffer.java b/src/org/python/core/buffer/SimpleStringBuffer.java
--- a/src/org/python/core/buffer/SimpleStringBuffer.java
+++ b/src/org/python/core/buffer/SimpleStringBuffer.java
@@ -1,7 +1,7 @@
package org.python.core.buffer;
import org.python.core.BufferPointer;
-import org.python.core.BufferProtocol;
+import org.python.core.PyBuffer;
import org.python.core.util.StringUtil;
/**
@@ -9,11 +9,11 @@
* but which is actually backed by a Java String. Some of the buffer API absolutely needs access to
* the data as a byte array (those parts that involve a {@link BufferPointer} result), and therefore
* this class must create a byte array from the String for them. However, it defers creation of a
- * byte array until that part of the API is actually used. This class overrides those methods in
- * SimpleReadonlyBuffer that would access the <code>buf</code> attribute to work out their results
- * from the String instead.
+ * byte array until that part of the API is actually used. Where possible, this class overrides
+ * those methods in SimpleBuffer that would otherwise access the byte array attribute to use the
+ * String instead.
*/
-public class SimpleStringBuffer extends SimpleReadonlyBuffer {
+public class SimpleStringBuffer extends SimpleBuffer {
/**
* The string backing this PyBuffer. A substitute for {@link #buf} until we can no longer avoid
@@ -22,44 +22,19 @@
private String bufString;
/**
- * Partial counterpart to CPython <code>PyBuffer_FillInfo()</code> specific to the simple type
- * of buffer and called from the constructor. The base constructor will already have been
- * called, filling {@link #bufString} and {@link #obj}. And the method
- * {@link #assignCapabilityFlags(int, int, int, int)} has set {@link #capabilityFlags}.
- */
- protected void fillInfo(String bufString) {
- /*
- * We will already have called: assignCapabilityFlags(flags, requiredFlags, allowedFlags,
- * impliedFlags); So capabilityFlags holds the requests for shape, strides, writable, etc..
- */
- // Save the backing string
- this.bufString = bufString;
-
- // Difference from CPython: never null, even when the consumer doesn't request it
- shape = new int[1];
- shape[0] = bufString.length();
-
- // Following CPython: provide strides only when the consumer requests it
- if ((capabilityFlags & STRIDES) == STRIDES) {
- strides = SIMPLE_STRIDES;
- }
-
- // Even when the consumer requests suboffsets, the exporter is allowed to supply null.
- // In theory, the exporter could require that it be requested and still supply null.
- }
-
- /**
- * Provide an instance of SimpleReadonlyBuffer meeting the consumer's expectations as expressed
- * in the flags argument.
+ * Provide an instance of SimpleStringBuffer meeting the consumer's expectations as expressed in
+ * the flags argument.
*
- * @param exporter the exporting object
* @param bufString storing the implementation of the object
* @param flags consumer requirements
*/
- public SimpleStringBuffer(BufferProtocol exporter, String bufString, int flags) {
- super(exporter, null);
- assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
- fillInfo(bufString);
+ public SimpleStringBuffer(int flags, String bufString) {
+ super();
+ // Save the backing string
+ this.bufString = bufString;
+ shape[0] = bufString.length();
+ // Check request is compatible with type
+ checkRequestFlags(flags);
}
/**
@@ -113,6 +88,33 @@
/**
* {@inheritDoc}
* <p>
+ * The <code>SimpleStringBuffer</code> implementation avoids creation of a byte buffer.
+ */
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int length) {
+ // The new string content is just a sub-string. (Non-copy operation in Java.)
+ return new SimpleStringView(getRoot(), flags, bufString.substring(start, start + length));
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The <code>SimpleStringBuffer</code> implementation creates an actual byte buffer.
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
+ if (stride == 1) {
+ // Unstrided slice of simple buffer is itself simple
+ return getBufferSlice(flags, start, length);
+ } else {
+ // Force creation of the actual byte buffer be a SimpleBuffer
+ getBuf();
+ return super.getBufferSlice(flags, start, length, stride);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
* This method creates an actual byte buffer from the String if none yet exists.
*/
@Override
@@ -146,4 +148,40 @@
return super.getPointer(indices);
}
+ /**
+ * A <code>SimpleStringBuffer.SimpleStringView</code> represents a contiguous subsequence of
+ * another <code>SimpleStringBuffer</code>.
+ */
+ static class SimpleStringView extends SimpleStringBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a SimpleStringBuffer.
+ *
+ * @param root buffer which will be acquired and must be released ultimately
+ * @param flags the request flags of the consumer that requested the slice
+ * @param buf becomes the buffer of bytes for this object
+ */
+ public SimpleStringView(PyBuffer root, int flags, String bufString) {
+ // Create a new SimpleStringBuffer on the string passed in
+ super(flags, bufString);
+ // Get a lease on the root PyBuffer
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void release() {
+ // We have to release both this and the root
+ super.release();
+ root.release();
+ }
+
+ }
}
diff --git a/src/org/python/core/buffer/SimpleBuffer.java b/src/org/python/core/buffer/SimpleWritableBuffer.java
copy from src/org/python/core/buffer/SimpleBuffer.java
copy to src/org/python/core/buffer/SimpleWritableBuffer.java
--- a/src/org/python/core/buffer/SimpleBuffer.java
+++ b/src/org/python/core/buffer/SimpleWritableBuffer.java
@@ -1,43 +1,44 @@
package org.python.core.buffer;
import org.python.core.BufferPointer;
-import org.python.core.BufferProtocol;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
/**
* Buffer API over a writable one-dimensional array of one-byte items.
*/
-public class SimpleBuffer extends SimpleReadonlyBuffer {
+public class SimpleWritableBuffer extends SimpleBuffer {
/**
- * <code>SimpleBuffer</code> allows consumer requests that are the same as
- * <code>SimpleReadonlyBuffer</code>, with the addition of WRITABLE.
+ * Provide an instance of <code>SimpleWritableBuffer</code>, on a slice of a byte array, meeting the consumer's expectations
+ * as expressed in the <code>flags</code> argument, which is checked against the capabilities of
+ * the buffer type.
+ *
+ * @param flags consumer requirements
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @param offset where the data starts in that array (item[0])
+ * @param size the number of bytes occupied
+ * @throws PyException (BufferError) when expectations do not correspond with the type
*/
- protected static final int ALLOWED_FLAGS = WRITABLE | SimpleReadonlyBuffer.ALLOWED_FLAGS;
-
- /**
- * Provide an instance of <code>SimpleBuffer</code> in a default, semi-constructed state. The
- * sub-class constructor takes responsibility for completing construction with a call to
- * {@link #assignCapabilityFlags(int, int, int, int)}.
- *
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
- */
- protected SimpleBuffer(BufferProtocol exporter, BufferPointer buf) {
- super(exporter, buf);
+ public SimpleWritableBuffer(int flags, byte[] storage, int offset, int size) throws PyException {
+ addFeatureFlags(WRITABLE);
+ // Wrap the exported data on a BufferPointer object
+ this.buf = new BufferPointer(storage, offset, size);
+ this.shape[0] = size; // Number of units in exported data
+ checkRequestFlags(flags); // Check request is compatible with type
}
/**
- * Provide an instance of SimpleBuffer meeting the consumer's expectations as expressed in the
- * flags argument.
+ * Provide an instance of <code>SimpleWritableBuffer</code>, on the entirety of a byte array, meeting the consumer's expectations
+ * as expressed in the <code>flags</code> argument, which is checked against the capabilities of
+ * the buffer type.
*
- * @param exporter the exporting object
- * @param buf wrapping the array of bytes storing the implementation of the object
* @param flags consumer requirements
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @throws PyException (BufferError) when expectations do not correspond with the type
*/
- public SimpleBuffer(BufferProtocol exporter, BufferPointer buf, int flags) {
- super(exporter, buf);
- assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
- fillInfo();
+ public SimpleWritableBuffer(int flags, byte[] storage) throws PyException {
+ this(flags, storage, 0, storage.length);
}
@Override
@@ -45,22 +46,134 @@
return false;
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
@Override
public void storeAt(byte value, int index) {
+ // Implement directly and don't ask whether read-only
buf.storage[buf.offset + index] = value;
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
@Override
public void storeAt(byte value, int... indices) {
- if (indices.length != 1) {
- checkDimension(indices);
- }
+ checkDimension(indices.length);
storeAt(value, indices[0]);
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
@Override
public void copyFrom(byte[] src, int srcPos, int destIndex, int length) {
System.arraycopy(src, srcPos, buf.storage, buf.offset + destIndex, length);
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public void copyFrom(PyBuffer src) throws IndexOutOfBoundsException, PyException {
+
+ if (src.getLen() != buf.size) {
+ throw differentStructure();
+ }
+
+ // Get the source to deliver efficiently to our byte storage
+ src.copyTo(buf.storage, buf.offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleWritableBuffer</code> provides an implementation ensuring the returned slice is
+ * writable.
+ */
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int length) {
+ // Translate relative to underlying buffer
+ int compIndex0 = buf.offset + start;
+ // Check the arguments define a slice within this buffer
+ checkInBuf(compIndex0, compIndex0 + length - 1);
+ // Create the slice from the sub-range of the buffer
+ return new SimpleView(getRoot(), flags, buf.storage, compIndex0, length);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleWritableBuffer</code> provides an implementation ensuring the returned slice is
+ * writable.
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
+
+ if (stride == 1) {
+ // Unstrided slice of simple buffer is itself simple
+ return getBufferSlice(flags, start, length);
+
+ } else {
+ // Translate relative to underlying buffer
+ int compIndex0 = buf.offset + start;
+ // Check the slice sits within the present buffer (first and last indexes)
+ checkInBuf(compIndex0, compIndex0 + (length - 1) * stride);
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new Strided1DWritableBuffer.SlicedView(getRoot(), flags, buf.storage,
+ compIndex0, length, stride);
+ }
+ }
+
+ /**
+ * A <code>SimpleWritableBuffer.SimpleView</code> represents a contiguous subsequence of
+ * another <code>SimpleWritableBuffer</code>.
+ */
+ static class SimpleView extends SimpleWritableBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a SimpleBuffer.
+ *
+ * @param root buffer which will be acquired and must be released ultimately
+ * @param flags the request flags of the consumer that requested the slice
+ * @param storage the array of bytes storing the implementation of the exporting object
+ * @param offset where the data starts in that array (item[0])
+ * @param size the number of bytes occupied
+ */
+ public SimpleView(PyBuffer root, int flags, byte[] storage, int offset, int size) {
+ // Create a new SimpleBuffer on the buffer passed in (part of the root)
+ super(flags, storage, offset, size);
+ // Get a lease on the root PyBuffer
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void release() {
+ // We have to release both this and the root
+ super.release();
+ root.release();
+ }
+
+ }
+
}
diff --git a/src/org/python/core/buffer/Strided1DBuffer.java b/src/org/python/core/buffer/Strided1DBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/Strided1DBuffer.java
@@ -0,0 +1,250 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+/**
+ * Read-only buffer API over a one-dimensional array of one-byte items, that are evenly-spaced in a
+ * storage array. The buffer has a <code>buf</code> property in the usual way, designating a slice
+ * (or all) of a byte array, but also a <code>stride</code> property (equal to
+ * <code>getStrides()[0]</code>).
+ * <p>
+ * Let this underlying buffer be the byte array <i>u(i)</i> for <i>i=a..a+N</i>, let <i>x</i> be the
+ * <code>Strided1DBuffer</code>, and let the stride be <i>p</i>. The storage works as follows.
+ * Designate by <i>x(j)</i>, for <i>j=0..L-1</i>, the byte at index <i>j</i>, that is, the byte
+ * retrieved by <code>x.byteAt(j)</code>. Then,
+ * <ul>
+ * <li>when <i>p>0</i>, we store <i>x(j)</i> at <i>u(a+pj)</i>, that is, <i>x(0)</i> is at
+ * <i>u(a)</i> and the byte array slice size should be <i>N = (L-1)p+1</i>.</li>
+ * <li>when <i>p<0</i>, we store <i>x(j)</i> at <i>u((a+N-1)+pj)</i>, that is, <i>x(0)</i> is at
+ * <i>u(a+N-1)</i>, and the byte array slice size should be <i>N = (L-1)(-p)+1</i>.</li>
+ * <li><i>p=0</i> is not a useful stride.</li>
+ * </ul>
+ * <p>
+ * The class may be used by exporters to create a strided slice (e.g. to export the diagonal of a
+ * matrix) and in particular by other buffers to create strided slices of themselves, such as to
+ * create the memoryview that is returned as an extended slice of a memoryview.
+ */
+public class Strided1DBuffer extends BaseBuffer {
+
+ /**
+ * Step size in the underlying buffer essential to correct translation of an index (or indices)
+ * into an index into the storage. The value is returned by {@link #getStrides()} is an array
+ * with this as the only element.
+ */
+ protected int stride;
+
+ /**
+ * Absolute index in <code>buf.storage</code> of <code>item[0]</code>. For a positive
+ * <code>stride</code> this is equal to <code>buf.offset</code>, and for a negative
+ * <code>stride</code> it is <code>buf.offset+buf.size-1</code>. It has to be used in most of
+ * the places that buf.offset would appear in the index calculations of simpler buffers (that
+ * have unit stride).
+ */
+ protected int index0;
+
+ /**
+ * Provide an instance of <code>Strided1DBuffer</code> with navigation variables partly
+ * initialised, for sub-class use. To complete initialisation, the sub-class normally must
+ * assign: {@link #buf}, {@link #shape}[0], and {@link #stride}, and call
+ * {@link #checkRequestFlags(int)} passing the consumer's request flags.
+ *
+ * <pre>
+ * this.buf = buf; // Wraps exported data
+ * setStride(stride); // Stride, shape[0] and index0 all set consistently
+ * checkRequestFlags(flags); // Check request is compatible with type
+ * </pre>
+ *
+ * The pre-defined {@link #strides} field remains <code>null</code> until {@link #getStrides} is
+ * called.
+ */
+ protected Strided1DBuffer() {
+ super(STRIDES);
+ // Initialise navigation
+ shape = new int[1];
+ // strides is created on demand;
+ // suboffsets is always null for this type.
+ }
+
+ /**
+ * Provide an instance of <code>Strided1DBuffer</code> on a particular array of bytes specifying
+ * a starting index, the number of items in the result, and a byte-indexing stride. The result
+ * of <code>byteAt(i)</code> will be equal to <code>storage[index0+stride*i]</code> (whatever
+ * the sign of <code>stride>0</code>), valid for <code>0<=i<length</code>.
+ * <p>
+ * The constructed <code>PyBuffer</code> meets the consumer's expectations as expressed in the
+ * <code>flags</code> argument, or an exception will be thrown if these are incompatible with
+ * the type (e.g. the consumer does not specify that it understands the strides array). Note
+ * that the actual range in the <code>storage</code> array, the lowest and highest index, is not
+ * explicitly passed, but is implicit in <code>index0</code>, <code>length</code> and
+ * <code>stride</code>. The caller is responsible for checking these fall within the array, or
+ * the sub-range the caller is allowed to use.
+ *
+ * @param flags consumer requirements
+ * @param storage raw byte array containing exported data
+ * @param index0 index into storage of item[0]
+ * @param length number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public Strided1DBuffer(int flags, byte[] storage, int index0, int length, int stride)
+ throws PyException {
+
+ // Arguments programme the object directly
+ this();
+ this.shape[0] = length;
+ this.index0 = index0;
+ this.stride = stride;
+
+ // Calculate buffer offset and size: start with distance of last item from first
+ int d = (length - 1) * stride;
+
+ if (stride >= 0) {
+ // Positive stride: indexing runs from first item
+ this.buf = new BufferPointer(storage, index0, 1 + d);
+ if (stride <= 1) {
+ // Really this is a simple buffer
+ addFeatureFlags(CONTIGUITY);
+ }
+ } else {
+ // Negative stride: indexing runs from last item
+ this.buf = new BufferPointer(storage, index0 + d, 1 - d);
+ }
+
+ checkRequestFlags(flags); // Check request is compatible with type
+ }
+
+ @Override
+ public boolean isReadonly() {
+ return true;
+ }
+
+ @Override
+ public byte byteAt(int index) throws IndexOutOfBoundsException {
+ return buf.storage[index0 + index * stride];
+ }
+
+ @Override
+ protected int calcIndex(int index) throws IndexOutOfBoundsException {
+ return index0 + index * stride;
+ }
+
+ @Override
+ protected int calcIndex(int... indices) throws IndexOutOfBoundsException {
+ // BaseBuffer implementation can be simplified since if indices.length!=1 we error.
+ checkDimension(indices.length); // throws if != 1
+ return calcIndex(indices[0]);
+ }
+
+ /**
+ * {@inheritDoc} <code>Strided1DBuffer</code> provides a version optimised for strided bytes in
+ * one dimension.
+ */
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ throws IndexOutOfBoundsException {
+ // Data is here in the buffers
+ int s = index0 + srcIndex * stride;
+ int d = destPos;
+
+ // Strategy depends on whether items are laid end-to-end contiguously or there are gaps
+ if (stride == 1) {
+ // stride == itemsize: straight copy of contiguous bytes
+ System.arraycopy(buf.storage, s, dest, d, length);
+
+ } else {
+ // Discontiguous copy: single byte items
+ int limit = s + length * stride;
+ for (; s != limit; s += stride) {
+ dest[d++] = buf.storage[s];
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>Strided1DBuffer</code> provides an implementation for slicing already-strided bytes in
+ * one dimension. In that case, <i>x(i) = u(r+ip)</i> for <i>i = 0..L-1</i> where u is the
+ * underlying buffer, and <i>r</i>, <i>p</i> and <i>L</i> are the start, stride and length with
+ * which <i>x</i> was created from <i>u</i>. Thus <i>y(k) = u(r+sp+kmp)</i>, that is, the
+ * composite offset is <i>r+sp</i> and the composite stride is <i>mp</i>.
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
+
+ // Translate relative to underlying buffer
+ int compStride = this.stride * stride;
+ int compIndex0 = index0 + start * stride;
+
+ // Check the slice sits within the present buffer (first and last indexes)
+ checkInBuf(compIndex0, compIndex0 + (length - 1) * compStride);
+
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new SlicedView(getRoot(), flags, buf.storage, compIndex0, length, compStride);
+ }
+
+ @Override
+ public BufferPointer getPointer(int index) {
+ return new BufferPointer(buf.storage, index0 + index, 1);
+ }
+
+ @Override
+ public BufferPointer getPointer(int... indices) {
+ // BaseBuffer implementation can be simplified since if indices.length!=1 we error.
+ checkDimension(indices.length);
+ return getPointer(indices[0]);
+ }
+
+ @Override
+ public int[] getStrides() {
+ if (strides == null) {
+ strides = new int[1];
+ strides[0] = stride;
+ }
+ return strides;
+ }
+
+ /**
+ * A <code>Strided1DBuffer.SlicedView</code> represents a discontiguous subsequence of a simple
+ * buffer.
+ */
+ static class SlicedView extends Strided1DBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a one-dimensional byte buffer.
+ *
+ * @param root on which release must be called when this is released
+ * @param flags consumer requirements
+ * @param storage raw byte array containing exported data
+ * @param index0 index into storage of item[0]
+ * @param len number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SlicedView(PyBuffer root, int flags, byte[] storage, int index0, int len, int stride)
+ throws PyException {
+ // Create a new on the buffer passed in (part of the root)
+ super(flags, storage, index0, len, stride);
+ // Get a lease on the root PyBuffer (read-only)
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void release() {
+ // We have to release both this and the root
+ super.release();
+ root.release();
+ }
+
+ }
+
+}
diff --git a/src/org/python/core/buffer/Strided1DWritableBuffer.java b/src/org/python/core/buffer/Strided1DWritableBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/Strided1DWritableBuffer.java
@@ -0,0 +1,159 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+/**
+ * Read-write buffer API over a one-dimensional array of one-byte items, that are evenly-spaced in a
+ * storage array. The storage conventions are described in {@link Strided1DBuffer} of which this is
+ * an extension providing write operations and a writable slice.
+ */
+public class Strided1DWritableBuffer extends Strided1DBuffer {
+
+ /**
+ * Provide an instance of <code>Strided1DWritableBuffer</code> on a particular array of bytes
+ * specifying a starting index, the number of items in the result, and a byte-indexing stride.
+ * The result of <code>byteAt(i)</code> will be equal to <code>storage[index0+stride*i]</code>
+ * (whatever the sign of <code>stride>0</code>), valid for <code>0<=i<length</code>.
+ * <p>
+ * The constructed <code>PyBuffer</code> meets the consumer's expectations as expressed in the
+ * <code>flags</code> argument, or an exception will be thrown if these are incompatible with
+ * the type (e.g. the consumer does not specify that it understands the strides array). Note
+ * that the actual range in the <code>storage</code> array, the lowest and highest index, is not
+ * explicitly passed, but is implicit in <code>index0</code>, <code>length</code> and
+ * <code>stride</code>. The caller is responsible for checking these fall within the array, or
+ * the sub-range the caller is allowed to use.
+ *
+ * @param flags consumer requirements
+ * @param storage raw byte array containing exported data
+ * @param index0 index into storage of item[0]
+ * @param length number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public Strided1DWritableBuffer(int flags, byte[] storage, int index0, int length, int stride)
+ throws PyException {
+
+ // Arguments programme the object directly
+ // this();
+ this.shape[0] = length;
+ this.index0 = index0;
+ this.stride = stride;
+
+ // Calculate buffer offset and size: start with distance of last item from first
+ int d = (length - 1) * stride;
+
+ if (stride >= 0) {
+ // Positive stride: indexing runs from first item
+ this.buf = new BufferPointer(storage, index0, 1 + d);
+ if (stride <= 1) {
+ // Really this is a simple buffer
+ addFeatureFlags(CONTIGUITY);
+ }
+ } else {
+ // Negative stride: indexing runs from last item
+ this.buf = new BufferPointer(storage, index0 + d, 1 - d);
+ }
+
+ checkRequestFlags(flags); // Check request is compatible with type
+ }
+
+ @Override
+ public boolean isReadonly() {
+ return false;
+ }
+
+ @Override
+ public void storeAt(byte value, int index) throws IndexOutOfBoundsException, PyException {
+ buf.storage[index0 + index * stride] = value;
+ }
+
+ /**
+ * {@inheritDoc} <code>Strided1DWritableBuffer</code> provides a version optimised for strided
+ * bytes in one dimension.
+ */
+ @Override
+ public void copyFrom(byte[] src, int srcPos, int destIndex, int length)
+ throws IndexOutOfBoundsException, PyException {
+
+ // Data is here in the buffers
+ int s = srcPos;
+ int d = index0 + destIndex * stride;
+
+ // Strategy depends on whether items are laid end-to-end or there are gaps
+ if (stride == 1) {
+ // Straight copy of contiguous bytes
+ System.arraycopy(src, srcPos, buf.storage, d, length);
+
+ } else {
+ // Discontiguous copy: single byte items
+ int limit = d + length * stride;
+ for (; d != limit; d += stride) {
+ buf.storage[d] = src[s++];
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>Strided1DWritableBuffer</code> provides an implementation that returns a writable
+ * slice.
+ */
+ public PyBuffer getBufferSlice(int flags, int start, int length, int stride) {
+
+ // Translate relative to underlying buffer
+ int compStride = this.stride * stride;
+ int compIndex0 = index0 + start * stride;
+
+ // Check the slice sits within the present buffer (first and last indexes)
+ checkInBuf(compIndex0, compIndex0 + (length - 1) * compStride);
+
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new SlicedView(getRoot(), flags, buf.storage, compIndex0, length, compStride);
+ }
+
+ /**
+ * A <code>Strided1DWritableBuffer.SlicedView</code> represents a discontiguous subsequence of a
+ * simple buffer.
+ */
+ static class SlicedView extends Strided1DWritableBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a one-dimensional byte buffer.
+ *
+ * @param root on which release must be called when this is released
+ * @param flags consumer requirements
+ * @param storage raw byte array containing exported data
+ * @param index0 index into storage of item[0]
+ * @param len number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SlicedView(PyBuffer root, int flags, byte[] storage, int index0, int len, int stride)
+ throws PyException {
+ // Create a new on the buffer passed in (part of the root)
+ super(flags, storage, index0, len, stride);
+ // Get a lease on the root PyBuffer (writable)
+ this.root = root.getBuffer(FULL);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void release() {
+ // We have to release both this and the root
+ super.release();
+ root.release();
+ }
+
+ }
+
+}
diff --git a/tests/java/org/python/core/BaseBytesTest.java b/tests/java/org/python/core/BaseBytesTest.java
--- a/tests/java/org/python/core/BaseBytesTest.java
+++ b/tests/java/org/python/core/BaseBytesTest.java
@@ -794,7 +794,7 @@
@Override
public PyBuffer getBuffer(int flags) {
- return new SimpleBuffer(this, new BufferPointer(store), flags);
+ return new SimpleBuffer(flags, store);
}
}
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -1,5 +1,8 @@
package org.python.core;
+import java.lang.ref.Reference;
+import java.lang.ref.SoftReference;
+import java.lang.ref.WeakReference;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
@@ -7,8 +10,8 @@
import junit.framework.TestCase;
import org.python.core.buffer.SimpleBuffer;
-import org.python.core.buffer.SimpleReadonlyBuffer;
import org.python.core.buffer.SimpleStringBuffer;
+import org.python.core.buffer.SimpleWritableBuffer;
import org.python.util.PythonInterpreter;
/**
@@ -63,10 +66,10 @@
interp = new PythonInterpreter();
// Tests using local examples
- queueWrite(new SimpleExporter(abcMaterial.getBytes()), abcMaterial);
- queueReadonly(new SimpleExporter(byteMaterial.getBytes(), true), byteMaterial);
+ queueWrite(new SimpleWritableExporter(abcMaterial.getBytes()), abcMaterial);
+ queueReadonly(new SimpleExporter(byteMaterial.getBytes()), byteMaterial);
queueReadonly(new StringExporter(stringMaterial.string), stringMaterial);
- queueWrite(new SimpleExporter(emptyMaterial.getBytes()), emptyMaterial);
+ queueWrite(new SimpleWritableExporter(emptyMaterial.getBytes()), emptyMaterial);
// Tests with PyByteArray
queueWrite(new PyByteArray(abcMaterial.getBytes()), abcMaterial);
@@ -108,15 +111,16 @@
private int[] validFlags = {PyBUF.SIMPLE, PyBUF.ND, PyBUF.STRIDES, PyBUF.INDIRECT};
/** To which we can add any of these (in one dimension, anyway) */
- private int[] validTassles = {PyBUF.FORMAT,
+ private int[] validTassles = {0,
+ PyBUF.FORMAT,
PyBUF.C_CONTIGUOUS,
PyBUF.F_CONTIGUOUS,
PyBUF.ANY_CONTIGUOUS};
/**
- * Test method for {@link org.python.core.PyBuffer#getBuf()}.
+ * Test method for {@link org.python.core.BufferProtocol#getBuffer()}.
*/
- public void testGetBuffer() {
+ public void testExporterGetBuffer() {
for (BufferTestPair test : buffersToRead) {
System.out.println("getBuffer(): " + test);
@@ -451,9 +455,9 @@
// A variety of lengths from zero to (n-destIndex)-ish
for (int length = 0; destIndex + length <= n; length = 2 * length + 1) {
- System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
- srcPos + length, n, destIndex, destIndex + length,
- actual.length);
+ // System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
+ // srcPos + length, n, destIndex, destIndex + length,
+ // actual.length);
// Initialise the object (have to do each time) and expected value
for (int i = 0; i < n; i++) {
@@ -475,9 +479,10 @@
// And from exactly n-destIndex down to zero-ish
for (int trim = 0; destIndex + trim <= n; trim = 2 * trim + 1) {
int length = n - destIndex - trim;
- System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
- srcPos + length, n, destIndex, destIndex + length,
- actual.length);
+
+ // System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
+ // srcPos + length, n, destIndex, destIndex + length,
+ // actual.length);
// Initialise the object (have to do each time) and expected value
for (int i = 0; i < n; i++) {
@@ -581,27 +586,43 @@
for (BufferTestPair test : buffersToRead) {
System.out.println("release: " + test);
BufferProtocol obj = test.exporter;
- // The object should already be exporting test.simple and test.strided
- PyBuffer a = test.simple; // 1
- PyBuffer b = test.strided; // 2
- PyBuffer c = obj.getBuffer(PyBUF.SIMPLE | PyBUF.FORMAT); // 3
+
+ // The object should already be exporting test.simple and test.strided = 2 exports
+ PyBuffer a = test.simple; // 1 exports
+ PyBuffer b = test.strided; // 2 exports
+ PyBuffer c = obj.getBuffer(PyBUF.SIMPLE | PyBUF.FORMAT); // = 3 exports
checkExporting(obj);
- // Multiple releases of the same buffer are just one release
- b.release(); // 2
- b.release();
- b.release();
- b.release();
+
+ // Now see that releasing in some other order works correctly
+ b.release(); // = 2 exports
+ a.release(); // = 1 export
checkExporting(obj);
- // Now see that releasing in some other order works correctly
- a.release(); // 1
+ int flags = PyBUF.STRIDES | PyBUF.FORMAT;
+
+ // You can get a buffer from a buffer (for SimpleExporter only c is alive)
+ PyBuffer d = c.getBuffer(flags); // = 2 exports
+ c.release(); // = 1 export
checkExporting(obj);
- PyBuffer d = obj.getBuffer(PyBUF.STRIDES | PyBUF.FORMAT); // 2
- c.release(); // 1
- checkExporting(obj);
- d.release(); // 0
+ d.release(); // = 0 exports
checkNotExporting(obj);
- d.release(); // 0
- checkNotExporting(obj);
+
+ // But fails if buffer has been finally released
+ try {
+ a = d.getBuffer(flags); // = 0 exports (since disallowed)
+ fail("getBuffer after final release not detected");
+ } catch (Exception e) {
+ // Detected *and* prevented?
+ checkNotExporting(obj);
+ }
+
+ // Further releases are also an error
+ try {
+ a.release(); // = -1 exports (oops)
+ fail("excess release not detected");
+ } catch (Exception e) {
+ // Success
+ }
+
}
}
@@ -611,12 +632,12 @@
* @param exporter
*/
private void checkExporting(BufferProtocol exporter) {
- if (exporter instanceof SimpleExporter) {
- assertTrue("exports not being counted", ((SimpleExporter)exporter).exportCount >= 1);
+ if (exporter instanceof TestableExporter) {
+ assertTrue("exports not being counted", ((TestableExporter)exporter).isExporting());
} else if (exporter instanceof PyByteArray) {
// Size-changing access should fail
try {
- ((PyByteArray)exporter).bytearray_extend(Py.One);
+ ((PyByteArray)exporter).bytearray_extend(Py.One); // Appends one zero byte
fail("bytearray_extend with exports should fail");
} catch (Exception e) {
// Success
@@ -631,8 +652,8 @@
* @param exporter
*/
private void checkNotExporting(BufferProtocol exporter) {
- if (exporter instanceof SimpleExporter) {
- assertFalse("exports falsely counted", ((SimpleExporter)exporter).exportCount >= 1);
+ if (exporter instanceof TestableExporter) {
+ assertFalse("exports falsely counted", ((TestableExporter)exporter).isExporting());
} else if (exporter instanceof PyByteArray) {
// Size-changing access should fail
try {
@@ -645,16 +666,37 @@
}
/**
+ * Check that reusable PyBuffer is re-used, and that non-reusable PyBuffer is not re-used.
+ *
+ * @param exporter
+ */
+ private void checkReusable(BufferProtocol exporter, PyBuffer previous, PyBuffer latest) {
+ assertNotNull("Re-used PyBuffer reference null", latest);
+ if (exporter instanceof PyByteArray) {
+ // Re-use prohibited because might have resized while released
+ assertFalse("PyByteArray buffer reused unexpectedly", latest == previous);
+ } else if (exporter instanceof TestableExporter && !((TestableExporter)exporter).reusable) {
+ // Special test case where re-use prohibited
+ assertFalse("PyBuffer reused unexpectedly", latest == previous);
+ } else {
+ // Other types of TestableExporter and PyString all re-use
+ assertTrue("PyBuffer not re-used as expected", latest == previous);
+ }
+ }
+
+ /**
* Test method for {@link org.python.core.PyBUF#getStrides()}.
*/
public void testGetStrides() {
for (BufferTestPair test : buffersToRead) {
System.out.println("getStrides: " + test);
- // When not requested ...
- assertNull(test.simple.getStrides());
- // When requested, ought to be as expected
- int[] strides = test.strided.getStrides();
- assertNotNull(strides);
+ // When not requested ... (different from CPython)
+ int[] strides = test.simple.getStrides();
+ assertNotNull("strides[] should always be provided", strides);
+ assertIntsEqual("simple.strides", test.strides, strides);
+ // And when requested, ought to be as expected
+ strides = test.strided.getStrides();
+ assertNotNull("strides[] not provided when requested", strides);
assertIntsEqual("strided.strides", test.strides, strides);
}
}
@@ -678,14 +720,17 @@
for (BufferTestPair test : buffersToRead) {
System.out.println("isContiguous: " + test);
// True for all test material and orders (since 1-dimensional)
- for (char order : validOrders) {
- assertTrue(test.simple.isContiguous(order));
- assertTrue(test.strided.isContiguous(order));
+ for (String orderMsg : validOrders) {
+ char order = orderMsg.charAt(0);
+ assertTrue(orderMsg, test.simple.isContiguous(order));
+ assertTrue(orderMsg, test.strided.isContiguous(order));
}
}
}
- private static final char[] validOrders = {'C', 'F', 'A'};
+ private static final String[] validOrders = {"C-contiguous test fail",
+ "F-contiguous test fail",
+ "Any-contiguous test fail"};
/**
* Test method for {@link org.python.core.PyBuffer#getFormat()}.
@@ -693,10 +738,10 @@
public void testGetFormat() {
for (BufferTestPair test : buffersToRead) {
System.out.println("getFormat: " + test);
- // Null for all test material
- assertNull(test.simple.getFormat());
- assertNull(test.strided.getFormat());
- // However, we can ask for it explicitly ...
+ // When not requested ... (different from CPython)
+ assertNotNull("format should always be provided", test.simple.getFormat());
+ assertNotNull("format should always be provided", test.strided.getFormat());
+ // And, we can ask for it explicitly ...
PyBuffer simpleWithFormat = test.exporter.getBuffer(PyBUF.SIMPLE | PyBUF.FORMAT);
PyBuffer stridedWithFormat = test.exporter.getBuffer(PyBUF.STRIDES | PyBUF.FORMAT);
// "B" for all test material where requested in flags
@@ -718,17 +763,19 @@
}
/**
- * A class to act as an exporter that uses the SimpleBuffer (or SimpleReadonlyBuffer). This
- * permits testing abstracted from the Jython interpreter.
+ * A class to act as an exporter that uses the SimpleReadonlyBuffer. This permits testing
+ * abstracted from the Jython interpreter.
+ * <p>
+ * The exporter exports a new PyBuffer object to each consumer (although each references the
+ * same internal storage) and it does not track their fate. You are most likely to use this
+ * approach with an exporting object that is immutable (or at least fixed in size).
*/
static class SimpleExporter implements BufferProtocol {
- byte[] storage;
- int exportCount;
- boolean readonly;
+ protected byte[] storage;
/**
- * Construct a simple exporter from the bytes supplied.
+ * Construct a simple read only exporter from the bytes supplied.
*
* @param storage
*/
@@ -736,47 +783,78 @@
this.storage = storage;
}
- /**
- * Construct a simple exporter from the bytes supplied, optionally read-only.
- *
- * @param storage
- * @param readonly
- */
- public SimpleExporter(byte[] storage, boolean readonly) {
- this.storage = storage;
- this.readonly = readonly;
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ return new SimpleBuffer(flags, storage);
}
- @Override
- public PyBuffer getBuffer(int flags) {
- BufferPointer mb = new BufferPointer(storage);
- exportCount++;
- if (readonly) {
- return new SimpleReadonlyBuffer(this, mb, flags) {
+ }
- protected void releaseAction() {
- --exportCount;
- }
- };
- } else {
- return new SimpleBuffer(this, mb, flags) {
+ /**
+ * Base class of certain exporters that permit testing abstracted from the Jython interpreter.
+ */
+ static abstract class TestableExporter implements BufferProtocol {
- protected void releaseAction() {
- --exportCount;
- }
- };
+ protected Reference<PyBuffer> export;
+
+ /**
+ * Try to re-use existing exported buffer, or return null if can't.
+ */
+ protected PyBuffer getExistingBuffer(int flags) {
+ PyBuffer pybuf = null;
+ if (export != null) {
+ // A buffer was exported at some time.
+ pybuf = export.get();
+ if (pybuf != null) {
+ // And this buffer still exists: expect this to provide a further reference
+ pybuf = pybuf.getBuffer(flags);
+ }
}
+ return pybuf;
}
+
+ /**
+ * Determine whether this object is exporting a buffer: modelled after
+ * {@link PyByteArray#resizeCheck()}.
+ *
+ * @return true iff exporting
+ */
+ public boolean isExporting() {
+ if (export != null) {
+ // A buffer was exported at some time.
+ PyBuffer pybuf = export.get();
+ if (pybuf != null) {
+ return !pybuf.isReleased();
+ } else {
+ // In fact the reference has expired: go quicker next time.
+ export = null;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Determine whether this object permits it's buffers to re-animate themselves. If not, a
+ * call to getBuffer on a released buffer should not return the same buffer.
+ */
+ public boolean reusable = true;
+
}
/**
* A class to act as an exporter that uses the SimpleStringBuffer. This permits testing
* abstracted from the Jython interpreter.
+ * <p>
+ * The exporter shares a single exported buffer between all consumers but does not need to take
+ * any action when that buffer is finally released. You are most likely to use this approach
+ * with an exporting object type that does not modify its behaviour while there are active
+ * exports, but where it is worth avoiding the cost of duplicate buffers. This is the case with
+ * PyString, where some buffer operations cause construction of a byte array copy of the Java
+ * String, which it is desirable to do only once.
*/
- static class StringExporter implements BufferProtocol {
+ static class StringExporter extends TestableExporter {
String storage;
- int exportCount;
/**
* Construct a simple exporter from the String supplied.
@@ -789,8 +867,63 @@
@Override
public PyBuffer getBuffer(int flags) {
- return new SimpleStringBuffer(this, storage, flags);
+ // If we have already exported a buffer it may still be available for re-use
+ PyBuffer pybuf = getExistingBuffer(flags);
+ if (pybuf == null) {
+ // No existing export we can re-use
+ pybuf = new SimpleStringBuffer(flags, storage);
+ // Hold a reference for possible re-use
+ export = new SoftReference<PyBuffer>(pybuf);
+ }
+ return pybuf;
}
+
+ }
+
+ /**
+ * A class to act as an exporter that uses the SimpleBuffer. This permits testing abstracted
+ * from the Jython interpreter.
+ * <p>
+ * The exporter shares a single exported buffer between all consumers and needs to take any
+ * action immediately when that buffer is finally released. You are most likely to use this
+ * approach with an exporting object type that modifies its behaviour while there are active
+ * exports, but where it is worth avoiding the cost of duplicate buffers. This is the case with
+ * PyByteArray, which prohibits operations that would resize it, while there are outstanding
+ * exports.
+ */
+ static class SimpleWritableExporter extends TestableExporter {
+
+ protected byte[] storage;
+
+ /**
+ * Construct a simple exporter from the bytes supplied.
+ *
+ * @param storage
+ */
+ public SimpleWritableExporter(byte[] storage) {
+ this.storage = storage;
+ reusable = false;
+ }
+
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ // If we have already exported a buffer it may still be available for re-use
+ PyBuffer pybuf = getExistingBuffer(flags);
+ if (pybuf == null) {
+ // No existing export we can re-use
+ pybuf = new SimpleWritableBuffer(flags, storage) {
+
+ protected void releaseAction() {
+ export = null;
+ }
+ };
+
+ // Hold a reference for possible re-use
+ export = new WeakReference<PyBuffer>(pybuf);
+ }
+ return pybuf;
+ }
+
}
/**
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list