From jython-checkins at python.org Sat Dec 1 14:57:51 2018 From: jython-checkins at python.org (jeff.allen) Date: Sat, 01 Dec 2018 19:57:51 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Restrict_the_buffer_API_in?= =?utf-8?q?_array=2Earray_and_unicode_=28fixes_=232715=29=2E?= Message-ID: <20181201195751.1.85FEC3C02E4FCB93@mg.python.org> https://hg.python.org/jython/rev/616f1bbd4ec9 changeset: 8199:616f1bbd4ec9 user: Jeff Allen date: Sat Dec 01 19:02:37 2018 +0000 summary: Restrict the buffer API in array.array and unicode (fixes #2715). We make array.array reject attempts to get the buffer API when a view other than flat bytes (SIMPLE) is requested, and the same mechanism is used to disable the buffer interface in unicode (which PyUnicode has only by inheritance from PyString). This is for better Python 2 conformance. On its own, this creates multiple regressions, which the bulk if the changeset is devoted to fixing. files: Lib/test/test_array.py | 93 +--- NEWS | 1 + src/org/python/core/BaseBytes.java | 54 +- src/org/python/core/BufferProtocol.java | 25 +- src/org/python/core/Py2kBuffer.java | 6 +- src/org/python/core/PyArray.java | 52 +- src/org/python/core/PyBUF.java | 2 +- src/org/python/core/PyByteArray.java | 34 +- src/org/python/core/PyFile.java | 8 +- src/org/python/core/PyMemoryView.java | 13 +- src/org/python/core/PyUnicode.java | 12 + src/org/python/modules/_io/PyFileIO.java | 14 +- src/org/python/modules/_io/PyIOBase.java | 60 +- src/org/python/modules/binascii.java | 267 +++++---- src/org/python/modules/posix/PosixModule.java | 9 +- src/org/python/modules/sre/PatternObject.java | 14 +- 16 files changed, 324 insertions(+), 340 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -63,6 +63,7 @@ a = array.array(self.typecode, self.example) self.assertEqual(len(a), len(self.example)) + @unittest.skipIf(test_support.is_jython, "Specific to CPython (memory address)") def test_buffer_info(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.buffer_info, 42) @@ -75,9 +76,7 @@ def test_byteswap(self): if test_support.is_jython and self.typecode == 'u': - # Jython unicodes are already decoded from utf16, - # so this doesn't make sense - return + self.skipTest("Jython unicode byteswap not implemented") a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.byteswap, 42) if a.itemsize in (1, 2, 4, 8): @@ -280,25 +279,6 @@ checkSlice(None, D-R-1, -1) checkSlice(R-1, None, -1) - def test_filewrite(self): - a = array.array(self.typecode, 2*self.example) - f = open(test_support.TESTFN, 'wb') - try: - f.write(a) - f.close() - b = array.array(self.typecode) - f = open(test_support.TESTFN, 'rb') - b.fromfile(f, len(self.example)) - self.assertEqual(b, array.array(self.typecode, self.example)) - self.assertNotEqual(a, b) - b.fromfile(f, len(self.example)) - self.assertEqual(a, b) - f.close() - finally: - if not f.closed: - f.close() - test_support.unlink(test_support.TESTFN) - def test_repr(self): a = array.array(self.typecode, 2*self.example) self.assertEqual(a, eval(repr(a), {"array": array.array})) @@ -836,6 +816,11 @@ b = buffer(a) self.assertEqual(b[0], a.tostring()[0]) + def test_not_memoryview(self): + # In Jython we are careful *not* to support memoryview + a = array.array(self.typecode, self.example) + self.assertRaises(TypeError, memoryview, a) + def test_weakref(self): s = array.array(self.typecode, self.example) p = proxy(s) @@ -859,66 +844,6 @@ # SF bug #1486663 -- this used to erroneously raise a TypeError ArraySubclassWithKwargs('b', newarg=1) - @unittest.skipUnless(test_support.is_jython, "array supports buffer interface in Jython") - def test_resize_forbidden(self): - # Test that array resizing is forbidden with buffer exports (Jython addition). - # Test adapted from corresponding one in test_bytes. - # We can't resize an array when there are buffer exports, even - # if it wouldn't reallocate the underlying array. - # Furthermore, no destructive changes to the buffer may be applied - # before raising the error. - a = array.array(self.typecode, self.example) - def resize(n): - "n = -1 -> Smaller, 0 -> the same, or 1 -> larger." - a[1:-1] = array.array(self.typecode, self.example[1-n:-1]) - - v = memoryview(a) - orig = a[:] - - self.assertRaises(BufferError, resize, -1) - self.assertEqual(a, orig) - #self.assertRaises(BufferError, resize, 0) - #self.assertEqual(a, orig) - self.assertRaises(BufferError, resize, 1) - self.assertEqual(a, orig) - - # Other operations implying resize - self.assertRaises(BufferError, a.pop, 0) - self.assertEqual(a, orig) - self.assertRaises(BufferError, a.remove, a[1]) - self.assertEqual(a, orig) - self.assertRaises(BufferError, a.append, self.outside) - self.assertEqual(a, orig) - self.assertRaises(BufferError, a.insert, 1, self.outside) - self.assertEqual(a, orig) - self.assertRaises(BufferError, a.extend, self.example) - self.assertEqual(a, orig) - - def iadd(x): - x += array.array(self.typecode, self.biggerexample) - self.assertRaises(BufferError, iadd, a) - self.assertEqual(a, orig) - - def imul(x): - x *= 3 - self.assertRaises(BufferError, imul, a) - self.assertEqual(a, orig) - - def delitem(): - del a[1] - self.assertRaises(BufferError, delitem) - self.assertEqual(a, orig) - - # deleting a non-contiguous slice - def delslice(): - del a[1:-1:2] - self.assertRaises(BufferError, delslice) - self.assertEqual(a, orig) - - # Show that releasing v releases the array for size change - v.release() - a.pop() - class StringTest(BaseTest): @@ -1221,10 +1146,6 @@ def test_main(verbose=None): import sys - if test_support.is_jython: - # CPython specific; returns a memory address - del BaseTest.test_buffer_info - test_support.run_unittest(*tests) # verify reference counting diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -32,6 +32,7 @@ - [ 2469 ] jython launcher fails on Windows if JAVA_HOME is set - [ 2650 ] Detail message is not set on PyException from PythonInterpreter - [ 2403 ] VerifyError when implementing interfaces containing default methods (Java 8) + - [ 2715 ] Restrict array.array support for buffer protocol New Features - The main program behaves more like CPython in many small ways, including a more correct diff --git a/src/org/python/core/BaseBytes.java b/src/org/python/core/BaseBytes.java --- a/src/org/python/core/BaseBytes.java +++ b/src/org/python/core/BaseBytes.java @@ -182,44 +182,31 @@ protected void init(PyObject arg) { if (arg == null) { - /* - * bytearray() Construct a zero-length bytearray. - */ + // bytearray() Construct a zero-length bytearray. setStorage(emptyStorage); - } else if (arg instanceof PyString) { + } else if (arg instanceof PyString) { // or PyUnicode /* * bytearray(string) Construct from a text string by default encoding and error policy. * Cases where encoding and error policy are specified explicitly are dealt with * elsewhere. */ - init((PyString)arg, (String)null, (String)null); // Casts select right init() + init((PyString) arg, (String) null, (String) null); // Casts select right init() } else if (arg.isIndex()) { - /* - * bytearray(int) Construct a zero-initialised bytearray of the given length. - */ + // bytearray(int) Construct a zero-initialised bytearray of the given length. init(arg.asIndex(Py.OverflowError)); // overflow if too big to be Java int } else if (arg instanceof BaseBytes) { - /* - * bytearray copy of bytearray (or bytes) -- do efficiently - */ - init((BaseBytes)arg); - - } else if (arg instanceof BufferProtocol) { - /* - * bytearray copy of object supporting Jython implementation of PEP 3118 - */ - init((BufferProtocol)arg); + // bytearray copy of bytearray (or bytes) -- do efficiently + init((BaseBytes) arg); + + } else if (initFromBuffer(arg)) { + // arg supports Jython implementation of PEP 3118. (We're done.) } else { - /* - * The remaining alternative is an iterable returning (hopefully) right-sized ints. If - * it isn't one, we get an exception about not being iterable, or about the values. - */ + // The remaining alternative is an iterable returning (we hope) right-sized ints. init(arg.asIterable()); - } } @@ -342,9 +329,9 @@ * * @param value an object bearing the Buffer API and consistent with the slice assignment */ - protected void init(BufferProtocol value) throws PyException { + protected void init(BufferProtocol value) throws PyException, ClassCastException { // Get the buffer view - try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) { + try (PyBuffer view = value.getBuffer(PyBUF.SIMPLE)) { // Create storage for the bytes and have the view drop them in newStorage(view.getLen()); view.copyTo(storage, offset); @@ -352,6 +339,23 @@ } /** + * Helper for __new__ and __init__ from objects that might + * support the Jython Buffer API. + * + * @param value an object possibly bearing the Buffer API + * @return {@code true} iff {@code value} allows the {@code getBuffer} + */ + private boolean initFromBuffer(PyObject value) throws PyException { + if (value instanceof BufferProtocol) { + try { + init((BufferProtocol) value); + return true; + } catch (ClassCastException e) { /* fall through to false */ } + } + return false; + } + + /** * Helper for __new__ and __init__ and the Java API constructor from * bytearray or bytes in subclasses. * diff --git a/src/org/python/core/BufferProtocol.java b/src/org/python/core/BufferProtocol.java --- a/src/org/python/core/BufferProtocol.java +++ b/src/org/python/core/BufferProtocol.java @@ -2,19 +2,32 @@ /** * Interface marking an object as capable of exposing its internal state as a {@link PyBuffer}. + *

+ * A few objects implement {@code BufferProtocol} (e.g. by inheritance) but cannot actually provide + * their value as a {@link PyBuffer}. These should throw {@code ClassCastException}, permitting the + * idiom: 

+ *     try (PyBuffer buf = ((BufferProtocol) obj).getBuffer(PyBUF.SIMPLE)) {
+ *         ... // Do something with buf
+ *     } catch (ClassCastException e) {
+ *         ... // expected bytes object or buffer not obj.getType()
+ *     }
+ *
The {@code catch} is executed identically whether the cause is the explicit cast of + * {@code obj} or {@code getBuffer}, and the try-with-resources releases the buffer if one was + * obtained. */ public interface BufferProtocol { /** * Method by which the consumer requests the buffer from the exporter. The consumer provides - * information on its intended method of navigation and the features the buffer object is asked - * (or assumed) to provide. Each consumer requesting a buffer in this way, when it has finished - * using it, should make a corresponding call to {@link PyBuffer#release()} on the buffer it - * obtained, since some objects alter their behaviour while buffers are exported. - * + * information on its ability to understand buffer navigation. Each consumer requesting a buffer + * in this way, when it has finished using it, should make a corresponding call to + * {@link PyBuffer#release()} on the buffer it obtained, or {@link PyBuffer#close()} using + * try-with-resources, since some objects alter their behaviour while buffers are exported. + * * @param flags specifying features demanded and the navigational capabilities of the consumer * @return exported buffer * @throws PyException (BufferError) when expectations do not correspond with the buffer + * @throws ClassCastException when the object only formally implements {@code BufferProtocol} */ - PyBuffer getBuffer(int flags) throws PyException; + PyBuffer getBuffer(int flags) throws PyException, ClassCastException; } diff --git a/src/org/python/core/Py2kBuffer.java b/src/org/python/core/Py2kBuffer.java --- a/src/org/python/core/Py2kBuffer.java +++ b/src/org/python/core/Py2kBuffer.java @@ -77,9 +77,9 @@ */ private PyBuffer getBuffer() { /* - * Ask for a simple one-dimensional byte view (not requiring strides, indirect, etc.) from - * the object, as we cannot deal with other navigation. Ask for read access. If the object - * is writable, the PyBuffer will be writable, but we won't write to it. + * Ask for a simple one-dimensional byte view from the object, as we cannot deal with more + * complex navigation. Ask for read access. If the object is writable, the PyBuffer will be + * writable, but we won't write to it. */ final int flags = PyBUF.SIMPLE; PyBuffer buf = object.getBuffer(flags); diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java --- a/src/org/python/core/PyArray.java +++ b/src/org/python/core/PyArray.java @@ -13,8 +13,8 @@ import java.nio.ByteBuffer; import org.python.core.buffer.BaseBuffer; +import org.python.core.buffer.SimpleBuffer; import org.python.core.buffer.SimpleStringBuffer; -import org.python.core.buffer.SimpleWritableBuffer; import org.python.core.util.ByteSwapper; import org.python.core.util.StringUtil; import org.python.expose.ExposedGet; @@ -398,7 +398,7 @@ PyArray otherArr = (PyArray)other; if (!otherArr.typecode.equals(this.typecode)) { throw Py.TypeError("can only append arrays of the same type, expected '" + this.type - + ", found " + otherArr.type); + + "', found '" + otherArr.type + "'"); } PyArray ret = new PyArray(this); ret.delegate.appendArray(otherArr.delegate.copyArray()); @@ -2066,34 +2066,40 @@ *

* The {@link PyBuffer} returned from this method is a one-dimensional array of single byte * items that allows modification of the object state. The existence of this export prohibits - * resizing the byte array. This prohibition is not only on the consumer of the view but - * extends to any other operations, such as any kind or insertion or deletion. + * resizing the array. This prohibition is not only on the consumer of the view but extends + * to operations on the underlying array, such as {@link #insert(int, PyObject)} or + * {@link #pop()}. */ @Override public synchronized PyBuffer getBuffer(int flags) { - // If we have already exported a buffer it may still be available for re-use - BaseBuffer pybuf = getExistingBuffer(flags); + if ((flags & ~PyBUF.WRITABLE) == PyBUF.SIMPLE) { + // Client requests a flat byte-oriented read-view, typically from buffer(a). + + // If we have already exported a buffer it may still be available for re-use + BaseBuffer pybuf = getExistingBuffer(flags); - if (pybuf == null) { - // No existing export we can re-use: create a new one - if ("b".equals(typecode)) { - // This is byte data, so we are within the state of the art - byte[] storage = (byte[])data; - int size = delegate.getSize(); - pybuf = new SimpleWritableBuffer(flags, this, storage, 0, size); - } else if ((flags & PyBUF.WRITABLE) == 0) { - // As the client only intends to read, fake the answer with a String - pybuf = new SimpleStringBuffer(flags, this, tostring()); - } else { - // For the time being ... - throw Py.NotImplementedError("only array('b') can export a writable buffer"); + if (pybuf == null) { + // No existing export we can re-use: create a new one + if ("b".equals(typecode)) { + // This is byte data, so we can export directly + byte[] storage = (byte[]) data; + int size = delegate.getSize(); + pybuf = new SimpleBuffer(flags, this, storage, 0, size); + } else { + // As the client only intends to read, fake the answer with a String + pybuf = new SimpleStringBuffer(flags, this, tostring()); + } + // Hold a reference for possible re-use + export = new WeakReference(pybuf); } - // Hold a reference for possible re-use - export = new WeakReference(pybuf); + + return pybuf; + + } else { + // Client request goes beyond Python 2 capability, typically from memoryview(a). + throw new ClassCastException("'array' supports only a byte-buffer view"); } - - return pybuf; } /** diff --git a/src/org/python/core/PyBUF.java b/src/org/python/core/PyBUF.java --- a/src/org/python/core/PyBUF.java +++ b/src/org/python/core/PyBUF.java @@ -116,7 +116,7 @@ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to * specify that it assumes a simple one-dimensional organisation of the exported storage with * item size of one. getBuffer will raise an exception if the consumer sets this - * flag and the exporter's buffer cannot be navigated that simply. + * flag and the exporter cannot represent itself as byte array data. */ static final int SIMPLE = 0; /** diff --git a/src/org/python/core/PyByteArray.java b/src/org/python/core/PyByteArray.java --- a/src/org/python/core/PyByteArray.java +++ b/src/org/python/core/PyByteArray.java @@ -482,12 +482,8 @@ */ setslice(start, stop, step, (BaseBytes)value); - } else if (value instanceof BufferProtocol) { - /* - * Value supports Jython implementation of PEP 3118, and can be can be inserted without - * making a copy. - */ - setslice(start, stop, step, (BufferProtocol)value); + } else if (setsliceFromBuffer(start, stop, step, value)) { + // Value supports Jython buffer API. (We're done.) } else { /* @@ -573,9 +569,9 @@ * @param value an object supporting the buffer API consistent with the slice assignment * @throws PyException (SliceSizeError) if the value size is inconsistent with an extended slice */ - private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException { + private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException, ClassCastException { - try (PyBuffer view = value.getBuffer(PyBUF.FULL_RO)) { + try (PyBuffer view = value.getBuffer(PyBUF.SIMPLE)) { int len = view.getLen(); @@ -599,6 +595,28 @@ } /** + * Sets the given range of elements according to Python slice assignment semantics from an + * object that might support the Jython Buffer API. + * + * @param start the position of the first element. + * @param stop one more than the position of the last element. + * @param step the step size. + * @param value an object possibly bearing the Buffer API + * @throws PyException (SliceSizeError) if the value size is inconsistent with an extended slice + */ + private boolean setsliceFromBuffer(int start, int stop, int step, PyObject value) + throws PyException { + if (value instanceof BufferProtocol) { + try { + setslice(start, stop, step, (BufferProtocol) value); + return true; + } catch (ClassCastException e) { /* fall through to false */ } + } + return false; + } + + + /** * Sets the given range of elements according to Python slice assignment semantics from a * bytearray (or bytes). * diff --git a/src/org/python/core/PyFile.java b/src/org/python/core/PyFile.java --- a/src/org/python/core/PyFile.java +++ b/src/org/python/core/PyFile.java @@ -496,11 +496,13 @@ } else if (obj instanceof PyArray && !binary) { // Fall through to TypeError. (If binary, BufferProtocol takes care of PyArray.) - } else if (obj instanceof BufferProtocol) { - // Try to get a byte-oriented buffer - try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)) { + } else { + // Try to get a simple byte-oriented buffer + try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE)) { // ... and treat those bytes as a String return buf.toString(); + } catch (ClassCastException e) { + // Does not implement BufferProtocol (in reality). Fall through to message. } } diff --git a/src/org/python/core/PyMemoryView.java b/src/org/python/core/PyMemoryView.java --- a/src/org/python/core/PyMemoryView.java +++ b/src/org/python/core/PyMemoryView.java @@ -43,8 +43,9 @@ * new lease on it. The buffer so obtained will be writable if the underlying object permits it. * * @param pybuf buffer exported by some underlying object + * @throws ClassCastException in cases where {@code pybuf.getBuffer} does so. */ - public PyMemoryView(BufferProtocol pybuf) { + public PyMemoryView(BufferProtocol pybuf) throws ClassCastException { super(TYPE); /* * Ask for the full set of facilities (strides, indirect, etc.) from the object in case they @@ -68,11 +69,13 @@ PyObject obj = ap.getPyObject(0); if (obj instanceof BufferProtocol) { - return new PyMemoryView((BufferProtocol)obj); - } else { - throw Py.TypeError("cannot make memory view because object does not have " - + "the buffer interface"); + // Certain types that implement BufferProtocol do not implement the buffer protocol + try { + return new PyMemoryView((BufferProtocol) obj); + } catch (ClassCastException e) { /* fall through to message */ } } + throw Py.TypeError( + "cannot make memory view because object does not have the buffer interface"); } // @ExposedGet(doc = obj_doc) // Not exposed in Python 2.7 diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java --- a/src/org/python/core/PyUnicode.java +++ b/src/org/python/core/PyUnicode.java @@ -134,6 +134,18 @@ return codePoints; } + /** + * {@code PyUnicode} implements the interface {@link BufferProtocol} technically by inheritance from {@link PyString}, + * but does not provide a buffer (in CPython). We therefore arrange that all calls to {@code getBuffer} + * raise an error. + * + * @return always throws a {@code ClassCastException} + */ + @Override + public synchronized PyBuffer getBuffer(int flags) throws ClassCastException { + throw new ClassCastException("'unicode' does not support the buffer protocol"); + } + // ------------------------------------------------------------------------------------------ // Index translation for Unicode beyond the BMP // ------------------------------------------------------------------------------------------ diff --git a/src/org/python/modules/_io/PyFileIO.java b/src/org/python/modules/_io/PyFileIO.java --- a/src/org/python/modules/_io/PyFileIO.java +++ b/src/org/python/modules/_io/PyFileIO.java @@ -250,16 +250,11 @@ } else { // Perform the operation through a buffer view on the object - PyBuffer pybuf = writablePyBuffer(buf); - - try { + try (PyBuffer pybuf = writablePyBuffer(buf)) { ByteBuffer byteBuffer = pybuf.getNIOByteBuffer(); synchronized (ioDelegate) { count = ioDelegate.readinto(byteBuffer); } - } finally { - // Must unlock the PyBuffer view from client's object - pybuf.release(); } } @@ -290,17 +285,12 @@ } else { // Get or synthesise a buffer API on the object to be written - PyBuffer pybuf = readablePyBuffer(buf); - - try { + try (PyBuffer pybuf = readablePyBuffer(buf)) { // Access the data as a java.nio.ByteBuffer [pos:limit] within possibly larger array ByteBuffer byteBuffer = pybuf.getNIOByteBuffer(); synchronized (ioDelegate) { count = ioDelegate.write(byteBuffer); } - } finally { - // Even if that went badly, we should release the lock on the client buffer - pybuf.release(); } } diff --git a/src/org/python/modules/_io/PyIOBase.java b/src/org/python/modules/_io/PyIOBase.java --- a/src/org/python/modules/_io/PyIOBase.java +++ b/src/org/python/modules/_io/PyIOBase.java @@ -543,9 +543,7 @@ if (peekResult.__nonzero__()) { // Get a look at the bytes themselves - PyBuffer peekBuffer = readablePyBuffer(peekResult); - - try { + try (PyBuffer peekBuffer = readablePyBuffer(peekResult)) { /* * Scan forwards in the peek buffer to see if there is an end-of-line. Most * frequently this succeeds. The number of bytes to scan is limited to the @@ -569,10 +567,6 @@ */ curr = readMethod.__call__(Py.newInteger(p)); remainingLimit -= p; - - } finally { - // We must let go of the buffer we were given - peekBuffer.release(); } } else { @@ -756,9 +750,15 @@ * @throws PyException (TypeError) if object not convertible to a byte array */ protected static PyBuffer readablePyBuffer(PyObject obj) throws PyException { - if (obj instanceof BufferProtocol) { + + // First consider special cases we can view as a String + if (obj instanceof PyUnicode) { + String s = ((PyUnicode) obj).encode(); + return new SimpleStringBuffer(PyBUF.SIMPLE, null, s); + + } else { try { - return ((BufferProtocol)obj).getBuffer(PyBUF.SIMPLE); + return ((BufferProtocol) obj).getBuffer(PyBUF.SIMPLE); } catch (PyException pye) { if (pye.match(Py.BufferError)) { // If we can't get a buffer on the object, say it's the wrong type @@ -766,20 +766,13 @@ } else { throw pye; } + } catch (ClassCastException pye) { + // fall through to message } - } else { - // Something else we can view as a String? - String s; - if (obj instanceof PyUnicode) { - s = ((PyUnicode)obj).encode(); - } else if (obj instanceof PyArray) { - s = ((PyArray)obj).tostring(); - } else { - // None of the above: complain - throw tailoredTypeError("read-write buffer", obj); - } - return new SimpleStringBuffer(PyBUF.SIMPLE, null, s); } + + // None of the above: complain + throw tailoredTypeError("read-write buffer", obj); } /** @@ -792,21 +785,20 @@ * @throws PyException (TypeError) if object not convertible to a byte array */ protected static PyBuffer writablePyBuffer(PyObject obj) throws PyException { - if (obj instanceof BufferProtocol) { - try { - return ((BufferProtocol)obj).getBuffer(PyBUF.WRITABLE); - } catch (PyException pye) { - if (pye.match(Py.BufferError)) { - // If we can't get a buffer on the object, say it's the wrong type - throw Py.TypeError(String.format("(BufferError) %s", pye.getMessage())); - } else { - throw pye; - } + try { + return ((BufferProtocol) obj).getBuffer(PyBUF.WRITABLE); + } catch (PyException pye) { + if (pye.match(Py.BufferError)) { + // If we can't get a buffer on the object, say it's the wrong type + throw Py.TypeError(String.format("(BufferError) %s", pye.getMessage())); + } else { + throw pye; } - } else { - // Can't be a buffer: complain - throw tailoredTypeError("read-write buffer", obj); + } catch (ClassCastException pye) { + // fall through to message } + // Can't be a buffer: complain + throw tailoredTypeError("read-write buffer", obj); } /** diff --git a/src/org/python/modules/binascii.java b/src/org/python/modules/binascii.java --- a/src/org/python/modules/binascii.java +++ b/src/org/python/modules/binascii.java @@ -21,6 +21,8 @@ import org.python.core.PyString; import org.python.core.PyStringMap; import org.python.core.PyTuple; +import org.python.core.PyUnicode; +import org.python.core.buffer.SimpleStringBuffer; import org.python.core.util.StringUtil; /** @@ -280,12 +282,11 @@ int leftchar = 0; StringBuilder bin_data = new StringBuilder(); - PyBuffer ascii_data = bp.getBuffer(PyBUF.SIMPLE); - try { + + try (PyBuffer ascii_data = getTextBuffer(bp)) { if (ascii_data.getLen() == 0) return new PyString(""); - char this_ch; int i; @@ -331,19 +332,18 @@ throw new PyException(Error, "Trailing garbage"); } } - + // finally, if we haven't decoded enough stuff, fill it up with zeros - for (; i ascii. Uuencode line of data" ); @@ -436,23 +436,21 @@ int leftchar = 0; int quad_pos = 0; - PyBuffer ascii_data = bp.getBuffer(PyBUF.SIMPLE); - int ascii_len = ascii_data.getLen(); + try (PyBuffer ascii_data = getTextBuffer(bp)) { + int ascii_len = ascii_data.getLen(); - int bin_len = 0; - StringBuilder bin_data = new StringBuilder(); + int bin_len = 0; + StringBuilder bin_data = new StringBuilder(); - try { - for(int i = 0; ascii_len > 0 ; ascii_len--, i++) { + for (int i = 0; ascii_len > 0; ascii_len--, i++) { // Skip some punctuation this_ch = (char) ascii_data.intAt(i); - if (this_ch > 0x7F || this_ch == '\r' || - this_ch == '\n' || this_ch == ' ') + if (this_ch > 0x7F || this_ch == '\r' || this_ch == '\n' || this_ch == ' ') continue; if (this_ch == BASE64_PAD) { - if (quad_pos < 2 || (quad_pos == 2 && - binascii_find_valid(ascii_data, i, 1) != BASE64_PAD)) + if (quad_pos < 2 || (quad_pos == 2 + && binascii_find_valid(ascii_data, i, 1) != BASE64_PAD)) continue; else { // A pad sequence means no more input. @@ -474,23 +472,24 @@ leftbits += 6; if (leftbits >= 8) { leftbits -= 8; - bin_data.append((char)((leftchar >> leftbits) & 0xff)); + bin_data.append((char) ((leftchar >> leftbits) & 0xff)); bin_len++; leftchar &= ((1 << leftbits) - 1); } } - } finally { - ascii_data.release(); + // Check that no bits are left + if (leftbits != 0) { + throw new PyException(Error, "Incorrect padding"); + } + + return new PyString(bin_data.toString()); + + } catch (ClassCastException e) { + throw argMustBeBytes("a2b_base64", bp); } - // Check that no bits are left - if (leftbits != 0) { - throw new PyException(Error, "Incorrect padding"); - } - return new PyString(bin_data.toString()); } - public static PyString __doc__b2a_base64 = new PyString( "(bin) -> ascii. Base64-code line of data" ); @@ -507,42 +506,42 @@ StringBuilder ascii_data = new StringBuilder(); - PyBuffer bin_data = bp.getBuffer(PyBUF.SIMPLE); - try { + try (PyBuffer bin_data = getTextBuffer(bp)) { int bin_len = bin_data.getLen(); if (bin_len > BASE64_MAXBIN) { - throw new PyException(Error,"Too much data for base64 line"); + throw new PyException(Error, "Too much data for base64 line"); } - for (int i = 0; bin_len > 0 ; bin_len--, i++) { + for (int i = 0; bin_len > 0; bin_len--, i++) { // Shift the data into our buffer - leftchar = (leftchar << 8) | (char) bin_data.intAt(i); //charAt(i); + leftchar = (leftchar << 8) | (char) bin_data.intAt(i); // charAt(i); leftbits += 8; // See if there are 6-bit groups ready while (leftbits >= 6) { - this_ch = (char)((leftchar >> (leftbits-6)) & 0x3f); + this_ch = (char) ((leftchar >> (leftbits - 6)) & 0x3f); leftbits -= 6; - ascii_data.append((char)table_b2a_base64[this_ch]); + ascii_data.append((char) table_b2a_base64[this_ch]); } } - } finally { - bin_data.release(); + + if (leftbits == 2) { + ascii_data.append((char) table_b2a_base64[(leftchar & 3) << 4]); + ascii_data.append(BASE64_PAD); + ascii_data.append(BASE64_PAD); + } else if (leftbits == 4) { + ascii_data.append((char) table_b2a_base64[(leftchar & 0xf) << 2]); + ascii_data.append(BASE64_PAD); + } + ascii_data.append('\n'); // Append a courtesy newline + + return new PyString(ascii_data.toString()); + + } catch (ClassCastException e) { + throw argMustBeBytes("b2a_base64", bp); } - if (leftbits == 2) { - ascii_data.append((char)table_b2a_base64[(leftchar&3) << 4]); - ascii_data.append(BASE64_PAD); - ascii_data.append(BASE64_PAD); - } else if (leftbits == 4) { - ascii_data.append((char)table_b2a_base64[(leftchar&0xf) << 2]); - ascii_data.append(BASE64_PAD); - } - ascii_data.append('\n'); // Append a courtesy newline - - return new PyString(ascii_data.toString()); } - public static PyString __doc__a2b_hqx = new PyString( "ascii -> bin, done. Decode .hqx coding" ); @@ -557,15 +556,14 @@ int leftbits = 0; char this_ch; int leftchar = 0; + boolean done = false; - boolean done = false; - PyBuffer ascii_data = bp.getBuffer(PyBUF.SIMPLE); - int len = ascii_data.getLen(); + try (PyBuffer ascii_data = getTextBuffer(bp)) { - StringBuilder bin_data = new StringBuilder(); + int len = ascii_data.getLen(); + StringBuilder bin_data = new StringBuilder(); - try { - for(int i = 0; len > 0 ; len--, i++) { + for (int i = 0; len > 0; len--, i++) { // Get the byte and look it up this_ch = (char) table_a2b_hqx[ascii_data.intAt(i)]; if (this_ch == SKIP) @@ -584,23 +582,22 @@ leftbits += 6; if (leftbits >= 8) { leftbits -= 8; - bin_data.append((char)((leftchar >> leftbits) & 0xff)); + bin_data.append((char) ((leftchar >> leftbits) & 0xff)); leftchar &= ((1 << leftbits) - 1); } } - } finally { - ascii_data.release(); - } + + if (leftbits != 0 && !done) { + throw new PyException(Incomplete, "String has incomplete number of bytes"); + } - if (leftbits != 0 && !done) { - throw new PyException(Incomplete, - "String has incomplete number of bytes"); + return new PyTuple(new PyString(bin_data.toString()), Py.newInteger(done ? 1 : 0)); + + } catch (ClassCastException e) { + throw argMustBeBytes("a2b_hqx", bp); } - - return new PyTuple(new PyString(bin_data.toString()), Py.newInteger(done ? 1 : 0)); } - public static PyString __doc__rlecode_hqx = new PyString( "Binhex RLE-code binary data" ); @@ -662,36 +659,35 @@ char this_ch; int leftchar = 0; - PyBuffer bin_data = bp.getBuffer(PyBUF.SIMPLE); - int len = bin_data.getLen(); + try (PyBuffer bin_data = getTextBuffer(bp)) { - StringBuilder ascii_data = new StringBuilder(); + int len = bin_data.getLen(); + StringBuilder ascii_data = new StringBuilder(); - try { - for(int i = 0; len > 0; len--, i++) { + for (int i = 0; len > 0; len--, i++) { // Shift into our buffer, and output any 6bits ready leftchar = (leftchar << 8) | (char) bin_data.intAt(i); leftbits += 8; while (leftbits >= 6) { - this_ch = (char) ((leftchar >> (leftbits-6)) & 0x3f); + this_ch = (char) ((leftchar >> (leftbits - 6)) & 0x3f); leftbits -= 6; ascii_data.append((char) table_b2a_hqx[this_ch]); } } - } finally { - bin_data.release(); - } + + // Output a possible runt byte + if (leftbits != 0) { + leftchar <<= (6 - leftbits); + ascii_data.append((char) table_b2a_hqx[leftchar & 0x3f]); + } - // Output a possible runt byte - if (leftbits != 0) { - leftchar <<= (6-leftbits); - ascii_data.append((char) table_b2a_hqx[leftchar & 0x3f]); + return new PyString(ascii_data.toString()); + + } catch (ClassCastException e) { + throw argMustBeBytes("b2a_hqx", bp); } - return new PyString(ascii_data.toString()); } - - public static PyString __doc__rledecode_hqx = new PyString( "Decode hexbin RLE-coded string" ); @@ -707,11 +703,11 @@ */ static public PyString rledecode_hqx(BufferProtocol bp) { char in_byte, in_repeat; - + PyBuffer in_data = bp.getBuffer(PyBUF.SIMPLE); int in_len = in_data.getLen(); int i = 0; - + StringBuilder out_data = new StringBuilder(); try { // Empty string is a special case @@ -888,25 +884,26 @@ ); public static PyString b2a_hex(BufferProtocol bp) { - PyBuffer argbuf = bp.getBuffer(PyBUF.SIMPLE); - int arglen = argbuf.getLen(); + + try (PyBuffer argbuf = getTextBuffer(bp)) { - StringBuilder retbuf = new StringBuilder(arglen*2); + int arglen = argbuf.getLen(); + StringBuilder retbuf = new StringBuilder(arglen * 2); - try { /* make hex version of string, taken from shamodule.c */ for (int i = 0; i < arglen; i++) { char ch = (char) argbuf.intAt(i); retbuf.append(hexdigit[(ch >>> 4) & 0xF]); retbuf.append(hexdigit[ch & 0xF]); } - } finally { - argbuf.release(); + + return new PyString(retbuf.toString()); + + } catch (ClassCastException e) { + throw argMustBeBytes("b2a_hex", bp); } - return new PyString(retbuf.toString()); } - public static PyString hexlify(BufferProtocol argbuf) { return b2a_hex(argbuf); } @@ -920,40 +917,40 @@ "This function is also available as \"unhexlify()\"" ); + public static PyString a2b_hex(BufferProtocol bp) { - public static PyString a2b_hex(BufferProtocol bp) { - PyBuffer argbuf = bp.getBuffer(PyBUF.SIMPLE); - int arglen = argbuf.getLen(); + try (PyBuffer argbuf = bp.getBuffer(PyBUF.SIMPLE)) { - StringBuilder retbuf = new StringBuilder(arglen/2); - /* XXX What should we do about strings with an odd length? Should - * we add an implicit leading zero, or a trailing zero? For now, - * raise an exception. - */ - try { + int arglen = argbuf.getLen(); + StringBuilder retbuf = new StringBuilder(arglen / 2); + /* + * XXX What should we do about strings with an odd length? Should we add an implicit + * leading zero, or a trailing zero? For now, raise an exception. + */ if (arglen % 2 != 0) throw Py.TypeError("Odd-length string"); for (int i = 0; i < arglen; i += 2) { int top = Character.digit(argbuf.intAt(i), 16); - int bot = Character.digit(argbuf.intAt(i+1), 16); + int bot = Character.digit(argbuf.intAt(i + 1), 16); if (top == -1 || bot == -1) throw Py.TypeError("Non-hexadecimal digit found"); retbuf.append((char) ((top << 4) + bot)); } - } finally { - argbuf.release(); + + return new PyString(retbuf.toString()); + + } catch (ClassCastException e) { + throw argMustBeBytes("a2b_hex", bp); } - return new PyString(retbuf.toString()); } - public static PyString unhexlify(BufferProtocol argbuf) { return a2b_hex(argbuf); } final private static char[] upper_hexdigit = "0123456789ABCDEF".toCharArray(); - + private static StringBuilder qpEscape(StringBuilder sb, char c) { sb.append('='); @@ -1143,20 +1140,44 @@ return new PyString(sb.toString()); } - -/* - public static void main(String[] args) { - String l = b2a_uu("Hello"); - System.out.println(l); - System.out.println(a2b_uu(l)); + + /** + * We use this when the argument given to a conversion method is to be interpreted as text. If + * it is byte-like, the bytes are used unchanged, assumed in the "intended" character set. It + * may be a {@code PyUnicode}, in which case the it will be decoded to bytes using the default + * encoding ({@code sys.getdefaultencoding()}. + * + * @param text an object with the buffer protocol + * @return a byte-buffer view of the ASCII text + * @throws ClassCastException where the text object does not implement the buffer protocol + */ + private static PyBuffer getTextBuffer(BufferProtocol text) + throws ClassCastException { + if (text instanceof PyUnicode) { + String s = ((PyUnicode) text).encode(); + return new SimpleStringBuffer(PyBUF.SIMPLE, null, s); + } else { + return text.getBuffer(PyBUF.SIMPLE); + } + } - l = b2a_base64("Hello"); - System.out.println(l); - System.out.println(a2b_base64(l)); + /** + * Convenience method providing the exception when an argument is not the expected type, in the + * format "f() argument 1 must bytes or unicode, not type(arg)." + * + * @param f name of function of error (or could be any text) + * @param arg argument provided from which actual type will be reported + * @return TypeError to throw + */ + private static PyException argMustBeBytes(String f, BufferProtocol arg) { + String fmt = "%s() argument 1 must bytes or unicode, not %s"; + String type = "null"; + if (arg instanceof PyObject) { + type = ((PyObject) arg).getType().fastGetName(); + } else if (arg != null) { + type = arg.getClass().getName(); + } + return Py.TypeError(String.format(fmt, f, type)); + } - l = b2a_hqx("Hello-"); - System.out.println(l); - System.out.println(a2b_hqx(l)); - } -*/ } diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java --- a/src/org/python/modules/posix/PosixModule.java +++ b/src/org/python/modules/posix/PosixModule.java @@ -1277,9 +1277,9 @@ public static PyString __doc__write = new PyString( "write(fd, string) -> byteswritten\n\n" + "Write a string to a file descriptor."); - public static int write(PyObject fd, BufferProtocol bytes) { - // Get a buffer view: we can cope with N-dimensional data, but not strided data. - try (PyBuffer buf = bytes.getBuffer(PyBUF.ND)) { + + public static int write(PyObject fd, PyObject bytes) { + try (PyBuffer buf = ((BufferProtocol) bytes).getBuffer(PyBUF.SIMPLE)) { // Get a ByteBuffer of that data, setting the position and limit to the real data. ByteBuffer bb = buf.getNIOByteBuffer(); Object javaobj = fd.__tojava__(RawIOBase.class); @@ -1292,6 +1292,9 @@ } else { return posix.write(getFD(fd).getIntFD(), bb, bb.position()); } + } catch (ClassCastException e) { + throw Py.TypeError( + "write() argument 2 must be string or buffer, not " + bytes.getType()); } } diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java --- a/src/org/python/modules/sre/PatternObject.java +++ b/src/org/python/modules/sre/PatternObject.java @@ -404,21 +404,19 @@ if (obj instanceof PyString) { // Easy case - return (PyString)obj; + return (PyString) obj; - } else if (obj instanceof BufferProtocol) { + } else { // Try to get a byte-oriented buffer - try (PyBuffer buf = ((BufferProtocol)obj).getBuffer(PyBUF.FULL_RO)){ - // ... and treat those bytes as a PyString + try (PyBuffer buf = ((BufferProtocol) obj).getBuffer(PyBUF.SIMPLE)) { + // ... and wrap those bytes as a PyString return new PyString(buf.toString()); + } catch (ClassCastException e) { + throw Py.TypeError("expected string or buffer, but got " + obj.getType()); } } - - // Neither of those things worked - throw Py.TypeError("expected string or buffer, but got " + obj.getType()); } - /* Traverseproc implementation */ @Override public int traverse(Visitproc visit, Object arg) { -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Dec 2 07:19:19 2018 From: jython-checkins at python.org (jeff.allen) Date: Sun, 02 Dec 2018 12:19:19 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Proper_processing_of_gzip_?= =?utf-8?q?trailer_without_resubmission?= Message-ID: <20181202121919.1.B0E5D14D302659E7@mg.python.org> https://hg.python.org/jython/rev/c25c2edd4b16 changeset: 8200:c25c2edd4b16 user: Ray Ferguson date: Sun Dec 02 08:14:35 2018 +0000 summary: Proper processing of gzip trailer without resubmission Supports pip and msgpack use with Jython. From https://github.com/jythontools/jython/pull/111 files: Lib/test/test_zlib_jy.py | 38 ++++++++++++++++++++++++ Lib/zlib.py | 43 ++++++++++++++++++--------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_zlib_jy.py b/Lib/test/test_zlib_jy.py --- a/Lib/test/test_zlib_jy.py +++ b/Lib/test/test_zlib_jy.py @@ -28,6 +28,44 @@ compressed = array('c', compress('jython')) self.assertEqual('jython', decompress(compressed)) + def test_decompress_gzip(self): + co = zlib.compressobj(wbits=31) # window 15 with gzip wrapper. + c = co.compress("Jenny: 867-5309") + c += co.flush() + dco = zlib.decompressobj(wbits=31) + d = dco.decompress(c) + self.assertEqual(b'', dco.unused_data, msg="dco.unused_data not empty after decompress.") + self.assertEqual(b'', dco.unconsumed_tail, msg="dco.unconsumed_tail not empty after decompress.") + self.assertEqual("Jenny: 867-5309", d) + + def test_decompress_badlen(self): + # Manipulating last two bytes to create invalid initial size check. + # RFC-1952: + # 0 1 2 3 4 5 6 7 + # +---+---+---+---+---+---+---+---+ + # | CRC32 | ISIZE | + # +---+---+---+---+---+---+---+---+turn: + # + c=b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0bJ\xacT(O,V\xc8H-J\x05\x00\xc2\xb0\x1e\xe5\x0d\x00\x00\x00' + dco = zlib.decompressobj(wbits=31) + self.assertRaisesRegexp(zlib.error, 'Error -3 while decompressing data: incorrect length check', + dco.decompress, c) + + def test_decompress_badcrc(self): + # Manipulating last crc bytes to create a crc check exception. + # RFC-1952: + # 0 1 2 3 4 5 6 7 + # +---+---+---+---+---+---+---+---+ + # | CRC32 | ISIZE | + # +---+---+---+---+---+---+---+---+turn: + # + c=b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0bJ\xacT(O,V\xc8H-J\x05\x00\xc2\xb0\x1f\xe5\x0c\x00\x00\x00' + dco = zlib.decompressobj(wbits=31) + self.assertRaisesRegexp(zlib.error, 'Error -3 while decompressing data: incorrect data check', + dco.decompress, c) + + + def test_main(): test_support.run_unittest(ArrayTestCase) diff --git a/Lib/zlib.py b/Lib/zlib.py --- a/Lib/zlib.py +++ b/Lib/zlib.py @@ -24,9 +24,6 @@ from java.util.zip import Adler32, CRC32, Deflater, Inflater, DataFormatException -class error(Exception): - pass - DEFLATED = 8 MAX_WBITS = 15 @@ -54,7 +51,8 @@ } -_ADLER_BASE = 65521 # largest prime smaller than 65536 +_ADLER_BASE = 65521 # largest prime smaller than 65536 +_MASK32 = 0xffffffffL # 2**32 - 1 used for unsigned mod 2**32 def adler32(s, value=1): # Although Java has an implmentation in java.util.zip.Adler32, @@ -110,6 +108,7 @@ # > system will be set to 255 (unknown). If a gzip stream is being # > written, strm->adler is a crc32 instead of an adler32. + class compressobj(object): # All jython uses wbits for is in deciding whether to skip the # header if it's negative or to set gzip. But we still raise @@ -118,10 +117,11 @@ GZIP_HEADER = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03" # NB: this format is little-endian, not big-endian as we might - # expect for network oriented protocols, as specified by RFCs; - # CRC32.getValue() returns an unsigned int as a long, so cope - # accordingly - GZIP_TRAILER_FORMAT = struct.Struct(" https://hg.python.org/jython/rev/8f1f265f7481 changeset: 8201:8f1f265f7481 user: Stefan Richthofer date: Tue Dec 04 17:44:05 2018 +0100 summary: Added Raymond Ferguson to ACKNOWLEDGEMENTS and his commit #111 to NEWS. files: ACKNOWLEDGMENTS | 1 + NEWS | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS --- a/ACKNOWLEDGMENTS +++ b/ACKNOWLEDGMENTS @@ -181,6 +181,7 @@ Alex Gaman Tom Bech Richie Bendall + Raymond Ferguson Local Variables: mode: indented-text diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Development tip Bugs fixed + - [ PR111 ] Proper processing of gzip trailer without resubmission - [ 2231 ] __doc__ of PyString objects is always "The most base type" - [ 2230 ] Jython evaluation blocks under heavy load with high multi-core systems - [ 2506 ] ensurepip is reporting an error -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Tue Dec 4 11:53:32 2018 From: jython-checkins at python.org (stefan.richthofer) Date: Tue, 04 Dec 2018 16:53:32 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Committed_=23114_=27Add_RE?= =?utf-8?q?ADME=2Emd=27?= Message-ID: <20181204165332.1.1B8131A6C1484521@mg.python.org> https://hg.python.org/jython/rev/231ce47330a9 changeset: 8202:231ce47330a9 user: James Mudd date: Tue Dec 04 17:53:19 2018 +0100 summary: Committed #114 'Add README.md' files: README.md | 20 ++++++++++++++++++++ 1 files changed, 20 insertions(+), 0 deletions(-) diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# Jython: Python for the Java Platform +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.python/jython-standalone/badge.svg)](https://search.maven.org/artifact/org.python/jython-standalone/) [![Javadocs](https://www.javadoc.io/badge/org.python/jython-standalone.svg)](https://www.javadoc.io/doc/org.python/jython-standalone) + +This is the development repository of Jython. Along with language and runtime compatibility with CPython 2.7, Jython 2.7 provides substantial support of the Python ecosystem. This includes built-in support of *pip/setuptools* (you can use with `bin/pip`) and a native launcher for Windows (`bin/jython.exe`), with the implication that you can finally install Jython scripts on Windows. + +**Note that if you have `JYTHON_HOME` set, you should unset it to avoid problems with the installer and pip/setuptools.** + +Jim Baker presented a talk at PyCon 2015 about Jython 2.7, including demos of new features: https://www.youtube.com/watch?v=hLm3garVQFo + +The release was compiled on OSX using JDK 7 and requires a minimum of Java 7 to run. + +Please try this release out and report any bugs at http://bugs.jython.org + +You can test your installation of Jython (not the standalone jar) by running the regression tests, with the command: +``` +jython -m test.regrtest -e -m regrtest_memo.txt +``` +For Windows, there is a simple script to do this: `jython_regrtest.bat`. In either case, the memo file `regrtest_memo.txt` will be useful in the bug report if you see test failures. The regression tests can take about half an hour. + +See [ACKNOWLEDGMENTS](ACKNOWLEDGMENTS) for details about Jython's copyright, license, contributors, and mailing lists; and [NEWS](NEWS) for detailed release notes, including bugs fixed, backwards breaking changes, and new features. We sincerely thank all who contribute to Jython, including - but not limited to - bug reports, patches, pull requests, documentation changes, support emails, and fantastic conversation on Freenode at #jython. Join us there for your questions and answers! -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Tue Dec 4 11:55:59 2018 From: jython-checkins at python.org (stefan.richthofer) Date: Tue, 04 Dec 2018 16:55:59 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Removed_README=2Etxt_in_fa?= =?utf-8?q?vor_of_new_README=2Emd?= Message-ID: <20181204165559.1.C8C2D9F9C824667E@mg.python.org> https://hg.python.org/jython/rev/e3aeea3760a3 changeset: 8203:e3aeea3760a3 user: James Mudd date: Tue Dec 04 17:55:48 2018 +0100 summary: Removed README.txt in favor of new README.md files: README.txt | 38 -------------------------------------- 1 files changed, 0 insertions(+), 38 deletions(-) diff --git a/README.txt b/README.txt deleted file mode 100644 --- a/README.txt +++ /dev/null @@ -1,38 +0,0 @@ -Jython: Python for the Java Platform - -Welcome to Jython 2.7.2a1. - -This is an alpha release of the 2.7.2 version of Jython. Along with -language and runtime compatibility with CPython 2.7, Jython 2.7 -provides substantial support of the Python ecosystem. This includes -built-in support of pip/setuptools (you can use with bin/pip) and a -native launcher for Windows (bin/jython.exe), with the implication -that you can finally install Jython scripts on Windows. - -**Note that if you have JYTHON_HOME set, you should unset it to avoid -problems with the installer and pip/setuptools.** - -Jim Baker presented a talk at PyCon 2015 about Jython 2.7, including -demos of new features: https://www.youtube.com/watch?v=hLm3garVQFo - -The release was compiled on OSX using JDK 7 and requires a minimum of -Java 7 to run. - -Please try this release out and report any bugs at -http://bugs.jython.org You can test your installation of Jython (not -the standalone jar) by running the regression tests, with the command: - -jython -m test.regrtest -e -m regrtest_memo.txt - -For Windows, there is a simple script to do this: jython_regrtest.bat. -In either case, the memo file regrtest_memo.txt will be useful in the -bug report if you see test failures. The regression tests can take -about half an hour. - -See ACKNOWLEDGMENTS for details about Jython's copyright, license, -contributors, and mailing lists; and NEWS for detailed release notes, -including bugs fixed, backwards breaking changes, and new features. We -sincerely thank all who contribute to Jython, including - but not -limited to - bug reports, patches, pull requests, documentation -changes, support emails, and fantastic conversation on Freenode at -#jython. Join us there for your questions and answers! -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Dec 20 09:38:10 2018 From: jython-checkins at python.org (jeff.allen) Date: Thu, 20 Dec 2018 14:38:10 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Locale=2C_windows/cygwin_a?= =?utf-8?q?nd_Java_version_failures_in_test=5Fos=5Fjy_fixes_=232714?= Message-ID: <20181220143810.1.91E49AFF4DD38D24@mg.python.org> https://hg.python.org/jython/rev/359e573ade00 changeset: 8204:359e573ade00 user: Adam Burke date: Thu Dec 20 05:48:58 2018 +0000 summary: Locale, windows/cygwin and Java version failures in test_os_jy fixes #2714 files: Lib/test/test_os_jy.py | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_os_jy.py b/Lib/test/test_os_jy.py --- a/Lib/test/test_os_jy.py +++ b/Lib/test/test_os_jy.py @@ -345,6 +345,7 @@ return available_codes # must be on posix and turkish locale supported + @unittest.skipIf(not test_support.is_jython_posix, "Not posix") def test_turkish_locale_posix_module(self): # Verifies fix of http://bugs.jython.org/issue1874 self.get_installed_locales(["tr_TR.UTF-8"], "Turkish locale not installed, cannot test") @@ -373,8 +374,8 @@ # # Note that JVMs seem to have some latitude here however, so support # either for now. - ["['i', u'\\u0131', 'I', u'\\u0130']\n", - "['i', u'i', 'I', u'I']\n"]) + ["['i', u'\\u0131', 'I', u'\\u0130']" + os.linesep, + "['i', u'i', 'I', u'I']" + os.linesep]) def test_strptime_locale(self): # Verifies fix of http://bugs.jython.org/issue2261 @@ -389,20 +390,26 @@ [sys.executable, "-c", 'import datetime; print(datetime.datetime.strptime("2015-01-22", "%Y-%m-%d"))'], env=newenv), - "2015-01-22 00:00:00\n") + "2015-01-22 00:00:00" + os.linesep) def test_strftime_japanese_locale(self): # Verifies fix of http://bugs.jython.org/issue2301 - produces # UTF-8 encoded output per what CPython does, rather than Unicode. # We will revisit in Jython 3.x! self.get_installed_locales("ja_JP.UTF-8") + if test_support.get_java_version() < (10,): + expected = "'\\xe6\\x97\\xa5 3 29 14:55:13 2015'" + else: + # From Java 10 onwards, Japanese formatting more correctly includes + # ?, the kanji character for month + expected = "'\\xe6\\x97\\xa5 3\\xe6\\x9c\\x88 29 14:55:13 2015'" self.assertEqual( subprocess.check_output( [sys.executable, "-J-Duser.country=JP", "-J-Duser.language=ja", "-c", "import time; print repr(time.strftime('%c', (2015, 3, 29, 14, 55, 13, 6, 88, 0)))"]), - "'\\xe6\\x97\\xa5 3 29 14:55:13 2015'\n") + expected + os.linesep) class SystemTestCase(unittest.TestCase): -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Dec 20 17:49:34 2018 From: jython-checkins at python.org (jeff.allen) Date: Thu, 20 Dec 2018 22:49:34 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_divergent_=5F=5Fstr=5F?= =?utf-8?q?=5F_and_=5F=5Frepr=5F=5F_of_type_buffer=2E_Fixes_=232719=2E?= Message-ID: <20181220224934.1.E6FB596E0E889540@mg.python.org> https://hg.python.org/jython/rev/70c3f2ca1fa0 changeset: 8205:70c3f2ca1fa0 user: Jeff Allen date: Thu Dec 20 21:11:58 2018 +0000 summary: Fix divergent __str__ and __repr__ of type buffer. Fixes #2719. This change addresses an issue explored by Ray Ferguson in GH-126 concerning a lack of clarity, and a potential stack overflow, in the way we expose __str__ and __repr__, and consequent errors in buffer.__str__. The lack of clarity is dealt with in a note (see PyObject.__repr__). A less confusing rework of __str__, __repr__ and toString may be in order, but we are not able to use Ray's approach in GH-126 directly. files: Lib/test/test_bytes_jy.py | 27 +++++++- NEWS | 12 ++- src/org/python/core/Py2kBuffer.java | 58 ++++++++-------- src/org/python/core/PyFunction.java | 16 ++++- src/org/python/core/PyObject.java | 31 +++++--- 5 files changed, 97 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_bytes_jy.py b/Lib/test/test_bytes_jy.py --- a/Lib/test/test_bytes_jy.py +++ b/Lib/test/test_bytes_jy.py @@ -16,8 +16,8 @@ self.assertEqual(len(s), 6) -class SimpleOperationsTest(unittest.TestCase): - # Things the CPython library did not test throughly enough +class BytesOperationsTest(unittest.TestCase): + # Tests additional to the CPython library, on bytes/str def test_irepeat(self) : @@ -178,10 +178,31 @@ self.assertEqual(1, len(s.rsplit()), "rsplit made in " + repr(s)) +class BufferOperationsTest(unittest.TestCase): + # Tests additional to the CPython library, on buffer + + def test_repr(self): + # Exact form of buffer.__repr__ + text = u"Le d?ner ? ?tretat".encode('latin-1') + buf = buffer(text) + expected = r"" + self.assertRegexpMatches(repr(buf), expected) + self.assertRegexpMatches(buf.__repr__(), expected) + + def test_str(self): + # Exact form of buffer.__str__ + text = u"Le d?ner ? ?tretat".encode('latin-1') + buf = buffer(text) + expected = text + self.assertEqual(str(buf), expected) + self.assertEqual(buf.__str__(), expected) + + def test_main(): test.test_support.run_unittest( ByteArraySubclassTest, - SimpleOperationsTest, + BytesOperationsTest, + BufferOperationsTest, ) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -1,10 +1,13 @@ Jython NEWS -For more details, please see https://hg.python.org/jython +For more details of issue [ n ], please see https://hg.python.org/jython, or for tags [ GH-n ] see +https://github.com/jythontools/jython Development tip Bugs fixed - - [ PR111 ] Proper processing of gzip trailer without resubmission + - [ 2719 ] Divergence of __str__ and __repr__ from CPython + - [ 2714 ] Locale and java version failures in test_os_jy + - [ GH-111 ] Proper processing of gzip trailer without resubmission - [ 2231 ] __doc__ of PyString objects is always "The most base type" - [ 2230 ] Jython evaluation blocks under heavy load with high multi-core systems - [ 2506 ] ensurepip is reporting an error @@ -45,6 +48,9 @@ change is for consistency with CPython and with our handling of other environment variables. A pure Java application that creates its own interpreter may use the system or registry key "python.path" to add to sys.path, as documented. + - We no longer recommend overriding toString in PyObject sub-classes as a way of defining + string forms. Override __repr__, and if necessary __str__, as is usual in Python. We + recommend toString should return the same as __str__ (or __unicode__). Jython 2.7.2a1 Bugs fixed @@ -144,7 +150,7 @@ - [ 2533 ] Opcode.java is outdated -> breaks PyBytecode.interpret - [ 2502 ] Missing OpenFlags enum entry makes Jython clash with JRuby dependency - [ 2446 ] Support SNI for SSL/TLS client sockets - - [ PR50 ] Calling Java vararg methods with no arguments fails + - [ GH-50 ] Calling Java vararg methods with no arguments fails - [ 2455 ] Java classes in packages with __init__.py not found - [ 2481 ] Update urllib2.py from 2.7.11 - [ 2514 ] Jython Class.__subclasses__() does not match Python output (not in load order) diff --git a/src/org/python/core/Py2kBuffer.java b/src/org/python/core/Py2kBuffer.java --- a/src/org/python/core/Py2kBuffer.java +++ b/src/org/python/core/Py2kBuffer.java @@ -41,7 +41,7 @@ * action performed obtains a new one and releases it. (Major difference from * memoryview.) Note that when size=-1 is given, the buffer reflects * the changing size of the underlying object. - * + * * @param object the object on which this is to be a buffer. * @param offset into the array exposed by the object (0 for start). * @param size of the slice or -1 for all of the object. @@ -72,7 +72,7 @@ /** * Every action on the buffer must obtain a new {@link PyBuffer} reflecting (this * buffer's slice of) the contents of the backing object. - * + * * @return a PyBuffer onto the specified slice. */ private PyBuffer getBuffer() { @@ -120,7 +120,7 @@ * except in the case of a {@link PyUnicode}, which will be converted to a {@link PyString} * according to Py2k semantics, equivalent to a UTF16BE encoding to bytes (for Py2k * compatibility). - * + * * @param obj the object to access. * @return PyObject supporting {@link BufferProtocol}, if not null. */ @@ -186,25 +186,25 @@ @Override public PyString __repr__() { - String fmt = ""; - String ret = String.format(fmt, Py.idstr((PyObject)object), size, offset, Py.idstr(this)); + return buffer___repr__(); + } + + @ExposedMethod(doc = BuiltinDocs.buffer___repr___doc) + final PyString buffer___repr__() { + String fmt = ""; + String ret = String.format(fmt, Py.idstr((PyObject) object), size, offset, Py.idstr(this)); return new PyString(ret); } @Override public PyString __str__() { - PyBuffer buf = getBuffer(); - try { - if (buf instanceof BaseBuffer) { - // In practice, it always is - return new PyString(buf.toString()); - } else { - // But just in case ... - String s = StringUtil.fromBytes(buf); - return new PyString(s); - } - } finally { - buf.release(); + return buffer___str__(); + } + + @ExposedMethod(doc = BuiltinDocs.buffer___str___doc) + final PyString buffer___str__() { + try (PyBuffer buf = getBuffer()) { + return new PyString(buf.toString()); } } @@ -292,7 +292,7 @@ * conventions, left-to-right (low to high index). Zero bytes are significant, even at the end * of the array: [65,66,67]<"ABC\u0000", for example and [] is less * than every non-empty b, while []=="". - * + * * @param a left-hand wrapped array in the comparison * @param b right-hand wrapped object in the comparison * @return 1, 0 or -1 as a>b, a==b, or a<b respectively @@ -333,7 +333,7 @@ /** * Comparison function between this buffer and any other object. The inequality * comparison operators are based on this. - * + * * @param b * @return 1, 0 or -1 as this>b, this==b, or this<b respectively, or -2 if the comparison is * not implemented @@ -377,7 +377,7 @@ * Fail-fast comparison function between byte array types and any other object, for when the * test is only for equality. The inequality comparison operators __eq__ and * __ne__ are based on this. - * + * * @param b * @return 0 if this==b, or +1 or -1 if this!=b, or -2 if the comparison is not implemented */ @@ -487,7 +487,7 @@ * Gets the indexed element of the buffer as a one byte string. This is an * extension point called by PySequence in its implementation of {@link #__getitem__}. It is * guaranteed by PySequence that the index is within the bounds of the buffer. - * + * * @param index index of the element to get. * @return one-character string formed from the byte at the index */ @@ -505,7 +505,7 @@ /** * Returns a slice of elements from this sequence as a PyString. - * + * * @param start the position of the first element. * @param stop one more than the position of the last element. * @param step the step size. @@ -530,7 +530,7 @@ /** * buffer*int represents repetition in Python, and returns a str ( * bytes) object. - * + * * @param count the number of times to repeat this. * @return a PyString repeating this buffer (as a str) that many times */ @@ -553,7 +553,7 @@ * {@link #__setitem__} It is guaranteed by PySequence that the index is within the bounds of * the buffer. Any other clients calling pyset(int, PyObject) must make * the same guarantee. - * + * * @param index index of the element to set. * @param value to set this element to, regarded as a buffer of length one unit. * @throws PyException(AttributeError) if value cannot be converted to an integer @@ -585,13 +585,13 @@ * Sets the given range of elements according to Python slice assignment semantics. If the step * size is one, it is a simple slice and the operation is equivalent to replacing that slice, * with the value, accessing the value via the buffer protocol. - * + * * 

      * a = bytearray(b'abcdefghijklmnopqrst')
      * m = buffer(a)
      * m[2:7] = "ABCDE"
      *
- * + * * Results in a=bytearray(b'abABCDEhijklmnopqrst'). *

* If the step size is one, but stop-start does not match the length of the right-hand-side a @@ -600,14 +600,14 @@ * If the step size is not one, and start!=stop, the slice defines a certain number of elements * to be replaced. This function is not available in Python 2.7 (but it is in Python 3.3). *

- * + * * 

      * a = bytearray(b'abcdefghijklmnopqrst')
      * a[2:12:2] = iter( [65, 66, 67, long(68), "E"] )
      *
- * + * * Results in a=bytearray(b'abAdBfChDjElmnopqrst') in Python 3.3. - * + * * @param start the position of the first element. * @param stop one more than the position of the last element. * @param step the step size. diff --git a/src/org/python/core/PyFunction.java b/src/org/python/core/PyFunction.java --- a/src/org/python/core/PyFunction.java +++ b/src/org/python/core/PyFunction.java @@ -273,12 +273,14 @@ return __dict__; } + @Override @ExposedGet(name = "__dict__") public PyObject getDict() { ensureDict(); return __dict__; } + @Override @ExposedSet(name = "__dict__") public void setDict(PyObject value) { if (!(value instanceof AbstractDict)) { @@ -287,6 +289,7 @@ __dict__ = value; } + @Override @ExposedDelete(name = "__dict__") public void delDict() { throw Py.TypeError("function's dictionary may not be deleted"); @@ -483,6 +486,16 @@ } @Override + public PyString __repr__() { + return function___repr__(); + } + + @ExposedMethod(doc = BuiltinDocs.buffer___repr___doc) + final PyString function___repr__() { + return Py.newString(toString()); + } + + @Override public String toString() { return String.format("", __name__, Py.idstr(this)); } @@ -525,6 +538,7 @@ return Proxy.newProxyInstance(c.getClassLoader(), new Class[]{c}, this); } + @Override public Object invoke( Object proxy, Method method, Object[] args ) throws Throwable { // Handle invocation when invoked through Proxy (as coerced to single method interface) if (method.getDeclaringClass() == Object.class) { @@ -583,7 +597,7 @@ if (retVal != 0) { return retVal; } - + // CPython also traverses the name, which is not stored // as a PyObject in Jython: // Py_VISIT(f->func_name); diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java --- a/src/org/python/core/PyObject.java +++ b/src/org/python/core/PyObject.java @@ -216,15 +216,21 @@ } /** - * Equivalent to the standard Python __repr__ method. This method should not typically need to - * be overrriden. The easiest way to configure the string representation of a - * PyObject is to override the standard Java toString method. + * Equivalent to the standard Python __repr__ method. Each sub-class of + * PyObject is likely to re-define this method to provide for its own reproduction. **/ /* - * counter-intuitively exposing this as __str__, otherwise stack overflow occurs during - * regression testing. + * The effect of exposing __repr__ as __str__ is that a Python call to o.__str__() will land + * here. (A Java call to o.__str__() lands here too because __str__ is defined to call + * __repr__.) This will continue to be true in any sub-class that does not expose a __str__ of + * its own. (Such a class should override Java __str__ to call the method exposed as Python + * __str__.) Note that we expose a non-final method, therefore in a class that (Java-)overrides + * __repr__, Python (and Java) calls like o.__str__() will land on the overridden __repr__. + * + * This design, though long-standing, has caused confusion to the implementors of types, and + * seems to make *Derived.java classes more complicated. We should seek a more transparent + * design. */ - // XXX: more detail for this comment is needed. @ExposedMethod(names = "__str__", doc = BuiltinDocs.object___str___doc) public PyString __repr__() { return new PyString(toString()); @@ -254,9 +260,12 @@ } /** - * Equivalent to the standard Python __str__ method. This method should not typically need to be - * overridden. The easiest way to configure the string representation of a PyObject - * is to override the standard Java toString method. + * Equivalent to the standard Python __str__ method. The default implementation (in + * PyObject) calls {@link #__repr__()}, making it unnecessary to override + * __str__ in sub-classes of PyObject where both forms are the same. A + * common choice is to provide the same implementation to __str__ and + * toString, for consistency in the printed form of objects between Python and + * Java. **/ public PyString __str__() { return __repr__(); @@ -533,8 +542,8 @@ } for (String keyword : keywords) { if (kwargs.__finditem__(keyword) != null) { - throw Py.TypeError(name + "got multiple values for keyword argument '" - + keyword + "'"); + throw Py.TypeError( + name + "got multiple values for keyword argument '" + keyword + "'"); } } argslen += kwargs.__len__(); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Dec 28 10:42:04 2018 From: jython-checkins at python.org (jeff.allen) Date: Fri, 28 Dec 2018 15:42:04 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ugrade_ANTLR_to_3=2E5=2E2?= Message-ID: <20181228154204.1.24383413B23C7BA3@mg.python.org> https://hg.python.org/jython/rev/547c523f81c7 changeset: 8206:547c523f81c7 user: Jeff Allen date: Thu Dec 27 11:01:53 2018 +0000 summary: Ugrade ANTLR to 3.5.2 The aim is to get rid of the stack dumps that ANTLR 3.1.3 produces running on Java 8. This requires some tweaks to the grammar actions since ANTLR 3.5.2 generates List where it used to have a raw List. Also, the ANTLR Lexer now returns real EOF tokens that CommonTokenStream buffers, which we are careful not to consume in PythonTokenSource in nextToken() or insertImaginaryIndentDedentTokens(). A minimum of change restores passing behaviour. Some irregularities noticed during debugging are left for later work. files: build.xml | 9 +- extlibs/antlr-3.1.3.jar | Bin extlibs/antlr-complete-3.5.2.jar | Bin extlibs/antlr-runtime-3.1.3.jar | Bin extlibs/antlr-runtime-3.5.2.jar | Bin grammar/Python.g | 83 ++++++--- grammar/PythonPartial.g | 31 ++- src/org/python/antlr/GrammarActions.java | 10 +- src/org/python/antlr/PythonTokenSource.java | 23 +- src/org/python/indexer/demos/Styler.java | 2 +- 10 files changed, 97 insertions(+), 61 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -144,9 +144,8 @@ - - - + + @@ -547,8 +546,8 @@ - - + + diff --git a/extlibs/antlr-3.1.3.jar b/extlibs/antlr-3.1.3.jar deleted file mode 100644 index 0ec52f864ebf29a8d3ba8fa21364c3ef687c7fdd..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/antlr-complete-3.5.2.jar b/extlibs/antlr-complete-3.5.2.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..260de7634889880167e5045e404774f1d3680b04 GIT binary patch [stripped] diff --git a/extlibs/antlr-runtime-3.1.3.jar b/extlibs/antlr-runtime-3.1.3.jar deleted file mode 100644 index b0a9ea69f5c29097145a48c26e127972920db440..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/extlibs/antlr-runtime-3.5.2.jar b/extlibs/antlr-runtime-3.5.2.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d48e3e867968c7fe114e45168afb71c9a3cdf124 GIT binary patch [stripped] diff --git a/grammar/Python.g b/grammar/Python.g --- a/grammar/Python.g +++ b/grammar/Python.g @@ -239,12 +239,15 @@ } /** - * Taken directly from antlr's Lexer.java -- needs to be re-integrated every time - * we upgrade from Antlr (need to consider a Lexer subclass, though the issue would - * remain). + * The text of this is mostly taken directly from ANTLR's Lexer.java, + * and ought to track changes there each time we get a new version, + * ... if there are any after 3.5.2. Also in PythonPartial.g. */ + @Override public Token nextToken() { + // -- begin Jython addition startPos = getCharPositionInLine(); + // -- end Jython addition while (true) { state.token = null; state.channel = Token.DEFAULT_CHANNEL; @@ -253,10 +256,12 @@ state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { + // -- begin Jython addition if (implicitLineJoiningLevel > 0) { eofWhileNested = true; } - return Token.EOF_TOKEN; + // -- end Jython addition + return getEOFToken(); } try { mTokens(); @@ -267,21 +272,30 @@ continue; } return state.token; + // -- begin Jython addition } catch (NoViableAltException nva) { reportError(nva); errorHandler.recover(this, nva); // throw out current char and try again } catch (FailedPredicateException fp) { - //XXX: added this for failed STRINGPART -- the FailedPredicateException - // hides a NoViableAltException. This should be the only - // FailedPredicateException that gets thrown by the lexer. + // Added this for failed STRINGPART -- the FailedPredicateException + // hides a NoViableAltException. This should be the only + // FailedPredicateException that gets thrown by the lexer. reportError(fp); errorHandler.recover(this, fp); // throw out current char and try again + // -- end Jython addition + } catch (MismatchedRangeException re) { + reportError(re); + // matchRange() routine has already called recover() + } catch (MismatchedTokenException re) { + reportError(re); + // match() routine has already called recover() } catch (RecognitionException re) { reportError(re); - // match() routine has already called recover() + recover(re); // throw out current char and try again } } } + @Override public void displayRecognitionError(String[] tokenNames, RecognitionException e) { //Do nothing. We will handle error display elsewhere. @@ -995,10 +1009,9 @@ //import_as_names: import_as_name (',' import_as_name)* [','] import_as_names returns [List atypes] - : n+=import_as_name (COMMA! n+=import_as_name)* - { - $atypes = $n; - } + @init{$atypes = new ArrayList();} + : n=import_as_name {$atypes.add($n.atype);} + (COMMA! n=import_as_name {$atypes.add($n.atype);})* ; //import_as_name: NAME [('as' | NAME) NAME] @@ -1029,32 +1042,31 @@ //dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_as_names returns [List atypes] - : d+=dotted_as_name (COMMA! d+=dotted_as_name)* - { - $atypes = $d; - } + @init{$atypes = new ArrayList();} + : d=dotted_as_name {$atypes.add($d.atype);} + (COMMA! d=dotted_as_name {$atypes.add($d.atype);})* ; //dotted_name: NAME ('.' NAME)* dotted_name returns [List names] - : NAME (DOT dn+=attr)* - { - $names = actions.makeDottedName($NAME, $dn); - } + @init{List dnList = new ArrayList<>();} + : NAME (DOT dn=attr {dnList.add($dn.tree);})* + {$names = actions.makeDottedName($NAME, dnList);} ; //global_stmt: 'global' NAME (',' NAME)* global_stmt @init { stmt stype = null; + List names = new ArrayList<>(); } @after { $global_stmt.tree = stype; } - : GLOBAL n+=NAME (COMMA n+=NAME)* + : GLOBAL n=NAME {names.add($n);} (COMMA n=NAME {names.add($n);})* { - stype = new Global($GLOBAL, actions.makeNames($n), actions.makeNameNodes($n)); + stype = new Global($GLOBAL, actions.makeNames(names), actions.makeNameNodes(names)); } ; @@ -1185,14 +1197,17 @@ try_stmt @init { stmt stype = null; + List exceptClauses = new ArrayList<>(); } @after { $try_stmt.tree = stype; } : TRY COLON trysuite=suite[!$suite.isEmpty() && $suite::continueIllegal] - ( e+=except_clause+ (ORELSE COLON elsesuite=suite[!$suite.isEmpty() && $suite::continueIllegal])? (FINALLY COLON finalsuite=suite[true])? + ( (e=except_clause {exceptClauses.add((excepthandler)$e.tree);})+ + (ORELSE COLON elsesuite=suite[!$suite.isEmpty() && $suite::continueIllegal])? + (FINALLY COLON finalsuite=suite[true])? { - stype = actions.makeTryExcept($TRY, $trysuite.stypes, $e, $elsesuite.stypes, $finalsuite.stypes); + stype = actions.makeTryExcept($TRY, $trysuite.stypes, exceptClauses, $elsesuite.stypes, $finalsuite.stypes); } | FINALLY COLON finalsuite=suite[true] { @@ -1205,14 +1220,15 @@ with_stmt @init { stmt stype = null; + List withList = new ArrayList<>(); } @after { $with_stmt.tree = stype; } - : WITH w+=with_item (options {greedy=true;}:COMMA w+=with_item)* COLON suite[false] - { - stype = actions.makeWith($WITH, $w, $suite.stypes); - } + : WITH w=with_item {withList.add((With)$w.tree);} + (options {greedy=true;}:COMMA w=with_item {withList.add((With)$w.tree);})* + COLON suite[false] + {stype = actions.makeWith($WITH, withList, $suite.stypes);} ; //with_item: test ['as' expr] @@ -1252,7 +1268,7 @@ //suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT suite - [boolean fromFinally] returns [List stypes] + [boolean fromFinally] returns [List stypes] scope { boolean continueIllegal; } @@ -1966,10 +1982,11 @@ //Needed as an exprlist that does not produce tuples for del_stmt. del_list returns [List etypes] - : e+=expr[expr_contextType.Del] (options {k=2;}: COMMA e+=expr[expr_contextType.Del])* (COMMA)? - { - $etypes = actions.makeDeleteList($e); - } + @init{List exprList = new ArrayList<>();} + : e=expr[expr_contextType.Del] {exprList.add($e.tree);} + (options {k=2;}: COMMA e=expr[expr_contextType.Del] + {exprList.add($e.tree);})* (COMMA)? + {$etypes = actions.makeDeleteList(exprList);} ; //testlist: test (',' test)* [','] diff --git a/grammar/PythonPartial.g b/grammar/PythonPartial.g --- a/grammar/PythonPartial.g +++ b/grammar/PythonPartial.g @@ -116,12 +116,15 @@ } /** - * Taken directly from antlr's Lexer.java -- needs to be re-integrated every time - * we upgrade from Antlr (need to consider a Lexer subclass, though the issue would - * remain). + * The text of this is mostly taken directly from ANTLR's Lexer.java, + * and ought to track changes there each time we get a new version, + * ... if there are any after 3.5.2. Also in PythonPartial.g. */ + @Override public Token nextToken() { + // -- begin Jython addition startPos = getCharPositionInLine(); + // -- end Jython addition while (true) { state.token = null; state.channel = Token.DEFAULT_CHANNEL; @@ -130,10 +133,12 @@ state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { + // -- begin Jython addition if (implicitLineJoiningLevel > 0) { eofWhileNested = true; } - return Token.EOF_TOKEN; + // -- end Jython addition + return getEOFToken(); } try { mTokens(); @@ -144,18 +149,28 @@ continue; } return state.token; + // -- begin Jython addition } catch (NoViableAltException nva) { errorHandler.reportError(this, nva); errorHandler.recover(this, nva); // throw out current char and try again } catch (FailedPredicateException fp) { - //XXX: added this for failed STRINGPART -- the FailedPredicateException - // hides a NoViableAltException. This should be the only - // FailedPredicateException that gets thrown by the lexer. + // Added this for failed STRINGPART -- the FailedPredicateException + // hides a NoViableAltException. This should be the only + // FailedPredicateException that gets thrown by the lexer. errorHandler.reportError(this, fp); errorHandler.recover(this, fp); // throw out current char and try again + // -- end Jython addition + } catch (MismatchedRangeException re) { + reportError(re); + // matchRange() routine has already called recover() + } catch (MismatchedTokenException re) { + reportError(re); + // match() routine has already called recover() } catch (RecognitionException re) { + // -- Jython replaces: reportError(this, re) with: errorHandler.reportError(this, re); - // match() routine has already called recover() + // -- end Jython replacement + recover(re); // throw out current char and try again } } } diff --git a/src/org/python/antlr/GrammarActions.java b/src/org/python/antlr/GrammarActions.java --- a/src/org/python/antlr/GrammarActions.java +++ b/src/org/python/antlr/GrammarActions.java @@ -129,10 +129,10 @@ return result; } - List makeNames(List names) { + List makeNames(List names) { List s = new ArrayList(); - for(int i=0;i makeNameNodes(List names) { List s = new ArrayList(); - for (int i=0; i 0) { Token t = tokens.firstElement(); - tokens.removeElementAt(0); - //System.out.println(filename + t); + if (t.getType() != Token.EOF) { // EOF stops further insertImaginaryIndentDedentTokens + tokens.removeElementAt(0); + } + // System.out.println(filename + t); return t; } @@ -165,7 +167,6 @@ protected void insertImaginaryIndentDedentTokens() { Token t = stream.LT(1); - stream.consume(); if (t.getType() == Token.EOF) { Token prev = stream.LT(-1); @@ -187,13 +188,12 @@ } else if (t.getType() == PythonLexer.NEWLINE) { // save NEWLINE in the queue //System.out.println("found newline: "+t+" stack is "+stackString()); - enqueueHiddens(t); - tokens.addElement(t); + enqueue(t); Token newline = t; + stream.consume(); // grab first token of next line t = stream.LT(1); - stream.consume(); List commentedNewlines = enqueueHiddens(t); @@ -204,13 +204,15 @@ cpos = -1; // pretend EOF always happens at left edge } else if (t.getType() == PythonLexer.LEADING_WS) { + stream.consume(); Token next = stream.LT(1); if (next != null && next.getType() == Token.EOF) { - stream.consume(); return; } else { cpos = t.getText().length(); } + } else { + stream.consume(); } //System.out.println("next token is: "+t); @@ -241,9 +243,10 @@ } else { enqueue(t); + stream.consume(); } } - + private void enqueue(Token t) { enqueueHiddens(t); tokens.addElement(t); @@ -276,7 +279,9 @@ } } } - List hiddenTokens = stream.getTokens(lastTokenAddedIndex + 1,t.getTokenIndex() - 1); + + List hiddenTokens = + stream.getTokens(lastTokenAddedIndex + 1, t.getTokenIndex() - 1); if (hiddenTokens != null) { tokens.addAll(hiddenTokens); } diff --git a/src/org/python/indexer/demos/Styler.java b/src/org/python/indexer/demos/Styler.java --- a/src/org/python/indexer/demos/Styler.java +++ b/src/org/python/indexer/demos/Styler.java @@ -151,7 +151,7 @@ }); Token tok; - while ((tok = lex.nextToken()) != Token.EOF_TOKEN) { + while ((tok = lex.nextToken()).getType() != Token.EOF) { switch (tok.getType()) { case PythonLexer.STRING: { int beg = ((CommonToken)tok).getStartIndex(); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Dec 28 10:42:04 2018 From: jython-checkins at python.org (jeff.allen) Date: Fri, 28 Dec 2018 15:42:04 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Respond_to_deprecation_war?= =?utf-8?q?nings_in_ASM_and_JFFI=2E?= Message-ID: <20181228154204.1.34B970CD175D83EE@mg.python.org> https://hg.python.org/jython/rev/d18511485bd5 changeset: 8207:d18511485bd5 user: Jeff Allen date: Thu Dec 27 14:08:48 2018 +0000 summary: Respond to deprecation warnings in ASM and JFFI. Mechanical replacement of calls with (what seems to be) the intended alternative. Regression tests have run ok. files: src/org/python/expose/generate/NewExposer.java | 2 +- src/org/python/modules/jffi/SkinnyMethodAdapter.java | 15 ++++++--- src/org/python/modules/jffi/StructLayout.java | 4 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/org/python/expose/generate/NewExposer.java b/src/org/python/expose/generate/NewExposer.java --- a/src/org/python/expose/generate/NewExposer.java +++ b/src/org/python/expose/generate/NewExposer.java @@ -50,7 +50,7 @@ mv.visitVarInsn(ALOAD, 2); mv.visitVarInsn(ALOAD, 3); mv.visitVarInsn(ALOAD, 4); - mv.visitMethodInsn(INVOKESTATIC, onType.getInternalName(), name, NEW_DESCRIPTOR); + mv.visitMethodInsn(INVOKESTATIC, onType.getInternalName(), name, NEW_DESCRIPTOR, false); endMethod(ARETURN); } diff --git a/src/org/python/modules/jffi/SkinnyMethodAdapter.java b/src/org/python/modules/jffi/SkinnyMethodAdapter.java --- a/src/org/python/modules/jffi/SkinnyMethodAdapter.java +++ b/src/org/python/modules/jffi/SkinnyMethodAdapter.java @@ -153,19 +153,19 @@ } public void invokestatic(String arg1, String arg2, String arg3) { - getMethodVisitor().visitMethodInsn(INVOKESTATIC, arg1, arg2, arg3); + getMethodVisitor().visitMethodInsn(INVOKESTATIC, arg1, arg2, arg3, false); } public void invokespecial(String arg1, String arg2, String arg3) { - getMethodVisitor().visitMethodInsn(INVOKESPECIAL, arg1, arg2, arg3); + getMethodVisitor().visitMethodInsn(INVOKESPECIAL, arg1, arg2, arg3, false); } public void invokevirtual(String arg1, String arg2, String arg3) { - getMethodVisitor().visitMethodInsn(INVOKEVIRTUAL, arg1, arg2, arg3); + getMethodVisitor().visitMethodInsn(INVOKEVIRTUAL, arg1, arg2, arg3, false); } public void invokeinterface(String arg1, String arg2, String arg3) { - getMethodVisitor().visitMethodInsn(INVOKEINTERFACE, arg1, arg2, arg3); + getMethodVisitor().visitMethodInsn(INVOKEINTERFACE, arg1, arg2, arg3, true); } public void aprintln() { @@ -840,8 +840,13 @@ getMethodVisitor().visitFieldInsn(arg0, arg1, arg2, arg3); } + @Deprecated public void visitMethodInsn(int arg0, String arg1, String arg2, String arg3) { - getMethodVisitor().visitMethodInsn(arg0, arg1, arg2, arg3); + getMethodVisitor().visitMethodInsn(arg0, arg1, arg2, arg3, arg0 == Opcodes.INVOKEINTERFACE); + } + + public void visitMethodInsn(int arg0, String arg1, String arg2, String arg3, boolean arg4) { + getMethodVisitor().visitMethodInsn(arg0, arg1, arg2, arg3, arg4); } public void visitJumpInsn(int arg0, Label arg1) { diff --git a/src/org/python/modules/jffi/StructLayout.java b/src/org/python/modules/jffi/StructLayout.java --- a/src/org/python/modules/jffi/StructLayout.java +++ b/src/org/python/modules/jffi/StructLayout.java @@ -140,8 +140,8 @@ } com.kenai.jffi.Type jffiType = isUnion - ? new com.kenai.jffi.Union(fieldTypes) - : new com.kenai.jffi.Struct(fieldTypes); + ? com.kenai.jffi.Union.newUnion(fieldTypes) + : com.kenai.jffi.Struct.newStruct(fieldTypes); return new StructLayout(fields, jffiType, MemoryOp.INVALID); } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Dec 28 10:42:05 2018 From: jython-checkins at python.org (jeff.allen) Date: Fri, 28 Dec 2018 15:42:05 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_failure_in_test=5Fjava?= =?utf-8?q?=5Fsubclasses=2E?= Message-ID: <20181228154205.1.308E640FFB668BDE@mg.python.org> https://hg.python.org/jython/rev/8c2c15c2aa62 changeset: 8209:8c2c15c2aa62 user: Jeff Allen date: Fri Dec 28 11:48:19 2018 +0000 summary: Fix failure in test_java_subclasses. Previous change to newInstance() in PyObject raises different exceptions to signify absence of a default constructor. Failed in test_java_subclasses.AutoSuperTest.test_no_default_constructor. files: src/org/python/core/PyObject.java | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java --- a/src/org/python/core/PyObject.java +++ b/src/org/python/core/PyObject.java @@ -189,15 +189,13 @@ try { try { proxy = (PyProxy) c.getDeclaredConstructor().newInstance(); - } catch (java.lang.InstantiationException e) { + } catch (InstantiationException | NoSuchMethodException e) { Class sup = c.getSuperclass(); - String msg = "Default constructor failed for Java superclass"; + String msg = "Default constructor failed/missing for Java superclass"; if (sup != null) { msg += " " + sup.getName(); } throw Py.TypeError(msg); - } catch (NoSuchMethodError nsme) { - throw Py.TypeError("constructor requires arguments"); } catch (Exception exc) { throw Py.JavaError(exc); } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Dec 28 10:42:05 2018 From: jython-checkins at python.org (jeff.allen) Date: Fri, 28 Dec 2018 15:42:05 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_A_little_less_noise_from_A?= =?utf-8?q?NTLR_during_the_build=2E?= Message-ID: <20181228154205.1.FD3E3D5E652C0000@mg.python.org> https://hg.python.org/jython/rev/828c41688a52 changeset: 8210:828c41688a52 user: Jeff Allen date: Fri Dec 28 14:25:56 2018 +0000 summary: A little less noise from ANTLR during the build. Introduces testlist1, as in the Python grammar, and answers a long-standing question in the lexer. files: grammar/Python.g | 30 +++++++++++++++++++++++----- grammar/PythonPartial.g | 16 ++++++++++---- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/grammar/Python.g b/grammar/Python.g --- a/grammar/Python.g +++ b/grammar/Python.g @@ -1770,9 +1770,9 @@ } ) RCURLY - | lb=BACKQUOTE testlist[expr_contextType.Load] rb=BACKQUOTE + | lb=BACKQUOTE testlist1[expr_contextType.Load] rb=BACKQUOTE { - etype = new Repr($lb, actions.castExpr($testlist.tree)); + etype = new Repr($lb, actions.castExpr($testlist1.tree)); } | name_or_print { @@ -2206,6 +2206,26 @@ } ; +// Variant of testlist used between BACKQUOTEs (the deprecated back-tick repr()) only +//testlist1: test (',' test)* +testlist1[expr_contextType ctype] + at init { + expr etype = null; +} + at after { + if (etype != null) { + $testlist1.tree = etype; + } +} + : t+=test[ctype] + ( + COMMA t+=test[ctype] + { + etype = new Tuple($testlist1.start, actions.castExprs($t), ctype); + } + )* + ; + //yield_expr: 'yield' [testlist] yield_expr returns [expr etype] @@ -2400,16 +2420,14 @@ } ; -/** the two '"'? cause a warning -- is there a way to avoid that? */ fragment TRIQUOTE - : '"'? '"'? (ESC|~('\\'|'"'))+ + : ('"' '"'?)? (ESC|~('\\'|'"'))+ ; -/** the two '\''? cause a warning -- is there a way to avoid that? */ fragment TRIAPOS - : '\''? '\''? (ESC|~('\\'|'\''))+ + : ('\'' '\''?)? (ESC|~('\\'|'\''))+ ; fragment diff --git a/grammar/PythonPartial.g b/grammar/PythonPartial.g --- a/grammar/PythonPartial.g +++ b/grammar/PythonPartial.g @@ -187,7 +187,7 @@ //eval_input: testlist NEWLINE* ENDMARKER eval_input - : LEADING_WS? (NEWLINE)* testlist? (NEWLINE)* EOF + : LEADING_WS? (NEWLINE)* (testlist (NEWLINE)*)? EOF ; //not in CPython's Grammar file @@ -744,7 +744,7 @@ | ) RCURLY - | BACKQUOTE testlist BACKQUOTE + | BACKQUOTE testlist1 BACKQUOTE | NAME | INT | LONGINT @@ -907,6 +907,12 @@ : IF test comp_iter? ; +// Variant of testlist used between BACKQUOTEs (the deprecated back-tick repr()) only +//testlist1: test (',' test)* +testlist1 + : test (COMMA test)* + ; + //yield_expr: 'yield' [testlist] yield_expr : YIELD testlist? @@ -1106,13 +1112,13 @@ /** the two '"'? cause a warning -- is there a way to avoid that? */ fragment TRIQUOTE - : '"'? '"'? (ESC|~('\\'|'"'))+ + : ('"' '"'?)? (ESC|~('\\'|'"'))+ ; /** the two '\''? cause a warning -- is there a way to avoid that? */ fragment TRIAPOS - : '\''? '\''? (ESC|~('\\'|'\''))+ + : ('\'' '\''?)? (ESC|~('\\'|'\''))+ ; fragment @@ -1133,7 +1139,7 @@ | nl=NEWLINE { extraNewlines = true; - } + } | ) { if (input.LA(1) == -1) { -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Dec 28 10:42:05 2018 From: jython-checkins at python.org (jeff.allen) Date: Fri, 28 Dec 2018 15:42:05 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Respond_to_most_deprecatio?= =?utf-8?q?n_warnings_from_Java_runtime_=28up_to_Java_9=29=2E?= Message-ID: <20181228154205.1.70D826F64994F479@mg.python.org> https://hg.python.org/jython/rev/b2a5ace2de1c changeset: 8208:b2a5ace2de1c user: Jeff Allen date: Thu Dec 27 16:32:55 2018 +0000 summary: Respond to most deprecation warnings from Java runtime (up to Java 9). The motivation is to reduce noise in the build so we can see when things go wrong. Although widespread, all are one of 2 simple types. Class.newInstance becomes Class.getDeclaredConstructor().newInstance(), and for a primitive wrapper T.valueOf(x) is preferred to new T(x). files: src/com/ziclix/python/sql/DataHandler.java | 4 +- src/com/ziclix/python/sql/Jython22DataHandler.java | 4 +- src/com/ziclix/python/sql/connect/Connectx.java | 2 +- src/com/ziclix/python/sql/pipe/Pipe.java | 4 +- src/com/ziclix/python/sql/zxJDBC.java | 3 +- src/org/python/compiler/ClassFile.java | 4 +- src/org/python/compiler/CodeCompiler.java | 2 +- src/org/python/compiler/LineNumberTable.java | 4 +- src/org/python/compiler/Module.java | 4 +- src/org/python/core/JavaProxyList.java | 31 ++++----- src/org/python/core/JavaProxyMap.java | 16 ++-- src/org/python/core/PyBeanEventProperty.java | 2 +- src/org/python/core/PyFloat.java | 4 +- src/org/python/core/PyInteger.java | 14 ++-- src/org/python/core/PyLong.java | 10 +- src/org/python/core/PyObject.java | 2 +- src/org/python/core/PySystemState.java | 4 +- src/org/python/core/imp.java | 18 +---- src/org/python/expose/generate/MethodExposer.java | 14 ++-- src/org/python/expose/generate/TypeExposer.java | 2 +- src/org/python/modules/posix/PyStatResult.java | 4 +- src/org/python/util/TemplateAntTask.java | 4 +- tests/java/javatests/Dict2JavaTest.java | 6 +- tests/java/javatests/ListTest.java | 4 +- tests/java/org/python/expose/generate/DescriptorExposerTest.java | 2 +- tests/java/org/python/expose/generate/ExposedTypeProcessorTest.java | 4 +- tests/java/org/python/expose/generate/NewExposerTest.java | 2 +- tests/java/org/python/expose/generate/OverridableNewExposerTest.java | 2 +- 28 files changed, 83 insertions(+), 93 deletions(-) diff --git a/src/com/ziclix/python/sql/DataHandler.java b/src/com/ziclix/python/sql/DataHandler.java --- a/src/com/ziclix/python/sql/DataHandler.java +++ b/src/com/ziclix/python/sql/DataHandler.java @@ -331,14 +331,14 @@ throw createUnsupportedTypeSQLException("STRUCT", col); default : - throw createUnsupportedTypeSQLException(new Integer(type), col); + throw createUnsupportedTypeSQLException(Integer.valueOf(type), col); } return set.wasNull() || obj == null ? Py.None : obj; } protected final SQLException createUnsupportedTypeSQLException(Object type, int col) { - Object[] vals = {type, new Integer(col)}; + Object[] vals = {type, Integer.valueOf(col)}; String msg = zxJDBC.getString("unsupportedTypeForColumn", vals); return new SQLException(msg); } diff --git a/src/com/ziclix/python/sql/Jython22DataHandler.java b/src/com/ziclix/python/sql/Jython22DataHandler.java --- a/src/com/ziclix/python/sql/Jython22DataHandler.java +++ b/src/com/ziclix/python/sql/Jython22DataHandler.java @@ -313,7 +313,7 @@ break; default : - throw createUnsupportedTypeSQLException(new Integer(type), col); + throw createUnsupportedTypeSQLException(Integer.valueOf(type), col); } return (set.wasNull() || (obj == null)) ? Py.None : obj; @@ -401,7 +401,7 @@ break; default : - throw createUnsupportedTypeSQLException(new Integer(type), col); + throw createUnsupportedTypeSQLException(Integer.valueOf(type), col); } return (stmt.wasNull() || (obj == null)) ? Py.None : obj; diff --git a/src/com/ziclix/python/sql/connect/Connectx.java b/src/com/ziclix/python/sql/connect/Connectx.java --- a/src/com/ziclix/python/sql/connect/Connectx.java +++ b/src/com/ziclix/python/sql/connect/Connectx.java @@ -58,7 +58,7 @@ try { String klass = (String) parser.arg(0).__tojava__(String.class); - datasource = Class.forName(klass).newInstance(); + datasource = Class.forName(klass).getDeclaredConstructor().newInstance(); } catch (Exception e) { throw zxJDBC.makeException(zxJDBC.DatabaseError, "unable to instantiate datasource"); } diff --git a/src/com/ziclix/python/sql/pipe/Pipe.java b/src/com/ziclix/python/sql/pipe/Pipe.java --- a/src/com/ziclix/python/sql/pipe/Pipe.java +++ b/src/com/ziclix/python/sql/pipe/Pipe.java @@ -87,8 +87,8 @@ // the purpose of the assert, but there's no need to create the buffer if I don't need it and I still // want to throw the AssertionError if required if ((sourceRunner.getCount() - sinkRunner.getCount()) != 0) { - Integer[] counts = {new Integer(sourceRunner.getCount()), - new Integer(sinkRunner.getCount())}; + Integer[] counts = {Integer.valueOf(sourceRunner.getCount()), + Integer.valueOf(sinkRunner.getCount())}; String msg = zxJDBC.getString("inconsistentRowCount", counts); Py.assert_(Py.Zero, Py.newString(msg)); diff --git a/src/com/ziclix/python/sql/zxJDBC.java b/src/com/ziclix/python/sql/zxJDBC.java --- a/src/com/ziclix/python/sql/zxJDBC.java +++ b/src/com/ziclix/python/sql/zxJDBC.java @@ -223,7 +223,8 @@ String className = props.getProperty(name).trim(); try { - connector = (PyObject) Class.forName(className).newInstance(); + connector = + (PyObject) Class.forName(className).getDeclaredConstructor().newInstance(); dict.__setitem__(name, connector); Py.writeComment("zxJDBC", "loaded connector [" + className + "] as [" + name + "]"); diff --git a/src/org/python/compiler/ClassFile.java b/src/org/python/compiler/ClassFile.java --- a/src/org/python/compiler/ClassFile.java +++ b/src/org/python/compiler/ClassFile.java @@ -243,11 +243,11 @@ AnnotationVisitor av = cw.visitAnnotation("Lorg/python/compiler/APIVersion;", true); // XXX: should imp.java really house this value or should imp.java point into // org.python.compiler? - av.visit("value", new Integer(imp.getAPIVersion())); + av.visit("value", Integer.valueOf(imp.getAPIVersion())); av.visitEnd(); av = cw.visitAnnotation("Lorg/python/compiler/MTime;", true); - av.visit("value", new Long(mtime)); + av.visit("value", Long.valueOf(mtime)); av.visitEnd(); if (sfilenameShort != null) { diff --git a/src/org/python/compiler/CodeCompiler.java b/src/org/python/compiler/CodeCompiler.java --- a/src/org/python/compiler/CodeCompiler.java +++ b/src/org/python/compiler/CodeCompiler.java @@ -105,7 +105,7 @@ public class CodeCompiler extends Visitor implements Opcodes, ClassConstants { - private static final Object Exit = new Integer(1); + private static final Object Exit = Integer.valueOf(1); private static final Object NoExit = null; private Module module; private Code code; diff --git a/src/org/python/compiler/LineNumberTable.java b/src/org/python/compiler/LineNumberTable.java --- a/src/org/python/compiler/LineNumberTable.java +++ b/src/org/python/compiler/LineNumberTable.java @@ -32,8 +32,8 @@ } public void addLine(int startpc, int lineno) { - lines.addElement(new Short((short) startpc)); - lines.addElement(new Short((short) lineno)); + lines.addElement(Short.valueOf((short) startpc)); + lines.addElement(Short.valueOf((short) lineno)); } public int length() { diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java --- a/src/org/python/compiler/Module.java +++ b/src/org/python/compiler/Module.java @@ -97,7 +97,7 @@ @Override void get(Code c) throws IOException { - c.ldc(new Double(value)); + c.ldc(Double.valueOf(value)); c.invokestatic(p(Py.class), "newFloat", sig(PyFloat.class, Double.TYPE)); } @@ -132,7 +132,7 @@ @Override void get(Code c) throws IOException { - c.ldc(new Double(value)); + c.ldc(Double.valueOf(value)); c.invokestatic(p(Py.class), "newImaginary", sig(PyComplex.class, Double.TYPE)); } diff --git a/src/org/python/core/JavaProxyList.java b/src/org/python/core/JavaProxyList.java --- a/src/org/python/core/JavaProxyList.java +++ b/src/org/python/core/JavaProxyList.java @@ -1,12 +1,5 @@ package org.python.core; -/** - * Proxy Java objects implementing java.util.List with Python methods - * corresponding to the standard list type - */ - -import org.python.util.Generic; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -16,7 +9,12 @@ import java.util.List; import java.util.ListIterator; +import org.python.util.Generic; +/** + * Proxy Java objects implementing java.util.List with Python methods corresponding to the standard + * list type + */ class JavaProxyList { @Untraversable @@ -35,10 +33,9 @@ protected List newList() { try { - return (List) asList().getClass().newInstance(); - } catch (IllegalAccessException e) { - throw Py.JavaError(e); - } catch (InstantiationException e) { + return (List) asList().getClass().getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException | SecurityException + | IllegalArgumentException e) { throw Py.JavaError(e); } } @@ -131,8 +128,9 @@ int n = PySequence.sliceLength(start, stop, step); List newList; try { - newList = list.getClass().newInstance(); - } catch (Exception e) { + newList = list.getClass().getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException | SecurityException + | IllegalArgumentException e) { throw Py.JavaError(e); } int j = 0; @@ -556,10 +554,9 @@ List jList = asList(); List jClone; try { - jClone = (List) jList.getClass().newInstance(); - } catch (IllegalAccessException e) { - throw Py.JavaError(e); - } catch (InstantiationException e) { + jClone = (List) jList.getClass().getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException | SecurityException + | IllegalArgumentException e) { throw Py.JavaError(e); } for (Object entry : jList) { diff --git a/src/org/python/core/JavaProxyMap.java b/src/org/python/core/JavaProxyMap.java --- a/src/org/python/core/JavaProxyMap.java +++ b/src/org/python/core/JavaProxyMap.java @@ -398,10 +398,10 @@ Map jmap = asMap(); Map jclone; try { - jclone = (Map) jmap.getClass().newInstance(); - } catch (IllegalAccessException e) { - throw Py.JavaError(e); - } catch (InstantiationException e) { + jclone = (Map) jmap.getClass().getDeclaredConstructor() + .newInstance(); + } catch (ReflectiveOperationException | SecurityException + | IllegalArgumentException e) { throw Py.JavaError(e); } for (Map.Entry entry : jmap.entrySet()) { @@ -514,14 +514,14 @@ * always injected to java.util.Map, so we know the class object we get from * asClass is subtype of java.util.Map */ - Map theMap = (Map) theClass.newInstance(); + Map theMap = (Map) theClass + .getDeclaredConstructor().newInstance(); for (PyObject key : keys.asIterable()) { theMap.put(Py.tojava(key, Object.class), defobj); } return Py.java2py(theMap); - } catch (InstantiationException e) { - throw Py.JavaError(e); - } catch (IllegalAccessException e) { + } catch (ReflectiveOperationException | SecurityException + | IllegalArgumentException e) { throw Py.JavaError(e); } } diff --git a/src/org/python/core/PyBeanEventProperty.java b/src/org/python/core/PyBeanEventProperty.java --- a/src/org/python/core/PyBeanEventProperty.java +++ b/src/org/python/core/PyBeanEventProperty.java @@ -114,7 +114,7 @@ return adapter; } try { - adapter = adapterClass.newInstance(); + adapter = adapterClass.getDeclaredConstructor().newInstance(); addMethod.invoke(self, adapter); } catch (Exception e) { throw Py.JavaError(e); diff --git a/src/org/python/core/PyFloat.java b/src/org/python/core/PyFloat.java --- a/src/org/python/core/PyFloat.java +++ b/src/org/python/core/PyFloat.java @@ -301,9 +301,9 @@ public Object __tojava__(Class c) { if (c == Double.TYPE || c == Number.class || c == Double.class || c == Object.class || c == Serializable.class) { - return new Double(getValue()); + return Double.valueOf(getValue()); } else if (c == Float.TYPE || c == Float.class) { - return new Float(getValue()); + return Float.valueOf((float) getValue()); } return super.__tojava__(c); } diff --git a/src/org/python/core/PyInteger.java b/src/org/python/core/PyInteger.java --- a/src/org/python/core/PyInteger.java +++ b/src/org/python/core/PyInteger.java @@ -226,27 +226,27 @@ public Object __tojava__(Class c) { if (c == Integer.TYPE || c == Number.class || c == Object.class || c == Integer.class || c == Serializable.class) { - return new Integer(getValue()); + return Integer.valueOf(getValue()); } if (c == Boolean.TYPE || c == Boolean.class) { - return new Boolean(getValue() != 0); + return Boolean.valueOf(getValue() != 0); } if (c == Byte.TYPE || c == Byte.class) { - return new Byte((byte)getValue()); + return Byte.valueOf((byte)getValue()); } if (c == Short.TYPE || c == Short.class) { - return new Short((short)getValue()); + return Short.valueOf((short)getValue()); } if (c == Long.TYPE || c == Long.class) { - return new Long(getValue()); + return Long.valueOf(getValue()); } if (c == Float.TYPE || c == Float.class) { - return new Float(getValue()); + return Float.valueOf(getValue()); } if (c == Double.TYPE || c == Double.class) { - return new Double(getValue()); + return Double.valueOf(getValue()); } return super.__tojava__(c); } diff --git a/src/org/python/core/PyLong.java b/src/org/python/core/PyLong.java --- a/src/org/python/core/PyLong.java +++ b/src/org/python/core/PyLong.java @@ -296,19 +296,19 @@ public Object __tojava__(Class c) { try { if (c == Boolean.TYPE || c == Boolean.class) { - return new Boolean(!getValue().equals(BigInteger.ZERO)); + return Boolean.valueOf(!getValue().equals(BigInteger.ZERO)); } if (c == Byte.TYPE || c == Byte.class) { - return new Byte((byte)getLong(Byte.MIN_VALUE, Byte.MAX_VALUE)); + return Byte.valueOf((byte)getLong(Byte.MIN_VALUE, Byte.MAX_VALUE)); } if (c == Short.TYPE || c == Short.class) { - return new Short((short)getLong(Short.MIN_VALUE, Short.MAX_VALUE)); + return Short.valueOf((short)getLong(Short.MIN_VALUE, Short.MAX_VALUE)); } if (c == Integer.TYPE || c == Integer.class) { - return new Integer((int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE)); + return Integer.valueOf((int)getLong(Integer.MIN_VALUE, Integer.MAX_VALUE)); } if (c == Long.TYPE || c == Long.class) { - return new Long(getLong(Long.MIN_VALUE, Long.MAX_VALUE)); + return Long.valueOf(getLong(Long.MIN_VALUE, Long.MAX_VALUE)); } if (c == Float.TYPE || c == Double.TYPE || c == Float.class || c == Double.class) { return __float__().__tojava__(c); diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java --- a/src/org/python/core/PyObject.java +++ b/src/org/python/core/PyObject.java @@ -188,7 +188,7 @@ ThreadContext.initializingProxy.set(new Object[] {this}); try { try { - proxy = (PyProxy) c.newInstance(); + proxy = (PyProxy) c.getDeclaredConstructor().newInstance(); } catch (java.lang.InstantiationException e) { Class sup = c.getSuperclass(); String msg = "Default constructor failed for Java superclass"; diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -1161,8 +1161,8 @@ return false; } try { - ((JythonInitializer) initializer.newInstance()).initialize(pre, post, argv, - sysClassLoader, adapter); + ((JythonInitializer) initializer.getDeclaredConstructor().newInstance()).initialize(pre, + post, argv, sysClassLoader, adapter); } catch (Exception e) { Py.writeWarning("initializer", "Failed initializing with class '" + className + "', continuing"); diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java --- a/src/org/python/core/imp.java +++ b/src/org/python/core/imp.java @@ -449,20 +449,12 @@ if (PyRunnable.class.isAssignableFrom(c)) { try { if (ContainsPyBytecode.class.isAssignableFrom(c)) { - try { - BytecodeLoader.fixPyBytecode((Class) c); - } catch (NoSuchFieldException e) { - throw Py.JavaError(e); - } catch (java.io.IOException e) { - throw Py.JavaError(e); - } catch (ClassNotFoundException e) { - throw Py.JavaError(e); - } + BytecodeLoader.fixPyBytecode((Class) c); } - return createFromCode(name, ((PyRunnable)c.newInstance()).getMain()); - } catch (InstantiationException e) { - throw Py.JavaError(e); - } catch (IllegalAccessException e) { + return createFromCode(name, + ((PyRunnable) c.getDeclaredConstructor().newInstance()).getMain()); + } catch (ReflectiveOperationException | SecurityException | IllegalArgumentException + | IOException e) { throw Py.JavaError(e); } } diff --git a/src/org/python/expose/generate/MethodExposer.java b/src/org/python/expose/generate/MethodExposer.java --- a/src/org/python/expose/generate/MethodExposer.java +++ b/src/org/python/expose/generate/MethodExposer.java @@ -249,26 +249,26 @@ // For primitive types, parse using the Java wrapper for that type and push onto the // stack as a constant. If the default is malformed, a NumberFormatException will be // raised. - mv.visitLdcInsn(new Long(def)); + mv.visitLdcInsn(Long.valueOf(def)); } else if(arg.equals(INT)) { - mv.visitLdcInsn(new Integer(def)); + mv.visitLdcInsn(Integer.valueOf(def)); } else if(arg.equals(BYTE)) { // byte, char, boolean and short go as int constants onto the stack, so convert them // to ints to get the right type - mv.visitLdcInsn(new Byte(def).intValue()); + mv.visitLdcInsn(Byte.valueOf(def).intValue()); } else if(arg.equals(SHORT)) { - mv.visitLdcInsn(new Short(def).intValue()); + mv.visitLdcInsn(Short.valueOf(def).intValue()); } else if(arg.equals(CHAR)) { if(def.length() != 1) { throwInvalid("A default for a char argument must be one character in length"); } - mv.visitLdcInsn((int)new Character(def.charAt(0)).charValue()); + mv.visitLdcInsn(def.charAt(0)); } else if(arg.equals(BOOLEAN)) { mv.visitLdcInsn(Boolean.valueOf(def) ? 1 : 0); } else if(arg.equals(Type.FLOAT_TYPE)) { - mv.visitLdcInsn(new Float(def)); + mv.visitLdcInsn(Float.valueOf(def)); } else if(arg.equals(Type.DOUBLE_TYPE)) { - mv.visitLdcInsn(new Double(def)); + mv.visitLdcInsn(Double.valueOf(def)); } } diff --git a/src/org/python/expose/generate/TypeExposer.java b/src/org/python/expose/generate/TypeExposer.java --- a/src/org/python/expose/generate/TypeExposer.java +++ b/src/org/python/expose/generate/TypeExposer.java @@ -90,7 +90,7 @@ } Class descriptor = load(l); try { - return (TypeBuilder)descriptor.newInstance(); + return (TypeBuilder)descriptor.getDeclaredConstructor().newInstance(); } catch(Exception e) { // If we're unable to create the generated class, the process is // definitely ill, but that shouldn't be the case most of the time diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java --- a/src/org/python/modules/posix/PyStatResult.java +++ b/src/org/python/modules/posix/PyStatResult.java @@ -121,7 +121,7 @@ private static Long zeroOrValue(Long value) { if (value == null) { - return new Long(0L); + return Long.valueOf(0L); } else { return value; } @@ -129,7 +129,7 @@ private static Integer zeroOrValue(Integer value) { if (value == null) { - return new Integer(0); + return Integer.valueOf(0); } else { return value; } diff --git a/src/org/python/util/TemplateAntTask.java b/src/org/python/util/TemplateAntTask.java --- a/src/org/python/util/TemplateAntTask.java +++ b/src/org/python/util/TemplateAntTask.java @@ -95,7 +95,7 @@ * Verbose flag. */ public void setVerbose(String in) { - verbose = (new Boolean(getProject().replaceProperties(in))).booleanValue(); + verbose = (Boolean.valueOf(getProject().replaceProperties(in))).booleanValue(); } /** @@ -107,7 +107,7 @@ * Lazy flag. */ public void setLazy(String in) { - lazy = (new Boolean(getProject().replaceProperties(in))).booleanValue(); + lazy = (Boolean.valueOf(getProject().replaceProperties(in))).booleanValue(); } public void execute() { diff --git a/tests/java/javatests/Dict2JavaTest.java b/tests/java/javatests/Dict2JavaTest.java --- a/tests/java/javatests/Dict2JavaTest.java +++ b/tests/java/javatests/Dict2JavaTest.java @@ -64,7 +64,7 @@ public boolean test_put_hig() { map.put("h", null); - map.put("i", new Integer(3)); + map.put("i", Integer.valueOf(3)); Object val = map.put("g", "3"); return val.equals("2"); } @@ -78,7 +78,7 @@ // Test a number hmap = new HashMap(); - hmap.put("i", new Integer(3)); + hmap.put("i", Integer.valueOf(3)); entry = hmap.entrySet().iterator().next(); if (!map.entrySet().contains(entry)) return false; @@ -95,7 +95,7 @@ public boolean test_entry_set_nulls() { Set> set = map.entrySet(); return set.contains(null) == false && set.remove(null) == false && - set.contains(new Boolean(true)) == false && set.remove(new String("")) == false; + set.contains(Boolean.TRUE) == false && set.remove(new String("")) == false; } diff --git a/tests/java/javatests/ListTest.java b/tests/java/javatests/ListTest.java --- a/tests/java/javatests/ListTest.java +++ b/tests/java/javatests/ListTest.java @@ -210,11 +210,11 @@ a.add(i, b.get(i)); } try { - a.add(a.size() + 1, new Integer(a.size() + 1)); + a.add(a.size() + 1, Integer.valueOf(a.size() + 1)); TestSupport.fail("expected IndexOutOfBoundsException"); } catch (IndexOutOfBoundsException e) {} try { - a.add(-1, new Integer(-1)); + a.add(-1, Integer.valueOf(-1)); TestSupport.fail("expected IndexOutOfBoundsException"); } catch (IndexOutOfBoundsException e) {} } diff --git a/tests/java/org/python/expose/generate/DescriptorExposerTest.java b/tests/java/org/python/expose/generate/DescriptorExposerTest.java --- a/tests/java/org/python/expose/generate/DescriptorExposerTest.java +++ b/tests/java/org/python/expose/generate/DescriptorExposerTest.java @@ -33,7 +33,7 @@ DescriptorExposer de = new DescriptorExposer(ASM_TYPE, name); setup.setup(de); Class descriptor = de.load(new BytecodeLoader.Loader()); - PyDataDescr descr = (PyDataDescr)descriptor.newInstance(); + PyDataDescr descr = (PyDataDescr)descriptor.getDeclaredConstructor().newInstance(); descr.setType(PY_TYPE); return descr; } diff --git a/tests/java/org/python/expose/generate/ExposedTypeProcessorTest.java b/tests/java/org/python/expose/generate/ExposedTypeProcessorTest.java --- a/tests/java/org/python/expose/generate/ExposedTypeProcessorTest.java +++ b/tests/java/org/python/expose/generate/ExposedTypeProcessorTest.java @@ -35,11 +35,11 @@ ice.getTypeExposer().load(loader); Class doctoredSimple = loader.loadClassFromBytes("org.python.expose.generate.SimpleExposed", ice.getBytecode()); - PyObject simp = (PyObject)doctoredSimple.newInstance(); + PyObject simp = (PyObject)doctoredSimple.getDeclaredConstructor().newInstance(); PyBuiltinCallable func = MethodExposerTest.instantiate(simple_method, "invisible"); PyBuiltinCallable bound = func.bind(simp); bound.__call__(); - PyDataDescr desc = (PyDataDescr)tostringDesc.newInstance(); + PyDataDescr desc = (PyDataDescr)tostringDesc.getDeclaredConstructor().newInstance(); desc.setType(simp.getType()); assertEquals(doctoredSimple.getField("toStringVal").get(simp), desc.__get__(simp, PyType.fromClass(doctoredSimple)).toString()); diff --git a/tests/java/org/python/expose/generate/NewExposerTest.java b/tests/java/org/python/expose/generate/NewExposerTest.java --- a/tests/java/org/python/expose/generate/NewExposerTest.java +++ b/tests/java/org/python/expose/generate/NewExposerTest.java @@ -22,7 +22,7 @@ assertEquals("org.python.expose.generate.NewExposerTest$Instantiable$exposed___new__", ne.getClassName()); Class descriptor = ne.load(new BytecodeLoader.Loader()); - PyNewWrapper instance = (PyNewWrapper)descriptor.newInstance(); + PyNewWrapper instance = (PyNewWrapper)descriptor.getDeclaredConstructor().newInstance(); instance.setWrappedType(PyType.fromClass(Instantiable.class)); assertSame("__new__", instance.__getattr__("__name__").toString()); assertEquals(Py.One, instance.__call__(PyType.fromClass(Instantiable.class))); diff --git a/tests/java/org/python/expose/generate/OverridableNewExposerTest.java b/tests/java/org/python/expose/generate/OverridableNewExposerTest.java --- a/tests/java/org/python/expose/generate/OverridableNewExposerTest.java +++ b/tests/java/org/python/expose/generate/OverridableNewExposerTest.java @@ -23,7 +23,7 @@ Type.getMethodDescriptor(VOID, new Type[] {APYOBJ, ASTRING}), new String[] {}); Class descriptor = ne.load(new BytecodeLoader.Loader()); - instance = (PyNewWrapper)descriptor.newInstance(); + instance = (PyNewWrapper)descriptor.getDeclaredConstructor().newInstance(); type = PyType.fromClass(Instantiable.class); instance.setWrappedType(type); } -- Repository URL: https://hg.python.org/jython