From jython-checkins at python.org Wed Feb 1 10:51:32 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 01 Feb 2017 15:51:32 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_issue_=232527=2E?= Message-ID: <20170201155131.127965.50786.C0F3195B@psf.io> https://hg.python.org/jython/rev/77f1bcd04321 changeset: 8012:77f1bcd04321 user: Stefan Richthofer date: Wed Feb 01 16:51:16 2017 +0100 summary: Fixed issue #2527. files: Lib/test/test_StringIO_jy.py | 6 +++ NEWS | 1 + src/org/python/core/PyString.java | 15 +++++-- src/org/python/modules/cStringIO.java | 29 ++++---------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_StringIO_jy.py b/Lib/test/test_StringIO_jy.py --- a/Lib/test/test_StringIO_jy.py +++ b/Lib/test/test_StringIO_jy.py @@ -12,6 +12,12 @@ self.assertEqual(u'foo', cStringIO.StringIO(u'foo').read()) self.assertEqual('foo', cStringIO.StringIO(u'foo').read()) + def test_unicode_encode_error(self): + # See issue #2527 + uc = u'\u4e2d\u6587' + self.assertRaises(UnicodeEncodeError, lambda: cStringIO.StringIO(uc)) + + class TestWrite(unittest.TestCase): def test_write_seek_write(self): f = cStringIO.StringIO() diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2527 ] cStringIO throws IllegalArgumentException with non-ASCII values - [ 2511 ] Percent operator calls __getattr__('__getitem__') - [ PR28 ] zipimporter supports multi-byte characters in pathnames - [ 2228 ] Re-use "makeCompiledFilename" function where possible diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java --- a/src/org/python/core/PyString.java +++ b/src/org/python/core/PyString.java @@ -707,7 +707,14 @@ @Override public Object __tojava__(Class c) { if (c.isAssignableFrom(String.class)) { - return getString(); + /* If c is a CharSequence we assume the caller is prepared + * to get maybe not an actual String. In that case we avoid + * conversion so the caller can do special stuff with the + * returned PyString or PyUnicode or whatever. + * (If c is Object.class, the caller usually expects to get + * actually a String) + */ + return c == CharSequence.class ? this : getString(); } if (c == Character.TYPE || c == Character.class) { @@ -4526,9 +4533,9 @@ if (args instanceof AbstractDict || (!(args instanceof PySequence) && // See issue 2511: __getitem__ should be looked up directly in the dict, rather - // than going through another __getattr__-call, so we use object___findattr__ - // instead of generic __findattr__. - args.object___findattr__("__getitem__") != null)) { + // than going through another __getattr__ call. We achieve this by using + // object___findattr__ instead of generic __findattr__. + args.object___findattr__("__getitem__".intern()) != null)) { dict = args; argIndex = -3; } diff --git a/src/org/python/modules/cStringIO.java b/src/org/python/modules/cStringIO.java --- a/src/org/python/modules/cStringIO.java +++ b/src/org/python/modules/cStringIO.java @@ -17,6 +17,7 @@ import org.python.core.PyList; import org.python.core.PyObject; import org.python.core.PyString; +import org.python.core.PyUnicode; import org.python.core.PyType; /** @@ -48,7 +49,7 @@ * @param buffer The initial value. * @return a new StringIO object. */ - public static StringIO StringIO(String buffer) { + public static StringIO StringIO(CharSequence buffer) { return new StringIO(buffer); } @@ -78,9 +79,9 @@ buf = new StringBuilder(); } - - public StringIO(String buffer) { - buf = new StringBuilder(buffer); + public StringIO(CharSequence buffer) { + buf = new StringBuilder(buffer instanceof PyUnicode ? + ((PyUnicode) buffer).encode() : buffer); } public StringIO(PyArray array) { @@ -99,6 +100,7 @@ return (int)val; } + @Override public void __setattr__(String name, PyObject value) { if (name == "softspace") { softspace = value.__nonzero__(); @@ -107,6 +109,7 @@ super.__setattr__(name, value); } + @Override public PyObject __iternext__() { _complain_ifclosed(); PyString r = readline(); @@ -127,7 +130,6 @@ // buf = null; } - /** * Return false. * @return false. @@ -137,7 +139,6 @@ return false; } - /** * Position the file pointer to the absolute position. * @param pos the position in the file. @@ -146,7 +147,6 @@ seek(pos, os.SEEK_SET); } - /** * Position the file pointer to the position in the . * @@ -187,8 +187,6 @@ return pos; } - - /** * Read all data until EOF is reached. * An empty string is returned when EOF is encountered immediately. @@ -198,7 +196,6 @@ return read(-1); } - /** * Read at most size bytes from the file (less if the read hits EOF). * If the size argument is negative, read all data until EOF is @@ -207,7 +204,6 @@ * @param size the number of characters to read. * @return A string containing the data read. */ - public synchronized PyString read(long size) { _complain_ifclosed(); _convert_to_int(size); @@ -236,7 +232,6 @@ return readline(-1); } - /** * Read one entire line from the file. A trailing newline character * is kept in the string (but may be absent when a file ends with an @@ -263,7 +258,6 @@ return new PyString(r); } - /** * Read and return a line without the trailing newline. * Usind by cPickle as an optimization. @@ -280,8 +274,6 @@ return new PyString(r); } - - /** * Read until EOF using readline() and return a list containing * the lines thus read. @@ -291,7 +283,6 @@ return readlines(0); } - /** * Read until EOF using readline() and return a list containing * the lines thus read. @@ -398,7 +389,6 @@ buf.setCharAt(pos++, ch); } - /** * Write a list of strings to the file. */ @@ -408,7 +398,6 @@ } } - /** * Flush the internal buffer. Does nothing. */ @@ -416,7 +405,6 @@ _complain_ifclosed(); } - /** * Retrieve the entire contents of the ``file'' at any time * before the StringIO object's close() method is called. @@ -429,6 +417,7 @@ } + private static String[] strings = new String[256]; static String getString(char ch) { if (ch > 255) { @@ -443,5 +432,5 @@ } return s; } +} -} -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Feb 1 13:35:28 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 01 Feb 2017 18:35:28 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_python-level_doc_to_c?= =?utf-8?q?StringIO=2E?= Message-ID: <20170201183527.16593.48361.4D8C3BC3@psf.io> https://hg.python.org/jython/rev/e3b697c6b4ec changeset: 8013:e3b697c6b4ec user: Stefan Richthofer date: Wed Feb 01 19:35:10 2017 +0100 summary: Added python-level doc to cStringIO. files: src/org/python/modules/cStringIO.java | 61 +++++++++++++++ 1 files changed, 61 insertions(+), 0 deletions(-) diff --git a/src/org/python/modules/cStringIO.java b/src/org/python/modules/cStringIO.java --- a/src/org/python/modules/cStringIO.java +++ b/src/org/python/modules/cStringIO.java @@ -28,6 +28,36 @@ * @version cStringIO.java,v 1.10 1999/05/20 18:03:20 fb Exp */ public class cStringIO { + public static final String __doc__ = + "A simple fast partial StringIO replacement.\n" + + "\n" + + "This module provides a simple useful replacement for\n" + + "the StringIO module that is written in Java. It does not provide the\n" + + "full generality of StringIO, but it provides enough for most\n" + + "applications and is especially useful in conjunction with the\n" + + "pickle module.\n" + + "\n" + + "Usage:\n" + + "\n" + + " from cStringIO import StringIO\n" + + "\n" + + " an_output_stream=StringIO()\n" + + " an_output_stream.write(some_stuff)\n" + + " ...\n" + + " value=an_output_stream.getvalue()\n" + + "\n" + + " an_input_stream=StringIO(a_string)\n" + + " spam=an_input_stream.readline()\n" + + " spam=an_input_stream.read(5)\n" + + " an_input_stream.seek(0) # OK, start over\n" + + " spam=an_input_stream.read() # and read it all\n" + + " \n" + + "If someone else wants to provide a more complete implementation,\n" + + "go for it. :-) \n" + + "\n" + + "cStringIO.java,v 1.10 1999/05/20 18:03:20 fb Exp\n" + + "Python-level doc was inserted on 2017/02/01, copied from\n" + + "cStringIO.c,v 1.29 1999/06/15 14:10:27 jim Exp\n"; // would be nicer if we directly imported from os, but crazy to do so // since in python code itself @@ -40,6 +70,9 @@ public static PyType InputType = PyType.fromClass(StringIO.class); public static PyType OutputType = PyType.fromClass(StringIO.class); + public static final String __doc__StringIO = + "StringIO([s]) -- Return a StringIO-like stream for reading or writing"; + public static StringIO StringIO() { return new StringIO(); } @@ -118,6 +151,7 @@ return r; } + public static final String __doc__close = "close(): explicitly release resources held."; /** * Free the memory buffer. */ @@ -130,6 +164,7 @@ // buf = null; } + public static final String __doc__isatty = "isatty(): always returns 0"; /** * Return false. * @return false. @@ -139,6 +174,9 @@ return false; } + public static final String __doc__seek = + "seek(position) -- set the current position\n" + + "seek(position, mode) -- mode 0: absolute; 1: relative; 2: relative to EOF"; /** * Position the file pointer to the absolute position. * @param pos the position in the file. @@ -171,6 +209,8 @@ } } + public static final String __doc__reset = + "reset() -- Reset the file position to the beginning"; /** * Reset the file position to the beginning of the file. */ @@ -178,6 +218,7 @@ pos = 0; } + public static final String __doc__tell = "tell() -- get the current position."; /** * Return the file position. * @return the position in the file. @@ -187,6 +228,8 @@ return pos; } + public static final String __doc__read = + "read([s]) -- Read s characters, or the rest of the string"; /** * Read all data until EOF is reached. * An empty string is returned when EOF is encountered immediately. @@ -221,6 +264,7 @@ return new PyString(substr); } + public static final String __doc__readline = "readline() -- Read one line"; /** * Read one entire line from the file. A trailing newline character * is kept in the string (but may be absent when a file ends with @@ -274,6 +318,7 @@ return new PyString(r); } + public static final String __doc__readlines = "readlines() -- Read all lines"; /** * Read until EOF using readline() and return a list containing * the lines thus read. @@ -305,6 +350,8 @@ return lines; } + public static final String __doc__truncate = + "truncate(): truncate the file at the current position."; /** * truncate the file at the current position. */ @@ -326,6 +373,9 @@ this.pos = pos_int; } + public static final String __doc__write = + "write(s) -- Write a string to the file" + + "\n\nNote (hack:) writing None resets the buffer"; /** * Write a string to the file. * @param obj The data to write. @@ -389,6 +439,11 @@ buf.setCharAt(pos++, ch); } + public static final String __doc__writelines = + "writelines(sequence_of_strings) -> None. Write the strings to the file.\n" + + "\n" + + "Note that newlines are not added. The sequence can be any iterable object\n" + + "producing strings. This is equivalent to calling write() for each string."; /** * Write a list of strings to the file. */ @@ -398,6 +453,7 @@ } } + public static final String __doc__flush = "flush(): does nothing."; /** * Flush the internal buffer. Does nothing. */ @@ -405,6 +461,11 @@ _complain_ifclosed(); } + public static final String __doc__getvalue = + "getvalue([use_pos]) -- Get the string value." + + "\n" + + "If use_pos is specified and is a true value, then the string returned\n" + + "will include only the text up to the current file position.\n"; /** * Retrieve the entire contents of the ``file'' at any time * before the StringIO object's close() method is called. -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 3 13:13:18 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Feb 2017 18:13:18 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232548=2E_As_a_conse?= =?utf-8?q?quence_this_commit_enables_support_for_SymPy_versions?= Message-ID: <20170203181318.16815.88046.5573C0FA@psf.io> https://hg.python.org/jython/rev/8e35d3c691fb changeset: 8014:8e35d3c691fb user: Stefan Richthofer date: Fri Feb 03 19:12:40 2017 +0100 summary: Fixed #2548. As a consequence this commit enables support for SymPy versions that incorporate https://github.com/sympy/sympy/pull/12120. See #1777 for details. files: Misc/make_ucnhashdat.py | 176 ++++- NEWS | 4 + src/org/python/modules/ucnhash.dat | Bin src/org/python/modules/ucnhash.java | 472 ++++++++++----- 4 files changed, 431 insertions(+), 221 deletions(-) diff --git a/Misc/make_ucnhashdat.py b/Misc/make_ucnhashdat.py --- a/Misc/make_ucnhashdat.py +++ b/Misc/make_ucnhashdat.py @@ -1,9 +1,10 @@ # Copyright Finn Bock +# Updated 2017 by Stefan Richthofer to support Unicode 9.0 # # Generate a ucnhash.dat file with mapping from unicode # names to codepoints. # -# python mkucnhashdat.py UnicodeData-3.0.0.txt +# python make_ucnhashdata.py UnicodeData.txt mph.exe # # The "mph" program must be available on the path. # This program is used to create the minimum perfect @@ -16,6 +17,7 @@ import fileinput, re, os, sys, struct, cStringIO +mph_exec = 'mph' # Formerly: 'mph.exe' def debug(str): print >>debugFile, str @@ -32,18 +34,14 @@ elif i == l-1: n = name[wordstart:i+1] if n: - #print " ", i, c, n wordstart = i if c == '-' and n != '': n += '-' if c == ' ' or c == '-': wordstart = i+1 - #print " ", n wordlist.append(n) - return wordlist - def readUnicodeDict(file): d = {} for l in fileinput.input(file): @@ -62,22 +60,17 @@ continue wordlist = splitIntoWords(name) - #print name, wordlist d[name] = (int(v, 16), wordlist, []) return d -#readUnicodeDict("nametest.txt") -#sys.exit() - def count(dict, index): c = dict.get(index) if c is None: c = 0 c += 1 dict[index] = c - def dumpUnicodeDict(title, dict): lst = [] i = 0 @@ -92,8 +85,6 @@ print "%.4X %s %s" % (v, k, p) - - class MphEmitter: def readint(self): @@ -155,17 +146,20 @@ self.inf = inf self.readconst(); - outf.write(struct.pack("!hhhhhh", self.n, - self.m, - self.minchar, - self.maxchar, - self.alphasz, - self.maxlen)) - self.readg().writeto(outf) + outf.write(struct.pack("!hhhhh", self.n, + self.m, + self.minchar, + self.alphasz, + self.maxlen)) + G = self.readg() + debug("G len: %d" % (G.size()/2)) + G.writeto(outf) outf.write(struct.pack("!h", self.d)) for t in range(self.d): - self.readT(t).writeto(outf) + T = self.readT(t) + debug("T%d len: %d" % (t, T.size()/2)) + T.writeto(outf) class Table: @@ -181,9 +175,15 @@ def write_UShort(self, v): self.buf.write(struct.pack("!H", v)) + def write_Int32(self, v): + self.buf.write(struct.pack("!i", v)) + + def write_UInt32(self, v): + self.buf.write(struct.pack("!I", v)) + def writeto(self, file): file.write('t') - file.write(struct.pack("!H", self.size())) + file.write(struct.pack("!I", self.size())) file.write(self.buf.getvalue()) def size(self): @@ -196,21 +196,16 @@ cnt += len(name) return cnt - - def calculateWords(unicodeDict): words = {} for key, (value, wordlist, rawlist) in unicodeDict.items(): for name in wordlist: wordlist = words.setdefault(name, []) wordlist.append(key) - return words - def replaceWord(word, index, charlist): replaced = 0 - for char in charlist: (v, wordlist, rawlist) = unicodeDict[char] try: @@ -224,11 +219,9 @@ def compress(): #dumpUnicodeDict("UnicodeDict before", unicodeDict) words = calculateWords(unicodeDict) - lenp = [(len(v), k, v) for k, v in words.items()] lenp.sort() lenp.reverse() - wordidx = len(chars) for (length, word, value) in lenp: # Do not lookup single char words or words only used once @@ -238,19 +231,12 @@ # be just as big. if len(word) == 2 and wordidx >= 238: continue - - #print length, word, len(value) - replaceWord(word, wordidx, value) wordmap[wordidx] = word - wordidx += 1 - #dumpUnicodeDict("UnicodeDict after", unicodeDict) - def writeUcnhashDat(): - cutoff = 255 - ((len(chars) + len(wordmap)) >> 8) debug("wordmap entries: %d" % len(wordmap)) @@ -259,20 +245,17 @@ worddata = Table() wordoffs = Table() wordfile = open("words.in", "wt"); - size = 0 l = [(k,v) for k,v in wordmap.items()] l.sort() for k,v in l: print >>wordfile, v wordoffs.write_UShort(worddata.size()) - mapv = ''.join(map(lambda x: chr(chardict.get(x)), v)) worddata.write_Str(mapv) - wordfile.close() - os.system("mph.exe -d3 -S1 -m4 -a < words.in > words.hash") + os.system(mph_exec+" -d3 -S1 -m4 -a < words.in > words.hash") outf = open("ucnhash.dat", "wb+") @@ -280,8 +263,8 @@ m.writeFile(open("words.hash"), outf) debug("wordhash size %d" % outf.tell()) - debug("wordoffs size %d" % wordoffs.size()) - debug("worddata size %d" % worddata.size()) + debug("wordoffs size %d" % (wordoffs.size()/2)) + debug("worddata size %d" % (worddata.size())) wordoffs.writeto(outf) worddata.writeto(outf) @@ -292,7 +275,7 @@ savewordlist = wordlist[:] # Map remaining strings to a list of bytes in chardict - # range: range(0,37) + # range: range(0, 40) l = len(wordlist) for i in range(l-1, -1, -1): part = wordlist[i] @@ -301,7 +284,6 @@ if i > 0 and type(wordlist[i-1]) == type(""): ipart[0:0] = [0] # index of space wordlist[i:i+1] = ipart - # Encode high values as two bytes for v in wordlist: if v <= cutoff: @@ -316,7 +298,6 @@ lst.append((rawlist, wordlist, key, value)) maxklen = max(maxklen, len(key)) lst.sort() - outf.write(struct.pack("!hhh", len(chars), cutoff, maxklen)); raw = Table() @@ -326,7 +307,7 @@ for r in rawlist: raw.write_Str(chr(r)) datasize.append((len(rawlist), value)) - debug("%d %s %r" % (i, key, rawlist)) + #debug("%d %s %r" % (i, key, rawlist)) i += 1 debug("Raw size = %d" % raw.size()) raw.writeto(outf) @@ -336,25 +317,45 @@ offset = 0 maxlen = 0 + maxvl = 0 # for debugging + # Formerly it was sufficient for rawindex and codepoint + # to be 16 bit. + # We leave the old 16 bit write instructions here as + # comments in case future debugging is necessary. + # Note that rawindex blocksize therefore shrunk from + # 5 to 3, which was adjusted in ucnhash.java accordingly. + # In line 'v = v | (long(size) << (j*5))' the '5' seemingly + # refers to blocksize, but for some reason it must not be + # adjusted to 3. + # (3 would break things, while it works well with 5) for i in range(0, len(datasize), 12): saveoffset = offset - rawindex.write_UShort(offset) + #rawindex.write_UShort(offset) + rawindex.write_UInt32(offset) v = 0L j = 0 for (size, value) in datasize[i:i+12]: + # we keep track of max value to confirm + # that 32 bit codepoint table is needed + if value > maxvl: + maxvl = value offset += size v = v | (long(size) << (j*5)) maxlen = max(maxlen, size) - codepoint.write_UShort(value) + #codepoint.write_UShort(value) + codepoint.write_UInt32(value) j += 1 - debug("%d %d %x" % (i/ 12, saveoffset, v)) - rawindex.write_UShort((v >> 48) & 0xFFFF) - rawindex.write_UShort((v >> 32) & 0xFFFF) - rawindex.write_UShort((v >> 16) & 0xFFFF) - rawindex.write_UShort(v & 0xFFFF) - - debug("rawindex size % d" % rawindex.size()) + #debug("%d %d %x" % (i/ 12, saveoffset, v)) + #rawindex.write_UShort((v >> 48) & 0xFFFF) + #rawindex.write_UShort((v >> 32) & 0xFFFF) + #rawindex.write_UShort((v >> 16) & 0xFFFF) + #rawindex.write_UShort(v & 0xFFFF) + rawindex.write_UInt32((v >> 32) & 0xFFFFFFFF) + rawindex.write_UInt32(v & 0xFFFFFFFF) + debug("maxval % d" % maxvl) + debug("rawindex size % d" % (rawindex.size()/4)) + debug("codepoint size % d" % (codepoint.size()/4)) rawindex.writeto(outf) codepoint.writeto(outf) @@ -362,9 +363,8 @@ outf.close(); - if __name__ == "__main__": - chars = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-" + chars = " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-()" chardict = {} for c in chars: chardict[c] = chars.index(c) @@ -372,12 +372,16 @@ debugChars = [] # [0x41, 0x20AC] debugFile = open("ucnhash.lst", "wt") + #debugFile = sys.stdout wordmap = {} - unicodeDataFile = "UnicodeData-3.0.0.txt" + # Called 2017 with UnicodeData.txt for Unicode 9.0 + unicodeDataFile = "UnicodeData.txt" if len(sys.argv) > 1: unicodeDataFile = sys.argv[1] + if len(sys.argv) > 2: + mph_exec = sys.argv[2] unicodeDict = readUnicodeDict(unicodeDataFile) print "Size:", calculateSize(unicodeDict) @@ -389,7 +393,63 @@ sys.exit(0) +# Debugging-hints: +#----------------- +# (with debugFile = sys.stdout, omitting mph-output) +# Output for "UnicodeData-3.0.0.txt": +# Size: 259126 +# compressed +# wordmap entries: 1384 +# wordmap cutoffs: 250 +# * d=3 +# * n=1703 +# * m=1384 +# * c=1.23 +# * maxlen=4 +# * minklen=2 +# * maxklen=18 +# * minchar=45 +# * maxchar=90 +# G len: 1703 +# T0 len: 184 +# T1 len: 184 +# T2 len: 184 +# wordhash size 4542 +# wordoffs size 1384 +# worddata size 7375 +# Raw size = 58531 +# maxval 65533 +# rawindex size 2577 +# codepoint size 10298 +# raw entries 10298 +# done - +# Output for "UnicodeData.txt", Unicode 9.0: +# Size: 755323 +# compressed +# wordmap entries: 3708 +# wordmap cutoffs: 241 +# * d=3 +# * n=4561 +# * m=3708 +# * c=1.23 +# * maxlen=4 +# * minklen=2 +# * maxklen=18 +# * minchar=32 +# * maxchar=90 +# G len: 4561 +# T0 len: 236 +# T1 len: 236 +# T2 len: 236 +# wordhash size 10570 +# wordoffs size 3708 +# worddata size 19399 +# Raw size = 184818 +# maxval 917999 +# rawindex size 7389 +# codepoint size 29545 +# raw entries 29545 +# done diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2548 ] Unicode u'\N{name}' frequently broken, because ucnhash.dat outdated - [ 2527 ] cStringIO throws IllegalArgumentException with non-ASCII values - [ 2511 ] Percent operator calls __getattr__('__getitem__') - [ PR28 ] zipimporter supports multi-byte characters in pathnames @@ -51,6 +52,9 @@ - [ 1767 ] Rich comparisons New Features + - SymPy is supported for versions that incorporate https://github.com/sympy/sympy/pull/12120. + - Updated ucnhash to support name lookup for Unicode 9.0 (like in u'\N{name}'). + This is a crucial step to allow SymPy support. - Jython doc-entries in Java-code (__doc__foo) now accept anything that implements java.lang.CharSequence (PyString implements CharSequence now as well). This allows Jython-friendly Java-code without PyString-induced dependency on Jython. diff --git a/src/org/python/modules/ucnhash.dat b/src/org/python/modules/ucnhash.dat index 2fd71d4741af7336a16adf2096c8a3714e448c85..fd3d767291dc47eb1761e7886b093a18888315f9 GIT binary patch [stripped] diff --git a/src/org/python/modules/ucnhash.java b/src/org/python/modules/ucnhash.java --- a/src/org/python/modules/ucnhash.java +++ b/src/org/python/modules/ucnhash.java @@ -1,10 +1,14 @@ -// Copyright 1998 Finn Bock. - +/* Copyright 1998 Finn Bock. + * Updated 2017 by Stefan Richthofer to support Unicode 9.0 + */ package org.python.modules; -import java.io.*; -import org.python.core.*; +import java.io.InputStream; +import java.io.DataInputStream; +import java.io.BufferedInputStream; +import java.io.IOException; +import org.python.core.ucnhashAPI; public class ucnhash implements ucnhashAPI { @@ -12,7 +16,6 @@ private static int n; private static int m; private static int minchar; - private static int maxchar; private static int alphasz; private static int maxlen; private static int maxidx; @@ -34,12 +37,10 @@ // The raw data and indexes into start of each name // The rawindex is sorted based on the wordindexes. private static byte[] rawdata; - private static char[] rawindex; + private static int[] rawindex; // The mapping from raw data index to unicode code points. - private static char[] codepoint; - - + private static int[] codepoint; public static String[] __depends__ = new String[] { @@ -48,23 +49,26 @@ public static void loadTables() throws Exception { - InputStream instream = ucnhash.class. - getResourceAsStream("ucnhash.dat"); + InputStream instream = + ucnhash.class.getResourceAsStream("ucnhash.dat"); if (instream == null) - throw new IOException("Unicode name database not found: " + - "ucnhash.dat"); + throw new IOException( + "Unicode name database not found: ucnhash.dat"); - DataInputStream in = new DataInputStream( - new BufferedInputStream(instream)); + DataInputStream in = + new DataInputStream(new BufferedInputStream(instream)); n = in.readShort(); m = in.readShort(); - minchar= in.readShort(); - maxchar = in.readShort(); + minchar = in.readShort(); alphasz = in.readShort(); maxlen = in.readShort(); maxidx = maxlen*alphasz-minchar; - + /* + if (debug) { + System.out.println("n "+n+" m "+m+" maxlen "+maxlen+ + " minchar "+minchar+" alphasz "+alphasz); + } */ G = readShortTable(in); if (in.readShort() != 3) throw new IOException("UnicodeNameMap file corrupt, " + @@ -76,24 +80,41 @@ wordoffs = readShortTable(in); worddata = readByteTable(in); + /* + if (debug) { + System.out.println("G "+G.length+" T0 "+T0.length+ + " T1 "+T1.length+" T2 "+T2.length); + System.out.println("wordoffs: "+wordoffs.length+ + " worddata: "+worddata.length); + }*/ wordstart = in.readShort(); wordcutoff = in.readShort(); maxklen = in.readShort(); rawdata = readByteTable(in); - rawindex = readCharTable(in); - codepoint = readCharTable(in); + // Formerly rawindex and codepoint were 16 bit + //rawindex = readCharTable(in); + //codepoint = readCharTable(in); + rawindex = readIntTable(in); + codepoint = readIntTable(in); + /* + if (debug) { + System.out.println("wordstart: "+wordstart+ + " wordcutoff: "+wordcutoff+" maxklen: "+maxklen); + System.out.println("rawdata: "+rawdata.length); + System.out.println("rawindex: "+rawindex.length+ + " codepoint: "+codepoint.length); + }*/ } - private static short[] readShortTable(DataInputStream in) - throws IOException + throws IOException { - if (in.read() != 't') + if (in.read() != 't') { throw new IOException("UnicodeNameMap file corrupt, shorttable"); - - int n = in.readUnsignedShort() / 2; + } + int n = in.readInt() / 2; short[] table = new short[n]; for (int i = 0; i < n; i++) { table[i] = in.readShort(); @@ -101,13 +122,27 @@ return table; } + private static int[] readIntTable(DataInputStream in) + throws IOException + { + if (in.read() != 't') { + throw new IOException("UnicodeNameMap file corrupt, inttable"); + } + int n = in.readInt() / 4; + int[] table = new int[n]; + for (int i = 0; i < n; i++) { + table[i] = in.readInt(); + } + return table; + } + private static char[] readCharTable(DataInputStream in) - throws IOException + throws IOException { - if (in.read() != 't') + if (in.read() != 't') { throw new IOException("UnicodeNameMap file corrupt, chartable"); - - int n = in.readUnsignedShort() / 2; + } + int n = in.readInt() / 2; char[] table = new char[n]; for (int i = 0; i < n; i++) { table[i] = in.readChar(); @@ -116,17 +151,17 @@ } private static byte[] readByteTable(DataInputStream in) - throws IOException + throws IOException { - if (in.read() != 't') + if (in.read() != 't') { throw new IOException("UnicodeNameMap file corrupt, byte table"); - int n = in.readUnsignedShort(); + } + int n = in.readInt(); byte[] table = new byte[n]; in.readFully(table); return table; } - public static int hash(String key) { return hash(key, 0, key.length()); } @@ -154,84 +189,80 @@ return (G[f0] + G[f1] + G[f2]) % m; } - private static final char[] charmap = - " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-".toCharArray(); + " ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-()".toCharArray(); private static String getWord(int idx) { - int offset = wordoffs[idx]; - int end = worddata.length; - if (idx < wordoffs.length-1) - end = wordoffs[idx+1]; - StringBuilder buf = new StringBuilder(); - for (int i = offset; i < end; i++) - buf.append(charmap[worddata[i]]); - return buf.toString(); + int offset = wordoffs[idx]; + int end = worddata.length; + if (idx < wordoffs.length-1) + end = wordoffs[idx+1]; + StringBuilder buf = new StringBuilder(); + for (int i = offset; i < end; i++) + buf.append(charmap[worddata[i]]); + return buf.toString(); } - private static boolean match(int idx, byte[] raw, int begin, int end) { - int woff = wordoffs[idx]; - int wend = worddata.length; - if (idx < wordoffs.length-1) - wend = wordoffs[idx+1]; - - if (end-begin != wend - woff) - return false; - int l = end-begin; - for (int i = 0; i < l; i++) { - if (worddata[woff + i] != raw[begin + i]) - return false; - } - return true; + int woff = wordoffs[idx]; + int wend = worddata.length; + if (idx < wordoffs.length-1) { + wend = wordoffs[idx+1]; + } + if (end-begin != wend - woff) { + return false; + } + int l = end-begin; + for (int i = 0; i < l; i++) { + if (worddata[woff + i] != raw[begin + i]) { + return false; + } + } + return true; } - - private static int compare(byte[] a1, int off1, int len1, byte[] a2, int off2, int len2) { for (int i = 0; i < len1 && i < len2; i++) { int d = (a1[off1 + i] & 0xFF) - (a2[off2 + i] & 0xFF); - if (d != 0) + if (d != 0) { return d; + } } return len1 - len2; } + // Was formerly 5, before rawindex changed to 32 bit: + private static final int raw_block = 3; private static int binarysearch(byte[] rawlist, int start, int end) { int floor = 0; - int ceiling = (rawindex.length) / 5; - + int ceiling = (rawindex.length) / raw_block; + int middle, off, len, d; while (floor < ceiling - 1) { - int middle = (floor + ceiling) / 2; - if (debug) - System.out.println("floor:" + floor + " ceiling:" + - ceiling +" => " + middle); - - int off = rawindex[middle*5]; - int len = rawindex[middle*5+4] & 0x1F; - int d = compare(rawlist, start, end - start, rawdata, off, len); - if (d < 0) - ceiling = middle; - else if (d > 0) - floor = middle; - else - return middle * 12; + middle = (floor + ceiling) / 2; + /*if (debug) + System.out.println("floor:" + floor + " ceiling:" + + ceiling +" => " + middle); */ + off = rawindex[middle*raw_block]; + len = rawindex[middle*raw_block+raw_block-1] & 0x1F; + d = compare(rawlist, start, end - start, rawdata, off, len); + if (d < 0) + ceiling = middle; + else if (d > 0) + floor = middle; + else + return middle * 12; } - int tmp = floor*5; - - int off = rawindex[tmp++]; - long lengths = ((long) rawindex[tmp++] << 48) | - ((long) rawindex[tmp++] << 32) | - ((long) rawindex[tmp++] << 16) | - rawindex[tmp++]; - + int tmp = floor*raw_block; + off = rawindex[tmp++]; + long lengths = (long) rawindex[tmp++] << 32 | + rawindex[tmp++] & 0xFFFFFFFFL; floor *= 12; for (int i = 0; i < 12; i++) { - int len = (int) (lengths >> (i * 5)) & 0x1F; + len = (int) (lengths >> (i * 5)) & 0x1F; if (compare(rawlist, start, end, rawdata, off, len) == 0) return floor; off += len; @@ -244,37 +275,109 @@ return lookup(name, 0, name.length()); } - private static int lookup(String name, int start, int end) { - byte[] rawlist = new byte[32]; int ridx = 0; int rbegin = 0; int rstart = 0; - int i; + int i, begin; char ch; byte v; while (true) { rbegin = ridx; - int begin = start; - for (i = start; i < end; i++) { - char ch = name.charAt(i); + begin = start; +lfor: for (i = start; i < end; i++) { + ch = name.charAt(i); + /* if (ch == ' ') { start = i+1; break; } - int v; if (ch >= 'a' && ch <= 'z') ch = (char) (ch - 'a' + 'A'); if (ch >= 'A' && ch <= 'Z') v = ch - 'A' + 1; else if (ch >= '0' && ch <= '9') v = ch - '0' + 27; - else if (ch == '-') - v = 37; - else - return -1; + else { + switch (ch) { + case '-': v = 37; break; + case '(': v = 38; break; + case ')': v = 39; break; + default: return -1; + } + } */ // Unfold this logic into one switch: + // (generated by printCharCases(), see below) + switch (ch) { + case ' ': start = i+1; break lfor; + case 'a': v = 1; break; + case 'b': v = 2; break; + case 'c': v = 3; break; + case 'd': v = 4; break; + case 'e': v = 5; break; + case 'f': v = 6; break; + case 'g': v = 7; break; + case 'h': v = 8; break; + case 'i': v = 9; break; + case 'j': v = 10; break; + case 'k': v = 11; break; + case 'l': v = 12; break; + case 'm': v = 13; break; + case 'n': v = 14; break; + case 'o': v = 15; break; + case 'p': v = 16; break; + case 'q': v = 17; break; + case 'r': v = 18; break; + case 's': v = 19; break; + case 't': v = 20; break; + case 'u': v = 21; break; + case 'v': v = 22; break; + case 'w': v = 23; break; + case 'x': v = 24; break; + case 'y': v = 25; break; + case 'z': v = 26; break; + case 'A': v = 1; break; + case 'B': v = 2; break; + case 'C': v = 3; break; + case 'D': v = 4; break; + case 'E': v = 5; break; + case 'F': v = 6; break; + case 'G': v = 7; break; + case 'H': v = 8; break; + case 'I': v = 9; break; + case 'J': v = 10; break; + case 'K': v = 11; break; + case 'L': v = 12; break; + case 'M': v = 13; break; + case 'N': v = 14; break; + case 'O': v = 15; break; + case 'P': v = 16; break; + case 'Q': v = 17; break; + case 'R': v = 18; break; + case 'S': v = 19; break; + case 'T': v = 20; break; + case 'U': v = 21; break; + case 'V': v = 22; break; + case 'W': v = 23; break; + case 'X': v = 24; break; + case 'Y': v = 25; break; + case 'Z': v = 26; break; + case '0': v = 27; break; + case '1': v = 28; break; + case '2': v = 29; break; + case '3': v = 30; break; + case '4': v = 31; break; + case '5': v = 32; break; + case '6': v = 33; break; + case '7': v = 34; break; + case '8': v = 35; break; + case '9': v = 36; break; + case '-': v = 37; break; + case '(': v = 38; break; + case ')': v = 39; break; + default: return -1; + } - rawlist[ridx++] = (byte) v; + rawlist[ridx++] = v; if (ch == '-' && start != i) { start = ++i; break; @@ -282,55 +385,71 @@ } int hash = hash(name, begin, i); + /* + We skip this try for now, because the issue doesn't occur + with Unicode 9.0 ucnhash.dat bundled with Jython. + Anyway, this might point to some subtle bug. + Todo: Investigate - if (debug) - System.out.println(name.substring(begin, i) + " " + hash); + int hash; + // Currently needed if with older Unicode a + // name containing '(' or ')' is searched: + try { + hash = hash(name, begin, i); + } catch (ArrayIndexOutOfBoundsException aexc) { + return -1; + } + */ - boolean isWord = hash >= 0 && - ridx - rbegin > 1 && - match(hash, rawlist, rbegin, ridx); + // if (debug) System.out.println(name.substring(begin, i) + " " + hash); - if (isWord) { - if (debug) - System.out.println("match " + getWord(hash)); + if (hash >= 0 && ridx - rbegin > 1 && + match(hash, rawlist, rbegin, ridx)) { + // if (debug) System.out.println("match " + getWord(hash)); hash += wordstart; ridx = rstart; if (hash > wordcutoff) { rawlist[ridx++] = (byte) ((hash >> 8) + wordcutoff); rawlist[ridx++] = (byte) (hash & 0xFF); - } else - rawlist[ridx++] = (byte) hash; - } - rstart = ridx; - - if (i >= end) - break; - - if (!isWord) { + } else { + rawlist[ridx++] = (byte) hash; + } + rstart = ridx; + if (i >= end) { + break; + } + } else { + rstart = ridx; + if (i >= end) { + break; + } rawlist[ridx++] = 0; } - } + /* if (debug) { System.out.print("rawdata: "); for (int k = 0; k < ridx; k++) System.out.print((rawlist[k] & 0xFF) + " "); System.out.println(); + } */ + + int idx = binarysearch(rawlist, 0, ridx); + if (idx < 0) { + return idx; } - int idx = binarysearch(rawlist, 0, ridx); - if (idx < 0) - return idx; + /* if (debug) { System.out.println("idx:" + idx); System.out.println("codepoint:" + codepoint[idx] + " " + Integer.toHexString(codepoint[idx])); - } + } */ + return codepoint[idx]; } - // From the ucnhashAPI interface public int getCchMax() { if (!initialized()) @@ -339,25 +458,26 @@ } - private static String cjkPrefix = "CJK COMPATIBILITY IDEOGRAPH-"; private static int cjkPrefixLen = cjkPrefix.length(); // From the ucnhashAPI interface public int getValue(String s, int start, int end) { - if (!initialized()) + if (!initialized()) { + System.out.println(349+" ucnhash Not initialized!"); return -1; + } if (s.regionMatches(start, cjkPrefix, 0, cjkPrefixLen)) { try { - String hex = s.substring(start + cjkPrefixLen, end); - int v = Integer.parseInt(hex, 16); - return v; + String hex = s.substring(start + cjkPrefixLen, end); + int v = Integer.parseInt(hex, 16); + return v; } catch (NumberFormatException exc) { - return -1; // Maybe fallthrough to the main algorithme. + System.out.println(359+" exc"); + return -1; // Maybe fall through to the main algorithm. } } - return lookup(s, start, end); } @@ -365,12 +485,10 @@ private static boolean initialized = false; private static boolean loaded = false; - private synchronized boolean initialized() { - if (initialized && loaded) - return true; - if (initialized) - return false; + if (initialized) { + return loaded; + } try { loadTables(); loaded = true; @@ -381,38 +499,66 @@ return true; } + + /* + public static int lookupChar(char ch) { + int v; + if (ch >= 'a' && ch <= 'z') + ch = (char) (ch - 'a' + 'A'); + if (ch >= 'A' && ch <= 'Z') + v = ch - 'A' + 1; + else if (ch >= '0' && ch <= '9') + v = ch - '0' + 27; + else { + switch (ch) { + case '-': v = 37; break; + case '(': v = 38; break; + case ')': v = 39; break; + default: return -1; + } + } + return v; + } + + public static void printCharCases() { + char[] charmapFull = + " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-()" + .toCharArray(); + for (char ch: charmapFull) { + System.out.println("case \'"+ch+"\': v = "+lookupChar(ch)+"; break;"); + } + } */ + + /* private static boolean debug = false; + public static void main(String[] args) throws Exception { + loadTables(); + debug = true; - public static void main(String[] args) throws Exception { - loadTables(); - - debug = true; - -/* - System.out.println(getWord(hash("ARABIC"))); - System.out.println(getWord(hash("SMALL"))); - System.out.println(getWord(hash("YI"))); - System.out.println(getWord(hash("SYLLABLE"))); - System.out.println(getWord(hash("WITH"))); - System.out.println(getWord(hash("LETTER"))); - - System.out.println(lookup("NULL")); - System.out.println(lookup("LATIN CAPITAL LETTER AFRICAN D")); - System.out.println(lookup("GURMUKHI TIPPI")); - System.out.println(lookup("TIBETAN MARK GTER YIG MGO -UM " + - "RNAM BCAD MA")); - System.out.println(lookup("HANGUL CHOSEONG PIEUP")); - System.out.println(lookup("SINGLE LOW-9 QUOTATION MARK")); -*/ - - System.out.println(lookup("BACKSPACE")); -// System.out.println(lookup("ACTIVATE SYMMETRIC SWAPPING")); - -/* - System.out.println(lookup("LATIN CAPITAL LETTER A")); - System.out.println(lookup("GREATER-THAN SIGN")); - System.out.println(lookup("EURO-CURRENCY SIGN")); -*/ - } + System.out.println(getWord(hash("ARABIC"))); + System.out.println(getWord(hash("SMALL"))); + System.out.println(getWord(hash("YI"))); + System.out.println(getWord(hash("SYLLABLE"))); + System.out.println(getWord(hash("WITH"))); + System.out.println(getWord(hash("LETTER"))); + + System.out.println(lookup("NULL")); // 0 + System.out.println(lookup("LATIN CAPITAL LETTER AFRICAN D")); // 393 + System.out.println(lookup("DOUBLE-STRUCK ITALIC SMALL D")); // 8518 + System.out.println(lookup("GURMUKHI TIPPI")); // 2672 + System.out.println(lookup("TIBETAN MARK GTER YIG MGO -UM" + + " RNAM BCAD MA")); // 3842 + System.out.println(lookup("HANGUL CHOSEONG PIEUP")); // 4359 + System.out.println(lookup("SINGLE LOW-9 QUOTATION MARK")); // 8218 + + System.out.println(lookup("BACKSPACE")); // 8 + System.out.println(lookup("ACTIVATE SYMMETRIC SWAPPING")); // 8299 + + System.out.println(lookup("LATIN CAPITAL LETTER A")); // 65 + System.out.println(lookup("GREATER-THAN SIGN")); // 62 + System.out.println(lookup("EURO-CURRENCY SIGN")); // 8352 + System.out.println(lookup("FORM FEED (FF)")); // 12 + System.out.println(lookup("FORM FEED (F")); // -1 + } */ } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 5 17:59:37 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sun, 05 Feb 2017 22:59:37 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Cleanup_of_previous_commit?= =?utf-8?q?=2E?= Message-ID: <20170205225937.15872.11436.0AFC3D16@psf.io> https://hg.python.org/jython/rev/1562e8943e3e changeset: 8015:1562e8943e3e user: Stefan Richthofer date: Sun Feb 05 23:58:53 2017 +0100 summary: Cleanup of previous commit. files: Misc/make_ucnhashdat.py | 2 +- src/org/python/modules/ucnhash.java | 2 -- 2 files changed, 1 insertions(+), 3 deletions(-) diff --git a/Misc/make_ucnhashdat.py b/Misc/make_ucnhashdat.py --- a/Misc/make_ucnhashdat.py +++ b/Misc/make_ucnhashdat.py @@ -4,7 +4,7 @@ # Generate a ucnhash.dat file with mapping from unicode # names to codepoints. # -# python make_ucnhashdata.py UnicodeData.txt mph.exe +# python make_ucnhashdat.py UnicodeData.txt mph.exe # # The "mph" program must be available on the path. # This program is used to create the minimum perfect diff --git a/src/org/python/modules/ucnhash.java b/src/org/python/modules/ucnhash.java --- a/src/org/python/modules/ucnhash.java +++ b/src/org/python/modules/ucnhash.java @@ -464,7 +464,6 @@ // From the ucnhashAPI interface public int getValue(String s, int start, int end) { if (!initialized()) { - System.out.println(349+" ucnhash Not initialized!"); return -1; } @@ -474,7 +473,6 @@ int v = Integer.parseInt(hex, 16); return v; } catch (NumberFormatException exc) { - System.out.println(359+" exc"); return -1; // Maybe fall through to the main algorithm. } } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 5 19:17:34 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 06 Feb 2017 00:17:34 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232539_=28via_workar?= =?utf-8?q?ound=29=2E_Todo=3A_Consider_to_file_a_Java_bug_report_about?= Message-ID: <20170206001734.128369.26833.61C1B320@psf.io> https://hg.python.org/jython/rev/2e5bc089096a changeset: 8016:2e5bc089096a user: Stefan Richthofer date: Mon Feb 06 01:17:24 2017 +0100 summary: Fixed #2539 (via workaround). Todo: Consider to file a Java bug report about this. files: src/org/python/util/InteractiveConsole.java | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diff --git a/src/org/python/util/InteractiveConsole.java b/src/org/python/util/InteractiveConsole.java --- a/src/org/python/util/InteractiveConsole.java +++ b/src/org/python/util/InteractiveConsole.java @@ -22,7 +22,13 @@ // Based on CPython-1.5.2's code module public class InteractiveConsole extends InteractiveInterpreter { - public static final String CONSOLE_FILENAME = ""; + /** + * Note: This field is actually final; don't modify. + *

+ * To work around an issue in javadoc with Java 8 we cannot have it final for now, see + * issue 2539 for details. + */ + public static String CONSOLE_FILENAME = ""; public String filename; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Feb 6 21:18:16 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Tue, 07 Feb 2017 02:18:16 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Partial_fix_of_=232476=3A_R?= =?utf-8?q?estore_workability_of_functools=2Ewraps=28=29_by_adding?= Message-ID: <20170207021815.26087.6228.2652AACE@psf.io> https://hg.python.org/jython/rev/f56f3eeca6f2 changeset: 8017:f56f3eeca6f2 user: Stefan Richthofer date: Tue Feb 07 03:17:39 2017 +0100 summary: Partial fix of #2476: Restore workability of functools.wraps() by adding __module__ field to PyReflectedFunction. Also moves forward #2475. files: src/org/python/core/PyJavaType.java | 28 ++++++++- src/org/python/core/PyReflectedFunction.java | 10 ++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/org/python/core/PyJavaType.java b/src/org/python/core/PyJavaType.java --- a/src/org/python/core/PyJavaType.java +++ b/src/org/python/core/PyJavaType.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Queue; @@ -304,6 +305,8 @@ Arrays.sort(methods, new MethodComparator(new ClassComparator())); boolean isInAwt = name.startsWith("java.awt.") && name.indexOf('.', 9) == -1; + ArrayList reflectedFuncs = new ArrayList<>(methods.length); + PyReflectedFunction reflfunc; for (Method meth : methods) { if (!declaredOnMember(baseClass, meth) || ignore(meth)) { continue; @@ -319,9 +322,11 @@ } String nmethname = normalize(methname); - PyReflectedFunction reflfunc = (PyReflectedFunction)dict.__finditem__(nmethname); + reflfunc = (PyReflectedFunction) dict.__finditem__(nmethname); if (reflfunc == null) { - dict.__setitem__(nmethname, new PyReflectedFunction(meth)); + reflfunc = new PyReflectedFunction(meth); + reflectedFuncs.add(reflfunc); + dict.__setitem__(nmethname, reflfunc); } else { reflfunc.addMethod(meth); } @@ -390,7 +395,7 @@ // Add superclass methods for (Method meth : methods) { String nmethname = normalize(meth.getName()); - PyReflectedFunction reflfunc = (PyReflectedFunction)dict.__finditem__(nmethname); + reflfunc = (PyReflectedFunction)dict.__finditem__(nmethname); if (reflfunc != null) { // The superclass method has the same name as one declared on this class, so add // the superclass version's arguments @@ -401,7 +406,9 @@ // visible from Java on this class, so do the same for Python here. This is the // flipside of what handleSuperMethodArgCollisions does for inherited public methods // on package protected classes. - dict.__setitem__(nmethname, new PyReflectedFunction(meth)); + reflfunc = new PyReflectedFunction(meth); + reflectedFuncs.add(reflfunc); + dict.__setitem__(nmethname, reflfunc); } } @@ -557,12 +564,13 @@ } } + PyObject nameSpecified = null; if (ClassDictInit.class.isAssignableFrom(forClass) && forClass != ClassDictInit.class) { try { Method m = forClass.getMethod("classDictInit", PyObject.class); m.invoke(null, dict); // allow the class to override its name after it is loaded - PyObject nameSpecified = dict.__finditem__("__name__"); + nameSpecified = dict.__finditem__("__name__"); if (nameSpecified != null) { name = nameSpecified.toString(); } @@ -571,6 +579,16 @@ } } + // Fill __module__ attribute of PyReflectedFunctions... + if (reflectedFuncs.size() > 0) { + if (nameSpecified == null) { + nameSpecified = Py.newString(name); + } + for (PyReflectedFunction func: reflectedFuncs) { + func.__module__ = nameSpecified; + } + } + if (baseClass != Object.class) { hasGet = getDescrMethod(forClass, "__get__", OO) != null || getDescrMethod(forClass, "_doget", PyObject.class) != null diff --git a/src/org/python/core/PyReflectedFunction.java b/src/org/python/core/PyReflectedFunction.java --- a/src/org/python/core/PyReflectedFunction.java +++ b/src/org/python/core/PyReflectedFunction.java @@ -13,6 +13,8 @@ public PyObject __doc__ = Py.None; + public PyObject __module__ = Py.None; + public ReflectedArgs[] argslist = new ReflectedArgs[1]; public int nargs; @@ -64,6 +66,7 @@ public PyReflectedFunction copy() { PyReflectedFunction func = new PyReflectedFunction(__name__); func.__doc__ = __doc__; + func.__module__ = __module__; func.nargs = nargs; func.argslist = new ReflectedArgs[nargs]; System.arraycopy(argslist, 0, func.argslist, 0, nargs); @@ -158,7 +161,6 @@ ReflectedCallData callData = new ReflectedCallData(); ReflectedArgs match = null; for (int i = 0; i < nargs && match == null; i++) { - // System.err.println(rargs.toString()); if (argslist[i].matches(self, args, keywords, callData)) { match = argslist[i]; } @@ -340,11 +342,15 @@ /* Traverseproc implementation */ @Override public int traverse(Visitproc visit, Object arg) { + if (__module__ != null) { + int res = visit.visit(__module__, arg); + if (res != 0) return res; + } return __doc__ != null ? visit.visit(__doc__, arg) : 0; } @Override public boolean refersDirectlyTo(PyObject ob) { - return ob != null && ob == __doc__; + return ob != null && (ob == __doc__ || ob == __module__); } } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Tue Feb 7 11:47:46 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Tue, 07 Feb 2017 16:47:46 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232549=2E?= Message-ID: <20170207164745.25629.20858.85B0B21C@psf.io> https://hg.python.org/jython/rev/a46658925074 changeset: 8018:a46658925074 user: Stefan Richthofer date: Tue Feb 07 17:46:25 2017 +0100 summary: Fixed #2549. files: NEWS | 1 + src/org/python/core/PyJavaType.java | 2 +- src/org/python/modules/posix/PyStatResult.java | 59 ++++----- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2549 ] test_posix fails on Linux - [ 2548 ] Unicode u'\N{name}' frequently broken, because ucnhash.dat outdated - [ 2527 ] cStringIO throws IllegalArgumentException with non-ASCII values - [ 2511 ] Percent operator calls __getattr__('__getitem__') diff --git a/src/org/python/core/PyJavaType.java b/src/org/python/core/PyJavaType.java --- a/src/org/python/core/PyJavaType.java +++ b/src/org/python/core/PyJavaType.java @@ -395,7 +395,7 @@ // Add superclass methods for (Method meth : methods) { String nmethname = normalize(meth.getName()); - reflfunc = (PyReflectedFunction)dict.__finditem__(nmethname); + reflfunc = (PyReflectedFunction) dict.__finditem__(nmethname); if (reflfunc != null) { // The superclass method has the same name as one declared on this class, so add // the superclass version's arguments diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java --- a/src/org/python/modules/posix/PyStatResult.java +++ b/src/org/python/modules/posix/PyStatResult.java @@ -42,7 +42,8 @@ public static final int n_sequence_fields = 10, n_fields = 10, n_unnamed_fields = 10; PyStatResult(PyObject... vals) { - super(TYPE, vals); + super(TYPE, new PyObject[] {vals[0], vals[1], vals[2], vals[3], vals[4], vals[5], vals[6], + vals[7].__int__(), vals[8].__int__(), vals[9].__int__()}); st_mode = vals[0]; st_ino = vals[1]; st_dev = vals[2]; @@ -55,6 +56,20 @@ st_ctime = vals[9]; } + protected PyStatResult(PyObject[] vals, PyObject st_atime, PyObject st_mtime, PyObject st_ctime) { + super(TYPE, vals); + st_mode = vals[0]; + st_ino = vals[1]; + st_dev = vals[2]; + st_nlink = vals[3]; + st_uid = vals[4]; + st_gid = vals[5]; + st_size = vals[6]; + this.st_atime = st_atime; + this.st_mtime = st_mtime; + this.st_ctime = st_ctime; + } + @ExposedNew static PyObject stat_result_new(PyNewWrapper wrapper, boolean init, PyType subtype, PyObject[] args, String[] keywords) { @@ -67,7 +82,12 @@ throw Py.TypeError(msg); } // tuples are immutable, so we can just use its underlying array - return new PyStatResult(((PyTuple)obj).getArray()); + if (obj instanceof PyStatResult) { + return new PyStatResult(((PyTuple) obj).getArray()); + } else { + return new PyStatResult(((PyTuple) obj).getArray(), ((PyStatResult) obj).st_atime, + ((PyStatResult) obj).st_mtime, ((PyStatResult) obj).st_ctime); + } } else { PyList seq = new PyList(obj); @@ -152,35 +172,6 @@ Py.newFloat(fromFileTime(stat.creationTime()))); } - // Override pyget, getslice to preserve backwards compatiblity that ints are returned for time elements - // if accessing by an index or slice - - private final static int ST_ATIME = 7; - private final static int ST_MTIME = 8; - private final static int ST_CTIME = 9; - - @Override - public PyObject pyget(int index) { - if (index == ST_ATIME || index == ST_MTIME || index == ST_CTIME) { - return super.pyget(index).__int__(); - } else { - return super.pyget(index); - } - } - - @Override - protected PyObject getslice(int start, int stop, int step) { - if (step > 0 && stop < start) { - stop = start; - } - int n = sliceLength(start, stop, step); - PyObject[] newArray = new PyObject[n]; - for (int i = start, j = 0; j < n; i += step, j++) { - newArray[j] = pyget(i); - } - return new PyTuple(newArray, false); - } - @Override public synchronized PyObject __eq__(PyObject o) { return stat_result___eq__(o); @@ -232,7 +223,11 @@ @Override public PyTuple __getnewargs__() { - return new PyTuple(new PyList(getArray())); + PyList lst = new PyList(getArray()); + lst.pyset(7, st_atime); + lst.pyset(8, st_mtime); + lst.pyset(9, st_ctime); + return new PyTuple(lst); } @Override -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Tue Feb 7 11:51:20 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Tue, 07 Feb 2017 16:51:20 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Correction_in_previous_comm?= =?utf-8?b?aXQuICgjMjU0OSk=?= Message-ID: <20170207165118.110885.222.D0E09617@psf.io> https://hg.python.org/jython/rev/923ffc94b619 changeset: 8019:923ffc94b619 user: Stefan Richthofer date: Tue Feb 07 17:51:12 2017 +0100 summary: Correction in previous commit. (#2549) files: src/org/python/modules/posix/PyStatResult.java | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java --- a/src/org/python/modules/posix/PyStatResult.java +++ b/src/org/python/modules/posix/PyStatResult.java @@ -83,10 +83,10 @@ } // tuples are immutable, so we can just use its underlying array if (obj instanceof PyStatResult) { - return new PyStatResult(((PyTuple) obj).getArray()); - } else { return new PyStatResult(((PyTuple) obj).getArray(), ((PyStatResult) obj).st_atime, ((PyStatResult) obj).st_mtime, ((PyStatResult) obj).st_ctime); + } else { + return new PyStatResult(((PyTuple) obj).getArray()); } } else { -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Feb 8 10:11:34 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 08 Feb 2017 15:11:34 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232550=2E_test=5Flis?= =?utf-8?q?t=5Fjy_now_passes_also_on_Java_8=2E?= Message-ID: <20170208151131.111010.1460.C8BA4D61@psf.io> https://hg.python.org/jython/rev/423d8c23acb5 changeset: 8020:423d8c23acb5 user: Stefan Richthofer date: Wed Feb 08 16:11:15 2017 +0100 summary: Fixed #2550. test_list_jy now passes also on Java 8. files: NEWS | 1 + src/org/python/core/JavaProxyList.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2550 ] test_list_jy fails on Java 8 - [ 2549 ] test_posix fails on Linux - [ 2548 ] Unicode u'\N{name}' frequently broken, because ucnhash.dat outdated - [ 2527 ] cStringIO throws IllegalArgumentException with non-ASCII values diff --git a/src/org/python/core/JavaProxyList.java b/src/org/python/core/JavaProxyList.java --- a/src/org/python/core/JavaProxyList.java +++ b/src/org/python/core/JavaProxyList.java @@ -630,13 +630,13 @@ listIAddProxy, new ListMulProxyClass("__mul__", 1), new ListMulProxyClass("__rmul__", 1), - listIMulProxy, - listSortProxy, + listIMulProxy }; } static PyBuiltinMethod[] getPostProxyMethods() { return new PyBuiltinMethod[]{ + listSortProxy, listRemoveOverrideProxy }; } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Feb 8 17:13:09 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 08 Feb 2017 22:13:09 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232551=2E_As_a_conse?= =?utf-8?q?quence_this_fixes_=232475_and_=231777_and_maybe_more=2E?= Message-ID: <20170208221308.7440.23615.FB895FA8@psf.io> https://hg.python.org/jython/rev/89747939dee1 changeset: 8021:89747939dee1 user: Stefan Richthofer date: Wed Feb 08 23:12:54 2017 +0100 summary: Fixed #2551. As a consequence this fixes #2475 and #1777 and maybe more. Detection of solid bases in PyType nedded an adjustment to deal correctly with the case described in #2551. files: Lib/test/test_class_jy.py | 15 +++++++++++++++ NEWS | 6 ++++-- src/org/python/core/PyType.java | 4 +++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_class_jy.py b/Lib/test/test_class_jy.py --- a/Lib/test/test_class_jy.py +++ b/Lib/test/test_class_jy.py @@ -184,6 +184,21 @@ class D(B, C): pass + def test_slotted_diamond_problem_bug_v2(self): + # see http://bugs.jython.org/issue2551 + class A(object): + __slots__ = 'foo' + class B(A): + __slots__ = 'bar' + pass + class C(A): + __slots__ = () + pass + # used to raise TypeError: multiple bases have instance lay-out + # conflict + class D(B, C): + pass + def test_getitem_exceptions(self): class A: def __getitem__(self, key): diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,9 @@ Jython 2.7.1rc1 Bugs fixed + - [ 1777 ] Help getting SymPy working with Jython + - [ 2475 ] Backported python 3 pathlib and pathlib2 do not work b/c of meta class issues + - [ 2551 ] __slots__ in diamond-style causes lay-out conflict (breaks pathlib and sympy) - [ 2550 ] test_list_jy fails on Java 8 - [ 2549 ] test_posix fails on Linux - [ 2548 ] Unicode u'\N{name}' frequently broken, because ucnhash.dat outdated @@ -54,9 +57,8 @@ - [ 1767 ] Rich comparisons New Features - - SymPy is supported for versions that incorporate https://github.com/sympy/sympy/pull/12120. + - SymPy is now workable. (However it runs somewhat slow; some profiling will be required.) - Updated ucnhash to support name lookup for Unicode 9.0 (like in u'\N{name}'). - This is a crucial step to allow SymPy support. - Jython doc-entries in Java-code (__doc__foo) now accept anything that implements java.lang.CharSequence (PyString implements CharSequence now as well). This allows Jython-friendly Java-code without PyString-induced dependency on Jython. diff --git a/src/org/python/core/PyType.java b/src/org/python/core/PyType.java --- a/src/org/python/core/PyType.java +++ b/src/org/python/core/PyType.java @@ -1127,7 +1127,9 @@ } private static boolean isSolidBase(PyType type) { - return type.underlying_class != null || (type.numSlots != 0 && !type.needs_userdict); + return type.underlying_class != null || + (type.numSlots - (type.base != null ? type.base.numSlots : 0) != 0 && + !type.needs_userdict); } /** -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 19 15:17:35 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sun, 19 Feb 2017 20:17:35 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Merged_https=3A//github=2Ec?= =?utf-8?q?om/jythontools/jython/pull/55=3A_Replace_deprecated_use_of?= Message-ID: <20170219201735.19463.52267.B32C5501@psf.io> https://hg.python.org/jython/rev/72a72b948afb changeset: 8022:72a72b948afb user: James Mudd date: Sun Feb 19 21:16:44 2017 +0100 summary: Merged https://github.com/jythontools/jython/pull/55: Replace deprecated use of ASCII constant with ascii(). CharMatcher.ASCII is now deprecated in Guava and will be removed in June 2018. This also eliminates some deprecation-warnings during build. files: Lib/test/test_class_jy.py | 2 -- NEWS | 1 + src/org/python/core/Py.java | 4 ++-- src/org/python/core/PyBaseCode.java | 2 +- src/org/python/core/PyUnicode.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_class_jy.py b/Lib/test/test_class_jy.py --- a/Lib/test/test_class_jy.py +++ b/Lib/test/test_class_jy.py @@ -190,10 +190,8 @@ __slots__ = 'foo' class B(A): __slots__ = 'bar' - pass class C(A): __slots__ = () - pass # used to raise TypeError: multiple bases have instance lay-out # conflict class D(B, C): diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ PR55 ] Replace deprecated use of ASCII constant with ascii() - [ 1777 ] Help getting SymPy working with Jython - [ 2475 ] Backported python 3 pathlib and pathlib2 do not work b/c of meta class issues - [ 2551 ] __slots__ in diamond-style causes lay-out conflict (breaks pathlib and sympy) diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java --- a/src/org/python/core/Py.java +++ b/src/org/python/core/Py.java @@ -667,7 +667,7 @@ * s. */ public static PyString newStringOrUnicode(PyObject precedent, String s) { - if (!(precedent instanceof PyUnicode) && CharMatcher.ASCII.matchesAllOf(s)) { + if (!(precedent instanceof PyUnicode) && CharMatcher.ascii().matchesAllOf(s)) { return Py.newString(s); } else { return Py.newUnicode(s); @@ -675,7 +675,7 @@ } public static PyString newStringUTF8(String s) { - if (CharMatcher.ASCII.matchesAllOf(s)) { + if (CharMatcher.ascii().matchesAllOf(s)) { // ascii of course is a subset of UTF-8 return Py.newString(s); } else { diff --git a/src/org/python/core/PyBaseCode.java b/src/org/python/core/PyBaseCode.java --- a/src/org/python/core/PyBaseCode.java +++ b/src/org/python/core/PyBaseCode.java @@ -256,7 +256,7 @@ co_name, Py.newUnicode(keyword).encode("ascii", "replace"))); } - if (CharMatcher.ASCII.matchesAllOf(keyword)) { + if (CharMatcher.ascii().matchesAllOf(keyword)) { kwdict.__setitem__(keyword, value); } else { kwdict.__setitem__(Py.newUnicode(keyword), value); diff --git a/src/org/python/core/PyUnicode.java b/src/org/python/core/PyUnicode.java --- a/src/org/python/core/PyUnicode.java +++ b/src/org/python/core/PyUnicode.java @@ -583,7 +583,7 @@ } public static String checkEncoding(String s) { - if (s == null || CharMatcher.ASCII.matchesAllOf(s)) { + if (s == null || CharMatcher.ascii().matchesAllOf(s)) { return s; } return codecs.PyUnicode_EncodeASCII(s, s.length(), null); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 19 15:55:36 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sun, 19 Feb 2017 20:55:36 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_issue_in_Lib/distutil?= =?utf-8?q?s/util=2Epy_that_had_been_introduced_with?= Message-ID: <20170219205536.15078.85021.48337E5D@psf.io> https://hg.python.org/jython/rev/b5c40b5e9da3 changeset: 8023:b5c40b5e9da3 user: Stefan Richthofer date: Sun Feb 19 21:55:18 2017 +0100 summary: Fixed issue in Lib/distutils/util.py that had been introduced with https://hg.python.org/jython/rev/7d7923408952. This e.g. affected pip (in addition to #2521). files: Lib/distutils/util.py | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -12,6 +12,8 @@ from distutils.spawn import spawn from distutils import log from distutils.errors import DistutilsByteCompileError +if sys.platform.startswith('java'): + from imp import _makeCompiledFilename def get_platform (): """Return a string that identifies the current platform. This is used @@ -546,7 +548,7 @@ # cfile - byte-compiled file # dfile - purported source filename (same as 'file' by default) if sys.platform.startswith('java'): - cfile = py_compile.imp._makeCompiledFilename(file) + cfile = _makeCompiledFilename(file) else: cfile = file + (__debug__ and "c" or "o") dfile = file -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 19 16:14:25 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sun, 19 Feb 2017 21:14:25 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Minor_improvement_in_previo?= =?utf-8?q?us_commit=2E?= Message-ID: <20170219211425.14554.69429.4F90BED7@psf.io> https://hg.python.org/jython/rev/bcc05ae4e312 changeset: 8024:bcc05ae4e312 user: Stefan Richthofer date: Sun Feb 19 22:14:15 2017 +0100 summary: Minor improvement in previous commit. files: Lib/distutils/util.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -13,7 +13,7 @@ from distutils import log from distutils.errors import DistutilsByteCompileError if sys.platform.startswith('java'): - from imp import _makeCompiledFilename + import _imp def get_platform (): """Return a string that identifies the current platform. This is used @@ -548,7 +548,7 @@ # cfile - byte-compiled file # dfile - purported source filename (same as 'file' by default) if sys.platform.startswith('java'): - cfile = _makeCompiledFilename(file) + cfile = _imp.makeCompiledFilename(file) else: cfile = file + (__debug__ and "c" or "o") dfile = file -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Feb 23 13:24:39 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 23 Feb 2017 18:24:39 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_LongInfo_and_FloatInf?= =?utf-8?q?o_to_CoreExposed=2Eincludes=2E_type=28sys=2Elong=5Finfo=29?= Message-ID: <20170223182438.122169.45071.4232D6A3@psf.io> https://hg.python.org/jython/rev/d856cb974c7b changeset: 8025:d856cb974c7b user: Stefan Richthofer date: Thu Feb 23 19:23:59 2017 +0100 summary: Added LongInfo and FloatInfo to CoreExposed.includes. type(sys.long_info) displayed wrong type, because FloatInfo and LongInfo were not exposed. files: CoreExposed.includes | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/CoreExposed.includes b/CoreExposed.includes --- a/CoreExposed.includes +++ b/CoreExposed.includes @@ -1,5 +1,7 @@ org/python/core/AstList.class org/python/core/ClasspathPyImporter.class +org/python/core/FloatInfo.class +org/python/core/LongInfo.class org/python/core/Py2kBuffer.class org/python/core/PyArray.class org/python/core/PyBaseString.class -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Feb 23 13:44:31 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 23 Feb 2017 18:44:31 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_traverseprocs_to_some?= =?utf-8?q?_classes_in_org=2Epython=2Emodules=2Ejffi=2E?= Message-ID: <20170223184430.71449.26233.4286D7B4@psf.io> https://hg.python.org/jython/rev/d57b03d7489c changeset: 8026:d57b03d7489c user: Stefan Richthofer date: Thu Feb 23 19:44:18 2017 +0100 summary: Added traverseprocs to some classes in org.python.modules.jffi. files: src/org/python/modules/jffi/AbstractMemoryCData.java | 1 + src/org/python/modules/jffi/CType.java | 46 +++++++- src/org/python/modules/jffi/DynamicLibrary.java | 5 +- src/org/python/modules/jffi/Function.java | 54 +++++++++- src/org/python/modules/jffi/JITInvoker.java | 4 - src/org/python/modules/jffi/JITInvoker0.java | 4 - src/org/python/modules/jffi/JITInvoker1.java | 3 - src/org/python/modules/jffi/JITInvoker2.java | 3 - src/org/python/modules/jffi/JITInvoker3.java | 3 - src/org/python/modules/jffi/JITInvoker4.java | 3 - src/org/python/modules/jffi/JITInvoker5.java | 3 - src/org/python/modules/jffi/JITInvoker6.java | 3 - src/org/python/modules/jffi/JITMethodGenerator.java | 3 - src/org/python/modules/jffi/JITRuntime.java | 3 - src/org/python/modules/jffi/JITSignature.java | 4 - src/org/python/modules/jffi/Memory.java | 1 - src/org/python/modules/jffi/MemoryOp.java | 1 - src/org/python/modules/jffi/NativeDataConverter.java | 4 - src/org/python/modules/jffi/NativeMemory.java | 1 - src/org/python/modules/jffi/NativeType.java | 2 - src/org/python/modules/jffi/NullMemory.java | 2 - src/org/python/modules/jffi/Pointer.java | 1 - src/org/python/modules/jffi/PointerCData.java | 1 - src/org/python/modules/jffi/ScalarCData.java | 30 ++++- src/org/python/modules/jffi/StringCData.java | 4 - src/org/python/modules/jffi/StructLayout.java | 43 +++++++- src/org/python/modules/jffi/Structure.java | 24 ++++- src/org/python/modules/jffi/Util.java | 1 - src/org/python/modules/jffi/jffi.java | 1 - 29 files changed, 186 insertions(+), 72 deletions(-) diff --git a/src/org/python/modules/jffi/AbstractMemoryCData.java b/src/org/python/modules/jffi/AbstractMemoryCData.java --- a/src/org/python/modules/jffi/AbstractMemoryCData.java +++ b/src/org/python/modules/jffi/AbstractMemoryCData.java @@ -10,6 +10,7 @@ super(subtype, type); this.memory = memory; } + @Override public boolean __nonzero__() { return !getMemory().isNull(); diff --git a/src/org/python/modules/jffi/CType.java b/src/org/python/modules/jffi/CType.java --- a/src/org/python/modules/jffi/CType.java +++ b/src/org/python/modules/jffi/CType.java @@ -8,6 +8,8 @@ import org.python.core.PyObject; import org.python.core.PyType; import org.python.core.Untraversable; +import org.python.core.Traverseproc; +import org.python.core.Visitproc; import org.python.expose.ExposeAsSuperclass; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; @@ -98,6 +100,7 @@ } } + @Untraversable @ExposedType(name = "jffi.Type.Custom", base = CType.class) static class Custom extends CType { final com.kenai.jffi.Type jffiType; @@ -127,7 +130,7 @@ } @ExposedType(name = "jffi.Type.Array", base = CType.class) - static final class Array extends CType.Custom { + static final class Array extends CType.Custom implements Traverseproc { public static final PyType TYPE = PyType.fromClass(Array.class); final CType componentType; final PyType pyComponentType; @@ -178,16 +181,33 @@ throw Py.TypeError("only scalar and struct types supported"); } } + + + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + if (componentType != null) { + int res = visit.visit(componentType, arg); + if (res != 0) { + return res; + } + } + return pyComponentType != null ? visit.visit(pyComponentType, arg) : 0; + } + + @Override + public boolean refersDirectlyTo(PyObject ob) { + return ob != null && (ob == componentType || ob == pyComponentType); + } } - @ExposedType(name = "jffi.Type.Pointer", base = CType.class) - final static class Pointer extends Custom { + final static class Pointer extends Custom implements Traverseproc { public static final PyType TYPE = PyType.fromClass(Pointer.class); private static final ConcurrentMap typeCache = new ConcurrentHashMap(); - + final CType componentType; final PyType pyComponentType; final MemoryOp componentMemoryOp; @@ -234,7 +254,7 @@ public final String toString() { return String.format("", componentType.toString()); } - + private static final class ScalarOp extends MemoryOp { private final MemoryOp op; private final PyType type; @@ -262,5 +282,21 @@ } + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + if (componentType != null) { + int res = visit.visit(componentType, arg); + if (res != 0) { + return res; + } + } + return pyComponentType != null ? visit.visit(pyComponentType, arg) : 0; + } + + @Override + public boolean refersDirectlyTo(PyObject ob) { + return ob != null && (ob == componentType || ob == pyComponentType); + } } } diff --git a/src/org/python/modules/jffi/DynamicLibrary.java b/src/org/python/modules/jffi/DynamicLibrary.java --- a/src/org/python/modules/jffi/DynamicLibrary.java +++ b/src/org/python/modules/jffi/DynamicLibrary.java @@ -57,6 +57,7 @@ return new DataSymbol(this, name.asString(), findSymbol(name)); } + @Untraversable @ExposedType(name = "jffi.DynamicLibrary.Symbol", base = PyObject.class) public static class Symbol extends BasePointer { public static final PyType TYPE = PyType.fromClass(Symbol.class); @@ -78,13 +79,15 @@ return memory; } } - + + @Untraversable public static final class TextSymbol extends Symbol implements ExposeAsSuperclass { public TextSymbol(DynamicLibrary lib, String name, long address) { super(lib, name, new SymbolMemory(lib, address)); } } + @Untraversable public static final class DataSymbol extends Symbol implements ExposeAsSuperclass { public DataSymbol(DynamicLibrary lib, String name, long address) { super(lib, name, new SymbolMemory(lib, address)); diff --git a/src/org/python/modules/jffi/Function.java b/src/org/python/modules/jffi/Function.java --- a/src/org/python/modules/jffi/Function.java +++ b/src/org/python/modules/jffi/Function.java @@ -9,13 +9,15 @@ import org.python.core.PySequenceList; import org.python.core.PyStringMap; import org.python.core.PyType; +import org.python.core.Traverseproc; +import org.python.core.Visitproc; import org.python.expose.ExposedGet; import org.python.expose.ExposedNew; import org.python.expose.ExposedSet; import org.python.expose.ExposedType; @ExposedType(name = "jffi.Function", base = PyObject.class) -public class Function extends BasePointer implements Pointer { +public class Function extends BasePointer implements Traverseproc { public static final PyType TYPE = PyType.fromClass(Function.class); private final Pointer pointer; @@ -262,4 +264,54 @@ return errcheck.__call__(invoker.invoke(arg1, arg2, arg3, arg4, arg5, arg6)); } } + + + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + int res = 0; + if (pointer != null && pointer instanceof PyObject) { + res = visit.visit((PyObject) pointer, arg); + if (res != 0) { + return res; + } + } + if (dict != null) { + res = visit.visit(dict, arg); + if (res != 0) { + return res; + } + } + if (restype != null) { + res = visit.visit(restype, arg); + if (res != 0) { + return res; + } + } + if (argtypes != null) { + for (PyObject obj: argtypes) { + res = visit.visit(obj, arg); + if (res != 0) { + return res; + } + } + } + return 0; + } + + @Override + public boolean refersDirectlyTo(PyObject ob) throws UnsupportedOperationException { + if (ob != null && (pointer == ob || dict == ob || restype == ob)) { + return true; + }; + if (argtypes != null) + { + for (PyObject obj: argtypes) { + if (obj == ob) { + return true; + } + } + } + return false; + } } diff --git a/src/org/python/modules/jffi/JITInvoker.java b/src/org/python/modules/jffi/JITInvoker.java --- a/src/org/python/modules/jffi/JITInvoker.java +++ b/src/org/python/modules/jffi/JITInvoker.java @@ -1,12 +1,8 @@ package org.python.modules.jffi; -import com.kenai.jffi.*; import org.python.core.Py; import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker extends Invoker { protected static final com.kenai.jffi.Invoker jffiInvoker = com.kenai.jffi.Invoker.getInstance(); protected final com.kenai.jffi.Function jffiFunction; diff --git a/src/org/python/modules/jffi/JITInvoker0.java b/src/org/python/modules/jffi/JITInvoker0.java --- a/src/org/python/modules/jffi/JITInvoker0.java +++ b/src/org/python/modules/jffi/JITInvoker0.java @@ -1,11 +1,7 @@ package org.python.modules.jffi; -import com.kenai.jffi.*; import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker0 extends JITInvoker { public JITInvoker0(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(0, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker1.java b/src/org/python/modules/jffi/JITInvoker1.java --- a/src/org/python/modules/jffi/JITInvoker1.java +++ b/src/org/python/modules/jffi/JITInvoker1.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker1 extends JITInvoker { public JITInvoker1(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(1, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker2.java b/src/org/python/modules/jffi/JITInvoker2.java --- a/src/org/python/modules/jffi/JITInvoker2.java +++ b/src/org/python/modules/jffi/JITInvoker2.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker2 extends JITInvoker { public JITInvoker2(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(2, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker3.java b/src/org/python/modules/jffi/JITInvoker3.java --- a/src/org/python/modules/jffi/JITInvoker3.java +++ b/src/org/python/modules/jffi/JITInvoker3.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker3 extends JITInvoker { public JITInvoker3(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(3, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker4.java b/src/org/python/modules/jffi/JITInvoker4.java --- a/src/org/python/modules/jffi/JITInvoker4.java +++ b/src/org/python/modules/jffi/JITInvoker4.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker4 extends JITInvoker { public JITInvoker4(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(4, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker5.java b/src/org/python/modules/jffi/JITInvoker5.java --- a/src/org/python/modules/jffi/JITInvoker5.java +++ b/src/org/python/modules/jffi/JITInvoker5.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker5 extends JITInvoker { public JITInvoker5(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(5, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITInvoker6.java b/src/org/python/modules/jffi/JITInvoker6.java --- a/src/org/python/modules/jffi/JITInvoker6.java +++ b/src/org/python/modules/jffi/JITInvoker6.java @@ -2,9 +2,6 @@ import org.python.core.PyObject; -/** - * - */ abstract public class JITInvoker6 extends JITInvoker { public JITInvoker6(com.kenai.jffi.Function function, Invoker fallbackInvoker) { super(6, function, fallbackInvoker); diff --git a/src/org/python/modules/jffi/JITMethodGenerator.java b/src/org/python/modules/jffi/JITMethodGenerator.java --- a/src/org/python/modules/jffi/JITMethodGenerator.java +++ b/src/org/python/modules/jffi/JITMethodGenerator.java @@ -1,8 +1,5 @@ package org.python.modules.jffi; -/** - * - */ public interface JITMethodGenerator { public boolean isSupported(JITSignature signature); diff --git a/src/org/python/modules/jffi/JITRuntime.java b/src/org/python/modules/jffi/JITRuntime.java --- a/src/org/python/modules/jffi/JITRuntime.java +++ b/src/org/python/modules/jffi/JITRuntime.java @@ -5,9 +5,6 @@ import java.math.BigInteger; -/** - * - */ public final class JITRuntime { private static final com.kenai.jffi.MemoryIO IO = com.kenai.jffi.MemoryIO.getInstance(); diff --git a/src/org/python/modules/jffi/JITSignature.java b/src/org/python/modules/jffi/JITSignature.java --- a/src/org/python/modules/jffi/JITSignature.java +++ b/src/org/python/modules/jffi/JITSignature.java @@ -1,12 +1,8 @@ package org.python.modules.jffi; import com.kenai.jffi.CallingConvention; - import java.util.Arrays; -/** - * - */ public final class JITSignature { private final NativeType resultType; private final NativeType[] parameterTypes; diff --git a/src/org/python/modules/jffi/Memory.java b/src/org/python/modules/jffi/Memory.java --- a/src/org/python/modules/jffi/Memory.java +++ b/src/org/python/modules/jffi/Memory.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; /** diff --git a/src/org/python/modules/jffi/MemoryOp.java b/src/org/python/modules/jffi/MemoryOp.java --- a/src/org/python/modules/jffi/MemoryOp.java +++ b/src/org/python/modules/jffi/MemoryOp.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import org.python.core.Py; diff --git a/src/org/python/modules/jffi/NativeDataConverter.java b/src/org/python/modules/jffi/NativeDataConverter.java --- a/src/org/python/modules/jffi/NativeDataConverter.java +++ b/src/org/python/modules/jffi/NativeDataConverter.java @@ -1,11 +1,7 @@ package org.python.modules.jffi; - import org.python.core.PyObject; -/** - * - */ abstract public class NativeDataConverter { private final boolean referenceRequired; private final boolean postInvokeRequired; diff --git a/src/org/python/modules/jffi/NativeMemory.java b/src/org/python/modules/jffi/NativeMemory.java --- a/src/org/python/modules/jffi/NativeMemory.java +++ b/src/org/python/modules/jffi/NativeMemory.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import com.kenai.jffi.Platform; diff --git a/src/org/python/modules/jffi/NativeType.java b/src/org/python/modules/jffi/NativeType.java --- a/src/org/python/modules/jffi/NativeType.java +++ b/src/org/python/modules/jffi/NativeType.java @@ -1,7 +1,5 @@ - package org.python.modules.jffi; - public enum NativeType { VOID, BOOL, diff --git a/src/org/python/modules/jffi/NullMemory.java b/src/org/python/modules/jffi/NullMemory.java --- a/src/org/python/modules/jffi/NullMemory.java +++ b/src/org/python/modules/jffi/NullMemory.java @@ -1,5 +1,3 @@ - - package org.python.modules.jffi; /** diff --git a/src/org/python/modules/jffi/Pointer.java b/src/org/python/modules/jffi/Pointer.java --- a/src/org/python/modules/jffi/Pointer.java +++ b/src/org/python/modules/jffi/Pointer.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; public interface Pointer { diff --git a/src/org/python/modules/jffi/PointerCData.java b/src/org/python/modules/jffi/PointerCData.java --- a/src/org/python/modules/jffi/PointerCData.java +++ b/src/org/python/modules/jffi/PointerCData.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import org.python.core.Py; diff --git a/src/org/python/modules/jffi/ScalarCData.java b/src/org/python/modules/jffi/ScalarCData.java --- a/src/org/python/modules/jffi/ScalarCData.java +++ b/src/org/python/modules/jffi/ScalarCData.java @@ -1,14 +1,11 @@ - package org.python.modules.jffi; import org.python.core.Py; import org.python.core.PyFloat; -import org.python.core.PyInteger; -import org.python.core.PyLong; import org.python.core.PyNewWrapper; import org.python.core.PyObject; -import org.python.core.PyObject.ConversionException; import org.python.core.PyType; +import org.python.core.Visitproc; import org.python.expose.ExposedClassMethod; import org.python.expose.ExposedGet; import org.python.expose.ExposedMethod; @@ -19,9 +16,9 @@ @ExposedType(name = "jffi.ScalarCData", base = CData.class) public class ScalarCData extends CData { public static final PyType TYPE = PyType.fromClass(ScalarCData.class); - static { +// static { // TYPE.fastGetDict().__setitem__("in_dll", new InDll()); - } +// } private PyObject value; @ExposedNew @@ -121,4 +118,25 @@ public final String toString() { return getType().getName() + "(" + getValue().toString() + ")"; } + + + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + if (value != null) { + int res = visit.visit(value, arg); + if (res != 0) { + return res; + } + } + return super.traverse(visit, arg); + } + + @Override + public boolean refersDirectlyTo(PyObject ob) { + if (ob != null && ob == value) { + return true; + } + return super.refersDirectlyTo(ob); + } } diff --git a/src/org/python/modules/jffi/StringCData.java b/src/org/python/modules/jffi/StringCData.java --- a/src/org/python/modules/jffi/StringCData.java +++ b/src/org/python/modules/jffi/StringCData.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import org.python.core.Py; @@ -52,7 +51,6 @@ : Py.None; } - @ExposedSet(name = "value") public void setValue(PyObject value) { byte[] str = value.asString().getBytes(); @@ -76,6 +74,4 @@ ? new String(m.getZeroTerminatedByteArray(0)) : null; } - - } diff --git a/src/org/python/modules/jffi/StructLayout.java b/src/org/python/modules/jffi/StructLayout.java --- a/src/org/python/modules/jffi/StructLayout.java +++ b/src/org/python/modules/jffi/StructLayout.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import java.util.Arrays; @@ -20,7 +19,7 @@ import org.python.expose.ExposedType; @ExposedType(name = "jffi.StructLayout", base = CType.class) -public class StructLayout extends CType.Custom { +public class StructLayout extends CType.Custom implements Traverseproc { public static final PyType TYPE = PyType.fromClass(StructLayout.class); static { TYPE.fastGetDict().__setitem__("Field", Field.TYPE); @@ -207,4 +206,44 @@ StructLayout.Field f = getField(key); return f != null ? f : Py.None; } + + + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + int res = 0; + if (fields != null) { + for (Field fld: fields) { + res = visit.visit(fld, arg); + if (res != 0) { + return res; + } + } + } + if (fieldMap != null) { + for (Object key: fieldMap.keySet()) { + if (key instanceof PyObject) { + res = visit.visit((PyObject) key, arg); + if (res != 0) { + return res; + } + } + } + } + return 0; + } + + @Override + public boolean refersDirectlyTo(PyObject ob) { + if (ob == null) { + return false; + } + if (fields != null && fields.contains(ob)) { + return true; + } + if (fieldMap != null && fieldMap.containsKey(ob)) { + return true; + } + return false; + } } diff --git a/src/org/python/modules/jffi/Structure.java b/src/org/python/modules/jffi/Structure.java --- a/src/org/python/modules/jffi/Structure.java +++ b/src/org/python/modules/jffi/Structure.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import java.util.List; @@ -6,12 +5,14 @@ import org.python.core.PyNewWrapper; import org.python.core.PyObject; import org.python.core.PyType; +import org.python.core.Traverseproc; +import org.python.core.Visitproc; import org.python.expose.ExposedClassMethod; import org.python.expose.ExposedNew; import org.python.expose.ExposedType; @ExposedType(name = "jffi.Structure", base = CData.class) -public class Structure extends CData implements Pointer { +public class Structure extends CData implements Pointer, Traverseproc { public static final PyType TYPE = PyType.fromClass(Structure.class); private final StructLayout layout; @@ -100,4 +101,23 @@ return getReferenceMemory(); } + /* Traverseproc implementation */ + @Override + public int traverse(Visitproc visit, Object arg) { + if (layout != null) { + int res = visit.visit(layout, arg); + if (res != 0) { + return res; + } + } + return super.traverse(visit, arg); + } + + @Override + public boolean refersDirectlyTo(PyObject ob) { + if (ob != null && layout == ob) { + return true; + } + return super.refersDirectlyTo(ob); + } } diff --git a/src/org/python/modules/jffi/Util.java b/src/org/python/modules/jffi/Util.java --- a/src/org/python/modules/jffi/Util.java +++ b/src/org/python/modules/jffi/Util.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import java.math.BigInteger; diff --git a/src/org/python/modules/jffi/jffi.java b/src/org/python/modules/jffi/jffi.java --- a/src/org/python/modules/jffi/jffi.java +++ b/src/org/python/modules/jffi/jffi.java @@ -1,4 +1,3 @@ - package org.python.modules.jffi; import com.kenai.jffi.Library; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Feb 23 18:54:52 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 23 Feb 2017 23:54:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_CPython-style_=5F=5Fr?= =?utf-8?q?epr=5F=5F_to_sys=2Elong=5Finfo_and_sys=2Efloat=5Finfo=2E?= Message-ID: <20170223235450.9293.4225.A03B6CE5@psf.io> https://hg.python.org/jython/rev/deff6432e139 changeset: 8027:deff6432e139 user: Stefan Richthofer date: Fri Feb 24 00:54:37 2017 +0100 summary: Added CPython-style __repr__ to sys.long_info and sys.float_info. files: src/org/python/core/PySystemState.java | 15 ++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-) diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -1895,6 +1895,14 @@ ); } + @Override + public PyString __repr__() { + return (PyString) Py.newString( + TYPE.fastGetName() + "(" + + "max=%r, max_exp=%r, max_10_exp=%r, min=%r, min_exp=%r, min_10_exp=%r, "+ + "dig=%r, mant_dig=%r, epsilon=%r, radix=%r, rounds=%r)").__mod__(this); + } + /* Traverseproc implementation */ @Override @@ -1993,6 +2001,13 @@ return new LongInfo(Py.newLong(30), Py.newLong(4)); } + @Override + public PyString __repr__() { + return (PyString) Py.newString( + TYPE.fastGetName() + "(" + + "bits_per_digit=%r, sizeof_digit=%r)").__mod__(this); + } + /* Traverseproc implementation */ @Override -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 04:44:26 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 09:44:26 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Removedor_reduced_Traversep?= =?utf-8?q?roc_implementation_of_named=5Ftuple-like_struct?= Message-ID: <20170224094425.115671.66132.DE5F4DB8@psf.io> https://hg.python.org/jython/rev/b263f56ea793 changeset: 8028:b263f56ea793 user: Stefan Richthofer date: Fri Feb 24 10:43:57 2017 +0100 summary: Removedor reduced Traverseproc implementation of named_tuple-like struct objects LongInfo, FloatInfo, PyStatResult. files: src/org/python/core/PySystemState.java | 98 +--------- src/org/python/modules/posix/PyStatResult.java | 52 +---- 2 files changed, 14 insertions(+), 136 deletions(-) diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -1904,79 +1904,10 @@ } - /* Traverseproc implementation */ - @Override - public int traverse(Visitproc visit, Object arg) { - int retVal = super.traverse(visit, arg); - if (max != null) { - retVal = visit.visit(max, arg); - if (retVal != 0) { - return retVal; - } - } - if (max_exp != null) { - retVal = visit.visit(max_exp, arg); - if (retVal != 0) { - return retVal; - } - } - if (max_10_exp != null) { - retVal = visit.visit(max_10_exp, arg); - if (retVal != 0) { - return retVal; - } - } - if (min != null) { - retVal = visit.visit(min, arg); - if (retVal != 0) { - return retVal; - } - } - if (min_exp != null) { - retVal = visit.visit(min_exp, arg); - if (retVal != 0) { - return retVal; - } - } - if (min_10_exp != null) { - retVal = visit.visit(min_10_exp, arg); - if (retVal != 0) { - return retVal; - } - } - if (dig != null) { - retVal = visit.visit(dig, arg); - if (retVal != 0) { - return retVal; - } - } - if (mant_dig != null) { - retVal = visit.visit(mant_dig, arg); - if (retVal != 0) { - return retVal; - } - } - if (epsilon != null) { - retVal = visit.visit(epsilon, arg); - if (retVal != 0) { - return retVal; - } - } - if (radix != null) { - retVal = visit.visit(radix, arg); - if (retVal != 0) { - return retVal; - } - } - return rounds == null ? 0 : visit.visit(rounds, arg); - } - - @Override - public boolean refersDirectlyTo(PyObject ob) { - return ob != null && (ob == max || ob == max_exp || ob == max_10_exp || ob == min - || ob == min_exp || ob == min_10_exp || ob == dig - || ob == mant_dig || ob == epsilon || ob == radix || ob == rounds); - } + /* Note for Traverseproc implementation: + * We needn't visit the fields, because they are also represented as tuple elements + * in the parent class. So deferring to super-implementation is sufficient. + */ } @@ -2009,21 +1940,8 @@ } - /* Traverseproc implementation */ - @Override - public int traverse(Visitproc visit, Object arg) { - int retVal = super.traverse(visit, arg); - if (bits_per_digit != null) { - retVal = visit.visit(bits_per_digit, arg); - if (retVal != 0) { - return retVal; - } - } - return sizeof_digit == null ? 0 : visit.visit(sizeof_digit, arg); - } - - @Override - public boolean refersDirectlyTo(PyObject ob) { - return ob != null && (ob == bits_per_digit || ob == sizeof_digit); - } + /* Note for Traverseproc implementation: + * We needn't visit the fields, because they are also represented as tuple elements + * in the parent class. So deferring to super-implementation is sufficient. + */ } diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java --- a/src/org/python/modules/posix/PyStatResult.java +++ b/src/org/python/modules/posix/PyStatResult.java @@ -239,55 +239,16 @@ } - /* Traverseproc implementation */ + /* Traverseproc implementation + * Note that there are more fields to traverse. However traverse in PyTuple handles those. + * Here we only traverse values that are not exactly referenced in PyTuple entries. + */ @Override public int traverse(Visitproc visit, Object arg) { int retVal = super.traverse(visit, arg); if (retVal != 0) { return retVal; } - if (st_mode != null) { - retVal = visit.visit(st_mode, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_ino != null) { - retVal = visit.visit(st_ino, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_dev != null) { - retVal = visit.visit(st_dev, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_nlink != null) { - retVal = visit.visit(st_nlink, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_uid != null) { - retVal = visit.visit(st_uid, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_gid != null) { - retVal = visit.visit(st_gid, arg); - if (retVal != 0) { - return retVal; - } - } - if (st_size != null) { - retVal = visit.visit(st_size, arg); - if (retVal != 0) { - return retVal; - } - } if (st_atime != null) { retVal = visit.visit(st_atime, arg); if (retVal != 0) { @@ -305,8 +266,7 @@ @Override public boolean refersDirectlyTo(PyObject ob) { - return ob != null && (ob == st_mode || ob == st_ino || ob == st_dev || ob == st_nlink - || ob == st_uid || ob == st_gid || ob == st_size || ob == st_atime - || ob == st_mtime || ob == st_ctime || super.refersDirectlyTo(ob)); + return ob != null && (ob == st_atime || ob == st_mtime + || ob == st_ctime || super.refersDirectlyTo(ob)); } } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 05:45:03 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 10:45:03 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Implemented_sys=2Egetwindow?= =?utf-8?q?sversion_to_support_pathlib_on_windows=2E?= Message-ID: <20170224104502.9550.85989.7C5791AC@psf.io> https://hg.python.org/jython/rev/9eac49911e8f changeset: 8029:9eac49911e8f user: Stefan Richthofer date: Fri Feb 24 11:44:32 2017 +0100 summary: Implemented sys.getwindowsversion to support pathlib on windows. files: CoreExposed.includes | 1 + src/org/python/core/PySystemState.java | 115 ++++++++++ src/org/python/modules/posix/PosixModule.java | 36 +--- 3 files changed, 117 insertions(+), 35 deletions(-) diff --git a/CoreExposed.includes b/CoreExposed.includes --- a/CoreExposed.includes +++ b/CoreExposed.includes @@ -55,6 +55,7 @@ org/python/core/PyUnicode.class org/python/core/PyVersionInfo.class org/python/core/PyXRange.class +org/python/core/WinVersion.class org/python/core/stringlib/MarkupIterator.class org/python/core/stringlib/FieldNameIterator.class org/python/modules/PyStruct.class diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -242,6 +242,10 @@ // XXX: Remove bean accessors for settrace/profile that we don't want dict.__setitem__("trace", null); dict.__setitem__("profile", null); + dict.__setitem__("windowsversion", null); + if (!System.getProperty("os.name").startsWith("Windows")) { + dict.__setitem__("getwindowsversion", null); + } } void reload() throws PyIgnoreMethodTag { @@ -329,6 +333,10 @@ platform = value; } + public WinVersion getwindowsversion() { + return WinVersion.getWinVersion(); + } + public synchronized codecs.CodecState getCodecState() { if (codecState == null) { codecState = new codecs.CodecState(); @@ -1614,6 +1622,50 @@ } + /** + * Backed as follows: + * Windows: cmd.exe /C ver (part after "Windows") + * Other: uname -v + */ + public static String getSystemVersionString() { + try { + String uname_sysver; + boolean win = System.getProperty("os.name").startsWith("Windows"); + Process p = Runtime.getRuntime().exec( + win ? "cmd.exe /C ver" : "uname -v"); + java.io.BufferedReader br = new java.io.BufferedReader( + new java.io.InputStreamReader(p.getInputStream())); + uname_sysver = br.readLine(); + while (uname_sysver != null && uname_sysver.length() == 0) { + uname_sysver = br.readLine(); + } + // to end the process sanely in case we deal with some + // implementation that emits additional new-lines: + while (br.readLine() != null) { + ; + } + br.close(); + if (p.waitFor() != 0) { + // No fallback for sysver available + uname_sysver = ""; + } + if (win && uname_sysver.length() > 0) { + int start = uname_sysver.toLowerCase().indexOf("version "); + if (start != -1) { + start += 8; + int end = uname_sysver.length(); + if (uname_sysver.endsWith("]")) { + --end; + } + uname_sysver = uname_sysver.substring(start, end); + } + } + return uname_sysver; + } catch (Exception e) { + return ""; + } + } + /* Traverseproc implementation */ @Override @@ -1945,3 +1997,66 @@ * in the parent class. So deferring to super-implementation is sufficient. */ } + + + at ExposedType(name = "sys.getwindowsversion", isBaseType = false) +class WinVersion extends PyTuple { + + @ExposedGet + public PyObject major, minor, build, platform, service_pack; + + public static final PyType TYPE = PyType.fromClass(WinVersion.class); + + private WinVersion(PyObject... vals) { + super(TYPE, vals); + + major = vals[0]; + minor = vals[1]; + build = vals[2]; + platform = vals[3]; + service_pack = vals[4]; + } + + public static WinVersion getWinVersion() { + try { + String sysver = PySystemState.getSystemVersionString(); + String[] sys_ver = sysver.split("\\."); + int major = Integer.parseInt(sys_ver[0]); + int minor = Integer.parseInt(sys_ver[1]); + int build = Integer.parseInt(sys_ver[2]); + if (major > 6) { + major = 6; minor = 2; build = 9200; + } else if (major == 6 && minor > 2) { + minor = 2; build = 9200; + } + // emulate deprecation behavior of GetVersionEx: + return new WinVersion( + Py.newInteger(major), // major + Py.newInteger(minor), // minor + Py.newInteger(build), // build + Py.newInteger(2), // platform + Py.EmptyString); // service_pack + } catch (Exception e) { + return new WinVersion(Py.EmptyString, Py.EmptyString, + Py.EmptyString, Py.EmptyString, Py.EmptyString); + } + } + + @Override + public PyString __repr__() { + return (PyString) Py.newString( + TYPE.fastGetName() + "(major=%r, minor=%r, build=%r, " + + "platform=%r, service_pack=%r)").__mod__(this); + } + + + /* Note for traverseproc implementation: + * We needn't visit the fields, because they are also represented as tuple elements + * in the parent class. So deferring to super-implementation is sufficient. + * + * (In CPython sys.getwindowsversion can have some keyword-only elements. So far + * we don't support these here. If that changes, an actual traverseproc implementation + * might be required. + */ +} + diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java --- a/src/org/python/modules/posix/PosixModule.java +++ b/src/org/python/modules/posix/PosixModule.java @@ -1030,7 +1030,6 @@ if (uname_cache != null) { return uname_cache; } -// todo: Giving os.uname a windows-implementation might break platform.uname. Check this! String sysname = System.getProperty("os.name"); String sysrelease; boolean win; @@ -1096,40 +1095,7 @@ } } - String uname_sysver; - try { - Process p = Runtime.getRuntime().exec( - win ? "cmd.exe /C ver" : "uname -v"); - java.io.BufferedReader br = new java.io.BufferedReader( - new java.io.InputStreamReader(p.getInputStream())); - uname_sysver = br.readLine(); - while (uname_sysver != null && uname_sysver.length() == 0) { - uname_sysver = br.readLine(); - } - // to end the process sanely in case we deal with some - // implementation that emits additional new-lines: - while (br.readLine() != null) { - ; - } - br.close(); - if (p.waitFor() != 0) { - // No fallback for sysver available - uname_sysver = ""; - } - if (win && uname_sysver.length() > 0) { - int start = uname_sysver.toLowerCase().indexOf("version "); - if (start != -1) { - start += 8; - int end = uname_sysver.length(); - if (uname_sysver.endsWith("]")) { - --end; - } - uname_sysver = uname_sysver.substring(start, end); - } - } - } catch (Exception e) { - uname_sysver = ""; - } + String uname_sysver = PySystemState.getSystemVersionString(); String uname_machine; try { -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 06:18:38 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 11:18:38 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Merged_https=3A//github=2Ec?= =?utf-8?q?om/jythontools/jython/pull/57_to_fix_=232534=2E?= Message-ID: <20170224111838.121663.17096.949C68D2@psf.io> https://hg.python.org/jython/rev/07724b0547e0 changeset: 8030:07724b0547e0 user: James Mudd date: Fri Feb 24 12:18:23 2017 +0100 summary: Merged https://github.com/jythontools/jython/pull/57 to fix #2534. files: src/org/python/modules/posix/PosixModule.java | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java --- a/src/org/python/modules/posix/PosixModule.java +++ b/src/org/python/modules/posix/PosixModule.java @@ -525,7 +525,12 @@ "Return the actual login name."); @Hide(value=OS.NT, posixImpl = PosixImpl.JAVA) public static PyObject getlogin() { - return new PyString(posix.getlogin()); + String login = posix.getlogin(); + if (login == null) { + throw Py.OSError( + "getlogin OS call failed. Preferentially use os.getenv('LOGNAME') instead."); + } + return new PyString(login); } public static PyString __doc__getppid = new PyString( -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 07:05:12 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 12:05:12 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Updated_NEWS-file=2E_Added_?= =?utf-8?q?code-comment_to_last_commit=2E?= Message-ID: <20170224120511.72366.81164.2526CFF2@psf.io> https://hg.python.org/jython/rev/137e1dee741f changeset: 8031:137e1dee741f user: Stefan Richthofer date: Fri Feb 24 13:05:03 2017 +0100 summary: Updated NEWS-file. Added code-comment to last commit. files: NEWS | 2 ++ src/org/python/modules/posix/PosixModule.java | 1 + 2 files changed, 3 insertions(+), 0 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2534 ] os.getlogin() returns a wrong user or returns an exception + - [ 2553 ] sys.getwindowsversion not implemented (breaks pathlib on Windows) - [ PR55 ] Replace deprecated use of ASCII constant with ascii() - [ 1777 ] Help getting SymPy working with Jython - [ 2475 ] Backported python 3 pathlib and pathlib2 do not work b/c of meta class issues diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java --- a/src/org/python/modules/posix/PosixModule.java +++ b/src/org/python/modules/posix/PosixModule.java @@ -527,6 +527,7 @@ public static PyObject getlogin() { String login = posix.getlogin(); if (login == null) { + // recommend according to https://docs.python.org/2/library/os.html#os.getlogin throw Py.OSError( "getlogin OS call failed. Preferentially use os.getenv('LOGNAME') instead."); } -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 08:34:04 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 13:34:04 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_2552_by_letting_jytho?= =?utf-8?q?ncompiler=2Epy_emit_a_warning_instead_of_an_error=2E?= Message-ID: <20170224133404.122356.81621.433D6287@psf.io> https://hg.python.org/jython/rev/24ef3218632f changeset: 8032:24ef3218632f user: Stefan Richthofer date: Fri Feb 24 14:33:56 2017 +0100 summary: Fixed 2552 by letting jythoncompiler.py emit a warning instead of an error. This is crucial for installing modules that contain an *optional* C-extension (e.g. scandir). files: Lib/distutils/jythoncompiler.py | 4 +++- NEWS | 1 + 2 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Lib/distutils/jythoncompiler.py b/Lib/distutils/jythoncompiler.py --- a/Lib/distutils/jythoncompiler.py +++ b/Lib/distutils/jythoncompiler.py @@ -6,6 +6,7 @@ from distutils.ccompiler import CCompiler from distutils.errors import CCompilerError +import warnings class JythonCompiler(CCompiler): @@ -16,6 +17,7 @@ def refuse_compilation(self, *args, **kwargs): """Refuse compilation""" - raise CCompilerError('Compiling extensions is not supported on Jython') + warnings.warn('Compiling extensions is not supported on Jython') + return [] preprocess = compile = create_static_lib = link = refuse_compilation diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2552 ] installing scandir via pip fails (breaks e.g. installing pathlib2 via pip) - [ 2534 ] os.getlogin() returns a wrong user or returns an exception - [ 2553 ] sys.getwindowsversion not implemented (breaks pathlib on Windows) - [ PR55 ] Replace deprecated use of ASCII constant with ascii() -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Feb 24 10:10:45 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 24 Feb 2017 15:10:45 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_PyShadowString_as_rep?= =?utf-8?q?lacement_for_os=2Ename_and_sys=2Eplatform_so_we=27ll_have?= Message-ID: <20170224151043.115837.38427.56AC64A7@psf.io> https://hg.python.org/jython/rev/a01c366576b8 changeset: 8033:a01c366576b8 user: Stefan Richthofer date: Fri Feb 24 16:10:28 2017 +0100 summary: Added PyShadowString as replacement for os.name and sys.platform so we'll have a fine-grained platform detection control. files: CoreExposed.includes | 1 + Lib/os.py | 4 +- src/org/python/core/PyString.java | 23 +++++++---- src/org/python/core/PySystemState.java | 28 +++++++++++++- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/CoreExposed.includes b/CoreExposed.includes --- a/CoreExposed.includes +++ b/CoreExposed.includes @@ -42,6 +42,7 @@ org/python/core/PyObject.class org/python/core/PyProperty.class org/python/core/PySet.class +org/python/core/PyShadowString.class org/python/core/PySlice.class org/python/core/PySlot.class org/python/core/PyStaticMethod.class diff --git a/Lib/os.py b/Lib/os.py --- a/Lib/os.py +++ b/Lib/os.py @@ -24,6 +24,7 @@ #' import sys, errno +from org.python.core import PyShadowString _names = sys.builtin_module_names @@ -38,7 +39,6 @@ except AttributeError: return [n for n in dir(module) if n[0] != '_'] -name = 'java' if 'posix' in _names: _name = 'posix' linesep = '\n' @@ -131,6 +131,8 @@ else: raise ImportError, 'no os specific module found' +name = PyShadowString('java', _name) + sys.modules['os.path'] = path from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep, devnull) diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java --- a/src/org/python/core/PyString.java +++ b/src/org/python/core/PyString.java @@ -49,6 +49,16 @@ this("", true); } + protected PyString(PyType subType, String string, boolean isBytes) { + super(subType); + if (string == null) { + throw new IllegalArgumentException("Cannot create PyString from null"); + } else if (!isBytes && !isBytes(string)) { + throw new IllegalArgumentException("Cannot create PyString with non-byte value"); + } + this.string = string; + } + /** * Fundamental constructor for PyString objects when the client provides a Java * String, necessitating that we range check the characters. @@ -57,13 +67,7 @@ * @param string a Java String to be wrapped */ public PyString(PyType subType, String string) { - super(subType); - if (string == null) { - throw new IllegalArgumentException("Cannot create PyString from null"); - } else if (!isBytes(string)) { - throw new IllegalArgumentException("Cannot create PyString with non-byte value"); - } - this.string = string; + this(subType, string, false); } public PyString(String string) { @@ -597,6 +601,9 @@ @Override public PyObject __eq__(PyObject other) { + if (other instanceof PyShadowString) { + return other.__eq__(this); + } return str___eq__(other); } @@ -894,7 +901,7 @@ * @return coerced value * @throws PyException if the coercion fails */ - private static String asUTF16StringOrError(PyObject obj) { + protected static String asUTF16StringOrError(PyObject obj) { // PyUnicode accepted here. Care required in the client if obj is not basic plane. String ret = asUTF16StringOrNull(obj); if (ret != null) { diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -134,7 +134,7 @@ public PyList warnoptions = new PyList(); public PyObject builtins; - private static PyObject defaultPlatform = new PyString("java"); + private static PyObject defaultPlatform = new PyShadowString("java", getNativePlatform()); public PyObject platform = defaultPlatform; public PyList meta_path; @@ -787,7 +787,31 @@ if (version.equals("12")) { version = "1.2"; } - defaultPlatform = new PyString("java" + version); + defaultPlatform = new PyShadowString("java" + version, getNativePlatform()); + } + + /** + * Emulates CPython's way to name sys.platform. + */ + public static String getNativePlatform() { + /* Works according to this table: + System Value + -------------------------- + Linux (2.x and 3.x) linux2 + Windows win32 + Windows/Cygwin cygwin + Mac OS X darwin + OS/2 os2 + OS/2 EMX os2emx + RiscOS riscos + AtheOS atheos + */ + String osname = System.getProperty("os.name"); + if (osname.equals("Linux")) return "linux2"; + if (osname.equals("Mac OS X")) return "darwin"; + if (osname.toLowerCase().contains("cygwin")) return "cygwin"; + if (osname.startsWith("Windows")) return "win32"; + return osname.replaceAll("[\\s/]", "").toLowerCase(); } private static void initRegistry(Properties preProperties, Properties postProperties, -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sat Feb 25 18:50:26 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sat, 25 Feb 2017 23:50:26 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Cleaned_unused_import_in_jy?= =?utf-8?q?thoncompiler=2Epy=2E?= Message-ID: <20170225235026.30069.82890.A17BCE23@psf.io> https://hg.python.org/jython/rev/37c7ab335238 changeset: 8034:37c7ab335238 user: Stefan Richthofer date: Sun Feb 26 00:49:46 2017 +0100 summary: Cleaned unused import in jythoncompiler.py. files: Lib/distutils/jythoncompiler.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/Lib/distutils/jythoncompiler.py b/Lib/distutils/jythoncompiler.py --- a/Lib/distutils/jythoncompiler.py +++ b/Lib/distutils/jythoncompiler.py @@ -5,7 +5,6 @@ """ from distutils.ccompiler import CCompiler -from distutils.errors import CCompilerError import warnings class JythonCompiler(CCompiler): -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sat Feb 25 18:54:02 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sat, 25 Feb 2017 23:54:02 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=27ValueError=3A_bad_?= =?utf-8?q?marshal_data=27_issue_occurring_on_Windows_when_processing?= Message-ID: <20170225235402.50715.25627.06D9AB23@psf.io> https://hg.python.org/jython/rev/5861112d6d39 changeset: 8035:5861112d6d39 user: Stefan Richthofer date: Sun Feb 26 00:53:20 2017 +0100 summary: Fixed 'ValueError: bad marshal data' issue occurring on Windows when processing bytecode for oversized functions; see #527524, http://bugs.jython.org/msg11130 files: src/org/python/compiler/Module.java | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java --- a/src/org/python/compiler/Module.java +++ b/src/org/python/compiler/Module.java @@ -717,7 +717,7 @@ String pyc_filename = filename+"c"; File pyc_file = new File(pyc_filename); if (pyc_file.exists()) { - PyFile f = new PyFile(pyc_filename, "r", 4096); + PyFile f = new PyFile(pyc_filename, "rb", 4096); byte[] bts = f.read(8).toBytes(); int magic = (bts[1]<< 8) & 0x0000FF00 | (bts[0]<< 0) & 0x000000FF; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Feb 26 19:22:33 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 27 Feb 2017 00:22:33 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_PyShadowString=2Ejava?= =?utf-8?q?=2E_I_had_forgotten_to_call_hg_add_on_that_file_when_I?= Message-ID: <20170227002233.107336.88316.C6845B0C@psf.io> https://hg.python.org/jython/rev/0d2b840f6df8 changeset: 8036:0d2b840f6df8 user: Stefan Richthofer date: Mon Feb 27 01:22:01 2017 +0100 summary: Added PyShadowString.java. I had forgotten to call hg add on that file when I committed https://hg.python.org/jython/rev/a01c366576b8 files: src/org/python/core/PyShadowString.java | 283 ++++++++++++ 1 files changed, 283 insertions(+), 0 deletions(-) diff --git a/src/org/python/core/PyShadowString.java b/src/org/python/core/PyShadowString.java new file mode 100644 --- /dev/null +++ b/src/org/python/core/PyShadowString.java @@ -0,0 +1,283 @@ +package org.python.core; + +import org.python.expose.ExposedMethod; +import org.python.expose.ExposedNew; +import org.python.expose.ExposedType; +import org.python.expose.MethodType; + + at Untraversable + at ExposedType(name = "shadowstr", base = PyString.class, isBaseType = false) +public class PyShadowString extends PyString { + public static final PyType TYPE = PyType.fromClass(PyShadowString.class); + + protected PyList targets; + + /** + * The shadow string is additionally used for some comparisons, especially for __eq__. + * __eq__ will evaluate positive if the other string equals the actual value + * *or* the shadow. The shadow persists slicing (is sliced accordingly) + * and is taken into account by startswith. + */ + protected String shadow; + + // for PyJavaClass.init() + public PyShadowString() { + this(TYPE, "", ""); + targets = new PyList(); + } + + public PyShadowString(String str, String shadow) { + super(TYPE, str); + this.shadow = shadow; + targets = new PyList(); + } + + public PyShadowString(String str, String shadow, boolean isBytes) { + super(TYPE, str, isBytes); + this.shadow = shadow; + targets = new PyList(); + } + + public PyShadowString(String str, String shadow, boolean isBytes, PyList targets) { + super(TYPE, str, isBytes); + this.shadow = shadow; + this.targets = targets; + } + + public PyShadowString(PyObject str, String shadow) { + super(str.toString()); + this.shadow = shadow; + targets = new PyList(); + } + + public PyShadowString(PyType subtype, String str, String shadow) { + super(subtype, str); + this.shadow = shadow; + targets = new PyList(); + } + + public PyShadowString(PyType subtype, PyObject str, String shadow) { + super(subtype, str.toString()); + this.shadow = shadow; + targets = new PyList(); + } + + @ExposedNew + static PyObject shadowstr_new(PyNewWrapper new_, boolean init, PyType subtype, + PyObject[] args, String[] keywords) { + ArgParser ap = new ArgParser("shadowstr", args, keywords, + new String[] {"string", "shadow"}, 0); + PyObject S = ap.getPyObject(0, null); + PyObject Sh = ap.getPyObject(1, null); + // Get the textual representation of the object into str/bytes form + String str, shd; + if (S == null) { + str = ""; + } else { + // Let the object tell us its representation: this may be str or unicode. + S = S.__str__(); + if (S instanceof PyUnicode) { + // Encoding will raise UnicodeEncodeError if not 7-bit clean. + str = codecs.encode((PyUnicode)S, null, null); + } else { + // Must be str/bytes, and should be 8-bit clean already. + str = S.toString(); + } + } + if (Sh == null) { + shd = ""; + } else { + // Let the object tell us its representation: this may be str or unicode. + Sh = Sh.__str__(); + if (Sh instanceof PyUnicode) { + // Encoding will raise UnicodeEncodeError if not 7-bit clean. + shd = codecs.encode((PyUnicode)Sh, null, null); + } else { + // Must be str/bytes, and should be 8-bit clean already. + shd = Sh.toString(); + } + } + return new PyShadowString(str, shd); + } + + private boolean isTarget() { + Exception exc = new Exception(); + PyObject obj; + boolean result; + for (StackTraceElement ste: exc.getStackTrace()) { + for (PyObject itm: targets.getList()) { + result = true; + obj = ((PyTuple) itm).__finditem__(0); + if (obj != null && obj != Py.None) { + if (!ste.getClassName().matches(obj.toString())) { + result = false; + } + } + if (result) { + obj = ((PyTuple) itm).__finditem__(1); + if (obj != null && obj != Py.None) { + if (!ste.getMethodName().matches(obj.toString())) { + result = false; + } + } + } + if (result) { + // we have a match + return true; + } + } + } + return false; + } + + public String getShadow() { + return shadow; + } + + public PyString getshadow() { + return (PyString) shadowstr_getshadow(); + } + + @ExposedMethod + public final PyObject shadowstr_getshadow() { + return Py.newString(shadow); + } + + public void addTarget(String className, String methodName) { + PyString classname = className == null ? null : Py.newString(className); + PyString methodname = methodName == null ? null : Py.newString(methodName); + shadowstr_addtarget(classname, methodname); + } + + @ExposedMethod(defaults = {"null"}) + public final void shadowstr_addtarget(PyObject classname, PyObject methodname) { + targets.add(methodname != null ? + new PyTuple(classname == null ? Py.None : classname, methodname) : + new PyTuple(classname == null ? Py.None : classname)); + } + + public PyList getTargets() { + return (PyList) shadowstr_gettargets(); + } + + @ExposedMethod + public final PyObject shadowstr_gettargets() { + return targets; + } + + @Override + public PyObject __eq__(PyObject other) { + if (!isTarget()) { + return str___eq__(other); + } + return shadowstr___eq__(other); + } + + @ExposedMethod(type = MethodType.BINARY) + final PyObject shadowstr___eq__(PyObject other) { + String s = other.toString(); + if (s != null && s.equals(shadow)) return Py.True; + return str___eq__(other); + } + + @Override + protected PyShadowString fromSubstring(int begin, int end) { + // Method is overridden in PyUnicode, so definitely a PyString + int shadowBegin = begin, shadowEnd = end; + if (begin > shadow.length()) { + shadowBegin = shadow.length(); + } + if (end > shadow.length()) { + shadowEnd = shadow.length(); + } + return new PyShadowString(getString().substring(begin, end), + shadow.substring(shadowBegin, shadowEnd), true, targets); + } + + @Override + protected PyObject getslice(int start, int stop, int step) { + if (step > 0 && stop < start) { + stop = start; + } + if (step == 1) { + return fromSubstring(start, stop); + } else { + int n = sliceLength(start, stop, step); + char new_chars[] = new char[n]; + int j = 0; + for (int i = start; j < n; i += step) { + new_chars[j++] = getString().charAt(i); + } + char new_shadow_chars[] = new char[n]; + j = 0; + try { + for (int i = start; j < n; i += step) { + new_chars[j] = shadow.charAt(i); + j++; // separate line, so in exception case j is clearly before increment + } + } catch (IndexOutOfBoundsException ioobe) + { + return new PyShadowString(new String(new_chars), + new String(new_shadow_chars, 0, j), true, targets); + } + return new PyShadowString(new String(new_chars), + new String(new_shadow_chars), true, targets); + } + } + + @Override + public boolean startswith(PyObject prefix) { + return shadowstr_startswith(prefix, null, null); + } + + @Override + public boolean startswith(PyObject prefix, PyObject start) { + return shadowstr_startswith(prefix, start, null); + } + + @Override + public boolean startswith(PyObject prefix, PyObject start, PyObject end) { + return shadowstr_startswith(prefix, start, end); + } + + @ExposedMethod(defaults = {"null", "null"}) + final boolean shadowstr_startswith(PyObject prefix, PyObject startObj, PyObject endObj) { + if (!isTarget()) { + return str_startswith(prefix, startObj, endObj); + } + int[] indices = translateIndices(startObj, endObj); + int start = indices[0]; + int sliceLen = indices[1] - start; + + if (!(prefix instanceof PyTuple)) { + // It ought to be PyUnicode or some kind of bytes with the buffer API. + String s = asUTF16StringOrError(prefix); + // If s is non-BMP, and this is a PyString (bytes), result will correctly be false. + return sliceLen >= s.length() && + (getString().startsWith(s, start) || shadow.startsWith(s, start)); + } else { + // Loop will return true if this slice starts with any prefix in the tuple + for (PyObject prefixObj : ((PyTuple)prefix).getArray()) { + // It ought to be PyUnicode or some kind of bytes with the buffer API. + String s = asUTF16StringOrError(prefixObj); + // If s is non-BMP, and this is a PyString (bytes), result will correctly be false. + if (sliceLen >= s.length() && + (getString().startsWith(s, start) || shadow.startsWith(s, start))) { + return true; + } + } + // None matched + return false; + } + } + + @Override + public PyString __repr__() { + return shadowstr___repr__(); + } + + @ExposedMethod + final PyString shadowstr___repr__() { + return new PyString(encode_UnicodeEscape(getString()+" ( =="+shadow+" for targets )", true)); + } +} -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Feb 27 12:24:29 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 27 Feb 2017 17:24:29 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Enabled_org=2Epython=2Ecomp?= =?utf-8?q?iler=2EModule=2EloadPyBytecode_to_automatically_run_CPython?= Message-ID: <20170227172428.28010.88978.A2953063@psf.io> https://hg.python.org/jython/rev/03f4808038f8 changeset: 8037:03f4808038f8 user: Stefan Richthofer date: Mon Feb 27 18:24:04 2017 +0100 summary: Enabled org.python.compiler.Module.loadPyBytecode to automatically run CPython 2.7 bytecode compilation, if a CPython 2.7 command is provided in property 'cpython_cmd'. Took care that this can be passed through pip e.g. using --global-option='-J-Dcpython_cmd=python'; note that pip launches separate Jython processes. Without that option, running e.g. pip install sympy is tedious, because it will prompt you several times to provide yet another pyc-file. files: NEWS | 4 + src/org/python/compiler/Module.java | 93 ++++++++++++++-- src/org/python/util/jython.java | 35 ++++++- 3 files changed, 119 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -61,6 +61,10 @@ - [ 1767 ] Rich comparisons New Features + - Recognize cpython_cmd property to automatically build CPython bytecode for oversized + functions (e.g. jython -J-Dcpython_cmd=python). This is especially convenient when + installing things like SymPy via pip; it would frequently prompt you to provide yet + another pyc-file. Now just run: pip install --global-option="-J-Dcpython_cmd=python" sympy - SymPy is now workable. (However it runs somewhat slow; some profiling will be required.) - Updated ucnhash to support name lookup for Unicode 9.0 (like in u'\N{name}'). - Jython doc-entries in Java-code (__doc__foo) now accept anything that implements diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java --- a/src/org/python/compiler/Module.java +++ b/src/org/python/compiler/Module.java @@ -10,6 +10,8 @@ import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; import java.io.File; +import java.io.BufferedReader; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Enumeration; import java.util.Hashtable; @@ -712,8 +714,20 @@ module.mainCode = main; } - private static PyBytecode loadPyBytecode(String filename) throws RuntimeException + private static PyBytecode loadPyBytecode(String filename, boolean try_cpython) + throws RuntimeException { + String cpython_cmd_msg = + "\n\nAlternatively provide proper CPython 2.7 execute command via"+ + "\ncpython_cmd property, e.g. call "+ + "\n jython -J-Dcpython_cmd=python"+ + "\nor if running pip on Jython:"+ + "\n pip install --global-option=\"-J-Dcpython_cmd=python\" "; + String large_method_msg = "\nEncountered too large method code in \n"+filename+"\n"; + String please_provide_msg = + "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+ + "\npython -m py_compile "+filename+"\nand try again."; + String pyc_filename = filename+"c"; File pyc_file = new File(pyc_filename); if (pyc_file.exists()) { @@ -726,11 +740,10 @@ // (bts[5]<< 8) & 0x0000FF00 | // (bts[4]<< 0) & 0x000000FF; if (magic != 62211) { // check Python 2.7 bytecode - throw new RuntimeException("Encountered too large method code in \n"+filename+ + throw new RuntimeException(large_method_msg+ "\n"+pyc_filename+ "\ncontains wrong bytecode version, not CPython 2.7 bytecode."+ - "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+ - "\npython -m py_compile "+filename+"\nand try again."); + please_provide_msg); } _marshal.Unmarshaller un = new _marshal.Unmarshaller(f); PyObject code = un.load(); @@ -738,15 +751,73 @@ if (code instanceof PyBytecode) { return (PyBytecode) code; } - throw new RuntimeException("Encountered too large method code in \n"+filename+ + throw new RuntimeException(large_method_msg+ "\n"+pyc_filename+ "\ncontains invalid bytecode."+ - "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+ - "\npython -m py_compile "+filename+"\nand try again."); + please_provide_msg); } else { - throw new RuntimeException("Encountered too large method code in \n"+filename+ - "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+ - "\npython -m py_compile "+filename+"\nand try again."); + String CPython_command = System.getProperty("cpython_cmd"); + if (try_cpython && CPython_command != null) { + // check version... + String command_ver = CPython_command+" --version"; + String command = CPython_command+" -m py_compile "+filename; + String tried_create_pyc_msg = "\nTried to create pyc-file by executing\n"+ + command+"\nThis failed because of\n"; + Exception exc = null; + int result = 0; + try { + Process p = Runtime.getRuntime().exec(command_ver); + // Python 2.7 writes version to error-stream for some reason: + BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream())); + String cp_version = br.readLine(); + while (br.readLine() != null) {} + br.close(); + if (cp_version == null) { + // Also try input-stream as fallback, just in case... + br = new BufferedReader(new InputStreamReader(p.getInputStream())); + cp_version = br.readLine(); + while (br.readLine() != null) {} + br.close(); + } + result = p.waitFor(); + if (!cp_version.startsWith("Python 2.7.")) { + throw new RuntimeException(large_method_msg+ + tried_create_pyc_msg+ + "wrong Python version: "+cp_version+"."+ + "\nRequired is Python 2.7.x.\n"+ + please_provide_msg+cpython_cmd_msg); + } + } + catch (InterruptedException ie) { + exc = ie; + } + catch (IOException ioe) { + exc = ioe; + } + if (exc == null && result == 0) { + try { + Process p = Runtime.getRuntime().exec(command); + result = p.waitFor(); + if (result == 0) { + return loadPyBytecode(filename, false); + } + } + catch (InterruptedException ie) { + exc = ie; + } + catch (IOException ioe) { + exc = ioe; + } + } + String exc_msg = large_method_msg+ + tried_create_pyc_msg+ + (exc != null ? exc.toString() : "bad return: "+result)+".\n"+ + please_provide_msg+cpython_cmd_msg; + throw exc != null ? new RuntimeException(exc_msg, exc) : new RuntimeException(exc_msg); + } else { + throw new RuntimeException(large_method_msg+ + please_provide_msg+cpython_cmd_msg); + } } } @@ -841,7 +912,7 @@ module.write(ostream); } catch (RuntimeException re) { if (re.getMessage() != null && re.getMessage().equals("Method code too large!")) { - PyBytecode btcode = loadPyBytecode(filename); + PyBytecode btcode = loadPyBytecode(filename, true); int thresh = 22000; // No idea, how to determine at this point if a method is oversized, so we just try // a threshold regarding Python code-length, while JVM restriction is actually about diff --git a/src/org/python/util/jython.java b/src/org/python/util/jython.java --- a/src/org/python/util/jython.java +++ b/src/org/python/util/jython.java @@ -682,6 +682,28 @@ } int n = args.length - index + 1; + + /* Exceptionally we allow -J-Dcpython_cmd=... also postpone the filename. + * E.g. the Linux launcher allows this already on launcher level for all + * -J flags, while the Windows launcher does not. + * + * Todo: Resolve this discrepancy! + * + * This is required to use cpython_cmd property in context of pip, e.g. + * pip install --global-option="-J-Dcpython_cmd=python" + * For details about the cpython_cmd property, look into + * org.python.compiler.Module.loadPyBytecode source. + */ + int cpython_cmd_pos = -1; + for (int i = index; i < args.length; i++) { + if (args[i].startsWith("-J-Dcpython_cmd=")) { + cpython_cmd_pos = i; + System.setProperty("cpython_cmd", args[i].substring(16)); + n--; + break; + } + } + argv = new String[n]; if (filename != null) { argv[0] = filename; @@ -691,8 +713,17 @@ argv[0] = ""; } - for (int i = 1; i < n; i++, index++) { - argv[i] = args[index]; + if (cpython_cmd_pos == -1) { + for (int i = 1; i < n; i++, index++) { + argv[i] = args[index]; + } + } else { + for (int i = 1; i < n; i++, index++) { + if (index == cpython_cmd_pos) { + index++; + } + argv[i] = args[index]; + } } return true; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Feb 27 23:31:51 2017 From: jython-checkins at python.org (frank.wierzbicki) Date: Tue, 28 Feb 2017 04:31:51 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7=2E1rc1_for?= =?utf-8?q?_changeset_03f4808038f8?= Message-ID: <20170228043150.27896.97880.5C9B091A@psf.io> https://hg.python.org/jython/rev/40802a3ef2b8 changeset: 8038:40802a3ef2b8 user: Frank Wierzbicki date: Tue Feb 28 04:30:32 2017 +0000 summary: Added tag v2.7.1rc1 for changeset 03f4808038f8 files: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -101,3 +101,4 @@ 44403bccae2163186f8ab46b4a544a48e137bba6 v2.7.1b1 6c0da6893570f3ae3cd37a33189cae954e2afebb v2.7.1b2 c3289f2a275446451605e586e5bef70092224184 v2.7.1b3 +03f4808038f8bbc246b6d6a022aecfde087eeb91 v2.7.1rc1 -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Feb 27 23:32:45 2017 From: jython-checkins at python.org (frank.wierzbicki) Date: Tue, 28 Feb 2017 04:32:45 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Prepare_for_2=2E7=2E1rc1_re?= =?utf-8?q?lease=2E?= Message-ID: <20170228043244.3109.6512.40B0C54D@psf.io> https://hg.python.org/jython/rev/330556fdad47 changeset: 8039:330556fdad47 user: Frank Wierzbicki date: Tue Feb 28 04:31:40 2017 +0000 summary: Prepare for 2.7.1rc1 release. files: README.txt | 4 ++-- build.xml | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.txt b/README.txt --- a/README.txt +++ b/README.txt @@ -1,8 +1,8 @@ Jython: Python for the Java Platform -Welcome to Jython 2.7.1 beta 3! +Welcome to Jython 2.7.1 release candidate 1! -This is the third beta release of the 2.7.1 version of Jython. Along with +This is the first release candidate of the 2.7.1 version of Jython. Along with language and runtime compatibility with CPython 2.7.1, Jython 2.7 provides substantial support of the Python ecosystem. This includes built-in support of pip/setuptools (you can use with bin/pip) and a native launcher for Windows diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -84,15 +84,15 @@ - - + + - - + + - + @@ -397,7 +397,7 @@ - ======================= @@ -432,7 +432,7 @@ - + -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Feb 27 23:33:00 2017 From: jython-checkins at python.org (frank.wierzbicki) Date: Tue, 28 Feb 2017 04:33:00 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_tag_v2=2E7=2E1rc1_for?= =?utf-8?q?_changeset_330556fdad47?= Message-ID: <20170228043300.50922.59632.CB635983@psf.io> https://hg.python.org/jython/rev/8a87a1e6d515 changeset: 8040:8a87a1e6d515 user: Frank Wierzbicki date: Tue Feb 28 04:32:00 2017 +0000 summary: Added tag v2.7.1rc1 for changeset 330556fdad47 files: .hgtags | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -102,3 +102,5 @@ 6c0da6893570f3ae3cd37a33189cae954e2afebb v2.7.1b2 c3289f2a275446451605e586e5bef70092224184 v2.7.1b3 03f4808038f8bbc246b6d6a022aecfde087eeb91 v2.7.1rc1 +03f4808038f8bbc246b6d6a022aecfde087eeb91 v2.7.1rc1 +330556fdad478b61f93a548643743c3d0214fd40 v2.7.1rc1 -- Repository URL: https://hg.python.org/jython